Query S3 data via Hive on local box
https://nikkisharma536.medium.com/query-s3-data-via-hive-on-local-box-55fd8fa654bb
hive-site.xml
<property>
<name>fs.s3a.access.key</name>
<value>ACCESS_KEY_HERE</value>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>ACCESS_SECRET_HERE</value>
</property>
core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/Users/nikki/hadoop/hdfs/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
<property>
<name>fs.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
</property>
<property>
<name>fs.s3a.access.key</name>
<value>ACCESS_KEY_HERE</value>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>ACCESS_SECRET_HERE</value>
</property>
</configuration>
mapred-site.xml
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>localhost:9010</value>
</property>
<property>
<name>fs.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
</property>
<configuration>
hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value></value>
</property>
<property>
<name>fs.s3a.access.key</name>
<value>ACCESS_KEY_HERE</value>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>ACCESS_SECRET_HERE</value>
</property>
</configuration>
How to generate the TPC-DS benchmarking data 1 TB in AWS S3?
https://stackoverflow.com/questions/53644143/how-to-generate-the-tpc-ds-benchmarking-data-1-tb-in-aws-s3
I did similar work several month ago, hive-testbench can be an option. Check the README.md about how to make it happen.
You need to configure fs.defaultFS in $HADOOP_HOME/etc/hadoop/core-site.xml to your AWS S3 bucket, the data will be generated in AWS directly. Pass data scale parameter to ./tpcds-setup.sh to generate date with different scale.
Configuring Access to S3 on CDP Private Cloud Base
https://docs.cloudera.com/runtime/7.2.10/cloud-data-access/topics/cr-cda-configuring-access-to-s3-data-center.html
To configure authentication with S3, explicitly declare the credentials in a configuration file such as core-site.xml.
<property>
<name>fs.s3a.access.key</name>
<value>ACCESS-KEY</value>
</property>
<property>
<name>fs.s3a.secret.key</name>
<value>SECRET-KEY</value>
</property>
Change core-site.xml
https://community.cloudera.com/t5/Support-Questions/Custom-configuration-in-core-site-xml/m-p/3809
Referencing S3 data in Applications
Cloudera repo
cat > /etc/yum.repos.d/cloudera-manager.repo <<EOF
[cloudera-manager]
name = Cloudera Manager, Version 7.4.4
baseurl = https://archive.cloudera.com/cm7/7.4.4/redhat8/yum
gpgcheck = 1
EOF
yum -y install openjdk8 cloudera-manager-daemons
Where is the classpath set for hadoop
# export HADOOP_HOME=${your hadoop install directory}, an example as follows:
export HADOOP_HOME=/usr/local/hadoop-2.10.1
export PATH=${HADOOP_HOME}/bin:${PATH}
export HADOOP_CLASSPATH=`hadoop classpath`
install hadoop hive
curl -O https://dlcdn.apache.org/hadoop/common/hadoop-3.3.2/hadoop-3.3.2.tar.gz
tar xzvf hadoop-3.3.2.tar.gz
curl -O https://dlcdn.apache.org/hive/hive-3.1.2/apache-hive-3.1.2-bin.tar.gz
tar xzvf apache-hive-3.1.2-bin.tar.gz
.bashrc
export HIVE_HOME="/root/apache-hive-3.1.2-bin"
export PATH="${HIVE_HOME}/bin:$PATH"
export HADOOP_HOME="/root/hadoop-3.3.2"
export PATH="${HADOOP_HOME}/bin:$PATH"
export JAVA_HOME="/usr/java/jdk1.8.0_232-cloudera"
export PATH="${JAVA_HOME}/bin:$PATH"
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/opt/cloudera/cm/common_jars/*
tpc-ds
git clone https://github.com/hortonworks/hive-testbench.git
# yum -y install java-11-openjdk java-11-openjdk-devel
# cloudera openjdk8 already installed
yum -y groupinstall "Development tools"
./tpcds-build.sh
tpcds-setup.sh changes
DIR=s3a://<bucket name>/tpcds-tests
# Do the actual data load.
#hdfs dfs -mkdir -p ${DIR}
#hdfs dfs -ls ${DIR}/${SCALE} > /dev/null
hadoop fs -mkdir -p ${DIR}
hadoop fs -ls ${DIR}/${SCALE} > /dev/null
#hdfs dfs -ls ${DIR}/${SCALE} > /dev/null
hadoop fs -ls ${DIR}/${SCALE} > /dev/null
hdfs
catalog_sales
s3
catalog_sales
HIVE
HIVE="beeline -n hive -u 'jdbc:hive2://worker01:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2?tez.queue.name=default' "
https://stackoverflow.com/questions/13178182/i-have-created-a-table-in-hive-i-would-like-to-know-which-directory-my-table-is
use tpcds_bin_partitioned_orc_3;
show tables;
+-------------------------+
| tab_name |
+-------------------------+
| call_center |
| catalog_page |
| catalog_returns |
| catalog_sales |
| customer |
| customer_address |
| customer_demographics |
| date_dim |
| household_demographics |
| income_band |
| inventory |
| item |
| promotion |
| reason |
| ship_mode |
| store |
| store_returns |
| store_sales |
| time_dim |
| warehouse |
| web_page |
| web_returns |
| web_sales |
| web_site |
+-------------------------+
show create table customer;
+----------------------------------------------------+
| createtab_stmt |
+----------------------------------------------------+
| CREATE TABLE `customer`( |
| `c_customer_sk` bigint, |
| `c_customer_id` char(16), |
| `c_current_cdemo_sk` bigint, |
| `c_current_hdemo_sk` bigint, |
| `c_current_addr_sk` bigint, |
| `c_first_shipto_date_sk` bigint, |
| `c_first_sales_date_sk` bigint, |
| `c_salutation` char(10), |
| `c_first_name` char(20), |
| `c_last_name` char(30), |
| `c_preferred_cust_flag` char(1), |
| `c_birth_day` int, |
| `c_birth_month` int, |
| `c_birth_year` int, |
| `c_birth_country` varchar(20), |
| `c_login` char(13), |
| `c_email_address` char(50), |
| `c_last_review_date_sk` bigint) |
| ROW FORMAT SERDE |
| 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' |
| STORED AS INPUTFORMAT |
| 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' |
| OUTPUTFORMAT |
| 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' |
| LOCATION |
| 'hdfs://worker01:8020/warehouse/tablespace/managed/hive/tpcds_bin_partitioned_orc_3.db/customer' |
| TBLPROPERTIES ( |
| 'bucketing_version'='2', |
| 'transactional'='true', |
| 'transactional_properties'='default', |
| 'transient_lastDdlTime'='1648620728') |
+----------------------------------------------------+
dir size
[root@worker01 ~]# HADOOP_USER_NAME=hive hadoop fs -du -s -h hdfs://worker01:8020/warehouse/tablespace/managed/hive/tpcds_bin_partitioned_orc_3.db
776.9 M 2.3 G hdfs://worker01:8020/warehouse/tablespace/managed/hive/tpcds_bin_partitioned_orc_3.db
scan test
run sample query
DB=tpcds_bin_partitioned_orc_3
HIVE="beeline -n hive -u 'jdbc:hive2://worker01:2181/${DB};serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2?tez.queue.name=default' "
$HIVE -f sample-queries-tpcds/query55.sql
INFO : Query ID = hive_20220330024221_7f7bfcba-ecec-4705-a34b-e5c2a09941f0
INFO : Total jobs = 1
INFO : Launching Job 1 out of 1
INFO : Starting task [Stage-1:MAPRED] in serial mode
INFO : Subscribed to counters: [] for queryId: hive_20220330024221_7f7bfcba-ecec-4705-a34b-e5c2a09941f0
INFO : Tez session hasn't been created yet. Opening session
INFO : Dag name: select i_brand_id brand_id, i_brand b...100 (Stage-1)
INFO : Setting tez.task.scale.memory.reserve-fraction to 0.30000001192092896
INFO : Status: Running (Executing on YARN cluster with App id application_1648605587507_0050)
----------------------------------------------------------------------------------------------
VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED
----------------------------------------------------------------------------------------------
Map 4 .......... container SUCCEEDED 1 1 0 0 0 0
Map 5 .......... container SUCCEEDED 1 1 0 0 0 0
Map 1 .......... container SUCCEEDED 4 4 0 0 0 0
Reducer 2 ...... container SUCCEEDED 2 2 0 0 0 0
Reducer 3 ...... container SUCCEEDED 1 1 0 0 0 0
----------------------------------------------------------------------------------------------
VERTICES: 05/05 [==========================>>] 100% ELAPSED TIME: 8.81 s
----------------------------------------------------------------------------------------------
INFO : Status: DAG finished successfully in 8.75 seconds
INFO :
INFO : Query Execution Summary
INFO : ----------------------------------------------------------------------------------------------
INFO : OPERATION DURATION
INFO : ----------------------------------------------------------------------------------------------
INFO : Compile Query 1.91s
INFO : Prepare Plan 0.28s
INFO : Get Query Coordinator (AM) 2.92s
INFO : Submit Plan 0.21s
INFO : Start DAG 0.69s
INFO : Run DAG 8.75s
INFO : ----------------------------------------------------------------------------------------------
INFO :
INFO : Task Execution Summary
INFO : ----------------------------------------------------------------------------------------------
INFO : VERTICES DURATION(ms) CPU_TIME(ms) GC_TIME(ms) INPUT_RECORDS OUTPUT_RECORDS
INFO : ----------------------------------------------------------------------------------------------
INFO : Map 1 4041.00 8,640 347 261,300 847
INFO : Map 4 2055.00 3,090 197 10,000 30
INFO : Map 5 2055.00 4,120 201 36,000 666
INFO : Reducer 2 2493.00 610 20 662 180
INFO : Reducer 3 0.00 520 10 180 0
INFO : ----------------------------------------------------------------------------------------------
INFO :
...
+-----------+----------------------------------------------------+------------+
| brand_id | brand | ext_price |
+-----------+----------------------------------------------------+------------+
| 5004001 | edu packscholar #1 | 337350.41 |
| 1001001 | amalgamalg #1 | 306888.37 |
| 4003001 | exportiedu pack #1 | 279769.61 |
| 2003001 | exportiimporto #1 | 274121.71 |
| 3004001 | edu packexporti #1 | 221208.38 |
| 1004001 | edu packamalg #1 | 200234.81 |
| 5003001 | exportischolar #1 | 190790.46 |
| 5001001 | amalgscholar #1 | 168600.18 |
| 3001001 | amalgexporti #1 | 159413.70 |
| 5002002 | importoscholar #2 | 148707.10 |
| 2002001 | importoimporto #1 | 147463.91 |
| 2001001 | amalgimporto #1 | 144902.28 |
| 4004001 | edu packedu pack #1 | 135995.32 |
| 1004002 | edu packamalg #2 | 123532.17 |
| 3003001 | exportiexporti #1 | 110458.28 |
| 2004001 | edu packimporto #1 | 104203.76 |
| 4001001 | amalgedu pack #1 | 101562.49 |
AWS S3 Bucket Metrics
how to activate?
https://docs.aws.amazon.com/AmazonS3/latest/userguide/configure-request-metrics-bucket.html
Report
TPC-DS EMR
https://partners-intl.aliyun.com/help/en/e-mapreduce/latest/run-the-tpc-ds-benchmark-in-an-emr-cluster
deps
yum -y groupinstall "development tools"
git clone https://github.com/hortonworks/hive-testbench.git
./tpcds-build.sh
edit tpcds-setup.sh
DIR=s3a://<bucket name>/tpcds-tests
# Do the actual data load.
#hdfs dfs -mkdir -p ${DIR}
#hdfs dfs -ls ${DIR}/${SCALE} > /dev/null
hadoop fs -mkdir -p ${DIR}
hadoop fs -ls ${DIR}/${SCALE} > /dev/null
#hdfs dfs -ls ${DIR}/${SCALE} > /dev/null
hadoop fs -ls ${DIR}/${SCALE} > /dev/null
HIVE="beeline -n hive -u 'jdbc:hive2://localhost:10000/' "
./tpcds-build.sh 2
run test
HIVE="beeline -n hive -u 'jdbc:hive2://emr-header-1:10000/tpcds_bin_partitioned_orc_3'"
$HIVE -f sample-queries-tpc/query55
INFO : Query ID = hive_20220402013012_83082d46-ab32-41f6-884a-2833e6a05093
INFO : Total jobs = 1
INFO : Launching Job 1 out of 1
INFO : Starting task [Stage-1:MAPRED] in serial mode
INFO : Tez session hasn't been created yet. Opening session
INFO : Dag name: select i_brand_id brand_id, i_brand b...100(Stage-1)
INFO : Setting tez.task.scale.memory.reserve-fraction to 0.30000001192092896
INFO : Status: Running (Executing on YARN cluster with App id application_1648857118863_0028)
INFO : Map 1: -/- Map 2: -/- Map 5: -/- Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 0/1 Map 2: -/- Map 5: 0/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 0(+1)/1 Map 2: -/- Map 5: 0(+1)/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 0(+1)/1 Map 2: -/- Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: -/- Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 0(+1)/3 Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 0(+2)/3 Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 1(+1)/3 Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 2(+1)/3 Map 5: 1/1 Reducer 3: 0(+1)/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 0(+1)/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 1(+0)/2 Reducer 4: 0(+1)/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 1(+1)/2 Reducer 4: 0(+1)/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 2/2 Reducer 4: 0(+1)/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 2/2 Reducer 4: 1/1
INFO : Completed executing command(queryId=hive_20220402013012_83082d46-ab32-41f6-884a-2833e6a05093); Time taken: 18.14 seconds
INFO : OK
+-----------+----------------------------------------------------+------------+
| brand_id | brand | ext_price |
+-----------+----------------------------------------------------+------------+
| 5004001 | edu packscholar #1 | 337350.41 |
| 1001001 | amalgamalg #1 | 306888.37 |
| 4003001 | exportiedu pack #1 | 279769.61 |
| 2003001 | exportiimporto #1 | 274121.71 |
| 3004001 | edu packexporti #1 | 221208.38 |
| 1004001 | edu packamalg #1 | 200234.81 |
| 5003001 | exportischolar #1 | 190790.46 |
| 5001001 | amalgscholar #1 | 168600.18 |
| 3001001 | amalgexporti #1 | 159413.70 |
| 5002002 | importoscholar #2 | 148707.10 |
| 2002001 | importoimporto #1 | 147463.91 |
| 2001001 | amalgimporto #1 | 144902.28 |
| 4004001 | edu packedu pack #1 | 135995.32 |
| 1004002 | edu packamalg #2 | 123532.17 |
| 3003001 | exportiexporti #1 | 110458.28 |
| 2004001 | edu packimporto #1 | 104203.76 |
| 4001001 | amalgedu pack #1 | 101562.49 |
| 5003002 | exportischolar #2 | 101009.84 |
| 10001014 | amalgunivamalg #14 | 93799.11 |
| 1001002 | amalgamalg #2 | 91408.09 |
| 7007005 | brandbrand #5 | 91309.27 |
| 1003002 | exportiamalg #2 | 86496.12 |
| 7016010 | corpnameless #10 | 78875.90 |
| 8004007 | edu packnameless #7 | 77913.67 |
| 7003009 | exportibrand #9 | 71169.58 |
| 3002001 | importoexporti #1 | 70594.15 |
| 5002001 | importoscholar #1 | 68968.43 |
| 1003001 | exportiamalg #1 | 68759.31 |
| 2003002 | exportiimporto #2 | 66479.39 |
| 6003002 | exporticorp #2 | 64201.40 |
| 5001002 | amalgscholar #2 | 64185.23 |
| 4002001 | importoedu pack #1 | 63383.69 |
| 7015003 | scholarnameless #3 | 61502.14 |
| 8005005 | scholarnameless #5 | 59744.16 |
| 7002006 | importobrand #6 | 58838.31 |
| 7004007 | edu packbrand #7 | 58665.96 |
| 8014005 | edu packmaxi #5 | 57592.98 |
| 7013009 | exportinameless #9 | 54815.47 |
| 4004002 | edu packedu pack #2 | 54804.14 |
| 7004006 | edu packbrand #6 | 54315.32 |
| 8001006 | amalgnameless #6 | 53890.12 |
| 10009011 | maxiunivamalg #11 | 52720.81 |
| 6007006 | brandcorp #6 | 51816.72 |
| 3002002 | importoexporti #2 | 51112.55 |
| 7007006 | brandbrand #6 | 50456.66 |
| 3001002 | amalgexporti #2 | 50407.70 |
| 6005005 | scholarcorp #5 | 49396.28 |
| 9010005 | univunivamalg #5 | 49286.34 |
| 10001002 | amalgunivamalg #2 | 45503.97 |
| 10011004 | amalgamalgamalg #4 | 45297.94 |
| 10013008 | exportiamalgamalg #8 | 45257.85 |
| 7010008 | univnameless #8 | 43472.64 |
| 2001002 | amalgimporto #2 | 43435.17 |
| 10010004 | univamalgamalg #4 | 42338.64 |
| 6002005 | importocorp #5 | 42048.29 |
| 5004002 | edu packscholar #2 | 40445.10 |
| 7008003 | namelessbrand #3 | 40378.98 |
| 8011009 | amalgmaxi #9 | 40192.71 |
| 10007014 | brandunivamalg #14 | 39431.69 |
| 9006008 | corpmaxi #8 | 39375.89 |
| 7007010 | brandbrand #10 | 39078.99 |
| 7004005 | edu packbrand #5 | 38992.73 |
| 9008009 | namelessmaxi #9 | 38916.30 |
| 9002009 | importomaxi #9 | 38747.03 |
| 9012005 | importounivamalg #5 | 38348.01 |
| 10008012 | namelessunivamalg #12 | 38326.22 |
| 10005012 | scholarunivamalg #12 | 37574.90 |
| 2004002 | edu packimporto #2 | 36220.23 |
| 2002002 | importoimporto #2 | 36209.05 |
| 7002008 | importobrand #8 | 36051.15 |
| 10008003 | namelessunivamalg #3 | 35948.00 |
| 10003017 | exportiunivamalg #17 | 35498.82 |
| 6003003 | exporticorp #3 | 35405.42 |
| 4002002 | importoedu pack #2 | 34862.77 |
| 10001007 | amalgunivamalg #7 | 34604.96 |
| 10015004 | scholaramalgamalg #4 | 34545.14 |
| 8002007 | importonameless #7 | 34317.26 |
| 8012010 | importomaxi #10 | 34282.28 |
| 8013003 | exportimaxi #3 | 34220.49 |
| 9014002 | edu packunivamalg #2 | 34218.12 |
| 8006003 | corpnameless #3 | 34136.00 |
| 7012003 | importonameless #3 | 33982.14 |
| 6014003 | edu packbrand #3 | 33864.88 |
| 10009013 | maxiunivamalg #13 | 33631.74 |
| 10014016 | edu packamalgamalg #16 | 33170.33 |
| 9006009 | corpmaxi #9 | 33117.48 |
| 9010008 | univunivamalg #8 | 32912.16 |
| 9011005 | amalgunivamalg #5 | 31786.29 |
| 9002008 | importomaxi #8 | 31781.75 |
| 8015001 | scholarmaxi #1 | 30692.59 |
| 10003002 | exportiunivamalg #2 | 30605.83 |
| 6014005 | edu packbrand #5 | 30496.25 |
| 6002006 | importocorp #6 | 30490.86 |
| 7012010 | importonameless #10 | 30466.62 |
| 10001013 | amalgunivamalg #13 | 29974.03 |
| 7008001 | namelessbrand #1 | 29754.49 |
| 6013003 | exportibrand #3 | 29662.15 |
| 9014008 | edu packunivamalg #8 | 29656.41 |
| 10013017 | exportiamalgamalg #17 | 29502.08 |
| 4001002 | amalgedu pack #2 | 29396.03 |
+-----------+----------------------------------------------------+------------+
100 rows selected (21.119 seconds)
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_>
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_> -- end query 1 in stream 0 using template query55.tpl
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_>
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_> Closing: 0: jdbc:hive2://emr-header-1:10000/tpcds_bin_partitioned_orc_3
more…
INFO : Query ID = hive_20220402013012_83082d46-ab32-41f6-884a-2833e6a05093
INFO : Total jobs = 1
INFO : Launching Job 1 out of 1
INFO : Starting task [Stage-1:MAPRED] in serial mode
INFO : Tez session hasn't been created yet. Opening session
INFO : Dag name: select i_brand_id brand_id, i_brand b...100(Stage-1)
INFO : Setting tez.task.scale.memory.reserve-fraction to 0.30000001192092896
INFO : Status: Running (Executing on YARN cluster with App id application_1648857118863_0028)
INFO : Map 1: -/- Map 2: -/- Map 5: -/- Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 0/1 Map 2: -/- Map 5: 0/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 0(+1)/1 Map 2: -/- Map 5: 0(+1)/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 0(+1)/1 Map 2: -/- Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: -/- Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 0(+1)/3 Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 0(+2)/3 Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 1(+1)/3 Map 5: 1/1 Reducer 3: 0/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 2(+1)/3 Map 5: 1/1 Reducer 3: 0(+1)/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 0(+1)/2 Reducer 4: 0/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 1(+0)/2 Reducer 4: 0(+1)/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 1(+1)/2 Reducer 4: 0(+1)/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 2/2 Reducer 4: 0(+1)/1
INFO : Map 1: 1/1 Map 2: 3/3 Map 5: 1/1 Reducer 3: 2/2 Reducer 4: 1/1
INFO : Completed executing command(queryId=hive_20220402013012_83082d46-ab32-41f6-884a-2833e6a05093); Time taken: 18.14 seconds
INFO : OK
+-----------+----------------------------------------------------+------------+
| brand_id | brand | ext_price |
+-----------+----------------------------------------------------+------------+
| 5004001 | edu packscholar #1 | 337350.41 |
| 1001001 | amalgamalg #1 | 306888.37 |
| 4003001 | exportiedu pack #1 | 279769.61 |
| 2003001 | exportiimporto #1 | 274121.71 |
| 3004001 | edu packexporti #1 | 221208.38 |
| 1004001 | edu packamalg #1 | 200234.81 |
| 5003001 | exportischolar #1 | 190790.46 |
| 5001001 | amalgscholar #1 | 168600.18 |
| 3001001 | amalgexporti #1 | 159413.70 |
| 5002002 | importoscholar #2 | 148707.10 |
| 2002001 | importoimporto #1 | 147463.91 |
| 2001001 | amalgimporto #1 | 144902.28 |
| 4004001 | edu packedu pack #1 | 135995.32 |
| 1004002 | edu packamalg #2 | 123532.17 |
| 3003001 | exportiexporti #1 | 110458.28 |
| 2004001 | edu packimporto #1 | 104203.76 |
| 4001001 | amalgedu pack #1 | 101562.49 |
| 5003002 | exportischolar #2 | 101009.84 |
| 10001014 | amalgunivamalg #14 | 93799.11 |
| 1001002 | amalgamalg #2 | 91408.09 |
| 7007005 | brandbrand #5 | 91309.27 |
| 1003002 | exportiamalg #2 | 86496.12 |
| 7016010 | corpnameless #10 | 78875.90 |
| 8004007 | edu packnameless #7 | 77913.67 |
| 7003009 | exportibrand #9 | 71169.58 |
| 3002001 | importoexporti #1 | 70594.15 |
| 5002001 | importoscholar #1 | 68968.43 |
| 1003001 | exportiamalg #1 | 68759.31 |
| 2003002 | exportiimporto #2 | 66479.39 |
| 6003002 | exporticorp #2 | 64201.40 |
| 5001002 | amalgscholar #2 | 64185.23 |
| 4002001 | importoedu pack #1 | 63383.69 |
| 7015003 | scholarnameless #3 | 61502.14 |
| 8005005 | scholarnameless #5 | 59744.16 |
| 7002006 | importobrand #6 | 58838.31 |
| 7004007 | edu packbrand #7 | 58665.96 |
| 8014005 | edu packmaxi #5 | 57592.98 |
| 7013009 | exportinameless #9 | 54815.47 |
| 4004002 | edu packedu pack #2 | 54804.14 |
| 7004006 | edu packbrand #6 | 54315.32 |
| 8001006 | amalgnameless #6 | 53890.12 |
| 10009011 | maxiunivamalg #11 | 52720.81 |
| 6007006 | brandcorp #6 | 51816.72 |
| 3002002 | importoexporti #2 | 51112.55 |
| 7007006 | brandbrand #6 | 50456.66 |
| 3001002 | amalgexporti #2 | 50407.70 |
| 6005005 | scholarcorp #5 | 49396.28 |
| 9010005 | univunivamalg #5 | 49286.34 |
| 10001002 | amalgunivamalg #2 | 45503.97 |
| 10011004 | amalgamalgamalg #4 | 45297.94 |
| 10013008 | exportiamalgamalg #8 | 45257.85 |
| 7010008 | univnameless #8 | 43472.64 |
| 2001002 | amalgimporto #2 | 43435.17 |
| 10010004 | univamalgamalg #4 | 42338.64 |
| 6002005 | importocorp #5 | 42048.29 |
| 5004002 | edu packscholar #2 | 40445.10 |
| 7008003 | namelessbrand #3 | 40378.98 |
| 8011009 | amalgmaxi #9 | 40192.71 |
| 10007014 | brandunivamalg #14 | 39431.69 |
| 9006008 | corpmaxi #8 | 39375.89 |
| 7007010 | brandbrand #10 | 39078.99 |
| 7004005 | edu packbrand #5 | 38992.73 |
| 9008009 | namelessmaxi #9 | 38916.30 |
| 9002009 | importomaxi #9 | 38747.03 |
| 9012005 | importounivamalg #5 | 38348.01 |
| 10008012 | namelessunivamalg #12 | 38326.22 |
| 10005012 | scholarunivamalg #12 | 37574.90 |
| 2004002 | edu packimporto #2 | 36220.23 |
| 2002002 | importoimporto #2 | 36209.05 |
| 7002008 | importobrand #8 | 36051.15 |
| 10008003 | namelessunivamalg #3 | 35948.00 |
| 10003017 | exportiunivamalg #17 | 35498.82 |
| 6003003 | exporticorp #3 | 35405.42 |
| 4002002 | importoedu pack #2 | 34862.77 |
| 10001007 | amalgunivamalg #7 | 34604.96 |
| 10015004 | scholaramalgamalg #4 | 34545.14 |
| 8002007 | importonameless #7 | 34317.26 |
| 8012010 | importomaxi #10 | 34282.28 |
| 8013003 | exportimaxi #3 | 34220.49 |
| 9014002 | edu packunivamalg #2 | 34218.12 |
| 8006003 | corpnameless #3 | 34136.00 |
| 7012003 | importonameless #3 | 33982.14 |
| 6014003 | edu packbrand #3 | 33864.88 |
| 10009013 | maxiunivamalg #13 | 33631.74 |
| 10014016 | edu packamalgamalg #16 | 33170.33 |
| 9006009 | corpmaxi #9 | 33117.48 |
| 9010008 | univunivamalg #8 | 32912.16 |
| 9011005 | amalgunivamalg #5 | 31786.29 |
| 9002008 | importomaxi #8 | 31781.75 |
| 8015001 | scholarmaxi #1 | 30692.59 |
| 10003002 | exportiunivamalg #2 | 30605.83 |
| 6014005 | edu packbrand #5 | 30496.25 |
| 6002006 | importocorp #6 | 30490.86 |
| 7012010 | importonameless #10 | 30466.62 |
| 10001013 | amalgunivamalg #13 | 29974.03 |
| 7008001 | namelessbrand #1 | 29754.49 |
| 6013003 | exportibrand #3 | 29662.15 |
| 9014008 | edu packunivamalg #8 | 29656.41 |
| 10013017 | exportiamalgamalg #17 | 29502.08 |
| 4001002 | amalgedu pack #2 | 29396.03 |
+-----------+----------------------------------------------------+------------+
100 rows selected (21.119 seconds)
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_>
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_> -- end query 1 in stream 0 using template query55.tpl
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_>
0: jdbc:hive2://emr-header-1:10000/tpcds_bin_> Closing: 0: jdbc:hive2://emr-header-1:10000/tpcds_bin_partitioned_orc_3
aws tpdds-setup 200
