In [63]:

import pandas as pd
from sqlalchemy import create_engine
from IPython.display import Image

In [1]:

!pip install sqlalchemy pyhive

Requirement already satisfied: sqlalchemy in /opt/conda/lib/python3.7/site-packages (1.3.4)
Collecting pyhive
  Downloading https://files.pythonhosted.org/packages/07/bf/11ce30d3c12ac24bad1d20df4c574010f6038c6f7ed92dfdbe1cfc1178c4/PyHive-0.6.5.tar.gz (44kB)
     |████████████████████████████████| 51kB 15.0MB/s 
Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from pyhive) (0.17.1)
Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.7/site-packages (from pyhive) (2.8.0)
Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil->pyhive) (1.12.0)
Building wheels for collected packages: pyhive
  Building wheel for pyhive (setup.py) ... done
  Stored in directory: /root/.cache/pip/wheels/71/13/19/e9c49c3457801c84baf45a1048113957f10f4718e2c7386523
Successfully built pyhive
Installing collected packages: pyhive
Successfully installed pyhive-0.6.5

In [23]:

!apt install -y libsasl2-dev

Reading package lists... Done
Building dependency tree       
Reading state information... Done
The following additional packages will be installed:
  libsasl2-2 libsasl2-modules libsasl2-modules-db
Suggested packages:
  libsasl2-modules-gssapi-mit | libsasl2-modules-gssapi-heimdal
  libsasl2-modules-ldap libsasl2-modules-otp libsasl2-modules-sql
The following NEW packages will be installed:
  libsasl2-dev
The following packages will be upgraded:
  libsasl2-2 libsasl2-modules libsasl2-modules-db
3 upgraded, 1 newly installed, 0 to remove and 119 not upgraded.
Need to get 586 kB of archives.
After this operation, 887 kB of additional disk space will be used.
Get:1 http://deb.debian.org/debian stretch/main amd64 libsasl2-modules-db amd64 2.1.27~101-g0780600+dfsg-3+deb9u1 [68.4 kB]
Get:2 http://deb.debian.org/debian stretch/main amd64 libsasl2-2 amd64 2.1.27~101-g0780600+dfsg-3+deb9u1 [105 kB]
Get:3 http://deb.debian.org/debian stretch/main amd64 libsasl2-dev amd64 2.1.27~101-g0780600+dfsg-3+deb9u1 [310 kB]
Get:4 http://deb.debian.org/debian stretch/main amd64 libsasl2-modules amd64 2.1.27~101-g0780600+dfsg-3+deb9u1 [102 kB]
Fetched 586 kB in 0s (688 kB/s)
perl: warning: Setting locale failed.
perl: warning: Please check that your locale settings:
	LANGUAGE = "en_US.UTF-8",
	LC_ALL = "en_US.UTF-8",
	LANG = "en_US.UTF-8"
    are supported and installed on your system.
perl: warning: Falling back to the standard locale ("C").
locale: Cannot set LC_CTYPE to default locale: No such file or directory
locale: Cannot set LC_MESSAGES to default locale: No such file or directory
locale: Cannot set LC_ALL to default locale: No such file or directory
debconf: delaying package configuration, since apt-utils is not installed

(Reading database ... 40408 files and directories currently installed.)
Preparing to unpack .../libsasl2-modules-db_2.1.27~101-g0780600+dfsg-3+deb9u1_amd64.deb ...
7Progress: [  0%] [..........................................................] 87Progress: [  4%] [##........................................................] 8Unpacking libsasl2-modules-db:amd64 (2.1.27~101-g0780600+dfsg-3+deb9u1) over (2.1.27~101-g0780600+dfsg-3) ...
7Progress: [  9%] [#####.....................................................] 87Progress: [ 14%] [########..................................................] 8Preparing to unpack .../libsasl2-2_2.1.27~101-g0780600+dfsg-3+deb9u1_amd64.deb ...
7Progress: [ 19%] [###########...............................................] 8Unpacking libsasl2-2:amd64 (2.1.27~101-g0780600+dfsg-3+deb9u1) over (2.1.27~101-g0780600+dfsg-3) ...
7Progress: [ 23%] [#############.............................................] 87Progress: [ 28%] [################..........................................] 8Selecting previously unselected package libsasl2-dev.
Preparing to unpack .../libsasl2-dev_2.1.27~101-g0780600+dfsg-3+deb9u1_amd64.deb ...
7Progress: [ 33%] [###################.......................................] 8Unpacking libsasl2-dev (2.1.27~101-g0780600+dfsg-3+deb9u1) ...
7Progress: [ 38%] [######################....................................] 87Progress: [ 42%] [########################..................................] 8Preparing to unpack .../libsasl2-modules_2.1.27~101-g0780600+dfsg-3+deb9u1_amd64.deb ...
7Progress: [ 47%] [###########################...............................] 8Unpacking libsasl2-modules:amd64 (2.1.27~101-g0780600+dfsg-3+deb9u1) over (2.1.27~101-g0780600+dfsg-3) ...
7Progress: [ 52%] [##############################............................] 87Progress: [ 57%] [#################################.........................] 8Setting up libsasl2-modules-db:amd64 (2.1.27~101-g0780600+dfsg-3+deb9u1) ...
7Progress: [ 61%] [###################################.......................] 87Progress: [ 66%] [######################################....................] 8Setting up libsasl2-2:amd64 (2.1.27~101-g0780600+dfsg-3+deb9u1) ...
7Progress: [ 71%] [#########################################.................] 87Progress: [ 76%] [############################################..............] 8Setting up libsasl2-dev (2.1.27~101-g0780600+dfsg-3+deb9u1) ...
7Progress: [ 80%] [##############################################............] 87Progress: [ 85%] [#################################################.........] 8Processing triggers for libc-bin (2.24-11+deb9u4) ...
Setting up libsasl2-modules:amd64 (2.1.27~101-g0780600+dfsg-3+deb9u1) ...
7Progress: [ 90%] [####################################################......] 87Progress: [ 95%] [#######################################################...] 8
78

In [27]:

!pip install thrift sasl thrift-sasl

Requirement already satisfied: thrift in /opt/conda/lib/python3.7/site-packages (0.15.0)
Requirement already satisfied: sasl in /opt/conda/lib/python3.7/site-packages (0.3.1)
Collecting thrift-sasl
  Downloading https://files.pythonhosted.org/packages/c3/9e/636c24ce1c0d46ce3020c5836c5a375d8e862fa81a240e0e352cc991dcf8/thrift_sasl-0.4.3-py2.py3-none-any.whl
Requirement already satisfied: six>=1.7.2 in /opt/conda/lib/python3.7/site-packages (from thrift) (1.12.0)
Collecting pure-sasl>=0.6.2 (from thrift-sasl)
  Downloading https://files.pythonhosted.org/packages/83/b7/a0d688f86c869073cc28c0640899394a1cf68a6d87ee78a09565e9037da6/pure-sasl-0.6.2.tar.gz
Building wheels for collected packages: pure-sasl
  Building wheel for pure-sasl (setup.py) ... done
  Stored in directory: /root/.cache/pip/wheels/8d/34/73/6ed9e69a7ffd204e8d71bf422f9fa028f551d0dac431dc1082
Successfully built pure-sasl
ERROR: thrift-sasl 0.4.3 has requirement six>=1.13.0, but you'll have six 1.12.0 which is incompatible.
Installing collected packages: pure-sasl, thrift-sasl
Successfully installed pure-sasl-0.6.2 thrift-sasl-0.4.3

In [65]:

#https://stackoverflow.com/questions/21370431/how-to-access-hive-via-python

In [64]:

Image("http://api-dupi2.localshi.com/image/1d9fa23fe86f487c9f8f6fc1b0f3c268.png")

Out[64]:

In [59]:

conn = "hive://hive:hive@hadoop:10000/default"

In [60]:

engine = create_engine(conn, connect_args={'auth': 'LDAP'})

In [61]:

data = pd.read_sql("select * from table3", con=engine)

In [62]:

data

Out[62]:

	col1	col2	col3	col4
0	499	["Poole","GBR"]	England	141000
1	501	["Blackburn","GBR"]	England	140000
2	500	["Bolton","GBR"]	England	139020
3	502	["Newport","GBR"]	Wales	139000
4	503	["PrestON","GBR"]	England	135000
5	504	["Stockport","GBR"]	England	132813

In [ ]: