Carlos Aguni

Highly motivated self-taught IT analyst. Always learning and ready to explore new skills. An eternal apprentice.


Python + Selenium + Firefox + pyvirtualdisplay + Tabs + HUE

12 Apr 2022 »

https://www.geeksforgeeks.org/python-opening-multiple-tabs-using-selenium/

yum -y install Xvfb tigervnc-server-minimal x11vnc
pip3 install pyvirtualdisplay
yum -y install firefox
# get geckodriver from github

install python3-webcokify for novnc

wget https://github.com/novnc/websockify/archive/refs/tags/v0.10.0.zip
yum -y install unzip python3.9
unzip v0.10.0.zip
cd websockify-0.10.0/
# need python3.9
python3.9 setup.py install
yum -y install https://rpmfind.net/linux/epel/9/Everything/x86_64/Packages/p/python3-websockify-0.10.0-1.el9.noarch.rpm
yum -y install novnc

#check if websockify is using python3.9
#sed '1c#!/usr/bin/python3.9' /usr/local/bin/websockify

yum -y install screen
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import os
from pyvirtualdisplay import Display
#display = Display(visible=0, size=(1024,768))
os.system("screen -x novnc -X quit")
ss = [i.strip() for i in os.popen("ss -npltu").read().split("\n") if "5904" in i]
if ss:
    print(ss)
    pid = ss[0].split("pid=")[1].split(",")[0]
    os.system(f"kill {pid}")
display = Display(backend="xvnc", size=(1024, 768), rfbport=5904)
display.start()
os.system("screen -dmS novnc bash -c \"novnc_server --listen 7900 --vnc localhost:5904\"")

print("start firefox")

options = webdriver.FirefoxOptions()
#options.add_argument("--headless")
driver = webdriver.Firefox(options=options)


url="https://crashlaker.github.io/"
driver.get(url)
time.sleep(3)

print("get ok")

# add new tab
# https://stackoverflow.com/questions/28431765/open-web-in-new-tab-selenium-python
for i in range(3):
    driver.execute_script("window.open('https://crashlaker.github.io','_blank');");
    time.sleep(2)


tabs = driver.window_handles
print("tabs", tabs)

for i in range(4):
    print("go to", i, tabs[i])
    driver.switch_to.window(tabs[i])
    time.sleep(8)


driver.get_screenshot_as_file("test.png")
time.sleep(30)
driver.quit()
display.stop()

HUE

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import os
from pyvirtualdisplay import Display
#display = Display(visible=0, size=(1024,768))
os.system("screen -x novnc -X quit")
ss = [i.strip() for i in os.popen("ss -npltu").read().split("\n") if "5904" in i]
if ss:
    print(ss)
    pid = ss[0].split("pid=")[1].split(",")[0]
    os.system(f"kill {pid}")
#display = Display(backend="xvnc", size=(1024, 768), rfbport=5904)
display = Display(backend="xvnc", size=(1368, 640), rfbport=5904)
display.start()
time.sleep(2)
os.system("screen -dmS novnc bash -c \"novnc_server --listen 7900 --vnc localhost:5904\"")

print("start firefox")

options = webdriver.FirefoxOptions()
#options.add_argument("--headless")
driver = webdriver.Firefox(options=options)
#driver.maximize_window()
#driver.implicitly_wait(30)
#driver.set_window_size(1024, 768)
driver.maximize_window()


url="http://worker01:8888/hue/accounts/login?next=/hue/editor/%3Ftype%3Dhive"
driver.get(url)
time.sleep(3)

driver.fullscreen_window()

driver.find_element_by_css_selector("input[name='username']").send_keys("admin")
driver.find_element_by_css_selector("input[name='password']").send_keys("admin")
driver.find_element_by_css_selector("input[type='submit']").click()

time.sleep(5)

print("textarea")

driver.execute_script("""
var editor_id = document.querySelector('div.ace-hue[id]').id
var query = `
use tpcds_bin_partitioned_orc_3;
select
  ca_state,
  cd_gender,
  cd_marital_status,
  cd_dep_count,
  count(*) cnt1,
  avg(cd_dep_count),
  min(cd_dep_count),
  stddev_samp(cd_dep_count),
  cd_dep_employed_count,
  count(*) cnt2,
  avg(cd_dep_employed_count),
  min(cd_dep_employed_count),
  stddev_samp(cd_dep_employed_count),
  cd_dep_college_count,
  count(*) cnt3,
  avg(cd_dep_college_count),
  min(cd_dep_college_count),
  stddev_samp(cd_dep_college_count)
 from
  customer c,customer_address ca,customer_demographics
 where
  c.c_current_addr_sk = ca.ca_address_sk and
  cd_demo_sk = c.c_current_cdemo_sk and
  exists (select *
          from store_sales,date_dim
          where c.c_customer_sk = ss_customer_sk and
                ss_sold_date_sk = d_date_sk and
                d_year = 2001 and
                d_qoy < 4) and
   (exists (select *
            from web_sales,date_dim
            where c.c_customer_sk = ws_bill_customer_sk and
                  ws_sold_date_sk = d_date_sk and
                  d_year = 2001 and
                  d_qoy < 4) or
    exists (select *
            from catalog_sales,date_dim
            where c.c_customer_sk = cs_ship_customer_sk and
                  cs_sold_date_sk = d_date_sk and
                  d_year = 2001 and
                  d_qoy < 4))
 group by ca_state,
          cd_gender,
          cd_marital_status,
          cd_dep_count,
          cd_dep_employed_count,
          cd_dep_college_count
 order by ca_state,
          cd_gender,
          cd_marital_status,
          cd_dep_count,
          cd_dep_employed_count,
          cd_dep_college_count
 limit 100;
`
ace.edit(editor_id).setValue(query)
ace.edit(editor_id).execCommand('execute')
""")


time.sleep(3000)
driver.quit()
display.stop()

document.querySelector('.fa-stop').parentElement.style.display
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import random
import os
import requests
from pyvirtualdisplay import Display
#display = Display(visible=0, size=(1024,768))
os.system("screen -x novnc -X quit")
os.system("screen -x fluxbox -X quit")
ss = [i.strip() for i in os.popen("ss -npltu").read().split("\n") if "5904" in i]
if ss:
    print(ss)
    pid = ss[0].split("pid=")[1].split(",")[0]
    os.system(f"kill {pid}")
display = Display(backend="xvnc", size=(1024, 768), rfbport=5904)
#display = Display(backend="xvnc", size=(1368, 640), rfbport=5904)
display.start()

displayid = os.popen("env | grep DISPLAY").read().strip()
displayid = displayid.split("=")[1]

time.sleep(2)
os.system("screen -dmS novnc bash -c \"novnc_server --listen 7900 --vnc localhost:5904\"")
os.system(f"screen -dmS fluxbox bash -c \"DISPLAY={displayid} fluxbox\"")

print("start firefox")

options = webdriver.FirefoxOptions()
#options.add_argument("--headless")
driver = webdriver.Firefox(options=options)
#driver.maximize_window()
#driver.implicitly_wait(30)
#driver.set_window_size(1024, 768)
#driver.maximize_window()


url="http://worker01:8888/hue/accounts/login?next=/hue/editor/%3Ftype%3Dhive"
driver.get(url)
time.sleep(3)

#driver.fullscreen_window()

driver.find_element_by_css_selector("input[name='username']").send_keys("admin")
driver.find_element_by_css_selector("input[name='password']").send_keys("admin")
#driver.find_element_by_css_selector("input[type='submit']").click()

submit = driver.find_element_by_css_selector("input[type='submit']")
actions = ActionChains(driver)
actions.move_to_element(submit).click().perform()


time.sleep(5)

if 0:
    driver.execute_script(f"window.open('{url}', '_blank')")

    time.sleep(5)

tabs = driver.window_handles

print("tabs", tabs)

queries = []
folder = "./sample-queries-tpcds"
files = [filename for filename in os.listdir(folder) if filename.endswith(".sql")]
for filename in random.sample(files, 10):
    print(filename)
    sql = open(folder+"/"+filename).read().strip()
    sql = "\n".join([i for i in sql.split("\n") if not i.startswith("--")])
    sql = sql.replace("`", "\\`")
    sql = ("`\n"
          "use tpcds_bin_partitioned_orc_3;\n"
          f"{sql}"
          "\n`")
    print(sql)
    queries.append(sql)

queries1 = queries

queries = []
folder = "./sample-queries-tpcds"
files = [filename for filename in os.listdir(folder) if filename.endswith(".sql")]
for filename in random.sample(files, 10):
    print(filename)
    sql = open(folder+"/"+filename).read().strip()
    sql = "\n".join([i for i in sql.split("\n") if not i.startswith("--")])
    sql = sql.replace("`", "\\`")
    sql = ("`\n"
          "use tpcds_bin_partitioned_orc_3;\n"
          f"{sql}"
          "\n`")
    print(sql)
    queries.append(sql)

queries2 = queries

print(len(queries))

def exec_queries(q):
    driver.execute_script("""

    var queries = [QUERIES]
    window.queries = [QUERIES]

    window.finished = false
    var state = 'pause' 

    var play = document.querySelector('a[title="Execute or CTRL + ENTER"]')
    var cancel = document.querySelector('a[title="Cancel operation"]')
    var errorcontainer = document.querySelector('div.snippet-error-container')
    var exectimer = document.querySelector('span.execution-timer')
    var editor_id = document.querySelector('div.ace-hue[id]').id
    var acem = ace.edit(editor_id)

    var isplaying = () => {
            return play.style.display == 'none'
    }

    window.result = []

    var loop = (idx) => {
            if (isplaying()){
                    console.log('is playing')
                    setTimeout(() => {
                            loop(idx)
                    }, 2000)
            }else{
                    console.log('not playing')
                    if (idx > 0){
                            if (errorcontainer.style.display != 'none'){
                                    // error found
                                    result.push([false, exectimer.innerHTML])
                            }else{
                                    result.push([true, exectimer.innerHTML])
                            }
                        if (window.result.length == queries.length){
                            setTimeout(() => {
                                window.finished = true
                            }, 3000)
                            return
                        }
                    }
                    state = 'running'
                    console.log('run query', idx)
                    acem.setValue(queries[idx])
                    acem.execCommand('execute')
                    setTimeout(() => {
                            loop(idx+1)
                    }, 1000)
            }
    }
    loop(0)
    """.replace("QUERIES", ",\n".join(q)))

driver.switch_to.window(tabs[0])
time.sleep(5)
exec_queries(queries1)

if 0:
    driver.switch_to.window(tabs[1])
    time.sleep(5)
    exec_queries(queries2)

allfinished = False
while not allfinished:
    allfinished = True
    for idx in range(len(tabs)):
        driver.switch_to.window(tabs[idx])
        finished = driver.execute_script("return window.finished")
        if not finished:
            allfinished = False
        result = driver.execute_script("return window.result")
        print(idx, finished, result)
        time.sleep(3)
    time.sleep(10)

print("finished")

time.sleep(3000)
driver.quit()
display.stop()
asdfsd

print("get ok")

# add new tab
# https://stackoverflow.com/questions/28431765/open-web-in-new-tab-selenium-python
for i in range(3):
    driver.execute_script("window.open('https://crashlaker.github.io','_blank');");
    time.sleep(2)


# https://stackoverflow.com/questions/10629815/how-to-switch-to-new-window-in-selenium-for-python
tabs = driver.window_handles
print("tabs", tabs)

for i in range(4):
    print("go to", i, tabs[i])
    driver.switch_to.window(tabs[i])
    time.sleep(8)


driver.get_screenshot_as_file("test.png")