├── .gitignore ├── chromedriver ├── 01_02_03_selenium_install_useragent_proxy_chrome.py ├── 04_selenium_form_chrome.py ├── 05_selenium_cookies_chrome.py ├── 06_selenium_webdriver_mode_chrome.py ├── 07_selenium_headless_chrome.py ├── 08_selenium_switch_to_window_chrome.py ├── 09_selenium_multiprocessing_chrome.py ├── 10_selenium_vps.py ├── auth_data.py ├── proxy_auth_data.py └── test.py ├── firefoxdriver ├── 01_02_03_selenium_install_useragent_proxy_firefox.py ├── 04_selenium_form_firefox.py ├── 05_selenium_cookies_firefox.py ├── 06_selenium_webdriver_mode_firefox.py ├── 07_selenium_headless_firefox.py ├── 08_selenium_switch_to_window_firefox.py ├── 09_selenium_multiprocessing_firefox.py ├── auth_data.py └── proxy_auth_data.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | venvSelenium 2 | *.pyc 3 | .idea 4 | chromedriver/chromedriver 5 | firefoxdriver/geckodriver 6 | -------------------------------------------------------------------------------- /chromedriver/01_02_03_selenium_install_useragent_proxy_chrome.py: -------------------------------------------------------------------------------- 1 | # from selenium import webdriver 2 | from seleniumwire import webdriver 3 | import time 4 | import random 5 | from fake_useragent import UserAgent 6 | from proxy_auth_data import login, password 7 | 8 | # url = "https://www.instagram.com/" 9 | 10 | user_agents_list = [ 11 | "hello_world", 12 | "best_of_the_best", 13 | "python_today" 14 | ] 15 | 16 | # options 17 | options = webdriver.ChromeOptions() 18 | 19 | # change useragent 20 | useragent = UserAgent() 21 | 22 | # options.add_argument("user-agent=HelloWorld:)") 23 | # options.add_argument("user-agent=Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36") 24 | # options.add_argument(f"user-agent={random.choice(user_agents_list)}") 25 | options.add_argument(f"user-agent={useragent.random}") 26 | 27 | # set proxy 28 | # options.add_argument("--proxy-server=138.128.91.65:8000") 29 | 30 | proxy_options = { 31 | "proxy": { 32 | "https": f"http://{login}:{password}@138.128.91.65:8000" 33 | } 34 | } 35 | 36 | # driver = webdriver.Chrome( 37 | # executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 38 | # options=options 39 | # ) 40 | driver = webdriver.Chrome( 41 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 42 | seleniumwire_options=proxy_options 43 | ) 44 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 45 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 46 | 47 | try: 48 | # driver.get(url="https://www.whatismybrowser.com/detect/what-is-my-user-agent") 49 | # time.sleep(5) 50 | # driver.get(url="https://stackoverflow.com/") 51 | # time.sleep(5) 52 | 53 | # driver.refresh() 54 | # driver.get_screenshot_as_file("1.png") 55 | # driver.get(url="https://stackoverflow.com/") 56 | # time.sleep(5) 57 | # driver.save_screenshot("2.png") 58 | # time.sleep(2) 59 | 60 | driver.get("https://2ip.ru") 61 | time.sleep(5) 62 | 63 | except Exception as ex: 64 | print(ex) 65 | finally: 66 | driver.close() 67 | driver.quit() 68 | -------------------------------------------------------------------------------- /chromedriver/04_selenium_form_chrome.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import vk_password 5 | 6 | # options 7 | options = webdriver.ChromeOptions() 8 | 9 | # user-agent 10 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 11 | 12 | driver = webdriver.Chrome( 13 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 14 | options=options 15 | ) 16 | 17 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 18 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 19 | 20 | try: 21 | driver.get("https://vk.com/") 22 | time.sleep(5) 23 | 24 | email_input = driver.find_element_by_id("index_email") 25 | email_input.clear() 26 | email_input.send_keys("79623618753") 27 | time.sleep(5) 28 | 29 | password_input = driver.find_element_by_id("index_pass") 30 | password_input.clear() 31 | password_input.send_keys(vk_password) 32 | time.sleep(3) 33 | password_input.send_keys(Keys.ENTER) 34 | 35 | # login_button = driver.find_element_by_id("index_login_button").click() 36 | time.sleep(10) 37 | 38 | news_link = driver.find_element_by_id("l_nwsf").click() 39 | time.sleep(5) 40 | 41 | except Exception as ex: 42 | print(ex) 43 | finally: 44 | driver.close() 45 | driver.quit() 46 | -------------------------------------------------------------------------------- /chromedriver/05_selenium_cookies_chrome.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import vk_password, vk_phone 5 | import pickle 6 | 7 | # options 8 | options = webdriver.ChromeOptions() 9 | 10 | # user-agent 11 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | driver = webdriver.Chrome( 14 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 15 | options=options 16 | ) 17 | 18 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 19 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 20 | 21 | try: 22 | # driver.get("https://vk.com/") 23 | # time.sleep(5) 24 | # 25 | # email_input = driver.find_element_by_id("index_email") 26 | # email_input.clear() 27 | # email_input.send_keys(vk_phone) 28 | # time.sleep(5) 29 | # 30 | # password_input = driver.find_element_by_id("index_pass") 31 | # password_input.clear() 32 | # password_input.send_keys(vk_password) 33 | # time.sleep(3) 34 | # password_input.send_keys(Keys.ENTER) 35 | # 36 | # # login_button = driver.find_element_by_id("index_login_button").click() 37 | # time.sleep(10) 38 | # 39 | # news_link = driver.find_element_by_id("l_nwsf").click() 40 | # time.sleep(5) 41 | # 42 | # # cookies 43 | # pickle.dump(driver.get_cookies(), open(f"{vk_phone}_cookies", "wb")) 44 | 45 | driver.get("https://vk.com/") 46 | time.sleep(5) 47 | 48 | for cookie in pickle.load(open(f"{vk_phone}_cookies", "rb")): 49 | driver.add_cookie(cookie) 50 | 51 | time.sleep(5) 52 | driver.refresh() 53 | time.sleep(10) 54 | 55 | except Exception as ex: 56 | print(ex) 57 | finally: 58 | driver.close() 59 | driver.quit() 60 | -------------------------------------------------------------------------------- /chromedriver/06_selenium_webdriver_mode_chrome.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | import time 3 | 4 | 5 | # options 6 | options = webdriver.ChromeOptions() 7 | 8 | # user-agent 9 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 10 | 11 | # disable webdriver mode 12 | 13 | # # for older ChromeDriver under version 79.0.3945.16 14 | # options.add_experimental_option("excludeSwitches", ["enable-automation"]) 15 | # options.add_experimental_option("useAutomationExtension", False) 16 | 17 | # for ChromeDriver version 79.0.3945.16 or over 18 | options.add_argument("--disable-blink-features=AutomationControlled") 19 | 20 | driver = webdriver.Chrome( 21 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 22 | options=options 23 | ) 24 | 25 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 26 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 27 | 28 | try: 29 | driver.get("https://intoli.com/blog/not-possible-to-block-chrome-headless/chrome-headless-test.html") 30 | time.sleep(10) 31 | except Exception as ex: 32 | print(ex) 33 | finally: 34 | driver.close() 35 | driver.quit() 36 | -------------------------------------------------------------------------------- /chromedriver/07_selenium_headless_chrome.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import vk_password, vk_phone 5 | import pickle 6 | 7 | # options 8 | options = webdriver.ChromeOptions() 9 | 10 | # user-agent 11 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | # for ChromeDriver version 79.0.3945.16 or over 14 | options.add_argument("--disable-blink-features=AutomationControlled") 15 | 16 | # headless mode 17 | # options.add_argument("--headless") 18 | options.headless = True 19 | 20 | driver = webdriver.Chrome( 21 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 22 | options=options 23 | ) 24 | 25 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 26 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 27 | 28 | try: 29 | driver.get("https://vk.com/") 30 | time.sleep(5) 31 | 32 | print("Passing authentication...") 33 | email_input = driver.find_element_by_id("index_email") 34 | email_input.clear() 35 | email_input.send_keys(vk_phone) 36 | time.sleep(5) 37 | 38 | password_input = driver.find_element_by_id("index_pass") 39 | password_input.clear() 40 | password_input.send_keys(vk_password) 41 | time.sleep(3) 42 | password_input.send_keys(Keys.ENTER) 43 | 44 | # login_button = driver.find_element_by_id("index_login_button").click() 45 | time.sleep(10) 46 | 47 | print("Going to the profile page...") 48 | profile_page = driver.find_element_by_id("l_pr").click() 49 | time.sleep(5) 50 | 51 | print("Start watching the video...") 52 | video_block = driver.find_element_by_class_name("VideoPreview__thumbWrap").click() 53 | time.sleep(5) 54 | print("Finish watching the video...") 55 | 56 | except Exception as ex: 57 | print(ex) 58 | finally: 59 | driver.close() 60 | driver.quit() 61 | -------------------------------------------------------------------------------- /chromedriver/08_selenium_switch_to_window_chrome.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | import time 3 | import datetime 4 | 5 | # options 6 | options = webdriver.ChromeOptions() 7 | 8 | # user-agent 9 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 10 | 11 | # for ChromeDriver version 79.0.3945.16 or over 12 | options.add_argument("--disable-blink-features=AutomationControlled") 13 | 14 | # headless mode 15 | # options.add_argument("--headless") 16 | # options.headless = True 17 | 18 | driver = webdriver.Chrome( 19 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 20 | options=options 21 | ) 22 | 23 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 24 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 25 | 26 | try: 27 | start_time = datetime.datetime.now() 28 | 29 | driver.get("https://www.avito.ru/moskva/tovary_dlya_kompyutera/komplektuyuschie/videokarty") 30 | # print(driver.window_handles) 31 | print(f"Currently URL is: {driver.current_url}") 32 | 33 | # time.sleep(5) 34 | driver.implicitly_wait(5) 35 | 36 | items = driver.find_elements_by_xpath("//div[@data-marker='item-photo']") 37 | items[0].click() 38 | # print(driver.window_handles) 39 | 40 | # time.sleep(5) 41 | driver.implicitly_wait(5) 42 | 43 | driver.switch_to.window(driver.window_handles[1]) 44 | 45 | # time.sleep(5) 46 | driver.implicitly_wait(5) 47 | 48 | print(f"Currently URL is: {driver.current_url}") 49 | 50 | username = driver.find_element_by_class_name("seller-info-name") 51 | print(f"User name is: {username.text}") 52 | print("#" * 20) 53 | 54 | # time.sleep(5) 55 | driver.implicitly_wait(5) 56 | 57 | driver.close() 58 | 59 | driver.switch_to.window(driver.window_handles[0]) 60 | 61 | # time.sleep(5) 62 | driver.implicitly_wait(5) 63 | 64 | print(f"Currently URL is: {driver.current_url}") 65 | 66 | items[1].click() 67 | 68 | # time.sleep(5) 69 | driver.implicitly_wait(5) 70 | 71 | driver.switch_to.window(driver.window_handles[1]) 72 | 73 | # time.sleep(5) 74 | driver.implicitly_wait(5) 75 | 76 | print(f"Currently URL is: {driver.current_url}") 77 | username = driver.find_element_by_xpath("//div[@data-marker='seller-info/name']") 78 | print(f"User name is: {username.text}") 79 | print("-" * 20) 80 | 81 | ad_date = driver.find_element_by_class_name("title-info-metadata-item-redesign") 82 | print(f"An ad date is: {ad_date.text}") 83 | print("-" * 20) 84 | 85 | joined_date = driver.find_elements_by_class_name("seller-info-value")[1] 86 | print(f"User since: {joined_date.text}") 87 | print("#" * 20) 88 | 89 | # time.sleep(5) 90 | driver.implicitly_wait(5) 91 | 92 | finish_time = datetime.datetime.now() 93 | spent_time = finish_time - start_time 94 | print(spent_time) 95 | 96 | except Exception as ex: 97 | print(ex) 98 | finally: 99 | driver.close() 100 | driver.quit() 101 | 102 | "0:00:47.315298" 103 | "0:00:14.850836" 104 | -------------------------------------------------------------------------------- /chromedriver/09_selenium_multiprocessing_chrome.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from selenium import webdriver 4 | import time 5 | from multiprocessing import Pool 6 | 7 | # options 8 | options = webdriver.ChromeOptions() 9 | 10 | # user-agent 11 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | # for ChromeDriver version 79.0.3945.16 or over 14 | options.add_argument("--disable-blink-features=AutomationControlled") 15 | 16 | # headless mode 17 | # options.add_argument("--headless") 18 | # options.headless = True 19 | 20 | # urls_list = ["https://stackoverflow.com", "https://instagram.com", "https://vk.com"] 21 | 22 | 23 | # def get_data(url): 24 | # try: 25 | # driver = webdriver.Chrome( 26 | # executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 27 | # options=options 28 | # ) 29 | # # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 30 | # # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 31 | # driver.get(url=url) 32 | # time.sleep(5) 33 | # driver.get_screenshot_as_file(f"media/{url.split('//')[1]}.png") 34 | # except Exception as ex: 35 | # print(ex) 36 | # finally: 37 | # driver.close() 38 | # driver.quit() 39 | # 40 | # 41 | # if __name__ == '__main__': 42 | # p = Pool(processes=2) 43 | # p.map(get_data, urls_list) 44 | 45 | def get_data(url): 46 | try: 47 | driver = webdriver.Chrome( 48 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 49 | options=options 50 | ) 51 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 52 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 53 | driver.get(url=url) 54 | time.sleep(5) 55 | driver.find_element_by_class_name("lazyload-wrapper").find_element_by_class_name("item-video-container").click() 56 | time.sleep(random.randrange(3, 10)) 57 | except Exception as ex: 58 | print(ex) 59 | finally: 60 | driver.close() 61 | driver.quit() 62 | 63 | 64 | if __name__ == '__main__': 65 | process_count = int(input("Enter the number of processes: ")) 66 | url = input("Enter the URL: ") 67 | urls_list = [url] * process_count 68 | print(urls_list) 69 | p = Pool(processes=process_count) 70 | p.map(get_data, urls_list) -------------------------------------------------------------------------------- /chromedriver/10_selenium_vps.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import vk_password, vk_phone 5 | import pickle 6 | 7 | # options 8 | options = webdriver.ChromeOptions() 9 | 10 | # user-agent 11 | options.add_argument("user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | # for ChromeDriver version 79.0.3945.16 or over 14 | options.add_argument("--disable-blink-features=AutomationControlled") 15 | 16 | # headless mode 17 | # options.add_argument("--headless") 18 | options.headless = True 19 | 20 | driver = webdriver.Chrome( 21 | executable_path="/home/cain/PycharmProjects/selenium_python/chromedriver/chromedriver", 22 | options=options 23 | ) 24 | 25 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 26 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 27 | 28 | try: 29 | driver.get("https://vk.com/") 30 | time.sleep(5) 31 | 32 | print("Passing authentication...") 33 | email_input = driver.find_element_by_id("index_email") 34 | email_input.clear() 35 | email_input.send_keys(vk_phone) 36 | time.sleep(5) 37 | 38 | password_input = driver.find_element_by_id("index_pass") 39 | password_input.clear() 40 | password_input.send_keys(vk_password) 41 | time.sleep(3) 42 | password_input.send_keys(Keys.ENTER) 43 | 44 | # login_button = driver.find_element_by_id("index_login_button").click() 45 | time.sleep(10) 46 | 47 | print("Going to the profile page...") 48 | profile_page = driver.find_element_by_id("l_pr").click() 49 | time.sleep(5) 50 | 51 | print("Click on the comment icon...") 52 | comment_icon = driver.find_element_by_class_name("_comment").find_element_by_class_name("like_button_icon").click() 53 | time.sleep(5) 54 | 55 | print("Writing a comment...") 56 | driver.find_elements_by_class_name("submit_post_field")[1].send_keys("Hey! I was here! (.)(.)" + Keys.ENTER) 57 | time.sleep(5) 58 | print("The work is done! Bye, dude!") 59 | 60 | 61 | except Exception as ex: 62 | print(ex) 63 | finally: 64 | driver.close() 65 | driver.quit() 66 | -------------------------------------------------------------------------------- /chromedriver/auth_data.py: -------------------------------------------------------------------------------- 1 | vk_phone = "your_phone" 2 | vk_password = "your_password" 3 | -------------------------------------------------------------------------------- /chromedriver/proxy_auth_data.py: -------------------------------------------------------------------------------- 1 | login = "your_proxy_login" 2 | password = "your_proxy_password" -------------------------------------------------------------------------------- /chromedriver/test.py: -------------------------------------------------------------------------------- 1 | str1 = "https://stackoverflow.com" 2 | print(str1.split('//')[1]) -------------------------------------------------------------------------------- /firefoxdriver/01_02_03_selenium_install_useragent_proxy_firefox.py: -------------------------------------------------------------------------------- 1 | # from selenium import webdriver 2 | from seleniumwire import webdriver 3 | import time 4 | from fake_useragent import UserAgent 5 | from proxy_auth_data import login, password 6 | 7 | # url = "https://www.vk.com/" 8 | 9 | # options 10 | options = webdriver.FirefoxOptions() 11 | 12 | # change useragent 13 | useragent = UserAgent() 14 | options.set_preference("general.useragent.override", useragent.random) 15 | 16 | # set proxy 17 | # proxy = "138.128.91.65:8000" 18 | # firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX 19 | # firefox_capabilities["marionette"] = True 20 | # 21 | # firefox_capabilities["proxy"] = { 22 | # "proxyType": "MANUAL", 23 | # "httpProxy": proxy, 24 | # "ftpProxy": proxy, 25 | # "sslProxy": proxy 26 | # } 27 | 28 | proxy_options = { 29 | "proxy": { 30 | "https": f"http://{login}:{password}@138.128.91.65:8000" 31 | } 32 | } 33 | 34 | # driver = webdriver.Firefox( 35 | # executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 36 | # options=options, proxy=proxy 37 | # ) 38 | 39 | driver = webdriver.Firefox( 40 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 41 | seleniumwire_options=proxy_options 42 | ) 43 | 44 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 45 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 46 | 47 | try: 48 | # driver.get(url="https://www.whatismybrowser.com/detect/what-is-my-user-agent") 49 | # driver.save_screenshot("vk.png") 50 | # time.sleep(5) 51 | # driver.get(url="https://stackoverflow.com/") 52 | # time.sleep(5) 53 | 54 | # driver.refresh() 55 | # driver.get_screenshot_as_file("1.png") 56 | # driver.get(url="https://stackoverflow.com/") 57 | # time.sleep(5) 58 | # driver.save_screenshot("2.png") 59 | # time.sleep(2) 60 | 61 | driver.get("https://2ip.ru") 62 | time.sleep(5) 63 | 64 | except Exception as ex: 65 | print(ex) 66 | finally: 67 | driver.close() 68 | driver.quit() 69 | -------------------------------------------------------------------------------- /firefoxdriver/04_selenium_form_firefox.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import instagram_password 5 | 6 | # options 7 | options = webdriver.FirefoxOptions() 8 | 9 | # user-agent 10 | options.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 11 | 12 | driver = webdriver.Firefox( 13 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 14 | options=options 15 | ) 16 | 17 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 18 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 19 | 20 | try: 21 | driver.get("https://instagram.com/") 22 | time.sleep(5) 23 | 24 | username_input = driver.find_element_by_name("username") 25 | username_input.clear() 26 | username_input.send_keys("python2day") 27 | time.sleep(5) 28 | 29 | password_input = driver.find_element_by_name("password") 30 | password_input.clear() 31 | password_input.send_keys(instagram_password) 32 | time.sleep(5) 33 | 34 | password_input.send_keys(Keys.ENTER) 35 | time.sleep(10) 36 | 37 | except Exception as ex: 38 | print(ex) 39 | finally: 40 | driver.close() 41 | driver.quit() 42 | -------------------------------------------------------------------------------- /firefoxdriver/05_selenium_cookies_firefox.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import instagram_password, instagram_login 5 | import pickle 6 | 7 | # options 8 | options = webdriver.FirefoxOptions() 9 | 10 | # user-agent 11 | options.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | driver = webdriver.Firefox( 14 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 15 | options=options 16 | ) 17 | 18 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 19 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 20 | 21 | try: 22 | # driver.get("https://instagram.com/") 23 | # time.sleep(5) 24 | # 25 | # username_input = driver.find_element_by_name("username") 26 | # username_input.clear() 27 | # username_input.send_keys(instagram_login) 28 | # time.sleep(5) 29 | # 30 | # password_input = driver.find_element_by_name("password") 31 | # password_input.clear() 32 | # password_input.send_keys(instagram_password) 33 | # time.sleep(5) 34 | # 35 | # password_input.send_keys(Keys.ENTER) 36 | # time.sleep(10) 37 | # 38 | # # # cookies 39 | # pickle.dump(driver.get_cookies(), open(f"{instagram_login}_cookies", "wb")) 40 | 41 | driver.get("https://instagram.com/") 42 | time.sleep(5) 43 | 44 | for cookie in pickle.load(open(f"{instagram_login}_cookies", "rb")): 45 | driver.add_cookie(cookie) 46 | 47 | time.sleep(5) 48 | driver.refresh() 49 | time.sleep(10) 50 | 51 | except Exception as ex: 52 | print(ex) 53 | finally: 54 | driver.close() 55 | driver.quit() 56 | -------------------------------------------------------------------------------- /firefoxdriver/06_selenium_webdriver_mode_firefox.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | import time 3 | 4 | 5 | # options 6 | options = webdriver.FirefoxOptions() 7 | 8 | # user-agent 9 | options.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 10 | 11 | # disable webdriver mode 12 | options.set_preference("dom.webdriver.enabled", False) 13 | 14 | driver = webdriver.Firefox( 15 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 16 | options=options 17 | ) 18 | 19 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 20 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 21 | 22 | try: 23 | driver.get("https://intoli.com/blog/not-possible-to-block-chrome-headless/chrome-headless-test.html") 24 | time.sleep(10) 25 | except Exception as ex: 26 | print(ex) 27 | finally: 28 | driver.close() 29 | driver.quit() 30 | -------------------------------------------------------------------------------- /firefoxdriver/07_selenium_headless_firefox.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.keys import Keys 3 | import time 4 | from auth_data import instagram_password, instagram_login 5 | import pickle 6 | 7 | # options 8 | options = webdriver.FirefoxOptions() 9 | 10 | # user-agent 11 | options.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | # disable webdriver mode 14 | options.set_preference("dom.webdriver.enabled", False) 15 | 16 | # headless mode 17 | options.headless = True 18 | 19 | driver = webdriver.Firefox( 20 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 21 | options=options 22 | ) 23 | 24 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 25 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 26 | 27 | try: 28 | driver.get("https://instagram.com/") 29 | time.sleep(5) 30 | 31 | print("Passing authentication...") 32 | username_input = driver.find_element_by_name("username") 33 | username_input.clear() 34 | username_input.send_keys(instagram_login) 35 | time.sleep(5) 36 | 37 | password_input = driver.find_element_by_name("password") 38 | password_input.clear() 39 | password_input.send_keys(instagram_password) 40 | time.sleep(5) 41 | 42 | password_input.send_keys(Keys.ENTER) 43 | time.sleep(10) 44 | 45 | print("Going to the post...") 46 | video_post = driver.get("https://www.instagram.com/p/your_post/") 47 | print("Start watching the video...") 48 | time.sleep(5) 49 | 50 | print("Unmuting audio...") 51 | unmute_audio = driver.find_element_by_xpath("/html/body/div[1]/section/main/div/div[1]/article/div[2]/div/div/span/div") 52 | unmute_audio.click() 53 | time.sleep(5) 54 | print("Finish watching the video...") 55 | 56 | except Exception as ex: 57 | print(ex) 58 | finally: 59 | driver.close() 60 | driver.quit() 61 | -------------------------------------------------------------------------------- /firefoxdriver/08_selenium_switch_to_window_firefox.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | import time 3 | 4 | 5 | # options 6 | options = webdriver.FirefoxOptions() 7 | 8 | # user-agent 9 | options.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 10 | 11 | # disable webdriver mode 12 | options.set_preference("dom.webdriver.enabled", False) 13 | 14 | # headless mode 15 | options.headless = True 16 | 17 | driver = webdriver.Firefox( 18 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 19 | options=options 20 | ) 21 | 22 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 23 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 24 | 25 | try: 26 | driver.get("https://www.avito.ru/moskva/transport") 27 | # print(driver.window_handles) 28 | print(f"Currently URL is: {driver.current_url}") 29 | time.sleep(5) 30 | 31 | items = driver.find_elements_by_xpath("//div[@data-marker='item-photo']") 32 | items[0].click() 33 | # print(driver.window_handles) 34 | time.sleep(5) 35 | 36 | driver.switch_to.window(driver.window_handles[1]) 37 | time.sleep(5) 38 | print(f"Currently URL is: {driver.current_url}") 39 | 40 | username = driver.find_element_by_class_name("seller-info-name") 41 | print(f"User name is: {username.text}") 42 | print("#" * 20) 43 | time.sleep(5) 44 | 45 | driver.close() 46 | 47 | driver.switch_to.window(driver.window_handles[0]) 48 | time.sleep(5) 49 | print(f"Currently URL is: {driver.current_url}") 50 | 51 | items[1].click() 52 | time.sleep(5) 53 | 54 | driver.switch_to.window(driver.window_handles[1]) 55 | time.sleep(5) 56 | print(f"Currently URL is: {driver.current_url}") 57 | username = driver.find_element_by_xpath("//div[@data-marker='seller-info/name']") 58 | print(f"User name is: {username.text}") 59 | print("-" * 20) 60 | 61 | ad_date = driver.find_element_by_class_name("title-info-metadata-item-redesign") 62 | print(f"An ad date is: {ad_date.text}") 63 | print("-" * 20) 64 | 65 | joined_date = driver.find_elements_by_class_name("seller-info-value")[1] 66 | print(f"User since: {joined_date.text}") 67 | print("#" * 20) 68 | 69 | time.sleep(5) 70 | except Exception as ex: 71 | print(ex) 72 | finally: 73 | driver.close() 74 | driver.quit() 75 | -------------------------------------------------------------------------------- /firefoxdriver/09_selenium_multiprocessing_firefox.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | import time 3 | from multiprocessing import Pool 4 | import random 5 | 6 | 7 | # options 8 | options = webdriver.FirefoxOptions() 9 | 10 | # user-agent 11 | options.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0") 12 | 13 | # disable webdriver mode 14 | options.set_preference("dom.webdriver.enabled", False) 15 | 16 | # headless mode 17 | # options.headless = True 18 | 19 | 20 | def get_data(url): 21 | try: 22 | driver = webdriver.Firefox( 23 | executable_path="/home/cain/PycharmProjects/selenium_python/firefoxdriver/geckodriver", 24 | options=options 25 | ) 26 | 27 | # "C:\\users\\selenium_python\\chromedriver\\chromedriver.exe" 28 | # r"C:\users\selenium_python\chromedriver\chromedriver.exe" 29 | driver.get(url=url) 30 | time.sleep(5) 31 | driver.find_element_by_class_name("lazyload-wrapper").find_element_by_class_name("item-video-container").click() 32 | time.sleep(random.randrange(3, 10)) 33 | except Exception as ex: 34 | print(ex) 35 | finally: 36 | driver.close() 37 | driver.quit() 38 | 39 | 40 | if __name__ == '__main__': 41 | process_count = int(input("Enter the number of processes: ")) 42 | url = input("Enter the URL: ") 43 | urls_list = [url] * process_count 44 | print(urls_list) 45 | p = Pool(processes=process_count) 46 | p.map(get_data, urls_list) 47 | -------------------------------------------------------------------------------- /firefoxdriver/auth_data.py: -------------------------------------------------------------------------------- 1 | instagram_login = "your_username" 2 | instagram_password = "your_password" 3 | -------------------------------------------------------------------------------- /firefoxdriver/proxy_auth_data.py: -------------------------------------------------------------------------------- 1 | login = "your_proxy_login" 2 | password = "your_proxy_password" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fake-useragent==0.1.11 2 | pkg-resources==0.0.0 3 | selenium==3.141.0 4 | selenium-wire==2.1.2 5 | urllib3==1.26.2 6 | --------------------------------------------------------------------------------