├── .gitignore ├── LICENSE.txt ├── README.md ├── fBrowser └── __init__.py ├── requirements.txt ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # Distribution / packaging 8 | .Python 9 | build/ 10 | develop-eggs/ 11 | dist/ 12 | downloads/ 13 | eggs/ 14 | .eggs/ 15 | lib/ 16 | lib64/ 17 | parts/ 18 | sdist/ 19 | var/ 20 | wheels/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | MANIFEST 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .nox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | .pytest_cache/ 48 | 49 | # PyBuilder 50 | target/ 51 | 52 | # pyenv 53 | .python-version 54 | 55 | # Environments 56 | .env 57 | .venv 58 | env/ 59 | venv/ 60 | ENV/ 61 | env.bak/ 62 | venv.bak/ 63 | 64 | .vscode/ 65 | geckodriver.log 66 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Abe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fBrowser | Quick-start Your Selenium Project 2 | 3 | This library was created in order to help users quick-start their webscraping project. It includes several useful functions including a browser handler, new tab functionality, and more. 4 | 5 | ## Installation 6 | 7 | ``` 8 | $ pip3 install fBrowser 9 | ``` 10 | 11 | ## Usage 12 | 13 | ### Browser Handler 14 | 15 | You can decorate your function with a browserHandler. This will create the driver and properly quit it if an exception occurs or when the program ends. 16 | 17 | ```python 18 | from time import sleep 19 | from fBrowser import browserHandler 20 | 21 | 22 | @browserHandler() 23 | def main(driver): 24 | driver.get('https://www.google.com') 25 | sleep(5) 26 | 27 | 28 | main() 29 | ``` 30 | 31 | You can also pass in these optional arguments: 32 | 33 | ```python 34 | @browserHandler(path='', firefox=False, proxy=':', 35 | headless=True, implicitWaitTime=60, incognito=True) 36 | ``` 37 | 38 | ### New Tab 39 | 40 | Run a function in a new tab of Chrome. It will automatically close and switch contexts when it finishes executing. 41 | 42 | You can use the `newTab` function to decorate your selenium functions. Doing so will execute your function in a new tab of the browser. Your decorated function must have its first parameter for the webdriver. 43 | 44 | ```python 45 | from time import sleep 46 | from fBrowser import browserHandler, newTab 47 | 48 | 49 | @newTab 50 | def loadTheVerge(driver): 51 | driver.get('https://www.theverge.com') 52 | sleep(2) 53 | 54 | 55 | @browserHandler() 56 | def main(driver): 57 | driver.get('https://www.google.com') 58 | sleep(2) 59 | loadTheVerge(driver) 60 | 61 | 62 | main() 63 | ``` 64 | 65 | ### Login 66 | 67 | Use this helper function to login into a site. Simply pass in your email / username and your password. (This function assumes that the input tags in the form have attributes @type=username, @type=email, or @type=password). 68 | 69 | ```python 70 | import fBrowser 71 | from fBrowser import browserHandler 72 | 73 | 74 | @browserHandler() 75 | def main(driver): 76 | driver.get( 77 | 'https://stackoverflow.com/users/login') 78 | fBrowser.login(driver, email='testing@gmail.com', password='abc123') 79 | 80 | 81 | main() 82 | ``` 83 | 84 | ### Fill Inputs 85 | 86 | Quickly fill multiple inputs with either a single value or multiple values. 87 | 88 | ```python 89 | import fBrowser 90 | from fBrowser import browserHandler 91 | 92 | 93 | @browserHandler() 94 | def main(driver): 95 | driver.get('example.com') 96 | # A list of xpaths to the inputs 97 | xpaths = ['//*[@name="foo"]', '//*[@name="bar"]', '//@name="fooBar"'] 98 | 99 | # Fill with multiple values 100 | fBrowser.fillInputs(driver, xpaths, ['value1', 'value2', 'value3']) 101 | 102 | # Or pass in one value 103 | fBrowser.fillInputs(driver, xpaths, 'hello world') 104 | 105 | 106 | main() 107 | ``` 108 | 109 | ### Human Type 110 | 111 | Don't want to get flagged as a bot? Fill your inputs using a human-like typing speed. 112 | 113 | ```python 114 | import fBrowser 115 | from time import sleep 116 | from fBrowser import browserHandler 117 | 118 | 119 | @browserHandler() 120 | def main(driver): 121 | driver.get('https://www.google.com') 122 | chatInput = driver.find_element_by_xpath('//*[@name="q"]') 123 | # Will input the string at a human-like typing speed 124 | fBrowser.humanType(driver, chatInput, 'Hello world!') 125 | sleep(5) 126 | 127 | 128 | main() 129 | ``` 130 | 131 | ## Contributing 132 | 133 | 1. Fork it () 134 | 2. Create your feature branch (`git checkout -b feature/fooBar`) 135 | 3. Commit your changes (`git commit -am 'Add some fooBar'`) 136 | 4. Push to the branch (`git push origin feature/fooBar`) 137 | 5. Create a new Pull Request 138 | -------------------------------------------------------------------------------- /fBrowser/__init__.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | from time import sleep 4 | from typing import Union 5 | from random import uniform 6 | from selenium import webdriver 7 | # TODO: TAB / CONTEXT MANAGER 8 | 9 | 10 | def getChromeBrowser(path: str = 'chromedriver', proxy: str = None, 11 | implicitWaitTime: int = 30, incognito: bool = False, 12 | headless: bool = False) -> webdriver.Chrome: 13 | """ 14 | Returns an instance of the webdriver for Chrome. 15 | 16 | :param path: The path to your chromedriver\n 17 | :param proxy: Proxy to connect to - ':\n 18 | :param implicitWaitTime: Implicit wait time for the browser\n 19 | :param incognito: Whether to open in incognito or not\n 20 | :param headless: Run the browser in headless mode 21 | """ 22 | chromeOptions = webdriver.ChromeOptions() 23 | chromeOptions.add_argument('disable-infobars') 24 | 25 | if proxy: 26 | chromeOptions.add_argument(f'--proxy-server={proxy}') 27 | if incognito: 28 | chromeOptions.add_argument("--incognito") 29 | if headless: 30 | chromeOptions.add_argument('--headless') 31 | 32 | driver = webdriver.Chrome( 33 | chrome_options=chromeOptions, executable_path=path) 34 | driver.implicitly_wait(implicitWaitTime) 35 | return driver 36 | 37 | 38 | def getFirefoxBrowser(path: str = 'geckodriver', proxy: str = None, 39 | implicitWaitTime: int = 30, incognito: bool = False, 40 | headless: bool = False) -> webdriver.Firefox: 41 | """ 42 | Returns an instance of the webdriver for FireFox. 43 | 44 | :param path: The path to your geckodriver\n 45 | :param proxy: Proxy to connect to - ':\n 46 | :param implicitWaitTime: Implicit wait time for the browser\n 47 | :param incognito: Whether to open in incognito or not\n 48 | :param headless: Run the browser in headless mode 49 | """ 50 | firefoxOptions = webdriver.FirefoxOptions() 51 | firefoxProfile = webdriver.FirefoxProfile() 52 | 53 | # This is to make sure that new tabs will not be opened in a new window 54 | firefoxProfile.set_preference("browser.link.open_newwindow", 3) 55 | firefoxProfile.set_preference("browser.link.open_newwindow.restriction", 2) 56 | 57 | if headless: 58 | firefoxOptions.set_headless() 59 | if incognito: 60 | firefoxProfile.set_preference( 61 | "browser.privatebrowsing.autostart", True) 62 | if proxy: 63 | host, port = proxy.split(':') 64 | firefoxProfile.set_preference("network.proxy.type", 1) 65 | firefoxProfile.set_preference("network.proxy.http", host) 66 | firefoxProfile.set_preference("network.proxy.http_port", int(port)) 67 | 68 | firefoxProfile.update_preferences() 69 | driver = webdriver.Firefox( 70 | firefox_options=firefoxOptions, executable_path=path, 71 | firefox_profile=firefoxProfile) 72 | driver.implicitly_wait(implicitWaitTime) 73 | return driver 74 | 75 | 76 | def browserHandler(path: str = None, firefox: bool = False, proxy: str = None, 77 | implicitWaitTime: int = 30, incognito: bool = False, 78 | headless: bool = False): 79 | """ 80 | Creates and handles the browser driver. Will automatically close 81 | if an exception occurs or when the program ends. 82 | 83 | :param path: The path to your webdriver. Leave blank to use the default one\n 84 | :param firefox: Boolean for whether to use Firefox over Chrome\n 85 | :param proxy: Proxy to connect to - ':\n 86 | :param implicitWaitTime: Implicit wait time for the browser\n 87 | :param incognito: Whether to open in incognito or not\n 88 | :param headless: Run the browser in headless mode 89 | """ 90 | if not path: 91 | path = 'geckodriver' if firefox else 'chromedriver' 92 | 93 | def bhWrapper(func): 94 | @functools.wraps(func) 95 | def bh(*args, **kwargs): 96 | driver = None 97 | if firefox: 98 | driver = getFirefoxBrowser(path, proxy, implicitWaitTime, 99 | incognito, headless) 100 | else: 101 | driver = getChromeBrowser(path, proxy, implicitWaitTime, 102 | incognito, headless) 103 | try: 104 | func(driver, *args, **kwargs) 105 | finally: 106 | driver.quit() 107 | return bh 108 | return bhWrapper 109 | 110 | 111 | def fillInputs(driver: webdriver.Chrome, inputXpaths: list = [], 112 | values: Union[str, list] = []) -> None: 113 | """ 114 | Fills a list of inputs with the specified value(s). If a list 115 | is passed, then the inputs will be filled with each value respectively. 116 | If the lists do not have equal lengths, then the amount of inputs filled will 117 | be the smallest length between the lists. If a string is passed, then all the 118 | inputs will be filled with that value. 119 | 120 | :param driver: The instance of the web driver\n 121 | :param inputXpaths: A list of xpaths pointing to the input elements\n 122 | :param values: A string or list to fill the input elements with 123 | """ 124 | inputs = [driver.find_element_by_xpath(i) for i in inputXpaths] 125 | 126 | if isinstance(values, str): 127 | for i in inputs: 128 | i.send_keys(values) 129 | 130 | elif isinstance(values, list): 131 | for i, j in zip(inputs, values): 132 | i.send_keys(j) 133 | 134 | 135 | def login(driver: webdriver.Chrome, email: str = '', 136 | username: str = '', password: str = '', 137 | oneAtTime: bool = False, humanType: bool = False) -> None: 138 | """ 139 | Will attempt to login to the current page. It will find elements 140 | that have the 'type' attribute associating it with an email, username, 141 | or password. If the password input only appears after you enter your 142 | email/username, make sure that `onAtTime` is set to True. 143 | 144 | :param driver: The instance of the web driver\n 145 | :param email: The email to sign in with\n 146 | :param username: The username to sign in with\n 147 | :param password: The password for the login\n 148 | :param oneAtTime: Bool indicating that an enter key needs to be pressed 149 | between each input\n 150 | :param humanType: Type at a human speed 151 | """ 152 | # TODO: CHECK IF ELEMENTS EXIST FIRST, IF NOT THROW ERROR 153 | if email: 154 | email = email + '\n' if oneAtTime else email 155 | x = driver.find_element_by_xpath('//*[@type="email"]') 156 | 157 | if humanType: 158 | humanType(driver, x, email) 159 | else: 160 | x.send_keys(email) 161 | 162 | if username: 163 | username = username + '\n' if oneAtTime else username 164 | x = driver.find_element_by_xpath('//*[@type="username"]') 165 | 166 | if humanType: 167 | humanType(driver, x, username) 168 | else: 169 | x.send_keys(username) 170 | 171 | sleep(1) 172 | if password: 173 | x = driver.find_element_by_xpath('//*[@type="password"]') 174 | if humanType: 175 | humanType(driver, x, password) 176 | else: 177 | x.send_keys(password + '\n') 178 | 179 | 180 | def humanType(driver: webdriver.Chrome, 181 | element: webdriver.remote.webelement, 182 | value: str) -> None: 183 | """ 184 | Fills an input with a human typing speed. Useful if you 185 | don't want to get flagged as a bot. 186 | 187 | :param driver: The instance of the web driver\n 188 | :param element: The input element to fill 189 | :param value: What to fill the input with 190 | """ 191 | for i in value: 192 | element.send_keys(i) 193 | # Sleep a random amount between each key press 194 | sleep(uniform(0.05, 0.10)) 195 | 196 | 197 | def newTab(func): 198 | """ 199 | Runs the function in a new tab, then closes and 200 | switches back context once it finishes. 201 | """ 202 | @functools.wraps(func) 203 | def nt(driver, *args, **kwargs): 204 | driver.execute_script("window.open('', '_blank')") 205 | oldWindow = driver.current_window_handle 206 | driver.switch_to_window(driver.window_handles[-1]) 207 | 208 | func(driver, *args, **kwargs) 209 | driver.close() 210 | driver.switch_to_window(oldWindow) 211 | return nt 212 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | selenium -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as f: 4 | longDescription = f.read() 5 | 6 | setuptools.setup( 7 | name="fBrowser", 8 | version="0.3.1", 9 | author="Abe", 10 | author_email="", 11 | description="Python3 library to make selenium webscraping easier", 12 | long_description=longDescription, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/FastestMolasses/fBrowser", 15 | packages=setuptools.find_packages(), 16 | classifiers=[ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ], 21 | ) 22 | --------------------------------------------------------------------------------