├── .gitignore ├── LICENSE ├── README.rst ├── example.py ├── finviz ├── __init__.py ├── config.py ├── helper_functions │ ├── __init__.py │ ├── display_functions.py │ ├── error_handling.py │ ├── request_functions.py │ ├── save_data.py │ └── scraper_functions.py ├── main_func.py ├── portfolio.py ├── screener.py └── tests │ └── test_screener.py ├── pyproject.toml ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | finviz/filters.json 2 | finviz-platform.code-workspace 3 | BACKLOG.md 4 | NOTES.md 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # IDE 14 | .idea/ 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | finviz/test.py 32 | finviz/portfolio.csv 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | .hypothesis/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # dotenv 92 | .env 93 | 94 | # virtualenv 95 | .venv 96 | venv/ 97 | ENV/ 98 | 99 | # Spyder project settings 100 | .spyderproject 101 | .spyproject 102 | 103 | # Rope project settings 104 | .ropeproject 105 | 106 | # mkdocs documentation 107 | /site 108 | 109 | # mypy 110 | .mypy_cache/ 111 | 112 | # CSV files 113 | *.csv 114 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 mariostoev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | finviz-api 2 | ########## 3 | *Unofficial Python API for FinViz* 4 | 5 | .. image:: https://badge.fury.io/py/finviz.svg 6 | :target: https://badge.fury.io/py/finviz 7 | 8 | .. image:: https://img.shields.io/badge/python-3.9-blue.svg 9 | :target: https://www.python.org/downloads/release/python-390/ 10 | 11 | .. image:: https://pepy.tech/badge/finviz 12 | :target: https://pepy.tech/project/finviz 13 | 14 | 15 | Downloading & Installation 16 | --------------------------- 17 | 18 | $ pip install -U git+https://github.com/mariostoev/finviz 19 | 20 | 21 | What is Finviz? 22 | ================ 23 | FinViz_ aims to make market information accessible and provides a lot of data in visual snapshots, allowing traders and investors to quickly find the stock, future or forex pair they are looking for. The site provides advanced screeners, market maps, analysis, comparative tools, and charts. 24 | 25 | .. _FinViz: https://finviz.com/?a=128493348 26 | 27 | **Important Information** 28 | 29 | Any quotes data displayed on finviz.com is delayed by 15 minutes for NASDAQ, and 20 minutes for NYSE and AMEX. This API should **NOT** be used for live trading, it's main purpose is financial analysis, research, and data scraping. 30 | 31 | Using Screener 32 | =============== 33 | 34 | Before using the Screener class, you have to manually go to the website's screener and enter your desired settings. The URL will automatically change every time you add a new setting. After you're done the URL will look something like this: 35 | 36 | .. image:: https://i.imgur.com/p8BLt06.png 37 | 38 | ``?v=111&s=ta_newhigh&f=cap_largeover,exch_nasd,fa_fpe_o10&o=-ticker&t=ZM`` are the extra parameters provided to the screener. Those parameters are a list of key/value pairs separated with the & symbol. Some keys have a clear intent - ``f=cap_largeover,exch_nasd,fa_fpe_o10`` are filters, ``o=-ticker`` is order and ``t=ZM`` are tickers - yet, some are ambiguous like ``v=111``, which stands for the type of table. 39 | 40 | To make matters easier inside the code you won't refer to tables by their number tag, but instead you will use their full name (ex. ``table=Performance``). 41 | 42 | .. code:: python 43 | 44 | from finviz.screener import Screener 45 | 46 | filters = ['exch_nasd', 'idx_sp500'] # Shows companies in NASDAQ which are in the S&P500 47 | stock_list = Screener(filters=filters, table='Performance', order='price') # Get the performance table and sort it by price ascending 48 | 49 | # Export the screener results to .csv 50 | stock_list.to_csv("stock.csv") 51 | 52 | # Create a SQLite database 53 | stock_list.to_sqlite("stock.sqlite3") 54 | 55 | for stock in stock_list[9:19]: # Loop through 10th - 20th stocks 56 | print(stock['Ticker'], stock['Price']) # Print symbol and price 57 | 58 | # Add more filters 59 | stock_list.add(filters=['fa_div_high']) # Show stocks with high dividend yield 60 | # or just stock_list(filters=['fa_div_high']) 61 | 62 | # Print the table into the console 63 | print(stock_list) 64 | 65 | .. image:: https://i.imgur.com/cb7UdxB.png 66 | 67 | Using Portfolio 68 | ================ 69 | .. code:: python 70 | 71 | from finviz.portfolio import Portfolio 72 | 73 | portfolio = Portfolio('', '', '') 74 | # Print the portfolio into the console 75 | print(portfolio) 76 | 77 | *Note that, portfolio name is optional - it would assume your default portfolio (if you have one) if you exclude it.* 78 | The Portfolio class can also create new portfolio from an existing ``.csv`` file. The ``.csv`` file must be in the following format: 79 | 80 | 81 | .. list-table:: 82 | :header-rows: 1 83 | 84 | * - Ticker 85 | - Transaction 86 | - Date (Opt.) 87 | - Shares 88 | - Price (Opt.) 89 | * - AAPL 90 | - 1 91 | - 05-25-2017 92 | - 34 93 | - 141.28 94 | * - NVDA 95 | - 2 96 | - 97 | - 250 98 | - 243.32 99 | * - WMT 100 | - 1 101 | - 01.19.2019 102 | - 45 103 | - 104 | 105 | Note that, if any *optional* fields are left empty, the API will assign them today's data. 106 | 107 | .. code:: python 108 | 109 | portfolio.create_portfolio('', '') 110 | 111 | Individual stocks 112 | ================== 113 | 114 | .. code:: pycon 115 | 116 | >>> import finviz 117 | >>> finviz.get_stock('AAPL') 118 | {'Index': 'DJIA S&P500', 'P/E': '12.91', 'EPS (ttm)': '12.15',... 119 | >>> finviz.get_insider('АAPL') 120 | [{'Insider Trading': 'KONDO CHRIS', 'Relationship': 'Principal Accounting Officer', 'Date': 'Nov 19', 'Transaction': 'Sale', 'Cost': '190.00', '#Shares': '3,408', 'Value ($)': '647,520', '#Shares Total': '8,940', 'SEC Form 4': 'Nov 21 06:31 PM'},... 121 | >>> finviz.get_news('AAPL') 122 | [('Chinas Economy Slows to the Weakest Pace Since 2009', 'https://finance.yahoo.com/news/china-economy-slows-weakest-pace- 020040147.html'),... 123 | >>> 124 | >>> finviz.get_analyst_price_targets('AAPL') 125 | [{'date': '2019-10-24', 'category': 'Reiterated', 'analyst': 'UBS', 'rating': 'Buy', 'price_from': 235, 'price_to': 275}, ... 126 | 127 | Downloading charts 128 | =================== 129 | 130 | .. code:: python 131 | 132 | # Monthly, Candles, Large, No Technical Analysis 133 | stock_list.get_charts(period='m', chart_type='c', size='l', ta='0') 134 | 135 | # period='d' > daily 136 | # period='w' > weekly 137 | # period='m' > monthly 138 | 139 | # chart_type='c' > candle 140 | # chart_type='l' > lines 141 | 142 | # size='m' > small 143 | # size='l' > large 144 | 145 | # ta='1' > display technical analysis 146 | # ta='0' > ignore technical analysis 147 | 148 | Environment Variables 149 | ====================== 150 | 151 | Set ``DISABLE_TQDM=1`` in your environment to disable the progress bar. 152 | 153 | Documentation 154 | ============== 155 | 156 | You can read the rest of the documentation inside the docstrings. 157 | 158 | Contributing 159 | ============= 160 | You can contribute to the project by reporting bugs, suggesting enhancements, or directly by extending and writing features (see the ongoing projects_). 161 | 162 | .. _projects: https://github.com/mariostoev/finviz/projects/1 163 | 164 | *You can also buy me a coffee!* 165 | 166 | .. image:: https://user-images.githubusercontent.com/8982949/33011169-6da4af5e-cddd-11e7-94e5-a52d776b94ba.png 167 | :target: https://www.paypal.me/finvizapi 168 | 169 | Disclaimer 170 | ----------- 171 | *Using the library to acquire data from FinViz is against their Terms of Service and robots.txt. Use it responsibly and at your own risk. This library is built purely for educational purposes.* 172 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from finviz.screener import Screener 4 | 5 | # Get dict of available filters 6 | # filters dict contains the corresponding filter tags 7 | filters = Screener.load_filter_dict() 8 | some_filters = [filters["PEG"]["Under 1"], filters["Exchange"]["AMEX"]] 9 | stock_list = Screener(filters=some_filters, order="ticker") 10 | print(stock_list) 11 | 12 | # Use raw filter tags in a list 13 | # filters = ['geo_usa'] 14 | filters = ["idx_sp500"] # Shows companies in the S&P500 15 | print("Screening stocks...") 16 | stock_list = Screener(filters=filters, order="ticker") 17 | print(stock_list) 18 | 19 | print("Retrieving stock data...") 20 | stock_data = stock_list.get_ticker_details() 21 | print(stock_data) 22 | 23 | # Export the screener results to CSV file 24 | stock_list.to_csv("sp500.csv") 25 | 26 | # Create a SQLite database 27 | # stock_list.to_sqlite("sp500.sqlite") 28 | -------------------------------------------------------------------------------- /finviz/__init__.py: -------------------------------------------------------------------------------- 1 | from finviz.main_func import (get_all_news, get_analyst_price_targets, 2 | get_insider, get_news, get_stock) 3 | from finviz.portfolio import Portfolio 4 | from finviz.screener import Screener 5 | -------------------------------------------------------------------------------- /finviz/config.py: -------------------------------------------------------------------------------- 1 | connection_settings = dict( 2 | CONCURRENT_CONNECTIONS=30, 3 | CONNECTION_TIMEOUT=30000, 4 | ) 5 | -------------------------------------------------------------------------------- /finviz/helper_functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mariostoev/finviz/bc02ca9bb980151c33fc7000276bbb0a0c729577/finviz/helper_functions/__init__.py -------------------------------------------------------------------------------- /finviz/helper_functions/display_functions.py: -------------------------------------------------------------------------------- 1 | def create_table_string(table_list): 2 | """ Used to create a readable representation of a table. """ 3 | 4 | col_size = [max(map(len, col)) for col in zip(*table_list)] 5 | format_str = " | ".join([f"{{:<{i}}}" for i in col_size]) 6 | table_list.insert(1, ["-" * i for i in col_size]) 7 | 8 | table_string = "" 9 | for item in table_list: 10 | table_string += format_str.format(*item) + "\n" 11 | 12 | return table_string 13 | -------------------------------------------------------------------------------- /finviz/helper_functions/error_handling.py: -------------------------------------------------------------------------------- 1 | from finviz.config import connection_settings 2 | 3 | 4 | class NoResults(Exception): 5 | """ Raise when there are no results found. """ 6 | 7 | def __init__(self, query): 8 | super(NoResults, self).__init__(f"No results found for query: {query}") 9 | 10 | 11 | class InvalidTableType(Exception): 12 | """ Raise when the given table type is invalid. """ 13 | 14 | def __init__(self, arg): 15 | super(InvalidTableType, self).__init__(f"Invalid table type called: {arg}") 16 | 17 | 18 | class TooManyRequests(Exception): 19 | """ Raise when HTTP request fails because too many requests were sent to FinViz at once. """ 20 | 21 | def __init__(self, arg): 22 | super(TooManyRequests, self).__init__(f"Too many HTTP requests at once: {arg}") 23 | 24 | 25 | class InvalidPortfolioID(Exception): 26 | """ Raise when the given portfolio id is invalid. """ 27 | 28 | def __int__(self, portfolio_id): 29 | super(InvalidPortfolioID, self).__init__( 30 | f"Invalid portfolio with ID: {portfolio_id}" 31 | ) 32 | 33 | 34 | class NonexistentPortfolioName(Exception): 35 | """ Raise when the given portfolio name is nonexistent. """ 36 | 37 | def __init__(self, name): 38 | super(NonexistentPortfolioName, self).__init__( 39 | f"Nonexistent portfolio with name: {name}" 40 | ) 41 | 42 | 43 | class NoPortfolio(Exception): 44 | """ Raise when the user has not created a portfolio. """ 45 | 46 | def __int__(self, func_name): 47 | super(NoPortfolio, self).__init__( 48 | "Function ({func_name}) cannot be called because " 49 | "there is no existing portfolio." 50 | ) 51 | 52 | 53 | class InvalidTicker(Exception): 54 | """ Raise when the given ticker is nonexistent or unavailable on FinViz. """ 55 | 56 | def __init__(self, ticker): 57 | super(InvalidTicker, self).__init__( 58 | f"Unable to find {ticker} since it is non-existent or unavailable on FinViz." 59 | ) 60 | 61 | 62 | class ConnectionTimeout(Exception): 63 | """ The request has timed out while trying to connect to the remote server. """ 64 | 65 | def __init__(self, webpage_link): 66 | super(ConnectionTimeout, self).__init__( 67 | f'Connection timed out after {connection_settings["CONNECTION_TIMEOUT"]} while trying to reach {webpage_link}' 68 | ) 69 | -------------------------------------------------------------------------------- /finviz/helper_functions/request_functions.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | from typing import Callable, Dict, List 4 | 5 | import aiohttp 6 | import requests 7 | import tenacity 8 | import urllib3 9 | from lxml import html 10 | from requests import Response 11 | from tqdm import tqdm 12 | from user_agent import generate_user_agent 13 | 14 | from finviz.config import connection_settings 15 | from finviz.helper_functions.error_handling import ConnectionTimeout 16 | 17 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 18 | 19 | 20 | def http_request_get( 21 | url, session=None, payload=None, parse=True, user_agent=generate_user_agent() 22 | ): 23 | """ Sends a GET HTTP request to a website and returns its HTML content and full url address. """ 24 | 25 | if payload is None: 26 | payload = {} 27 | 28 | try: 29 | if session: 30 | content = session.get( 31 | url, 32 | params=payload, 33 | verify=False, 34 | headers={"User-Agent": user_agent}, 35 | ) 36 | else: 37 | content = requests.get( 38 | url, 39 | params=payload, 40 | verify=False, 41 | headers={"User-Agent": user_agent}, 42 | ) 43 | 44 | content.raise_for_status() # Raise HTTPError for bad requests (4xx or 5xx) 45 | if parse: 46 | return html.fromstring(content.text), content.url 47 | else: 48 | return content.text, content.url 49 | except (asyncio.TimeoutError, requests.exceptions.Timeout): 50 | raise ConnectionTimeout(url) 51 | 52 | 53 | @tenacity.retry(wait=tenacity.wait_exponential()) 54 | def finviz_request(url: str, user_agent: str) -> Response: 55 | response = requests.get(url, headers={"User-Agent": user_agent}) 56 | if response.text == "Too many requests.": 57 | raise Exception("Too many requests.") 58 | return response 59 | 60 | 61 | def sequential_data_scrape( 62 | scrape_func: Callable, urls: List[str], user_agent: str, *args, **kwargs 63 | ) -> List[Dict]: 64 | data = [] 65 | 66 | for url in tqdm(urls, disable="DISABLE_TQDM" in os.environ): 67 | try: 68 | response = finviz_request(url, user_agent) 69 | kwargs["URL"] = url 70 | data.append(scrape_func(response, *args, **kwargs)) 71 | except Exception as exc: 72 | raise exc 73 | 74 | return data 75 | 76 | 77 | class Connector: 78 | """ Used to make asynchronous HTTP requests. """ 79 | 80 | def __init__( 81 | self, 82 | scrape_function: Callable, 83 | urls: List[str], 84 | user_agent: str, 85 | *args, 86 | css_select: bool = False 87 | ): 88 | self.scrape_function = scrape_function 89 | self.urls = urls 90 | self.user_agent = user_agent 91 | self.arguments = args 92 | self.css_select = css_select 93 | self.data = [] 94 | 95 | async def __http_request__async( 96 | self, 97 | url: str, 98 | session: aiohttp.ClientSession, 99 | ): 100 | """ Sends asynchronous http request to URL address and scrapes the webpage. """ 101 | 102 | try: 103 | async with session.get( 104 | url, headers={"User-Agent": self.user_agent} 105 | ) as response: 106 | page_html = await response.read() 107 | 108 | if page_html.decode("utf-8") == "Too many requests.": 109 | raise Exception("Too many requests.") 110 | 111 | if self.css_select: 112 | return self.scrape_function( 113 | html.fromstring(page_html), *self.arguments 114 | ) 115 | return self.scrape_function(page_html, *self.arguments) 116 | except (asyncio.TimeoutError, requests.exceptions.Timeout): 117 | raise ConnectionTimeout(url) 118 | 119 | async def __async_scraper(self): 120 | """ Adds a URL's into a list of tasks and requests their response asynchronously. """ 121 | 122 | async_tasks = [] 123 | conn = aiohttp.TCPConnector( 124 | limit_per_host=connection_settings["CONCURRENT_CONNECTIONS"] 125 | ) 126 | timeout = aiohttp.ClientTimeout(total=connection_settings["CONNECTION_TIMEOUT"]) 127 | 128 | async with aiohttp.ClientSession( 129 | connector=conn, timeout=timeout, headers={"User-Agent": self.user_agent} 130 | ) as session: 131 | for url in self.urls: 132 | async_tasks.append(self.__http_request__async(url, session)) 133 | 134 | self.data = await asyncio.gather(*async_tasks) 135 | 136 | def run_connector(self): 137 | """ Starts the asynchronous loop and returns the scraped data. """ 138 | 139 | asyncio.set_event_loop(asyncio.SelectorEventLoop()) 140 | loop = asyncio.get_event_loop() 141 | loop.run_until_complete(self.__async_scraper()) 142 | 143 | return self.data 144 | -------------------------------------------------------------------------------- /finviz/helper_functions/save_data.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import io 3 | import re 4 | import sqlite3 5 | 6 | 7 | def create_connection(sqlite_file): 8 | """ Creates a database connection. """ 9 | 10 | try: 11 | conn = sqlite3.connect(sqlite_file) 12 | return conn 13 | except sqlite3.Error as error: 14 | raise ( 15 | "An error has occurred while connecting to the database: ", 16 | error.args[0], 17 | ) 18 | 19 | 20 | def __write_csv_to_stream(stream, headers, data): 21 | """Writes the data in CSV format to a stream.""" 22 | 23 | dict_writer = csv.DictWriter(stream, headers) 24 | dict_writer.writeheader() 25 | dict_writer.writerows(data) 26 | 27 | 28 | def export_to_csv(headers, data, filename=None, mode="w", newline=""): 29 | """Exports the generated table into a CSV file if a file is mentioned. 30 | Returns the CSV table as a string if no file is mentioned.""" 31 | 32 | if filename: 33 | with open(filename, mode, newline=newline) as output_file: 34 | __write_csv_to_stream(output_file, headers, data) 35 | return None 36 | stream = io.StringIO() 37 | __write_csv_to_stream(stream, headers, data) 38 | return stream.getvalue() 39 | 40 | 41 | def export_to_db(headers, data, filename): 42 | """ Exports the generated table into a SQLite database into a file.""" 43 | 44 | field_list = "" 45 | table_name = "screener_results" # name of the table to be created 46 | conn = create_connection(filename) 47 | c = conn.cursor() 48 | 49 | for field in headers: 50 | field_cleaned = re.sub(r"[^\w\s]", "", field) 51 | field_cleaned = field_cleaned.replace(" ", "") 52 | field_cleaned = field_cleaned.replace("50DHigh", "High50D") 53 | field_cleaned = field_cleaned.replace("50DLow", "Low50D") 54 | field_cleaned = field_cleaned.replace("52WHigh", "High52W") 55 | field_cleaned = field_cleaned.replace("52WLow", "Low52W") 56 | field_list += field_cleaned + " TEXT, " 57 | 58 | c.execute(f"CREATE TABLE IF NOT EXISTS {table_name} ({field_list[:-2]});") 59 | 60 | inserts = "" 61 | for row in data: 62 | 63 | insert_fields = "(" 64 | for field, value in row.items(): 65 | 66 | insert_fields += '"' + value + '", ' 67 | 68 | inserts += insert_fields[:-2] + "), " 69 | 70 | insert_lines = inserts[:-2] 71 | 72 | try: 73 | c.execute(f"INSERT INTO {table_name} VALUES {insert_lines};") 74 | except sqlite3.Error as error: 75 | print("An error has occurred", error.args[0]) 76 | 77 | conn.commit() 78 | conn.close() 79 | -------------------------------------------------------------------------------- /finviz/helper_functions/scraper_functions.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import time 4 | 5 | import requests 6 | from lxml import etree, html 7 | 8 | 9 | def get_table(page_html: requests.Response, headers, rows=None, **kwargs): 10 | """ Private function used to return table data inside a list of dictionaries. """ 11 | if isinstance(page_html, str): 12 | page_parsed = html.fromstring(page_html) 13 | else: 14 | page_parsed = html.fromstring(page_html.text) 15 | # When we call this method from Portfolio we don't fill the rows argument. 16 | # Conversely, we always fill the rows argument when we call this method from Screener. 17 | # Also, in the portfolio page, we don't need the last row - it's redundant. 18 | if rows is None: 19 | rows = -2 # We'll increment it later (-1) and use it to cut the last row 20 | 21 | data_sets = [] 22 | # Select the HTML of the rows and append each column text to a list 23 | all_rows = [ 24 | column.xpath("td//text()") 25 | for column in page_parsed.cssselect('tr[valign="top"]') 26 | ] 27 | 28 | # If rows is different from -2, this function is called from Screener 29 | if rows != -2: 30 | for row_number, row_data in enumerate(all_rows, 1): 31 | data_sets.append(dict(zip(headers, row_data))) 32 | if row_number == rows: # If we have reached the required end 33 | break 34 | else: 35 | # Zip each row values to the headers and append them to data_sets 36 | [data_sets.append(dict(zip(headers, row))) for row in all_rows] 37 | 38 | return data_sets 39 | 40 | 41 | def get_total_rows(page_content): 42 | """ Returns the total number of rows(results). """ 43 | 44 | options=[('class="count-text whitespace-nowrap">#1 / ',' Total'),('class="count-text">#1 / ',' Total')] 45 | page_text = str(html.tostring(page_content)) 46 | for option_beg,option_end in options: 47 | if option_beg in page_text: 48 | total_number = page_text.split(option_beg)[1].split(option_end)[0] 49 | try: 50 | return int(total_number) 51 | except ValueError: 52 | return 0 53 | return 0 54 | 55 | 56 | def get_page_urls(page_content, rows, url): 57 | """ Returns a list containing all of the page URL addresses. """ 58 | 59 | total_pages = int( 60 | [i.text.split("/")[1] for i in page_content.cssselect('option[value="1"]')][0] 61 | ) 62 | urls = [] 63 | 64 | for page_number in range(1, total_pages + 1): 65 | sequence = 1 + (page_number - 1) * 20 66 | 67 | if sequence - 20 <= rows < sequence: 68 | break 69 | urls.append(url + f"&r={str(sequence)}") 70 | 71 | return urls 72 | 73 | 74 | def download_chart_image(page_content: requests.Response, **kwargs): 75 | """ Downloads a .png image of a chart into the "charts" folder. """ 76 | file_name = f"{kwargs['URL'].split('t=')[1]}_{int(time.time())}.png" 77 | 78 | if not os.path.exists("charts"): 79 | os.mkdir("charts") 80 | 81 | with open(os.path.join("charts", file_name), "wb") as handle: 82 | handle.write(page_content.content) 83 | 84 | 85 | def get_analyst_price_targets_for_export( 86 | ticker=None, page_content=None, last_ratings=5 87 | ): 88 | analyst_price_targets = [] 89 | 90 | try: 91 | table = page_content.cssselect('table[class="fullview-ratings-outer"]')[0] 92 | ratings_list = [row.xpath("td//text()") for row in table] 93 | ratings_list = [ 94 | [val for val in row if val != "\n"] for row in ratings_list 95 | ] # remove new line entries 96 | 97 | headers = [ 98 | "ticker", 99 | "date", 100 | "category", 101 | "analyst", 102 | "rating", 103 | "price_from", 104 | "price_to", 105 | ] # header names 106 | count = 0 107 | 108 | for row in ratings_list: 109 | if count == last_ratings: 110 | break 111 | 112 | price_from, price_to = ( 113 | 0, 114 | 0, 115 | ) # default values for len(row) == 4 , that is there is NO price information 116 | if len(row) == 5: 117 | strings = row[4].split("→") 118 | if len(strings) == 1: 119 | price_to = ( 120 | strings[0].strip(" ").strip("$") 121 | ) # if only ONE price is available then it is 'price_to' value 122 | else: 123 | price_from = ( 124 | strings[0].strip(" ").strip("$") 125 | ) # both '_from' & '_to' prices available 126 | price_to = strings[1].strip(" ").strip("$") 127 | 128 | elements = [ 129 | ticker, 130 | datetime.datetime.strptime(row[0], "%b-%d-%y").strftime("%Y-%m-%d"), 131 | ] 132 | elements.extend(row[1:3]) 133 | elements.append(row[3].replace("→", "->")) 134 | elements.append(price_from) 135 | elements.append(price_to) 136 | data = dict(zip(headers, elements)) 137 | analyst_price_targets.append(data) 138 | count += 1 139 | except Exception: 140 | pass 141 | 142 | return analyst_price_targets 143 | 144 | 145 | def download_ticker_details(page_content: requests.Response, **kwargs): 146 | data = {} 147 | ticker = kwargs["URL"].split("=")[1] 148 | page_parsed = html.fromstring(page_content.text) 149 | 150 | all_rows = [ 151 | row.xpath("td//text()") 152 | for row in page_parsed.cssselect('tr[class="table-dark-row"]') 153 | ] 154 | 155 | for row in all_rows: 156 | for column in range(0, 11): 157 | if column % 2 == 0: 158 | data[row[column]] = row[column + 1] 159 | 160 | if len(data) == 0: 161 | print(f"-> Unable to parse page for ticker: {ticker}") 162 | 163 | return {ticker: [data, get_analyst_price_targets_for_export(ticker, page_parsed)]} 164 | -------------------------------------------------------------------------------- /finviz/main_func.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from lxml import etree 4 | 5 | from finviz.helper_functions.request_functions import http_request_get 6 | from finviz.helper_functions.scraper_functions import get_table 7 | 8 | STOCK_URL = "https://finviz.com/quote.ashx" 9 | NEWS_URL = "https://finviz.com/news.ashx" 10 | CRYPTO_URL = "https://finviz.com/crypto_performance.ashx" 11 | STOCK_PAGE = {} 12 | 13 | 14 | def get_page(ticker): 15 | global STOCK_PAGE 16 | 17 | if ticker not in STOCK_PAGE: 18 | STOCK_PAGE[ticker], _ = http_request_get( 19 | url=STOCK_URL, payload={"t": ticker}, parse=True 20 | ) 21 | 22 | 23 | def get_stock(ticker): 24 | """ 25 | Returns a dictionary containing stock data. 26 | 27 | :param ticker: stock symbol 28 | :type ticker: str 29 | :return dict 30 | """ 31 | 32 | get_page(ticker) 33 | page_parsed = STOCK_PAGE[ticker] 34 | 35 | title = page_parsed.cssselect('table[class="fullview-title"]')[0] 36 | keys = ["Ticker","Company", "Sector", "Industry", "Country"] 37 | fields = [f.text_content() for f in title.cssselect('a[class="tab-link"]')] 38 | data = dict(zip(keys, fields)) 39 | 40 | company_link = title.cssselect('a[class="tab-link"]')[0].attrib["href"] 41 | data["Website"] = company_link if company_link.startswith("http") else None 42 | 43 | all_rows = [ 44 | row.xpath("td//text()") 45 | for row in page_parsed.cssselect('tr[class="table-dark-row"]') 46 | ] 47 | 48 | for row in all_rows: 49 | for column in range(0, 11, 2): 50 | if row[column] == "EPS next Y" and "EPS next Y" in data.keys(): 51 | data["EPS growth next Y"] = row[column + 1] 52 | continue 53 | elif row[column] == "Volatility": 54 | vols = row[column + 1].split() 55 | data["Volatility (Week)"] = vols[0] 56 | data["Volatility (Month)"] = vols[1] 57 | continue 58 | 59 | data[row[column]] = row[column + 1] 60 | 61 | return data 62 | 63 | 64 | def get_insider(ticker): 65 | """ 66 | Returns a list of dictionaries containing all recent insider transactions. 67 | 68 | :param ticker: stock symbol 69 | :return: list 70 | """ 71 | 72 | get_page(ticker) 73 | page_parsed = STOCK_PAGE[ticker] 74 | outer_table = page_parsed.cssselect('table[class="body-table insider-trading-table"]') 75 | 76 | if len(outer_table) == 0: 77 | return [] 78 | 79 | table = outer_table[0] 80 | headers = table[0].xpath("td//text()") 81 | 82 | data = [dict(zip( 83 | headers, 84 | [etree.tostring(elem, method="text", encoding="unicode") for elem in row] 85 | )) for row in table[1:]] 86 | 87 | return data 88 | 89 | 90 | def get_news(ticker): 91 | """ 92 | Returns a list of sets containing news headline and url 93 | 94 | :param ticker: stock symbol 95 | :return: list 96 | """ 97 | 98 | get_page(ticker) 99 | page_parsed = STOCK_PAGE[ticker] 100 | news_table = page_parsed.cssselect('table[id="news-table"]') 101 | 102 | if len(news_table) == 0: 103 | return [] 104 | 105 | rows = news_table[0].xpath("./tr[not(@id)]") 106 | 107 | results = [] 108 | date = None 109 | for row in rows: 110 | raw_timestamp = row.xpath("./td")[0].xpath("text()")[0][0:-2] 111 | 112 | if len(raw_timestamp) > 8: 113 | parsed_timestamp = datetime.strptime(raw_timestamp, "%b-%d-%y %I:%M%p") 114 | date = parsed_timestamp.date() 115 | else: 116 | parsed_timestamp = datetime.strptime(raw_timestamp, "%I:%M%p").replace( 117 | year=date.year, month=date.month, day=date.day) 118 | 119 | results.append(( 120 | parsed_timestamp.strftime("%Y-%m-%d %H:%M"), 121 | row.xpath("./td")[1].cssselect('a[class="tab-link-news"]')[0].xpath("text()")[0], 122 | row.xpath("./td")[1].cssselect('a[class="tab-link-news"]')[0].get("href"), 123 | row.xpath("./td")[1].cssselect('div[class="news-link-right"] span')[0].xpath("text()")[0][1:] 124 | )) 125 | 126 | return results 127 | 128 | 129 | def get_all_news(): 130 | """ 131 | Returns a list of sets containing time, headline and url 132 | :return: list 133 | """ 134 | 135 | page_parsed, _ = http_request_get(url=NEWS_URL, parse=True) 136 | all_dates = [ 137 | row.text_content() for row in page_parsed.cssselect('td[class="nn-date"]') 138 | ] 139 | all_headlines = [ 140 | row.text_content() for row in page_parsed.cssselect('a[class="nn-tab-link"]') 141 | ] 142 | all_links = [ 143 | row.get("href") for row in page_parsed.cssselect('a[class="nn-tab-link"]') 144 | ] 145 | 146 | return list(zip(all_dates, all_headlines, all_links)) 147 | 148 | 149 | def get_crypto(pair): 150 | """ 151 | 152 | :param pair: crypto pair 153 | :return: dictionary 154 | """ 155 | 156 | page_parsed, _ = http_request_get(url=CRYPTO_URL, parse=True) 157 | page_html, _ = http_request_get(url=CRYPTO_URL, parse=False) 158 | crypto_headers = page_parsed.cssselect('tr[valign="middle"]')[0].xpath("td//text()") 159 | crypto_table_data = get_table(page_html, crypto_headers) 160 | 161 | return crypto_table_data[pair] 162 | 163 | 164 | def get_analyst_price_targets(ticker, last_ratings=5): 165 | """ 166 | Returns a list of dictionaries containing all analyst ratings and Price targets 167 | - if any of 'price_from' or 'price_to' are not available in the DATA, then those values are set to default 0 168 | :param ticker: stock symbol 169 | :param last_ratings: most recent ratings to pull 170 | :return: list 171 | """ 172 | 173 | analyst_price_targets = [] 174 | 175 | try: 176 | get_page(ticker) 177 | page_parsed = STOCK_PAGE[ticker] 178 | table = page_parsed.cssselect( 179 | 'table[class="js-table-ratings fullview-ratings-outer"]' 180 | )[0] 181 | 182 | for row in table: 183 | rating = row.xpath("td//text()") 184 | rating = [val.replace("→", "->").replace("$", "") for val in rating if val != "\n"] 185 | rating[0] = datetime.strptime(rating[0], "%b-%d-%y").strftime("%Y-%m-%d") 186 | 187 | data = { 188 | "date": rating[0], 189 | "category": rating[1], 190 | "analyst": rating[2], 191 | "rating": rating[3], 192 | } 193 | if len(rating) == 5: 194 | if "->" in rating[4]: 195 | rating.extend(rating[4].replace(" ", "").split("->")) 196 | del rating[4] 197 | data["target_from"] = float(rating[4]) 198 | data["target_to"] = float(rating[5]) 199 | else: 200 | data["target"] = float(rating[4]) 201 | 202 | analyst_price_targets.append(data) 203 | except Exception as e: 204 | pass 205 | 206 | return analyst_price_targets[:last_ratings] 207 | -------------------------------------------------------------------------------- /finviz/portfolio.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | import requests 4 | from lxml import html 5 | from user_agent import generate_user_agent 6 | 7 | from finviz.helper_functions.display_functions import create_table_string 8 | from finviz.helper_functions.error_handling import (InvalidPortfolioID, 9 | InvalidTicker, 10 | NonexistentPortfolioName) 11 | from finviz.helper_functions.request_functions import http_request_get 12 | from finviz.helper_functions.scraper_functions import get_table 13 | 14 | LOGIN_URL = "https://finviz.com/login_submit.ashx" 15 | PRICE_REQUEST_URL = "https://finviz.com/request_quote.ashx" 16 | PORTFOLIO_URL = "https://finviz.com/portfolio.ashx" 17 | PORTFOLIO_SUBMIT_URL = "https://finviz.com/portfolio_submit.ashx" 18 | PORTFOLIO_DIGIT_COUNT = 9 # Portfolio ID is always 9 digits 19 | PORTFOLIO_HEADERS = [ 20 | "No.", 21 | "Ticker", 22 | "Company", 23 | "Price", 24 | "Change%", 25 | "Volume", 26 | "Transaction", 27 | "Date", 28 | "Shares", 29 | "Cost", 30 | "Market Value", 31 | "Gain$", 32 | "Gain%", 33 | "Change$", 34 | ] 35 | 36 | 37 | class Portfolio(object): 38 | """ Used to interact with FinViz Portfolio. """ 39 | 40 | def __init__(self, email, password, portfolio=None): 41 | """ 42 | Logs in to FinViz and send a GET request to the portfolio. 43 | """ 44 | 45 | payload = {"email": email, "password": password} 46 | 47 | # Create a session and log in by sending a POST request 48 | self._session = requests.session() 49 | auth_response = self._session.post( 50 | LOGIN_URL, data=payload, headers={"User-Agent": generate_user_agent()} 51 | ) 52 | 53 | if not auth_response.ok: # If the post request wasn't successful 54 | auth_response.raise_for_status() 55 | 56 | # Get the parsed HTML and the URL of the base portfolio page 57 | self._page_content, self.portfolio_url = http_request_get( 58 | url=PORTFOLIO_URL, session=self._session, parse=False 59 | ) 60 | 61 | # If the user has not created a portfolio it redirects the request to ?v=2) 62 | self.created = True 63 | if self.portfolio_url == f"{PORTFOLIO_URL}?v=2": 64 | self.created = False 65 | 66 | if self.created: 67 | if portfolio: 68 | self._page_content, _ = self.__get_portfolio_url(portfolio) 69 | 70 | self.data = get_table(self._page_content, PORTFOLIO_HEADERS) 71 | 72 | def __str__(self): 73 | """ Returns a readable representation of a table. """ 74 | 75 | table_list = [PORTFOLIO_HEADERS] 76 | 77 | for row in self.data: 78 | table_list.append([row[col] or "" for col in PORTFOLIO_HEADERS]) 79 | 80 | return create_table_string(table_list) 81 | 82 | def create_portfolio(self, name, file, drop_invalid_ticker=False): 83 | """ 84 | Creates a new portfolio from a .csv file. 85 | 86 | The .csv file must be in the following format: 87 | Ticker,Transaction,Date,Shares,Price 88 | NVDA,2,14-04-2018,43,148.26 89 | AAPL,1,01-05-2019,12 90 | WMT,1,25-02-2015,20 91 | ENGH:CA,1,,1, 92 | 93 | (!) For transaction - 1 = BUY, 2 = SELL 94 | (!) Note that if the price is omitted the function will take today's ticker price 95 | """ 96 | 97 | data = { 98 | "portfolio_id": "0", 99 | "portfolio_name": name, 100 | } 101 | 102 | with open(file, "r") as infile: 103 | reader = csv.reader(infile) 104 | next(reader, None) # Skip the headers 105 | 106 | for row_number, row in enumerate(reader, 0): 107 | row_number_string = str(row_number) 108 | data["ticker" + row_number_string] = row[0] 109 | data["transaction" + row_number_string] = row[1] 110 | data["date" + row_number_string] = row[2] 111 | data["shares" + row_number_string] = row[3] 112 | 113 | try: 114 | # empty string is no price, so try get today's price 115 | assert data["price" + row_number_string] != "" 116 | data["price" + row_number_string] = row[4] 117 | except (IndexError, KeyError): 118 | current_price_page, _ = http_request_get( 119 | PRICE_REQUEST_URL, payload={"t": row[0]}, parse=True 120 | ) 121 | 122 | # if price not available on finviz don't upload that ticker to portfolio 123 | if current_price_page.text == "NA": 124 | if not drop_invalid_ticker: 125 | raise InvalidTicker(row[0]) 126 | del data["ticker" + row_number_string] 127 | del data["transaction" + row_number_string] 128 | del data["date" + row_number_string] 129 | del data["shares" + row_number_string] 130 | else: 131 | data["price" + row_number_string] = current_price_page.text 132 | self._session.post(PORTFOLIO_SUBMIT_URL, data=data) 133 | 134 | def __get_portfolio_url(self, portfolio_name): 135 | """ Private function used to return the portfolio url from a given id/name. """ 136 | 137 | # If the user has provided an ID (Portfolio ID is always an int) 138 | if isinstance(portfolio_name, int): 139 | # Raise error for invalid portfolio ID 140 | if not len(str(portfolio_name)) == PORTFOLIO_DIGIT_COUNT: 141 | raise InvalidPortfolioID(portfolio_name) 142 | else: 143 | return http_request_get( 144 | url=f"{PORTFOLIO_URL}?pid={portfolio_name}", 145 | session=self._session, 146 | parse=False, 147 | ) 148 | else: # else the user has passed a name 149 | # We remove the first element, since it's redundant 150 | for portfolio in html.fromstring(self._page_content).cssselect("option")[ 151 | 1: 152 | ]: 153 | if portfolio.text == portfolio_name: 154 | return http_request_get( 155 | url=f"{PORTFOLIO_URL}?pid={portfolio.get('value')}", 156 | session=self._session, 157 | parse=False, 158 | ) 159 | # Raise Non-existing PortfolioName if none of the names match 160 | raise NonexistentPortfolioName(portfolio_name) 161 | -------------------------------------------------------------------------------- /finviz/screener.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pathlib 3 | import urllib.request 4 | from urllib.parse import parse_qs as urlparse_qs 5 | from urllib.parse import urlencode, urlparse 6 | 7 | from bs4 import BeautifulSoup 8 | from user_agent import generate_user_agent 9 | 10 | import finviz.helper_functions.scraper_functions as scrape 11 | from finviz.helper_functions.display_functions import create_table_string 12 | from finviz.helper_functions.error_handling import InvalidTableType, NoResults 13 | from finviz.helper_functions.request_functions import (Connector, 14 | http_request_get, 15 | sequential_data_scrape) 16 | from finviz.helper_functions.save_data import export_to_csv, export_to_db 17 | 18 | TABLE_TYPES = { 19 | "Overview": "111", 20 | "Valuation": "121", 21 | "Ownership": "131", 22 | "Performance": "141", 23 | "Custom": "152", 24 | "Financial": "161", 25 | "Technical": "171", 26 | } 27 | 28 | 29 | class Screener(object): 30 | """ Used to download data from https://www.finviz.com/screener.ashx. """ 31 | 32 | @classmethod 33 | def init_from_url(cls, url, rows=None): 34 | """ 35 | Initializes from url 36 | 37 | :param url: screener url 38 | :type url: string 39 | :param rows: total number of rows to get 40 | :type rows: int 41 | """ 42 | 43 | split_query = urlparse_qs(urlparse(url).query) 44 | 45 | tickers = split_query["t"][0].split(",") if "t" in split_query else None 46 | filters = split_query["f"][0].split(",") if "f" in split_query else None 47 | custom = split_query["c"][0].split(",") if "c" in split_query else None 48 | order = split_query["o"][0] if "o" in split_query else "" 49 | signal = split_query["s"][0] if "s" in split_query else "" 50 | 51 | table = "Overview" 52 | if "v" in split_query: 53 | table_numbers_types = {v: k for k, v in TABLE_TYPES.items()} 54 | table_number_string = split_query["v"][0][0:3] 55 | try: 56 | table = table_numbers_types[table_number_string] 57 | except KeyError: 58 | raise InvalidTableType(split_query["v"][0]) 59 | 60 | return cls(tickers, filters, rows, order, signal, table, custom) 61 | 62 | def __init__( 63 | self, 64 | tickers=None, 65 | filters=None, 66 | rows=None, 67 | order="", 68 | signal="", 69 | table=None, 70 | custom=None, 71 | user_agent=generate_user_agent(), 72 | request_method="sequential", 73 | ): 74 | """ 75 | Initializes all variables to its values 76 | 77 | :param tickers: collection of ticker strings eg.: ['AAPL', 'AMD', 'WMT'] 78 | :type tickers: list 79 | :param filters: collection of filters strings eg.: ['exch_nasd', 'idx_sp500', 'fa_div_none'] 80 | :type filters: list 81 | :param rows: total number of rows to get 82 | :type rows: int 83 | :param order: table order eg.: '-price' (to sort table by descending price) 84 | :type order: str 85 | :param signal: show by signal eg.: 'n_majornews' (for stocks with major news) 86 | :type signal: str 87 | :param table: table type eg.: 'Performance' 88 | :type table: str 89 | :param custom: collection of custom columns eg.: ['1', '21', '23', '45'] 90 | :type custom: list 91 | :var self.data: list of dictionaries containing row data 92 | :type self.data: list 93 | """ 94 | 95 | if tickers is None: 96 | self._tickers = [] 97 | else: 98 | self._tickers = tickers 99 | 100 | if filters is None: 101 | self._filters = [] 102 | else: 103 | self._filters = filters 104 | 105 | if table is None: 106 | self._table = "111" 107 | else: 108 | self._table = self.__check_table(table) 109 | 110 | if custom is None: 111 | self._custom = [] 112 | else: 113 | self._table = "152" 114 | self._custom = custom 115 | 116 | if ( 117 | "0" not in self._custom 118 | ): # 0 (No.) is required for the sequence algorithm to work 119 | self._custom = ["0"] + self._custom 120 | 121 | self._rows = rows 122 | self._order = order 123 | self._signal = signal 124 | self._user_agent = user_agent 125 | self._request_method = request_method 126 | 127 | self.analysis = [] 128 | self.data = self.__search_screener() 129 | 130 | def __call__( 131 | self, 132 | tickers=None, 133 | filters=None, 134 | rows=None, 135 | order="", 136 | signal="", 137 | table=None, 138 | custom=None, 139 | ): 140 | """ 141 | Adds more filters to the screener. Example usage: 142 | 143 | stock_list = Screener(filters=['cap_large']) # All the stocks with large market cap 144 | # After analyzing you decide you want to see which of the stocks have high dividend yield 145 | # and show their performance: 146 | stock_list(filters=['fa_div_high'], table='Performance') 147 | # Shows performance of stocks with large market cap and high dividend yield 148 | """ 149 | 150 | if tickers: 151 | [self._tickers.append(item) for item in tickers] 152 | 153 | if filters: 154 | [self._filters.append(item) for item in filters] 155 | 156 | if table: 157 | self._table = self.__check_table(table) 158 | 159 | if order: 160 | self._order = order 161 | 162 | if signal: 163 | self._signal = signal 164 | 165 | if rows: 166 | self._rows = rows 167 | 168 | if custom: 169 | self._custom = custom 170 | 171 | self.analysis = [] 172 | self.data = self.__search_screener() 173 | 174 | add = __call__ 175 | 176 | def __str__(self): 177 | """ Returns a readable representation of a table. """ 178 | 179 | table_list = [self.headers] 180 | 181 | for row in self.data: 182 | table_list.append([row[col] or "" for col in self.headers]) 183 | 184 | return create_table_string(table_list) 185 | 186 | def __repr__(self): 187 | """ Returns a string representation of the parameter's values. """ 188 | 189 | values = ( 190 | f"tickers: {tuple(self._tickers)}\n" 191 | f"filters: {tuple(self._filters)}\n" 192 | f"rows: {self._rows}\n" 193 | f"order: {self._order}\n" 194 | f"signal: {self._signal}\n" 195 | f"table: {self._table}\n" 196 | f"table: {self._custom}" 197 | ) 198 | 199 | return values 200 | 201 | def __len__(self): 202 | """ Returns an int with the number of total rows. """ 203 | 204 | return int(self._rows) 205 | 206 | def __getitem__(self, position): 207 | """ Returns a dictionary containing specific row data. """ 208 | 209 | return self.data[position] 210 | 211 | get = __getitem__ 212 | 213 | @staticmethod 214 | def __check_table(input_table): 215 | """ Checks if the user input for table type is correct. Otherwise, raises an InvalidTableType error. """ 216 | 217 | try: 218 | table = TABLE_TYPES[input_table] 219 | return table 220 | except KeyError: 221 | raise InvalidTableType(input_table) 222 | 223 | @staticmethod 224 | def load_filter_dict(reload=True): 225 | """ 226 | Get dict of available filters. File containing json specification of filters will be built if it doesn't exist 227 | or if reload is False 228 | """ 229 | 230 | # Get location of filter.json 231 | json_directory = pathlib.Path(__file__).parent 232 | json_file = pathlib.Path.joinpath(json_directory, "filters.json") 233 | 234 | # Reload the filters JSON file if present and requested 235 | if reload and json_file.is_file(): 236 | with open(json_file, "r") as fp: 237 | return json.load(fp) 238 | 239 | # Get html from main filter page, ft=4 ensures all filters are present 240 | hdr = { 241 | "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) " 242 | "Chrome/23.0.1271.64 Safari/537.11" 243 | } 244 | url = "https://finviz.com/screener.ashx?ft=4" 245 | req = urllib.request.Request(url, headers=hdr) 246 | with urllib.request.urlopen(req) as response: 247 | html = response.read().decode("utf-8") 248 | 249 | # Parse html and locate table we are interested in. 250 | # Use one of the text values and get the parent table from that 251 | bs = BeautifulSoup(html, "html.parser") 252 | filters_table = None 253 | for td in bs.find_all("td"): 254 | if td.get_text().strip() == "Exchange": 255 | filters_table = td.find_parent("table") 256 | if filters_table is None: 257 | raise Exception("Could not locate filter parameters") 258 | 259 | # Delete all div tags, we don't need them 260 | for div in filters_table.find_all("div"): 261 | div.decompose() 262 | 263 | # Populate dict with filtering options and corresponding filter tags 264 | filter_dict = {} 265 | td_list = filters_table.find_all("td") 266 | 267 | for i in range(0, len(td_list) - 2, 2): 268 | current_dict = {} 269 | if td_list[i].get_text().strip() == "": 270 | continue 271 | 272 | # Even td elements contain filter name (as shown on web page) 273 | filter_text = td_list[i].get_text().strip() 274 | 275 | # Odd td elements contain the filter tag and options 276 | selections = td_list[i + 1].find("select") 277 | filter_name = selections.get("data-filter").strip() 278 | 279 | # Store filter options for current filter 280 | options = selections.find_all("option", {"value": True}) 281 | for opt in options: 282 | # Encoded filter string 283 | value = opt.get("value").strip() 284 | 285 | # String shown in pull-down menu 286 | text = opt.get_text() 287 | 288 | # Filter out unwanted items 289 | if value is None or "Elite" in text: 290 | continue 291 | 292 | # Make filter string and store in dict 293 | current_dict[text] = f"{filter_name}_{value}" 294 | 295 | # Store current filter dict 296 | filter_dict[filter_text] = current_dict 297 | 298 | # Save filter dict to finviz directory 299 | try: 300 | with open(json_file, "w") as fp: 301 | json.dump(filter_dict, fp) 302 | except Exception as e: 303 | print(e) 304 | print("Unable to write to file{}".format(json_file)) 305 | 306 | return filter_dict 307 | 308 | def to_sqlite(self, filename): 309 | """Exports the generated table into a SQLite database. 310 | 311 | :param filename: SQLite database file path 312 | :type filename: str 313 | """ 314 | 315 | export_to_db(self.headers, self.data, filename) 316 | 317 | def to_csv(self, filename: str): 318 | """Exports the generated table into a CSV file. 319 | Returns a CSV string if filename is None. 320 | 321 | :param filename: CSV file path 322 | :type filename: str 323 | """ 324 | 325 | if filename and filename.endswith(".csv"): 326 | filename = filename[:-4] 327 | 328 | if len(self.analysis) > 0: 329 | export_to_csv( 330 | [ 331 | "ticker", 332 | "date", 333 | "category", 334 | "analyst", 335 | "rating", 336 | "price_from", 337 | "price_to", 338 | ], 339 | self.analysis, 340 | f"{filename}-analysts.csv", 341 | ) 342 | 343 | return export_to_csv(self.headers, self.data, f"{filename}.csv") 344 | 345 | def get_charts(self, period="d", size="l", chart_type="c", ta="1"): 346 | """ 347 | Downloads the charts of all tickers shown by the table. 348 | 349 | :param period: table period eg. : 'd', 'w' or 'm' for daily, weekly and monthly periods 350 | :type period: str 351 | :param size: table size eg.: 'l' for large or 's' for small - choose large for better quality but higher size 352 | :type size: str 353 | :param chart_type: chart type: 'c' for candles or 'l' for lines 354 | :type chart_type: str 355 | :param ta: technical analysis eg.: '1' to show ta '0' to hide ta 356 | :type ta: str 357 | """ 358 | 359 | encoded_payload = urlencode( 360 | {"ty": chart_type, "ta": ta, "p": period, "s": size} 361 | ) 362 | 363 | sequential_data_scrape( 364 | scrape.download_chart_image, 365 | [ 366 | f"https://finviz.com/chart.ashx?{encoded_payload}&t={row.get('Ticker')}" 367 | for row in self.data 368 | ], 369 | self._user_agent, 370 | ) 371 | 372 | def get_ticker_details(self): 373 | """ 374 | Downloads the details of all tickers shown by the table. 375 | """ 376 | 377 | ticker_data = sequential_data_scrape( 378 | scrape.download_ticker_details, 379 | [ 380 | f"https://finviz.com/quote.ashx?&t={row.get('Ticker')}" 381 | for row in self.data 382 | ], 383 | self._user_agent, 384 | ) 385 | 386 | for entry in ticker_data: 387 | for key, value in entry.items(): 388 | for ticker_generic in self.data: 389 | if ticker_generic.get("Ticker") == key: 390 | if "Sales" not in self.headers: 391 | self.headers.extend(list(value[0].keys())) 392 | 393 | ticker_generic.update(value[0]) 394 | self.analysis.extend(value[1]) 395 | 396 | return self.data 397 | 398 | def __check_rows(self): 399 | """ 400 | Checks if the user input for row number is correct. 401 | Otherwise, modifies the number or raises NoResults error. 402 | """ 403 | 404 | self._total_rows = scrape.get_total_rows(self._page_content) 405 | 406 | if self._total_rows == 0: 407 | raise NoResults(self._url.split("?")[1]) 408 | elif self._rows is None or self._rows > self._total_rows: 409 | return self._total_rows 410 | else: 411 | return self._rows 412 | 413 | def __get_table_headers(self): 414 | """ Private function used to return table headers. """ 415 | headers = [] 416 | 417 | header_elements = self._page_content.cssselect('tr[valign="middle"]')[0].xpath("td") 418 | 419 | for header_element in header_elements: 420 | # Use normalize-space to extract text content while ignoring internal elements 421 | header_text = header_element.xpath("normalize-space()") 422 | 423 | if header_text: 424 | headers.append(header_text) 425 | 426 | return headers 427 | 428 | def __search_screener(self): 429 | """ Private function used to return data from the FinViz screener. """ 430 | 431 | self._page_content, self._url = http_request_get( 432 | "https://finviz.com/screener.ashx", 433 | payload={ 434 | "v": self._table, 435 | "t": ",".join(self._tickers), 436 | "f": ",".join(self._filters), 437 | "o": self._order, 438 | "s": self._signal, 439 | "c": ",".join(self._custom), 440 | }, 441 | user_agent=self._user_agent, 442 | ) 443 | 444 | self._rows = self.__check_rows() 445 | self.headers = self.__get_table_headers() 446 | 447 | if self._request_method == "async": 448 | async_connector = Connector( 449 | scrape.get_table, 450 | scrape.get_page_urls(self._page_content, self._rows, self._url), 451 | self._user_agent, 452 | self.headers, 453 | self._rows, 454 | css_select=True, 455 | ) 456 | pages_data = async_connector.run_connector() 457 | else: 458 | pages_data = sequential_data_scrape( 459 | scrape.get_table, 460 | scrape.get_page_urls(self._page_content, self._rows, self._url), 461 | self._user_agent, 462 | self.headers, 463 | self._rows, 464 | ) 465 | 466 | data = [] 467 | for page in pages_data: 468 | for row in page: 469 | data.append(row) 470 | 471 | return data 472 | -------------------------------------------------------------------------------- /finviz/tests/test_screener.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import lxml 4 | from dateutil.parser import parse 5 | 6 | from finviz.main_func import get_all_news, get_analyst_price_targets 7 | from finviz.screener import Screener 8 | 9 | 10 | class TestScreener: 11 | """ Unit tests for Screener app """ 12 | 13 | def test_get_screener_data_sequential_requests(self): 14 | """ Tests that basic Screener example returns correct number of stocks. """ 15 | stock_list = Screener( 16 | filters=["sh_curvol_o300", "ta_highlow52w_b0to10h", "ind_stocksonly"] 17 | ) 18 | 19 | count = 0 20 | for _ in stock_list: 21 | count += 1 22 | 23 | assert len(stock_list) == count 24 | 25 | def test_screener_stability(self): 26 | """ Requested in #77: https://github.com/mariostoev/finviz/issues/77 """ 27 | filters = [ 28 | "sh_avgvol_o100", # average daily volume > 100k shares 29 | "sh_curvol_o100", # current daily volume > 100k shares 30 | "sh_float_u50", # floating stocks < 50m shares 31 | "sh_price_u3", # current price is under $3 32 | ] 33 | stock_list = Screener(filters=filters, table="Performance") 34 | 35 | count = 0 36 | for _ in stock_list: 37 | count += 1 38 | 39 | assert len(stock_list) == count 40 | 41 | def test_get_ticker_details_sequential_requests(self): 42 | """ Tests that `get_ticker_details` method returns correct number of ticker details. """ 43 | stocks = Screener( 44 | filters=[ 45 | "sh_curvol_o300", 46 | "ta_highlow52w_b0to10h", 47 | "ind_stocksonly", 48 | "sh_outstanding_o1000", 49 | ] 50 | ) 51 | ticker_details = stocks.get_ticker_details() 52 | 53 | count = 0 54 | for _ in ticker_details: 55 | count += 1 56 | 57 | assert len(stocks) == count == len(ticker_details) 58 | 59 | @patch("finviz.screener.scrape.download_chart_image") 60 | def test_get_charts_sequential_requests(self, patched_download_chart_image): 61 | """ Tests that `get_charts` method returns correct number of valid charts. """ 62 | 63 | stocks = Screener( 64 | filters=[ 65 | "sh_curvol_o20000", 66 | "ta_highlow52w_b0to10h", 67 | "ind_stocksonly", 68 | "sh_outstanding_o1000", 69 | ] 70 | ) 71 | 72 | count = 0 73 | for _ in stocks: 74 | count += 1 75 | 76 | stocks.get_charts() 77 | 78 | for call in patched_download_chart_image.call_args_list: 79 | assert call.kwargs["URL"] 80 | assert call.args[0] 81 | assert len(stocks) == count == patched_download_chart_image.call_count 82 | 83 | 84 | def test_get_analyst_price_targets(): 85 | """ Verifies `get_analyst_price_targets` results' types are valid. """ 86 | 87 | price_target = get_analyst_price_targets("AAPL")[0] 88 | assert price_target 89 | 90 | try: 91 | parse(price_target["date"]) 92 | except ValueError: 93 | assert False 94 | 95 | assert type(price_target["category"]) == lxml.etree._ElementUnicodeResult 96 | assert type(price_target["analyst"]) == lxml.etree._ElementUnicodeResult 97 | assert type(price_target["rating"]) == lxml.etree._ElementUnicodeResult 98 | assert type(price_target.get("price_from", 0.0)) == float 99 | assert type(price_target.get("price_to", 0.0)) == float 100 | 101 | 102 | def test_get_all_news(): 103 | """ Verifies news results are greater than 0. """ 104 | news = get_all_news() 105 | assert len(news) > 0 106 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools.packages] 6 | find = {} 7 | 8 | [project] 9 | name = "finviz-platform" 10 | version = "0.0.1" 11 | authors = [ 12 | {name = "Alberto Rincones", email = "code4road@gmail.com"}, 13 | ] 14 | description = "Finviz scrapper with additinal datascience functionality" 15 | keywords = ["finviz", "finance", "datascience"] 16 | dependencies =[ 17 | "wheel", 18 | "lxml", 19 | "requests", 20 | "aiohttp", 21 | "urllib3", 22 | "cssselect", 23 | "user_agent", 24 | "beautifulsoup4", 25 | "tqdm", 26 | "tenacity", 27 | ] 28 | 29 | requires-python = ">=3.8" 30 | 31 | [tools.setuptools] 32 | packages = ["finviz"] 33 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | finviz~=1.4.4 2 | beautifulsoup4~=4.9.3 3 | requests~=2.25.1 4 | aiohttp~=3.7.4 5 | lxml~=4.9.2 6 | urllib3~=1.26.5 7 | user_agent~=0.1.9 8 | cssselect~=1.1.0 9 | tqdm~=4.61.1 10 | tenacity~=7.0.0 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup() 4 | --------------------------------------------------------------------------------