├── .github └── workflows │ └── python-app.yml ├── .gitignore ├── Dockerfile ├── Makefile ├── README.md ├── app_screenshot.png ├── dev-vars.env ├── docker-compose.prod.yml ├── docker-compose.yml ├── requirements.txt ├── tests ├── __init__.py ├── mypy.ini └── test_stock_info_scraper.py ├── utils ├── __init__.py ├── alembic_helpers.py ├── config.py ├── etl_base.py ├── models │ ├── __init__.py │ ├── _view_magic_formula_score.py │ ├── _view_piotroski_score.py │ ├── _view_screen_results.py │ ├── balance_sheet_statements.py │ ├── base.py │ ├── cash_flow_statements.py │ ├── income_statements.py │ ├── prices.py │ └── stocks.py ├── queries.py ├── stock_financial_statements_etl.py ├── stock_info_etl.py └── stock_valuation_etl.py ├── web ├── app.py └── static │ ├── css │ ├── index.css │ ├── login.css │ └── tabulator_simple.min.css │ ├── img │ └── favicon.png │ ├── index.html │ ├── js │ ├── tabulator.min.js │ └── update_table.js │ └── login.html └── worker ├── alembic.ini ├── alembic ├── README ├── env.py ├── script.py.mako └── versions │ ├── 00b80735370b_stock_financial_tables.py │ ├── 0e87ac62cba4_magic_formula_view.py │ ├── 4f406bad1840_screener_results_view.py │ ├── 7848c183a286_re_create_screener_results_view.py │ ├── 7bfdef0682d4_new_stock_prices_table.py │ ├── aeee2aa4ad8f_stocks_table.py │ └── f5cebdfa9f70_piotroski_view.py ├── worker.py └── worker_entrypoint.sh /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- 1 | name: Running Python tests 2 | on: push 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v2 8 | - name: Build the containers 9 | run: make run 10 | - name: Run tests 11 | run: make tests 12 | - name: Stop containers 13 | run: make stop 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | .bash_history 7 | .python_history 8 | .vscode/ 9 | .mypy_cache/ 10 | 11 | postgres-data/ 12 | stock_data/* 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | env/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *,cover 55 | .hypothesis/ 56 | .local 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # dotenv 89 | .env 90 | 91 | # virtualenv 92 | .venv/ 93 | venv/ 94 | ENV/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | 99 | # Rope project settings 100 | .ropeproject -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | RUN useradd -ms /bin/bash worker \ 4 | && apt-get update \ 5 | && apt-get install -y git netcat 6 | COPY requirements.txt /requirements.txt 7 | RUN python3 -m pip install -U pip setuptools \ 8 | && python3 -m pip install -U -r /requirements.txt 9 | 10 | USER worker 11 | WORKDIR /home/worker 12 | ENV PYTHONPATH="${PYTHONPATH}:${HOME}" 13 | 14 | EXPOSE 5000 5050 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run run_prod tests stop 2 | 3 | run: 4 | @echo 'Starting containers...' 5 | @docker-compose up -d --build 6 | 7 | run_prod: 8 | @echo 'Starting containers...' 9 | @docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build 10 | 11 | tests: 12 | @echo 'Running tests...' 13 | @docker-compose run \ 14 | --rm \ 15 | --no-deps \ 16 | --entrypoint='' \ 17 | -e APP_ENV=test \ 18 | worker \ 19 | bash -c "python -m unittest discover tests/ -v \ 20 | && python -m mypy --config-file tests/mypy.ini utils/" 21 | 22 | stop: 23 | @docker-compose down 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Running Python tests](https://github.com/lseffer/stock_screener/workflows/Running%20Python%20tests/badge.svg) 2 | 3 | # Stock screener 4 | For stocks listed on Nasdaq OMX Nordic. 5 | 6 | ![](app_screenshot.png?raw=true) 7 | 8 | This is a hobby project that helpes me make better investment decisions, while helping me learn new things. I haven't followed any investment strategy to the letter, but the screener helps me find good stocks and weed out the crap. 9 | 10 | I'm working on this project very sporadically and the code could be more beautiful but who has time for that? 11 | 12 | __Loosely based on:__ 13 | 14 | Piotroski F-Score - https://en.wikipedia.org/wiki/Piotroski_F-Score 15 | 16 | Magic Formula - https://en.wikipedia.org/wiki/Magic_formula_investing 17 | 18 | NCAV - Net Current Asset Value - https://www.oldschoolvalue.com/blog/investing-strategy/backtest-graham-nnwc-ncav-screen/ 19 | 20 | ## Architecture 21 | 22 | This stock screener consists of 3 services 23 | 24 | * Postgres database 25 | * Worker service scheduling and executing jobs that fetch the data about the stocks and stores that in the database 26 | * Web server for serving a front end with a login portal 27 | * Flask app served by gunicorn 28 | * Configured for nginx 29 | 30 | ## Usage: 31 | * Install Docker and Docker-compose 32 | * run the `make run` target 33 | * Go to `localhost:5000` and login with credentials found in `dev-vars.env` 34 | * For a production setup use the `make run_prod` target with your own secret `prod-vars.env` 35 | 36 | ## TO DO: 37 | * Add more screening methods 38 | * Add more tests 39 | * Possibly refactor the ETL job scripts 40 | * Add a job for sending same data to Google Sheets (like in previous version) 41 | * Ask for feedback 42 | -------------------------------------------------------------------------------- /app_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lseffer/stock_screener/5c42cbd9e22522131a1b68054230cd83fda531c8/app_screenshot.png -------------------------------------------------------------------------------- /dev-vars.env: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=test 2 | POSTGRES_PASSWORD=test 3 | POSTGRES_DB=master 4 | POSTGRES_HOST=database 5 | STOCKS_USERNAME=hello 6 | STOCKS_PASSWORD=yoyo 7 | SESSION_SECRET=897f35f8d104e4a9e4a67f8d757b0dbd4028f85aed24c721ee4d79e618cc9560 8 | -------------------------------------------------------------------------------- /docker-compose.prod.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | 3 | services: 4 | database: 5 | networks: 6 | - backend 7 | env_file: 8 | - prod-vars.env 9 | 10 | web: 11 | networks: 12 | - backend 13 | - frontend 14 | env_file: 15 | - prod-vars.env 16 | 17 | worker: 18 | networks: 19 | - backend 20 | - frontend 21 | env_file: 22 | - prod-vars.env 23 | 24 | networks: 25 | backend: 26 | internal: true 27 | frontend: 28 | internal: false 29 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | 3 | services: 4 | database: 5 | image: postgres:11.2 6 | restart: always 7 | env_file: 8 | - dev-vars.env 9 | ports: 10 | - "5432:5432" 11 | volumes: 12 | - ./postgres-data:/var/lib/postgresql/data 13 | 14 | web: 15 | build: . 16 | command: gunicorn -w 3 -k gevent -b 0.0.0.0:5000 app:app 17 | env_file: 18 | - dev-vars.env 19 | volumes: 20 | - ./web:/home/worker 21 | - ./utils:/home/worker/utils 22 | ports: 23 | - "5000:5000" 24 | depends_on: 25 | - database 26 | restart: always 27 | 28 | worker: 29 | build: . 30 | command: ./worker_entrypoint.sh 31 | volumes: 32 | - ./worker:/home/worker 33 | - ./utils:/home/worker/utils 34 | - ./tests:/home/worker/tests 35 | depends_on: 36 | - database 37 | env_file: 38 | - dev-vars.env 39 | restart: always 40 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | alembic==1.0.9 2 | attrs==19.1.0 3 | beautifulsoup4==4.7.1 4 | certifi==2019.3.9 5 | chardet==3.0.4 6 | Click==7.0 7 | decorator==4.4.0 8 | Flask==1.0.2 9 | Flask-Compress==1.4.0 10 | flask_jsontools==0.1.1.post0 11 | gevent==1.4.0 12 | greenlet==0.4.15 13 | gunicorn==19.9.0 14 | idna==2.8 15 | ipython-genutils==0.2.0 16 | itsdangerous==1.1.0 17 | Jinja2==2.10.1 18 | jsonschema==3.0.1 19 | lxml==4.3.3 20 | Mako==1.0.9 21 | MarkupSafe==1.1.1 22 | mypy==0.701 23 | mypy-extensions==0.4.1 24 | psycopg2-binary==2.8.2 25 | PyMySQL==0.9.3 26 | pyrsistent==0.14.11 27 | python-dateutil==2.8.0 28 | python-editor==1.0.4 29 | pytz==2019.1 30 | requests==2.21.0 31 | retrying==1.3.3 32 | schedule==0.6.0 33 | six==1.12.0 34 | soupsieve==1.9.1 35 | SQLAlchemy==1.3.3 36 | sqlalchemy-stubs==0.1 37 | traitlets==4.3.2 38 | typed-ast==1.3.4 39 | typing-extensions==3.7.2 40 | urllib3==1.24.2 41 | Werkzeug==0.15.3 42 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lseffer/stock_screener/5c42cbd9e22522131a1b68054230cd83fda531c8/tests/__init__.py -------------------------------------------------------------------------------- /tests/mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.7 3 | warn_return_any = True 4 | warn_unused_configs = True 5 | plugins = sqlmypy 6 | ignore_missing_imports = True 7 | 8 | [mypy-sqlalchemy.*] 9 | ignore_missing_imports = True 10 | -------------------------------------------------------------------------------- /tests/test_stock_info_scraper.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from utils.models import Stock 3 | 4 | class TestStockInfoScraper(unittest.TestCase): 5 | 6 | mock_record = { 7 | 'isin': 'FISHIT', 8 | 'symbol': 'shit', 9 | 'currency': 'EUR', 10 | 'name': 'test', 11 | 'sector': 'crap' 12 | } 13 | 14 | def test_normal_get_yahoo_ticker(self): 15 | res = Stock.parse_yahoo_ticker_from_isin(self.mock_record) 16 | self.assertEqual(res, 'shit.HE') 17 | 18 | def test_crazy_get_yahoo_ticker(self): 19 | mk_rec = self.mock_record.copy() 20 | mk_rec['isin'] = 'ZZ' 21 | mk_rec['currency'] = 'NOK' 22 | res = Stock.parse_yahoo_ticker_from_isin(mk_rec) 23 | self.assertEqual(res, 'shit.OL') 24 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import Dict, Any, List 3 | from utils.config import YAHOO_API_BASE_URL, YAHOO_API_PARAMS 4 | from flask_jsontools import DynamicJSONEncoder 5 | from datetime import date, datetime 6 | 7 | 8 | class ApiJSONEncoder(DynamicJSONEncoder): 9 | def default(self, o): 10 | # Custom formats 11 | if isinstance(o, datetime): 12 | return o.isoformat(' ') 13 | if isinstance(o, date): 14 | return o.isoformat() 15 | if isinstance(o, set): 16 | return list(o) 17 | 18 | # Fallback 19 | return super(DynamicJSONEncoder, self).default(o) 20 | 21 | 22 | def get_nested(dict_: Dict, *keys: str, default=None) -> Any: 23 | # Recursive helper function for traversing nested dictionaries 24 | if not isinstance(dict_, dict): 25 | return default 26 | elem = dict_.get(keys[0], default) 27 | if len(keys) == 1: 28 | return elem 29 | return get_nested(elem, *keys[1:], default=default) 30 | 31 | 32 | def make_yahoo_request(yahoo_ticker: str, params: Dict) -> Dict: 33 | response: Dict = requests.get(YAHOO_API_BASE_URL.format(yahoo_ticker), params=params).json() 34 | payload: Dict = get_nested(response, 'quoteSummary', 'result')[0] 35 | return payload 36 | 37 | 38 | def fetch_yahoo_data(yahoo_ticker: str, modules: str) -> Dict: 39 | params: Dict = YAHOO_API_PARAMS.copy() 40 | params['modules'] = modules 41 | response = make_yahoo_request(yahoo_ticker, params) 42 | return response 43 | 44 | 45 | def union_of_list_elements(*lists: List[Any]) -> List[Any]: 46 | lists_appended: List[Any] = sum(lists, []) 47 | return list(set(lists_appended)) 48 | -------------------------------------------------------------------------------- /utils/alembic_helpers.py: -------------------------------------------------------------------------------- 1 | from alembic.operations import Operations, MigrateOperation 2 | 3 | 4 | class ReversibleOp(MigrateOperation): 5 | def __init__(self, target): 6 | self.target = target 7 | 8 | @classmethod 9 | def invoke_for_target(cls, operations, target): 10 | op = cls(target) 11 | return operations.invoke(op) 12 | 13 | def reverse(self): 14 | raise NotImplementedError() 15 | 16 | @classmethod 17 | def _get_object_from_version(cls, operations, ident): 18 | version, objname = ident.split(".") 19 | 20 | module = operations.get_context().script.get_revision(version).module 21 | obj = getattr(module, objname) 22 | return obj 23 | 24 | @classmethod 25 | def replace(cls, operations, target, replaces=None, replace_with=None): 26 | 27 | if replaces: 28 | old_obj = cls._get_object_from_version(operations, replaces) 29 | drop_old = cls(old_obj).reverse() 30 | create_new = cls(target) 31 | elif replace_with: 32 | old_obj = cls._get_object_from_version(operations, replace_with) 33 | drop_old = cls(target).reverse() 34 | create_new = cls(old_obj) 35 | else: 36 | raise TypeError("replaces or replace_with is required") 37 | 38 | operations.invoke(drop_old) 39 | operations.invoke(create_new) 40 | 41 | 42 | @Operations.register_operation("create_view", "invoke_for_target") 43 | @Operations.register_operation("replace_view", "replace") 44 | class CreateViewOp(ReversibleOp): 45 | def reverse(self): 46 | return DropViewOp(self.target) 47 | 48 | 49 | @Operations.register_operation("drop_view", "invoke_for_target") 50 | class DropViewOp(ReversibleOp): 51 | def reverse(self): 52 | return CreateViewOp(self.view) 53 | 54 | 55 | @Operations.register_operation("create_sp", "invoke_for_target") 56 | @Operations.register_operation("replace_sp", "replace") 57 | class CreateSPOp(ReversibleOp): 58 | def reverse(self): 59 | return DropSPOp(self.target) 60 | 61 | 62 | @Operations.register_operation("drop_sp", "invoke_for_target") 63 | class DropSPOp(ReversibleOp): 64 | def reverse(self): 65 | return CreateSPOp(self.target) 66 | 67 | 68 | @Operations.implementation_for(CreateViewOp) 69 | def create_view(operations, operation): 70 | operations.execute("CREATE OR REPLACE VIEW %s AS %s" % ( 71 | operation.target.name, 72 | operation.target.sqltext 73 | )) 74 | 75 | 76 | @Operations.implementation_for(DropViewOp) 77 | def drop_view(operations, operation): 78 | operations.execute("DROP VIEW %s" % operation.target.name) 79 | 80 | 81 | @Operations.implementation_for(CreateSPOp) 82 | def create_sp(operations, operation): 83 | operations.execute( 84 | "CREATE OR REPLACE FUNCTION %s %s" % ( 85 | operation.target.name, operation.target.sqltext 86 | ) 87 | ) 88 | 89 | 90 | @Operations.implementation_for(DropSPOp) 91 | def drop_sp(operations, operation): 92 | operations.execute("DROP FUNCTION %s" % operation.target.name) 93 | -------------------------------------------------------------------------------- /utils/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | from datetime import datetime, date 5 | from logging.handlers import TimedRotatingFileHandler 6 | from sqlalchemy.orm import sessionmaker 7 | from sqlalchemy import create_engine 8 | from sqlalchemy.engine import Engine 9 | 10 | 11 | def get_last_year() -> date: 12 | return date(datetime.utcnow().year - 1, 12, 31) 13 | 14 | 15 | APP_ENV = os.getenv('APP_ENV') 16 | HOME = os.getenv('HOME', None) 17 | POSTGRES_USER = os.getenv('POSTGRES_USER', None) 18 | POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD', None) 19 | POSTGRES_DB = os.getenv('POSTGRES_DB', None) 20 | POSTGRES_HOST = os.getenv('POSTGRES_HOST', None) 21 | 22 | YAHOO_API_BASE_URL = "https://query1.finance.yahoo.com/v11/finance/quoteSummary/{}" 23 | YAHOO_API_PARAMS = {"formatted": "false", 24 | "lang": "en-US", 25 | "region": "US", 26 | "corsDomain": "finance.yahoo.com"} 27 | 28 | 29 | def create_pg_engine() -> Engine: 30 | engine = create_engine('postgresql://%s:%s@%s/%s' % 31 | (POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_DB)) 32 | return engine 33 | 34 | 35 | engine = create_pg_engine() 36 | Session = sessionmaker(bind=engine) 37 | 38 | 39 | def setup_logging(level: int = logging.INFO) -> logging.Logger: 40 | log = logging.getLogger() 41 | log.setLevel(level) 42 | formatter = logging.Formatter('%(asctime)s - %(module)s - %(funcName)s - %(levelname)s - %(message)s') 43 | 44 | stdout_handler = logging.StreamHandler(sys.stdout) 45 | stdout_handler.setFormatter(formatter) 46 | stdout_handler.setLevel(level) 47 | log.addHandler(stdout_handler) 48 | 49 | if APP_ENV != 'test': 50 | timed_filehandler = TimedRotatingFileHandler('%s/worker.log' % HOME, when='D', interval=14) 51 | timed_filehandler.setFormatter(formatter) 52 | timed_filehandler.setLevel(level) 53 | log.addHandler(timed_filehandler) 54 | 55 | return log 56 | 57 | 58 | logger = setup_logging(logging.INFO) 59 | -------------------------------------------------------------------------------- /utils/etl_base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from utils.config import Session, logger 3 | from traceback import format_exc 4 | from typing import List 5 | from utils.models import Base 6 | 7 | class ETLBase(ABC): 8 | 9 | @staticmethod 10 | def load_data(data: List[Base]) -> None: 11 | if len(data) > 0: 12 | session = Session() 13 | for idx, record in enumerate(data): 14 | try: 15 | session.merge(record) 16 | except Exception: 17 | logger.info('Something went wrong: %s' % record) 18 | logger.error(format_exc()) 19 | continue 20 | logger.debug(record) 21 | if idx > 0 and idx % 100 == 0: 22 | session.commit() 23 | logger.info('Chunked commit at %s records' % idx) 24 | session.commit() 25 | logger.info('Chunked commit at %s records' % idx) 26 | session.close() 27 | else: 28 | logger.info('No data to load') 29 | 30 | @staticmethod 31 | @abstractmethod 32 | def job() -> None: 33 | pass 34 | -------------------------------------------------------------------------------- /utils/models/__init__.py: -------------------------------------------------------------------------------- 1 | from utils.models.base import Base # NOQA 2 | from utils.models.stocks import Stock # NOQA 3 | from utils.models.prices import Price # NOQA 4 | from utils.models.balance_sheet_statements import BalanceSheetStatement # NOQA 5 | from utils.models.income_statements import IncomeStatement # NOQA 6 | from utils.models.cash_flow_statements import CashFlowStatement # NOQA 7 | from utils.models._view_piotroski_score import PiotroskiScore # NOQA 8 | from utils.models._view_magic_formula_score import MagicFormulaScore # NOQA 9 | from utils.models._view_screen_results import ScreenResults # NOQA 10 | -------------------------------------------------------------------------------- /utils/models/_view_magic_formula_score.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, String, Float, Date 2 | from .base import Base 3 | 4 | 5 | class MagicFormulaScore(Base): 6 | __tablename__ = 'magic_formula_score' 7 | isin = Column(String, primary_key=True) 8 | report_date = Column(Date, primary_key=True) 9 | market_date = Column(Date) 10 | roic = Column(Float) 11 | ev_ebitda_ratio_inv = Column(Float) 12 | shareholder_yield_stock = Column(Float) 13 | shareholder_yield_dividends = Column(Float) 14 | price_to_sales = Column(Float) 15 | price_to_cash_flow = Column(Float) 16 | ncav_ratio = Column(Float) 17 | price = Column(Float) 18 | target_median_price = Column(Float) 19 | recommendation = Column(Float) 20 | number_of_analyst_opinions = Column(Float) 21 | ebitda = Column(Float) 22 | market_cap = Column(Float) 23 | trailing_pe = Column(Float) 24 | forward_pe = Column(Float) 25 | ev_ebitda_ratio = Column(Float) 26 | magic_formula_score = Column(Float) 27 | name = __tablename__ 28 | sqltext = """ 29 | SELECT 30 | *, 31 | CASE 32 | WHEN 33 | SIGN(roic) = -1 AND SIGN(ev_ebitda_ratio_inv) = -1 THEN NULL 34 | ELSE roic * ev_ebitda_ratio_inv 35 | END AS magic_formula_score 36 | FROM 37 | ( 38 | SELECT 39 | isin, 40 | report_date, 41 | (ebit * (1-(COALESCE(income_tax_expense, 0) / NULLIF(COALESCE(income_before_tax, 0), 0)))) / NULLIF( ( 42 | (COALESCE(total_assets, 0) - COALESCE(other_assets, 0) - COALESCE(total_current_liabilities, 0) - COALESCE 43 | (cash, 0)) + LAG((COALESCE(total_assets, 0) - COALESCE(other_assets, 0) - COALESCE 44 | (total_current_liabilities, 0) - COALESCE(cash, 0))) OVER (partition BY isin ORDER BY report_date ASC)) / 45 | 2.0, 0) AS roic, 46 | 1.0 / NULLIF(ev_ebitda_ratio, 0) AS ev_ebitda_ratio_inv, 47 | (LAG(common_stock) OVER (PARTITION BY isin ORDER BY report_date ASC) - common_stock) / NULLIF(LAG 48 | (common_stock) OVER (PARTITION BY isin ORDER BY report_date ASC), 0) AS shareholder_yield_stock, 49 | ABS(dividends_paid) / NULLIF(market_cap, 0) AS shareholder_yield_dividends, 50 | market_cap / NULLIF(a.total_revenue, 0) AS price_to_sales, 51 | market_cap / NULLIF(b.total_cash_from_operating_activities, 0) AS price_to_cash_flow, 52 | (COALESCE(total_current_assets, 0) - COALESCE(total_liab, 0)) / NULLIF(market_cap, 0) AS ncav_ratio, 53 | price, 54 | target_median_price, 55 | recommendation, 56 | number_of_analyst_opinions, 57 | ebitda, 58 | market_cap, 59 | trailing_pe, 60 | forward_pe, 61 | ev_ebitda_ratio, 62 | market_date 63 | FROM 64 | income_statements AS a 65 | FULL JOIN 66 | cash_flow_statements AS b 67 | USING 68 | (isin, report_date) 69 | FULL JOIN 70 | balance_sheet_statements AS c 71 | USING 72 | (isin, report_date) 73 | LEFT JOIN 74 | ( 75 | SELECT 76 | a.* 77 | FROM 78 | prices AS a 79 | INNER JOIN 80 | ( 81 | SELECT 82 | isin, 83 | MAX(market_date) AS market_date 84 | FROM 85 | prices 86 | GROUP BY 87 | 1) AS b 88 | USING 89 | (isin, market_date)) AS d 90 | USING 91 | (isin) ) AS a 92 | """ 93 | -------------------------------------------------------------------------------- /utils/models/_view_piotroski_score.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, String, Integer, Date 2 | from .base import Base 3 | 4 | 5 | class PiotroskiScore(Base): 6 | __tablename__ = 'piotroski_score' 7 | isin = Column(String, primary_key=True) 8 | report_date = Column(Date, primary_key=True) 9 | p_score_1 = Column(Integer) 10 | p_score_2 = Column(Integer) 11 | p_score_3 = Column(Integer) 12 | p_score_4 = Column(Integer) 13 | p_score_5 = Column(Integer) 14 | p_score_6 = Column(Integer) 15 | p_score_7 = Column(Integer) 16 | p_score_8 = Column(Integer) 17 | p_score_9 = Column(Integer) 18 | p_score = Column(Integer) 19 | name = __tablename__ 20 | sqltext = """ 21 | SELECT 22 | isin, 23 | report_date, 24 | CASE 25 | WHEN p_score_1 26 | THEN 1 27 | ELSE 0 28 | END AS p_score_1, 29 | CASE 30 | WHEN p_score_2 31 | THEN 1 32 | ELSE 0 33 | END AS p_score_2, 34 | CASE 35 | WHEN p_score_3 36 | THEN 1 37 | ELSE 0 38 | END AS p_score_3, 39 | CASE 40 | WHEN p_score_4 41 | THEN 1 42 | ELSE 0 43 | END AS p_score_4, 44 | CASE 45 | WHEN p_score_5 46 | THEN 1 47 | ELSE 0 48 | END AS p_score_5, 49 | CASE 50 | WHEN p_score_6 51 | THEN 1 52 | ELSE 0 53 | END AS p_score_6, 54 | CASE 55 | WHEN p_score_7 56 | THEN 1 57 | ELSE 0 58 | END AS p_score_7, 59 | CASE 60 | WHEN p_score_8 61 | THEN 1 62 | ELSE 0 63 | END AS p_score_8, 64 | CASE 65 | WHEN p_score_9 66 | THEN 1 67 | ELSE 0 68 | END AS p_score_9, 69 | CASE 70 | WHEN p_score_1 71 | THEN 1 72 | ELSE 0 73 | END + 74 | CASE 75 | WHEN p_score_2 76 | THEN 1 77 | ELSE 0 78 | END + 79 | CASE 80 | WHEN p_score_3 81 | THEN 1 82 | ELSE 0 83 | END + 84 | CASE 85 | WHEN p_score_4 86 | THEN 1 87 | ELSE 0 88 | END + 89 | CASE 90 | WHEN p_score_5 91 | THEN 1 92 | ELSE 0 93 | END + 94 | CASE 95 | WHEN p_score_6 96 | THEN 1 97 | ELSE 0 98 | END + 99 | CASE 100 | WHEN p_score_7 101 | THEN 1 102 | ELSE 0 103 | END + 104 | CASE 105 | WHEN p_score_8 106 | THEN 1 107 | ELSE 0 108 | END + 109 | CASE 110 | WHEN p_score_9 111 | THEN 1 112 | ELSE 0 113 | END AS p_score 114 | FROM 115 | ( 116 | SELECT 117 | isin, 118 | report_date, 119 | COALESCE(return_on_assets > 0, FALSE) AS p_score_1, 120 | COALESCE(total_cash_from_operating_activities > 0.0, FALSE) AS p_score_2, 121 | COALESCE(return_on_assets > LAG(return_on_assets) OVER (PARTITION BY isin ORDER BY report_date ASC), FALSE 122 | ) AS p_score_3, 123 | COALESCE(total_cash_from_operating_activities > net_income, FALSE) AS p_score_4, 124 | COALESCE(long_term_debt < LAG(long_term_debt) OVER (PARTITION BY isin ORDER BY report_date ASC), FALSE) AS 125 | p_score_5, 126 | COALESCE(current_ratio > LAG(current_ratio) OVER (PARTITION BY isin ORDER BY report_date ASC), FALSE) AS 127 | p_score_6, 128 | COALESCE(net_shares_issued <= 0, FALSE) AS p_score_7, 129 | COALESCE(gross_margin_pct > LAG(gross_margin_pct) OVER (PARTITION BY isin ORDER BY report_date ASC), FALSE 130 | ) AS p_score_8, 131 | COALESCE(asset_turnover > LAG(asset_turnover) OVER (PARTITION BY isin ORDER BY report_date ASC), FALSE) AS 132 | p_score_9 133 | FROM 134 | ( 135 | SELECT 136 | isin, 137 | report_date, 138 | (total_revenue - cost_of_revenue) / NULLIF(total_revenue, 0) AS gross_margin_pct, 139 | total_current_assets / NULLIF(total_current_liabilities, 0) AS current_ratio, 140 | total_revenue / ( 141 | CASE 142 | WHEN LAG(total_assets) OVER (PARTITION BY isin ORDER BY report_date ASC) IS NULL 143 | THEN total_assets 144 | ELSE (LAG(total_assets) OVER (PARTITION BY isin ORDER BY report_date ASC) + total_assets) 145 | / 2 146 | END) AS asset_turnover, 147 | COALESCE(issuance_of_stock, 0.0) + COALESCE(repurchase_of_stock, 0.0) AS net_shares_issued, 148 | a.net_income / NULLIF(total_assets, 0) AS return_on_assets, 149 | total_cash_from_operating_activities, 150 | a.net_income, 151 | long_term_debt 152 | FROM 153 | income_statements AS a 154 | FULL JOIN 155 | cash_flow_statements AS b 156 | USING 157 | (isin, report_date) 158 | FULL JOIN 159 | balance_sheet_statements AS c 160 | USING 161 | (isin, report_date) ) AS a) AS a 162 | """ 163 | -------------------------------------------------------------------------------- /utils/models/_view_screen_results.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, String, Float, Date, Integer 2 | from .base import Base 3 | 4 | 5 | class ScreenResults(Base): 6 | __tablename__ = 'screen_results' 7 | isin = Column(String, primary_key=True) 8 | company_name = Column(String) 9 | symbol = Column(String) 10 | currency = Column(String) 11 | sector = Column(String) 12 | yahoo_ticker = Column(String) 13 | report_date = Column(Date, primary_key=True) 14 | market_date = Column(Date) 15 | p_score = Column(Integer) 16 | roic = Column(Float) 17 | ev_ebitda_ratio_inv = Column(Float) 18 | shareholder_yield_stock = Column(Float) 19 | shareholder_yield_dividends = Column(Float) 20 | price_to_sales = Column(Float) 21 | price_to_cash_flow = Column(Float) 22 | ncav_ratio = Column(Float) 23 | price = Column(Float) 24 | target_median_price = Column(Float) 25 | number_of_analyst_opinions = Column(Float) 26 | ebitda = Column(Float) 27 | market_cap = Column(Float) 28 | trailing_pe = Column(Float) 29 | forward_pe = Column(Float) 30 | ev_ebitda_ratio = Column(Float) 31 | magic_formula_score = Column(Float) 32 | name = __tablename__ 33 | sqltext = """ 34 | SELECT 35 | isin, 36 | name AS company_name, 37 | symbol, 38 | currency, 39 | sector, 40 | yahoo_ticker, 41 | report_date, 42 | market_date, 43 | CASE 44 | WHEN p_score IN ('Infinity'::Float, -'Infinity'::Float) THEN 45 | NULL 46 | ELSE 47 | p_score 48 | END AS p_score, 49 | CASE 50 | WHEN roic IN ('Infinity'::Float, -'Infinity'::Float) THEN 51 | NULL 52 | ELSE 53 | roic 54 | END AS roic, 55 | CASE 56 | WHEN ev_ebitda_ratio_inv IN ('Infinity'::Float, -'Infinity'::Float) THEN 57 | NULL 58 | ELSE 59 | ev_ebitda_ratio_inv 60 | END AS ev_ebitda_ratio_inv, 61 | CASE 62 | WHEN shareholder_yield_stock IN ('Infinity'::Float, -'Infinity'::Float) THEN 63 | NULL 64 | ELSE 65 | shareholder_yield_stock 66 | END AS shareholder_yield_stock, 67 | CASE 68 | WHEN shareholder_yield_dividends IN ('Infinity'::Float, -'Infinity'::Float) THEN 69 | NULL 70 | ELSE 71 | shareholder_yield_dividends 72 | END AS shareholder_yield_dividends, 73 | CASE 74 | WHEN price_to_sales IN ('Infinity'::Float, -'Infinity'::Float) THEN 75 | NULL 76 | ELSE 77 | price_to_sales 78 | END AS price_to_sales, 79 | CASE 80 | WHEN price_to_cash_flow IN ('Infinity'::Float, -'Infinity'::Float) THEN 81 | NULL 82 | ELSE 83 | price_to_cash_flow 84 | END AS price_to_cash_flow, 85 | CASE 86 | WHEN ncav_ratio IN ('Infinity'::Float, -'Infinity'::Float) THEN 87 | NULL 88 | ELSE 89 | ncav_ratio 90 | END AS ncav_ratio, 91 | CASE 92 | WHEN price IN ('Infinity'::Float, -'Infinity'::Float) THEN 93 | NULL 94 | ELSE 95 | price 96 | END AS price, 97 | CASE 98 | WHEN target_median_price IN ('Infinity'::Float, -'Infinity'::Float) THEN 99 | NULL 100 | ELSE 101 | target_median_price 102 | END AS target_median_price, 103 | CASE 104 | WHEN number_of_analyst_opinions IN ('Infinity'::Float, -'Infinity'::Float) THEN 105 | NULL 106 | ELSE 107 | number_of_analyst_opinions 108 | END AS number_of_analyst_opinions, 109 | CASE 110 | WHEN ebitda IN ('Infinity'::Float, -'Infinity'::Float) THEN 111 | NULL 112 | ELSE 113 | ebitda 114 | END AS ebitda, 115 | CASE 116 | WHEN market_cap IN ('Infinity'::Float, -'Infinity'::Float) THEN 117 | NULL 118 | ELSE 119 | market_cap 120 | END AS market_cap, 121 | CASE 122 | WHEN trailing_pe IN ('Infinity'::Float, -'Infinity'::Float) THEN 123 | NULL 124 | ELSE 125 | trailing_pe 126 | END AS trailing_pe, 127 | CASE 128 | WHEN forward_pe IN ('Infinity'::Float, -'Infinity'::Float) THEN 129 | NULL 130 | ELSE 131 | forward_pe 132 | END AS forward_pe, 133 | CASE 134 | WHEN ev_ebitda_ratio IN ('Infinity'::Float, -'Infinity'::Float) THEN 135 | NULL 136 | ELSE 137 | ev_ebitda_ratio 138 | END AS ev_ebitda_ratio, 139 | CASE 140 | WHEN magic_formula_score IN ('Infinity'::Float, -'Infinity'::Float) THEN 141 | NULL 142 | ELSE 143 | magic_formula_score 144 | END AS magic_formula_score 145 | FROM 146 | piotroski_score AS a 147 | FULL JOIN 148 | magic_formula_score AS b 149 | USING 150 | (isin, report_date) 151 | LEFT JOIN 152 | stocks AS c 153 | USING 154 | (isin) 155 | """ 156 | -------------------------------------------------------------------------------- /utils/models/balance_sheet_statements.py: -------------------------------------------------------------------------------- 1 | from .base import Base 2 | from sqlalchemy import Column, String, DateTime, Float, Date 3 | from datetime import datetime 4 | from typing import Dict 5 | from utils import get_nested 6 | 7 | 8 | class BalanceSheetStatement(Base): 9 | __tablename__ = 'balance_sheet_statements' 10 | isin = Column(String, primary_key=True) 11 | report_date = Column(Date, primary_key=True) 12 | cash = Column(Float) 13 | short_term_investments = Column(Float) 14 | net_receivables = Column(Float) 15 | total_current_assets = Column(Float) 16 | property_plant_equipment = Column(Float) 17 | intangible_assets = Column(Float) 18 | other_assets = Column(Float) 19 | deferred_long_term_asset_charges = Column(Float) 20 | total_assets = Column(Float) 21 | accounts_payable = Column(Float) 22 | short_long_term_debt = Column(Float) 23 | other_current_liab = Column(Float) 24 | long_term_debt = Column(Float) 25 | other_liab = Column(Float) 26 | deferred_long_term_liab = Column(Float) 27 | total_current_liabilities = Column(Float) 28 | total_liab = Column(Float) 29 | common_stock = Column(Float) 30 | retained_earnings = Column(Float) 31 | treasury_stock = Column(Float) 32 | other_stockholder_equity = Column(Float) 33 | total_stockholder_equity = Column(Float) 34 | net_tangible_assets = Column(Float) 35 | dw_created = Column(DateTime, default=datetime.utcnow) 36 | dw_modified = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) 37 | 38 | @classmethod 39 | def process_response(cls, response: Dict, isin: str) -> Base: 40 | record = { 41 | 'isin': isin, 42 | 'report_date': datetime.fromtimestamp(get_nested(response, 'endDate', 'raw')).date(), 43 | 'cash': get_nested(response, 'cash', 'raw'), 44 | 'short_term_investments': get_nested(response, 'shortTermInvestments', 'raw'), 45 | 'net_receivables': get_nested(response, 'netReceivables', 'raw'), 46 | 'total_current_assets': get_nested(response, 'totalCurrentAssets', 'raw'), 47 | 'property_plant_equipment': get_nested(response, 'propertyPlantEquipment', 'raw'), 48 | 'intangible_assets': get_nested(response, 'intangibleAssets', 'raw'), 49 | 'other_assets': get_nested(response, 'otherAssets', 'raw'), 50 | 'deferred_long_term_asset_charges': get_nested(response, 'deferredLongTermAssetCharges', 'raw'), 51 | 'total_assets': get_nested(response, 'totalAssets', 'raw'), 52 | 'accounts_payable': get_nested(response, 'accountsPayable', 'raw'), 53 | 'short_long_term_debt': get_nested(response, 'shortLongTermDebt', 'raw'), 54 | 'other_current_liab': get_nested(response, 'otherCurrentLiab', 'raw'), 55 | 'long_term_debt': get_nested(response, 'longTermDebt', 'raw'), 56 | 'other_liab': get_nested(response, 'otherLiab', 'raw'), 57 | 'deferred_long_term_liab': get_nested(response, 'deferredLongTermLiab', 'raw'), 58 | 'total_current_liabilities': get_nested(response, 'totalCurrentLiabilities', 'raw'), 59 | 'total_liab': get_nested(response, 'totalLiab', 'raw'), 60 | 'common_stock': get_nested(response, 'commonStock', 'raw'), 61 | 'retained_earnings': get_nested(response, 'retainedEarnings', 'raw'), 62 | 'treasury_stock': get_nested(response, 'treasuryStock', 'raw'), 63 | 'other_stockholder_equity': get_nested(response, 'otherStockholderEquity', 'raw'), 64 | 'total_stockholder_equity': get_nested(response, 'totalStockholderEquity', 'raw'), 65 | 'net_tangible_assets': get_nested(response, 'netTangibleAssets', 'raw') 66 | } 67 | result: Base = cls(**record) 68 | return result 69 | -------------------------------------------------------------------------------- /utils/models/base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.ext.declarative import declarative_base 2 | from flask_jsontools import JsonSerializableBase 3 | 4 | Base = declarative_base(cls=(JsonSerializableBase,)) 5 | -------------------------------------------------------------------------------- /utils/models/cash_flow_statements.py: -------------------------------------------------------------------------------- 1 | from .base import Base 2 | from sqlalchemy import Column, String, DateTime, Float, Date 3 | from datetime import datetime 4 | from typing import Dict 5 | from utils import get_nested 6 | 7 | 8 | class CashFlowStatement(Base): 9 | __tablename__ = 'cash_flow_statements' 10 | isin = Column(String, primary_key=True) 11 | report_date = Column(Date, primary_key=True) 12 | net_income = Column(Float) 13 | change_to_netincome = Column(Float) 14 | change_to_account_receivables = Column(Float) 15 | change_to_liabilities = Column(Float) 16 | total_cash_from_operating_activities = Column(Float) 17 | capital_expenditures = Column(Float) 18 | other_cashflows_from_investing_activities = Column(Float) 19 | total_cashflows_from_investing_activities = Column(Float) 20 | dividends_paid = Column(Float) 21 | net_borrowings = Column(Float) 22 | other_cashflows_from_financing_activities = Column(Float) 23 | total_cash_from_financing_activities = Column(Float) 24 | effect_of_exchange_rate = Column(Float) 25 | change_in_cash = Column(Float) 26 | repurchase_of_stock = Column(Float) 27 | issuance_of_stock = Column(Float) 28 | dw_created = Column(DateTime, default=datetime.utcnow) 29 | dw_modified = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) 30 | 31 | @classmethod 32 | def process_response(cls, response: Dict, isin: str) -> Base: 33 | record = { 34 | 'isin': isin, 35 | 'report_date': datetime.fromtimestamp(get_nested(response, 'endDate', 'raw')).date(), 36 | 'net_income': get_nested(response, 'netIncome', 'raw'), 37 | 'change_to_netincome': get_nested(response, 'changeToNetincome', 'raw'), 38 | 'change_to_account_receivables': get_nested(response, 'changeToAccountReceivables', 'raw'), 39 | 'change_to_liabilities': get_nested(response, 'changeToLiabilities', 'raw'), 40 | 'total_cash_from_operating_activities': get_nested(response, 'totalCashFromOperatingActivities', 'raw'), 41 | 'capital_expenditures': get_nested(response, 'capitalExpenditures', 'raw'), 42 | 'other_cashflows_from_investing_activities': get_nested(response, 43 | 'otherCashflowsFromInvestingActivities', 'raw'), 44 | 'total_cashflows_from_investing_activities': get_nested(response, 45 | 'totalCashflowsFromInvestingActivities', 'raw'), 46 | 'dividends_paid': get_nested(response, 'dividendsPaid', 'raw'), 47 | 'net_borrowings': get_nested(response, 'netBorrowings', 'raw'), 48 | 'other_cashflows_from_financing_activities': get_nested(response, 'otherCashflowsFromFinancingActivities', 49 | 'raw'), 50 | 'total_cash_from_financing_activities': get_nested(response, 'totalCashFromFinancingActivities', 'raw'), 51 | 'effect_of_exchange_rate': get_nested(response, 'effectOfExchangeRate', 'raw'), 52 | 'change_in_cash': get_nested(response, 'changeInCash', 'raw'), 53 | 'repurchase_of_stock': get_nested(response, 'repurchaseOfStock', 'raw'), 54 | 'issuance_of_stock': get_nested(response, 'issuanceOfStock', 'raw') 55 | } 56 | result: Base = cls(**record) 57 | return result 58 | -------------------------------------------------------------------------------- /utils/models/income_statements.py: -------------------------------------------------------------------------------- 1 | from .base import Base 2 | from sqlalchemy import Column, String, DateTime, Float, Date 3 | from datetime import datetime 4 | from typing import Dict 5 | from utils import get_nested 6 | 7 | 8 | class IncomeStatement(Base): 9 | __tablename__ = 'income_statements' 10 | isin = Column(String, primary_key=True) 11 | report_date = Column(Date, primary_key=True) 12 | total_revenue = Column(Float) 13 | cost_of_revenue = Column(Float) 14 | gross_profit = Column(Float) 15 | research_development = Column(Float) 16 | selling_general_administrative = Column(Float) 17 | non_recurring = Column(Float) 18 | other_operating_expenses = Column(Float) 19 | total_operating_expenses = Column(Float) 20 | operating_income = Column(Float) 21 | total_other_income_expense_net = Column(Float) 22 | ebit = Column(Float) 23 | interest_expense = Column(Float) 24 | income_before_tax = Column(Float) 25 | income_tax_expense = Column(Float) 26 | minority_interest = Column(Float) 27 | net_income_from_continuing_ops = Column(Float) 28 | discontinued_operations = Column(Float) 29 | extraordinary_items = Column(Float) 30 | effect_of_accounting_charges = Column(Float) 31 | other_items = Column(Float) 32 | net_income = Column(Float) 33 | net_income_applicable_to_common_shares = Column(Float) 34 | dw_created = Column(DateTime, default=datetime.utcnow) 35 | dw_modified = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) 36 | 37 | @classmethod 38 | def process_response(cls, response: Dict, isin: str) -> Base: 39 | record = { 40 | 'isin': isin, 41 | 'report_date': datetime.fromtimestamp(get_nested(response, 'endDate', 'raw')).date(), 42 | 'total_revenue': get_nested(response, 'totalRevenue', 'raw'), 43 | 'cost_of_revenue': get_nested(response, 'costOfRevenue', 'raw'), 44 | 'gross_profit': get_nested(response, 'grossProfit', 'raw'), 45 | 'research_development': get_nested(response, 'researchDevelopment', 'raw'), 46 | 'selling_general_administrative': get_nested(response, 'sellingGeneralAdministrative', 'raw'), 47 | 'non_recurring': get_nested(response, 'nonRecurring', 'raw'), 48 | 'other_operating_expenses': get_nested(response, 'otherOperatingExpenses', 'raw'), 49 | 'total_operating_expenses': get_nested(response, 'totalOperatingExpenses', 'raw'), 50 | 'operating_income': get_nested(response, 'operatingIncome', 'raw'), 51 | 'total_other_income_expense_net': get_nested(response, 'totalOtherIncomeExpenseNet', 'raw'), 52 | 'ebit': get_nested(response, 'ebit', 'raw'), 53 | 'interest_expense': get_nested(response, 'interestExpense', 'raw'), 54 | 'income_before_tax': get_nested(response, 'incomeBeforeTax', 'raw'), 55 | 'income_tax_expense': get_nested(response, 'incomeTaxExpense', 'raw'), 56 | 'minority_interest': get_nested(response, 'minorityInterest', 'raw'), 57 | 'net_income_from_continuing_ops': get_nested(response, 'netIncomeFromContinuingOps', 'raw'), 58 | 'discontinued_operations': get_nested(response, 'discontinuedOperations', 'raw'), 59 | 'extraordinary_items': get_nested(response, 'extraordinaryItems', 'raw'), 60 | 'effect_of_accounting_charges': get_nested(response, 'effectOfAccountingCharges', 'raw'), 61 | 'other_items': get_nested(response, 'otherItems', 'raw'), 62 | 'net_income': get_nested(response, 'netIncome', 'raw'), 63 | 'net_income_applicable_to_common_shares': get_nested(response, 'netIncomeApplicableToCommonShares', 'raw') 64 | } 65 | result: Base = cls(**record) 66 | return result 67 | -------------------------------------------------------------------------------- /utils/models/prices.py: -------------------------------------------------------------------------------- 1 | from utils.models.base import Base 2 | from sqlalchemy import Column, String, DateTime, Float, Date 3 | from datetime import datetime 4 | from utils import get_nested 5 | from typing import Dict 6 | 7 | 8 | class Price(Base): 9 | __tablename__ = 'prices' 10 | isin = Column(String, primary_key=True) 11 | market_date = Column(Date, primary_key=True) 12 | price = Column(Float) 13 | target_median_price = Column(Float) 14 | recommendation = Column(Float) 15 | number_of_analyst_opinions = Column(Float) 16 | ebitda = Column(Float) 17 | market_cap = Column(Float) 18 | trailing_pe = Column(Float) 19 | forward_pe = Column(Float) 20 | ev_ebitda_ratio = Column(Float) 21 | dw_created = Column(DateTime, default=datetime.utcnow) 22 | dw_modified = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) 23 | 24 | @classmethod 25 | def process_response(cls, response: Dict, isin: str) -> Base: 26 | record = { 27 | 'isin': isin, 28 | 'market_date': datetime.fromtimestamp(get_nested(response, 'price', 'regularMarketTime')).date(), 29 | 'price': get_nested(response, 'financialData', 'currentPrice', 'raw'), 30 | 'target_median_price': get_nested(response, 'financialData', 'targetMedianPrice', 'raw'), 31 | 'recommendation': get_nested(response, 'financialData', 'recommendationKey', 'raw'), 32 | 'number_of_analyst_opinions': get_nested(response, 'financialData', 'numberOfAnalystOpinions', 'raw'), 33 | 'ebitda': get_nested(response, 'financialData', 'ebitda', 'raw'), 34 | 'market_cap': get_nested(response, 'summaryDetail', 'marketCap', 'raw'), 35 | 'trailing_pe': get_nested(response, 'summaryDetail', 'trailingPE', 'raw'), 36 | 'forward_pe': get_nested(response, 'summaryDetail', 'forwardPE', 'raw'), 37 | 'ev_ebitda_ratio': get_nested(response, 'defaultKeyStatistics', 'enterpriseToEbitda', 'raw') 38 | } 39 | result: Base = cls(**record) 40 | return result 41 | -------------------------------------------------------------------------------- /utils/models/stocks.py: -------------------------------------------------------------------------------- 1 | from utils.models.base import Base 2 | from sqlalchemy import Column, String, DateTime 3 | from datetime import datetime 4 | from typing import List, Dict 5 | 6 | 7 | class Stock(Base): 8 | __tablename__ = 'stocks' 9 | isin = Column(String, primary_key=True) 10 | name = Column(String) 11 | symbol = Column(String) 12 | currency = Column(String) 13 | sector = Column(String) 14 | yahoo_ticker = Column(String) 15 | dw_created = Column(DateTime, default=datetime.utcnow) 16 | dw_modified = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) 17 | 18 | @staticmethod 19 | def parse_yahoo_ticker_from_isin(record: Dict[str, str]) -> str: 20 | symbol: str = record.get('symbol', '').replace(' ', '-') 21 | isin_country: str = record.get('isin', '')[:2] 22 | currency: str = record.get('currency', '') 23 | if isin_country == 'DK': 24 | return symbol + '.CO' 25 | elif isin_country == 'SE': 26 | return symbol + '.ST' 27 | elif isin_country == 'FI': 28 | return symbol + '.HE' 29 | elif isin_country == 'NO': 30 | return symbol.replace('o', '') + '.OL' 31 | elif currency == 'DKK': 32 | return symbol + '.CO' 33 | elif currency == 'ISK': 34 | return symbol + '.CO' 35 | elif currency == 'SEK': 36 | return symbol + '.ST' 37 | elif currency == 'EUR': 38 | return symbol + '.HE' 39 | elif currency == 'NOK': 40 | return symbol.replace('o', '') + '.OL' 41 | else: 42 | return '' 43 | 44 | @classmethod 45 | def process_response(cls, response: List) -> Base: 46 | record: Dict[str, str] = { 47 | 'isin': response[3], 48 | 'name': response[0], 49 | 'symbol': response[1], 50 | 'currency': response[2], 51 | 'sector': response[4] 52 | } 53 | record['yahoo_ticker'] = cls.parse_yahoo_ticker_from_isin(record.copy()) 54 | result: Base = cls(**record) 55 | return result 56 | -------------------------------------------------------------------------------- /utils/queries.py: -------------------------------------------------------------------------------- 1 | from utils.models import Stock, IncomeStatement, BalanceSheetStatement, CashFlowStatement, ScreenResults 2 | from typing import List, Tuple, Union, Dict 3 | from utils.config import Session, get_last_year 4 | from sqlalchemy import func 5 | 6 | 7 | def fetch_all_tickers_from_database() -> List[Tuple]: 8 | session = Session() 9 | res: List[Tuple] = session.query(Stock.isin, Stock.yahoo_ticker).group_by(Stock.isin, Stock.yahoo_ticker).all() 10 | session.close() 11 | return res 12 | 13 | 14 | def fetch_isins_not_updated_financials(Model: Union[IncomeStatement, 15 | BalanceSheetStatement, 16 | CashFlowStatement]) -> List[Tuple]: 17 | session = Session() 18 | res: List[Tuple] = session.query(Stock.isin, Stock.yahoo_ticker).filter(~Stock.isin.in_( 19 | session.query(Model.isin).filter(func.extract('year', Model.report_date) == get_last_year().year).all() 20 | )).group_by(Stock.isin, Stock.yahoo_ticker).all() 21 | return res 22 | 23 | 24 | def screened_stocks() -> List[Dict]: 25 | session = Session() 26 | res: List[Tuple] = session.query(ScreenResults)\ 27 | .filter(func.extract('year', ScreenResults.report_date) == get_last_year().year).all() 28 | res1 = [row.__json__() for row in res] # type: ignore 29 | return res1 30 | -------------------------------------------------------------------------------- /utils/stock_financial_statements_etl.py: -------------------------------------------------------------------------------- 1 | from utils import fetch_yahoo_data, get_nested, union_of_list_elements 2 | from utils.queries import fetch_isins_not_updated_financials 3 | from utils.models import Base, BalanceSheetStatement, CashFlowStatement, IncomeStatement 4 | from utils.config import logger 5 | from utils.etl_base import ETLBase 6 | from typing import Dict, List, Tuple, Union, Any 7 | from traceback import format_exc 8 | 9 | 10 | def fetch_yahoo_responses() -> List[Tuple]: 11 | tickers: List[List] = [] 12 | for model in [IncomeStatement, BalanceSheetStatement, CashFlowStatement]: 13 | tickers.append(fetch_isins_not_updated_financials(model)) # type: ignore 14 | tickers_unique: List[Tuple] = union_of_list_elements(*tickers) 15 | logger.info('Fetching financials from %s stocks' % len(tickers_unique)) 16 | responses: List[Tuple[Any, ...]] = [] 17 | for ticker_tuple in tickers_unique: 18 | if len(ticker_tuple) == 2: 19 | isin: str = ticker_tuple[0] 20 | yahoo_ticker: str = ticker_tuple[1] 21 | try: 22 | response = fetch_yahoo_data(yahoo_ticker, 23 | 'balanceSheetHistory,incomeStatementHistory,cashflowStatementHistory') 24 | logger.info('Succeeded getting ticker, isin: %s, %s' % (yahoo_ticker, isin)) 25 | except Exception: 26 | logger.error('Something went wrong getting ticker, isin: %s, %s' % (yahoo_ticker, isin)) 27 | logger.error(format_exc()) 28 | continue 29 | responses.append((response, isin)) 30 | else: 31 | continue 32 | return responses 33 | 34 | 35 | def traverse_statement_history(Model: Union[IncomeStatement, 36 | BalanceSheetStatement, 37 | CashFlowStatement], 38 | isin: str, 39 | statements: List[Dict]) -> List[Base]: 40 | data = [] 41 | for statement in statements: 42 | data.append(Model.process_response(statement, isin)) 43 | return data 44 | 45 | 46 | class StockFinancialStatementsETL(ETLBase): 47 | 48 | @staticmethod 49 | def job() -> None: 50 | data: List[Base] = [] 51 | responses: List[Tuple] = fetch_yahoo_responses() 52 | for response in responses: 53 | payload: Dict = response[0] 54 | isin: str = response[1] 55 | income_statement_response: List = get_nested(payload, 56 | 'incomeStatementHistory', 'incomeStatementHistory', 57 | default=[]) 58 | data = data + traverse_statement_history(Model=IncomeStatement, # type: ignore 59 | isin=isin, statements=income_statement_response) 60 | cash_flow_statement_response: List = get_nested(payload, 61 | 'cashflowStatementHistory', 'cashflowStatements', 62 | default=[]) 63 | data = data + traverse_statement_history(Model=CashFlowStatement, # type: ignore 64 | isin=isin, statements=cash_flow_statement_response) 65 | balance_sheet_statement_response: List = get_nested(payload, 66 | 'balanceSheetHistory', 'balanceSheetStatements', 67 | default=[]) 68 | data = data + traverse_statement_history(Model=BalanceSheetStatement, # type: ignore 69 | isin=isin, statements=balance_sheet_statement_response) 70 | ETLBase.load_data(data) 71 | -------------------------------------------------------------------------------- /utils/stock_info_etl.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from requests import Response 3 | import bs4 as bs 4 | from bs4 import BeautifulSoup, Tag 5 | from utils.models import Stock 6 | from utils.etl_base import ETLBase 7 | from utils.models import Base 8 | from typing import List 9 | 10 | STOCK_INFO_URLS = [ 11 | 'http://www.nasdaqomxnordic.com/aktier/listed-companies/copenhagen', 12 | 'http://www.nasdaqomxnordic.com/aktier/listed-companies/helsinki', 13 | 'http://www.nasdaqomxnordic.com/aktier/listed-companies/stockholm', 14 | 'http://www.nasdaqomxnordic.com/aktier/listed-companies/first-north', 15 | 'http://www.nasdaqomxnordic.com/aktier/listed-companies/norwegian-listed-shares' 16 | ] 17 | 18 | 19 | def get_stock_info_soup_table(response: Response) -> Tag: 20 | soup: BeautifulSoup = bs.BeautifulSoup(response.text, 'lxml') 21 | table: Tag = soup.find('table', {'id': 'listedCompanies'}) 22 | return table 23 | 24 | 25 | def create_data_from_soup(soup: Tag) -> List[Base]: 26 | data: List[Base] = [] 27 | # Iterate all rows in table (skip header) 28 | for row in soup.findAll('tr')[1:]: 29 | values = [cell.string for cell in row.findChildren('td')] 30 | record = Stock.process_response(values) 31 | data.append(record) 32 | return data 33 | 34 | 35 | class StockInfoETL(ETLBase): 36 | 37 | @staticmethod 38 | def job() -> None: 39 | data: List[Base] = [] 40 | for url in STOCK_INFO_URLS: 41 | response = requests.get(url) 42 | soup_table = get_stock_info_soup_table(response) 43 | data = data + create_data_from_soup(soup_table) 44 | ETLBase.load_data(data) 45 | -------------------------------------------------------------------------------- /utils/stock_valuation_etl.py: -------------------------------------------------------------------------------- 1 | from utils import fetch_yahoo_data 2 | from utils.queries import fetch_all_tickers_from_database 3 | from utils.models import Base, Price 4 | from utils.config import logger 5 | from utils.etl_base import ETLBase 6 | from typing import List 7 | 8 | 9 | def create_database_records() -> List[Base]: 10 | tickers = fetch_all_tickers_from_database() 11 | data: List[Base] = [] 12 | for ticker_tuple in tickers: 13 | if len(ticker_tuple) == 2: 14 | try: 15 | response = fetch_yahoo_data(ticker_tuple[1], 'summaryDetail,financialData,price,defaultKeyStatistics') 16 | record = Price.process_response(response, ticker_tuple[0]) 17 | except Exception: 18 | logger.error('Something went wrong getting ticker %s' % ticker_tuple[1]) 19 | continue 20 | data.append(record) 21 | else: 22 | continue 23 | return data 24 | 25 | 26 | class StockValuationETL(ETLBase): 27 | 28 | @staticmethod 29 | def job() -> None: 30 | data = create_database_records() 31 | ETLBase.load_data(data) 32 | -------------------------------------------------------------------------------- /web/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, session, redirect, abort, url_for 2 | from flask_jsontools import jsonapi 3 | from datetime import timedelta 4 | from utils.queries import screened_stocks 5 | from utils import ApiJSONEncoder 6 | import os 7 | 8 | 9 | class ReverseProxied(object): 10 | '''Wrap the application in this middleware and configure the 11 | front-end server to add these headers, to let you quietly bind 12 | this to a URL other than / and to an HTTP scheme that is 13 | different than what is used locally. 14 | 15 | In nginx: 16 | location /myprefix { 17 | proxy_pass http://192.168.0.1:5001; 18 | proxy_set_header Host $host; 19 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 20 | proxy_set_header X-Scheme $scheme; 21 | proxy_set_header X-Script-Name /myprefix; 22 | } 23 | 24 | :param app: the WSGI application 25 | ''' 26 | 27 | def __init__(self, app): 28 | self.app = app 29 | 30 | def __call__(self, environ, start_response): 31 | script_name = environ.get('HTTP_X_SCRIPT_NAME', '') 32 | if script_name: 33 | environ['SCRIPT_NAME'] = script_name 34 | path_info = environ['PATH_INFO'] 35 | if path_info.startswith(script_name): 36 | environ['PATH_INFO'] = path_info[len(script_name):] 37 | 38 | scheme = environ.get('HTTP_X_SCHEME', '') 39 | if scheme: 40 | environ['wsgi.url_scheme'] = scheme 41 | return self.app(environ, start_response) 42 | 43 | 44 | app = Flask(__name__) 45 | app.secret_key = os.getenv('SESSION_SECRET') 46 | app.json_encoder = ApiJSONEncoder 47 | app.wsgi_app = ReverseProxied(app.wsgi_app) 48 | 49 | 50 | @app.route("/stocks", methods=["GET"]) 51 | @jsonapi 52 | def stocks(): 53 | if session.get('logged_in'): 54 | result = screened_stocks() 55 | return result 56 | else: 57 | return abort(401) 58 | 59 | 60 | @app.before_request 61 | def make_session_permanent(): 62 | session.permanent = True 63 | app.permanent_session_lifetime = timedelta(minutes=60) 64 | 65 | 66 | @app.route('/') 67 | def index(): 68 | if session.get('logged_in'): 69 | return app.send_static_file('index.html') 70 | else: 71 | return app.send_static_file('login.html') 72 | 73 | 74 | @app.route('/login', methods=['POST']) 75 | def login(): 76 | if request.form.get('password') == os.getenv('STOCKS_PASSWORD') and \ 77 | request.form.get('username') == os.getenv('STOCKS_USERNAME'): 78 | session['logged_in'] = True 79 | return redirect(url_for('.index', _external=True)) 80 | 81 | 82 | @app.route('/logout', methods=['POST']) 83 | def logout(): 84 | session['logged_in'] = False 85 | return redirect(url_for('.index', _external=True)) 86 | 87 | 88 | if __name__ == '__main__': 89 | app.run(debug=True, host='0.0.0.0', port=5000) 90 | -------------------------------------------------------------------------------- /web/static/css/index.css: -------------------------------------------------------------------------------- 1 | html { 2 | min-height: 100%; 3 | background: #ffffff; 4 | /* Old Browsers */ 5 | background: -moz-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 6 | /* FF3.6+ */ 7 | background: -webkit-gradient(left top, left bottom, color-stop(0%, #ffffff), color-stop(47%, #f6f6f6), color-stop(100%, #ededed)); 8 | /* Chrome, Safari4+ */ 9 | background: -webkit-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 10 | /* Chrome10+,Safari5.1+ */ 11 | background: -o-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 12 | /* Opera 11.10+ */ 13 | background: -ms-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 14 | /* IE 10+ */ 15 | background: linear-gradient(to bottom, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 16 | /* W3C */ 17 | filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#ffffff', endColorstr='#ededed', GradientType=0); 18 | /* IE6-9 */ 19 | font-family: "Open Sans", Helvetica, Verdana, sans-serif; 20 | font-size: 14px; 21 | -webkit-font-smoothing: antialiased; 22 | -moz-osx-font-smoothing: grayscale; 23 | } 24 | 25 | @media only screen and (max-width: 600px) { 26 | .buttonribbon { 27 | display: flex; 28 | flex-direction: column; 29 | flex-wrap: nowrap; 30 | justify-content: center; 31 | align-items: center; 32 | padding: 10px; 33 | } 34 | .buttonribbon h1 { 35 | margin-bottom: auto; 36 | } 37 | .actionbtn { 38 | border-radius: 25px; 39 | width: 100%; 40 | font-family: "Open Sans", Helvetica, Verdana, sans-serif; 41 | text-transform: uppercase; 42 | outline: 0; 43 | background: #4CAF50; 44 | border: 0; 45 | padding: 15px; 46 | color: #FFFFFF; 47 | -webkit-transition: all 0.3 ease; 48 | transition: all 0.3 ease; 49 | cursor: pointer; 50 | margin-top: 10px; 51 | margin-bottom: 10px; 52 | } 53 | .actionbtn:hover, .form .actionbtn:active, .form .actionbtn:focus { 54 | background: #43A047; 55 | } 56 | #logout { 57 | width: 100%; 58 | } 59 | #download_data_link { 60 | width: 100%; 61 | } 62 | } 63 | 64 | @media only screen and (min-width: 600px) { 65 | .buttonribbon { 66 | display: flex; 67 | flex-direction: row; 68 | flex-wrap: nowrap; 69 | justify-content: flex-end; 70 | align-items: center; 71 | padding: 10px; 72 | height: 100px; 73 | } 74 | .buttonribbon h1 { 75 | margin-right: auto; 76 | } 77 | .actionbtn { 78 | border-radius: 25px; 79 | width: 120px; 80 | font-family: "Open Sans", Helvetica, Verdana, sans-serif; 81 | text-transform: uppercase; 82 | outline: 0; 83 | background: #4CAF50; 84 | border: 0; 85 | padding: 15px; 86 | color: #FFFFFF; 87 | -webkit-transition: all 0.3 ease; 88 | transition: all 0.3 ease; 89 | cursor: pointer; 90 | margin-left: 20px; 91 | } 92 | .actionbtn:hover, .form .actionbtn:active, .form .actionbtn:focus { 93 | background: #43A047; 94 | } 95 | } 96 | 97 | .main { 98 | padding: 2%; 99 | } 100 | 101 | .tablewrapper { 102 | box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.2), 0 5px 5px 0 rgba(0, 0, 0, 0.24); 103 | } 104 | -------------------------------------------------------------------------------- /web/static/css/login.css: -------------------------------------------------------------------------------- 1 | .login-page { 2 | width: 360px; 3 | padding: 8% 0 0; 4 | margin: auto; 5 | } 6 | 7 | .form { 8 | position: relative; 9 | z-index: 1; 10 | background: #FFFFFF; 11 | max-width: 360px; 12 | margin: 0 auto 100px; 13 | padding: 45px; 14 | text-align: center; 15 | box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.2), 0 5px 5px 0 rgba(0, 0, 0, 0.24); 16 | } 17 | 18 | .form input { 19 | font-family: "Open Sans", Helvetica, Verdana, sans-serif; 20 | outline: 0; 21 | background: #f2f2f2; 22 | width: 100%; 23 | border: 0; 24 | margin: 0 0 15px; 25 | padding: 15px; 26 | box-sizing: border-box; 27 | font-size: 14px; 28 | } 29 | 30 | .form button { 31 | font-family: "Open Sans", Helvetica, Verdana, sans-serif; 32 | text-transform: uppercase; 33 | outline: 0; 34 | background: #4CAF50; 35 | width: 100%; 36 | border: 0; 37 | padding: 15px; 38 | color: #FFFFFF; 39 | font-size: 14px; 40 | -webkit-transition: all 0.3 ease; 41 | transition: all 0.3 ease; 42 | cursor: pointer; 43 | } 44 | 45 | .form button:hover, .form button:active, .form button:focus { 46 | background: #43A047; 47 | } 48 | 49 | .form .message { 50 | margin: 15px 0 0; 51 | color: #b3b3b3; 52 | font-size: 12px; 53 | } 54 | 55 | .form .message a { 56 | color: #4CAF50; 57 | text-decoration: none; 58 | } 59 | 60 | .form .register-form { 61 | display: none; 62 | } 63 | 64 | .container { 65 | position: relative; 66 | z-index: 1; 67 | max-width: 300px; 68 | margin: 0 auto; 69 | } 70 | 71 | .container:before, .container:after { 72 | content: ""; 73 | display: block; 74 | clear: both; 75 | } 76 | 77 | .container .info { 78 | margin: 50px auto; 79 | text-align: center; 80 | } 81 | 82 | .container .info h1 { 83 | margin: 0 0 15px; 84 | padding: 0; 85 | font-size: 36px; 86 | font-weight: 300; 87 | color: #1a1a1a; 88 | } 89 | 90 | .container .info span { 91 | color: #4d4d4d; 92 | font-size: 12px; 93 | } 94 | 95 | .container .info span a { 96 | color: #000000; 97 | text-decoration: none; 98 | } 99 | 100 | .container .info span .fa { 101 | color: #EF3B3A; 102 | } 103 | 104 | html, body { 105 | height: 100%; 106 | background: #ffffff; 107 | /* Old Browsers */ 108 | background: -moz-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 109 | /* FF3.6+ */ 110 | background: -webkit-gradient(left top, left bottom, color-stop(0%, #ffffff), color-stop(47%, #f6f6f6), color-stop(100%, #ededed)); 111 | /* Chrome, Safari4+ */ 112 | background: -webkit-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 113 | /* Chrome10+,Safari5.1+ */ 114 | background: -o-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 115 | /* Opera 11.10+ */ 116 | background: -ms-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 117 | /* IE 10+ */ 118 | background: linear-gradient(to bottom, #ffffff 0%, #f6f6f6 47%, #ededed 100%); 119 | /* W3C */ 120 | filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#ffffff', endColorstr='#ededed', GradientType=0); 121 | /* IE6-9 */ 122 | font-family: "Open Sans", Helvetica, Verdana, sans-serif; 123 | -webkit-font-smoothing: antialiased; 124 | -moz-osx-font-smoothing: grayscale; 125 | } 126 | -------------------------------------------------------------------------------- /web/static/css/tabulator_simple.min.css: -------------------------------------------------------------------------------- 1 | /* Tabulator v4.2.7 (c) Oliver Folkerd */ 2 | .tabulator{position:relative;background-color:#fff;overflow:hidden;font-size:14px;text-align:left;transform:translatez(0)}.tabulator[tabulator-layout=fitDataFill] .tabulator-tableHolder .tabulator-table{min-width:100%}.tabulator.tabulator-block-select{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.tabulator .tabulator-header{width:100%;border-bottom:1px solid #999;color:#555;font-weight:700;white-space:nowrap;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none}.tabulator .tabulator-header,.tabulator .tabulator-header .tabulator-col{position:relative;box-sizing:border-box;background-color:#fff;overflow:hidden}.tabulator .tabulator-header .tabulator-col{display:inline-block;border-right:1px solid #ddd;text-align:left;vertical-align:bottom}.tabulator .tabulator-header .tabulator-col.tabulator-moving{position:absolute;border:1px solid #999;background:#e6e6e6;pointer-events:none}.tabulator .tabulator-header .tabulator-col .tabulator-col-content{box-sizing:border-box;position:relative;padding:4px}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title{box-sizing:border-box;width:100%;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-title-editor{box-sizing:border-box;width:100%;border:1px solid #999;padding:1px;background:#fff}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-arrow{display:inline-block;position:absolute;top:9px;right:8px;width:0;height:0;border-left:6px solid transparent;border-right:6px solid transparent;border-bottom:6px solid #bbb}.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols{position:relative;display:-ms-flexbox;display:flex;border-top:1px solid #ddd;overflow:hidden}.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols .tabulator-col:last-child{margin-right:-1px}.tabulator .tabulator-header .tabulator-col:first-child .tabulator-col-resize-handle.prev{display:none}.tabulator .tabulator-header .tabulator-col.ui-sortable-helper{position:absolute;background-color:#e6e6e6!important;border:1px solid #ddd}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter{position:relative;box-sizing:border-box;margin-top:2px;width:100%;text-align:center}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter textarea{height:auto!important}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter svg{margin-top:3px}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input::-ms-clear{width:0;height:0}.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title{padding-right:25px}.tabulator .tabulator-header .tabulator-col.tabulator-sortable:hover{cursor:pointer;background-color:#e6e6e6}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-arrow{border-top:none;border-bottom:6px solid #bbb}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=asc] .tabulator-col-content .tabulator-arrow{border-top:none;border-bottom:6px solid #666}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=desc] .tabulator-col-content .tabulator-arrow{border-top:6px solid #666;border-bottom:none}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical .tabulator-col-content .tabulator-col-title{-webkit-writing-mode:vertical-rl;-ms-writing-mode:tb-rl;writing-mode:vertical-rl;text-orientation:mixed;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-col-vertical-flip .tabulator-col-title{transform:rotate(180deg)}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-title{padding-right:0;padding-top:20px}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable.tabulator-col-vertical-flip .tabulator-col-title{padding-right:0;padding-bottom:20px}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-arrow{right:calc(50% - 6px)}.tabulator .tabulator-header .tabulator-frozen{display:inline-block;position:absolute;z-index:10}.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-left{border-right:2px solid #ddd}.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-right{border-left:2px solid #ddd}.tabulator .tabulator-header .tabulator-calcs-holder{box-sizing:border-box;min-width:400%;background:#f2f2f2!important;border-top:1px solid #ddd;border-bottom:1px solid #999;overflow:hidden}.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row{background:#f2f2f2!important}.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle{display:none}.tabulator .tabulator-header .tabulator-frozen-rows-holder{min-width:400%}.tabulator .tabulator-header .tabulator-frozen-rows-holder:empty{display:none}.tabulator .tabulator-tableHolder{position:relative;width:100%;white-space:nowrap;overflow:auto;-webkit-overflow-scrolling:touch}.tabulator .tabulator-tableHolder:focus{outline:none}.tabulator .tabulator-tableHolder .tabulator-placeholder{box-sizing:border-box;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;width:100%}.tabulator .tabulator-tableHolder .tabulator-placeholder[tabulator-render-mode=virtual]{position:absolute;top:0;left:0;height:100%}.tabulator .tabulator-tableHolder .tabulator-placeholder span{display:inline-block;margin:0 auto;padding:10px;color:#000;font-weight:700;font-size:20px}.tabulator .tabulator-tableHolder .tabulator-table{position:relative;display:inline-block;background-color:#fff;white-space:nowrap;overflow:visible;color:#333}.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs{font-weight:700;background:#f2f2f2!important}.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-top{border-bottom:2px solid #ddd}.tabulator .tabulator-tableHolder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-bottom{border-top:2px solid #ddd}.tabulator .tabulator-col-resize-handle{position:absolute;right:0;top:0;bottom:0;width:5px}.tabulator .tabulator-col-resize-handle.prev{left:0;right:auto}.tabulator .tabulator-col-resize-handle:hover{cursor:ew-resize}.tabulator .tabulator-footer{padding:5px 10px;border-top:1px solid #999;background-color:#fff;text-align:right;color:#555;font-weight:700;white-space:nowrap;-ms-user-select:none;user-select:none;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none}.tabulator .tabulator-footer .tabulator-calcs-holder{box-sizing:border-box;width:calc(100% + 20px);margin:-5px -10px 5px;text-align:left;background:#f2f2f2!important;border-bottom:1px solid #fff;border-top:1px solid #ddd;overflow:hidden}.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row{background:#f2f2f2!important}.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle{display:none}.tabulator .tabulator-footer .tabulator-calcs-holder:only-child{margin-bottom:-5px;border-bottom:none}.tabulator .tabulator-footer .tabulator-paginator{color:#555;font-family:inherit;font-weight:inherit;font-size:inherit}.tabulator .tabulator-footer .tabulator-page-size{display:inline-block;margin:0 5px;padding:2px 5px;border:1px solid #aaa;border-radius:3px}.tabulator .tabulator-footer .tabulator-pages{margin:0 7px}.tabulator .tabulator-footer .tabulator-page{display:inline-block;margin:0 2px;padding:2px 5px;border:1px solid #aaa;border-radius:3px;background:hsla(0,0%,100%,.2)}.tabulator .tabulator-footer .tabulator-page.active{color:#d00}.tabulator .tabulator-footer .tabulator-page:disabled{opacity:.5}.tabulator .tabulator-footer .tabulator-page:not(.disabled):hover{cursor:pointer;background:rgba(0,0,0,.2);color:#fff}.tabulator .tabulator-loader{position:absolute;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;top:0;left:0;z-index:100;height:100%;width:100%;background:rgba(0,0,0,.4);text-align:center}.tabulator .tabulator-loader .tabulator-loader-msg{display:inline-block;margin:0 auto;padding:10px 20px;border-radius:10px;background:#fff;font-weight:700;font-size:16px}.tabulator .tabulator-loader .tabulator-loader-msg.tabulator-loading{border:4px solid #333;color:#000}.tabulator .tabulator-loader .tabulator-loader-msg.tabulator-error{border:4px solid #d00;color:#590000}.tabulator-row{position:relative;box-sizing:border-box;min-height:22px;border-bottom:1px solid #ddd}.tabulator-row,.tabulator-row:nth-child(2n){background-color:#fff}.tabulator-row.tabulator-selectable:hover{background-color:#bbb;cursor:pointer}.tabulator-row.tabulator-selected{background-color:#9abcea}.tabulator-row.tabulator-selected:hover{background-color:#769bcc;cursor:pointer}.tabulator-row.tabulator-moving{position:absolute;border-top:1px solid #ddd;border-bottom:1px solid #ddd;pointer-events:none!important;z-index:15}.tabulator-row .tabulator-row-resize-handle{position:absolute;right:0;bottom:0;left:0;height:5px}.tabulator-row .tabulator-row-resize-handle.prev{top:0;bottom:auto}.tabulator-row .tabulator-row-resize-handle:hover{cursor:ns-resize}.tabulator-row .tabulator-frozen{display:inline-block;position:absolute;background-color:inherit;z-index:10}.tabulator-row .tabulator-frozen.tabulator-frozen-left{border-right:2px solid #ddd}.tabulator-row .tabulator-frozen.tabulator-frozen-right{border-left:2px solid #ddd}.tabulator-row .tabulator-responsive-collapse{box-sizing:border-box;padding:5px;border-top:1px solid #ddd;border-bottom:1px solid #ddd}.tabulator-row .tabulator-responsive-collapse:empty{display:none}.tabulator-row .tabulator-responsive-collapse table{font-size:14px}.tabulator-row .tabulator-responsive-collapse table tr td{position:relative}.tabulator-row .tabulator-responsive-collapse table tr td:first-of-type{padding-right:10px}.tabulator-row .tabulator-cell{display:inline-block;position:relative;box-sizing:border-box;padding:4px;border-right:1px solid #ddd;vertical-align:middle;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.tabulator-row .tabulator-cell:last-of-type{border-right:none}.tabulator-row .tabulator-cell.tabulator-editing{border:1px solid #1d68cd;padding:0}.tabulator-row .tabulator-cell.tabulator-editing input,.tabulator-row .tabulator-cell.tabulator-editing select{border:1px;background:transparent}.tabulator-row .tabulator-cell.tabulator-validation-fail{border:1px solid #d00}.tabulator-row .tabulator-cell.tabulator-validation-fail input,.tabulator-row .tabulator-cell.tabulator-validation-fail select{border:1px;background:transparent;color:#d00}.tabulator-row .tabulator-cell:first-child .tabulator-col-resize-handle.prev{display:none}.tabulator-row .tabulator-cell.tabulator-row-handle{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none}.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box{width:80%}.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box .tabulator-row-handle-bar{width:100%;height:3px;margin-top:2px;background:#666}.tabulator-row .tabulator-cell .tabulator-data-tree-branch{display:inline-block;vertical-align:middle;height:9px;width:7px;margin-top:-9px;margin-right:5px;border-bottom-left-radius:1px;border-left:2px solid #ddd;border-bottom:2px solid #ddd}.tabulator-row .tabulator-cell .tabulator-data-tree-control{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-pack:center;justify-content:center;-ms-flex-align:center;align-items:center;vertical-align:middle;height:11px;width:11px;margin-right:5px;border:1px solid #333;border-radius:2px;background:rgba(0,0,0,.1);overflow:hidden}.tabulator-row .tabulator-cell .tabulator-data-tree-control:hover{cursor:pointer;background:rgba(0,0,0,.2)}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse{display:inline-block;position:relative;height:7px;width:1px;background:transparent}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after{position:absolute;content:"";left:-3px;top:3px;height:1px;width:7px;background:#333}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand{display:inline-block;position:relative;height:7px;width:1px;background:#333}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after{position:absolute;content:"";left:-3px;top:3px;height:1px;width:7px;background:#333}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none;height:15px;width:15px;border-radius:20px;background:#666;color:#fff;font-weight:700;font-size:1.1em}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle:hover{opacity:.7}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-close{display:initial}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-open,.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle .tabulator-responsive-collapse-toggle-close{display:none}.tabulator-row .tabulator-cell .tabulator-traffic-light{display:inline-block;height:14px;width:14px;border-radius:14px}.tabulator-row.tabulator-group{box-sizing:border-box;border-bottom:1px solid #999;border-right:1px solid #ddd;border-top:1px solid #999;padding:5px;padding-left:10px;background:#fafafa;font-weight:700;min-width:100%}.tabulator-row.tabulator-group:hover{cursor:pointer;background-color:rgba(0,0,0,.1)}.tabulator-row.tabulator-group.tabulator-group-visible .tabulator-arrow{margin-right:10px;border-left:6px solid transparent;border-right:6px solid transparent;border-top:6px solid #666;border-bottom:0}.tabulator-row.tabulator-group.tabulator-group-level-1{padding-left:30px}.tabulator-row.tabulator-group.tabulator-group-level-2{padding-left:50px}.tabulator-row.tabulator-group.tabulator-group-level-3{padding-left:70px}.tabulator-row.tabulator-group.tabulator-group-level-4{padding-left:90px}.tabulator-row.tabulator-group.tabulator-group-level-5{padding-left:110px}.tabulator-row.tabulator-group .tabulator-arrow{display:inline-block;width:0;height:0;margin-right:16px;border-top:6px solid transparent;border-bottom:6px solid transparent;border-right:0;border-left:6px solid #666;vertical-align:middle}.tabulator-row.tabulator-group span{margin-left:10px;color:#666}.tabulator-edit-select-list{position:absolute;display:inline-block;box-sizing:border-box;max-height:200px;background:#fff;border:1px solid #ddd;font-size:14px;overflow-y:auto;-webkit-overflow-scrolling:touch;z-index:10000}.tabulator-edit-select-list .tabulator-edit-select-list-item{padding:4px;color:#333}.tabulator-edit-select-list .tabulator-edit-select-list-item.active{color:#fff;background:#1d68cd}.tabulator-edit-select-list .tabulator-edit-select-list-item:hover{cursor:pointer;color:#fff;background:#1d68cd}.tabulator-edit-select-list .tabulator-edit-select-list-group{border-bottom:1px solid #ddd;padding:4px;padding-top:6px;color:#333;font-weight:700} 3 | /*# sourceMappingURL=tabulator_simple.min.css.map */ 4 | -------------------------------------------------------------------------------- /web/static/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lseffer/stock_screener/5c42cbd9e22522131a1b68054230cd83fda531c8/web/static/img/favicon.png -------------------------------------------------------------------------------- /web/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | Nordic stock screener 14 | 15 | 16 | 17 |
18 |
19 |

Nordic stock screener

20 | 21 | 22 | 23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 | 31 | -------------------------------------------------------------------------------- /web/static/js/update_table.js: -------------------------------------------------------------------------------- 1 | function ajax_get(url, callback) { 2 | var xmlhttp = new XMLHttpRequest(); 3 | xmlhttp.onreadystatechange = function () { 4 | if (xmlhttp.readyState == 4 && xmlhttp.status == 200) { 5 | console.log('responseText:' + xmlhttp.responseText); 6 | try { 7 | var data = JSON.parse(xmlhttp.responseText); 8 | } catch (err) { 9 | console.log(err.message + " in " + xmlhttp.responseText); 10 | return; 11 | } 12 | callback(data); 13 | } 14 | }; 15 | 16 | xmlhttp.open("GET", url, true); 17 | xmlhttp.send(); 18 | } 19 | 20 | function get_screen_resolution() { 21 | var width = window.innerWidth 22 | || document.documentElement.clientWidth 23 | || document.body.clientWidth; 24 | 25 | var height = window.innerHeight 26 | || document.documentElement.clientHeight 27 | || document.body.clientHeight; 28 | 29 | return { "width": width, "height": height } 30 | } 31 | 32 | function frozen_cols_checker() { 33 | dims = get_screen_resolution(); 34 | if (dims["width"] <= 600) { 35 | return false; 36 | } else { 37 | return true; 38 | } 39 | } 40 | 41 | function table_height_large_screen() { 42 | dims = get_screen_resolution(); 43 | if (dims["height"] >= 800) { 44 | return "700px"; 45 | } else { 46 | return "500px"; 47 | } 48 | } 49 | 50 | //define table 51 | function update_table(data) { 52 | 53 | var table = new Tabulator("#example-table", { 54 | columns: [ 55 | { 56 | title: "isin", field: "isin", frozen: frozen_cols_checker(), headerFilter: true, formatter: "link", formatterParams: { 57 | labelField: "isin", 58 | urlPrefix: "https://www.google.com/search?q=", 59 | target: "_blank", 60 | } 61 | }, 62 | { title: "company_name", field: "company_name", frozen: frozen_cols_checker(), headerFilter: true }, 63 | { title: "symbol", field: "symbol", headerFilter: true }, 64 | { title: "currency", field: "currency", headerFilter: true }, 65 | { title: "sector", field: "sector", headerFilter: true }, 66 | { title: "yahoo_ticker", field: "yahoo_ticker", headerFilter: true }, 67 | { title: "report_date", field: "report_date", headerFilter: true }, 68 | { title: "market_date", field: "market_date", headerFilter: true }, 69 | { title: "p_score", field: "p_score", headerFilter: "number", headerFilterFunc: ">=" }, 70 | { title: "roic", field: "roic", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 71 | { title: "ev_ebitda_ratio_inv", field: "ev_ebitda_ratio_inv", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 72 | { title: "shareholder_yield_stock", field: "shareholder_yield_stock", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 73 | { title: "shareholder_yield_dividends", field: "shareholder_yield_dividends", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 74 | { title: "price_to_sales", field: "price_to_sales", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 75 | { title: "price_to_cash_flow", field: "price_to_cash_flow", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 76 | { title: "ncav_ratio", field: "ncav_ratio", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 77 | { title: "price", field: "price", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 78 | { title: "target_median_price", field: "target_median_price", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 79 | { title: "number_of_analyst_opinions", field: "number_of_analyst_opinions", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 80 | { title: "ebitda", field: "ebitda", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 81 | { title: "market_cap", field: "market_cap", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 82 | { title: "trailing_pe", field: "trailing_pe", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 83 | { title: "forward_pe", field: "forward_pe", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 84 | { title: "ev_ebitda_ratio", field: "ev_ebitda_ratio", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" }, 85 | { title: "magic_formula_score", field: "magic_formula_score", headerFilter: "number", headerFilterFunc: ">=", formatter: "money" } 86 | ], 87 | height: table_height_large_screen(), 88 | data: data, 89 | autoColumns: false, 90 | layout: "fitDataFill", 91 | persistenceMode: "cookie" 92 | }); 93 | 94 | var dl_link = document.getElementById("download_data_link"); 95 | 96 | dl_link.onclick = function (e) { 97 | e.preventDefault(); 98 | table.download("csv", "data.csv"); 99 | return false; 100 | } 101 | }; 102 | 103 | function reload_table_onclick(e) { 104 | e.preventDefault(); 105 | reload_table(); 106 | return false; 107 | }; 108 | 109 | function reload_table() { 110 | ajax_get('./stocks', function (data) { 111 | update_table(data) 112 | }); 113 | }; 114 | 115 | function logout_onclick(e) { 116 | e.preventDefault(); 117 | xhr = new XMLHttpRequest(); 118 | xhr.open("POST", "logout", true); 119 | xhr.onload = function () { 120 | location.reload(); 121 | }; 122 | xhr.send(); 123 | }; 124 | -------------------------------------------------------------------------------- /web/static/login.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Login 11 | 12 | 13 | 14 |
15 |
16 | 21 |
22 |
23 | 24 | -------------------------------------------------------------------------------- /worker/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = alembic 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # timezone to use when rendering the date 11 | # within the migration file as well as the filename. 12 | # string value is passed to dateutil.tz.gettz() 13 | # leave blank for localtime 14 | # timezone = 15 | 16 | # max length of characters to apply to the 17 | # "slug" field 18 | # truncate_slug_length = 40 19 | 20 | # set to 'true' to run the environment during 21 | # the 'revision' command, regardless of autogenerate 22 | # revision_environment = false 23 | 24 | # set to 'true' to allow .pyc and .pyo files without 25 | # a source .py file to be detected as revisions in the 26 | # versions/ directory 27 | # sourceless = false 28 | 29 | # version location specification; this defaults 30 | # to alembic/versions. When using multiple version 31 | # directories, initial revisions must be specified with --version-path 32 | # version_locations = %(here)s/bar %(here)s/bat alembic/versions 33 | 34 | # the output encoding used when revision files 35 | # are written from script.py.mako 36 | # output_encoding = utf-8 37 | 38 | sqlalchemy.url = 39 | 40 | 41 | # Logging configuration 42 | [loggers] 43 | keys = root,sqlalchemy,alembic 44 | 45 | [handlers] 46 | keys = console 47 | 48 | [formatters] 49 | keys = generic 50 | 51 | [logger_root] 52 | level = WARN 53 | handlers = console 54 | qualname = 55 | 56 | [logger_sqlalchemy] 57 | level = WARN 58 | handlers = 59 | qualname = sqlalchemy.engine 60 | 61 | [logger_alembic] 62 | level = INFO 63 | handlers = 64 | qualname = alembic 65 | 66 | [handler_console] 67 | class = StreamHandler 68 | args = (sys.stderr,) 69 | level = NOTSET 70 | formatter = generic 71 | 72 | [formatter_generic] 73 | format = %(levelname)-5.5s [%(name)s] %(message)s 74 | datefmt = %H:%M:%S 75 | -------------------------------------------------------------------------------- /worker/alembic/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /worker/alembic/env.py: -------------------------------------------------------------------------------- 1 | from logging.config import fileConfig 2 | 3 | from sqlalchemy import engine_from_config 4 | from sqlalchemy import pool 5 | 6 | from alembic import context 7 | from utils.config import POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB 8 | from utils.models import Base 9 | from utils.alembic_helpers import create_view, drop_view # NOQA 10 | 11 | # this is the Alembic Config object, which provides 12 | # access to the values within the .ini file in use. 13 | config = context.config 14 | config.set_main_option('sqlalchemy.url', 'postgresql://%s:%s@database/%s' % 15 | (POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB)) 16 | 17 | # Interpret the config file for Python logging. 18 | # This line sets up loggers basically. 19 | fileConfig(config.config_file_name) 20 | 21 | # add your model's MetaData object here 22 | # for 'autogenerate' support 23 | # from myapp import mymodel 24 | # target_metadata = mymodel.Base.metadata 25 | target_metadata = Base.metadata 26 | 27 | # other values from the config, defined by the needs of env.py, 28 | # can be acquired: 29 | # my_important_option = config.get_main_option("my_important_option") 30 | # ... etc. 31 | 32 | 33 | def run_migrations_offline(): 34 | """Run migrations in 'offline' mode. 35 | 36 | This configures the context with just a URL 37 | and not an Engine, though an Engine is acceptable 38 | here as well. By skipping the Engine creation 39 | we don't even need a DBAPI to be available. 40 | 41 | Calls to context.execute() here emit the given string to the 42 | script output. 43 | 44 | """ 45 | url = config.get_main_option("sqlalchemy.url") 46 | context.configure( 47 | url=url, target_metadata=target_metadata, literal_binds=True 48 | ) 49 | 50 | with context.begin_transaction(): 51 | context.run_migrations() 52 | 53 | 54 | def run_migrations_online(): 55 | """Run migrations in 'online' mode. 56 | 57 | In this scenario we need to create an Engine 58 | and associate a connection with the context. 59 | 60 | """ 61 | connectable = engine_from_config( 62 | config.get_section(config.config_ini_section), 63 | prefix="sqlalchemy.", 64 | poolclass=pool.NullPool, 65 | ) 66 | 67 | with connectable.connect() as connection: 68 | context.configure( 69 | connection=connection, target_metadata=target_metadata 70 | ) 71 | 72 | with context.begin_transaction(): 73 | context.run_migrations() 74 | 75 | 76 | if context.is_offline_mode(): 77 | run_migrations_offline() 78 | else: 79 | run_migrations_online() 80 | -------------------------------------------------------------------------------- /worker/alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /worker/alembic/versions/00b80735370b_stock_financial_tables.py: -------------------------------------------------------------------------------- 1 | """stock financial tables 2 | 3 | Revision ID: 00b80735370b 4 | Revises: 7bfdef0682d4 5 | Create Date: 2019-04-28 17:20:58.775476 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '00b80735370b' 14 | down_revision = '7bfdef0682d4' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('balance_sheet_statements', 22 | sa.Column('isin', sa.String(), nullable=False), 23 | sa.Column('report_date', sa.Date(), nullable=False), 24 | sa.Column('cash', sa.Float(), nullable=True), 25 | sa.Column('short_term_investments', sa.Float(), nullable=True), 26 | sa.Column('net_receivables', sa.Float(), nullable=True), 27 | sa.Column('total_current_assets', sa.Float(), nullable=True), 28 | sa.Column('property_plant_equipment', sa.Float(), nullable=True), 29 | sa.Column('intangible_assets', sa.Float(), nullable=True), 30 | sa.Column('other_assets', sa.Float(), nullable=True), 31 | sa.Column('deferred_long_term_asset_charges', sa.Float(), nullable=True), 32 | sa.Column('total_assets', sa.Float(), nullable=True), 33 | sa.Column('accounts_payable', sa.Float(), nullable=True), 34 | sa.Column('short_long_term_debt', sa.Float(), nullable=True), 35 | sa.Column('other_current_liab', sa.Float(), nullable=True), 36 | sa.Column('long_term_debt', sa.Float(), nullable=True), 37 | sa.Column('other_liab', sa.Float(), nullable=True), 38 | sa.Column('deferred_long_term_liab', sa.Float(), nullable=True), 39 | sa.Column('total_current_liabilities', sa.Float(), nullable=True), 40 | sa.Column('total_liab', sa.Float(), nullable=True), 41 | sa.Column('common_stock', sa.Float(), nullable=True), 42 | sa.Column('retained_earnings', sa.Float(), nullable=True), 43 | sa.Column('treasury_stock', sa.Float(), nullable=True), 44 | sa.Column('other_stockholder_equity', sa.Float(), nullable=True), 45 | sa.Column('total_stockholder_equity', sa.Float(), nullable=True), 46 | sa.Column('net_tangible_assets', sa.Float(), nullable=True), 47 | sa.Column('dw_created', sa.DateTime(), nullable=True), 48 | sa.Column('dw_modified', sa.DateTime(), nullable=True), 49 | sa.PrimaryKeyConstraint('isin', 'report_date') 50 | ) 51 | op.create_table('cash_flow_statements', 52 | sa.Column('isin', sa.String(), nullable=False), 53 | sa.Column('report_date', sa.Date(), nullable=False), 54 | sa.Column('net_income', sa.Float(), nullable=True), 55 | sa.Column('change_to_netincome', sa.Float(), nullable=True), 56 | sa.Column('change_to_account_receivables', sa.Float(), nullable=True), 57 | sa.Column('change_to_liabilities', sa.Float(), nullable=True), 58 | sa.Column('total_cash_from_operating_activities', sa.Float(), nullable=True), 59 | sa.Column('capital_expenditures', sa.Float(), nullable=True), 60 | sa.Column('other_cashflows_from_investing_activities', sa.Float(), nullable=True), 61 | sa.Column('total_cashflows_from_investing_activities', sa.Float(), nullable=True), 62 | sa.Column('dividends_paid', sa.Float(), nullable=True), 63 | sa.Column('net_borrowings', sa.Float(), nullable=True), 64 | sa.Column('other_cashflows_from_financing_activities', sa.Float(), nullable=True), 65 | sa.Column('total_cash_from_financing_activities', sa.Float(), nullable=True), 66 | sa.Column('effect_of_exchange_rate', sa.Float(), nullable=True), 67 | sa.Column('change_in_cash', sa.Float(), nullable=True), 68 | sa.Column('repurchase_of_stock', sa.Float(), nullable=True), 69 | sa.Column('issuance_of_stock', sa.Float(), nullable=True), 70 | sa.Column('dw_created', sa.DateTime(), nullable=True), 71 | sa.Column('dw_modified', sa.DateTime(), nullable=True), 72 | sa.PrimaryKeyConstraint('isin', 'report_date') 73 | ) 74 | op.create_table('income_statements', 75 | sa.Column('isin', sa.String(), nullable=False), 76 | sa.Column('report_date', sa.Date(), nullable=False), 77 | sa.Column('total_revenue', sa.Float(), nullable=True), 78 | sa.Column('cost_of_revenue', sa.Float(), nullable=True), 79 | sa.Column('gross_profit', sa.Float(), nullable=True), 80 | sa.Column('research_development', sa.Float(), nullable=True), 81 | sa.Column('selling_general_administrative', sa.Float(), nullable=True), 82 | sa.Column('non_recurring', sa.Float(), nullable=True), 83 | sa.Column('other_operating_expenses', sa.Float(), nullable=True), 84 | sa.Column('total_operating_expenses', sa.Float(), nullable=True), 85 | sa.Column('operating_income', sa.Float(), nullable=True), 86 | sa.Column('total_other_income_expense_net', sa.Float(), nullable=True), 87 | sa.Column('ebit', sa.Float(), nullable=True), 88 | sa.Column('interest_expense', sa.Float(), nullable=True), 89 | sa.Column('income_before_tax', sa.Float(), nullable=True), 90 | sa.Column('income_tax_expense', sa.Float(), nullable=True), 91 | sa.Column('minority_interest', sa.Float(), nullable=True), 92 | sa.Column('net_income_from_continuing_ops', sa.Float(), nullable=True), 93 | sa.Column('discontinued_operations', sa.Float(), nullable=True), 94 | sa.Column('extraordinary_items', sa.Float(), nullable=True), 95 | sa.Column('effect_of_accounting_charges', sa.Float(), nullable=True), 96 | sa.Column('other_items', sa.Float(), nullable=True), 97 | sa.Column('net_income', sa.Float(), nullable=True), 98 | sa.Column('net_income_applicable_to_common_shares', sa.Float(), nullable=True), 99 | sa.Column('dw_created', sa.DateTime(), nullable=True), 100 | sa.Column('dw_modified', sa.DateTime(), nullable=True), 101 | sa.PrimaryKeyConstraint('isin', 'report_date') 102 | ) 103 | # ### end Alembic commands ### 104 | 105 | 106 | def downgrade(): 107 | # ### commands auto generated by Alembic - please adjust! ### 108 | op.drop_table('income_statements') 109 | op.drop_table('cash_flow_statements') 110 | op.drop_table('balance_sheet_statements') 111 | # ### end Alembic commands ### 112 | -------------------------------------------------------------------------------- /worker/alembic/versions/0e87ac62cba4_magic_formula_view.py: -------------------------------------------------------------------------------- 1 | """magic formula view 2 | 3 | Revision ID: 0e87ac62cba4 4 | Revises: f5cebdfa9f70 5 | Create Date: 2019-06-11 17:35:08.715102 6 | 7 | """ 8 | from alembic import op 9 | from utils.models import MagicFormulaScore 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0e87ac62cba4' 14 | down_revision = 'f5cebdfa9f70' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_view(MagicFormulaScore) 21 | 22 | 23 | def downgrade(): 24 | op.drop_view(MagicFormulaScore) 25 | -------------------------------------------------------------------------------- /worker/alembic/versions/4f406bad1840_screener_results_view.py: -------------------------------------------------------------------------------- 1 | """screener results view 2 | 3 | Revision ID: 4f406bad1840 4 | Revises: 0e87ac62cba4 5 | Create Date: 2019-06-11 20:03:12.716812 6 | 7 | """ 8 | from alembic import op 9 | from utils.models import ScreenResults 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '4f406bad1840' 14 | down_revision = '0e87ac62cba4' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_view(ScreenResults) 21 | 22 | 23 | def downgrade(): 24 | op.drop_view(ScreenResults) 25 | -------------------------------------------------------------------------------- /worker/alembic/versions/7848c183a286_re_create_screener_results_view.py: -------------------------------------------------------------------------------- 1 | """re-create screener results view 2 | 3 | Revision ID: 7848c183a286 4 | Revises: 4f406bad1840 5 | Create Date: 2019-12-08 10:46:02.145678 6 | 7 | """ 8 | from alembic import op 9 | from utils.models import ScreenResults 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '7848c183a286' 14 | down_revision = '4f406bad1840' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_view(ScreenResults) 21 | 22 | 23 | def downgrade(): 24 | op.drop_view(ScreenResults) 25 | -------------------------------------------------------------------------------- /worker/alembic/versions/7bfdef0682d4_new_stock_prices_table.py: -------------------------------------------------------------------------------- 1 | """New stock prices table 2 | 3 | Revision ID: 7bfdef0682d4 4 | Revises: aeee2aa4ad8f 5 | Create Date: 2019-04-24 05:06:43.720260 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '7bfdef0682d4' 14 | down_revision = 'aeee2aa4ad8f' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('prices', 22 | sa.Column('isin', sa.String(), nullable=False), 23 | sa.Column('market_date', sa.Date(), nullable=False), 24 | sa.Column('price', sa.Float(), nullable=True), 25 | sa.Column('target_median_price', sa.Float(), nullable=True), 26 | sa.Column('recommendation', sa.Float(), nullable=True), 27 | sa.Column('number_of_analyst_opinions', sa.Float(), nullable=True), 28 | sa.Column('ebitda', sa.Float(), nullable=True), 29 | sa.Column('market_cap', sa.Float(), nullable=True), 30 | sa.Column('trailing_pe', sa.Float(), nullable=True), 31 | sa.Column('forward_pe', sa.Float(), nullable=True), 32 | sa.Column('ev_ebitda_ratio', sa.Float(), nullable=True), 33 | sa.Column('dw_created', sa.DateTime(), nullable=True), 34 | sa.Column('dw_modified', sa.DateTime(), nullable=True), 35 | sa.PrimaryKeyConstraint('isin', 'market_date') 36 | ) 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade(): 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | op.drop_table('prices') 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /worker/alembic/versions/aeee2aa4ad8f_stocks_table.py: -------------------------------------------------------------------------------- 1 | """stocks table 2 | 3 | Revision ID: aeee2aa4ad8f 4 | Revises: 5 | Create Date: 2019-04-21 16:27:53.284552 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from datetime import datetime 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'aeee2aa4ad8f' 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('stocks', 22 | sa.Column('isin', sa.String(), nullable=False), 23 | sa.Column('name', sa.String(), nullable=True), 24 | sa.Column('symbol', sa.String(), nullable=True), 25 | sa.Column('currency', sa.String(), nullable=True), 26 | sa.Column('sector', sa.String(), nullable=True), 27 | sa.Column('yahoo_ticker', sa.String(), nullable=True), 28 | sa.Column('dw_created', sa.DateTime(), nullable=True, default=datetime.utcnow), 29 | sa.Column('dw_modified', sa.DateTime(), nullable=True, default=datetime.utcnow), 30 | sa.PrimaryKeyConstraint('isin') 31 | ) 32 | # ### end Alembic commands ### 33 | 34 | 35 | def downgrade(): 36 | # ### commands auto generated by Alembic - please adjust! ### 37 | op.drop_table('stocks') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /worker/alembic/versions/f5cebdfa9f70_piotroski_view.py: -------------------------------------------------------------------------------- 1 | """piotroski view 2 | 3 | Revision ID: f5cebdfa9f70 4 | Revises: 00b80735370b 5 | Create Date: 2019-05-05 16:15:14.365580 6 | 7 | """ 8 | from alembic import op 9 | from utils.models import PiotroskiScore 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'f5cebdfa9f70' 13 | down_revision = '00b80735370b' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade(): 19 | op.create_view(PiotroskiScore) 20 | 21 | 22 | def downgrade(): 23 | op.drop_view(PiotroskiScore) 24 | -------------------------------------------------------------------------------- /worker/worker.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | from traceback import format_exc 4 | import datetime 5 | from schedule import Scheduler 6 | import logging 7 | from typing import Callable 8 | from utils.config import logger 9 | from utils.stock_info_etl import StockInfoETL 10 | from utils.stock_valuation_etl import StockValuationETL 11 | from utils.stock_financial_statements_etl import StockFinancialStatementsETL 12 | 13 | 14 | class SafeScheduler(Scheduler): 15 | """ 16 | An implementation of Scheduler that catches jobs that fail, logs their 17 | exception tracebacks as errors, optionally reschedules the jobs for their 18 | next run time, and keeps going. 19 | Use this to run jobs that may or may not crash without worrying about 20 | whether other jobs will run or if they'll crash the entire script. 21 | """ 22 | 23 | def __init__(self, reschedule_on_failure: bool = True, logger: logging.Logger = None) -> None: 24 | """ 25 | If reschedule_on_failure is True, jobs will be rescheduled for their 26 | next run as if they had completed successfully. If False, they'll run 27 | on the next run_pending() tick. 28 | """ 29 | self.logger = logger 30 | self.reschedule_on_failure = reschedule_on_failure 31 | super().__init__() 32 | 33 | def _run_job(self, job: Callable) -> None: 34 | try: 35 | super()._run_job(job) 36 | except Exception: 37 | if self.logger: 38 | self.logger.error(format_exc()) 39 | job.last_run = datetime.datetime.now() 40 | job._schedule_next_run() 41 | 42 | 43 | def run_threaded(job_func: Callable) -> None: 44 | job_thread = threading.Thread(target=job_func) 45 | job_thread.start() 46 | 47 | 48 | if __name__ == '__main__': 49 | scheduler = SafeScheduler(logger=logger) 50 | scheduler.every().saturday.at("00:00").do(run_threaded, StockInfoETL.job) 51 | scheduler.every().saturday.at("03:00").do(run_threaded, StockValuationETL.job) 52 | scheduler.every().sunday.at("03:00").do(run_threaded, StockFinancialStatementsETL.job) 53 | while True: 54 | logger.debug('Heartbeat 5 seconds') 55 | scheduler.run_pending() 56 | time.sleep(5) 57 | -------------------------------------------------------------------------------- /worker/worker_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | TRY_LOOP="20" 6 | 7 | wait_for_port() { 8 | local name="$1" host="$2" port="$3" 9 | local j=0 10 | while ! nc -z "$host" "$port" >/dev/null 2>&1 < /dev/null; do 11 | j=$((j+1)) 12 | if [ $j -ge $TRY_LOOP ]; then 13 | echo >&2 "$(date) - $host:$port still not reachable, giving up" 14 | exit 1 15 | fi 16 | echo "$(date) - waiting for $name... $j/$TRY_LOOP" 17 | sleep 5 18 | done 19 | } 20 | 21 | wait_for_port "Postgres" ${POSTGRES_HOST} "5432" 22 | sleep 10 23 | 24 | alembic upgrade head 25 | python worker.py 26 | --------------------------------------------------------------------------------