├── .gitignore ├── Dockerfile ├── README.md ├── config.example.json ├── data └── .gitkeep ├── docker-compose.yml ├── docker-compose.yml.example-second-dashboard ├── images ├── dca.png └── futures.png ├── metabase.db └── metabase.db.mv.db ├── metabase_rpi └── Dockerfile └── scraper_root ├── __init__.py ├── requirements.txt ├── scraper.py └── scraper ├── __init__.py ├── binancefutures.py ├── binancespot.py ├── bitgetfutures.py ├── bybitderivatives.py ├── data_classes.py ├── kucoinfutures.py ├── persistence ├── __init__.py ├── lockable_session.py ├── orm_classes.py └── repository.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | /data 132 | *.json 133 | 134 | *.db.trace.db 135 | 136 | .idea 137 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM python:3.10-alpine 3 | RUN apk add --no-cache gcc musl-dev linux-headers libffi-dev g++ 4 | RUN pip install --upgrade pip 5 | COPY scraper_root /scraper/scraper_root 6 | RUN pip install -r /scraper/scraper_root/requirements.txt 7 | COPY config*.json /scraper/ 8 | WORKDIR /scraper 9 | ENV PYTHONPATH "${PYTHONPATH}:/scraper" 10 | CMD ["python3", "scraper_root/scraper.py"] 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Exchanges dashboard 2 | 3 | This repository is meant to provide an easy-to-run (local) web-UI that provides insight into your account(s) activities on exchanges. It uses a custom script to retrieve data from one or more exchanges (via websocket and/or REST), which is then inserted into a database (sqlite by default). A metabase container is then launched with a default configuration to display this information. 4 | 5 | # Screenshots 6 | 7 | Overview of the dashboard: 8 | ![Futures](images/futures.png?raw=true "Futures") 9 | 10 | When a value is entered in the DCA field (top left), the DCA quantity and the DCA price (x% above the current price) will be displayed in the additional fields in the table. 11 | ![DCA](images/dca.png?raw=true "DCA") 12 | 13 | # Metabase configuration: 14 | 15 | ## Credentials 16 | * First name: First 17 | * Last name: Last 18 | * Email: exchanges@dashboard.com 19 | * Password: ExchangesDashboard1! 20 | * Department: Department of Awesome 21 | 22 | ## Database 23 | * Database type: SQLite 24 | * Name: Exchanges 25 | * Filename: /data/exchanges_db.sqlite 26 | 27 | # Starting the dashboard 28 | * Clone this repo: 29 | * `git clone https://github.com/hoeckxer/exchanges_dashboard.git` 30 | * Copy the config.example.json to config.json 31 | * `cp config.example.json config.json` 32 | * Enter the api-key & secret 33 | * `nano config.json` 34 | * Start the dashboard 35 | * Run `docker-compose up -d` 36 | * Go to http://localhost:3000 37 | 38 | | WARNING: If you change the config.json file, make sure you rebuild the container using `docker-compose up -d --build` | 39 | | --- | 40 | 41 | | INFO: If you get a message about `$PWD` being unknown, replace `$PWD` the docker-compose.yaml file with the full path | 42 | | --- | 43 | 44 | # Stopping the dashboard 45 | * Run `docker-compose stop` 46 | 47 | # Multiple dashboard 48 | * If you want to deploy a second dashboard instance on the same machine, please clone the repo and update the `config.json` and `docker-compose.yaml` files. 49 | * In the `docker-compose.yaml` you'll have to modify container names, exposed port and network interface. An example is provided in `docker-compose.yaml.example-second-dashboard` 50 | 51 | # Running on Raspberry Pi 52 | * The metabase image is built for x86. 53 | Therefore, on Raspberry Pi, a new image needs to be created using an ARM based image. 54 | * I created a `Dockerfile` that creates this kind of image, which supposed to be identical to the `metabase` one. 55 | * To use the new image, replace line 13 of `docker-compose.yml` from `image: metabase/metabase:latest` to `build: metabase_rpi` -------------------------------------------------------------------------------- /config.example.json: -------------------------------------------------------------------------------- 1 | // Copy this file to config.json in the root folder 2 | // supported: binance_futures, bybit_derivatives (NOTE: USDT_Perpetual only) and bitget_future (NOTE: USDT-M only) 3 | 4 | { 5 | "accounts": [ 6 | { 7 | "alias": "demo_binance", 8 | "exchange": "binance_futures", 9 | "api_key": "", 10 | "api_secret": "" 11 | }, 12 | { 13 | "alias": "demo_bybit", 14 | "exchange": "bybit_derivatives", 15 | "unified": false, // false by default, set this to true if you've got a unified bybit account 16 | "api_key": "", 17 | "api_secret": "" 18 | }, 19 | { 20 | "alias": "demo_bitget", 21 | "exchange": "bitget_futures", 22 | "api_key": "", 23 | "api_secret": "", 24 | "api_passphrase": "" 25 | }, 26 | { 27 | "alias": "demo_kucoin", 28 | "exchange": "kucoin_futures", 29 | "api_key": "", 30 | "api_secret": "", 31 | "api_passphrase": "" 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/data/.gitkeep -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | services: 3 | scraper: 4 | build: . 5 | environment: 6 | - CONFIG_FILE=/scraper/config.json 7 | - DATABASE_PATH=sqlite:////data/exchanges_db.sqlite 8 | volumes: 9 | - $PWD/data:/data 10 | - /etc/timezone:/etc/timezone:ro 11 | - /etc/localtime:/etc/localtime:ro 12 | metabase-app: 13 | image: metabase/metabase:latest 14 | # image: metabase/metabase:v0.43.1 15 | container_name: metabase 16 | hostname: metabase 17 | volumes: 18 | - /dev/urandom:/dev/random:ro 19 | - $PWD:/metabase-data 20 | - $PWD/data:/data 21 | ports: 22 | - 3000:3000 23 | environment: 24 | MB_DB_FILE: /metabase-data/metabase.db 25 | networks: 26 | - metanet1 27 | depends_on: 28 | - scraper 29 | networks: 30 | metanet1: 31 | driver: bridge -------------------------------------------------------------------------------- /docker-compose.yml.example-second-dashboard: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | services: 3 | scraper2: 4 | build: . 5 | environment: 6 | - CONFIG_FILE=/scraper/config.json 7 | - DATABASE_PATH=sqlite:////data/exchanges_db.sqlite 8 | volumes: 9 | - $PWD/data:/data 10 | - /etc/timezone:/etc/timezone:ro 11 | - /etc/localtime:/etc/localtime:ro 12 | metabase-app2: 13 | image: metabase/metabase:v0.43.1 14 | container_name: metabase2 15 | hostname: metabase2 16 | volumes: 17 | - /dev/urandom:/dev/random:ro 18 | - $PWD:/metabase-data 19 | - $PWD/data:/data 20 | ports: 21 | - 3002:3000 22 | environment: 23 | MB_DB_FILE: /metabase-data/metabase.db 24 | networks: 25 | - metanet2 26 | depends_on: 27 | - scraper2 28 | networks: 29 | metanet2: 30 | driver: bridge 31 | -------------------------------------------------------------------------------- /images/dca.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/images/dca.png -------------------------------------------------------------------------------- /images/futures.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/images/futures.png -------------------------------------------------------------------------------- /metabase.db/metabase.db.mv.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/metabase.db/metabase.db.mv.db -------------------------------------------------------------------------------- /metabase_rpi/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | ENV FC_LANG en-US LC_CTYPE en_US.UTF-8 4 | 5 | # dependencies 6 | RUN apt-get update -yq && apt-get install -yq bash fonts-dejavu-core fonts-dejavu-extra fontconfig curl openjdk-11-jre-headless && \ 7 | apt-get clean && \ 8 | rm -rf /var/lib/{apt,dpkg,cache,log}/ && \ 9 | mkdir -p /app/certs && \ 10 | curl https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem -o /app/certs/rds-combined-ca-bundle.pem && \ 11 | keytool -noprompt -import -trustcacerts -alias aws-rds -file /app/certs/rds-combined-ca-bundle.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \ 12 | curl https://cacerts.digicert.com/DigiCertGlobalRootG2.crt.pem -o /app/certs/DigiCertGlobalRootG2.crt.pem && \ 13 | keytool -noprompt -import -trustcacerts -alias azure-cert -file /app/certs/DigiCertGlobalRootG2.crt.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \ 14 | mkdir -p /plugins && chmod a+rwx /plugins && \ 15 | useradd --shell /bin/bash metabase 16 | 17 | 18 | WORKDIR /app 19 | 20 | # copy app from the official image 21 | COPY --from=metabase/metabase:latest /app /app 22 | 23 | RUN chown -R metabase /app 24 | 25 | USER metabase 26 | # expose our default runtime port 27 | EXPOSE 3000 28 | 29 | # run it 30 | ENTRYPOINT ["/app/run_metabase.sh"] 31 | -------------------------------------------------------------------------------- /scraper_root/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/scraper_root/__init__.py -------------------------------------------------------------------------------- /scraper_root/requirements.txt: -------------------------------------------------------------------------------- 1 | unicorn-binance-websocket-api==1.41.0 2 | unicorn-binance-rest-api==1.5.0 3 | unicorn-fy==0.12.2 4 | SQLAlchemy==1.4.44 5 | hjson==3.1.0 6 | pybit==5.6.2 7 | python-bitget==1.0.7 8 | kucoin-futures-python==1.0.9 -------------------------------------------------------------------------------- /scraper_root/scraper.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time 4 | from types import SimpleNamespace 5 | from typing import List 6 | 7 | import hjson 8 | from scraper.binancefutures import BinanceFutures 9 | from scraper.bybitderivatives import BybitDerivatives 10 | from scraper.bitgetfutures import BitgetFutures 11 | from scraper_root.scraper.binancespot import BinanceSpot 12 | from scraper_root.scraper.data_classes import ScraperConfig, Account 13 | from scraper_root.scraper.kucoinfutures import KucoinFutures 14 | from scraper_root.scraper.persistence.repository import Repository 15 | 16 | logging.basicConfig( 17 | format='%(asctime)s %(levelname)-8s %(message)s', 18 | level=logging.INFO, 19 | datefmt='%Y-%m-%d %H:%M:%S') 20 | 21 | logger = logging.getLogger() 22 | 23 | if __name__ == '__main__': 24 | config_file_path = os.environ.get('CONFIG_FILE', 'config.json') 25 | logger.info(f"Using config file {config_file_path}") 26 | with open(config_file_path) as config_file: 27 | user_config = hjson.load(config_file, object_hook=lambda d: SimpleNamespace(**d)) 28 | 29 | scraper_config = ScraperConfig() 30 | for key in user_config: 31 | if hasattr(scraper_config, key): 32 | setattr(scraper_config, key, user_config[key]) 33 | parsed_accounts = [] 34 | for account in scraper_config.accounts: 35 | parsed_accounts.append(Account(**account)) 36 | scraper_config.accounts = parsed_accounts 37 | 38 | if 'BTCUSDT' not in scraper_config.symbols: 39 | scraper_config.symbols.append('BTCUSDT') 40 | 41 | scrapers: List = [] 42 | repository = Repository(accounts=[account.alias for account in scraper_config.accounts]) 43 | scraper = None 44 | for account in scraper_config.accounts: 45 | if account.exchange == 'binance_futures': 46 | scraper = BinanceFutures(account=account, symbols=scraper_config.symbols, repository=repository) 47 | elif account.exchange == 'binance_spot': 48 | scraper = BinanceSpot(account=account, symbols=scraper_config.symbols, repository=repository) 49 | elif account.exchange == 'bybit_derivatives': 50 | scraper = BybitDerivatives(account=account, symbols=scraper_config.symbols, repository=repository, unified_account=account.unified) 51 | elif account.exchange == 'bitget_futures': 52 | scraper = BitgetFutures(account=account, symbols=scraper_config.symbols, repository=repository) 53 | elif account.exchange == 'kucoin_futures': 54 | scraper = KucoinFutures(account=account, symbols=scraper_config.symbols, repository=repository) 55 | else: 56 | raise Exception(f'Encountered unsupported exchange {account.exchange}') 57 | 58 | try: 59 | scraper.start() 60 | logger.info(f'Started scraping account {account.alias} ({account.exchange})') 61 | except Exception as e: 62 | logger.error(f"Failed to start exchange: {e}") 63 | 64 | while True: 65 | try: 66 | time.sleep(10) 67 | except: 68 | break 69 | 70 | logger.info('Scraper shut down') 71 | -------------------------------------------------------------------------------- /scraper_root/scraper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/scraper_root/scraper/__init__.py -------------------------------------------------------------------------------- /scraper_root/scraper/binancefutures.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import os 4 | import threading 5 | import time 6 | from random import randint 7 | from typing import List 8 | 9 | from unicorn_binance_rest_api import BinanceRestApiManager 10 | from unicorn_binance_websocket_api import BinanceWebSocketApiManager 11 | 12 | from scraper_root.scraper.data_classes import AssetBalance, Position, ScraperConfig, Tick, Balance, \ 13 | Income, Order, Account 14 | from scraper_root.scraper.persistence.repository import Repository 15 | 16 | logger = logging.getLogger() 17 | 18 | 19 | def is_asset_usd_or_derivative(asset: str): 20 | return asset.lower() in ["usdt", "busd", "usd", "usdc", "bnfcr"] 21 | 22 | 23 | class BinanceFutures: 24 | def __init__(self, account: Account, symbols: List[str], repository: Repository, 25 | exchange: str = "binance.com-futures"): 26 | print('Binance initialized') 27 | self.account = account 28 | self.symbols = symbols 29 | self.api_key = self.account.api_key 30 | self.secret = self.account.api_secret 31 | self.repository = repository 32 | self.ws_manager = BinanceWebSocketApiManager(exchange=exchange, throw_exception_if_unrepairable=True, 33 | warn_on_update=False) 34 | self.rest_manager = BinanceRestApiManager(self.api_key, api_secret=self.secret) 35 | 36 | self.rest_manager.FUTURES_URL = f"{os.getenv('FUTURES_URL', 'https://fapi.binance.com')}/fapi" 37 | self.rest_manager.FUTURES_DATA_URL = f"{os.getenv('FUTURES_DATA_URL', 'https://fapi.binance.com')}/futures/data" 38 | self.rest_manager.FUTURES_COIN_URL = f"{os.getenv('FUTURES_DATA_URL', 'https://fapi.binance.com')}/fapi" 39 | self.rest_manager.FUTURES_COIN_DATA_URL = f"{os.getenv('FUTURES_DATA_URL', 'https://dapi.binance.com')}/futures/data" 40 | 41 | self.tick_symbols = [] 42 | 43 | def start(self): 44 | logger.info('Starting binance futures scraper') 45 | 46 | for symbol in self.symbols: 47 | symbol_trade_thread = threading.Thread( 48 | name=f'trade_thread_{symbol}', target=self.process_trades, args=(symbol,), daemon=True) 49 | symbol_trade_thread.start() 50 | 51 | sync_balance_thread = threading.Thread( 52 | name=f'sync_balance_thread', target=self.sync_account, daemon=True) 53 | sync_balance_thread.start() 54 | 55 | sync_trades_thread = threading.Thread( 56 | name=f'sync_trades_thread', target=self.sync_trades, daemon=True) 57 | sync_trades_thread.start() 58 | 59 | sync_orders_thread = threading.Thread( 60 | name=f'sync_orders_thread', target=self.sync_open_orders, daemon=True) 61 | sync_orders_thread.start() 62 | 63 | def sync_trades(self): 64 | max_fetches_in_cycle = 3 65 | first_trade_reached = False 66 | while True: 67 | try: 68 | counter = 0 69 | while first_trade_reached is False and counter < max_fetches_in_cycle: 70 | counter += 1 71 | oldest_income = self.repository.get_oldest_income(account=self.account.alias) 72 | if oldest_income is None: 73 | # API will return inclusive, don't want to return the oldest record again 74 | oldest_timestamp = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000) 75 | else: 76 | oldest_timestamp = oldest_income.timestamp 77 | logger.warning(f'Synced trades before {oldest_timestamp}') 78 | 79 | exchange_incomes = self.rest_manager.futures_income_history( 80 | **{'limit': 1000, 'endTime': oldest_timestamp - 1}) 81 | logger.info(f"Length of older trades fetched up to {oldest_timestamp}: {len(exchange_incomes)}") 82 | incomes = [] 83 | for exchange_income in exchange_incomes: 84 | if not is_asset_usd_or_derivative(exchange_income['asset']): 85 | exchange_income['income'] = self.income_to_usdt( 86 | float(exchange_income['income']), 87 | int(exchange_income['time']), 88 | exchange_income['asset']) 89 | exchange_income['asset'] = "USDT" 90 | 91 | income = Income(symbol=exchange_income['symbol'], 92 | asset=exchange_income['asset'], 93 | type=exchange_income['incomeType'], 94 | income=float(exchange_income['income']), 95 | timestamp=exchange_income['time'], 96 | transaction_id=exchange_income['tranId']) 97 | incomes.append(income) 98 | self.repository.process_incomes(incomes, account=self.account.alias) 99 | if len(exchange_incomes) < 1: 100 | first_trade_reached = True 101 | 102 | # WARNING: don't use forward-walking only, because binance only returns max 7 days when using forward-walking 103 | # If this logic is ever changed, make sure that it's still able to retrieve all the account history 104 | newest_trade_reached = False 105 | while newest_trade_reached is False and counter < max_fetches_in_cycle: 106 | counter += 1 107 | newest_income = self.repository.get_newest_income(account=self.account.alias) 108 | if newest_income is None: 109 | # Binance started in September 2017, so no trade can be before that 110 | newest_timestamp = int( 111 | datetime.datetime.fromisoformat('2017-09-01 00:00:00+00:00').timestamp() * 1000) 112 | else: 113 | newest_timestamp = newest_income.timestamp 114 | logger.warning(f'Synced newer trades since {newest_timestamp}') 115 | 116 | exchange_incomes = self.rest_manager.futures_income_history( 117 | **{'limit': 1000, 'startTime': newest_timestamp + 1}) 118 | logger.info(f"Length of newer trades fetched from {newest_timestamp}: {len(exchange_incomes)}") 119 | incomes = [] 120 | for exchange_income in exchange_incomes: 121 | if not is_asset_usd_or_derivative(exchange_income['asset']): 122 | exchange_income['income'] = self.income_to_usdt( 123 | float(exchange_income['income']), 124 | int(exchange_income['time']), 125 | exchange_income['asset']) 126 | exchange_income['asset'] = "USDT" 127 | 128 | income = Income(symbol=exchange_income['symbol'], 129 | asset=exchange_income['asset'], 130 | type=exchange_income['incomeType'], 131 | income=float(exchange_income['income']), 132 | timestamp=exchange_income['time'], 133 | transaction_id=exchange_income['tranId']) 134 | incomes.append(income) 135 | self.repository.process_incomes(incomes, account=self.account.alias) 136 | if len(exchange_incomes) < 1: 137 | newest_trade_reached = True 138 | 139 | logger.warning('Synced trades') 140 | except Exception as e: 141 | logger.error(f'{self.account.alias} Failed to process trades: {e}') 142 | 143 | time.sleep(60) 144 | 145 | def income_to_usdt(self, income: float, income_timestamp: int, asset: str) -> float: 146 | if is_asset_usd_or_derivative(asset): 147 | return income 148 | 149 | # Can't get the latest aggr_trades on just the endTime, so this is 'best effort' 150 | symbol = f"{asset}USDT" 151 | candles = self.rest_manager.futures_klines(symbol=symbol, 152 | interval='1m', 153 | startTime=int(income_timestamp) - 1000, 154 | limit=1) 155 | 156 | close_price = candles[-1][4] 157 | income *= float(close_price) 158 | 159 | return income 160 | 161 | def sync_account(self): 162 | while True: 163 | try: 164 | account = self.rest_manager._request('get', self.rest_manager.FUTURES_URL + '/v2/account', True, data={}) 165 | asset_balances = [AssetBalance(asset=asset['asset'], 166 | balance=float( 167 | asset['walletBalance']), 168 | unrealizedProfit=float( 169 | asset['unrealizedProfit']) 170 | ) for asset in account['assets']] 171 | 172 | usd_assets = [asset for asset in account['assets'] if asset['asset'] in ['BUSD', 'USDT', 'USDC', 'BNFCR']] 173 | total_wallet_balance = sum([float(asset['walletBalance']) for asset in usd_assets]) 174 | total_upnl = sum([float(asset['unrealizedProfit']) for asset in usd_assets]) 175 | 176 | logger.info(f'Wallet balance: {total_wallet_balance}, upnl: {total_upnl}') 177 | 178 | balance = Balance(totalBalance=total_wallet_balance, 179 | totalUnrealizedProfit=total_upnl, 180 | assets=asset_balances) 181 | self.repository.process_balances(balance, account=self.account.alias) 182 | 183 | positions = [Position(symbol=position['symbol'], 184 | entry_price=float( 185 | position['entryPrice']), 186 | position_size=float( 187 | position['positionAmt']), 188 | side=position['positionSide'], 189 | unrealizedProfit=float( 190 | position['unrealizedProfit']), 191 | initial_margin=float(position['initialMargin']) 192 | ) for position in account['positions'] if position['positionSide'] != 'BOTH'] 193 | self.repository.process_positions(positions, account=self.account.alias) 194 | mark_prices = self.rest_manager.futures_mark_price() 195 | 196 | for position in positions: 197 | if position.position_size != 0.0: 198 | symbol = position.symbol 199 | mark_price = [p for p in mark_prices if p['symbol'] == symbol][0] 200 | tick = Tick(symbol=symbol, 201 | price=float(mark_price['markPrice']), 202 | qty=-1, 203 | timestamp=int(mark_price['time'])) 204 | self.repository.process_tick(tick, account=self.account.alias) 205 | logger.debug(f'Synced recent trade price for {symbol}') 206 | # [self.add_to_ticker(position.symbol) for position in positions if position.position_size > 0.0] 207 | logger.warning('Synced account') 208 | except Exception as e: 209 | logger.error(f'{self.account.alias} Failed to process balance: {e}') 210 | time.sleep(20) 211 | 212 | 213 | def sync_open_orders(self): 214 | while True: 215 | orders = [] 216 | try: 217 | open_orders = self.rest_manager.futures_get_open_orders() 218 | for open_order in open_orders: 219 | order = Order() 220 | order.symbol = open_order['symbol'] 221 | order.price = float(open_order['price']) 222 | order.quantity = float(open_order['origQty']) 223 | order.side = open_order['side'] 224 | order.position_side = open_order['positionSide'] 225 | order.type = open_order['type'] 226 | orders.append(order) 227 | self.repository.process_orders(orders, account=self.account.alias) 228 | logger.warning(f'Synced orders') 229 | 230 | headers = self.rest_manager.response.headers._store 231 | logger.info(f'API weight: {int(headers["x-mbx-used-weight-1m"][1])}') 232 | except Exception as e: 233 | logger.error(f'{self.account.alias} Failed to process open orders for symbol: {e}') 234 | 235 | time.sleep(30) 236 | 237 | def add_to_ticker(self, symbol: str): 238 | if symbol not in self.tick_symbols: 239 | symbol_trade_thread = threading.Thread( 240 | name=f'trade_thread_{symbol}', target=self.process_trades, args=(symbol,), daemon=True) 241 | symbol_trade_thread.start() 242 | 243 | def process_trades(self, symbol: str): 244 | if symbol in self.tick_symbols: 245 | logger.error(f'Already listening to ticks for {symbol}, not starting new processing!') 246 | return 247 | self.tick_symbols.append(symbol) 248 | 249 | # stream buffer is set to length 1, because we're only interested in the most recent tick 250 | self.ws_manager.create_stream(channels=['aggTrade'], 251 | markets=symbol, 252 | stream_buffer_name=f"trades_{symbol}", 253 | output="UnicornFy", 254 | stream_buffer_maxlen=1) 255 | logger.info(f"Trade stream started for {symbol}") 256 | while True: 257 | try: 258 | if self.ws_manager.is_manager_stopping(): 259 | logger.debug('Stopping trade-stream processing...') 260 | break 261 | event = self.ws_manager.pop_stream_data_from_stream_buffer( 262 | stream_buffer_name=f"trades_{symbol}") 263 | if event and 'event_type' in event and event['event_type'] == 'aggTrade': 264 | logger.debug(event) 265 | tick = Tick(symbol=event['symbol'], 266 | price=float(event['price']), 267 | qty=float(event['quantity']), 268 | timestamp=int(event['trade_time'])) 269 | logger.debug(f"Processed tick for {tick.symbol}") 270 | self.repository.process_tick(tick, account=self.account.alias) 271 | except Exception as e: 272 | logger.warning(f'Error processing tick: {e}') 273 | # Price update every 5 seconds is fast enough 274 | time.sleep(5) 275 | logger.warning('Stopped trade-stream processing') 276 | -------------------------------------------------------------------------------- /scraper_root/scraper/binancespot.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import threading 4 | import time 5 | from typing import List 6 | 7 | from unicorn_binance_rest_api import BinanceRestApiManager 8 | from unicorn_binance_websocket_api import BinanceWebSocketApiManager 9 | 10 | from scraper_root.scraper.data_classes import AssetBalance, Position, ScraperConfig, Tick, Balance, \ 11 | Income, Order, Trade, Account 12 | from scraper_root.scraper.persistence.orm_classes import TradeEntity 13 | from scraper_root.scraper.persistence.repository import Repository 14 | 15 | logger = logging.getLogger() 16 | 17 | 18 | class BinanceSpot: 19 | def __init__(self, account: Account, symbols: List[str], repository: Repository, exchange: str = "binance.com"): 20 | print('Binance spot initialized') 21 | self.account = account 22 | self.symbols = symbols 23 | self.api_key = self.account.api_key 24 | self.secret = self.account.api_secret 25 | self.repository = repository 26 | self.ws_manager = BinanceWebSocketApiManager(exchange=exchange, throw_exception_if_unrepairable=True, 27 | warn_on_update=False) 28 | 29 | self.rest_manager = BinanceRestApiManager(self.api_key, api_secret=self.secret) 30 | self.exchange_information = None 31 | self.tick_symbols = [] 32 | 33 | def start(self): 34 | print('Starting binance spot scraper') 35 | 36 | self.exchange_information = self.rest_manager.get_exchange_info() 37 | sorted_symbols = [s for s in self.exchange_information['symbols'] if 38 | s['status'] == 'TRADING' and s['quoteAsset'] in ['BTC', 'USDT', 'BUSD', 'USDC', 'USDP']] 39 | sorted_symbols.extend([s for s in self.exchange_information['symbols'] if s not in sorted_symbols]) 40 | self.exchange_information['symbols'] = sorted_symbols 41 | symbol_search_thread = threading.Thread(name=f'userdata_thread', 42 | target=self.find_new_traded_symbols, 43 | daemon=True) 44 | symbol_search_thread.start() 45 | 46 | # userdata_thread = threading.Thread(name=f'userdata_thread', target=self.process_userdata, daemon=True) 47 | # userdata_thread.start() 48 | 49 | for symbol in self.symbols: 50 | symbol_trade_thread = threading.Thread( 51 | name=f'trade_thread_{symbol}', target=self.process_trades, args=(symbol,), daemon=True) 52 | symbol_trade_thread.start() 53 | 54 | sync_balance_thread = threading.Thread( 55 | name=f'sync_balance_thread', target=self.sync_account, daemon=True) 56 | sync_balance_thread.start() 57 | 58 | sync_trades_thread = threading.Thread( 59 | name=f'sync_trades_thread', target=self.sync_trades, daemon=True) 60 | sync_trades_thread.start() 61 | 62 | sync_orders_thread = threading.Thread( 63 | name=f'sync_orders_thread', target=self.sync_open_orders, daemon=True) 64 | sync_orders_thread.start() 65 | 66 | def find_new_traded_symbols(self): 67 | while True: 68 | try: 69 | counter = 0 70 | for item in self.exchange_information['symbols']: 71 | if item['status'] != 'TRADING': 72 | continue # for performance reasons 73 | symbol = item['symbol'] 74 | if symbol not in self.repository.get_symbol_checks(account=self.account.alias): 75 | if not self.repository.is_symbol_traded(symbol, account=self.account.alias) and counter < 3: 76 | trades = self.rest_manager.get_my_trades(**{'limit': 1, 'symbol': symbol}) 77 | counter += 1 78 | self.repository.process_symbol_checked(symbol, account=self.account.alias) 79 | if len(trades) > 0: 80 | logger.info(f'Trades found for {symbol}, adding to sync list') 81 | self.repository.process_traded_symbol(symbol, account=self.account.alias) 82 | except Exception as e: 83 | logger.error(f'Failed to verify unchecked symbols: {e}') 84 | 85 | logger.info('Updated new traded symbols') 86 | 87 | # TODO: once in a while the checked symbols that are not in the DB should be checked 88 | time.sleep(20) 89 | 90 | def get_asset(self, symbol: str) -> str: 91 | symbol_informations = self.exchange_information['symbols'] 92 | for symbol_information in symbol_informations: 93 | if symbol_information['symbol'] == symbol: 94 | return symbol_information['baseAsset'] 95 | raise Exception(f'No asset found for symbol {symbol}') 96 | 97 | def get_quote_asset(self, symbol: str) -> str: 98 | symbol_informations = self.exchange_information['symbols'] 99 | for symbol_information in symbol_informations: 100 | if symbol_information['symbol'] == symbol: 101 | return symbol_information['quoteAsset'] 102 | raise Exception(f'No asset found for symbol {symbol}') 103 | 104 | def sync_trades(self): 105 | first_trade_reached = {} # key: symbol, value: bool 106 | max_downloads = 10 107 | while True: 108 | try: 109 | iteration_symbols = [] 110 | counter = 0 111 | while counter < max_downloads: 112 | # TODO: sync symbol of open position first if it was more than 5 minutes ago 113 | symbol = self.repository.get_next_traded_symbol(account=self.account.alias) 114 | logger.warning(f'Updating trades for {symbol}') 115 | if symbol is not None: 116 | self.repository.update_trades_last_downloaded(symbol=symbol, account=self.account.alias) 117 | if symbol is None or symbol in iteration_symbols: 118 | counter += 1 119 | continue 120 | iteration_symbols.append(symbol) 121 | if symbol not in first_trade_reached: 122 | first_trade_reached[symbol] = False 123 | while first_trade_reached[symbol] is False and counter < max_downloads: 124 | counter += 1 125 | oldest_trade = self.repository.get_oldest_trade(symbol=symbol, account=self.account.alias) 126 | if oldest_trade is None: 127 | # API will return inclusive, don't want to return the oldest record again 128 | oldest_timestamp = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000) 129 | else: 130 | oldest_timestamp = oldest_trade.timestamp 131 | logger.warning(f'Synced trades before {oldest_timestamp} for {symbol}') 132 | 133 | exchange_trades = self.rest_manager.get_my_trades(**{'symbol': symbol, 'limit': 1000, 134 | 'endTime': oldest_timestamp - 1}) 135 | logger.info( 136 | f"Length of older trades fetched up to {oldest_timestamp}: {len(exchange_trades)} for {symbol}") 137 | trades = [] 138 | for exchange_trade in exchange_trades: 139 | trade = Trade(symbol=exchange_trade['symbol'], 140 | asset=self.get_asset(exchange_trade['symbol']), 141 | order_id=exchange_trade['orderId'], 142 | quantity=exchange_trade['qty'], 143 | price=exchange_trade['price'], 144 | type='REALIZED_PNL', 145 | side='BUY' if exchange_trade['isBuyer'] is True else 'SELL', 146 | timestamp=int(exchange_trade['time'])) 147 | trades.append(trade) 148 | self.repository.process_trades(trades=trades, account=self.account.alias) 149 | if len(exchange_trades) < 1: 150 | first_trade_reached[symbol] = True 151 | 152 | # WARNING: don't use forward-walking only, because binance only returns max 7 days when using forward-walking 153 | # If this logic is ever changed, make sure that it's still able to retrieve all the account history 154 | newest_trade_reached = False 155 | while newest_trade_reached is False and counter < max_downloads: 156 | counter += 1 157 | newest_trade = self.repository.get_newest_trade(symbol=symbol, account=self.account.alias) 158 | if newest_trade is None: 159 | # Binance started in September 2017, so no trade can be before that 160 | # newest_timestamp = int(datetime.datetime.fromisoformat('2017-09-01 00:00:00+00:00').timestamp() * 1000) 161 | newest_order_id = 0 162 | else: 163 | # newest_timestamp = newest_trade.timestamp 164 | newest_order_id = newest_trade.order_id 165 | # logger.warning(f'Synced newer trades since {newest_timestamp}') 166 | logger.warning(f'Synced newer trades since {newest_order_id}') 167 | 168 | exchange_trades = self.rest_manager.get_my_trades(**{'symbol': symbol, 169 | # 'limit': 1000, 170 | 'orderId': newest_order_id + 1}) 171 | # 'startTime': newest_timestamp + 1}) 172 | logger.info( 173 | f"Length of newer trades fetched from id {newest_order_id}: {len(exchange_trades)} for {symbol}") 174 | trades = [] 175 | for exchange_trade in exchange_trades: 176 | trade = Trade(symbol=exchange_trade['symbol'], 177 | asset=self.get_asset(exchange_trade['symbol']), 178 | order_id=exchange_trade['orderId'], 179 | quantity=exchange_trade['qty'], 180 | price=exchange_trade['price'], 181 | type='REALIZED_PNL', 182 | side='BUY' if exchange_trade['isBuyer'] is True else 'SELL', 183 | timestamp=int(exchange_trade['time'])) 184 | trades.append(trade) 185 | self.repository.process_trades(trades=trades, account=self.account.alias) 186 | if len(exchange_trades) < 1: 187 | newest_trade_reached = True 188 | 189 | if newest_trade_reached: # all trades downloaded 190 | # calculate incomes 191 | incomes = self.calculate_incomes(symbol=symbol, 192 | trades=self.repository.get_trades(symbol=symbol, 193 | account=self.account.alias)) 194 | self.repository.process_incomes(incomes=incomes, account=self.account.alias) 195 | logger.warning('Synced trades') 196 | except Exception as e: 197 | logger.error(f'Failed to process trades: {e}') 198 | 199 | time.sleep(60) 200 | 201 | def calc_long_pprice(self, long_psize, trades: List[TradeEntity]): 202 | trades.sort(key=lambda x: x.timestamp) 203 | psize, pprice = 0.0, 0.0 204 | for trade in trades: 205 | abs_qty = abs(trade.quantity) 206 | if trade.side == 'BUY': 207 | new_psize = psize + abs_qty 208 | pprice = pprice * (psize / new_psize) + trade.price * (abs_qty / new_psize) 209 | psize = new_psize 210 | else: 211 | psize = max(0.0, psize - abs_qty) 212 | return pprice 213 | 214 | def calc_long_pnl(self, entry_price, close_price, qty, inverse, c_mult) -> float: 215 | if inverse: 216 | if entry_price == 0.0 or close_price == 0.0: 217 | return 0.0 218 | return abs(qty) * c_mult * (1.0 / entry_price - 1.0 / close_price) 219 | else: 220 | return abs(qty) * (close_price - entry_price) 221 | 222 | def calculate_incomes(self, symbol: str, trades: List[TradeEntity]) -> List[Income]: 223 | incomes = [] 224 | psize, pprice = 0.0, 0.0 225 | for trade in trades: 226 | if trade.side == 'BUY': 227 | new_psize = psize + trade.quantity 228 | pprice = pprice * (psize / new_psize) + trade.price * (trade.quantity / new_psize) 229 | psize = new_psize 230 | elif psize > 0.0: 231 | income = Income(symbol=symbol, 232 | asset=trade.asset, 233 | type='REALIZED_PNL', 234 | income=self.calc_long_pnl(pprice, trade.price, trade.quantity, False, 1.0), 235 | timestamp=trade.timestamp, 236 | transaction_id=trade.order_id) 237 | incomes.append(income) 238 | psize = max(0.0, psize - trade.quantity) 239 | return incomes 240 | 241 | def sync_account(self): 242 | while True: 243 | try: 244 | account = self.rest_manager.get_account() 245 | current_prices = self.rest_manager.get_all_tickers() 246 | total_usdt_wallet_balance = 0.0 247 | total_unrealized_profit = 0.0 248 | asset_balances = [] 249 | positions = [] 250 | for balance in account['balances']: 251 | asset = balance['asset'] 252 | free = float(balance['free']) 253 | locked = float(balance['locked']) 254 | asset_quantity = free + locked 255 | if asset_quantity > 0.0: 256 | if asset in ['USDT', 'BUSD', 'USDC', 'USDP']: 257 | total_usdt_wallet_balance += asset_quantity 258 | else: 259 | current_usd_prices = [p for p in current_prices if 260 | p['symbol'] in [f'{asset}BTC', f'{asset}USDT', f'{asset}BUSD', 261 | f'{asset}USDC', f'{asset}USDP']] 262 | if len(current_usd_prices) > 0: 263 | asset_usd_balance = 0.0 264 | unrealized_profit = 0.0 265 | asset_positions = [] 266 | for current_usd_price in current_usd_prices: 267 | symbol = current_usd_price['symbol'] 268 | symbol_trades = self.repository.get_trades_by_asset(symbol, 269 | account=self.account.alias) 270 | 271 | if len(symbol_trades) > 0: # and len(self.repository.get_open_orders(symbol)) > 0: 272 | position_price = self.calc_long_pprice(long_psize=asset_quantity, 273 | trades=symbol_trades) 274 | 275 | # position size is already bigger than 0, so there is a position 276 | unrealized_profit = (self.get_current_price( 277 | symbol) - position_price) * asset_quantity 278 | total_unrealized_profit += unrealized_profit 279 | 280 | position = Position(symbol=symbol, 281 | entry_price=position_price, 282 | position_size=asset_quantity, 283 | side='LONG', 284 | unrealizedProfit=unrealized_profit, 285 | initial_margin=0.0) 286 | asset_positions.append(position) 287 | logger.debug(f'Processed position for {symbol}') 288 | 289 | position_with_open_orders = [position for position in asset_positions 290 | if len(self.repository.get_open_orders(position.symbol, 291 | account=self.account.alias)) > 0] 292 | selected_position = None 293 | if len(position_with_open_orders) == 1: 294 | selected_position = position_with_open_orders[0] 295 | elif len(position_with_open_orders) > 1: 296 | selected_position = position_with_open_orders[0] 297 | logger.warning(f'Found multiple different symbols ' 298 | f'({[pos.symbol for pos in position_with_open_orders]}) with open ' 299 | f'orders for asset {asset}, using {selected_position.symbol}') 300 | else: 301 | overall_latest_trade_date = None 302 | for position in asset_positions: 303 | symbol_trades = self.repository.get_trades(symbol=position.symbol, 304 | account=self.account.alias) 305 | if len(symbol_trades) > 0: 306 | latest_trade = max([trade.timestamp for trade in symbol_trades]) 307 | if overall_latest_trade_date is None or latest_trade > overall_latest_trade_date: 308 | overall_latest_trade_date = latest_trade 309 | selected_position = position 310 | continue 311 | 312 | if selected_position is not None: 313 | asset_usd_balance = asset_quantity * selected_position.entry_price 314 | positions.append(selected_position) 315 | 316 | asset_balance = AssetBalance(asset=balance['asset'], 317 | balance=asset_usd_balance, 318 | unrealizedProfit=unrealized_profit) 319 | asset_balances.append(asset_balance) 320 | else: 321 | logger.debug(f'NO PRICE FOUND FOR ASSET {asset}') 322 | 323 | positions_to_use = [] 324 | for position in positions: 325 | base_asset = self.get_asset(position.symbol) 326 | quote_asset = self.get_quote_asset(position.symbol) 327 | 328 | quote_based_position_found = False 329 | 330 | for inspected_position in positions: 331 | inspected_base_asset = self.get_asset(inspected_position.symbol) 332 | inspected_quote_asset = self.get_quote_asset(inspected_position.symbol) 333 | if inspected_quote_asset == base_asset and \ 334 | len(self.repository.get_trades(symbol=inspected_position.symbol, 335 | account=self.account.alias)) > 0: 336 | [positions_to_use.remove(p) for p in positions_to_use 337 | if self.get_asset(p.symbol) == inspected_quote_asset 338 | and self.get_quote_asset(p.symbol) == inspected_base_asset] 339 | 340 | quote_based_position_found = True 341 | break 342 | 343 | if not quote_based_position_found: 344 | positions_to_use.append(position) 345 | 346 | coin_usdt_balance = sum([b.balance for b in asset_balances]) 347 | total_usdt_wallet_balance += coin_usdt_balance 348 | logger.info(f"Total wallet balance in USDT = {total_usdt_wallet_balance}") 349 | 350 | total_balance = Balance(totalBalance=total_usdt_wallet_balance, 351 | totalUnrealizedProfit=total_unrealized_profit, 352 | assets=asset_balances) 353 | 354 | self.repository.process_balances(total_balance, account=self.account.alias) 355 | self.repository.process_positions(positions_to_use, account=self.account.alias) 356 | logger.warning('Synced account') 357 | except Exception as e: 358 | logger.error(f'Failed to process balance: {e}') 359 | 360 | time.sleep(20) 361 | 362 | def sync_open_orders(self): 363 | while True: 364 | orders = [] 365 | try: 366 | open_orders = self.rest_manager.get_open_orders() 367 | for open_order in open_orders: 368 | order = Order() 369 | order.symbol = open_order['symbol'] 370 | order.price = float(open_order['price']) 371 | order.quantity = float(open_order['origQty']) 372 | order.side = open_order['side'] 373 | order.position_side = 'LONG' 374 | order.type = open_order['type'] 375 | orders.append(order) 376 | except Exception as e: 377 | logger.error(f'Failed to process open orders for symbol: {e}') 378 | self.repository.process_orders(orders, account=self.account.alias) 379 | 380 | logger.warning('Synced open orders') 381 | 382 | time.sleep(30) 383 | 384 | def get_current_price(self, symbol: str) -> float: 385 | if symbol not in self.tick_symbols: 386 | symbol_trade_thread = threading.Thread( 387 | name=f'trade_thread_{symbol}', target=self.process_trades, args=(symbol,), daemon=True) 388 | symbol_trade_thread.start() 389 | 390 | curr_price = self.repository.get_current_price(symbol, account=self.account.alias) 391 | return curr_price.price if curr_price else 0.0 392 | 393 | def process_trades(self, symbol: str): 394 | if symbol in self.tick_symbols: 395 | logger.error(f'Already listening to ticks for {symbol}, not starting new processing!') 396 | return 397 | self.tick_symbols.append(symbol) 398 | # stream buffer is set to length 1, because we're only interested in the most recent tick 399 | self.ws_manager.create_stream(channels=['aggTrade'], 400 | markets=symbol, 401 | stream_buffer_name=f"trades_{symbol}", 402 | output="UnicornFy", 403 | stream_buffer_maxlen=1) 404 | logger.info(f"Trade stream started for {symbol}") 405 | while True: 406 | if self.ws_manager.is_manager_stopping(): 407 | logger.debug('Stopping trade-stream processing...') 408 | break 409 | event = self.ws_manager.pop_stream_data_from_stream_buffer( 410 | stream_buffer_name=f"trades_{symbol}") 411 | if event and 'event_type' in event and event['event_type'] == 'aggTrade': 412 | logger.debug(event) 413 | tick = Tick(symbol=event['symbol'], 414 | price=float(event['price']), 415 | qty=float(event['quantity']), 416 | timestamp=int(event['trade_time'])) 417 | logger.debug(f"Processed tick for {tick.symbol}") 418 | self.repository.process_tick(tick, account=self.account.alias) 419 | # Price update every 5 seconds is fast enough 420 | time.sleep(5) 421 | logger.warning('Stopped trade-stream processing') 422 | self.tick_symbols.remove(symbol) 423 | -------------------------------------------------------------------------------- /scraper_root/scraper/bitgetfutures.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging 3 | import threading 4 | import time 5 | from typing import List 6 | 7 | from pybitget import Client 8 | from pybitget import utils 9 | 10 | from scraper_root.scraper.data_classes import AssetBalance, Position, Tick, Balance, Income, Order, \ 11 | Account 12 | from scraper_root.scraper.persistence.repository import Repository 13 | 14 | logger = logging.getLogger() 15 | 16 | 17 | class BitgetFutures: 18 | def __init__(self, account: Account, symbols: List[str], repository: Repository, exchange: str = "bitget"): 19 | logger.info(f"Bitget initializing") 20 | self.account = account 21 | self.alias = self.account.alias 22 | self.symbols = symbols 23 | self.api_key = self.account.api_key 24 | self.secret = self.account.api_secret 25 | self.passphrase = self.account.api_passphrase 26 | self.repository = repository 27 | # bitget connection 28 | self.rest_manager_bitget = Client(self.api_key, self.secret, passphrase=self.passphrase) 29 | 30 | # pull all USDT symbols and create a list. 31 | self.linearsymbols = [] 32 | linearsymbolslist = self.rest_manager_bitget.mix_get_symbols_info(productType='UMCBL') 33 | try: 34 | for i in linearsymbolslist['data']: 35 | if i['quoteCoin'] == 'USDT': 36 | self.linearsymbols.append(i['symbol'][:-6]) 37 | except Exception as e: 38 | logger.error(f'{self.alias}: Failed to pull linearsymbols: {e}') 39 | 40 | self.activesymbols = [] # list 41 | 42 | def start(self): 43 | logger.info(f'{self.alias}: Starting Bitget Futures scraper') 44 | 45 | for symbol in self.symbols: 46 | symbol_trade_thread = threading.Thread(name=f'trade_thread_{symbol}', target=self.process_trades, 47 | args=(symbol,), daemon=True) 48 | symbol_trade_thread.start() 49 | 50 | sync_balance_thread = threading.Thread(name=f'sync_balance_thread', target=self.sync_account, daemon=True) 51 | sync_balance_thread.start() 52 | 53 | sync_positions_thread = threading.Thread(name=f'sync_positions_thread', target=self.sync_positions, daemon=True) 54 | sync_positions_thread.start() 55 | 56 | sync_trades_thread = threading.Thread(name=f'sync_trades_thread', target=self.sync_trades, daemon=True) 57 | sync_trades_thread.start() 58 | 59 | sync_orders_thread = threading.Thread(name=f'sync_orders_thread', target=self.sync_open_orders, daemon=True) 60 | sync_orders_thread.start() 61 | 62 | def sync_account(self): 63 | while True: 64 | try: 65 | account = self.rest_manager_bitget.mix_get_accounts(productType='UMCBL') 66 | assets = account['data'] 67 | asset_balances = [AssetBalance(asset=asset['marginCoin'], balance=float(asset['available']), unrealizedProfit=float(asset['unrealizedPL'])) for asset in assets] 68 | 69 | # bitget has no total assets balance, assuming USDT 70 | for asset in assets: 71 | if asset['marginCoin'] == 'USDT': 72 | balance = Balance(totalBalance=asset['available'], totalUnrealizedProfit=asset['unrealizedPL'], assets=asset_balances) 73 | self.repository.process_balances(balance=balance, account=self.alias) 74 | logger.warning(f'{self.alias}: Synced balance') 75 | time.sleep(100) 76 | except Exception as e: 77 | logger.error(f'{self.alias}: Failed to process balance: {e}') 78 | time.sleep(360) 79 | pass 80 | 81 | def sync_positions(self): 82 | while True: 83 | try: 84 | # global activesymbols 85 | self.activesymbols = ["BTCUSDT"] 86 | positions = [] 87 | for i in self.linearsymbols: 88 | exchange_position = self.rest_manager_bitget.mix_get_single_position(symbol="{}_UMCBL".format(i),marginCoin='USDT') 89 | for x in exchange_position['data']: 90 | if x['total'] != '0': # filter only items that have positions 91 | if x['holdSide'] == "long": # recode long / short into LONG / SHORT 92 | side = "LONG" 93 | else: 94 | side = "SHORT" 95 | self.activesymbols.append(x['symbol'][:-6]) 96 | 97 | positions.append(Position(symbol=x['symbol'][:-6], 98 | entry_price=float(x['averageOpenPrice']), 99 | position_size=float(x['total']), 100 | side=side, 101 | # make it the same as binance data, bitget data is : item['holdside'], 102 | unrealizedProfit=float(x['unrealizedPL']), 103 | initial_margin=float(x['margin'])) 104 | ) 105 | self.repository.process_positions(positions=positions, account=self.alias) 106 | logger.warning(f'{self.alias}: Synced positions') 107 | # logger.info(f'test: {self.activesymbols}') 108 | time.sleep(250) 109 | except Exception as e: 110 | logger.error(f'{self.alias}: Failed to process positions: {e}') 111 | time.sleep(360) 112 | pass 113 | 114 | def sync_open_orders(self): 115 | while True: 116 | orders = [] 117 | if len(self.activesymbols) > 1: # if activesymbols has more than 1 item do stuff 118 | for i in self.activesymbols: 119 | try: # when there a new symbols a pnl request fails with an error and scripts stops. so in a try and pass. 120 | open_orders = self.rest_manager_bitget.mix_get_open_order(symbol="{}_UMCBL".format(i)) 121 | if not open_orders['data']: # note: None = empty. 122 | pass 123 | else: 124 | for item in open_orders["data"]: 125 | order = Order() 126 | order.symbol = item['symbol'][:-6] 127 | order.price = float(item['price']) 128 | order.quantity = float(item['size']) 129 | if item['side'].startswith('open'): # recode open / close into BUY / SELL 130 | side = "BUY" 131 | else: 132 | side = "SELL" 133 | order.side = side 134 | order.position_side = item['posSide'].upper() # upper() to make it the same as binance 135 | order.type = item['orderType'] 136 | orders.append(order) 137 | except Exception as e: 138 | logger.warning(f'{self.alias}: Failed to process orders: {e}') 139 | time.sleep(360) 140 | pass 141 | logger.warning(f'{self.alias}: Synced orders') 142 | self.repository.process_orders(orders=orders, account=self.alias) 143 | time.sleep(140) # pause after 1 complete run 144 | 145 | # #WS stream bybit; for future use, cannot limit ws stream 146 | # def process_trades(self, symbol: str): 147 | # subs = [ 148 | # "trade."[symbol] 149 | # ] 150 | # self.ws_trades = WebSocket( 151 | # "wss://stream-testnet.bybit.com/realtime_public", 152 | # subscriptions=subs 153 | # ) 154 | # logger.info(f"Trade stream started") 155 | # while True: 156 | # if self.ws_trades.is_trades_stopping(): 157 | # logger.debug('Stopping trade-stream processing...') 158 | # break 159 | # # event = self.ws_manager.pop_stream_data_from_stream_buffer(stream_buffer_name=f"trades_{symbol}") 160 | # # if event and 'event_type' in event and event['event_type'] == 'aggTrade': 161 | # # logger.debug(event) 162 | # # tick = Tick(symbol=event['symbol'], 163 | # # price=float(event['price']), 164 | # # qty=float(event['quantity']), 165 | # # timestamp=int(event['trade_time'])) 166 | # # logger.debug(f"Processed tick for {tick.symbol}") 167 | # # self.repository.process_tick(tick) 168 | # # # Price update every 5 seconds is fast enough 169 | # # time.sleep(5) 170 | # # logger.warning('Stopped trade-stream processing') 171 | # data = self.ws_trades.fetch(subs[0]) 172 | # if data: 173 | # print(data) 174 | 175 | def process_trades(self, symbol: str): 176 | while True: 177 | # logger.info(f"Trade stream started") 178 | if len(self.activesymbols) > 1: # if activesymbols has more than 1 item do stuff 179 | try: 180 | for i in self.activesymbols: 181 | event = self.rest_manager_bitget.mix_get_fills(symbol="{}_UMCBL".format(i), limit='1') 182 | event1 = event['data'][0] 183 | tick = Tick(symbol=event1['symbol'][:-6], 184 | price=float(event1['price']), 185 | qty=float(event1['size']), 186 | timestamp=int(event1['timestamp'])) 187 | self.repository.process_tick(tick=tick, account=self.alias) 188 | logger.info(f"{self.alias}: Processed ticks") 189 | time.sleep(130) 190 | except Exception as e: 191 | logger.warning(f'{self.alias}: Failed to process trades: {e}') 192 | time.sleep(360) 193 | pass 194 | 195 | def sync_trades(self): 196 | lastEnd = '' 197 | startT = '0' 198 | while True: 199 | endT = utils.get_timestamp() 200 | try: # bitget returns all income, the symbol spcified dosn't matter 201 | exchange_pnl = self.rest_manager_bitget.mix_get_accountBill(symbol='BTCUSDT_UMCBL', marginCoin='USDT',startTime=startT,endTime=endT, pageSize='100') 202 | if not exchange_pnl['data']['result']: # note: None = empty. 203 | pass 204 | else: 205 | while True: 206 | incomes = [] 207 | for exchange_income in exchange_pnl["data"]['result']: 208 | if exchange_income['symbol']: 209 | if exchange_income['business'].startswith('close'): 210 | income_type = 'REALIZED_PNL' 211 | else: 212 | income_type = exchange_income['business'] 213 | income = Income(symbol=exchange_income['symbol'][:-6], 214 | asset='USDT', 215 | type=income_type, 216 | income=float(exchange_income['amount'])+float(exchange_income['fee']), 217 | timestamp=int(int(exchange_income['cTime'])/1000)*1000, 218 | transaction_id=exchange_income['id']) 219 | incomes.append(income) 220 | self.repository.process_incomes(incomes=incomes, account=self.alias) 221 | time.sleep(5) # pause to not overload the api limit 222 | if not exchange_pnl['data']['nextFlag']: 223 | startT = endT 224 | break 225 | lastEnd = exchange_pnl['data']['lastEndId'] 226 | exchange_pnl = self.rest_manager_bitget.mix_get_accountBill(symbol='BTCUSDT_UMCBL', marginCoin='USDT',startTime=startT,endTime=endT, pageSize='100', lastEndId=lastEnd) 227 | except Exception: 228 | time.sleep(360) 229 | pass 230 | logger.info(f'{self.alias}: Synced trades') 231 | time.sleep(120) 232 | -------------------------------------------------------------------------------- /scraper_root/scraper/bybitderivatives.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | import time 4 | import datetime 5 | 6 | from dateutil import parser 7 | from typing import List, Dict 8 | 9 | from pybit.exceptions import FailedRequestError 10 | from pybit.unified_trading import HTTP 11 | 12 | from scraper_root.scraper.data_classes import AssetBalance, Position, Tick, Balance, Income, Order, \ 13 | Account 14 | from scraper_root.scraper.persistence.repository import Repository 15 | from scraper_root.scraper.utils import readable 16 | 17 | logger = logging.getLogger() 18 | 19 | 20 | def is_asset_usd_or_derivative(symbol: str): 21 | if 'usdt' in symbol.lower(): 22 | return True 23 | if 'usd' in symbol.lower(): 24 | return True 25 | if 'usdc' in symbol.lower(): 26 | return True 27 | return False 28 | 29 | 30 | class BybitDerivatives: 31 | def __init__(self, account: Account, symbols: List[str], repository: Repository, unified_account: bool): 32 | logger.info(f"Bybit initializing") 33 | self.account = account 34 | self.unified_account = unified_account 35 | self.alias = self.account.alias 36 | self.symbols = symbols 37 | self.api_key = self.account.api_key 38 | self.secret = self.account.api_secret 39 | self.repository = repository 40 | # self.ws_manager = BybitWebsocket(wsURL="wss://stream-testnet.bybit.com/realtime_private", 41 | # api_key=self.api_key, api_secret=self.secret) 42 | # bybit connection 43 | self.rest_manager2 = HTTP(testnet=False, api_key=self.api_key, api_secret=self.secret) 44 | 45 | # check if i am able to login 46 | test = self.rest_manager2.get_api_key_information() 47 | if test['retCode'] == 0: 48 | logger.info(f"{self.alias}: rest login succesfull") 49 | else: 50 | logger.error(f"{self.alias}: failed to login") 51 | logger.error(f"{self.alias}: exiting") 52 | raise SystemExit() 53 | 54 | # pull all USDT symbols and create a list. 55 | self.linearsymbols = [] 56 | linearsymbolslist = self.rest_manager2.get_instruments_info(category='linear', 57 | limit=1000, 58 | status='Trading') 59 | try: 60 | for i in linearsymbolslist['result']['list']: 61 | if i['quoteCoin'] == 'USDT': 62 | self.linearsymbols.append(i['symbol']) 63 | except Exception as e: 64 | logger.error(f'{self.alias}: Failed to pull linearsymbols: {e}') 65 | 66 | # globals 67 | # global activesymbols 68 | self.activesymbols = [] # list 69 | self.asset_symbol: Dict[str, str] = {} 70 | 71 | def start(self): 72 | logger.info(f'{self.alias}: Starting Bybit Derivatives scraper') 73 | 74 | for symbol in self.symbols: 75 | symbol_trade_thread = threading.Thread(name=f'trade_thread_{symbol}', target=self.sync_current_price, 76 | args=(symbol,), daemon=True) 77 | symbol_trade_thread.start() 78 | 79 | sync_balance_thread = threading.Thread(name=f'sync_balance_thread', target=self.sync_account, daemon=True) 80 | sync_balance_thread.start() 81 | 82 | sync_positions_thread = threading.Thread(name=f'sync_positions_thread', target=self.sync_positions, daemon=True) 83 | sync_positions_thread.start() 84 | 85 | sync_trades_thread = threading.Thread(name=f'sync_trades_thread', target=self.sync_trades, daemon=True) 86 | sync_trades_thread.start() 87 | 88 | sync_orders_thread = threading.Thread(name=f'sync_orders_thread', target=self.sync_open_orders, daemon=True) 89 | sync_orders_thread.start() 90 | 91 | def sync_account(self): 92 | while True: 93 | try: 94 | accounttype = "UNIFIED" if self.unified_account is True else "CONTRACT" 95 | account = self.rest_manager2.get_wallet_balance(accountType=accounttype) 96 | assets = account['result']['list'] 97 | balances = [] 98 | total_usdt_balance = 0 99 | total_upnl = 0 100 | for asset in assets: 101 | for coin in asset['coin']: 102 | if coin['coin'] == 'USDT': 103 | total_usdt_balance += float(coin['walletBalance']) 104 | total_upnl += float(coin['unrealisedPnl']) 105 | else: 106 | balances.append(AssetBalance(asset=coin['coin'], balance=float(coin['walletBalance']), unrealizedProfit=float(coin['unrealisedPnl']))) 107 | 108 | # bybit has no total assets balance, assuming USDT 109 | balance = Balance(totalBalance=total_usdt_balance, 110 | totalUnrealizedProfit=total_upnl, 111 | assets=balances) 112 | self.repository.process_balances(balance=balance, account=self.alias) 113 | logger.warning(f'{self.alias}: Synced balance') 114 | time.sleep(100) 115 | except Exception as e: 116 | logger.error(f'{self.alias}: Failed to process balance: {e}') 117 | time.sleep(360) 118 | pass 119 | 120 | def sync_positions(self): 121 | while True: 122 | try: 123 | # global activesymbols 124 | self.activesymbols = ["BTCUSDT"] 125 | positions = [] 126 | exchange_position = self.rest_manager2.get_positions(category='linear', settleCoin="USDT") 127 | for x in exchange_position['result']['list']: 128 | if float(x['size']) != 0: # filter only items that have positions 129 | if x['positionIdx'] == 1: # recode buy / sell into long / short 130 | side = "LONG" 131 | else: 132 | side = "SHORT" 133 | self.activesymbols.append(x['symbol']) 134 | 135 | positions.append(Position(symbol=x['symbol'], 136 | entry_price=float(x['avgPrice']), 137 | position_size=float(x['size']), 138 | side=side, 139 | # make it the same as binance data, bybit data is : item['side'], 140 | unrealizedProfit=float(x['unrealisedPnl']), 141 | initial_margin=0.0, # TODO: float(x['position_margin']) 142 | market_price=float(x['markPrice'])) 143 | ) 144 | self.repository.process_positions(positions=positions, account=self.alias) 145 | logger.warning(f'{self.alias}: Synced positions') 146 | # logger.info(f'test: {self.activesymbols}') 147 | time.sleep(250) 148 | except Exception as e: 149 | logger.error(f'{self.alias}: Failed to process positions: {e}') 150 | time.sleep(360) 151 | 152 | def sync_open_orders(self): 153 | while True: 154 | orders = [] 155 | if len(self.activesymbols) > 1: # if activesymbols has more than 1 item do stuff 156 | for symbol in self.activesymbols: 157 | try: # when there a new symbols a pnl request fails with an error and scripts stops. so in a try and pass. 158 | open_orders = self.rest_manager2.get_open_orders(category='linear', symbol=symbol, limit=50) # queries open orders only by default 159 | for item in open_orders["result"]['list']: 160 | order = Order() 161 | order.symbol = item['symbol'] 162 | order.price = float(item['price']) 163 | order.quantity = float(item['qty']) 164 | order.side = item['side'].upper() # upper() to make it the same as binance 165 | # bybit has no 'position side', assuming 'side' 166 | # if item['side'] == "Buy": # recode buy / sell into long / short 167 | # side = "SHORT" # note: reversed. buy=short,sell = long 168 | # else: 169 | # side = "LONG" 170 | if item['side'] == "Buy": # recode buy / sell into long / short 171 | if item['reduceOnly']: 172 | side = "SHORT" 173 | else: 174 | side = "LONG" 175 | else: 176 | if item['reduceOnly']: 177 | side = "LONG" 178 | else: 179 | side = "SHORT" 180 | order.position_side = side 181 | order.type = item['orderType'] 182 | orders.append(order) 183 | except: 184 | logger.exception(f'{self.alias}: Failed to process orders') 185 | time.sleep(30) 186 | logger.warning(f'{self.alias}: Synced orders') 187 | self.repository.process_orders(orders=orders, account=self.alias) 188 | time.sleep(120) # pause after 1 complete run 189 | 190 | # #WS stream bybit; for future use, cannot limit ws stream 191 | # def process_trades(self, symbol: str): 192 | # subs = [ 193 | # "trade."[symbol] 194 | # ] 195 | # self.ws_trades = WebSocket( 196 | # "wss://stream-testnet.bybit.com/realtime_public", 197 | # subscriptions=subs 198 | # ) 199 | # logger.info(f"Trade stream started") 200 | # while True: 201 | # if self.ws_trades.is_trades_stopping(): 202 | # logger.debug('Stopping trade-stream processing...') 203 | # break 204 | # # event = self.ws_manager.pop_stream_data_from_stream_buffer(stream_buffer_name=f"trades_{symbol}") 205 | # # if event and 'event_type' in event and event['event_type'] == 'aggTrade': 206 | # # logger.debug(event) 207 | # # tick = Tick(symbol=event['symbol'], 208 | # # price=float(event['price']), 209 | # # qty=float(event['quantity']), 210 | # # timestamp=int(event['trade_time'])) 211 | # # logger.debug(f"Processed tick for {tick.symbol}") 212 | # # self.repository.process_tick(tick) 213 | # # # Price update every 5 seconds is fast enough 214 | # # time.sleep(5) 215 | # # logger.warning('Stopped trade-stream processing') 216 | # data = self.ws_trades.fetch(subs[0]) 217 | # if data: 218 | # print(data) 219 | 220 | def sync_current_price(self, symbol: str): 221 | while True: 222 | # logger.info(f"Trade stream started") 223 | try: 224 | for i in self.activesymbols: 225 | event = self.rest_manager2.get_public_trade_history(category="linear", symbol="{}".format(i), limit='1') 226 | event1 = event['result']['list'][0] 227 | tick = Tick(symbol=event1['symbol'], 228 | price=float(event1['price']), 229 | qty=float(event1['size']), 230 | timestamp=int(event1['time'])) 231 | self.repository.process_tick(tick=tick, account=self.alias) 232 | logger.info(f"{self.alias}: Processed ticks") 233 | time.sleep(60) 234 | except Exception as e: 235 | logger.warning(f'{self.alias}: Failed to process ticks: {e}') 236 | time.sleep(120) 237 | pass 238 | 239 | def sync_trades(self): 240 | max_fetches_in_cycle = 3 241 | first_trade_reached = False 242 | one_day_ms = 24 * 60 * 60 * 1000 243 | while True: 244 | try: 245 | two_years_ago = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=2*365-1)) 246 | two_years_ago = int(two_years_ago.timestamp() * 1000) 247 | 248 | counter = 0 249 | while first_trade_reached is False and counter < max_fetches_in_cycle: 250 | counter += 1 251 | oldest_income = self.repository.get_oldest_income(account=self.account.alias) 252 | if oldest_income is None: 253 | # API will return inclusive, don't want to return the oldest record again 254 | oldest_timestamp = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000) 255 | else: 256 | oldest_timestamp = oldest_income.timestamp 257 | logger.warning(f'Synced trades before {readable(oldest_timestamp)}') 258 | 259 | oldest_timestamp = max(oldest_timestamp, two_years_ago) 260 | 261 | exchange_incomes = self.rest_manager2.get_closed_pnl(category="linear", limit='100', startTime=oldest_timestamp - one_day_ms, endTime=oldest_timestamp - 1) 262 | logger.info(f"Length of older trades fetched up to {readable(oldest_timestamp)}: {len(exchange_incomes['result']['list'])}") 263 | incomes = [] 264 | 265 | for exchange_income in exchange_incomes['result']['list']: 266 | asset = self.get_asset(exchange_income['symbol']) 267 | if not is_asset_usd_or_derivative(exchange_income['symbol']): 268 | exchange_income['income'] = self.income_to_usdt( 269 | float(exchange_income['income']), 270 | int(exchange_income['updatedTime']), 271 | asset) 272 | 273 | income = Income(symbol=exchange_income['symbol'], 274 | asset="USDT", 275 | type='REALIZED_PNL', 276 | income=float(exchange_income['closedPnl']), 277 | timestamp=int(exchange_income['createdTime']), 278 | transaction_id=exchange_income['orderId']) 279 | incomes.append(income) 280 | 281 | while exchange_incomes['result']['nextPageCursor'] != '': 282 | logger.info(f"{self.alias}: Retrieving orders for page cursor {exchange_incomes['result']['nextPageCursor']}'") 283 | for exchange_income in exchange_incomes['result']['list']: 284 | asset = self.get_asset(exchange_income['symbol']) 285 | if not is_asset_usd_or_derivative(exchange_income['symbol']): 286 | exchange_income['income'] = self.income_to_usdt( 287 | float(exchange_income['income']), 288 | int(exchange_income['time']), 289 | asset) 290 | 291 | income = Income(symbol=exchange_income['symbol'], 292 | asset="USDT", 293 | type='REALIZED_PNL', 294 | income=float(exchange_income['closedPnl']), 295 | timestamp=int(exchange_income['createdTime']), 296 | transaction_id=exchange_income['orderId']) 297 | incomes.append(income) 298 | 299 | exchange_incomes = self.rest_manager2.get_closed_pnl(category="linear", limit='100', endTime=oldest_timestamp - 1, 300 | cursor=exchange_incomes['result']['nextPageCursor']) 301 | self.repository.process_incomes(incomes, account=self.account.alias) 302 | if len(incomes) < 1: 303 | first_trade_reached = True 304 | 305 | # WARNING: don't use forward-walking only, because binance only returns max 7 days when using forward-walking 306 | # If this logic is ever changed, make sure that it's still able to retrieve all the account history 307 | newest_trade_reached = False 308 | while newest_trade_reached is False and counter < max_fetches_in_cycle: 309 | counter += 1 310 | newest_income = self.repository.get_newest_income(account=self.account.alias) 311 | if newest_income is None: 312 | # only support trades after 2020, so no trade can be before that 313 | newest_timestamp = int(datetime.datetime.fromisoformat('2020-01-01 00:00:00+00:00').timestamp() * 1000) 314 | else: 315 | newest_timestamp = newest_income.timestamp 316 | logger.warning(f'Synced newer trades since {readable(newest_timestamp)}') 317 | 318 | newest_timestamp = max(newest_timestamp, two_years_ago) 319 | 320 | exchange_incomes = self.rest_manager2.get_closed_pnl(category="linear", limit='100', startTime=newest_timestamp + 1) 321 | logger.info(f"Length of newer trades fetched from {readable(newest_timestamp)}: {len(exchange_incomes['result']['list'])}") 322 | incomes = [] 323 | for exchange_income in exchange_incomes['result']['list']: 324 | asset = self.get_asset(exchange_income['symbol']) 325 | if not is_asset_usd_or_derivative(exchange_income['symbol']): 326 | exchange_income['income'] = self.income_to_usdt( 327 | float(exchange_income['income']), 328 | int(exchange_income['updatedTime']), 329 | asset) 330 | 331 | income = Income(symbol=exchange_income['symbol'], 332 | asset="USDT", 333 | type='REALIZED_PNL', 334 | income=float(exchange_income['closedPnl']), 335 | timestamp=int(exchange_income['createdTime']), 336 | transaction_id=exchange_income['orderId']) 337 | incomes.append(income) 338 | 339 | while exchange_incomes['result']['nextPageCursor'] != '': 340 | for exchange_income in exchange_incomes['result']['list']: 341 | asset = self.get_asset(exchange_income['symbol']) 342 | if not is_asset_usd_or_derivative(exchange_income['symbol']): 343 | exchange_income['income'] = self.income_to_usdt( 344 | float(exchange_income['income']), 345 | int(exchange_income['updatedTime']), 346 | asset) 347 | 348 | income = Income(symbol=exchange_income['symbol'], 349 | asset="USDT", 350 | type='REALIZED_PNL', 351 | income=float(exchange_income['closedPnl']), 352 | timestamp=int(exchange_income['createdTime']), 353 | transaction_id=exchange_income['orderId']) 354 | incomes.append(income) 355 | 356 | exchange_incomes = self.rest_manager2.get_closed_pnl(category="linear", limit='100', startTime=newest_timestamp + 1, 357 | cursor=exchange_incomes['result']['nextPageCursor']) 358 | self.repository.process_incomes(incomes, account=self.account.alias) 359 | if len(incomes) < 1: 360 | newest_trade_reached = True 361 | 362 | logger.warning('Synced trades') 363 | except Exception as e: 364 | logger.exception(f'{self.account.alias} Failed to process trades: {e}') 365 | 366 | time.sleep(60) 367 | 368 | def income_to_usdt(self, income: float, income_timestamp: int, asset: str) -> float: 369 | if is_asset_usd_or_derivative(asset): 370 | return income 371 | 372 | # Can't get the latest aggr_trades on just the endTime, so this is 'best effort' 373 | symbol = f"{asset}USDT" 374 | candles = self.rest_manager2.get_kline(category="linear", 375 | symbol=symbol, 376 | interval='1', 377 | start=int(income_timestamp) - 1000, 378 | limit=1) 379 | 380 | close_price = candles['result']['list'][-1][4] 381 | income *= float(close_price) 382 | 383 | return income 384 | 385 | def get_asset(self, symbol: str): 386 | if symbol not in self.asset_symbol: 387 | try: 388 | self.asset_symbol[symbol] = self.rest_manager2.get_instruments_info(category="linear", symbol=symbol)['result']['list'][0]['quoteCoin'] 389 | return self.asset_symbol[symbol] 390 | except: 391 | logger.exception(f"Failed to retrieve quoteCoin for symbol {symbol}, falling back to USDT") 392 | return 'USDT' 393 | -------------------------------------------------------------------------------- /scraper_root/scraper/data_classes.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | from dataclasses import dataclass, field 4 | from enum import Enum 5 | from typing import List 6 | 7 | 8 | class Timeframe(Enum): 9 | ONE_MINUTE = '1m', int(60 * 1000) 10 | THREE_MINUTES = '3m', int(3 * 60 * 1000) 11 | FIVE_MINUTES = '5m', int(5 * 60 * 1000) 12 | FIFTEEN_MINUTES = '15m', int(15 * 60 * 1000) 13 | THIRTY_MINUTES = '30m', int(30 * 60 * 1000) 14 | ONE_HOUR = '1h', int(1 * 60 * 60 * 1000) 15 | TWO_HOURS = '2h', int(2 * 60 * 60 * 1000) 16 | FOUR_HOURS = '4h', int(4 * 60 * 60 * 1000) 17 | SIX_HOURS = '6h', int(6 * 60 * 60 * 1000) 18 | EIGHT_HOURS = '8h', int(8 * 60 * 60 * 1000) 19 | TWELVE_HOURS = '12h', int(12 * 60 * 60 * 1000) 20 | ONE_DAY = '1d', int(1 * 24 * 60 * 60 * 1000) 21 | THREE_DAYS = '3d', int(3 * 24 * 60 * 60 * 1000) 22 | ONE_WEEK = '1w', int(7 * 24 * 60 * 60 * 1000) 23 | ONE_MONTH = '1M', int( 24 | 31 * 24 * 60 * 60 * 1000) # TODO: implementation of 1 month is more complicated than this, assuming 31 days in a month here 25 | 26 | @property 27 | def code(self): 28 | return self.value[0] 29 | 30 | @property 31 | def milliseconds(self): 32 | return self.value[1] 33 | 34 | 35 | class OrderStatus(Enum): 36 | NEW = 'NEW' 37 | PARTIALLY_FILLED = 'PARTIALLY_FILLED' 38 | FILLED = 'FILLED' 39 | CANCELED = 'CANCELED' 40 | PENDING_CANCEL = 'PENDING_CANCEL' 41 | REJECTED = 'REJECTED' 42 | EXPIRED = 'EXPIRED' 43 | 44 | 45 | @dataclass 46 | class SymbolInformation: 47 | minimum_quantity: float = 0.0 48 | maximum_quantity: float = 0.0 49 | quantity_step: float = 0.0 50 | price_step: float = 0.0 51 | minimum_price: float = 0.0 52 | maximum_price: float = 0.0 53 | minimal_cost: float = 0.0 54 | asset: str = '' 55 | symbol: str = '' 56 | 57 | 58 | @dataclass 59 | class Position: 60 | symbol: str 61 | entry_price: float 62 | position_size: float 63 | unrealizedProfit: float 64 | side: str 65 | initial_margin: float 66 | market_price: float 67 | 68 | 69 | @dataclass 70 | class Income: 71 | symbol: str 72 | asset: str 73 | type: str 74 | income: float 75 | timestamp: float 76 | transaction_id: int 77 | 78 | 79 | @dataclass 80 | class Trade: 81 | symbol: str 82 | asset: str 83 | type: str 84 | timestamp: float 85 | order_id: int 86 | quantity: float 87 | price: float 88 | side: str 89 | 90 | 91 | class OrderType(Enum): 92 | LIMIT = 'LIMIT' 93 | TAKE_PROFIT = 'TAKE_PROFIT' 94 | STOP = 'STOP' 95 | TAKE_PROFIT_MARKET = 'TAKE_PROFIT_MARKET' 96 | STOP_MARKET = 'STOP_MARKET' 97 | TRAILING_STOP_MARKET = 'TRAILING_STOP_MARKET' 98 | MARKET = 'MARKET' 99 | LIQUIDATION = 'LIQUIDATION' 100 | 101 | 102 | @dataclass 103 | class Order: 104 | symbol: str = None 105 | quantity: float = None 106 | side: str = None 107 | position_side: str = None 108 | status: OrderStatus = None 109 | type: OrderType = None 110 | price: float = None 111 | 112 | 113 | @dataclass 114 | class AssetBalance: 115 | asset: str 116 | balance: float 117 | unrealizedProfit: float 118 | 119 | 120 | @dataclass 121 | class Balance: 122 | totalBalance: float 123 | totalUnrealizedProfit: float 124 | assets: List[AssetBalance] = field(default_factory=lambda: []) 125 | 126 | 127 | @dataclass 128 | class Tick: 129 | symbol: str 130 | price: float 131 | qty: float 132 | timestamp: int 133 | 134 | 135 | @dataclass 136 | class Account: 137 | alias: str = '' 138 | api_key: str = '' 139 | api_secret: str = '' 140 | api_passphrase: str = '' 141 | exchange: str = '' 142 | unified: bool = False 143 | 144 | 145 | @dataclass 146 | class ScraperConfig: 147 | accounts: List[Account] = field(default_factory=lambda: []) 148 | symbols: List[str] = field(default_factory=lambda: []) 149 | -------------------------------------------------------------------------------- /scraper_root/scraper/kucoinfutures.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import logging 4 | import threading 5 | import time 6 | from typing import List 7 | 8 | from kucoin_futures.client import Market, User, Trade 9 | 10 | from scraper_root.scraper.data_classes import AssetBalance, Position, Tick, Balance, \ 11 | Order, Account, Income 12 | from scraper_root.scraper.persistence.repository import Repository 13 | 14 | logger = logging.getLogger() 15 | 16 | 17 | def is_asset_usd_or_derivative(asset: str): 18 | return asset.lower() in ["usdt", "busd", "usd", "usdc"] 19 | 20 | 21 | class KucoinFutures: 22 | def __init__(self, account: Account, symbols: List[str], repository: Repository): 23 | logger.info('Kucoin initialized') 24 | self.account = account 25 | self.symbols = ['XBTUSDTM'] 26 | self.api_key = self.account.api_key 27 | self.secret = self.account.api_secret 28 | self.passphrase = self.account.api_passphrase 29 | self.repository = repository 30 | self.market = Market() 31 | self.user = User(key=self.api_key, secret=self.secret, passphrase=self.passphrase) 32 | self.trade = Trade(key=self.api_key, secret=self.secret, passphrase=self.passphrase) 33 | 34 | self.tick_symbols = [] 35 | 36 | def start(self): 37 | logger.info('Starting binance futures scraper') 38 | 39 | for symbol in self.symbols: 40 | symbol_trade_thread = threading.Thread( 41 | name=f'trade_thread_{symbol}', target=self.process_trades, args=(symbol,), daemon=True) 42 | symbol_trade_thread.start() 43 | 44 | sync_balance_thread = threading.Thread( 45 | name=f'sync_balance_thread', target=self.sync_account, daemon=True) 46 | sync_balance_thread.start() 47 | 48 | sync_trades_thread = threading.Thread( 49 | name=f'sync_trades_thread', target=self.sync_trades, daemon=True) 50 | sync_trades_thread.start() 51 | 52 | sync_orders_thread = threading.Thread( 53 | name=f'sync_orders_thread', target=self.sync_open_orders, daemon=True) 54 | sync_orders_thread.start() 55 | 56 | def sync_trades(self): 57 | max_fetches_in_cycle = 3 58 | first_trade_reached = False 59 | while True: 60 | try: 61 | counter = 0 62 | while first_trade_reached is False and counter < max_fetches_in_cycle: 63 | counter += 1 64 | oldest_income = self.repository.get_oldest_income(account=self.account.alias) 65 | if oldest_income is None: 66 | # API will return inclusive, don't want to return the oldest record again 67 | oldest_timestamp = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000) 68 | else: 69 | oldest_timestamp = oldest_income.timestamp 70 | logger.warning(f'Synced trades before {oldest_timestamp}') 71 | 72 | exchange_incomes = self.user.get_transaction_history(type='RealisedPNL', endAt=oldest_timestamp - 1, maxCount=1000)['dataList'] 73 | logger.info(f"Length of older trades fetched up to {oldest_timestamp}: {len(exchange_incomes)}") 74 | incomes = [] 75 | for exchange_income in exchange_incomes: 76 | if not is_asset_usd_or_derivative(exchange_income['currency']): 77 | exchange_income['amount'] = self.income_to_usdt( 78 | float(exchange_income['amount']), 79 | int(exchange_income['time']), 80 | exchange_income['currency']) 81 | exchange_income['currency'] = "USDT" 82 | 83 | income = Income(symbol=exchange_income['remark'], 84 | asset=exchange_income['currency'], 85 | type=exchange_income['type'], 86 | income=float(exchange_income['amount']), 87 | timestamp=exchange_income['time'], 88 | transaction_id=exchange_income['offset']) 89 | incomes.append(income) 90 | self.repository.process_incomes(incomes, account=self.account.alias) 91 | if len(exchange_incomes) < 1: 92 | first_trade_reached = True 93 | 94 | # WARNING: don't use forward-walking only, because binance only returns max 7 days when using forward-walking 95 | # If this logic is ever changed, make sure that it's still able to retrieve all the account history 96 | newest_trade_reached = False 97 | while newest_trade_reached is False and counter < max_fetches_in_cycle: 98 | counter += 1 99 | newest_income = self.repository.get_newest_income(account=self.account.alias) 100 | if newest_income is None: 101 | # Binance started in September 2017, so no trade can be before that 102 | newest_timestamp = int( 103 | datetime.datetime.fromisoformat('2017-09-01 00:00:00+00:00').timestamp() * 1000) 104 | else: 105 | newest_timestamp = newest_income.timestamp 106 | logger.warning(f'Synced newer trades since {newest_timestamp}') 107 | 108 | exchange_incomes = self.user.get_transaction_history(type='RealisedPNL', startAt=newest_timestamp + 1, maxCount=1000)['dataList'] 109 | logger.info(f"Length of newer trades fetched from {newest_timestamp}: {len(exchange_incomes)}") 110 | incomes = [] 111 | for exchange_income in exchange_incomes: 112 | if not is_asset_usd_or_derivative(exchange_income['currency']): 113 | exchange_income['amount'] = self.income_to_usdt( 114 | float(exchange_income['amount']), 115 | int(exchange_income['time']), 116 | exchange_income['currency']) 117 | exchange_income['currency'] = "USDT" 118 | 119 | income = Income(symbol=exchange_income['remark'], 120 | asset=exchange_income['currency'], 121 | type=exchange_income['type'], 122 | income=float(exchange_income['amount']), 123 | timestamp=exchange_income['time'], 124 | transaction_id=exchange_income['offset']) 125 | incomes.append(income) 126 | self.repository.process_incomes(incomes, account=self.account.alias) 127 | if len(exchange_incomes) < 1: 128 | newest_trade_reached = True 129 | 130 | logger.warning('Synced trades') 131 | except Exception as e: 132 | logger.error(f'{self.account.alias} Failed to process trades: {e}') 133 | 134 | time.sleep(60) 135 | 136 | def income_to_usdt(self, income: float, income_timestamp: int, asset: str) -> float: 137 | if is_asset_usd_or_derivative(asset): 138 | return income 139 | 140 | # Can't get the latest aggr_trades on just the endTime, so this is 'best effort' 141 | symbol = f"{asset}USDT" 142 | start_time = int(income_timestamp) - 1000 143 | candles = self.market.get_kline_data(symbol=symbol, granularity=1, begin_t=start_time, end_t=start_time + 61_000) 144 | close_price = candles[-1][4] 145 | income *= float(close_price) 146 | 147 | return income 148 | 149 | def sync_account(self): 150 | while True: 151 | try: 152 | account = self.user.get_account_overview(currency='USDT') 153 | usd_assets = [AssetBalance(asset=account['currency'], 154 | balance=float(account['marginBalance']), 155 | unrealizedProfit=float(account['unrealisedPNL']))] 156 | 157 | total_wallet_balance = sum([asset.balance for asset in usd_assets]) 158 | total_upnl = sum([asset.unrealizedProfit for asset in usd_assets]) 159 | 160 | logger.info(f'Wallet balance: {total_wallet_balance}, upnl: {total_upnl}') 161 | 162 | balance = Balance(totalBalance=total_wallet_balance, 163 | totalUnrealizedProfit=total_upnl, 164 | assets=usd_assets) 165 | self.repository.process_balances(balance, account=self.account.alias) 166 | 167 | positions = [] 168 | positions_response = self.trade.get_all_position() 169 | for position in positions_response: 170 | quantity = float(position['currentQty']) 171 | position_side = 'LONG' 172 | if quantity < 0: 173 | position_side = 'SHORT' 174 | positions.append(Position(symbol=position['symbol'], 175 | entry_price=float(position['avgEntryPrice']), 176 | position_size=quantity, 177 | side=position_side, 178 | unrealizedProfit=float(position['unrealisedPnl']), 179 | initial_margin=float(position['posMargin']) 180 | )) 181 | self.repository.process_positions(positions, account=self.account.alias) 182 | 183 | for position_symbol in [position.symbol for position in positions]: 184 | mark_price = self.market.get_current_mark_price(symbol=position_symbol) 185 | logger.debug(mark_price) 186 | tick = Tick(symbol=mark_price['symbol'], 187 | price=float(mark_price['value']), 188 | qty=float('0'), 189 | timestamp=int(mark_price['timePoint'])) 190 | logger.debug(f"Processed tick for {tick.symbol}") 191 | self.repository.process_tick(tick, account=self.account.alias) 192 | # [self.add_to_ticker(position.symbol) for position in positions if position.position_size > 0.0] 193 | logger.warning('Synced account') 194 | except Exception as e: 195 | logger.error(f'{self.account.alias} Failed to process balance: {e}') 196 | time.sleep(20) 197 | 198 | def sync_open_orders(self): 199 | while True: 200 | orders = [] 201 | try: 202 | open_orders = self.trade.get_order_list(pageSize=1000, status='active') 203 | contract_list = self.market.get_contracts_list() 204 | price_per_symbol = {} 205 | for contract in contract_list: 206 | price_per_symbol[contract['symbol']] = contract['lastTradePrice'] 207 | # go through all pages 208 | for open_order in open_orders['items']: 209 | order = Order() 210 | order.symbol = open_order['symbol'] 211 | order.price = float(open_order['price']) 212 | order.quantity = float(open_order['size']) 213 | order.side = str(open_order['side']).upper() 214 | current_price = price_per_symbol[order.symbol] 215 | if order.price < current_price and order.side == 'BUY': 216 | order.position_side = 'LONG' 217 | elif order.price < current_price and order.side == 'SELL': 218 | order.position_side = 'SHORT' 219 | elif order.price > current_price and order.side == 'BUY': 220 | order.position_side = 'SHORT' 221 | elif order.price > current_price and order.side == 'SELL': 222 | order.position_side = 'LONG' 223 | else: 224 | logger.error(f'{self.account.alias}: Failed to identify positionside for order {open_order}') 225 | 226 | order.type = open_order['type'] 227 | orders.append(order) 228 | self.repository.process_orders(orders, account=self.account.alias) 229 | logger.warning(f'Synced orders') 230 | except: 231 | logger.error(f'{self.account.alias} Failed to process open orders') 232 | 233 | time.sleep(30) 234 | 235 | # def add_to_ticker(self, symbol: str): 236 | # if symbol not in self.tick_symbols: 237 | # symbol_trade_thread = threading.Thread( 238 | # name=f'trade_thread_{symbol}', target=self.process_trades, args=(symbol,), daemon=True) 239 | # symbol_trade_thread.start() 240 | 241 | def process_trades(self, symbol: str): 242 | if symbol in self.tick_symbols: 243 | logger.error(f'Already listening to ticks for {symbol}, not starting new processing!') 244 | return 245 | self.tick_symbols.append(symbol) 246 | 247 | logger.info(f"Trade stream started for {symbol}") 248 | while True: 249 | try: 250 | mark_price = self.market.get_current_mark_price(symbol=symbol) 251 | logger.debug(mark_price) 252 | tick_symbol = 'BTCUSDT' if symbol == 'XBTUSDTM' else mark_price['symbol'] 253 | tick = Tick(symbol=tick_symbol, 254 | price=float(mark_price['value']), 255 | qty=float('0'), 256 | timestamp=int(mark_price['timePoint'])) 257 | logger.debug(f"Processed tick for {tick.symbol}") 258 | self.repository.process_tick(tick, account=self.account.alias) 259 | except Exception as e: 260 | logger.warning(f'Error processing tick: {e}') 261 | # Price update every 5 seconds is fast enough 262 | time.sleep(5) 263 | -------------------------------------------------------------------------------- /scraper_root/scraper/persistence/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HawkeyeBot/exchanges_dashboard/4fcdf8a3ba0aaa0668eee7b96f8e567c1b3710ab/scraper_root/scraper/persistence/__init__.py -------------------------------------------------------------------------------- /scraper_root/scraper/persistence/lockable_session.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | from sqlalchemy.orm import sessionmaker 4 | 5 | 6 | class LockableSession: 7 | def __init__(self, engine): 8 | self.lock = threading.RLock() 9 | self.session_maker = sessionmaker() 10 | self.session_maker.configure(bind=engine) 11 | self.session = None 12 | 13 | def __enter__(self): 14 | self.lock.acquire() 15 | self.session = self.session_maker() 16 | return self.session 17 | 18 | def __exit__(self, type, value, traceback): 19 | try: 20 | self.session.close() 21 | except: 22 | pass 23 | finally: 24 | self.session = None 25 | self.lock.release() 26 | -------------------------------------------------------------------------------- /scraper_root/scraper/persistence/orm_classes.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import UniqueConstraint, PrimaryKeyConstraint, Column, Integer, DateTime, func, String, Float, Boolean, \ 2 | ForeignKey 3 | from sqlalchemy.ext.declarative import declarative_base 4 | from sqlalchemy.orm import relationship 5 | from sqlalchemy.sql.schema import UniqueConstraint 6 | 7 | _DECL_BASE = declarative_base() 8 | 9 | 10 | class OrderEntity(_DECL_BASE): 11 | __tablename__ = 'ORDERS' 12 | id = Column(Integer, primary_key=True) 13 | order_id = Column(Integer) 14 | registration_datetime = Column(DateTime, default=func.now()) 15 | type = Column(String) 16 | symbol = Column(String) 17 | quantity = Column(Float) 18 | side = Column(String) 19 | position_side = Column(String) 20 | status = Column(String) 21 | price = Column(Float) 22 | stop_price = Column(Float) 23 | timeInForce = Column(String) 24 | activation_price = Column(Float) 25 | callback_rate = Column(Float) 26 | close_position = Column(Boolean) 27 | account = Column(String) 28 | 29 | 30 | class DailyBalanceEntity(_DECL_BASE): 31 | __tablename__ = 'DAILY_BALANCE' 32 | id = Column(Integer, primary_key=True) 33 | registration_datetime = Column(DateTime, default=func.now()) 34 | day = Column(DateTime) 35 | totalWalletBalance = Column(Float) 36 | account = Column(String) 37 | 38 | 39 | class BalanceEntity(_DECL_BASE): 40 | __tablename__ = 'BALANCE' 41 | id = Column(Integer, primary_key=True) 42 | registration_datetime = Column(DateTime, default=func.now()) 43 | totalWalletBalance = Column(Float) 44 | totalUnrealizedProfit = Column(Float) 45 | totalEquity = Column(Float) 46 | account = Column(String) 47 | assets = relationship("AssetBalanceEntity", 48 | back_populates="balance", cascade="all, delete") 49 | 50 | 51 | class AssetBalanceEntity(_DECL_BASE): 52 | __tablename__ = 'ASSET_BALANCE' 53 | id = Column(Integer, primary_key=True) 54 | registration_datetime = Column(DateTime, default=func.now()) 55 | asset = Column(String) 56 | walletBalance = Column(Float) 57 | unrealizedProfit = Column(Float) 58 | balance_id = Column(Integer, ForeignKey('BALANCE.id')) 59 | balance = relationship("BalanceEntity", back_populates="assets") 60 | account = Column(String) 61 | 62 | 63 | class PositionEntity(_DECL_BASE): 64 | __tablename__ = 'POSITION' 65 | id = Column(Integer, primary_key=True) 66 | registration_datetime = Column(DateTime, default=func.now()) 67 | symbol = Column(String) 68 | side = Column(String) 69 | unrealizedProfit = Column(Float) 70 | entryPrice = Column(Float) 71 | quantity = Column(Float) 72 | initialMargin = Column(Float) 73 | account = Column(String) 74 | 75 | class PositionHistoryEntity(_DECL_BASE): 76 | __tablename__ = 'POSITION_HISTORY' 77 | id = Column(Integer, primary_key=True) 78 | registration_datetime = Column(DateTime, default=func.now()) 79 | symbol = Column(String) 80 | side = Column(String) 81 | unrealizedProfit = Column(Float) 82 | market_price = Column(Float) 83 | entryPrice = Column(Float) 84 | quantity = Column(Float) 85 | initialMargin = Column(Float) 86 | account = Column(String) 87 | balance_id = Column(Integer, ForeignKey('BALANCE.id')) 88 | 89 | 90 | class CurrentPriceEntity(_DECL_BASE): 91 | __tablename__ = 'PRICE' 92 | id = Column(Integer, primary_key=True) 93 | registration_datetime = Column(DateTime, default=func.now()) 94 | symbol = Column(String) 95 | price = Column(Float) 96 | account = Column(String) 97 | 98 | 99 | class IncomeEntity(_DECL_BASE): 100 | __tablename__ = 'INCOME' 101 | id = Column(Integer, primary_key=True) 102 | registration_datetime = Column(DateTime, default=func.now()) 103 | transaction_id = Column(Integer, nullable=False, 104 | unique=True, sqlite_on_conflict_unique='IGNORE') 105 | symbol = Column(String) 106 | incomeType = Column(String) 107 | income = Column(Float) 108 | asset = Column(String) 109 | time = Column(DateTime) 110 | timestamp = Column(Integer) 111 | account = Column(String) 112 | 113 | __table_args__ = ( 114 | (UniqueConstraint('transaction_id', sqlite_on_conflict='IGNORE')), 115 | ) 116 | 117 | 118 | class TradeEntity(_DECL_BASE): 119 | __tablename__ = 'Trade' 120 | id = Column(Integer, primary_key=True) 121 | registration_datetime = Column(DateTime, default=func.now()) 122 | order_id = Column(Integer, nullable=False, unique=True, sqlite_on_conflict_unique='IGNORE') 123 | symbol = Column(String) 124 | incomeType = Column(String) 125 | asset = Column(String) 126 | quantity = Column(Float) 127 | price = Column(Float) 128 | side = Column(String) 129 | time = Column(DateTime) 130 | timestamp = Column(Integer) 131 | account = Column(String) 132 | 133 | __table_args__ = ( 134 | (UniqueConstraint('order_id', sqlite_on_conflict='IGNORE')), 135 | ) 136 | 137 | 138 | 139 | class TradedSymbolEntity(_DECL_BASE): 140 | __tablename__ = 'TRADED_SYMBOL' 141 | id = Column(Integer, primary_key=True) 142 | registration_datetime = Column(DateTime, default=func.now()) 143 | symbol = Column(String, unique=True, nullable=False) 144 | last_trades_downloaded = Column(DateTime) 145 | account = Column(String) 146 | 147 | 148 | class SymbolCheckEntity(_DECL_BASE): 149 | __tablename__ = 'CHECKED_SYMBOL' 150 | id = Column(Integer, primary_key=True) 151 | registration_datetime = Column(DateTime, default=func.now()) 152 | symbol = Column(String, unique=True, nullable=False, default=func.now()) 153 | last_checked_datetime = Column(DateTime, default=func.now()) 154 | account = Column(String) 155 | -------------------------------------------------------------------------------- /scraper_root/scraper/persistence/repository.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import threading 4 | import time 5 | from datetime import datetime, date, timedelta, timezone 6 | from typing import List, Dict 7 | 8 | from sqlalchemy import create_engine, func, Table, asc, nulls_first 9 | from sqlalchemy.orm import sessionmaker 10 | 11 | from scraper_root.scraper.data_classes import Order, Tick, Position, Balance, Income, Trade 12 | from scraper_root.scraper.persistence.lockable_session import LockableSession 13 | from scraper_root.scraper.persistence.orm_classes import _DECL_BASE, CurrentPriceEntity, \ 14 | BalanceEntity, AssetBalanceEntity, PositionEntity, PositionHistoryEntity, IncomeEntity, OrderEntity, DailyBalanceEntity, TradeEntity, \ 15 | TradedSymbolEntity, SymbolCheckEntity 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class Repository: 21 | def __init__(self, accounts: List[str]): 22 | self.engine = create_engine(url=os.getenv( 23 | 'DATABASE_PATH', 'sqlite:///data/exchanges_db.sqlite'), echo=False) 24 | _DECL_BASE.metadata.create_all(self.engine) 25 | 26 | self.lockable_session = LockableSession(self.engine) 27 | self.accounts: List[str] = accounts 28 | 29 | update_daily_balance_thread = threading.Thread( 30 | name=f'sync_balance_thread', target=self.update_daily_balance, args=(accounts, ), daemon=True) 31 | update_daily_balance_thread.start() 32 | 33 | def update_daily_balance(self, accounts: List[str]): 34 | while True: 35 | with self.lockable_session as session: 36 | try: 37 | for account in accounts: 38 | session.query(DailyBalanceEntity).filter(DailyBalanceEntity.account == account).delete() 39 | session.commit() 40 | result = session.query(BalanceEntity.totalWalletBalance) \ 41 | .order_by(BalanceEntity.registration_datetime.desc()) \ 42 | .filter(BalanceEntity.account == account).first() 43 | current_balance = 0 44 | if result is not None: 45 | current_balance = result[0] 46 | 47 | daily_balances = [] 48 | with self.engine.connect() as con: 49 | oldest_income = self.get_oldest_income(account=account) 50 | if oldest_income is not None: 51 | day = oldest_income.time.date() 52 | end_date = date.today() 53 | while day <= end_date: 54 | rs = con.execute( 55 | f'SELECT sum("INCOME"."income") AS "sum" FROM "INCOME" ' 56 | f'WHERE "INCOME"."time" >= date(\'{day.strftime("%Y-%m-%d")}\') ' 57 | f'AND "Income"."account" = \'{account}\'') 58 | for row in rs: 59 | if row[0] is None: 60 | income = 0 61 | else: 62 | income = float(row[0]) 63 | daily_balance = DailyBalanceEntity() 64 | daily_balance.day = day 65 | daily_balance.totalWalletBalance = current_balance - income 66 | daily_balance.account = account 67 | daily_balances.append(daily_balance) 68 | 69 | day += timedelta(days=1) 70 | 71 | [session.add(balance) for balance in daily_balances] 72 | session.commit() 73 | logger.info('Updated daily balances') 74 | except Exception as e: 75 | logger.error(f'Failed to update daily balance: {e}') 76 | 77 | time.sleep(60) 78 | 79 | def process_order_update(self, order: Order): 80 | # if order is already in the database, update it 81 | # if order is not in the database, insert it 82 | pass 83 | 84 | def process_tick(self, tick: Tick, account: str): 85 | with self.lockable_session as session: 86 | # find entry in database 87 | query = session.query(CurrentPriceEntity) 88 | query = query.filter(CurrentPriceEntity.symbol == tick.symbol).filter(CurrentPriceEntity.account == account) 89 | currentEntity = query.first() 90 | if currentEntity is None: 91 | currentEntity = CurrentPriceEntity() 92 | session.add(currentEntity) 93 | 94 | currentEntity.symbol = tick.symbol 95 | currentEntity.price = tick.price 96 | currentEntity.registration_datetime = func.now() 97 | currentEntity.account = account 98 | session.commit() 99 | logger.debug(f'Tick processed: {tick}') 100 | 101 | def get_current_price(self, symbol: str, account: str) -> CurrentPriceEntity: 102 | with self.lockable_session as session: 103 | return session.query(CurrentPriceEntity).filter(CurrentPriceEntity.symbol == symbol) \ 104 | .filter(CurrentPriceEntity.account == account).first() 105 | 106 | def process_balances(self, balance: Balance, account: str): 107 | with self.lockable_session as session: 108 | logger.debug('Updating balances') 109 | session.query(AssetBalanceEntity).filter(AssetBalanceEntity.account == account).delete() 110 | session.query(BalanceEntity).filter(BalanceEntity.account == account, BalanceEntity.registration_datetime < datetime.now() - timedelta(days=90)).delete() 111 | session.commit() 112 | 113 | balanceEntity = BalanceEntity() 114 | balanceEntity.totalWalletBalance = balance.totalBalance 115 | balanceEntity.totalUnrealizedProfit = balance.totalUnrealizedProfit 116 | balanceEntity.totalEquity = float(balance.totalBalance) + float(balance.totalUnrealizedProfit) 117 | balanceEntity.account = account 118 | 119 | asset_balance_entities = [] 120 | for asset in balance.assets: 121 | asset_balance_entity = AssetBalanceEntity() 122 | asset_balance_entity.asset = asset.asset 123 | asset_balance_entity.walletBalance = asset.balance 124 | asset_balance_entity.unrealizedProfit = asset.unrealizedProfit 125 | asset_balance_entity.account = account 126 | asset_balance_entities.append(asset_balance_entity) 127 | balanceEntity.assets = asset_balance_entities 128 | session.add(balanceEntity) 129 | session.commit() 130 | 131 | def process_positions(self, positions: List[Position], account: str): 132 | with self.lockable_session as session: 133 | logger.debug('Updating positions') 134 | balance = session.query(BalanceEntity).filter(BalanceEntity.account == account) \ 135 | .order_by(BalanceEntity.registration_datetime.desc()).first() 136 | session.query(PositionEntity).filter(PositionEntity.account == account).delete() 137 | 138 | for position in positions: 139 | position_entity = PositionEntity() 140 | position_entity.symbol = position.symbol 141 | position_entity.side = position.side 142 | position_entity.quantity = position.position_size 143 | position_entity.entryPrice = position.entry_price 144 | position_entity.unrealizedProfit = position.unrealizedProfit 145 | position_entity.initialMargin = position.initial_margin 146 | position_entity.account = account 147 | session.add(position_entity) 148 | 149 | for position in positions: 150 | position_history_entity = PositionHistoryEntity() 151 | position_history_entity.symbol = position.symbol 152 | position_history_entity.side = position.side 153 | position_history_entity.quantity = position.position_size 154 | position_history_entity.market_price = position.market_price 155 | position_history_entity.entryPrice = position.entry_price 156 | position_history_entity.unrealizedProfit = position.unrealizedProfit 157 | position_history_entity.initialMargin = position.initial_margin 158 | position_history_entity.account = account 159 | position_history_entity.balance_id = balance.id 160 | session.add(position_history_entity) 161 | 162 | session.commit() 163 | 164 | def get_oldest_trade(self, symbol: str, account: str) -> TradeEntity: 165 | with self.lockable_session as session: 166 | logger.debug('Getting oldest trade') 167 | result = session.query(TradeEntity).filter(TradeEntity.symbol == symbol) \ 168 | .filter(TradeEntity.account == account).order_by(TradeEntity.time.asc()).first() 169 | return result 170 | 171 | def get_trades(self, symbol: str, account: str) -> List[TradeEntity]: 172 | with self.lockable_session as session: 173 | logger.debug(f'Getting all trades for {symbol}') 174 | result = session.query(TradeEntity).filter(TradeEntity.symbol == symbol) \ 175 | .filter(TradeEntity.account == account).all() 176 | return result 177 | 178 | def get_trades_by_asset(self, asset: str, account: str) -> TradeEntity: 179 | with self.lockable_session as session: 180 | logger.debug(f'Getting all trades for asset {asset}') 181 | result = session.query(TradeEntity).filter(TradeEntity.symbol.like(f'{asset}%')) \ 182 | .filter(TradeEntity.account == account).all() 183 | return result 184 | 185 | def get_newest_trade(self, symbol: str, account: str) -> TradeEntity: 186 | with self.lockable_session as session: 187 | logger.debug('Getting newest trade') 188 | result = session.query(TradeEntity).filter(TradeEntity.symbol == symbol) \ 189 | .filter(TradeEntity.account == account).order_by(TradeEntity.time.desc()).first() 190 | return result 191 | 192 | def get_oldest_income(self, account: str) -> IncomeEntity: 193 | with self.lockable_session as session: 194 | logger.debug('Getting oldest income') 195 | result = session.query(IncomeEntity).filter(IncomeEntity.account == account).order_by( 196 | IncomeEntity.time.asc()).first() 197 | return result 198 | 199 | def get_newest_income(self, account: str) -> IncomeEntity: 200 | with self.lockable_session as session: 201 | logger.debug('Getting newest income') 202 | result = session.query(IncomeEntity).filter(IncomeEntity.account == account).order_by( 203 | IncomeEntity.time.desc()).first() 204 | return result 205 | 206 | def process_incomes(self, incomes: List[Income], account: str): 207 | if len(incomes) == 0: 208 | return 209 | with self.lockable_session as session: 210 | logger.warning(f'{account}: Processing incomes') 211 | 212 | session.execute( 213 | IncomeEntity.__table__.insert(), 214 | params=[{ 215 | "transaction_id": income.transaction_id, 216 | "symbol": income.symbol, 217 | "incomeType": income.type, 218 | "income": income.income, 219 | "asset": income.asset, 220 | "time": datetime.utcfromtimestamp(income.timestamp / 1000), 221 | "timestamp": income.timestamp, 222 | "account": account} 223 | for income in incomes], 224 | ) 225 | session.commit() 226 | 227 | def process_trades(self, trades: List[Trade], account: str): 228 | if len(trades) == 0: 229 | return 230 | with self.lockable_session as session: 231 | logger.warning('Processing trades') 232 | 233 | session.execute( 234 | TradeEntity.__table__.insert(), 235 | params=[{ 236 | "order_id": trade.order_id, 237 | "symbol": trade.symbol, 238 | "incomeType": trade.type, 239 | "asset": trade.asset, 240 | "quantity": trade.quantity, 241 | "price": trade.price, 242 | "side": trade.side, 243 | "time": datetime.utcfromtimestamp(trade.timestamp / 1000), 244 | "timestamp": trade.timestamp, 245 | "account": account} 246 | for trade in trades], 247 | ) 248 | session.commit() 249 | 250 | def process_orders(self, orders: List[Order], account: str): 251 | with self.lockable_session as session: 252 | logger.debug('Processing orders') 253 | session.query(OrderEntity).filter(OrderEntity.account == account).delete() 254 | 255 | for order in orders: 256 | order_entity = OrderEntity() 257 | order_entity.symbol = order.symbol 258 | order_entity.price = order.price 259 | order_entity.type = order.type 260 | order_entity.quantity = order.quantity 261 | order_entity.position_side = order.position_side 262 | order_entity.side = order.side 263 | order_entity.status = order.status 264 | order_entity.account = account 265 | session.add(order_entity) 266 | session.commit() 267 | 268 | def get_open_orders(self, symbol: str, account: str) -> List[OrderEntity]: 269 | with self.lockable_session as session: 270 | logger.debug(f'Getting orders for {symbol}') 271 | return session.query(OrderEntity).filter(OrderEntity.symbol == symbol)\ 272 | .filter(OrderEntity.account == account).all() 273 | 274 | def is_symbol_traded(self, symbol: str, account: str) -> bool: 275 | with self.lockable_session as session: 276 | logger.debug('Getting traded symbol') 277 | query = session.query(TradedSymbolEntity).filter(TradedSymbolEntity.symbol == symbol)\ 278 | .filter(TradedSymbolEntity.account == account) 279 | return session.query(query.exists()).scalar() 280 | 281 | def get_all_traded_symbols(self, account: str) -> List[str]: 282 | with self.lockable_session as session: 283 | logger.debug('Getting all traded symbol') 284 | return [e.symbol for e in session.query(TradedSymbolEntity).filter(TradedSymbolEntity.account == account).all()] 285 | 286 | def get_traded_symbol(self, symbol: str, account: str) -> TradedSymbolEntity: 287 | with self.lockable_session as session: 288 | logger.debug(f'Getting traded symbol for {symbol}') 289 | return session.query(TradedSymbolEntity).filter(TradedSymbolEntity.symbol == symbol)\ 290 | .filter(TradedSymbolEntity.account == account).first() 291 | 292 | def process_traded_symbol(self, symbol: str, account: str): 293 | with self.lockable_session as session: 294 | logger.debug('Processing traded symbol') 295 | traded_symbol_entity = TradedSymbolEntity() 296 | traded_symbol_entity.symbol = symbol 297 | traded_symbol_entity.account = account 298 | session.add(traded_symbol_entity) 299 | session.commit() 300 | 301 | def update_trades_last_downloaded(self, symbol: str, account: str): 302 | with self.lockable_session as session: 303 | logger.debug('Updating trades last downloaded') 304 | traded_symbol = session.query(TradedSymbolEntity).filter(TradedSymbolEntity.symbol == symbol)\ 305 | .filter(TradedSymbolEntity.account == account).first() 306 | traded_symbol.last_trades_downloaded = datetime.now() 307 | session.commit() 308 | 309 | def get_symbol_checks(self, account: str) -> List[SymbolCheckEntity]: 310 | with self.lockable_session as session: 311 | logger.debug('Getting all symbol checks') 312 | return [e.symbol for e in session.query(SymbolCheckEntity).filter(SymbolCheckEntity.account == account).all()] 313 | 314 | def process_symbol_checked(self, symbol: str, account: str): 315 | with self.lockable_session as session: 316 | existing_symbol_check = session.query(SymbolCheckEntity).filter(SymbolCheckEntity.symbol == symbol)\ 317 | .filter(SymbolCheckEntity.account == account).first() 318 | if existing_symbol_check is None: 319 | existing_symbol_check = SymbolCheckEntity() 320 | existing_symbol_check.symbol = symbol 321 | existing_symbol_check.account = account 322 | session.add(existing_symbol_check) 323 | existing_symbol_check.last_checked_datetime = datetime.now() 324 | session.commit() 325 | 326 | def get_next_traded_symbol(self, account: str) -> TradedSymbolEntity: 327 | with self.lockable_session as session: 328 | # first check any of the open positions 329 | open_positions = self.open_positions(account=account) 330 | for open_position in open_positions: 331 | position_symbol = open_position.symbol 332 | traded_symbol = self.get_traded_symbol(position_symbol, account=account) 333 | if traded_symbol.last_trades_downloaded < datetime.utcnow() + timedelta(minutes=5): 334 | # open positions are synced at least every 5 minutes 335 | return session.query(TradedSymbolEntity).filter(TradedSymbolEntity.symbol == position_symbol)\ 336 | .filter(TradedSymbolEntity.account == account).first().symbol 337 | next_symbol = session.query(TradedSymbolEntity).filter(TradedSymbolEntity.account == account).order_by( 338 | nulls_first(TradedSymbolEntity.last_trades_downloaded.asc())).first() 339 | if next_symbol is None: 340 | return None 341 | else: 342 | return next_symbol.symbol 343 | 344 | def open_positions(self, account: str) -> List[PositionEntity]: 345 | with self.lockable_session as session: 346 | return session.query(PositionEntity).filter(PositionEntity.account == account).all() 347 | -------------------------------------------------------------------------------- /scraper_root/scraper/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | def from_timestamp(timestamp: int) -> datetime.datetime: 8 | if timestamp > 2000000000: 9 | return datetime.datetime.fromtimestamp(int(timestamp / 1000), datetime.timezone.utc) 10 | else: 11 | return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) 12 | 13 | 14 | def readable(timestamp: int): 15 | if timestamp is None: 16 | return 'None' 17 | try: 18 | human_readable = from_timestamp(timestamp) 19 | except TypeError: 20 | return 'None' 21 | except Exception: 22 | logger.exception(f'Failed to convert timestamp {timestamp} to readable form') 23 | return f'{timestamp} (CHECK EXCEPTION!)' 24 | return f'{human_readable} ({timestamp})' 25 | --------------------------------------------------------------------------------