├── ai ├── __init__.py ├── blueprints │ ├── __init__.py │ ├── junior.py │ ├── blp5m1117.py │ ├── luckyantelope.py │ └── base.py └── blueprint.py ├── core ├── __init__.py ├── bots │ ├── __init__.py │ ├── enums.py │ ├── live.py │ ├── backtest.py │ ├── paper.py │ └── base.py ├── constants.py ├── tradeaction.py ├── wallet.py ├── common.py ├── plot.py ├── report.py └── engine.py ├── dojo ├── __init__.py └── dojo.py ├── lib ├── __init__.py └── indicators │ ├── __init__.py │ ├── epc.py │ ├── ropc.py │ ├── macd.py │ ├── elderray.py │ └── stoploss.py ├── stats ├── __init__.py └── stats.py ├── utils ├── __init__.py ├── fetch2gcp.sh ├── telegrambot.py ├── blueprints2gcp.py ├── postman.py └── walletlense.py ├── backfill ├── __init__.py ├── base.py ├── candles.py └── trades.py ├── exchanges ├── __init__.py ├── bittrex │ ├── __init__.py │ └── bittrexclient.py ├── poloniex │ ├── __init__.py │ └── polo.py ├── base.py └── exchange.py ├── strategies ├── __init__.py ├── ai │ ├── __init__.py │ ├── scikitbase.py │ └── luckyantelope.py ├── enums.py ├── base.py └── ema.py ├── .coveragerc ├── tests └── test.py ├── .coveralls.yml ├── images ├── mosquito.png ├── console_sample.png └── mosquito_plot.png ├── .travis.yml ├── .gitmodules ├── logging.ini ├── requirements.txt ├── .gitignore ├── lense.py ├── dojo.py ├── examples └── exchange.py ├── blueprint.py ├── backfill.py ├── .circleci └── config.yml ├── mosquito.py ├── mosquito.sample.ini ├── README.md └── market_stats.py /ai/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dojo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stats/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /backfill/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /core/bots/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exchanges/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /strategies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ai/blueprints/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/indicators/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /strategies/ai/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exchanges/bittrex/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exchanges/poloniex/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | tests/* -------------------------------------------------------------------------------- /core/constants.py: -------------------------------------------------------------------------------- 1 | 2 | SECONDS_IN_DAY = 86400 3 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | print("some tests to come here") 2 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-pro 2 | repo_token: lsmDGkMqa8uNiNjrvBM0U9Jlf9bVwmV4h -------------------------------------------------------------------------------- /images/mosquito.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miro-ka/mosquito/HEAD/images/mosquito.png -------------------------------------------------------------------------------- /images/console_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miro-ka/mosquito/HEAD/images/console_sample.png -------------------------------------------------------------------------------- /images/mosquito_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miro-ka/mosquito/HEAD/images/mosquito_plot.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | 4 | - "3.8" 5 | - "3.9" 6 | - "3.9-dev" # 3.9 development branch 7 | 8 | before_install: 9 | 10 | # command to install dependencies 11 | install: 12 | 13 | script: 14 | - python3 tests/test.py 15 | 16 | after_success: 17 | coveralls -------------------------------------------------------------------------------- /strategies/enums.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class TradeState(Enum): 5 | """ 6 | Enum class for holding all available trade states 7 | """ 8 | none = 0 9 | buy = 1 10 | buying = 2 11 | bought = 3 12 | sell = 4 13 | selling = 5 14 | sold = 6 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "exchanges/poloniex/python-poloniex"] 2 | path = exchanges/poloniex/python-poloniex 3 | url = https://github.com/s4w3d0ff/python-poloniex.git 4 | [submodule "exchanges/bittrex/python-bittrex"] 5 | path = exchanges/bittrex/python-bittrex 6 | url = https://github.com/miti0/python-bittrex.git 7 | -------------------------------------------------------------------------------- /logging.ini: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=consoleHandler 6 | 7 | [formatters] 8 | keys=simpleFormatter 9 | 10 | [logger_root] 11 | level=DEBUG 12 | handlers=consoleHandler 13 | 14 | [handler_consoleHandler] 15 | class=StreamHandler 16 | level=DEBUG 17 | formatter=simpleFormatter 18 | args=(sys.stdout,) 19 | 20 | [formatter_simpleFormatter] 21 | format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 22 | datefmt= -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi>=2021.5.30 2 | chardet>=4.0.0 3 | charset-normalizer>=2.0.6 4 | ConfigArgParse>=1.5.3 5 | idna>=3.2 6 | numpy>=1.24.3 7 | pandas>=1.3.3 8 | pandas-ta>=0.3.14b0 9 | plotly>=5.3.1 10 | poloniexapi>=0.5.7 11 | pymongo>=3.12.0 12 | python-bittrex>=0.3.0 13 | python-dateutil>=2.8.2 14 | pytz>=2021.3 15 | requests>=2.26.0 16 | retrying>=1.3.3 17 | six>=1.16.0 18 | tenacity>=8.0.1 19 | termcolor>=1.1.0 20 | tzlocal>=3.0 21 | urllib3>=1.26.7 22 | websocket-client>=1.2.1 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Extensions 2 | *.html 3 | *.zip 4 | *.csv 5 | *.prof 6 | .DS_Store 7 | 8 | # Folders 9 | out/ 10 | notebooks/ 11 | __pycache__/ 12 | core/__pycache__/ 13 | core/bots/__pycache__/ 14 | images/mosquito_imgs/ 15 | .idea/ 16 | exchanges/__pycache__/ 17 | exchanges/poloniex/__pycache__/ 18 | strategies/__pycache__/ 19 | notebooks/.ipynb_checkpoints/* 20 | 21 | # Files 22 | config.ini 23 | mosquito.ini 24 | images/rendered_plain_small.png 25 | config.bumblebee.ini 26 | config.mosquito.ini 27 | mosquito_ai.ini 28 | notebooks/.ipynb_checkpoints 29 | -------------------------------------------------------------------------------- /core/bots/enums.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class TradeMode(Enum): 5 | """ 6 | Enum class for holding all available trade modes 7 | """ 8 | backtest = 0 9 | paper = 1 10 | live = 2 11 | 12 | 13 | class BuySellMode(Enum): 14 | """ 15 | Enum class holding buy/sell mode the bot should use 16 | """ 17 | all = 0 # Only 1 currency will be used for trading 18 | fixed = 1 # Currencies will be bought only for given amount 19 | user_defined = 2 # Buy/sell amount is specified by the user 20 | -------------------------------------------------------------------------------- /lib/indicators/epc.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def epc(array, distance=5): 4 | """ 5 | Calculates percentage change between 2 elements between give space distance 6 | close: price numpy.ndarray 7 | distance: space distance check 8 | """ 9 | 10 | dataset_size = array.size 11 | if dataset_size < distance-1: 12 | print('Error in ropc.py: passed not enough data! Required: ' + str(distance) + 13 | ' passed: ' + str(dataset_size)) 14 | return None 15 | 16 | return (array[-1]*100.0 / array[-distance]) - 100.0 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /utils/fetch2gcp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Simple scripts which fetched ticker data, runs blueprint and uploads data to gcp 4 | 5 | DAYS=2 6 | BUCKET=mosquito 7 | BUCKER_DIR=data/ 8 | PAIRS=BTC_ETH 9 | BLEUPRINT=blp5m1117 10 | 11 | 12 | echo fetching ticker data 13 | cd .. 14 | python3 backfill.py --days $DAYS --pairs $PAIRS 15 | 16 | echo generating blueprint 17 | python3 blueprint.py --pairs $PAIRS --days $DAYS --features $BLEUPRINT 18 | 19 | 20 | echo uploading to gcp 21 | cd utils 22 | python3 blueprints2gcp.py --bucket $BUCKET --bucket_dir $BUCKER_DIR 23 | 24 | echo done 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /lense.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | from utils.walletlense import WalletLense 3 | 4 | """ 5 | Lense: Returns actual wallet statistics with simple daily digest (winners / losers) 6 | """ 7 | 8 | 9 | def main(): 10 | lense = WalletLense() 11 | lense.get_stats() 12 | 13 | 14 | if __name__ == "__main__": 15 | arg_parser = configargparse.get_argument_parser() 16 | arg_parser.add('-c', '--config', is_config_file=True, help='config file path', default='mosquito.ini') 17 | arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true') 18 | args = arg_parser.parse_known_args()[0] 19 | 20 | main() 21 | 22 | -------------------------------------------------------------------------------- /core/tradeaction.py: -------------------------------------------------------------------------------- 1 | from strategies.enums import TradeState 2 | from core.bots.enums import BuySellMode 3 | 4 | 5 | class TradeAction: 6 | """ 7 | Trade Action class 8 | """ 9 | 10 | def __init__(self, pair, 11 | action=TradeState.none, 12 | amount=0.0, 13 | rate=0.0, 14 | buy_sell_mode=BuySellMode.all): 15 | """ 16 | Definition of Trace Action 17 | """ 18 | 19 | self.pair = pair 20 | self.action = action 21 | self.amount = amount 22 | self.rate = rate 23 | self.buy_sell_mode = buy_sell_mode 24 | self.order_number = None 25 | 26 | -------------------------------------------------------------------------------- /dojo.py: -------------------------------------------------------------------------------- 1 | import time 2 | import configargparse 3 | from dojo.dojo import Dojo 4 | 5 | 6 | def run(): 7 | """ 8 | Start blueprint 9 | """ 10 | dojo = Dojo() 11 | start_time = time.time() 12 | args = arg_parser.parse_known_args()[0] 13 | dojo.train(blueprint=args.blueprint) 14 | end_time = time.time() 15 | time_delta = end_time - start_time 16 | print('Finished in ' + str(int(time_delta)) + ' sec.') 17 | 18 | 19 | if __name__ == '__main__': 20 | arg_parser = configargparse.get_argument_parser() 21 | arg_parser.add('-v', '--verbose', help='Verbosity', action='store_true') 22 | arg_parser.add('-b', '--blueprint', help='blueprint csv.file') 23 | run() 24 | -------------------------------------------------------------------------------- /examples/exchange.py: -------------------------------------------------------------------------------- 1 | import time 2 | import configargparse 3 | from exchanges.exchange import Exchange 4 | 5 | 6 | def trade_history(): 7 | """ 8 | Gets sample trades history 9 | """ 10 | exchange = Exchange() 11 | end = time.time() 12 | start = end - 3600 13 | data = exchange.get_market_history(start=start, 14 | end=end, 15 | currency_pair='BTC_ETH') 16 | print(data) 17 | 18 | 19 | if __name__ == "__main__": 20 | arg_parser = configargparse.get_argument_parser() 21 | arg_parser.add('-c', '--config', is_config_file=True, help='config file path', default='../mosquito.ini') 22 | options = arg_parser.parse_known_args()[0] 23 | 24 | trade_history() 25 | -------------------------------------------------------------------------------- /core/wallet.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | 3 | 4 | class Wallet: 5 | """ 6 | Class holding current status of wallet (assets and currencies) 7 | """ 8 | arg_parser = configargparse.get_argument_parser() 9 | arg_parser.add('--wallet_currency', help='Wallet currency (separated by comma)') 10 | arg_parser.add("--wallet_amount", help='Wallet amount (separated by comma)') 11 | 12 | def __init__(self): 13 | args = self.arg_parser.parse_known_args()[0] 14 | currency = args.wallet_currency.replace(" ", "").split(',') 15 | amount = args.wallet_amount.replace(" ", "").split(',') 16 | amount = [float(i) for i in amount] 17 | self.initial_balance = dict(zip(currency, amount)) 18 | self.current_balance = self.initial_balance.copy() 19 | -------------------------------------------------------------------------------- /blueprint.py: -------------------------------------------------------------------------------- 1 | import time 2 | import configargparse 3 | from ai.blueprint import Blueprint 4 | import logging 5 | 6 | 7 | def run(): 8 | """ 9 | Start blueprint 10 | """ 11 | blueprint = Blueprint() 12 | start_time = time.time() 13 | blueprint.run() 14 | end_time = time.time() 15 | time_delta = end_time - start_time 16 | logger = logging.getLogger(__name__) 17 | logger.info('Finished in ' + str(int(time_delta)) + ' sec.') 18 | 19 | 20 | if __name__ == '__main__': 21 | arg_parser = configargparse.get_argument_parser() 22 | arg_parser.add('--pairs', help='Pairs to run blueprint on. For ex. [BTC_ETH, BTC_* (to get all BTC_* prefixed pairs]') 23 | arg_parser.add('-c', '--config', is_config_file=True, help='config file path', default='mosquito.ini') 24 | run() 25 | -------------------------------------------------------------------------------- /backfill.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | from backfill.candles import Candles 3 | from backfill.trades import Trades 4 | 5 | 6 | def main(args): 7 | if args.full: 8 | trades_client = Trades() 9 | trades_client.run() 10 | candles_client = Candles() 11 | candles_client.run() 12 | return 13 | 14 | if args.backfilltrades: 15 | backfill_client = Trades() 16 | backfill_client.run() 17 | else: 18 | backfill_client = Candles() 19 | backfill_client.run() 20 | 21 | 22 | if __name__ == "__main__": 23 | arg_parser = configargparse.get_argument_parser() 24 | arg_parser.add('--full', 25 | help='Backfill candle and trades', 26 | action='store_true') 27 | 28 | options = arg_parser.parse_known_args()[0] 29 | main(options) 30 | -------------------------------------------------------------------------------- /lib/indicators/ropc.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def ropc(close, timeperiod=5): 4 | """ 5 | Calculates percentage change between n last elements 6 | close: price numpy.ndarray 7 | timeperiod: size to check 8 | """ 9 | 10 | dataset_size = close.size 11 | if dataset_size < timeperiod-1: 12 | print('Error in ropc.py: passed not enough data! Required: ' + str(timeperiod) + 13 | ' passed: ' + str(dataset_size)) 14 | return None 15 | 16 | close_list = close.tolist() 17 | prev_value = None 18 | price_diff_sum = 0.0 19 | for value in close_list: 20 | if not prev_value: 21 | prev_value = value 22 | continue 23 | value_diff = value - prev_value 24 | perc_change = (value_diff*100/prev_value) 25 | price_diff_sum += perc_change 26 | 27 | return price_diff_sum 28 | 29 | 30 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Use the latest 2.1 version of CircleCI pipeline process engine. 2 | # See: https://circleci.com/docs/2.0/configuration-reference 3 | version: 2.1 4 | 5 | # Define a job to be invoked later in a workflow. 6 | # See: https://circleci.com/docs/2.0/configuration-reference/#jobs 7 | jobs: 8 | say-hello: 9 | # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. 10 | # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor 11 | docker: 12 | - image: cimg/base:stable 13 | # Add steps to the job 14 | # See: https://circleci.com/docs/2.0/configuration-reference/#steps 15 | steps: 16 | - checkout 17 | - run: 18 | name: "Say hello" 19 | command: "echo Hello, World!" 20 | 21 | # Invoke jobs via workflows 22 | # See: https://circleci.com/docs/2.0/configuration-reference/#workflows 23 | workflows: 24 | say-hello-workflow: 25 | jobs: 26 | - say-hello 27 | -------------------------------------------------------------------------------- /utils/telegrambot.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | import logging 3 | import telegram 4 | 5 | 6 | def run(token, chat_id): 7 | """ 8 | Telegram bot connection 9 | """ 10 | 11 | bot = telegram.Bot(token=token) 12 | logger = logging.getLogger(__name__) 13 | bot.send_message(chat_id=chat_id, text="I'm sorry Dave I'm afraid I can't do that.") 14 | logger.info(bot.get_me()) 15 | """ 16 | 17 | file = open('mosquito_stats.html', 'w') 18 | file.write(body) 19 | file.close() 20 | 21 | token = '572035357:AAEZ0na7xvdIUk53o9OTfLzwZkX52_nTAY4' 22 | chat_id = '406903247' 23 | bot = telegram.Bot(token=token) 24 | bot.send_document(chat_id=chat_id, document=open('mosquito_stats.html', 'rb')) 25 | """ 26 | 27 | 28 | if __name__ == '__main__': 29 | arg_parser = configargparse.get_argument_parser() 30 | arg_parser.add('--telegram_token', help='Telegram token', required=True) 31 | arg_parser.add('--chat_id', help='Telegram Chat id', required=True) 32 | 33 | args = arg_parser.parse_known_args()[0] 34 | run(token=args.telegram_token, chat_id=args.chat_id) 35 | -------------------------------------------------------------------------------- /lib/indicators/macd.py: -------------------------------------------------------------------------------- 1 | import talib 2 | 3 | 4 | def macd(close, previous_macds=[], fast_period=12, slow_period=26, signal_period=9): 5 | """ 6 | MACD - Moving Average Convergence Divergence 7 | previous_macd: numpy.ndarray of previous MACDs 8 | Returns: 9 | - macd 10 | - macd_line 11 | """ 12 | dataset_size = close.size 13 | if dataset_size < slow_period-1: 14 | print('Error in macd.py: passed not enough data! Required: ' + str(slow_period) + 15 | ' passed: ' + str(dataset_size)) 16 | return None, None 17 | 18 | try: 19 | ema_slow = talib.EMA(close, timeperiod=slow_period)[-1] 20 | ema_fast = talib.EMA(close[-fast_period:], timeperiod=fast_period)[-1] 21 | macd_value = ema_fast - ema_slow 22 | 23 | # print('previous_macds:', previous_macds) 24 | if len(previous_macds) < signal_period: 25 | signal_line = None 26 | else: 27 | signal_line = talib.EMA(previous_macds[-signal_period:], timeperiod=signal_period)[-1] 28 | except Exception as e: 29 | print('Got Exception in macd.py. Details: ' + str(e) + '. Data: ' + str(previous_macds)) 30 | return None, None 31 | 32 | return macd_value, signal_line 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /mosquito.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | from core.engine import Engine 3 | 4 | 5 | def main(): 6 | engine = Engine() 7 | engine.run() 8 | 9 | 10 | def has_mandatory_fields(options): 11 | """ 12 | Checks if command arguments contain all mandatory arguments 13 | """ 14 | if not options.backtest and not options.live and not options.paper: 15 | return False 16 | return True 17 | 18 | 19 | if __name__ == "__main__": 20 | arg_parser = configargparse.get_argument_parser() 21 | arg_parser.add('-c', '--config', is_config_file=True, help='config file path', default='mosquito.ini') 22 | arg_parser.add('--backtest', help='Simulate your strategy on history ticker data', action='store_true') 23 | arg_parser.add("--paper", help="Simulate your strategy on real ticker", action='store_true') 24 | arg_parser.add("--live", help="REAL trading mode", action='store_true') 25 | arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true') 26 | arg_parser.add('--strategy', help='Strategy') 27 | arg_parser.add('--fixed_trade_amount', help='Fixed trade amount') 28 | args = arg_parser.parse_known_args()[0] 29 | 30 | if not has_mandatory_fields(args): 31 | print("Missing trade mode argument (backtest, paper or live). See --help for more details.") 32 | exit(0) 33 | 34 | main() 35 | 36 | -------------------------------------------------------------------------------- /strategies/ai/scikitbase.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | import configargparse 3 | from sklearn.externals import joblib 4 | from termcolor import colored 5 | 6 | 7 | class ScikitBase(ABC): 8 | """ 9 | Base class for AI strategies 10 | """ 11 | arg_parser = configargparse.get_argument_parser() 12 | arg_parser.add('-p', '--pipeline', help='trained model/pipeline (*.pkl file)', required=True) 13 | arg_parser.add('-f', '--feature_names', help='List of features list pipeline (*.pkl file)') 14 | pipeline = None 15 | 16 | def __init__(self): 17 | args = self.arg_parser.parse_known_args()[0] 18 | super(ScikitBase, self).__init__() 19 | self.pipeline = self.load_pipeline(args.pipeline) 20 | if args.feature_names: 21 | self.feature_names = self.load_pipeline(args.feature_names) 22 | 23 | @staticmethod 24 | def load_pipeline(pipeline_file): 25 | """ 26 | Loads scikit model/pipeline 27 | """ 28 | print(colored('Loading pipeline: ' + pipeline_file, 'green')) 29 | return joblib.load(pipeline_file) 30 | 31 | def fetch_pipeline_from_server(self): 32 | """ 33 | Method fetches pipeline from server/cloud 34 | """ 35 | # TODO 36 | pass 37 | 38 | def predict(self, df): 39 | """ 40 | Returns predictions based on the model/pipeline 41 | """ 42 | try: 43 | return self.pipeline.predict(df) 44 | except (ValueError, TypeError): 45 | print(colored('Got ValueError while using scikit model.. ', 'red')) 46 | return None 47 | 48 | -------------------------------------------------------------------------------- /utils/blueprints2gcp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import google.cloud.storage 3 | import configargparse 4 | import logging 5 | import glob 6 | 7 | 8 | def get_last_file(path): 9 | """ 10 | :param dir: 11 | :return: terates through all files that are under the given path and 12 | returns last created/edited file 13 | """ 14 | 15 | list_of_files = glob.glob(path + '*') 16 | return max(list_of_files, key=os.path.getctime) 17 | 18 | 19 | def run(root_dir, bucket_name, bucket_dir): 20 | """ 21 | Simple script which fetches last generated blueprint and uploads 22 | """ 23 | logger = logging.getLogger(__name__) 24 | logger.info('Starting to upload blueprints to gcp bucket: ' + bucket_name + 25 | ', bucket_dir: ' + bucket_dir) 26 | 27 | # Get last modified/created blueprint file 28 | last_blueprint_file = get_last_file(root_dir) 29 | logger.info('last_blueprint_file:' + last_blueprint_file) 30 | 31 | # Create a storage client. 32 | source_file_name = last_blueprint_file 33 | storage_client = google.cloud.storage.Client() 34 | bucket = storage_client.get_bucket(bucket_name) 35 | blob = bucket.blob(bucket_dir + os.path.basename(source_file_name)) 36 | blob.upload_from_filename(source_file_name) 37 | logger.info('File ' + source_file_name + ' uploaded to ' + str(bucket)) 38 | 39 | 40 | if __name__ == '__main__': 41 | arg_parser = configargparse.get_argument_parser() 42 | arg_parser.add('--path', help='Path to root dir from where the script should read files (default out/blueprints)') 43 | arg_parser.add('--bucket', help='GCP Bucket name and path', required=True) 44 | arg_parser.add('--bucket_dir', help='GCP Bucket directory') 45 | 46 | args = arg_parser.parse_known_args()[0] 47 | 48 | if args.path is None: 49 | args.path = '../out/blueprints/' 50 | run(root_dir=args.path, bucket_name=args.bucket, bucket_dir=args.bucket_dir) 51 | -------------------------------------------------------------------------------- /lib/indicators/elderray.py: -------------------------------------------------------------------------------- 1 | import talib 2 | 3 | 4 | def elderray(close): 5 | """ 6 | Elder ray Indicator (Bulls/Bears Power) 7 | Returns: 8 | 0 - No condition met 9 | 1 - Green Price Bar: (13-period EMA > previous 13-period EMA) and (MACD-Histogram > previous period's MACD-Histogram) 10 | 2 - Red Price Bar: (13-period EMA < previous 13-period EMA) and (MACD-Histogram < previous period's MACD-Histogram) 11 | 12 | Price bars are colored blue when conditions for a Red Price Bar or Green Price Bar are not met. The MACD-Histogram 13 | is based on MACD(12,26,9). 14 | """ 15 | 16 | min_dataset_size = 36 17 | dataset_size = close.size 18 | if dataset_size < min_dataset_size: 19 | print('Error in elderray.py: passed not enough data! Required: ' + str(min_dataset_size) + 20 | ' passed: ' + str(dataset_size)) 21 | return None 22 | 23 | # Calc EMA 24 | ema_period = 13 25 | ema = talib.EMA(close[-ema_period:], timeperiod=ema_period)[-1] 26 | ema_prev = talib.EMA(close[-ema_period-1:len(close)-1], 27 | timeperiod=ema_period)[-1] 28 | 29 | # Calc MACD 30 | macd_period = 34 31 | macd, macd_signal, _ = talib.MACD(close[-macd_period:], 32 | fastperiod=12, 33 | slowperiod=26, 34 | signalperiod=9) 35 | macd = macd[-1:] 36 | 37 | macd_prev, macd_signal_prev, _ = talib.MACD(close[-macd_period-1:len(close)-1], 38 | fastperiod=12, 39 | slowperiod=26, 40 | signalperiod=9) 41 | macd_prev = macd_prev[-1:] 42 | 43 | # Green Price Bar 44 | if ema > ema_prev and macd > macd_prev: 45 | return 1 46 | 47 | # Red Price Bar 48 | if ema < ema_prev and macd < macd_prev: 49 | return 2 50 | 51 | return 0 52 | 53 | 54 | -------------------------------------------------------------------------------- /lib/indicators/stoploss.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | 3 | 4 | class StopLoss: 5 | """ 6 | StopLoss class with normal and trailing stop-loss functionality 7 | :param: interval: Interval in minutes for checking price difference 8 | :param stop_loss_limit: Percentage value for stop-loss (how much should the price drop to send stop-loss signal 9 | :param trailing: If True stop_loss Trailing stop-loss will be applied. If False first price will be used 10 | for a static stop-loss limit 11 | :param ticker_size: Ticker size 12 | """ 13 | 14 | arg_parser = configargparse.get_argument_parser() 15 | arg_parser.add('--stoploss_interval', help='Stop-loss interval in minutes') 16 | base_price = None 17 | 18 | def __init__(self, ticker_size, interval=30, stop_loss_limit=-0.1, trailing=True, ): 19 | self.trailing = trailing 20 | self.checkpoint = int(interval/ticker_size) 21 | self.stop_loss_limit = stop_loss_limit 22 | 23 | def set_base_price(self, price): 24 | """ 25 | Sets base price, which is compared to trailing-stop 26 | :param price: 27 | """ 28 | self.base_price = price 29 | 30 | def calculate(self, price): 31 | """ 32 | 33 | :param price: numpy array of price values 34 | :return: Returns True if Stop-Loss met conditions 35 | """ 36 | 37 | # Check if array has data 38 | if len(price) < self.checkpoint: 39 | print('StopLoss: not enough data.') 40 | return False 41 | 42 | if not self.base_price: 43 | self.base_price = price[-1] 44 | print('StopLoss: setting base-price to:', self.base_price) 45 | return False 46 | 47 | last_price = price[-1] 48 | checkpoint_price = price[-self.checkpoint] 49 | percentage_change = last_price*100/checkpoint_price - 100.0 50 | if percentage_change <= self.stop_loss_limit: 51 | return True 52 | 53 | # Handle trailing 54 | if self.trailing: 55 | if last_price > self.base_price: 56 | self.base_price = last_price 57 | 58 | -------------------------------------------------------------------------------- /core/common.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from importlib import import_module 3 | from termcolor import colored 4 | 5 | 6 | def load_module(module_prefix, module_name): 7 | """ 8 | Loads strategy module based on given name. 9 | """ 10 | if module_name is None: 11 | print(colored('Not provided module,. please add it as an argument or in config file', 'red')) 12 | sys.exit() 13 | mod = import_module(module_prefix + module_name) 14 | module_class = getattr(mod, module_name.split('.')[-1].capitalize()) 15 | return module_class 16 | 17 | 18 | def handle_buffer_limits(df, max_size): 19 | """ 20 | Handles dataframe limits (drops df, if the df > max_size) 21 | """ 22 | df_size = len(df.index) 23 | if df_size > max_size: 24 | # print(colored('Max buffer memory exceeded, cleaning', 'yellow')) 25 | rows_to_delete = df_size - max_size 26 | df = df.ix[rows_to_delete:] 27 | df = df.reset_index(drop=True) 28 | return df 29 | 30 | 31 | def parse_pairs(exchange, in_pairs): 32 | """ 33 | Returns list of available pairs from exchange based on the given pairs string/list 34 | """ 35 | all_pairs = exchange.get_pairs() 36 | if in_pairs == 'all': 37 | print('setting_all_pairs') 38 | return all_pairs 39 | else: 40 | pairs = [] 41 | parsed_pairs = in_pairs.replace(" ", "").split(',') 42 | for in_pair in parsed_pairs: 43 | if '*' in in_pair: 44 | prefix = in_pair.replace('*', '') 45 | pairs_list = [p for p in all_pairs if prefix in p] 46 | pairs.extend(pairs_list) 47 | # remove duplicates 48 | # pairs = list(set(pairs)) 49 | else: 50 | pairs.append(in_pair) 51 | return pairs 52 | 53 | 54 | def get_dataset_count(df, group_by_field='pair'): 55 | """ 56 | Returns count of dataset and pairs_count (group by provided string) 57 | """ 58 | pairs_group = df.groupby([group_by_field]) 59 | # cnt = pairs_group.count() 60 | pairs_count = len(pairs_group.groups.keys()) 61 | dataset_cnt = pairs_group.size().iloc[0] 62 | return dataset_cnt, pairs_count 63 | -------------------------------------------------------------------------------- /utils/postman.py: -------------------------------------------------------------------------------- 1 | import smtplib 2 | import configargparse 3 | from email.mime.text import MIMEText 4 | from email.mime.multipart import MIMEMultipart 5 | from premailer import transform 6 | 7 | 8 | class Postman: 9 | """ 10 | Simple email/postman module 11 | ! Currently supported only for gmail 12 | """ 13 | arg_parser = configargparse.get_argument_parser() 14 | arg_parser.add('--mail_username', help='Email username (supported only gmail)') 15 | arg_parser.add("--mail_password", help='Email password (supported only gmail)') 16 | arg_parser.add("--mail_recipients", help='Email recipients') 17 | 18 | def __init__(self): 19 | self.args = self.arg_parser.parse_known_args()[0] 20 | self.username = self.args.mail_username 21 | self.password = self.args.mail_password 22 | self.recipients = self.args.mail_recipients 23 | 24 | def send_mail(self, subject, body): 25 | """ 26 | Send email to configured account with given subject and body 27 | """ 28 | mail_from = self.username 29 | # mail_to = self.recipients if type(self.recipients) is list else [self.recipients] 30 | mail_to = self.recipients 31 | 32 | msg = MIMEMultipart('alternative') 33 | msg['Subject'] = subject 34 | msg['From'] = mail_from 35 | msg['To'] = mail_to 36 | 37 | # body = self.html_style() + body 38 | # msg.attach(MIMEText(body, 'html')) 39 | body = transform(body) 40 | #body = '

Peter

Hej

' 41 | msg.attach(MIMEText(body, 'html')) 42 | mail = smtplib.SMTP("smtp.gmail.com", 587) 43 | mail.ehlo() 44 | mail.starttls() 45 | mail.login(self.username, self.password) 46 | mail.sendmail(mail_from, mail_to, msg.as_string()) 47 | mail.close() 48 | print('mail successfully sent') 49 | 50 | @staticmethod 51 | def html_style(): 52 | """ 53 | Email css styles 54 | """ 55 | style = ''' 56 | 62 | ''' 63 | return style 64 | -------------------------------------------------------------------------------- /dojo/dojo.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import configargparse 3 | 4 | 5 | class Dojo: 6 | """ 7 | Main module responsible for training 8 | """ 9 | arg_parser = configargparse.get_argument_parser() 10 | 11 | def __init__(self): 12 | args = self.arg_parser.parse_known_args()[0] 13 | self.verbose = args.verbose 14 | 15 | def train(self, blueprint=None, automatic_search=True, models=None, minimum_score=0.7): 16 | """ 17 | Trains input data by automatic_search (GA) or with given models 18 | :param blueprint: multi-currency features csv file (for example generated by blueprint module) 19 | :param automatic_search: automatically find the best models 20 | :param models: list of models to be used. If automatic_search is True, models will be included into model search 21 | :param minimum_score: minimum_score the model needs to have so that it is returned 22 | :return: list of best models with their score 23 | """ 24 | if not blueprint: 25 | print("Required blueprint csv. argument value is missing, nothing to do here.") 26 | return 27 | 28 | print('Loading dataset') 29 | df_pair_groups = self.load_blueprint(blueprint) 30 | # Get trained models for every pair 31 | for pair, df in df_pair_groups: 32 | pair_models = self.train_pair(pair, df, automatic_search, models, minimum_score) 33 | # TODO: Store pair_models 34 | print(pair) 35 | 36 | @staticmethod 37 | def train_pair(pair, df, automatic_search, models, minimum_score): 38 | """ 39 | Function that trains given dataset for a specific pair 40 | """ 41 | print('Training model for pair:', pair) 42 | # TODO: Train model 43 | return pair, None 44 | 45 | @staticmethod 46 | def load_blueprint(blueprint_file): 47 | """ 48 | Loads blueprint csv file and returns paired groups (grouped by pair) 49 | """ 50 | print('Loading dataset') 51 | df = pd.read_csv(blueprint_file) 52 | df_pair_groups = df.groupby(['pair']) 53 | pairs_names = list(df_pair_groups.groups.keys()) 54 | print('Training total of: ', len(pairs_names), 'pairs and ', df.shape[0], 'records') 55 | return df_pair_groups 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /core/bots/live.py: -------------------------------------------------------------------------------- 1 | from .base import Base 2 | from core.bots.enums import TradeMode 3 | import time 4 | 5 | 6 | class Live(Base): 7 | """ 8 | Main class for Live Trading 9 | """ 10 | mode = TradeMode.live 11 | 12 | def __init__(self): 13 | super(Live, self).__init__(self.mode) 14 | self.counter = 0 15 | # open_orders = self.exchange.get_open_orders() 16 | # print(open_orders) 17 | 18 | def get_next(self, interval_in_min): 19 | """ 20 | Returns next state 21 | Interval: Interval in minutes 22 | """ 23 | interval_in_sec = interval_in_min*60 24 | epoch_now = int(time.time()) 25 | if self.last_tick_epoch > 0: 26 | next_ticker_time = (self.last_tick_epoch + interval_in_sec) 27 | delay_second = epoch_now - next_ticker_time 28 | if delay_second < 0: 29 | print('Going to sleep for: ', abs(delay_second), ' seconds.') 30 | time.sleep(abs(delay_second)) 31 | 32 | if not self.ticker_df.empty: 33 | self.ticker_df.drop(self.ticker_df.index, inplace=True) 34 | 35 | print('Fetching data for ' + str(len(self.pairs)) + ' ticker/tickers.', end='', flush=True) 36 | 37 | epoch_now = int(time.time()) 38 | epoch_start = epoch_now - interval_in_sec*5 # just to be sure get extra 5 datasets 39 | epoch_end = epoch_now 40 | for pair in self.pairs: 41 | new_df = self.exchange.get_candles_df(pair, epoch_start, epoch_end, interval_in_sec) 42 | # df = self.exchange.get_symbol_ticker(pair, interval_in_min) 43 | if self.ticker_df.empty: 44 | self.ticker_df = new_df.copy() 45 | else: 46 | self.ticker_df = self.ticker_df.append(new_df, ignore_index=True) 47 | print('.', end='', flush=True) 48 | 49 | print('..done') 50 | self.last_tick_epoch = epoch_now 51 | return self.ticker_df 52 | 53 | def get_balance(self): 54 | """ 55 | Returns wallet balance 56 | """ 57 | return self.exchange.get_balances() 58 | 59 | def trade(self, actions, wallet, trades, force_sell=True): 60 | """ 61 | Simulate currency buy/sell (places fictive buy/sell orders) 62 | """ 63 | # TODO: we need to deal with trades-buffer (trades) 64 | return self.exchange.trade(actions, wallet, TradeMode.live) 65 | -------------------------------------------------------------------------------- /ai/blueprints/junior.py: -------------------------------------------------------------------------------- 1 | import talib 2 | from .base import Base 3 | 4 | 5 | class Junior(Base): 6 | """ 7 | Mid-size blueprint - EMA, RCI, CCI, OBV 8 | """ 9 | 10 | def __init__(self, pairs): 11 | super(Junior, self).__init__('junior', pairs) 12 | self.min_history_ticks = 35 13 | 14 | @staticmethod 15 | def calculate_features(df): 16 | """ 17 | Method which calculates and generates features 18 | """ 19 | close = df['close'].values 20 | high = df['high'].values 21 | low = df['low'].values 22 | volume = df['volume'].values 23 | last_row = df.tail(1).copy() 24 | 25 | periods = [2, 4, 8, 12, 16, 20] 26 | for period in periods: 27 | # ************** Calc EMAs 28 | ema = talib.EMA(close[-period:], timeperiod=period)[-1] 29 | last_row['ema' + str(period)] = ema 30 | 31 | # ************** Calc OBVs 32 | obv = talib.OBV(close[-period:], volume[-period:])[-1] 33 | last_row['obv' + str(period)] = obv 34 | 35 | # ************** Calc RSIs 36 | rsi_periods = [5] 37 | for rsi_period in rsi_periods: 38 | rsi = talib.RSI(close[-rsi_period:], timeperiod=rsi_period-1)[-1] 39 | last_row['rsi' + str(rsi_period)] = rsi 40 | last_row['rsi_above_50' + str(rsi_period)] = int(rsi > 50.0) 41 | 42 | # ************** Calc CCIs 43 | cci_periods = [5] 44 | for cci_period in cci_periods: 45 | cci = talib.CCI(high[-cci_period:], 46 | low[-cci_period:], 47 | close[-cci_period:], 48 | timeperiod=cci_period)[-1] 49 | last_row['cci' + str(cci_period)] = cci 50 | 51 | # ************** Calc MACD 1 52 | macd_periods = [34] 53 | for macd_period in macd_periods: 54 | macd, macd_signal, _ = talib.MACD(close[-macd_period:], 55 | fastperiod=12, 56 | slowperiod=26, 57 | signalperiod=9) 58 | macd = macd[-1] 59 | signal_line = macd_signal[-1] 60 | last_row['macd_above_signal' + str(macd_period)] = int(macd > signal_line) 61 | last_row['macd_above_zero' + str(macd_period)] = int(macd > 0.0) 62 | 63 | return last_row 64 | -------------------------------------------------------------------------------- /backfill/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import configargparse 3 | from logging.config import fileConfig 4 | from abc import ABC, abstractmethod 5 | from pymongo import MongoClient 6 | from exchanges.exchange import Exchange 7 | 8 | 9 | class Base(ABC): 10 | """ 11 | Base class for data back-filling 12 | """ 13 | arg_parser = configargparse.get_argument_parser() 14 | arg_parser.add('-c', '--config', is_config_file=True, help='config file path', default='mosquito.ini') 15 | arg_parser.add('--pairs', help='Pairs to backfill. For ex. [BTC_ETH, BTC_* (to get all BTC_* prefixed pairs]') 16 | arg_parser.add("--all", help='Backfill data for ALL currencies', action='store_true') 17 | arg_parser.add("--days", help="Number of days to backfill", required=True, type=int, default=1) 18 | arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true') 19 | logging.config.fileConfig('logging.ini') 20 | 21 | def __init__(self): 22 | super(Base, self).__init__() 23 | args = self.arg_parser.parse_known_args()[0] 24 | self.exchange = Exchange() 25 | self.exchange_name = self.exchange.get_exchange_name() 26 | self.db = self.initialize_db(args) 27 | 28 | @staticmethod 29 | def initialize_db(args): 30 | """ 31 | DB Initializer 32 | """ 33 | db = args.db 34 | port = int(args.db_port) 35 | url = args.db_url 36 | # Init DB 37 | client = MongoClient(url, port) 38 | return client[db] 39 | 40 | def get_backfill_pairs(self, backfill_all_pairs=False, pairs_list=None): 41 | """ 42 | Returns list of exchange pairs that were ordered to backfill 43 | """ 44 | all_pairs = self.exchange.get_pairs() 45 | if backfill_all_pairs: 46 | return all_pairs 47 | elif pairs_list is not None: 48 | tmp_pairs = [pairs_list] 49 | pairs = [] 50 | # Handle * suffix pairs 51 | for pair in tmp_pairs: 52 | if '*' in pair: 53 | prefix = pair.replace('*', '') 54 | pairs_list = [p for p in all_pairs if prefix in p] 55 | pairs.extend(pairs_list) 56 | # remove duplicates 57 | pairs = list(set(pairs)) 58 | else: 59 | pairs.append(pair) 60 | return pairs 61 | 62 | @abstractmethod 63 | def run(self): 64 | """ 65 | Backfill/fetch data 66 | """ 67 | pass 68 | -------------------------------------------------------------------------------- /core/bots/backtest.py: -------------------------------------------------------------------------------- 1 | from .base import Base 2 | import time 3 | import pandas as pd 4 | from core.bots.enums import TradeMode 5 | import configargparse 6 | 7 | 8 | DAY_IN_SECONDS = 86400 9 | 10 | 11 | class Backtest(Base): 12 | """ 13 | Main class for Backtest trading 14 | """ 15 | arg_parser = configargparse.get_argument_parser() 16 | arg_parser.add('--backtest_from', help='Backtest epoch start datetime') 17 | arg_parser.add("--backtest_to", help='Backtest epoch end datetime') 18 | arg_parser.add("--days", help='Number of history days the simulation should start from') 19 | 20 | mode = TradeMode.backtest 21 | sim_start = None 22 | sim_end = None 23 | sim_days = None 24 | 25 | def __init__(self, wallet): 26 | args = self.arg_parser.parse_known_args()[0] 27 | super(Backtest, self).__init__(self.mode) 28 | self.counter = 0 29 | if args.backtest_from: 30 | self.sim_start = int(args.backtest_from) 31 | if args.backtest_to: 32 | self.sim_end = int(args.backtest_to) 33 | if args.days: 34 | self.sim_days = int(args.days) 35 | self.sim_epoch_start = self.get_sim_epoch_start(self.sim_days, self.sim_start) 36 | self.current_epoch = self.sim_epoch_start 37 | self.balance = wallet 38 | 39 | @staticmethod 40 | def get_sim_epoch_start(sim_days, sim_start): 41 | if sim_start: 42 | return sim_start 43 | elif sim_days: 44 | epoch_now = int(time.time()) 45 | return epoch_now - (DAY_IN_SECONDS * sim_days) 46 | 47 | def get_next(self, interval_in_min): 48 | """ 49 | Returns next state of current_time + interval (in minutes) 50 | """ 51 | if self.sim_end and self.current_epoch > self.sim_end: 52 | return pd.DataFrame() 53 | self.ticker_df = self.exchange.get_offline_ticker(self.current_epoch, self.pairs) 54 | df_trades = self.exchange.get_offline_trades(self.current_epoch, self.pairs) 55 | 56 | if df_trades.empty: 57 | self.ticker_df = self.ticker_df 58 | else: 59 | self.ticker_df = self.ticker_df.merge(df_trades) 60 | 61 | self.current_epoch += interval_in_min*60 62 | return self.ticker_df 63 | 64 | def trade(self, actions, wallet, trades, force_sell=True): 65 | """ 66 | Simulate currency buy/sell (places fictive buy/sell orders) 67 | """ 68 | return super(Backtest, self).trade(actions, wallet, trades, force_sell=False) 69 | 70 | -------------------------------------------------------------------------------- /ai/blueprints/blp5m1117.py: -------------------------------------------------------------------------------- 1 | import talib 2 | from .base import Base 3 | 4 | 5 | class Blp5m1117(Base): 6 | """ 7 | Full blown blueprint - using 5m ticker 8 | """ 9 | 10 | def __init__(self, pairs): 11 | super(Blp5m1117, self).__init__('blp5m1117', pairs) 12 | self.min_history_ticks = 35 13 | 14 | @staticmethod 15 | def calculate_features(df): 16 | """ 17 | Method which calculates and generates features 18 | """ 19 | close = df['close'].values 20 | high = df['high'].values 21 | low = df['low'].values 22 | volume = df['volume'].values 23 | last_row = df.tail(1).copy() 24 | 25 | # ************** Calc EMAs 26 | ema_periods = [2, 4, 8, 12, 16, 20] 27 | for ema_period in ema_periods: 28 | ema = talib.EMA(close[-ema_period:], timeperiod=ema_period)[-1] 29 | last_row['ema' + str(ema_period)] = ema 30 | 31 | # ************** Calc RSIs 32 | rsi_periods = [5] 33 | for rsi_period in rsi_periods: 34 | rsi = talib.RSI(close[-rsi_period:], timeperiod=rsi_period-1)[-1] 35 | last_row['rsi' + str(rsi_period)] = rsi 36 | last_row['rsi_above_50' + str(rsi_period)] = int(rsi > 50.0) 37 | 38 | # ************** Calc CCIs 39 | cci_periods = [5] 40 | for cci_period in cci_periods: 41 | cci = talib.CCI(high[-cci_period:], 42 | low[-cci_period:], 43 | close[-cci_period:], 44 | timeperiod=cci_period)[-1] 45 | last_row['cci' + str(cci_period)] = cci 46 | 47 | # ************** Calc MACD 1 48 | macd_periods = [34] 49 | for macd_period in macd_periods: 50 | macd, macd_signal, _ = talib.MACD(close[-macd_period:], 51 | fastperiod=12, 52 | slowperiod=26, 53 | signalperiod=9) 54 | macd = macd[-1] 55 | signal_line = macd_signal[-1] 56 | last_row['macd_above_signal' + str(macd_period)] = int(macd > signal_line) 57 | last_row['macd_above_zero' + str(macd_period)] = int(macd > 0.0) 58 | 59 | # ************** Calc OBVs 60 | obv_periods = [2, 4, 8, 12, 16, 20] 61 | for obv_period in obv_periods: 62 | obv = talib.OBV(close[-obv_period:], volume[-obv_period:])[-1] 63 | last_row['obv' + str(obv_period)] = obv 64 | 65 | return last_row 66 | 67 | 68 | -------------------------------------------------------------------------------- /ai/blueprints/luckyantelope.py: -------------------------------------------------------------------------------- 1 | import talib 2 | from .base import Base 3 | 4 | 5 | class Luckyantelope(Base): 6 | """ 7 | Full blown blueprint - using 2h ticker 8 | """ 9 | 10 | def __init__(self, pairs): 11 | super(Luckyantelope, self).__init__('luckyantelope', pairs) 12 | self.min_history_ticks = 21 13 | 14 | @staticmethod 15 | def calculate_features(df): 16 | """ 17 | Method which calculates and generates features 18 | """ 19 | close = df['close'].values 20 | high = df['high'].values 21 | low = df['low'].values 22 | volume = df['volume'].values 23 | last_row = df.tail(1).copy() 24 | 25 | # ************** Calc EMAs 26 | ema_periods = [2, 4, 8, 12, 16, 20] 27 | for ema_period in ema_periods: 28 | ema = talib.EMA(close[-ema_period:], timeperiod=ema_period)[-1] 29 | last_row['ema' + str(ema_period)] = ema 30 | 31 | # ************** Calc RSIs 32 | rsi_periods = [5] 33 | for rsi_period in rsi_periods: 34 | rsi = talib.RSI(close[-rsi_period:], timeperiod=rsi_period-1)[-1] 35 | last_row['rsi' + str(rsi_period)] = rsi 36 | last_row['rsi_above_50' + str(rsi_period)] = int(rsi > 50.0) 37 | 38 | # ************** Calc CCIs 39 | cci_periods = [5] 40 | for cci_period in cci_periods: 41 | cci = talib.CCI(high[-cci_period:], 42 | low[-cci_period:], 43 | close[-cci_period:], 44 | timeperiod=cci_period)[-1] 45 | last_row['cci' + str(cci_period)] = cci 46 | 47 | # ************** Calc MACD 1 48 | macd_periods = [20] 49 | for macd_period in macd_periods: 50 | macd, macd_signal, _ = talib.MACD(close[-macd_period:], 51 | fastperiod=6, 52 | slowperiod=12, 53 | signalperiod=8) 54 | macd = macd[-1] 55 | signal_line = macd_signal[-1] 56 | last_row['macd_above_signal' + str(macd_period)] = int(macd > signal_line) 57 | last_row['macd_above_zero' + str(macd_period)] = int(macd > 0.0) 58 | 59 | # ************** Calc OBVs 60 | obv_periods = [2, 4, 8, 12, 16, 20] 61 | for obv_period in obv_periods: 62 | obv = talib.OBV(close[-obv_period:], volume[-obv_period:])[-1] 63 | last_row['obv' + str(obv_period)] = obv 64 | 65 | return last_row 66 | 67 | 68 | -------------------------------------------------------------------------------- /strategies/base.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | from termcolor import colored 3 | from abc import ABC, abstractmethod 4 | from .enums import TradeState as ts 5 | from strategies.enums import TradeState 6 | 7 | 8 | class Base(ABC): 9 | """ 10 | Base class for all strategies 11 | """ 12 | arg_parser = configargparse.get_argument_parser() 13 | action_request = ts.none 14 | actions = [] 15 | 16 | def __init__(self): 17 | super(Base, self).__init__() 18 | args = self.arg_parser.parse_known_args()[0] 19 | self.verbosity = args.verbosity 20 | self.min_history_ticks = 5 21 | self.group_by_field = 'pair' 22 | 23 | def get_min_history_ticks(self): 24 | """ 25 | Returns min_history_ticks 26 | """ 27 | return self.min_history_ticks 28 | 29 | @staticmethod 30 | def get_delimiter(df): 31 | if df.empty: 32 | print('Error: get_delimiter! Got empty df!') 33 | pair = df.iloc[-1].pair 34 | return '_' if '_' in pair else '-' 35 | 36 | @staticmethod 37 | def parse_pairs(pairs): 38 | return [x.strip() for x in pairs.split(',')] 39 | 40 | @abstractmethod 41 | def calculate(self, look_back, wallet): 42 | """ 43 | Main Strategy function, which takes recent history data and returns recommended list of actions 44 | """ 45 | None 46 | 47 | @staticmethod 48 | def get_price(trade_action, df, pair): 49 | """ 50 | Returns price based on on the given action and dataset. 51 | """ 52 | 53 | if df.empty: 54 | print(colored('get_price: got empty dataframe (pair): ' + pair + ', skipping!', 'red')) 55 | return 0.0 56 | 57 | pair_df = df.loc[df['pair'] == pair].sort_values('date') 58 | if pair_df.empty: 59 | print(colored('get_price: got empty dataframe for pair: ' + pair + ', skipping!', 'red')) 60 | return 0.0 61 | 62 | pair_df = pair_df.iloc[-1] 63 | close_price = float(pair_df.get('close')) 64 | price = None 65 | 66 | if trade_action == TradeState.buy: 67 | if 'lowestAsk' in pair_df: 68 | price = float(pair_df.get('lowestAsk')) 69 | elif trade_action == TradeState.sell: 70 | if 'highestBid' in pair_df: 71 | price = float(pair_df.get('highestBid')) 72 | 73 | # Check if we don't have nan 74 | if not price or price != price: 75 | if close_price != close_price: 76 | print(colored('got Nan price for pair: ' + pair + '. Dataframe: ' + str(pair_df), 'red')) 77 | return 0.0 78 | else: 79 | return close_price 80 | 81 | return price 82 | -------------------------------------------------------------------------------- /strategies/ema.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | import pandas as pd 3 | import pandas_ta as ta 4 | from .base import Base 5 | import core.common as common 6 | from .enums import TradeState 7 | from core.bots.enums import BuySellMode 8 | from core.tradeaction import TradeAction 9 | from lib.indicators.stoploss import StopLoss 10 | 11 | 12 | class Ema(Base): 13 | """ 14 | Ema strategy 15 | About: Buy when close_price > ema20, sell when close_price < ema20 and below death_cross 16 | """ 17 | arg_parser = configargparse.get_argument_parser() 18 | 19 | def __init__(self): 20 | args = self.arg_parser.parse_known_args()[0] 21 | super(Ema, self).__init__() 22 | self.name = 'ema' 23 | self.min_history_ticks = 30 24 | self.pair = self.parse_pairs(args.pairs)[0] 25 | self.buy_sell_mode = BuySellMode.all 26 | self.stop_loss = StopLoss(int(args.ticker_size)) 27 | 28 | def calculate(self, look_back, wallet): 29 | """ 30 | Main strategy logic (the meat of the strategy) 31 | """ 32 | (dataset_cnt, _) = common.get_dataset_count(look_back, self.group_by_field) 33 | 34 | # Wait until we have enough data 35 | if dataset_cnt < self.min_history_ticks: 36 | print('dataset_cnt:', dataset_cnt) 37 | return self.actions 38 | 39 | self.actions.clear() 40 | new_action = TradeState.none 41 | 42 | # Calculate indicators 43 | df = look_back.tail(self.min_history_ticks) 44 | close = df['close'] 45 | 46 | # ************** Calc EMA 47 | ema5 = ta.ema(close, length=5).values[-1] 48 | ema10 = ta.ema(close, length=10).values[-1] 49 | ema20 = ta.ema(close, length=20).values[-1] 50 | 51 | close_price = self.get_price(TradeState.none, df.tail(), self.pair) 52 | 53 | print('close_price:', close_price, 'ema:', ema20) 54 | if close_price < ema10 or close_price < ema20: 55 | new_action = TradeState.sell 56 | elif close_price > ema5 and close_price > ema10: 57 | new_action = TradeState.buy 58 | 59 | trade_price = self.get_price(new_action, df.tail(), self.pair) 60 | 61 | # Get stop-loss 62 | if new_action == TradeState.buy and self.stop_loss.calculate(close.values): 63 | print('stop-loss detected,..selling') 64 | new_action = TradeState.sell 65 | 66 | action = TradeAction(self.pair, 67 | new_action, 68 | amount=None, 69 | rate=trade_price, 70 | buy_sell_mode=self.buy_sell_mode) 71 | 72 | self.actions.append(action) 73 | return self.actions 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /backfill/candles.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | from pymongo import ASCENDING 4 | from backfill.base import Base 5 | from core.constants import SECONDS_IN_DAY 6 | 7 | 8 | class Candles(Base): 9 | """ 10 | Back-fills ticker candle data 11 | """ 12 | 13 | def __init__(self): 14 | super(Candles, self).__init__() 15 | self.args = self.arg_parser.parse_known_args()[0] 16 | self.db_ticker = self.db.ticker 17 | self.db_ticker.create_index([('id', ASCENDING)], unique=True) 18 | 19 | def run(self): 20 | """ 21 | Run actual backfill job 22 | """ 23 | # Get list of all currencies 24 | logger = logging.getLogger(__name__) 25 | time_start = time.time() 26 | pairs = self.get_backfill_pairs(self.args.all, self.args.pairs) 27 | logger.info("Back-filling candles for total currencies:" + str(len(pairs))) 28 | 29 | # Get the candlestick data 30 | epoch_now = int(time.time()) 31 | 32 | for pair in pairs: 33 | for day in reversed(range(1, int(self.args.days) + 1)): 34 | epoch_from = epoch_now - (SECONDS_IN_DAY * day) 35 | epoch_to = epoch_now if day == 1 else epoch_now - (SECONDS_IN_DAY * (day - 1)) 36 | logger.info('Getting currency data: ' + pair + ', days left: ' + str(day)) 37 | candles = self.exchange.get_candles(pair, 38 | epoch_from, 39 | epoch_to, 40 | 300) # by default 5 minutes candles (minimum) 41 | 42 | logger.info(' (got total candles: ' + str(len(candles)) + ')') 43 | for candle in candles: 44 | if candle['date'] == 0: 45 | logger.warning('Found nothing for pair: ' + pair) 46 | continue 47 | # Convert strings to number (float or int) 48 | for key, value in candle.items(): 49 | if key == 'date': 50 | candle[key] = int(value) 51 | else: 52 | candle[key] = float(value) 53 | new_db_item = candle.copy() 54 | # Add identifier 55 | new_db_item['exchange'] = self.exchange_name 56 | new_db_item['pair'] = pair 57 | unique_id = self.exchange_name + '-' + pair + '-' + str(candle['date']) 58 | new_db_item['id'] = unique_id 59 | # Store to DB 60 | self.db_ticker.update_one({'id': unique_id}, {'$set': new_db_item}, upsert=True) 61 | 62 | time_end = time.time() 63 | duration_in_sec = int(time_end-time_start) 64 | logger.info("Backfill done in (sec) " + str(duration_in_sec)) 65 | -------------------------------------------------------------------------------- /core/bots/paper.py: -------------------------------------------------------------------------------- 1 | from .base import Base 2 | from core.bots.enums import TradeMode 3 | import time 4 | import configargparse 5 | 6 | 7 | class Paper(Base): 8 | """ 9 | Main class for Paper trading 10 | """ 11 | arg_parser = configargparse.get_argument_parser() 12 | arg_parser.add('--use_real_wallet', help='Use/not use fictive wallet (only for paper simulation)', 13 | action='store_true') 14 | mode = TradeMode.paper 15 | ticker_df = None 16 | 17 | def __init__(self, wallet): 18 | args = self.arg_parser.parse_known_args()[0] 19 | super(Paper, self).__init__(self.mode) 20 | self.use_real_wallet = args.use_real_wallet 21 | if not self.use_real_wallet: 22 | self.balance = wallet.copy() 23 | 24 | def get_next(self, interval_in_min): 25 | """ 26 | Returns next state 27 | Interval: Interval in minutes 28 | """ 29 | interval_in_sec = interval_in_min*60 30 | epoch_now = int(time.time()) 31 | if self.last_tick_epoch > 0: 32 | next_ticker_time = (self.last_tick_epoch + interval_in_sec) 33 | delay_second = epoch_now - next_ticker_time 34 | if delay_second < 0: 35 | print('Going to sleep for: ', abs(delay_second), ' seconds.') 36 | time.sleep(abs(delay_second)) 37 | 38 | if not self.ticker_df.empty: 39 | self.ticker_df.drop(self.ticker_df.index, inplace=True) 40 | 41 | epoch_now = int(time.time()) 42 | epoch_start = epoch_now - interval_in_sec*5 # just to be sure get extra 5 datasets 43 | epoch_end = epoch_now 44 | for pair in self.pairs: 45 | # print('getting candles for period:', str(epoch_start) + '---' + str(epoch_end) + '----' 46 | # + str(interval_in_sec) + pair) 47 | new_df = self.exchange.get_candles_df(pair, epoch_start, epoch_end, interval_in_sec) 48 | # rint('new_df____:', new_df) 49 | if self.ticker_df.empty: 50 | self.ticker_df = new_df.copy() 51 | else: 52 | self.ticker_df = self.ticker_df.append(new_df, ignore_index=True) 53 | # Remove duplicates 54 | # self.ticker_df.drop_duplicates(subset=['date', 'pair'], inplace=True, keep='last') 55 | 56 | self.last_tick_epoch = epoch_now 57 | return self.ticker_df.copy() 58 | 59 | def get_balance(self): 60 | """ 61 | Returns wallet balance 62 | """ 63 | if self.use_real_wallet: 64 | return self.exchange.get_balances() 65 | else: 66 | return self.balance.copy() 67 | 68 | def trade(self, actions, wallet, trades, force_sell=True): 69 | """ 70 | Simulate currency buy/sell (places fictive buy/sell orders) 71 | """ 72 | return super(Paper, self).trade(actions, wallet, trades, force_sell=True) 73 | -------------------------------------------------------------------------------- /core/plot.py: -------------------------------------------------------------------------------- 1 | import plotly.graph_objs as go 2 | import pandas as pd 3 | from plotly.offline import plot 4 | from tzlocal import get_localzone 5 | 6 | 7 | class Plot: 8 | """ 9 | Main plotting class 10 | """ 11 | 12 | def __init__(self): 13 | pass 14 | 15 | @staticmethod 16 | def draw(df, df_trades, pair, strategy_info): 17 | """ 18 | Candle-stick plot 19 | """ 20 | if df.empty: 21 | print('No data to plot!') 22 | return 23 | 24 | df = df[df['pair'] == pair] 25 | 26 | if df.empty: 27 | print('Plot: Empty dataframe, nothing to draw!') 28 | return 29 | 30 | pd.options.mode.chained_assignment = None 31 | 32 | df['date'] = pd.to_datetime(df['date'], unit='s', utc=True) 33 | df_trades['date'] = pd.to_datetime(df_trades['date'], unit='s', utc=True) 34 | local_tz = get_localzone() 35 | df = df.set_index(['date']) 36 | df.tz_convert(local_tz) 37 | # Convert datetime to current time-zone 38 | df_index = df.index.tz_localize(None) 39 | 40 | df_trades = df_trades.set_index(['date']) 41 | df_trades.tz_convert(local_tz) 42 | 43 | # plotly.offline.init_notebook_mode() 44 | 45 | trace = go.Candlestick(x=df_index, 46 | open=df.open, 47 | high=df.high, 48 | low=df.low, 49 | close=df.close) 50 | data = [trace] 51 | 52 | # Create buy/sell annotations 53 | annotations = [] 54 | for index, row in df_trades.iterrows(): 55 | d = dict(x=index.tz_localize(None), 56 | y=row['close_price'], 57 | xref='x', 58 | yref='y', 59 | ax=0, 60 | ay=40 if row['action'] == 'buy' else -40, 61 | showarrow=True, 62 | arrowhead=2, 63 | arrowsize=3, 64 | arrowwidth=1, 65 | arrowcolor='red' if row['action'] == 'sell' else 'green', 66 | bordercolor='#c7c7c7') 67 | annotations.append(d) 68 | 69 | # Unpack the report string 70 | title = '' 71 | for item in strategy_info: 72 | s = str(item) 73 | title = title + '
' + s 74 | 75 | layout = go.Layout( 76 | title=title, 77 | titlefont=dict( 78 | family='Courier New, monospace', 79 | size=14, 80 | color='#606060' 81 | ), 82 | autosize=True, 83 | showlegend=False, 84 | annotations=annotations 85 | ) 86 | 87 | figure = go.Figure(data=data, layout=layout) 88 | 89 | # Auto-open html page 90 | plot(figure, 91 | auto_open=True, 92 | image_filename='plot_image', 93 | validate=False) 94 | 95 | -------------------------------------------------------------------------------- /mosquito.sample.ini: -------------------------------------------------------------------------------- 1 | # --- Main configuration file for mosquito --- 2 | # All parameters can be overridden with command arguments 3 | 4 | [General] 5 | # Overall info verbosity true-on, false-off 6 | verbosity = false 7 | 8 | 9 | 10 | [Trade] 11 | # Valid values (polo, bittrex) 12 | exchange = polo 13 | # List of pairs that the ticker should be retrieved/monitored 14 | # Valid values: 15 | # all - get ticker for ALL currencies 16 | # comma separated list of pairs, for example BTC_DGB 17 | # prefix_* - log all pairs with given prefix. For example BTC_* or USDT_* 18 | # all 19 | pairs = BTC_ETH 20 | # Buffer size in days (how many days of data samples we need to save in memory) 21 | buffer_size = 30 22 | # Step interval in minutes 23 | # !! Supported intervals for Poloniex are 5, 15, 30, 120, 240, and 1440 minutes 24 | ticker_size = 5 25 | strategy = ema 26 | # Currency towards which the balance will be calculated 27 | root_report_currency = BTC 28 | # Trade amount for a fixed BuySellMode (used only for BuySellMode.fixed)!! 29 | # Current implementation works only on 1 pair type - for example BTC_* 30 | fixed_trade_amount = 0.0015 31 | # Prefetches data from history exchange ticker 32 | prefetch = true 33 | 34 | 35 | [Report] 36 | # Currency which will be plotted (!currently supported in single currency simulation only) 37 | plot_pair = BTC_ETH 38 | 39 | 40 | # ⋅⋅⋅⋅⋅⋅⋅ Mongo DB ⋅⋅⋅⋅⋅⋅⋅ 41 | [MongoDB] 42 | db_url = localhost 43 | db_port = 27017 44 | db = mosquito 45 | 46 | 47 | 48 | # ⋅⋅⋅⋅⋅⋅⋅ Exchanges ⋅⋅⋅⋅⋅⋅⋅ 49 | [Poloniex] 50 | polo_api_key = 51 | polo_secret = 52 | # fillOrKill, immediateOrCancel. (postOnly - not supported yet!) 53 | polo_buy_order = immediateOrCancel 54 | # fillOrKill, immediateOrCancel. (postOnly - not supported yet!) 55 | polo_sell_order = immediateOrCancel 56 | polo_txn_fee = 0.2 57 | 58 | 59 | 60 | [Bittrex] 61 | bittrex_api_key = 62 | bittrex_secret = 63 | bittrex_txn_fee = 0.25 64 | 65 | 66 | # ⋅⋅⋅⋅⋅⋅⋅ Backtest ⋅⋅⋅⋅⋅⋅⋅ 67 | [Backtest] 68 | # Backtest epoch start datetime 69 | # backtest_from 70 | # Backtest epoch end datetime 71 | # backtest_to 72 | # Number of history days the simulation should start from 73 | days = 10 74 | 75 | # ⋅⋅⋅⋅⋅⋅⋅ Paper ⋅⋅⋅⋅⋅⋅⋅ 76 | [Paper] 77 | # Set to false if you want to use fictive wallet (defined below) 78 | use_real_wallet = false 79 | 80 | 81 | # ⋅⋅⋅⋅⋅⋅⋅ Wallet ⋅⋅⋅⋅⋅⋅⋅ 82 | [Wallet] 83 | # Comma separated list of currencies 84 | # If left blank exchange will try to get data from defined Exchange wallet 85 | wallet_currency=BTC, ETH 86 | # Comma separated list of currency represented values 87 | # If left blank exchange will try to get data from defined Exchange wallet 88 | wallet_amount=1, 2 89 | 90 | 91 | # ⋅⋅⋅⋅⋅⋅⋅ Email ⋅⋅⋅⋅⋅⋅⋅ 92 | # Currently supported only gmail 93 | [Email] 94 | mail_username = 95 | mail_password = 96 | mail_recipients = 97 | 98 | 99 | # ⋅⋅⋅⋅⋅⋅⋅ Lense ⋅⋅⋅⋅⋅⋅⋅ 100 | # Used for gettin your wallets statistics 101 | [Lense] 102 | mail_username= 103 | mail_password= 104 | # Comma separated list of recepients 105 | mail_recipients= -------------------------------------------------------------------------------- /strategies/ai/luckyantelope.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | import pandas as pd 3 | from strategies.base import Base 4 | import core.common as common 5 | from ai.blueprints.luckyantelope import Luckyantelope as Model 6 | from strategies.ai.scikitbase import ScikitBase 7 | from core.bots.enums import BuySellMode 8 | from strategies.enums import TradeState 9 | from core.tradeaction import TradeAction 10 | 11 | 12 | class Luckyantelope(Base, ScikitBase): 13 | """ 14 | Luckyantelope strategy 15 | About: Strategy using trained model from Luckyantelope blueprint factory 16 | """ 17 | arg_parser = configargparse.get_argument_parser() 18 | trade_history = pd.DataFrame(columns=['close', 'predicted']) 19 | 20 | def __init__(self): 21 | args = self.arg_parser.parse_known_args()[0] 22 | super(Luckyantelope, self).__init__() 23 | self.name = 'luckyantelope' 24 | self.min_history_ticks = 21 25 | self.pair = self.parse_pairs(args.pairs)[0] 26 | self.buy_sell_mode = BuySellMode.fixed 27 | 28 | def calculate(self, look_back, wallet): 29 | """ 30 | Main strategy logic (the meat of the strategy) 31 | """ 32 | (dataset_cnt, _) = common.get_dataset_count(look_back, self.group_by_field) 33 | 34 | # Wait until we have enough data 35 | if dataset_cnt < self.min_history_ticks: 36 | print('dataset_cnt:', dataset_cnt, ',..waiting for more data..') 37 | return self.actions 38 | 39 | self.actions.clear() 40 | 41 | df = look_back.tail(self.min_history_ticks) 42 | df_blueprint = Model.calculate_features(df) 43 | 44 | # Remove not-used columns 45 | df_blueprint = df_blueprint[self.feature_names] 46 | 47 | # Re-ordering column names 48 | column_names = self.feature_names 49 | x = df_blueprint[column_names] 50 | price_now = x.close.iloc[0] 51 | price_predicted = self.predict(x) 52 | 53 | if price_predicted is None: 54 | return self.actions 55 | else: 56 | price_predicted = price_predicted[0] 57 | 58 | if price_predicted > 0.08 or price_predicted < 0.05: 59 | return self.actions 60 | 61 | price_change = ((price_predicted * 100) / price_now) - 100 62 | 63 | if price_change > 10: 64 | return self.actions 65 | 66 | self.trade_history = self.trade_history.append({'close': price_now, 67 | 'predicted': price_predicted}, ignore_index=True) 68 | self.trade_history.to_csv('out/luckyantelope_out.csv', index=False) 69 | 70 | print('price_change:' + str(price_change) + ', close_price: ' + str(x.close.iloc[0]) + ', predicted: ' + str(price_predicted)) 71 | 72 | new_action = TradeState.buy if price_predicted > price_now else TradeState.sell 73 | trade_price = self.get_price(new_action, df.tail(), self.pair) 74 | 75 | action = TradeAction(self.pair, 76 | new_action, 77 | amount=None, 78 | rate=trade_price, 79 | buy_sell_mode=self.buy_sell_mode) 80 | 81 | self.actions.append(action) 82 | return self.actions 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /backfill/trades.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import numpy as np 4 | import logging 5 | import pandas as pd 6 | import configargparse 7 | from pymongo import ASCENDING 8 | from backfill.base import Base 9 | from core.constants import SECONDS_IN_DAY 10 | 11 | 12 | class Trades(Base): 13 | """ 14 | Back-fills ticker trade data 15 | """ 16 | arg_parser = configargparse.get_argument_parser() 17 | arg_parser.add("--backfilltrades", help="Fetch /backfill and store trade history", action='store_true') 18 | 19 | def __init__(self): 20 | super(Trades, self).__init__() 21 | self.args = self.arg_parser.parse_known_args()[0] 22 | self.fetch_interval = 3600*6 # 6 hour batches 23 | self.db_trades = self.db.trades 24 | self.db_trades.create_index([('id', ASCENDING)], unique=True) 25 | 26 | def run(self): 27 | """ 28 | Run actual backfill job 29 | """ 30 | # Get list of all currencies 31 | logger = logging.getLogger(__name__) 32 | pairs = self.get_backfill_pairs(self.args.all, self.args.pairs) 33 | logger.info("Back-filling trade orders for total currencies: " + str(len(pairs))) 34 | time_start = time.time() 35 | 36 | # Get the candlestick data 37 | init_date_end = int(time.time()) 38 | init_date_start = init_date_end - (SECONDS_IN_DAY*int(self.args.days)) 39 | fetch_interval = 3600 * 6 40 | 41 | for pair in pairs: 42 | fetch_end_date = init_date_start 43 | date_start = init_date_start 44 | while fetch_end_date < init_date_end: 45 | fetch_end_date = date_start + fetch_interval 46 | date_start_string = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(date_start)) 47 | fetch_end_string = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(fetch_end_date)) 48 | logger.debug('fetching intervals: ' + pair + ', ' + date_start_string + ' - ' + fetch_end_string) 49 | trades = self.exchange.get_market_history(currency_pair=pair, 50 | start=date_start, 51 | end=fetch_end_date) 52 | date_start = fetch_end_date+1 53 | if len(trades) == 0: 54 | logger.debug('No trades for: ' + pair + str(date_start)) 55 | continue 56 | 57 | df = pd.DataFrame(trades) 58 | df = df.infer_objects() 59 | df['exchange'] = self.exchange_name 60 | df['pair'] = pair 61 | df = df.apply(pd.to_numeric, errors="ignore") 62 | # df = df.convert_objects(convert_numeric=True) 63 | df['date'] = (pd.to_datetime(df.date).view(np.int64)/10e8).astype(int) 64 | df['id'] = self.exchange_name + '-' + pair + '-' + df['globalTradeID'].astype(str) 65 | id_list = list(df['id']) 66 | # self.db_ticker.update_one({'id': unique_id}, {'$set': new_db_item}, upsert=True) 67 | existing_ids_df = pd.DataFrame(list(self.db_trades.find({'id': {'$in': id_list}}, {'id': 1}))) 68 | # Check if db contains already values that we want to insert 69 | if not existing_ids_df.empty: 70 | existing_ids = list(existing_ids_df['id']) 71 | df = df[~df['id'].isin(existing_ids)] 72 | # Drop existing records 73 | df = df[~df['id'].isin(existing_ids)] 74 | if df.empty: 75 | continue 76 | records = json.loads(df.T.to_json()).values() 77 | self.db_trades.insert(records) 78 | 79 | time_end = time.time() 80 | duration_in_sec = int(time_end-time_start) 81 | logger.info("Backfill done in (sec) " + str(duration_in_sec)) 82 | -------------------------------------------------------------------------------- /exchanges/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from strategies.enums import TradeState 3 | from termcolor import colored 4 | import configargparse 5 | 6 | 7 | class Base(ABC): 8 | """ 9 | Base class for all exchanges 10 | """ 11 | arg_parser = configargparse.get_argument_parser() 12 | arg_parser.add('--fixed_trade_amount', help='Fixed trade amount') 13 | 14 | transaction_fee = 0.0 15 | pair_delimiter = '_' 16 | 17 | def __init__(self): 18 | super(Base, self).__init__() 19 | args = self.arg_parser.parse_known_args()[0] 20 | self.pair_delimiter = '_' 21 | self.fixed_trade_amount = float(args.fixed_trade_amount) 22 | 23 | @abstractmethod 24 | def get_market_history(self, date_from, date_to, currency_pair='all'): 25 | """ 26 | Returns market trade history 27 | """ 28 | pass 29 | 30 | @abstractmethod 31 | def get_open_orders(self, currency_pair='all'): 32 | """ 33 | Returns your open orders 34 | """ 35 | pass 36 | 37 | def get_pair_delimiter(self): 38 | """ 39 | Returns exchanges pair delimiter 40 | """ 41 | return self.pair_delimiter 42 | 43 | def get_transaction_fee(self): 44 | """ 45 | Returns exchanges transaction fee 46 | """ 47 | return self.transaction_fee 48 | 49 | @abstractmethod 50 | def cancel_order(self, order_number): 51 | """ 52 | Cancels order for given order number 53 | """ 54 | pass 55 | 56 | @classmethod 57 | def trade(cls, actions, wallet, trade_mode): 58 | """ 59 | Apply given actions and returns updated wallet - Base class only simulates buy/sell. 60 | For exchange the buy/sel logic should be implemented here 61 | """ 62 | 63 | for action in actions: 64 | print('applying action:', action.action, ', for pair: ', action.pair) 65 | # TODO: check if we already don't have the same action in process 66 | 67 | # In simulation we are just buying straight currency 68 | return wallet 69 | 70 | def get_buy_sell_all_amount(self, wallet, action): 71 | """ 72 | Calculates total amount for ALL assets in wallet 73 | """ 74 | if action.action == TradeState.none: 75 | return 0.0 76 | 77 | if action.rate == 0.0: 78 | print(colored('Got zero rate!. Can not calc. buy_sell_amount for pair: ' + action.pair, 'red')) 79 | return 0.0 80 | 81 | (symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter)) 82 | amount = 0.0 83 | if action.action == TradeState.buy and symbol_1 in wallet: 84 | assets = wallet.get(symbol_1) 85 | amount = assets / action.rate 86 | elif action.action == TradeState.sell and symbol_2 in wallet: 87 | assets = wallet.get(symbol_2) 88 | amount = assets 89 | 90 | if amount <= 0.0: 91 | return 0.0 92 | 93 | txn_fee_amount = (self.transaction_fee * amount) / 100.0 94 | amount -= txn_fee_amount 95 | return amount 96 | 97 | def get_fixed_trade_amount(self, wallet, action): 98 | """ 99 | Calculates fixed trade amount given action 100 | """ 101 | if action.action == TradeState.none: 102 | return 0.0 103 | 104 | if action.rate == 0.0: 105 | print(colored('Got zero rate!. Can not calc. buy_sell_amount for pair: ' + action.pair, 'red')) 106 | return 0.0 107 | 108 | (symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter)) 109 | amount = 0.0 110 | if action.action == TradeState.buy and symbol_1 in wallet: 111 | assets = self.fixed_trade_amount 112 | amount = assets / action.rate 113 | elif action.action == TradeState.sell and symbol_2 in wallet: 114 | assets = wallet.get(symbol_2) 115 | amount = assets 116 | 117 | if amount <= 0.0: 118 | return 0.0 119 | return amount 120 | -------------------------------------------------------------------------------- /ai/blueprints/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from termcolor import colored 3 | import configargparse 4 | import pandas as pd 5 | 6 | 7 | class Base(ABC): 8 | """ 9 | Base class for all simulation types (sim, paper, trade) 10 | """ 11 | arg_parser = configargparse.get_argument_parser() 12 | arg_parser.add('--price_intervals', 13 | help='Price intervals saved in dataset (minutes) ', 14 | default='120, 360, 720') 15 | 16 | feature_names = [] 17 | scans_container = [] 18 | min_history_ticks = 0 19 | 20 | def __init__(self, name, pairs): 21 | super(Base, self).__init__() 22 | self.name = name 23 | self.pairs = pairs 24 | arg_parser = configargparse.get_argument_parser() 25 | self.args = arg_parser.parse_known_args()[0] 26 | self.price_intervals = [int(x.strip()) for x in self.args.price_intervals.split(',')] 27 | self.Y_prefix = 'Y_' 28 | self.Yt_prefix = 'Yt_' 29 | self.Yt_column_names = self.create_yt_column_names(self.price_intervals, self.Yt_prefix) 30 | 31 | def get_feature_names(self): 32 | """ 33 | Returns feature names 34 | """ 35 | if len(self.scans_container) == 0: 36 | print(colored('Not enough data to get features name!', 'red')) 37 | return [] 38 | df = self.scans_container[0][2] 39 | columns = df.columns.values.tolist() 40 | return columns 41 | 42 | @staticmethod 43 | @abstractmethod 44 | def calculate_features(df): 45 | """ 46 | Method which calculates and generates features 47 | """ 48 | 49 | @staticmethod 50 | def create_yt_column_names(intervals, prefix): 51 | return [prefix + str(interval) for interval in intervals] 52 | 53 | def scan(self, 54 | ticker_df=None, 55 | ticker_size=5): 56 | """ 57 | Function that generates a blueprint from given dataset 58 | """ 59 | final_scan_df = pd.DataFrame() 60 | if ticker_df.empty: 61 | return final_scan_df 62 | 63 | for pair_name in self.pairs: 64 | # Check if we have enough datasets 65 | pair_ticker_df = ticker_df.loc[ticker_df['pair'] == pair_name].sort_values('date') 66 | if len(pair_ticker_df.index) < self.min_history_ticks: 67 | continue 68 | 69 | # Create features 70 | features_df = self.calculate_features(pair_ticker_df.copy()) 71 | # Initial output fields 72 | features_df = self.add_empty_outputs(features_df) 73 | self.scans_container.append((pair_name, 1, features_df)) 74 | # Update stored scans 75 | final_scan = self.update(pair_name, pair_ticker_df, ticker_size) 76 | if final_scan: 77 | df_t = final_scan[2].copy() 78 | final_scan_df = final_scan_df.append(df_t, ignore_index=True) 79 | return final_scan_df 80 | 81 | def add_empty_outputs(self, df): 82 | """ 83 | Creates interval columns with its interval as value 84 | """ 85 | for interval in self.price_intervals: 86 | df[self.Yt_prefix+str(interval)] = None 87 | return df 88 | 89 | def update(self, pair_name, df, ticker_size): 90 | """ 91 | Updates Y price intervals 92 | """ 93 | # 1) get all scans for particular pair_name 94 | for idx, (pair, iter_counter, scan_df) in enumerate(self.scans_container[:]): 95 | if pair != pair_name: 96 | continue 97 | 98 | passed_interval = iter_counter * ticker_size 99 | scan_complete = True 100 | pair_df = df.loc[df['pair'] == pair] 101 | 102 | if pair_df.empty: 103 | print(colored('Got empty pair_df', 'red')) 104 | continue 105 | 106 | # Update Yt (target) intervals 107 | for Yt_name in self.Yt_column_names: 108 | interval = int(Yt_name.replace(self.Yt_prefix, '')) 109 | 110 | # If we have enough data and our target value is empty save it 111 | if passed_interval >= interval and not scan_df.iloc[-1].get(Yt_name): 112 | interval_date = scan_df['date'].iloc[-1] + interval*60 113 | interval_df_idx = (pair_df['date'].searchsorted(interval_date, side='right'))[0] 114 | if interval_df_idx > 0: 115 | interval_df_idx -= 1 116 | interval_df = pair_df.iloc[interval_df_idx] 117 | interval_df_date = interval_df['date'] 118 | if interval_df_date > interval_date: 119 | print(colored("Problem while blueprint scan - invalid dates!!", 'red')) 120 | exit(1) 121 | close_price = interval_df['close'] 122 | scan_df[Yt_name] = close_price 123 | # print('______________' + Yt_name + ', passed_interval: ' + str(passed_interval) + ', interval: ' 124 | # + str(interval)) 125 | # print('adding_close_value:', interval_df) 126 | # print('scan_df:', scan_df) 127 | else: 128 | column_value = scan_df.iloc[-1].get(Yt_name) 129 | if not column_value: 130 | scan_complete = False 131 | 132 | tmp_scan_list = list(self.scans_container[idx]) 133 | tmp_scan_list[1] = iter_counter + 1 134 | tmp_scan_list[2] = scan_df.copy() 135 | self.scans_container[idx] = tuple(tmp_scan_list) 136 | 137 | # Check if we have all Yt data. If yes, return it 138 | if scan_complete: 139 | final_scan_df = self.scans_container[idx] 140 | self.scans_container.remove(final_scan_df) 141 | return final_scan_df 142 | return None 143 | -------------------------------------------------------------------------------- /ai/blueprint.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import logging 4 | import pandas as pd 5 | import configargparse 6 | import core.common as common 7 | from exchanges.exchange import Exchange 8 | from termcolor import colored 9 | 10 | 11 | class Blueprint: 12 | """ 13 | Main module for generating and handling datasets for AI. Application will generate datasets including 14 | future target/output parameters. 15 | """ 16 | arg_parser = configargparse.get_argument_parser() 17 | arg_parser.add('--days', help='Days to start blueprint from', default=30) 18 | arg_parser.add('-f', '--features', help='Blueprints module name to be used to generated features', required=True) 19 | arg_parser.add('--ticker_size', help='Size of the candle ticker (minutes)', default=5) 20 | arg_parser.add('--pairs', help='Pairs to blueprint') 21 | arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true') 22 | arg_parser.add("--buffer_size", help="Maximum Buffer size (days)", default=30) 23 | arg_parser.add("--output_dir", help="Output directory") 24 | 25 | logger = logging.getLogger(__name__) 26 | features_list = None 27 | exchange = None 28 | blueprint = None 29 | out_dir = 'out/blueprints/' 30 | 31 | def __init__(self): 32 | args = self.arg_parser.parse_known_args()[0] 33 | self.blueprint_days = args.days 34 | self.ticker_size = int(args.ticker_size) 35 | self.blueprint_end_time = int(time.time()) 36 | self.start_time = self.blueprint_end_time - int(self.blueprint_days)*86400 37 | self.ticker_epoch = self.start_time 38 | self.exchange = Exchange(None, ticker_size=self.ticker_size) 39 | self.pairs = common.parse_pairs(self.exchange, args.pairs) 40 | blueprints_module = common.load_module('ai.blueprints.', args.features) 41 | self.blueprint = blueprints_module(self.pairs) 42 | self.max_buffer_size = int(int(args.buffer_size) * (1440 / self.ticker_size) * len(self.pairs)) 43 | self.df_ticker_buffer = pd.DataFrame() 44 | self.df_blueprint = pd.DataFrame() 45 | self.output_dir = args.output_dir 46 | self.export_file_name = self.get_output_file_path(self.output_dir, self.blueprint.name) 47 | self.export_file_initialized = False 48 | 49 | # Crete output dir 50 | if not os.path.exists(self.out_dir): 51 | os.makedirs(self.out_dir) 52 | 53 | @staticmethod 54 | def get_output_file_path(dir_path, blueprint_name): 55 | filename = 'blueprint_' + blueprint_name + '_' + str(int(time.time())) + '.csv' 56 | if dir_path: 57 | if not dir_path.endswith(os.path.sep): 58 | dir_path += os.path.sep 59 | filename = dir_path + filename 60 | return filename 61 | 62 | def print_progress_dot(self, counter): 63 | """ 64 | Prints progress 65 | """ 66 | if counter % 100 == 0: 67 | print('.', end='', flush=True) 68 | if counter > 101: 69 | counter = 0 70 | self.write_to_file() 71 | return counter+1 72 | 73 | def write_to_file(self): 74 | """ 75 | Writes df to file 76 | """ 77 | if self.df_blueprint.empty: 78 | print('Blueprint is empty, nothing to write to file.') 79 | return 80 | 81 | export_df = self.df_blueprint.copy() 82 | dropping_columns = ['_id', 'id', 'curr_1', 'curr_2', 'exchange'] 83 | df_columns = self.blueprint.get_feature_names() 84 | df_columns = [x for x in df_columns if x not in dropping_columns] 85 | export_df = export_df.drop(dropping_columns, axis=1) 86 | export_df = export_df[df_columns] 87 | dt = export_df.tail(1).date.iloc[0] 88 | dt_string = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(dt)) 89 | print('saving,..(last df date: ' + dt_string + ')') 90 | if not self.export_file_initialized: 91 | export_df.to_csv(self.out_dir + self.export_file_name, index=False, columns=df_columns) 92 | self.export_file_initialized = True 93 | else: 94 | export_df.to_csv(self.out_dir + self.export_file_name, 95 | mode='a', 96 | header=False, 97 | index=False, 98 | columns=df_columns) 99 | 100 | self.df_blueprint = self.df_blueprint[0:0] 101 | 102 | def run(self): 103 | """ 104 | Calculates and stores dataset 105 | """ 106 | info_text = 'Starting generating data for Blueprint ' + self.blueprint.name + ', back-days ' + \ 107 | self.blueprint_days + ' (This might take several hours/days,.so please stay back and relax)' 108 | print(colored(info_text, 'yellow')) 109 | dot_counter = 0 110 | while True: 111 | # Get new dataset 112 | time_now = int(time.time()) 113 | df_ticker = self.exchange.get_offline_ticker(self.ticker_epoch, self.pairs) 114 | time_diff = int(time.time()) - time_now 115 | self.logger.debug('df_ticker fetched in sec:' + str(time_diff)) 116 | 117 | time_now = int(time.time()) 118 | df_trades = self.exchange.get_offline_trades(self.ticker_epoch, self.pairs) 119 | time_diff = int(time.time()) - time_now 120 | self.logger.debug('df_trades fetched in sec:' + str(time_diff)) 121 | 122 | if df_trades.empty: 123 | df = df_ticker 124 | else: 125 | df = df_ticker.merge(df_trades) 126 | 127 | # Check if the simulation is finished 128 | if self.ticker_epoch >= self.blueprint_end_time: 129 | self.write_to_file() 130 | return 131 | 132 | # Store ticker to buffer 133 | if not self.df_ticker_buffer.empty: 134 | df = df[list(self.df_ticker_buffer)] 135 | self.df_ticker_buffer = self.df_ticker_buffer.append(df, ignore_index=True) 136 | else: 137 | self.df_ticker_buffer = self.df_ticker_buffer.append(df, ignore_index=True) 138 | self.df_ticker_buffer = common.handle_buffer_limits(self.df_ticker_buffer, self.max_buffer_size) 139 | 140 | scan_df = self.blueprint.scan(ticker_df=self.df_ticker_buffer, 141 | ticker_size=self.ticker_size) 142 | if not scan_df.empty: 143 | dot_counter = self.print_progress_dot(dot_counter) 144 | self.df_blueprint = self.df_blueprint.append(scan_df, ignore_index=True) 145 | 146 | self.ticker_epoch += self.ticker_size*60 147 | 148 | @staticmethod 149 | def add_trades(ticker_df, trades_df): 150 | """ 151 | Merges trades with ticker data 152 | """ 153 | print('w') -------------------------------------------------------------------------------- /utils/walletlense.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import pandas as pd 4 | import configargparse 5 | from termcolor import colored 6 | from backfill.candles import Candles 7 | from exchanges.exchange import Exchange 8 | from utils.postman import Postman 9 | import telegram 10 | 11 | 12 | class WalletLense: 13 | """ 14 | Lense: Returns actual wallet statistics with simple daily digest (winners / losers) 15 | """ 16 | arg_parser = configargparse.get_argument_parser() 17 | analysis_days = 1 18 | time_intervals_hours = [1, 3, 6, 12, 24] 19 | 20 | def __init__(self): 21 | self.args = self.arg_parser.parse_known_args()[0] 22 | print(colored('Starting lense on exchange: ' + self.args.exchange, 'yellow')) 23 | self.exchange = Exchange() 24 | self.postman = Postman() 25 | 26 | def get_stats(self): 27 | """ 28 | Returns current statistics (market and wallet) 29 | """ 30 | self.fetch_last_ticker(self.analysis_days) 31 | df_candles = self.get_ticker(self.exchange.get_pairs(), self.analysis_days) 32 | df_candles.to_csv('test_ticker.csv', index=False) 33 | # df_candles = pd.read_csv('test_ticker.csv') 34 | 35 | # Parse all df's to html 36 | html_body = [''] 37 | html_body.append('Mosquito') 38 | winners, losers = self.get_winners_losers(df_candles) 39 | html_body.append(self.parse_winners_losers_to_html(winners, losers)) 40 | 41 | # wallet_stats = self.get_wallet_stats(ticker) 42 | print('wallet stats:') 43 | html_body.append('') 44 | self.send_email(html_body) 45 | 46 | @staticmethod 47 | def df_to_html(df, header, bar_color='lightblue'): 48 | """ 49 | Converts DataFrame to html text 50 | """ 51 | df_header = '

' + header + '

' 52 | table = (df.style.set_properties(**{'font-size': '9pt', 'font-family': 'Calibri'}) 53 | .set_precision(3) 54 | # .background_gradient(subset=['price_change'], cmap='PuBu', low=-100.0, high=100.0) 55 | .set_table_styles([{'selector': '.row_heading, .blank', 'props': [('display', 'none;')]}]) 56 | .bar(subset=['price_change'], color=bar_color) 57 | .render() 58 | ) 59 | return df_header + table 60 | 61 | def send_email(self, body_list): 62 | """ 63 | Sending email module 64 | """ 65 | 66 | body = '' 67 | for body_item in body_list: 68 | body += '\n' + body_item 69 | self.postman.send_mail('mosquito_stats', body) 70 | 71 | def get_wallet_stats(self, ticker): 72 | """ 73 | Returns simple wallet stats 74 | """ 75 | wallet = self.exchange.get_balances() 76 | return wallet 77 | 78 | def parse_winners_losers_to_html(self, winners, losers): 79 | """ 80 | Converts Winners and Losers df's to html 81 | """ 82 | html = '

Winners

' 83 | grouped_winners = winners.groupby(['hour_spam']) 84 | for key, df in grouped_winners: 85 | df = df.drop(['hour_spam'], axis=1) 86 | # Reorder columns 87 | df = df[['price_change', 'pair', 'V', 'Vq']] 88 | html += self.df_to_html(df, str(key) + '-Hour', bar_color='lightgreen') 89 | 90 | html += '
' 91 | 92 | html += '

Losers

' 93 | grouped_losers = losers.groupby(['hour_spam']) 94 | for key, df in grouped_losers: 95 | df = df.drop(['hour_spam'], axis=1) 96 | df = df.sort_values(['price_change'], ascending=[1]) 97 | df['price_change'] = df['price_change'] * -1.0 98 | # Reorder columns 99 | df = df[['price_change', 'pair', 'V', 'Vq']] 100 | html += self.df_to_html(df, str(key) + '-Hour', bar_color='lightpink') 101 | 102 | return html 103 | 104 | def get_ticker(self, pairs, history_days): 105 | """ 106 | Gets ticker for given list of pairs and given perion 107 | """ 108 | print('Getting tickers.. (might take a while)') 109 | ticker_to = int(time.time()) 110 | ticker_from = ticker_to - (24 * history_days * 3600) 111 | df_pairs_ticker = pd.DataFrame() 112 | for pair in pairs: 113 | ticker_list = self.exchange.get_candles(pair, ticker_from, ticker_to, period=1800) 114 | df_ticker = pd.DataFrame(ticker_list) 115 | df_ticker['pair'] = pair 116 | df_pairs_ticker = df_pairs_ticker.append(df_ticker, ignore_index=True) 117 | return df_pairs_ticker 118 | 119 | def get_winners_losers(self, df_ticker): 120 | """ 121 | Get winners/losers 122 | """ 123 | grouped = df_ticker.groupby(['pair']) 124 | df_stats = pd.DataFrame() 125 | for name, df_group in grouped: 126 | pair_stat = self.get_pair_stats(name, df_group, self.time_intervals_hours) 127 | df_s = pd.DataFrame(pair_stat) 128 | df_stats = df_stats.append(df_s, ignore_index=True) 129 | grouped_stats = df_stats.groupby(['hour_spam']) 130 | winners = pd.DataFrame() 131 | losers = pd.DataFrame() 132 | for interval, df_group in grouped_stats: 133 | sorted_intervals = df_group.sort_values('price_change', ascending=False) 134 | winners = winners.append(sorted_intervals.head(5)) 135 | losers = losers.append(sorted_intervals.tail(5)) 136 | return winners, losers 137 | 138 | def get_pair_stats(self, pair, df, hour_intervals): 139 | """ 140 | Returns statistics summary 141 | """ 142 | df_now = df.tail(1) 143 | date_end = df_now['date'].iloc[0] 144 | dates = df['date'] 145 | stats = [] 146 | for hour_interval in hour_intervals: 147 | next_epoch = hour_interval * 3600 148 | closest_date_idx = self.find_nearest(dates, date_end - next_epoch) 149 | closest_df = df.loc[closest_date_idx] 150 | df_interval = df.loc[df['date'] == closest_df.date] 151 | pair_stats = self.calc_pair_stats(df_now.iloc[0], df_interval.iloc[0]) 152 | pair_stats['pair'] = pair 153 | pair_stats['hour_spam'] = hour_interval 154 | stats.append(pair_stats) 155 | return stats 156 | 157 | @staticmethod 158 | def calc_pair_stats(ticker_now, ticker_past): 159 | stats = dict() 160 | price_change = ((float(ticker_past.close) * 100.0) / float(ticker_now.close)) - 100.0 161 | volume_perc_change = ((float(ticker_past.volume) * 100.0) / float(ticker_now.volume)) - 100.0 162 | quote_volume_perc_change = ((float(ticker_past.quoteVolume) * 100.0) / float(ticker_now.quoteVolume)) - 100.0 163 | stats['price_change'] = price_change 164 | stats['V'] = volume_perc_change 165 | stats['Vq'] = quote_volume_perc_change 166 | return stats 167 | 168 | @staticmethod 169 | def find_nearest(array, value): 170 | idx = (np.abs(array - value)).argmin() 171 | return idx 172 | 173 | def fetch_last_ticker(self, prefetch_days): 174 | """ 175 | Prefetch data for all pairs for given days 176 | """ 177 | self.args.days = prefetch_days 178 | self.args.all = True 179 | backfill_client = Candles() 180 | backfill_client.run() 181 | -------------------------------------------------------------------------------- /stats/stats.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | import core.common as common 3 | import pandas as pd 4 | from core.bots.paper import Paper 5 | from termcolor import colored 6 | from core.bots.backtest import Backtest 7 | from core.bots.enums import TradeMode 8 | from core.bots.live import Live 9 | from core.plot import Plot 10 | from core.report import Report 11 | from core.wallet import Wallet 12 | 13 | 14 | class Stats: 15 | """ 16 | Main class for statistics 17 | """ 18 | 19 | arg_parser = configargparse.get_argument_parser() 20 | 21 | def __init__(self): 22 | self.args = self.arg_parser.parse_known_args()[0] 23 | 24 | def run(self): 25 | print("running stats") 26 | 27 | 28 | """ 29 | arg_parser.add("--plot", help="Generate a candle stick plot at simulation end", action='store_true') 30 | arg_parser.add("--ticker_size", help="Simulation ticker size", default=5) 31 | arg_parser.add("--root_report_currency", help="Root currency used in final plot") 32 | arg_parser.add("--prefetch", help="Prefetch data from history DB", action='store_true') 33 | arg_parser.add("--all", help="Include all currencies/tickers") 34 | arg_parser.add("--days", help="Days to backtest") 35 | 36 | buffer_size = None 37 | interval = None 38 | pairs = None 39 | verbosity = None 40 | ticker = None 41 | look_back = None 42 | history = None 43 | bot = None 44 | report = None 45 | plot = None 46 | plot_pair = None 47 | trade_mode = None 48 | root_report_currency = None 49 | config_strategy_name = None 50 | actions = None 51 | prefetch = None 52 | first_ticker = None 53 | last_valid_ticker = None 54 | 55 | def __init__(self): 56 | self.args = self.arg_parser.parse_known_args()[0] 57 | self.parse_config() 58 | strategy_class = common.load_module('strategies.', self.args.strategy) 59 | self.wallet = Wallet() 60 | self.history = pd.DataFrame() 61 | trade_columns = ['date', 'pair', 'close_price', 'action'] 62 | self.trades = pd.DataFrame(columns=trade_columns, index=None) 63 | if self.args.backtest: 64 | self.bot = Backtest(self.wallet.initial_balance.copy()) 65 | self.trade_mode = TradeMode.backtest 66 | elif self.args.paper: 67 | self.bot = Paper(self.wallet.initial_balance.copy()) 68 | self.trade_mode = TradeMode.paper 69 | self.wallet.initial_balance = self.bot.get_balance() 70 | self.wallet.current_balance = self.bot.get_balance() 71 | elif self.args.live: 72 | self.bot = Live() 73 | self.trade_mode = TradeMode.live 74 | self.wallet.initial_balance = self.bot.get_balance() 75 | self.wallet.current_balance = self.bot.get_balance() 76 | self.strategy = strategy_class() 77 | self.pairs = self.bot.get_pairs() 78 | self.look_back = pd.DataFrame() 79 | self.max_lookback_size = int(self.buffer_size*(1440/self.interval)*len(self.pairs)) 80 | self.initialize() 81 | 82 | def initialize(self): 83 | # Initialization 84 | self.report = Report(self.wallet.initial_balance, 85 | self.pairs, 86 | self.root_report_currency, 87 | self.bot.get_pair_delimiter()) 88 | self.report.set_verbosity(self.verbosity) 89 | self.plot = Plot() 90 | 91 | def parse_config(self): 92 | 93 | self.root_report_currency = self.args.root_report_currency 94 | self.buffer_size = self.args.buffer_size 95 | self.prefetch = self.args.prefetch 96 | if self.buffer_size != '': 97 | self.buffer_size = int(self.buffer_size) 98 | self.interval = self.args.ticker_size 99 | if self.interval != '': 100 | self.interval = int(self.interval) 101 | self.config_strategy_name = self.args.strategy 102 | self.plot_pair = self.args.plot_pair 103 | self.verbosity = self.args.verbosity 104 | 105 | def on_simulation_done(self): 106 | 107 | print('shutting down and writing final statistics!') 108 | strategy_info = self.report.write_final_stats(self.first_ticker.copy(), 109 | self.last_valid_ticker.copy(), 110 | self.wallet, self.trades) 111 | if self.args.plot: 112 | plot_title = ['Simulation: ' + str(self.trade_mode) + ' Strategy: ' + self.config_strategy_name + ', Pair: ' 113 | + str(self.pairs)] 114 | strategy_info = plot_title + strategy_info 115 | self.plot.draw(self.history, 116 | self.trades, 117 | self.plot_pair, 118 | strategy_info) 119 | 120 | @staticmethod 121 | def validate_ticker(df): 122 | 123 | columns_to_check = ['close', 'volume'] 124 | df_column_names = list(df) 125 | if not set(columns_to_check).issubset(df_column_names): 126 | return False 127 | nan_columns = df.columns[df.isnull().any()].tolist() 128 | nans = [i for i in nan_columns if i in columns_to_check] 129 | if len(nans) > 0: 130 | return False 131 | return True 132 | 133 | def simulation_finished(self, new_ticker): 134 | 135 | # If we got an empty dataset finish simulation 136 | if new_ticker.empty: 137 | print('New ticker is empty,..') 138 | return True 139 | 140 | # If we have not data just continue 141 | if self.ticker is None: 142 | return False 143 | 144 | # If we have received the same date,..we assume that we have no more data,..finish simulation 145 | if not self.ticker.empty and (self.ticker is not None and new_ticker.equals(self.ticker)): 146 | print(colored('Received ticker data that equal to previous data (this can happen,.but not too often): ' 147 | + str(new_ticker.date[0]), 'yellow')) 148 | if self.trade_mode == TradeMode.backtest: 149 | return True 150 | else: 151 | return False 152 | 153 | return False 154 | 155 | def run(self): 156 | 157 | if self.bot is None: 158 | print(colored('The bots type is NOT specified. You need to choose one action (--sim, --paper, --trade)', 'red')) 159 | exit(0) 160 | 161 | print(colored('Starting simulation: ' + str(self.trade_mode) + ', Strategy: ' + self.config_strategy_name, 'yellow')) 162 | 163 | # Prefetch Buffer Data (if enabled in config) 164 | if self.prefetch: 165 | self.history = self.bot.prefetch(self.strategy.get_min_history_ticks(), self.interval) 166 | self.look_back = self.history.copy() 167 | 168 | try: 169 | while True: 170 | # Get next ticker 171 | new_ticker = self.bot.get_next(self.interval) 172 | # print('new_ticker____:', new_ticker) 173 | if self.simulation_finished(new_ticker): 174 | print("No more data,..simulation done,. quitting") 175 | exit(0) 176 | 177 | self.ticker = new_ticker 178 | 179 | # Check if ticker is valid 180 | if not self.validate_ticker(self.ticker): 181 | print(colored('Received invalid ticker, will have to skip it! Details:\n' + str(self.ticker), 'red')) 182 | continue 183 | 184 | # Save ticker to buffer 185 | self.history = self.history.append(self.ticker, ignore_index=True) 186 | self.look_back = self.look_back.append(self.ticker, ignore_index=True) 187 | 188 | # Check if buffer is not overflown 189 | self.look_back = common.handle_buffer_limits(self.look_back, self.max_lookback_size) 190 | 191 | if self.first_ticker is None or self.first_ticker.empty: 192 | self.first_ticker = self.ticker.copy() 193 | self.last_valid_ticker = self.ticker.copy() 194 | 195 | # Get next actions 196 | self.actions = self.strategy.calculate(self.look_back, self.wallet) 197 | 198 | # Set trade 199 | self.actions = self.bot.trade(self.actions, 200 | self.wallet.current_balance, 201 | self.trades) 202 | 203 | # Get wallet balance 204 | self.wallet.current_balance = self.bot.get_balance() 205 | 206 | # Write report 207 | self.report.calc_stats(self.ticker, self.wallet) 208 | 209 | except KeyboardInterrupt: 210 | self.on_simulation_done() 211 | 212 | except SystemExit: 213 | self.on_simulation_done() 214 | 215 | """ 216 | 217 | 218 | 219 | 220 | 221 | -------------------------------------------------------------------------------- /core/report.py: -------------------------------------------------------------------------------- 1 | import time 2 | import pandas as pd 3 | from dateutil.tz import * 4 | from termcolor import colored 5 | from datetime import datetime 6 | 7 | 8 | class Report: 9 | """ 10 | Main reporting class (overall score and plots) 11 | """ 12 | 13 | initial_closing_prices = None 14 | initial_balance = 0.0 15 | verbosity = None 16 | 17 | def __init__(self, 18 | initial_wallet, 19 | pairs, 20 | root_report_currency, 21 | pair_delimiter): 22 | self.initial_wallet = initial_wallet 23 | self.pairs = pairs 24 | self.initial_closing_prices = pd.DataFrame() 25 | self.root_report_currency = root_report_currency 26 | self.sim_start_epoch = int(time.time()) 27 | self.pair_delimiter = pair_delimiter 28 | 29 | def set_verbosity(self, verbosity): 30 | self.verbosity = verbosity 31 | 32 | def write_final_stats(self, ticker_start, ticker_end, wallet, trades): 33 | """ 34 | Final statistics report 35 | """ 36 | curr_bal_percent = self.calc_balance(ticker_end, wallet.current_balance) 37 | buy_and_hold = self.calc_buy_and_hold(ticker_end, wallet.initial_balance) 38 | print('') 39 | print('****************************************************') 40 | print('* Final simulation report: *') 41 | print('****************************************************') 42 | complete_report = [] 43 | wallet_start_text = 'Wallet at Start:' + self.get_wallet_text(wallet.initial_balance) 44 | print(wallet_start_text) 45 | # complete_report.append(wallet_start_text) 46 | wallet_end_text = 'Wallet at End:' + self.get_wallet_text(wallet.current_balance) 47 | print(wallet_end_text) 48 | # complete_report.append(wallet_end_text) 49 | print(self.get_color_text('Strategy result: ', curr_bal_percent)) 50 | strategy_result = 'Strategy result: ' + str(round(curr_bal_percent, 2)) + '%, ' + 'Buy & Hold: ' + \ 51 | str(round(buy_and_hold, 2)) + '%' 52 | complete_report.append(strategy_result) 53 | print(self.get_color_text('Buy & Hold: ', buy_and_hold)) 54 | print(self.get_color_text('Strategy vs Buy & Hold: ', curr_bal_percent-buy_and_hold)) 55 | # Get real sim time values 56 | sim_start = ticker_start.date.iloc[0] 57 | sim_end = ticker_end.date.iloc[0] 58 | minutes, seconds = divmod(sim_end-sim_start, 60) 59 | hours, minutes = divmod(minutes, 60) 60 | days, hours = divmod(hours, 24) 61 | print('Total txn:', len(trades)) 62 | complete_report.append('Total txn: ' + str(len(trades)) + ' in ' + str(days) + ' days ' + str(hours) + ' hours and ' 63 | + str(minutes) + ' minutes') 64 | print('Simulated (data time):', days, 'days,', hours, 'hours and', minutes, 'minutes') 65 | if sim_end-sim_start == 0: 66 | txn_per_hour = len(trades) 67 | else: 68 | txn_per_hour = len(trades)/((sim_end-sim_start)/3600.0) 69 | print('Transactions per hour:', round(txn_per_hour, 2)) 70 | # Get simulation run time values 71 | time_now = int(time.time()) 72 | time_diff = time_now - self.sim_start_epoch 73 | minutes, seconds = divmod(time_diff, 60) 74 | hours, minutes = divmod(minutes, 60) 75 | days, hours = divmod(hours, 24) 76 | print('Simulation run time:', hours, 'hours', minutes, 'minutes and', seconds, 'seconds') 77 | return complete_report 78 | 79 | def calc_stats(self, ticker_data, wallet): 80 | """ 81 | Creates ticker report 82 | """ 83 | # Store initial closing price and initial overall wallet balance 84 | if len(self.initial_closing_prices.index) < len(self.pairs): 85 | self.initialize_start_price(ticker_data) 86 | self.initial_balance = self.initialize_start_balance(self.initial_wallet, 87 | self.initial_closing_prices) 88 | epoch_dt = ticker_data['date'].iloc[0] 89 | utc_dt = datetime.fromtimestamp(epoch_dt, tz=tzutc()) 90 | local_dt = utc_dt.astimezone(tzlocal()) 91 | 92 | date_time = 'Local timestamp: ' + local_dt.strftime('%c') + ',' 93 | # current_close = 'close:' + format(ticker_data.iloc[0]['close'], '2f') 94 | # Wallet 95 | wallet_text = self.get_wallet_text(wallet.current_balance) 96 | # Balance 97 | balance = self.calc_balance(ticker_data, wallet.current_balance) 98 | balance_text = self.get_color_text('$: ', balance) + ',' 99 | # Buy & Hold 100 | bh = self.calc_buy_and_hold(ticker_data, wallet.initial_balance) 101 | bh_text = self.get_color_text('b&h: ', bh) 102 | print(date_time, 103 | balance_text, 104 | bh_text, 105 | wallet_text 106 | ) 107 | 108 | return 0 109 | 110 | @staticmethod 111 | def get_wallet_text(wallet, currencies=None): 112 | """ 113 | Returns wallet balance in string. By default it returns balance of the entire wallet. 114 | You can specify currencies which you would like to receive update 115 | """ 116 | # TODO return only wallet of given currencies 117 | wallet_string = '' 118 | for symbol, balance in wallet.items(): 119 | if balance > 0: 120 | wallet_string += '| ' + str(balance) + ' ' + symbol 121 | wallet_string += ' |' 122 | return wallet_string 123 | 124 | @staticmethod 125 | def get_color_text(text, value): 126 | """ 127 | Returns colored text 128 | """ 129 | v = round(value, 2) + 0.0 130 | output_text = text + str(round(v, 2)) + '%' 131 | color = 'green' if round(v, 2) >= 0 else 'red' 132 | return colored(output_text, color) 133 | 134 | def get_exchange_rate_value(self, currency, ticker_data, value, root_currency): 135 | """ 136 | Returns currencies exchange value towards the root_currency 137 | """ 138 | # 1) If currency is root one, just return it 139 | if currency == root_currency: 140 | return value 141 | 142 | # 2) If we have exchange rate towards root_currency, return that one 143 | pair = root_currency + self.pair_delimiter + currency 144 | pair_tick = ticker_data.loc[ticker_data.pair == pair] 145 | # pair_tick = ticker_data.loc[ticker_data['pair'] == pair] 146 | if not pair_tick.empty: 147 | closing_price = pair_tick['close'].iloc[0] 148 | return closing_price * value 149 | 150 | # 2) If we didn't find root-currency try to find currency-pair ticker 151 | pair = currency + self.pair_delimiter + root_currency 152 | pair_tick = ticker_data.loc[ticker_data.pair == pair] 153 | # pair_tick = ticker_data.loc[ticker_data['pair'] == pair] 154 | if not pair_tick.empty: 155 | closing_price = pair_tick['close'].iloc[0] 156 | if closing_price == 0.0: 157 | return 0.0 158 | return value/closing_price 159 | 160 | if self.verbosity: 161 | print(colored("Couldn't find exchange rate for:" + currency + '. Report data invalid!', 'red')) 162 | 163 | return 0.0 164 | 165 | def calc_balance(self, ticker_data, wallet_balance): 166 | """ 167 | Calculates current balance (profit/loss) 168 | """ 169 | current_balance = 0 170 | for currency, value in wallet_balance.items(): 171 | pair_value = self.get_exchange_rate_value(currency, ticker_data, value, self.root_report_currency) 172 | current_balance += pair_value 173 | 174 | price_diff = current_balance - self.initial_balance 175 | if self.initial_balance == 0.0: 176 | return 0.0 177 | perc_change = ((price_diff*100.0)/self.initial_balance) 178 | return perc_change 179 | 180 | def initialize_start_price(self, ticker_data): 181 | """ 182 | Save initial closing price 183 | """ 184 | # Get only currencies that have not been initialized yet 185 | for index, row in ticker_data.iterrows(): 186 | pair = row['pair'] 187 | if self.initial_closing_prices.empty: 188 | self.initial_closing_prices = self.initial_closing_prices.append(row, ignore_index=True) 189 | elif self.initial_closing_prices.pair[self.initial_closing_prices.pair == pair].count() == 0: 190 | self.initial_closing_prices = self.initial_closing_prices.append(row, ignore_index=True) 191 | 192 | def initialize_start_balance(self, wallet, close_prices): 193 | """ 194 | Calculate overall wallet balance in bitcoins 195 | """ 196 | balance = 0.0 197 | for currency, value in wallet.items(): 198 | pair_value = self.get_exchange_rate_value(currency, close_prices, value, self.root_report_currency) 199 | balance += pair_value 200 | return balance 201 | 202 | def calc_buy_and_hold(self, ticker_data, initial_balance): 203 | """ 204 | Calculate Buy & Hold price 205 | """ 206 | return self.calc_balance(ticker_data, initial_balance) 207 | -------------------------------------------------------------------------------- /core/engine.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | import core.common as common 3 | import pandas as pd 4 | from core.bots.paper import Paper 5 | from termcolor import colored 6 | from core.bots.backtest import Backtest 7 | from core.bots.enums import TradeMode 8 | from core.bots.live import Live 9 | from core.plot import Plot 10 | from core.report import Report 11 | from core.wallet import Wallet 12 | 13 | 14 | class Engine: 15 | """ 16 | Main class for Simulation Engine (main class where all is happening 17 | """ 18 | 19 | arg_parser = configargparse.get_argument_parser() 20 | arg_parser.add("--strategy", help="Name of strategy to be run (if not set, the default one will be used") 21 | arg_parser.add("--plot", help="Generate a candle stick plot at simulation end", action='store_true') 22 | arg_parser.add("--ticker_size", help="Simulation ticker size", default=5) 23 | arg_parser.add("--root_report_currency", help="Root currency used in final plot") 24 | arg_parser.add("--buffer_size", help="Buffer size in days", default=30) 25 | arg_parser.add("--prefetch", help="Prefetch data from history DB", action='store_true') 26 | arg_parser.add("--plot_pair", help="Plot pair") 27 | arg_parser.add("--all", help="Include all currencies/tickers") 28 | arg_parser.add("--days", help="Days to backtest") 29 | 30 | buffer_size = None 31 | interval = None 32 | pairs = None 33 | verbosity = None 34 | ticker = None 35 | look_back = None 36 | history = None 37 | bot = None 38 | report = None 39 | plot = None 40 | plot_pair = None 41 | trade_mode = None 42 | root_report_currency = None 43 | config_strategy_name = None 44 | actions = None 45 | prefetch = None 46 | first_ticker = None 47 | last_valid_ticker = None 48 | 49 | def __init__(self): 50 | self.args = self.arg_parser.parse_known_args()[0] 51 | self.parse_config() 52 | strategy_class = common.load_module('strategies.', self.args.strategy) 53 | self.wallet = Wallet() 54 | self.history = pd.DataFrame() 55 | trade_columns = ['date', 'pair', 'close_price', 'action'] 56 | self.trades = pd.DataFrame(columns=trade_columns, index=None) 57 | if self.args.backtest: 58 | self.bot = Backtest(self.wallet.initial_balance.copy()) 59 | self.trade_mode = TradeMode.backtest 60 | elif self.args.paper: 61 | self.bot = Paper(self.wallet.initial_balance.copy()) 62 | self.trade_mode = TradeMode.paper 63 | self.wallet.initial_balance = self.bot.get_balance() 64 | self.wallet.current_balance = self.bot.get_balance() 65 | elif self.args.live: 66 | self.bot = Live() 67 | self.trade_mode = TradeMode.live 68 | self.wallet.initial_balance = self.bot.get_balance() 69 | self.wallet.current_balance = self.bot.get_balance() 70 | self.strategy = strategy_class() 71 | self.pairs = self.bot.get_pairs() 72 | self.look_back = pd.DataFrame() 73 | self.max_lookback_size = int(self.buffer_size*(1440/self.interval)*len(self.pairs)) 74 | self.initialize() 75 | 76 | def initialize(self): 77 | # Initialization 78 | self.report = Report(self.wallet.initial_balance, 79 | self.pairs, 80 | self.root_report_currency, 81 | self.bot.get_pair_delimiter()) 82 | self.report.set_verbosity(self.verbosity) 83 | self.plot = Plot() 84 | 85 | def parse_config(self): 86 | """ 87 | Parsing of config.ini file 88 | """ 89 | self.root_report_currency = self.args.root_report_currency 90 | self.buffer_size = self.args.buffer_size 91 | self.prefetch = self.args.prefetch 92 | if self.buffer_size != '': 93 | self.buffer_size = int(self.buffer_size) 94 | self.interval = self.args.ticker_size 95 | if self.interval != '': 96 | self.interval = int(self.interval) 97 | self.config_strategy_name = self.args.strategy 98 | self.plot_pair = self.args.plot_pair 99 | self.verbosity = self.args.verbosity 100 | 101 | def on_simulation_done(self): 102 | """ 103 | Last function called when the simulation is finished 104 | """ 105 | print('shutting down and writing final statistics!') 106 | strategy_info = self.report.write_final_stats(self.first_ticker.copy(), 107 | self.last_valid_ticker.copy(), 108 | self.wallet, self.trades) 109 | if self.args.plot: 110 | plot_title = ['Simulation: ' + str(self.trade_mode) + ' Strategy: ' + self.config_strategy_name + ', Pair: ' 111 | + str(self.pairs)] 112 | strategy_info = plot_title + strategy_info 113 | self.plot.draw(self.history, 114 | self.trades, 115 | self.plot_pair, 116 | strategy_info) 117 | 118 | @staticmethod 119 | def validate_ticker(df): 120 | """ 121 | Validates if the given dataframe contains mandatory fields 122 | """ 123 | columns_to_check = ['close', 'volume'] 124 | df_column_names = list(df) 125 | if not set(columns_to_check).issubset(df_column_names): 126 | return False 127 | nan_columns = df.columns[df.isnull().any()].tolist() 128 | nans = [i for i in nan_columns if i in columns_to_check] 129 | if len(nans) > 0: 130 | return False 131 | return True 132 | 133 | def simulation_finished(self, new_ticker): 134 | """ 135 | Checks if simulation is finished based on new data 136 | """ 137 | # If we got an empty dataset finish simulation 138 | if new_ticker.empty: 139 | print('New ticker is empty,..') 140 | return True 141 | 142 | # If we have not data just continue 143 | if self.ticker is None: 144 | return False 145 | 146 | # If we have received the same date,..we assume that we have no more data,..finish simulation 147 | if not self.ticker.empty and (self.ticker is not None and new_ticker.equals(self.ticker)): 148 | print(colored('Received ticker data that equal to previous data (this can happen,.but not too often): ' 149 | + str(new_ticker.date[0]), 'yellow')) 150 | if self.trade_mode == TradeMode.backtest: 151 | return True 152 | else: 153 | return False 154 | 155 | return False 156 | 157 | def run(self): 158 | """ 159 | This is the main simulation loop 160 | """ 161 | if self.bot is None: 162 | print(colored('The bots type is NOT specified. You need to choose one action (--sim, --paper, --trade)', 'red')) 163 | exit(0) 164 | 165 | print(colored('Starting simulation: ' + str(self.trade_mode) + ', Strategy: ' + self.config_strategy_name, 'yellow')) 166 | 167 | # Prefetch Buffer Data (if enabled in config) 168 | if self.prefetch: 169 | self.history = self.bot.prefetch(self.strategy.get_min_history_ticks(), self.interval) 170 | self.look_back = self.history.copy() 171 | 172 | try: 173 | while True: 174 | # Get next ticker 175 | new_ticker = self.bot.get_next(self.interval) 176 | # print('new_ticker____:', new_ticker) 177 | if self.simulation_finished(new_ticker): 178 | print("No more data,..simulation done,. quitting") 179 | exit(0) 180 | 181 | self.ticker = new_ticker 182 | 183 | # Check if ticker is valid 184 | if not self.validate_ticker(self.ticker): 185 | print(colored('Received invalid ticker, will have to skip it! Details:\n' + str(self.ticker), 'red')) 186 | continue 187 | 188 | # Save ticker to buffer 189 | self.history = self.history.append(self.ticker, ignore_index=True) 190 | self.look_back = self.look_back.append(self.ticker, ignore_index=True) 191 | 192 | # Check if buffer is not overflown 193 | self.look_back = common.handle_buffer_limits(self.look_back, self.max_lookback_size) 194 | 195 | if self.first_ticker is None or self.first_ticker.empty: 196 | self.first_ticker = self.ticker.copy() 197 | self.last_valid_ticker = self.ticker.copy() 198 | 199 | # Get next actions 200 | self.actions = self.strategy.calculate(self.look_back, self.wallet) 201 | 202 | # Set trade 203 | self.actions = self.bot.trade(self.actions, 204 | self.wallet.current_balance, 205 | self.trades) 206 | 207 | # Get wallet balance 208 | self.wallet.current_balance = self.bot.get_balance() 209 | 210 | # Write report 211 | self.report.calc_stats(self.ticker, self.wallet) 212 | 213 | except KeyboardInterrupt: 214 | self.on_simulation_done() 215 | 216 | except SystemExit: 217 | self.on_simulation_done() 218 | 219 | 220 | 221 | 222 | 223 | 224 | -------------------------------------------------------------------------------- /core/bots/base.py: -------------------------------------------------------------------------------- 1 | import re 2 | import math 3 | import time 4 | import pandas as pd 5 | import configargparse 6 | import core.common as common 7 | from termcolor import colored 8 | from abc import ABC, abstractmethod 9 | from backfill.candles import Candles 10 | from exchanges.exchange import Exchange 11 | from strategies.enums import TradeState 12 | from core.bots.enums import BuySellMode 13 | 14 | 15 | class Base(ABC): 16 | """ 17 | Base class for all simulation types (sim, paper, trade) 18 | """ 19 | ticker_df = pd.DataFrame() 20 | pairs = [] 21 | exchange = None 22 | balance = None 23 | arg_parser = configargparse.get_argument_parser() 24 | 25 | def __init__(self, trade_mode): 26 | super(Base, self).__init__() 27 | self.args = self.arg_parser.parse_known_args()[0] 28 | self.exchange = Exchange(trade_mode) 29 | self.transaction_fee = self.exchange.get_transaction_fee() 30 | self.ticker_df = pd.DataFrame() 31 | self.verbosity = self.args.verbosity 32 | self.pairs = common.parse_pairs(self.exchange, self.args.pairs) 33 | self.fixed_trade_amount = float(self.args.fixed_trade_amount) 34 | self.pair_delimiter = self.exchange.get_pair_delimiter() 35 | self.last_tick_epoch = 0 36 | 37 | def get_pair_delimiter(self): 38 | """ 39 | Returns exchanges pair delimiter 40 | """ 41 | return self.pair_delimiter 42 | 43 | def prefetch(self, min_ticker_size, ticker_interval): 44 | """ 45 | Method pre-fetches data to ticker buffer 46 | """ 47 | prefetch_epoch_size = ticker_interval * min_ticker_size * 60 48 | prefetch_days = math.ceil(prefetch_epoch_size / 86400) 49 | # Prefetch/Backfill data 50 | self.args.days = prefetch_days 51 | orig_pair = self.args.pairs 52 | backfill_candles = Candles() 53 | backfill_candles.run() 54 | self.args.pairs = orig_pair 55 | # Load data to our ticker buffer 56 | prefetch_epoch_size = ticker_interval * min_ticker_size * 60 57 | epoch_now = int(time.time()) 58 | prefetch_epoch = epoch_now - prefetch_epoch_size 59 | print('Going to prefetch data of size (minutes): ', ticker_interval * min_ticker_size) 60 | df = pd.DataFrame() 61 | while prefetch_epoch < epoch_now: 62 | data = self.exchange.get_offline_ticker(prefetch_epoch, self.pairs) 63 | df = df.append(data, ignore_index=True) 64 | prefetch_epoch += (ticker_interval * 60) 65 | print('Fetching done..') 66 | return df 67 | 68 | def get_pairs(self): 69 | """ 70 | Returns the pairs the bot is working with 71 | """ 72 | return self.pairs 73 | 74 | @abstractmethod 75 | def get_next(self, interval_in_min): 76 | """ 77 | Gets next data set 78 | :param interval_in_min: 79 | :return: New data in DataFrame 80 | """ 81 | pass 82 | 83 | def get_balance(self): 84 | """ 85 | Returns current balance 86 | """ 87 | # Remove all items with zero amount 88 | self.balance = {k: v for k, v in self.balance.items() if v != 0} 89 | return self.balance 90 | 91 | def sell_all_assets(self, trades, wallet, pair_to_hold): 92 | """ 93 | Sells all available assets in wallet 94 | """ 95 | assets = wallet.copy() 96 | del assets['BTC'] 97 | for asset, amount in assets.items(): 98 | if amount == 0.0: 99 | continue 100 | pair = 'BTC' + self.pair_delimiter + asset 101 | # If we have the same pair that we want to buy, lets not sell it 102 | if pair == pair_to_hold: 103 | continue 104 | ticker = self.ticker_df.loc[self.ticker_df['pair'] == pair] 105 | if ticker.empty: 106 | print('No currency data for pair: ' + pair + ', skipping') 107 | continue 108 | close_price = ticker['close'].iloc[0] 109 | fee = self.transaction_fee * float(amount) / 100.0 110 | print('txn fee:', fee, ', balance before: ', amount, ', after: ', amount - fee) 111 | print(colored('Sold: ' + str(amount) + ', pair: ' + pair + ', price: ' + str(close_price), 'yellow')) 112 | amount -= fee 113 | earned_balance = close_price * amount 114 | root_symbol = 'BTC' 115 | currency = wallet[root_symbol] 116 | # Store trade history 117 | trades.loc[len(trades)] = [ticker['date'].iloc[0], pair, close_price, 'sell'] 118 | wallet[root_symbol] = currency + earned_balance 119 | wallet[asset] = 0.0 120 | 121 | def trade(self, actions, wallet, trades, force_sell=True): 122 | """ 123 | force_sell: Sells ALL assets before buying new one 124 | Simulate currency buy/sell (places fictive buy/sell orders). 125 | Returns remaining / not - processed actions 126 | """ 127 | self.balance = wallet 128 | if self.ticker_df.empty: 129 | print('Can not trade with empty dataframe, skipping trade') 130 | return actions 131 | 132 | for action in actions: 133 | # If action is None, just skip it 134 | if action.action == TradeState.none: 135 | actions.remove(action) 136 | continue 137 | 138 | # If we are forcing_sell, we will first sell all our assets 139 | if force_sell: 140 | self.sell_all_assets(trades, wallet, action.pair) 141 | 142 | # Get pairs current closing price 143 | (currency_symbol, asset_symbol) = tuple(re.split('[-_]', action.pair)) 144 | ticker = self.ticker_df.loc[self.ticker_df['pair'] == action.pair] 145 | 146 | if len(ticker.index) == 0: 147 | print('Could not find pairs ticker, skipping trade') 148 | continue 149 | 150 | close_price = action.rate 151 | 152 | currency_balance = asset_balance = 0.0 153 | if currency_symbol in wallet: 154 | currency_balance = wallet[currency_symbol] 155 | if asset_symbol in wallet: 156 | asset_balance = wallet[asset_symbol] 157 | 158 | if action.buy_sell_mode == BuySellMode.all: 159 | action.amount = self.get_buy_sell_all_amount(wallet, action) 160 | elif action.buy_sell_mode == BuySellMode.fixed: 161 | action.amount = self.get_fixed_trade_amount(wallet, action) 162 | 163 | fee = self.transaction_fee * float(action.amount) / 100.0 164 | # *** Buy *** 165 | if action.action == TradeState.buy: 166 | if currency_balance <= 0: 167 | print('Want to buy ' + action.pair + ', not enough money, or everything already bought..') 168 | actions.remove(action) 169 | continue 170 | print(colored('Bought: ' + str(action.amount) + ', pair: ' + action.pair + ', price: ' + str(close_price), 'green')) 171 | wallet[asset_symbol] = asset_balance + action.amount - fee 172 | wallet[currency_symbol] = currency_balance - (action.amount*action.rate) 173 | # Append trade 174 | trades.loc[len(trades)] = [ticker['date'].iloc[0], action.pair, close_price, 'buy'] 175 | actions.remove(action) 176 | continue 177 | 178 | # *** Sell *** 179 | elif action.action == TradeState.sell: 180 | if asset_balance <= 0: 181 | print('Want to sell ' + action.pair + ', not enough assets, or everything already sold..') 182 | actions.remove(action) 183 | continue 184 | print(colored('Sold: ' + str(action.amount) + '' + action.pair + ', price: ' + str(close_price), 'yellow')) 185 | wallet[currency_symbol] = currency_balance + ((action.amount-fee)*action.rate) 186 | wallet[asset_symbol] = asset_balance - action.amount 187 | # Append trade 188 | trades.loc[len(trades)] = [ticker['date'].iloc[0], action.pair, close_price, 'sell'] 189 | actions.remove(action) 190 | continue 191 | self.balance = wallet 192 | return actions 193 | 194 | def get_buy_sell_all_amount(self, wallet, action): 195 | """ 196 | Calculates total amount for ALL assets in wallet 197 | """ 198 | if action.action == TradeState.none: 199 | return 0.0 200 | 201 | if action.rate == 0.0: 202 | print(colored('Got zero rate!. Can not calc. buy_sell_amount for pair: ' + action.pair, 'red')) 203 | return 0.0 204 | 205 | (symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter)) 206 | amount = 0.0 207 | if action.action == TradeState.buy and symbol_1 in wallet: 208 | assets = wallet.get(symbol_1) 209 | amount = assets / action.rate 210 | elif action.action == TradeState.sell and symbol_2 in wallet: 211 | assets = wallet.get(symbol_2) 212 | amount = assets 213 | 214 | if amount <= 0.0: 215 | return 0.0 216 | return amount 217 | 218 | def get_fixed_trade_amount(self, wallet, action): 219 | """ 220 | Calculates fixed trade amount given action 221 | """ 222 | if action.action == TradeState.none: 223 | return 0.0 224 | 225 | if action.rate == 0.0: 226 | print(colored('Got zero rate!. Can not calc. buy_sell_amount for pair: ' + action.pair, 'red')) 227 | return 0.0 228 | 229 | (symbol_1, symbol_2) = tuple(action.pair.split(self.pair_delimiter)) 230 | amount = 0.0 231 | if action.action == TradeState.buy and symbol_1 in wallet: 232 | assets = self.fixed_trade_amount 233 | amount = assets / action.rate 234 | elif action.action == TradeState.sell and symbol_2 in wallet: 235 | assets = wallet.get(symbol_2) 236 | amount = assets 237 | 238 | if amount <= 0.0: 239 | return 0.0 240 | return amount 241 | -------------------------------------------------------------------------------- /exchanges/poloniex/polo.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | import datetime 4 | import pandas as pd 5 | import configargparse 6 | from termcolor import colored 7 | from exchanges.base import Base 8 | from json import JSONDecodeError 9 | from poloniex import Poloniex, PoloniexError 10 | from core.bots.enums import TradeMode 11 | from strategies.enums import TradeState 12 | from core.bots.enums import BuySellMode 13 | 14 | 15 | class Polo(Base): 16 | """ 17 | Poloniex interface 18 | """ 19 | arg_parser = configargparse.get_argument_parser() 20 | arg_parser.add('--polo_api_key', help='Poloniex API key') 21 | arg_parser.add("--polo_secret", help='Poloniex secret key') 22 | arg_parser.add("--polo_txn_fee", help='Poloniex txn. fee') 23 | arg_parser.add("--polo_buy_order", help='Poloniex buy order type') 24 | arg_parser.add("--polo_sell_order", help='Poloniex sell order type') 25 | valid_candle_intervals = [300, 900, 1800, 7200, 14400, 86400] 26 | 27 | def __init__(self): 28 | super(Polo, self).__init__() 29 | args = self.arg_parser.parse_known_args()[0] 30 | api_key = args.polo_api_key 31 | secret = args.polo_secret 32 | self.transaction_fee = float(args.polo_txn_fee) 33 | self.polo = Poloniex(api_key, secret) 34 | self.buy_order_type = args.polo_buy_order 35 | self.sell_order_type = args.polo_sell_order 36 | self.pair_delimiter = '_' 37 | self.tickers_cache_refresh_interval = 50 # If the ticker request is within the interval, get data from cache 38 | self.last_tickers_fetch_epoch = 0 39 | self.last_tickers_cache = None # Cache for storing immediate tickers 40 | 41 | def get_balances(self): 42 | """ 43 | Return available account balances (function returns ONLY currencies > 0) 44 | """ 45 | try: 46 | balances = self.polo.returnBalances() 47 | only_non_zeros = {k: float(v) for k, v in balances.items() if float(v) > 0.0} 48 | except PoloniexError as e: 49 | print(colored('!!! Got exception (polo.get_balances): ' + str(e), 'red')) 50 | only_non_zeros = dict() 51 | 52 | return only_non_zeros 53 | 54 | def get_symbol_ticker(self, symbol, candle_size=5): 55 | """ 56 | Returns real-time ticker Data-Frame for given symbol/pair 57 | Info: Currently Poloniex returns tickers for ALL pairs. To speed the queries and avoid 58 | unnecessary API calls, this method implements temporary cache 59 | """ 60 | epoch_now = int(time.time()) 61 | if epoch_now < (self.last_tickers_fetch_epoch + self.tickers_cache_refresh_interval): 62 | # If the ticker request is within cache_fetch_interval, try to get data from cache 63 | pair_ticker = self.last_tickers_cache[symbol].copy() 64 | else: 65 | # If cache is too old fetch data from Poloniex API 66 | try: 67 | ticker = self.polo.returnTicker() 68 | pair_ticker = ticker[symbol] 69 | self.last_tickers_fetch_epoch = int(time.time()) 70 | self.last_tickers_cache = ticker.copy() 71 | except (PoloniexError, JSONDecodeError) as e: 72 | print(colored('!!! Got exception in get_symbol_ticker. Details: ' + str(e), 'red')) 73 | pair_ticker = self.last_tickers_cache[symbol].copy() 74 | pair_ticker = dict.fromkeys(pair_ticker, None) 75 | 76 | df = pd.DataFrame.from_dict(pair_ticker, orient="index") 77 | df = df.T 78 | # We will use 'last' price as closing one 79 | df = df.rename(columns={'last': 'close', 'baseVolume': 'volume'}) 80 | df['close'] = df['close'].astype(float) 81 | df['volume'] = df['volume'].astype(float) 82 | df['pair'] = symbol 83 | df['date'] = int(datetime.datetime.utcnow().timestamp()) 84 | return df 85 | 86 | def return_ticker(self): 87 | """ 88 | Returns ticker for all currencies 89 | """ 90 | return self.polo.returnTicker() 91 | 92 | def cancel_order(self, order_number): 93 | """ 94 | Cancels order for given order number 95 | """ 96 | return self.polo.cancelOrder(order_number) 97 | 98 | def get_open_orders(self, currency_pair='all'): 99 | """ 100 | Returns your open orders 101 | """ 102 | return self.polo.returnOpenOrders(currency_pair) 103 | 104 | def get_pairs(self): 105 | """ 106 | Returns ticker pairs for all currencies 107 | """ 108 | ticker = self.polo.returnTicker() 109 | return list(ticker) 110 | 111 | def get_candles_df(self, currency_pair, epoch_start, epoch_end, period=False): 112 | """ 113 | Returns candlestick chart data in pandas dataframe 114 | """ 115 | try: 116 | data = self.get_candles(currency_pair, epoch_start, epoch_end, period) 117 | df = pd.DataFrame(data) 118 | df = df.tail(1) 119 | df['close'] = df['close'].astype(float) 120 | df['volume'] = df['volume'].astype(float) 121 | df['pair'] = currency_pair 122 | return df 123 | except (PoloniexError, JSONDecodeError) as e: 124 | print() 125 | print(colored('!!! Got exception while retrieving polo data:' + str(e) + ', pair: ' + currency_pair, 'red')) 126 | return pd.DataFrame() 127 | 128 | def get_candles(self, currency_pair, epoch_start, epoch_end, interval_in_sec=300): 129 | """ 130 | Returns candlestick chart data 131 | """ 132 | candle_interval = self.get_valid_candle_interval(interval_in_sec) 133 | data = [] 134 | try: 135 | data = self.polo.returnChartData(currency_pair, candle_interval, epoch_start, epoch_end) 136 | except (PoloniexError, JSONDecodeError) as e: 137 | print() 138 | print(colored('!!! Got exception while retrieving polo data: ' + str(e) + ', pair: ' + currency_pair, 'red')) 139 | 140 | if not isinstance(data, list): 141 | print(colored('!!! Received invalid candle payload. Details: ' + str(data), 'red')) 142 | return[] 143 | return data 144 | 145 | def get_market_history(self, start, end, currency_pair='all'): 146 | """ 147 | Returns market trade history 148 | """ 149 | data = [] 150 | try: 151 | data = self.polo.marketTradeHist(currencyPair=currency_pair, 152 | start=start, 153 | end=end) 154 | except (PoloniexError, JSONDecodeError) as e: 155 | logger = logging.getLogger(__name__) 156 | logger.error('Got exception while retrieving polo data:' + str(e) + ', pair: ' + currency_pair, e) 157 | return data 158 | 159 | def get_valid_candle_interval(self, period_in_sec): 160 | """ 161 | Returns closest value from valid candle intervals 162 | """ 163 | if not period_in_sec: 164 | return period_in_sec 165 | 166 | if period_in_sec in self.valid_candle_intervals: 167 | return period_in_sec 168 | # Find the closest valid interval 169 | return min(self.valid_candle_intervals, key=lambda x: abs(x - period_in_sec)) 170 | 171 | def trade(self, actions, wallet, trade_mode): 172 | if trade_mode == TradeMode.backtest: 173 | return Base.trade(actions, wallet, trade_mode) 174 | else: 175 | actions = self.life_trade(actions) 176 | return actions 177 | 178 | def life_trade(self, actions): 179 | """ 180 | Places orders and returns order number 181 | !!! For now we are NOT handling postOnly type of orders !!! 182 | """ 183 | for action in actions: 184 | 185 | if action.action == TradeState.none: 186 | actions.remove(action) 187 | continue 188 | 189 | # Handle buy_sell mode 190 | wallet = self.get_balances() 191 | if action.buy_sell_mode == BuySellMode.all: 192 | action.amount = self.get_buy_sell_all_amount(wallet, action) 193 | elif action.buy_sell_mode == BuySellMode.fixed: 194 | action.amount = self.get_fixed_trade_amount(wallet, action) 195 | 196 | print('Processing live-action: ' + str(action.action) + 197 | ', amount:', str(action.amount) + 198 | ', pair:', action.pair + 199 | ', rate:', str(action.rate) + 200 | ', buy_sell_mode:', action.buy_sell_mode) 201 | 202 | # If we don't have enough assets, just skip/remove the action 203 | if action.amount == 0.0: 204 | print(colored('No assets to buy/sell, ...skipping: ' + str(action.amount) + ' ' + action.pair, 'green')) 205 | actions.remove(action) 206 | continue 207 | 208 | # ** Buy Action ** 209 | if action.action == TradeState.buy: 210 | try: 211 | print(colored('Setting buy order: ' + str(action.amount) + '' + action.pair, 'green')) 212 | action.order_number = self.polo.buy(action.pair, action.rate, action.amount, self.buy_order_type) 213 | except PoloniexError as e: 214 | print(colored('Got exception: ' + str(e) + ' Txn: buy-' + action.pair, 'red')) 215 | continue 216 | amount_unfilled = action.order_number.get('amountUnfilled') 217 | if float(amount_unfilled) == 0.0: 218 | actions.remove(action) 219 | print(colored('Bought: ' + str(action.amount) + '' + action.pair, 'green')) 220 | else: 221 | action.amount = amount_unfilled 222 | print(colored('Not filled 100% buy txn. Unfilled amount: ' + str(amount_unfilled) + '' + action.pair, 'red')) 223 | 224 | # ** Sell Action ** 225 | elif action.action == TradeState.sell: 226 | try: 227 | print(colored('Setting sell order: ' + str(action.amount) + '' + action.pair, 'yellow')) 228 | action.order_number = self.polo.sell(action.pair, action.rate, action.amount, self.buy_order_type) 229 | except PoloniexError as e: 230 | print(colored('Got exception: ' + str(e) + ' Txn: sell-' + action.pair, 'red')) 231 | continue 232 | amount_unfilled = action.order_number.get('amountUnfilled') 233 | if float(amount_unfilled) == 0.0: 234 | actions.remove(action) 235 | print(colored('Sold: ' + str(action.amount) + '' + action.pair, 'yellow')) 236 | else: 237 | action.amount = amount_unfilled 238 | print(colored('Not filled 100% sell txn. Unfilled amount: ' + str(amount_unfilled) + '' + action.pair, 'red')) 239 | return actions 240 | 241 | -------------------------------------------------------------------------------- /exchanges/bittrex/bittrexclient.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | import pandas as pd 4 | from bittrex.bittrex import Bittrex 5 | from core.bots.enums import TradeMode 6 | from exchanges.base import Base 7 | from strategies.enums import TradeState 8 | from termcolor import colored 9 | import datetime 10 | from datetime import timezone 11 | from dateutil.tz import * 12 | from dateutil.parser import * 13 | from core.bots.enums import BuySellMode 14 | import configargparse 15 | from socket import gaierror 16 | 17 | 18 | class BittrexClient(Base): 19 | """ 20 | Bittrex interface 21 | """ 22 | arg_parser = configargparse.get_argument_parser() 23 | arg_parser.add('--bittrex_api_key', help='Bittrex API key') 24 | arg_parser.add("--bittrex_secret", help='Bittrex secret key') 25 | arg_parser.add("--bittrex_txn_fee", help='Bittrex txn. fee') 26 | open_orders = [] 27 | 28 | def __init__(self): 29 | args = self.arg_parser.parse_known_args()[0] 30 | super(BittrexClient, self).__init__() 31 | api_key = args.bittrex_api_key 32 | secret = args.bittrex_secret 33 | self.transaction_fee = float(args.bittrex_txn_fee) 34 | self.bittrex = Bittrex(api_key, secret) 35 | self.pair_delimiter = '-' 36 | self.verbosity = args.verbosity 37 | 38 | def get_pairs(self): 39 | """ 40 | Returns ticker pairs for all currencies 41 | """ 42 | markets = self.bittrex.get_market_summaries() 43 | if markets is None: 44 | print(colored('\n! Got empty markets', 'red')) 45 | return None 46 | 47 | res = markets['result'] 48 | pairs = [] 49 | for market in res: 50 | pair = market['MarketName'] 51 | # pair = pair.replace('-', '_') 52 | pairs.append(pair) 53 | return pairs 54 | 55 | def get_candles_df(self, currency_pair, epoch_start, epoch_end, interval_in_sec=300): 56 | """ 57 | Returns candlestick chart data in pandas dataframe 58 | """ 59 | dict_data = self.get_candles(currency_pair, epoch_start, epoch_end, interval_in_sec) 60 | df = pd.DataFrame(dict_data) 61 | df['pair'] = currency_pair 62 | return df 63 | 64 | def get_candles(self, currency_pair, epoch_start, epoch_end, interval_in_sec=300): 65 | """ 66 | Returns candlestick chart data 67 | """ 68 | currency_pair = currency_pair.replace('_', self.pair_delimiter) 69 | try: 70 | res = self.bittrex.get_ticks(currency_pair, 'fiveMin') 71 | except gaierror as e: 72 | print(colored('\n! Got gaierror exception from Bittrex client. Details: ' + e, 'red')) 73 | return dict() 74 | except Exception as e: 75 | print(colored('\n! Got exception from Bittrex client. Details: ' + e, 'red')) 76 | return dict() 77 | 78 | if res is None: 79 | print(colored('\n! Got empty result for pair: ' + currency_pair, 'red')) 80 | return dict() 81 | 82 | tickers = res['result'] 83 | got_min_epoch_ticker = False 84 | raw_tickers = [] 85 | 86 | if tickers is None: 87 | print(colored('\n! Got empty tickers for pair: ' + currency_pair, 'red')) 88 | return dict() 89 | 90 | # Parse tickers 91 | for ticker in tickers: 92 | naive_dt = parse(ticker['T']) 93 | utc_dt = naive_dt.replace(tzinfo=tzutc()) 94 | epoch = int(utc_dt.timestamp()) 95 | 96 | if epoch <= epoch_start: 97 | got_min_epoch_ticker = True 98 | 99 | # Skip/remove older than wanted tickers (adding extra hours to be sure that we have the data) 100 | if epoch < (epoch_start - 6*3600): 101 | continue 102 | 103 | raw_ticker = dict() 104 | raw_ticker['high'] = ticker['H'] 105 | raw_ticker['low'] = ticker['L'] 106 | raw_ticker['open'] = ticker['O'] 107 | raw_ticker['close'] = ticker['C'] 108 | raw_ticker['volume'] = ticker['V'] 109 | raw_ticker['quoteVolume'] = ticker['BV'] 110 | raw_ticker['date'] = epoch 111 | raw_ticker['weightedAverage'] = 0.0 112 | 113 | raw_tickers.append(raw_ticker) 114 | if not got_min_epoch_ticker: 115 | print(colored('Not able to get all data (data not available) for pair: ' + currency_pair, 'red')) 116 | 117 | # Create/interpolate raw tickers to fit our interval ticker 118 | out_tickers = [] 119 | for ticker_epoch in range(epoch_start, epoch_end, interval_in_sec): 120 | items = [element for element in raw_tickers if element['date'] <= ticker_epoch] 121 | if len(items) <= 0: 122 | print(colored('Could not found a ticker for:' + currency_pair + ', epoch:' + str(ticker_epoch), 'red')) 123 | continue 124 | # Get the last item (should be closest to search epoch) 125 | item = items[-1].copy() 126 | item['date'] = ticker_epoch 127 | out_tickers.append(item) 128 | return out_tickers.copy() 129 | # return self.bittrex.returnChartData(currency_pair, period, start, end) 130 | 131 | def get_balances(self): 132 | """ 133 | Return available account balances (function returns ONLY currencies > 0) 134 | """ 135 | resp = self.bittrex.get_balances() 136 | balances = resp['result'] 137 | pairs = dict() 138 | for item in balances: 139 | currency = item['Currency'] 140 | pairs[currency] = item['Available'] 141 | 142 | return pairs 143 | 144 | @staticmethod 145 | def get_volume_from_history(history, candle_size): 146 | """ 147 | Returns volume for given candle_size 148 | :param history: history data 149 | :param candle_size: in minutes 150 | :return: Calculated volume for given candle_size 151 | """ 152 | volume = 0.0 153 | epoch_now = int(time.time()) 154 | epoch_candle_start = epoch_now - candle_size * 60 155 | pattern = '%Y-%m-%dT%H:%M:%S' 156 | for item in history: 157 | time_string = item['TimeStamp'].split('.', 1)[0] 158 | dt = datetime.datetime.strptime(time_string, pattern) 159 | item_epoch = dt.replace(tzinfo=timezone.utc).timestamp() 160 | if item_epoch >= epoch_candle_start: 161 | quantity = item['Quantity'] 162 | volume += quantity 163 | return volume 164 | 165 | def get_symbol_ticker(self, symbol, candle_size=5): 166 | """ 167 | Returns real-time ticker Data-Frame 168 | :candle_size: size in minutes to calculate the interval 169 | """ 170 | market = symbol.replace('_', self.pair_delimiter) 171 | 172 | ticker = self.bittrex.get_ticker(market) 173 | history = self.bittrex.get_market_history(market, 100)['result'] 174 | volume = self.get_volume_from_history(history, candle_size) 175 | 176 | df = pd.DataFrame.from_dict(ticker['result'], orient="index") 177 | df = df.T 178 | # We will use 'last' price as closing one 179 | df = df.rename(columns={'Last': 'close', 'Ask': 'lowestAsk', 'Bid': 'highestBid'}) 180 | df['volume'] = volume 181 | df['pair'] = symbol 182 | df['date'] = int(datetime.datetime.utcnow().timestamp()) 183 | return df 184 | 185 | def get_market_history(self, start, end, currency_pair='all'): 186 | """ 187 | Returns market trade history 188 | """ 189 | logger = logging.getLogger(__name__) 190 | logger.warning('Not implemented!') 191 | pass 192 | 193 | def trade(self, actions, wallet, trade_mode): 194 | """ 195 | Places actual buy/sell orders 196 | """ 197 | if trade_mode == TradeMode.backtest: 198 | return Base.trade(actions, wallet, trade_mode) 199 | else: 200 | actions = self.life_trade(actions) 201 | return actions 202 | 203 | def life_trade(self, actions): 204 | """ 205 | Places orders and returns order number 206 | """ 207 | for action in actions: 208 | market = action.pair.replace('_', self.pair_delimiter) 209 | 210 | # Handle buy/sell mode 211 | wallet = self.get_balances() 212 | if action.buy_sell_mode == BuySellMode.all: 213 | action.amount = self.get_buy_sell_all_amount(wallet, action) 214 | elif action.buy_sell_mode == BuySellMode.fixed: 215 | action.amount = self.get_fixed_trade_amount(wallet, action) 216 | 217 | if self.verbosity: 218 | print('Processing live-action: ' + str(action.action) + 219 | ', amount:', str(action.amount) + 220 | ', pair:', market + 221 | ', rate:', str(action.rate) + 222 | ', buy_sell_mode:', action.buy_sell_mode) 223 | if action.action == TradeState.none: 224 | actions.remove(action) 225 | continue 226 | 227 | # If we don't have enough assets, just skip/remove the action 228 | if action.amount == 0.0: 229 | print(colored('No assets to buy/sell, ...skipping: ' + str(action.amount) + ' ' + market, 'green')) 230 | actions.remove(action) 231 | continue 232 | 233 | # ** Buy Action ** 234 | if action.action == TradeState.buy: 235 | print(colored('setting buy order: ' + str(action.amount) + '' + market, 'green')) 236 | ret = self.bittrex.buy_limit(market, action.amount, action.rate) 237 | if not ret['success']: 238 | print(colored('Error: ' + ret['message'] + '. Txn: buy-' + market, 'red')) 239 | continue 240 | else: 241 | uuid = ret['result']['uuid'] 242 | self.open_orders.append(uuid) 243 | print(colored('Buy order placed (uuid): ' + uuid, 'green')) 244 | print(ret) 245 | 246 | # ** Sell Action ** 247 | elif action.action == TradeState.sell: 248 | print(colored('setting sell order: ' + str(action.amount) + '' + market, 'yellow')) 249 | ret = self.bittrex.sell_limit(market, action.amount, action.rate) 250 | if not ret['success']: 251 | print(colored('Error: ' + ret['message'] + '. Txn: sell-' + market, 'red')) 252 | continue 253 | else: 254 | uuid = ret['result']['uuid'] 255 | self.open_orders.append(uuid) 256 | print(colored('Sell order placed (uuid): ' + uuid, 'green')) 257 | print(ret) 258 | return actions 259 | 260 | def cancel_order(self, order_number): 261 | """ 262 | Cancels order for given order number 263 | """ 264 | return self.bittrex.cancel(order_number) 265 | 266 | def get_open_orders(self, currency_pair=''): 267 | """ 268 | Returns open orders 269 | """ 270 | return self.bittrex.get_open_orders(currency_pair) 271 | 272 | def get_trade_history(self, date_from, date_to, currency_pair='all'): 273 | """ 274 | Returns trade history 275 | """ 276 | # TODO 277 | pass 278 | -------------------------------------------------------------------------------- /exchanges/exchange.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | import pymongo 4 | import pandas as pd 5 | import configargparse 6 | from termcolor import colored 7 | from .poloniex.polo import Polo 8 | from .bittrex.bittrexclient import BittrexClient 9 | from core.bots.enums import TradeMode 10 | 11 | 12 | class Exchange: 13 | """ 14 | Main interface to all exchanges 15 | """ 16 | 17 | exchange = None 18 | exchange_name = None 19 | arg_parser = configargparse.get_argument_parser() 20 | arg_parser.add('--exchange', help='Exchange') 21 | arg_parser.add('--db_url', help='Mongo db url') 22 | arg_parser.add('--db_port', help='Mongo db port') 23 | arg_parser.add('--db', help='Mongo db') 24 | arg_parser.add('--pairs', help='Pairs') 25 | logger = logging.getLogger(__name__) 26 | 27 | def __init__(self, trade_mode=TradeMode.backtest, ticker_size=5): 28 | self.args = self.arg_parser.parse_known_args()[0] 29 | self.exchange = self.load_exchange() 30 | self.trade_mode = trade_mode 31 | self.db = self.initialize_db() 32 | self.ticker_client = self.db.ticker 33 | self.trades_client = self.db.trades 34 | self.ticker_size = ticker_size 35 | 36 | def get_pair_delimiter(self): 37 | """ 38 | Returns exchanges pair delimiter 39 | """ 40 | return self.exchange.get_pair_delimiter() 41 | 42 | def get_transaction_fee(self): 43 | """ 44 | Returns exchanges transaction fee 45 | """ 46 | return self.exchange.get_transaction_fee() 47 | 48 | def get_pairs(self): 49 | """ 50 | Returns ticker for all pairs 51 | """ 52 | return self.exchange.get_pairs() 53 | 54 | def get_symbol_ticker(self, symbol, candle_size=5): 55 | """ 56 | Returns ticker for given symbol 57 | """ 58 | return self.exchange.get_symbol_ticker(symbol, candle_size) 59 | 60 | def initialize_db(self): 61 | """ 62 | DB Initialization 63 | """ 64 | db = self.args.db 65 | port = int(self.args.db_port) 66 | url = self.args.db_url 67 | client = pymongo.MongoClient(url, port) 68 | db = client[db] 69 | return db 70 | 71 | def get_exchange_name(self): 72 | """ 73 | Returns name of the exchange 74 | """ 75 | return self.exchange_name 76 | 77 | def load_exchange(self): 78 | """ 79 | Loads exchange files 80 | """ 81 | self.exchange_name = self.args.exchange 82 | 83 | if self.exchange_name == 'polo': 84 | return Polo() 85 | elif self.exchange_name == 'bittrex': 86 | return BittrexClient() 87 | else: 88 | print('Trying to use not defined exchange!') 89 | return None 90 | 91 | def trade(self, actions, wallet, trade_mode): 92 | """ 93 | Main class for setting up buy/sell orders 94 | """ 95 | return self.exchange.trade(actions, wallet, trade_mode) 96 | 97 | def cancel_order(self, order_number): 98 | """ 99 | Cancels order for given order number 100 | """ 101 | return self.exchange.cancel_order(order_number) 102 | 103 | def get_balances(self): 104 | """ 105 | Returns all available account balances 106 | """ 107 | return self.exchange.get_balances() 108 | 109 | def get_candles_df(self, currency_pair, epoch_start, epoch_end, period=300): 110 | """ 111 | Returns candlestick chart data in pandas dataframe 112 | """ 113 | return self.exchange.get_candles_df(currency_pair, epoch_start, epoch_end, period) 114 | 115 | def get_candles(self, currency_pair, epoch_start, epoch_end, period=300): 116 | """ 117 | Returns candlestick chart data 118 | """ 119 | return self.exchange.get_candles(currency_pair, epoch_start, epoch_end, period) 120 | 121 | def get_open_orders(self, currency_pair='all'): 122 | """ 123 | Returns your open orders 124 | """ 125 | return self.exchange.get_open_orders(currency_pair) 126 | 127 | def get_market_history(self, start, end, currency_pair='all'): 128 | """ 129 | Returns trade history 130 | """ 131 | return self.exchange.get_market_history(start=int(start), 132 | end=int(end), 133 | currency_pair=currency_pair) 134 | 135 | # Do not use!!!: It turns out that group method is very consuming (takes approx 3x then get_offline_ticker) 136 | def get_offline_tickers(self, epoch, pairs): 137 | """ 138 | Returns offline data from DB 139 | """ 140 | pipeline = [ 141 | {"$match": {"date": {"$lte": epoch}, "pair": {"$in": pairs}}}, 142 | {"$group": {"_id": "$pair", 143 | "pair": {"$last": "$pair"}, 144 | "high": {"$last": "$high"}, 145 | "low": {"$last": "$low"}, 146 | "open": {"$last": "$open"}, 147 | "close": {"$last": "$close"}, 148 | "volume": {"$last": "$volume"}, 149 | "quoteVolume": {"$last": "$quoteVolume"}, 150 | "weightedAverage": {"$last": "$weightedAverage"}, 151 | "date": {"$last": "$date"} 152 | } 153 | } 154 | ] 155 | 156 | db_list = list(self.ticker_client.aggregate(pipeline, allowDiskUse=True)) 157 | ticker_df = pd.DataFrame(db_list) 158 | df_pair = ticker_df['pair'].str.split('_', 1, expand=True) 159 | ticker_df = pd.concat([ticker_df, df_pair], axis=1) 160 | ticker_df = ticker_df.drop(['_id'], axis=1) 161 | return ticker_df 162 | 163 | def get_offline_ticker(self, epoch, pairs): 164 | """ 165 | Returns offline ticker data from DB 166 | """ 167 | ticker = pd.DataFrame() 168 | # print(' Getting offline ticker for total pairs: ' + str(len(pairs)) + ', epoch:', str(epoch)) 169 | for pair in pairs: 170 | db_doc = self.ticker_client.find_one({"$and": [{"date": {"$lte": epoch}}, 171 | {"pair": pair}, 172 | {"exchange": self.exchange_name}]}, 173 | sort=[("date", pymongo.DESCENDING)]) 174 | 175 | if db_doc is None: 176 | local_dt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(epoch)) 177 | print(colored('No offline data for pair: ' + pair + ', epoch: ' + str(epoch) + ' (local: ' 178 | + str(local_dt) + ')', 'yellow')) 179 | continue 180 | 181 | dict_keys = list(db_doc.keys()) 182 | df = pd.DataFrame([db_doc], columns=dict_keys) 183 | df_pair = df['pair'].str.split('_', 1, expand=True) 184 | df = pd.concat([df, df_pair], axis=1) 185 | df.rename(columns={0: 'curr_1', 1: 'curr_2'}, inplace=True) 186 | ticker = ticker.append(df, ignore_index=True) 187 | return ticker 188 | 189 | def get_offline_trades(self, epoch, pairs): 190 | """ 191 | Returns offline trades data from DB 192 | """ 193 | trades = pd.DataFrame() 194 | date_start = epoch - self.ticker_size*60 195 | for pair in pairs: 196 | timer_start = time.time() 197 | db_doc = self.trades_client.find({"$and": [{"date": {"$gte": date_start, "$lte": epoch}}, 198 | {"pair": pair}, 199 | {"exchange": self.exchange_name}]}) 200 | timer_duration = int(time.time() - timer_start) 201 | self.logger.info('data fetched from DB in sec:' + str(timer_duration)) 202 | 203 | if db_doc.count() == 0: 204 | local_dt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(epoch)) 205 | print(colored('No offline data for pair: ' + pair + ', epoch: ' + str(epoch) + ' (local: ' 206 | + str(local_dt) + ')', 'yellow')) 207 | 208 | # Add empty/zeroes dataframe 209 | single_trades_df = self.summarize_trades(pd.DataFrame(), pair) 210 | trades = trades.append(single_trades_df, ignore_index=True) 211 | continue 212 | 213 | timer_start = time.time() 214 | dict_keys = list(db_doc[0].keys()) 215 | pair_trades_tmp = pd.DataFrame(list(db_doc), columns=dict_keys) 216 | timer_duration = int(time.time() - timer_start) 217 | self.logger.info('mongo doc parsed in sec:' + str(timer_duration)) 218 | 219 | timer_start = time.time() 220 | single_trades_df = self.summarize_trades(pair_trades_tmp, pair) 221 | timer_duration = int(time.time() - timer_start) 222 | self.logger.info('trades data summarized in sec:' + str(timer_duration)) 223 | trades = trades.append(single_trades_df, ignore_index=True) 224 | return trades 225 | 226 | def summarize_trades(self, df, pair): 227 | """ 228 | Summarizes trades 229 | """ 230 | trades_dict = {'pair': pair, 231 | 'exchange': self.exchange_name, 232 | 'buys_count': 0, 233 | 'buys_min': 0.0, 234 | 'buys_max': 0.0, 235 | 'buys_mean': 0, 236 | 'buys_volume': 0.0, 237 | 'buys_rate_mean': 0.0, 238 | 'buys_rate_spam': 0.0, 239 | 'sells_count': 0, 240 | 'sells_min': 0.0, 241 | 'sells_max': 0.0, 242 | 'sells_mean': 0, 243 | 'sells_volume': 0.0, 244 | 'sells_rate_mean': 0.0, 245 | 'sells_rate_spam': 0.0, 246 | } 247 | 248 | if not df.empty: 249 | # Buy trades 250 | buys_df = df[df['type'] == 'buy'] 251 | trades_dict['pair'] = df.pair.iloc[0] 252 | trades_dict['exchange'] = df.exchange.iloc[0] 253 | trades_dict['buys_count'] = int(buys_df.shape[0]) 254 | trades_dict['buys_min'] = buys_df.amount.min() 255 | trades_dict['buys_max'] = buys_df.amount.max() 256 | trades_dict['buys_mean'] = buys_df.amount.mean() 257 | trades_dict['buys_volume'] = buys_df.amount.sum() 258 | trades_dict['buys_rate_mean'] = buys_df.rate.mean() 259 | trades_dict['buys_rate_spam'] = buys_df.rate.max() - buys_df.rate.min() 260 | # Sell trades 261 | sells_df = df[df['type'] == 'sell'] 262 | trades_dict['sells_count'] = int(sells_df.shape[0]) 263 | trades_dict['sells_min'] = sells_df.amount.min() 264 | trades_dict['sells_max'] = sells_df.amount.max() 265 | trades_dict['sells_mean'] = sells_df.amount.mean() 266 | trades_dict['sells_volume'] = sells_df.amount.sum() 267 | trades_dict['sells_rate_mean'] = sells_df.rate.mean() 268 | trades_dict['sells_rate_spam'] = sells_df.rate.max() - sells_df.rate.min() 269 | 270 | ticker_df = pd.DataFrame.from_dict(trades_dict, orient="index").transpose() 271 | return ticker_df 272 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Flexible Trading Bot with main focus on Machine Learning and Genetic Algorithms, inspired by [zenbot.](https://github.com/carlos8f/zenbot) 4 | 5 | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/294dac01b9d74557a874797ab5856b72)](https://www.codacy.com/gh/miro-ka/mosquito/dashboard?utm_source=github.com&utm_medium=referral&utm_content=miro-ka/mosquito&utm_campaign=Badge_Grade) 6 | [![codebeat badge](https://codebeat.co/badges/46a04bf1-ce92-41ad-84ba-6627e08f54d5)](https://codebeat.co/projects/github-com-miti0-mosquito-master) 7 | 8 | ## About 9 | 10 | ![Mosquito - backtesting](images/console_sample.png) 11 | 12 | Mosquito is a cryptocurrency trading bot written in Python, with main focus on modularity, 13 | so it is straight forward to plug-in new exchange. 14 | 15 | The idea to build a new bot came because of missing following easy-access features in all of available open-source bots: 16 | * **Multi-currency bot** - Be able to monitor and exchange several currencies in one strategy (no need to run 5 same strategies for 5 different currencies). 17 | * **Easy AI plug & play** - Possibility to easily plug any of the existing AI/ML libraries (for ex. scikit or keras) 18 | 19 | > **Please be AWARE that Mosquito is still in beta and under heavy development. Please use it in Live trading VERY carefully.** 20 | 21 | 22 | ### Supported Exchanges 23 | Mosquito currently supports following exchanges: 24 | * **Poloniex** - supporting *fillOrKill* and *immediateOrCancel* trading types. *postOnly* type is not supported. You can 25 | read more about trading types [here.](https://github.com/s4w3d0ff/python-poloniex/blob/master/poloniex/__init__.py) 26 | * **Bittrex** - supporting *Trade Limit Buy/Sell Orders* 27 | * **Kucoin** - work in progress 28 | 29 | 30 | 31 | ## Requirements 32 | * Python 3.9 33 | * mongodb 34 | * Depending on the exchange you want to use: 35 | * [Poloniex-api](https://github.com/s4w3d0ff/python-poloniex) 36 | * [Bittrex-api](https://github.com/miti0/python-bittrex) 37 | 38 | 39 | 40 | ## Quick Start 41 | 42 | 43 | 44 | ### Install 45 | 1. Clone repo 46 | ``` 47 | git clone https://github.com/miti0/mosquito.git 48 | ``` 49 | 2. Install requirements (ideally in separate virtual environment) 50 | ``` 51 | pip install -r requirements.txt 52 | ``` 53 | 3. Install [mongodb](https://www.mongodb.com/try/download/community) 54 | 55 | 4. Set-up mosquito.ini (if you want to use sample config, just rename mosquito.sample.ini to mosquito.ini) 56 | 57 | 5. Run desired command (full list of commands below) 58 | 59 | All parameters in the program can be overridden with input arguments. 60 | You can get list of all available arguments with: 61 | 62 | ``` 63 | python mosquito.py --help 64 | ``` 65 | 66 | 67 | ## Backfill 68 | Backfill gets history data from exchange and stores them to mongodb. Data can be after that used for testing your simulation strategies. 69 | 70 | usage: backfill.py [-h] [--pairs PAIRS] [--all] --days DAYS 71 | 72 | ``` 73 | optional arguments: 74 | -h, --help show this help message and exit 75 | --pairs PAIRS PairS to backfill. For ex. [BTC_ETH, BTC_* (to get all BTC_* 76 | prefixed pairs] 77 | --all Backfill data for ALL currencies 78 | --days DAYS Number of days to backfill 79 | ``` 80 | 81 | 82 | Example 1) Load historical data for BTC_ETH pair for the last 5 days: 83 | ``` 84 | python backfill.py --days 5 --pairs USDT_BTC 85 | ``` 86 | 87 | Example 2) Load historical data for ALL pairs for the last 2 days 88 | ``` 89 | python backfill.py --days 3 --all 90 | ``` 91 | 92 | Example 3) Load historical data for all pairs starting with BTC_ for the last day 93 | ``` 94 | python backfill.py --days 1 --pairs BTC_* 95 | ``` 96 | 97 | 98 | 99 | ## Trading 100 | This is the main module that handles passed strategy and places buy/sell orders. 101 | 102 | Architecture and logic of mosquito is made so, that it should be easy to set and tune all strategy parameters with program arguments. Below is a list of main arguments that can be either configured via the mosquito.ini config file or by passing the value/values as argument. 103 | 104 | ``` 105 | -h, --help show this help message and exit 106 | --polo_api_key POLO_API_KEY 107 | Poloniex API key (default: None) 108 | --polo_secret POLO_SECRET 109 | Poloniex secret key (default: None) 110 | --polo_txn_fee POLO_TXN_FEE 111 | Poloniex txn. fee (default: None) 112 | --polo_buy_order POLO_BUY_ORDER 113 | Poloniex buy order type (default: None) 114 | --polo_sell_order POLO_SELL_ORDER 115 | Poloniex sell order type (default: None) 116 | --bittrex_api_key BITTREX_API_KEY 117 | Bittrex API key (default: None) 118 | --bittrex_secret BITTREX_SECRET 119 | Bittrex secret key (default: None) 120 | --bittrex_txn_fee BITTREX_TXN_FEE 121 | Bittrex txn. fee (default: None) 122 | --exchange EXCHANGE Exchange (default: None) 123 | --db_url DB_URL Mongo db url (default: None) 124 | --db_port DB_PORT Mongo db port (default: None) 125 | --db DB Mongo db (default: None) 126 | --pairs PAIRS Pairs (default: None) 127 | --use_real_wallet Use/not use fictive wallet (only for paper simulation) 128 | (default: False) 129 | --backtest_from BACKTEST_FROM 130 | Backtest epoch start datetime (default: None) 131 | --backtest_to BACKTEST_TO 132 | Backtest epoch end datetime (default: None) 133 | --backtest_days BACKTEST_DAYS 134 | Number of history days the simulation should start 135 | from (default: None) 136 | --wallet_currency WALLET_CURRENCY 137 | Wallet currency (separated by comma) (default: None) 138 | --wallet_amount WALLET_AMOUNT 139 | Wallet amount (separated by comma) (default: None) 140 | --backtest Simulate your strategy on history ticker data 141 | (default: False) 142 | --paper Simulate your strategy on real ticker (default: False) 143 | --live REAL trading mode (default: False) 144 | --plot Generate a candle stick plot at simulation end 145 | (default: False) 146 | --ticker_size TICKER_SIZE Simulation ticker_size (default: 5) 147 | --root_report_currency ROOT_REPORT_CURRENCY 148 | Root currency used in final plot (default: None) 149 | --buffer_size BUFFER_SIZE 150 | Buffer size in days (default: 30) 151 | --prefetch Prefetch data from history DB (default: False) 152 | --plot_pair PLOT_PAIR 153 | Plot pair (default: None) 154 | --all ALL Include all currencies/tickers (default: None) 155 | --days DAYS Days to pre-fill (default: None) 156 | -c CONFIG, --config CONFIG 157 | config file path (default: mosquito.ini) 158 | -v, --verbosity Verbosity (default: False) 159 | --strategy STRATEGY Strategy (default: None) 160 | --fixed_trade_amount FIXED_TRADE_AMOUNT 161 | Fixed trade amount (default: None) 162 | 163 | ``` 164 | 165 | Currently Trading supports following modes: 166 | * **Backtest** - fast simulation mode using past data and placing fictive buy/sell orders. 167 | * **Paper** - mode simulating live ticker with placing fictive buy/sell orders. 168 | * **Live** - live trading with placing REAL buy/sell orders. 169 | 170 | > Backtest and Paper trading are using immediate buy/sell orders by using the last ticker 171 | closing price. This results to NOT 100% accurate strategy results, what you should be aware of. 172 | 173 | 174 | ### Backtest 175 | Fast simulation mode using past data and placing fictive buy/sell orders. Simulation configuration is done via 176 | *config.ini* file (some of the parameters can be overridden with command line arguments). 177 | 178 | Below is an example of running a backtest together with final buy/sell plot generated at the end of the simulation. 179 | ``` 180 | python3 mosquito.py --backtest --plot 181 | ``` 182 | > ! Please be aware that Backtest should 99% work, but it is currently under final verification test. 183 | 184 | 185 | ### Paper 186 | Trading mode that simulates live ticker with placing fictive buy/sell orders. Simulation configuration is done via 187 | *config.ini* file (some of the parameters can be overridden with command line arguments). 188 | 189 | Below is an example of running a backtest together with final buy/sell plot generated at the end of the simulation. 190 | ``` 191 | python mosquito.py --paper 192 | ``` 193 | > ! Please be aware that Paper should 99% work, but it is currently under final verification test. 194 | 195 | 196 | ### Live 197 | Live trading with placing REAL buy/sell orders. Configuration is done via *config.ini* file (some of the parameters can be overridden with command line arguments). 198 | Below is an example of running a backtest together with final buy/sell plot generated at the end of the simulation. 199 | ``` 200 | python mosquito.py --live 201 | ``` 202 | > ! Please be aware that Live should 99% work, but it is currently under final verification test. 203 | 204 | 205 | 206 | ## Plot and Statistics 207 | Mosquito has a simple plot utility for visualizing current pair combined with trading history. 208 | Visualization uses external library [plotly](https://plot.ly/). Below You can see an example visualizing ticker price plot, together with simulated buy/sell orders. 209 | 210 | 211 | 212 | Below is an example of Final Simulation Report summary: 213 | ``` 214 | **************************************************** 215 | Final simulation report: 216 | **************************************************** 217 | Wallet at Start: | 50.0DGB | 218 | Wallet at End: | 51.3464723121DGB | 219 | Strategy result: -5.68% 220 | Buy & Hold: -8.16% 221 | Strategy vs Buy & Hold: 2.47% 222 | Total txn: 10 223 | Simulated (data time): 0 days, 4 hours and 55 minutes 224 | Transactions per hour: 2.03 225 | Simulation run time: 0 hours 1 minutes and 13 seconds 226 | ``` 227 | 228 | ## AI 229 | 230 | ### Blueprint 231 | Blueprint is a part of AI package. Main function of the module is to generate datasets which can be used for training AI. Logic of Blueprint module is following: 232 | 233 | 1. Create a blueprint file/module which contains features, indicators and output parameters. As an example you can take a look at ai/blueprints/minimal.py or ai/blueprints/junior.py 234 | 235 | 2. Decide how many days you would like to run the Blueprint. Backfield data for that period. 236 | 237 | 3. Choose which pair/pairs you would like to include. Following combinations should work [BTC_ETH] - single pair, [BTC_ETH, BTC_LTC] - list of pairs, [BTC_*] - all pairs with prefix BTC 238 | 239 | 4. Start blueprint with following parameters (example below) 240 | 241 | ``` 242 | python blueprint.py --features junior --days 200 243 | ``` 244 | 245 | As a result you should see *.csv file in your Mosquito's **out/blueprints** folder, which should contain the dataset. 246 | 247 | 248 | ## Utilities 249 | 250 | ### Wallet Lense 251 | Simple module which sends up to 24h winners/losers market pairs summary by email in user specified intervals (sample below). 252 | 253 | 254 | 255 | #### Usage 256 | ``` 257 | # You need to have configured email parameters in ini file, or pass them as input arguments. 258 | python lense.py 259 | ``` 260 | 261 | 262 | --- 263 | 264 | 265 | 266 | ### License: GNU GENERAL PUBLIC LICENSE 267 | - Copyright (C) 2023 (miro-ka) 268 | 269 | 270 | The GNU General Public License is a free, copyleft license for 271 | software and other kinds of works. 272 | 273 | The licenses for most software and other practical works are designed 274 | to take away your freedom to share and change the works. By contrast, 275 | the GNU General Public License is intended to guarantee your freedom to 276 | share and change all versions of a program--to make sure it remains free 277 | software for all its users. We, the Free Software Foundation, use the 278 | GNU General Public License for most of our software; it applies also to 279 | any other work released this way by its authors. You can apply it to 280 | your programs, too. 281 | 282 | 283 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 284 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 285 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 286 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 287 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 288 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 289 | SOFTWARE. 290 | -------------------------------------------------------------------------------- /market_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | import configargparse 3 | from stats.stats import Stats 4 | 5 | 6 | def main(): 7 | stats = Stats() 8 | stats.run() 9 | 10 | 11 | if __name__ == "__main__": 12 | arg_parser = configargparse.get_argument_parser() 13 | arg_parser.add('-c', '--config', is_config_file=True, help='config file path', default='mosquito.ini') 14 | arg_parser.add("--live", help="REAL trading mode", action='store_true') 15 | arg_parser.add('-v', '--verbosity', help='Verbosity', action='store_true') 16 | args = arg_parser.parse_known_args()[0] 17 | 18 | main() 19 | """ 20 | 21 | 22 | # Works on python3 / requires: pandas, numpy, pymongo, bokeh 23 | # BTC: 1A7K4kgXLSSzvDRjvoGwomvhrNU4CKezEp 24 | # LTC: LWShTeRrZpYS4aJhb6JdP3R9tNFMnZiDo2 25 | 26 | import logging 27 | from operator import itemgetter 28 | from math import pi 29 | from time import time 30 | 31 | from pymongo import MongoClient 32 | import pandas as pd 33 | import numpy as np 34 | from bokeh.plotting import figure, show 35 | from bokeh.models import NumeralTickFormatter 36 | from bokeh.models import LinearAxis, Range1d 37 | 38 | logger = logging.getLogger(__name__) 39 | 40 | 41 | def rsi(df, window, targetcol='weightedAverage', colname='rsi'): 42 | """ Calculates the Relative Strength Index (RSI) from a pandas dataframe 43 | http://stackoverflow.com/a/32346692/3389859 44 | """ 45 | series = df[targetcol] 46 | delta = series.diff().dropna() 47 | u = delta * 0 48 | d = u.copy() 49 | u[delta > 0] = delta[delta > 0] 50 | d[delta < 0] = -delta[delta < 0] 51 | # first value is sum of avg gains 52 | u[u.index[window - 1]] = np.mean(u[:window]) 53 | u = u.drop(u.index[:(window - 1)]) 54 | # first value is sum of avg losses 55 | d[d.index[window - 1]] = np.mean(d[:window]) 56 | d = d.drop(d.index[:(window - 1)]) 57 | rs = u.ewm(com=window - 1, 58 | ignore_na=False, 59 | min_periods=0, 60 | adjust=False).mean() / d.ewm(com=window - 1, 61 | ignore_na=False, 62 | min_periods=0, 63 | adjust=False).mean() 64 | df[colname] = 100 - 100 / (1 + rs) 65 | df[colname].fillna(df[colname].mean(), inplace=True) 66 | return df 67 | 68 | 69 | def sma(df, window, targetcol='close', colname='sma'): 70 | """ Calculates Simple Moving Average on a 'targetcol' in a pandas dataframe 71 | """ 72 | df[colname] = df[targetcol].rolling( 73 | min_periods=1, window=window, center=False).mean() 74 | return df 75 | 76 | 77 | def ema(df, window, targetcol='close', colname='ema', **kwargs): 78 | """ Calculates Expodential Moving Average on a 'targetcol' in a pandas 79 | dataframe """ 80 | df[colname] = df[targetcol].ewm( 81 | span=window, 82 | min_periods=kwargs.get('min_periods', 1), 83 | adjust=kwargs.get('adjust', True), 84 | ignore_na=kwargs.get('ignore_na', False) 85 | ).mean() 86 | df[colname].fillna(df[colname].mean(), inplace=True) 87 | return df 88 | 89 | 90 | def macd(df, fastcol='emafast', slowcol='sma', colname='macd'): 91 | """ Calculates the differance between 'fastcol' and 'slowcol' in a pandas 92 | dataframe """ 93 | df[colname] = df[fastcol] - df[slowcol] 94 | return df 95 | 96 | 97 | def bbands(df, window, targetcol='close', stddev=2.0): 98 | """ Calculates Bollinger Bands for 'targetcol' of a pandas dataframe """ 99 | if not 'sma' in df: 100 | df = sma(df, window, targetcol) 101 | df['sma'].fillna(df['sma'].mean(), inplace=True) 102 | df['bbtop'] = df['sma'] + stddev * df[targetcol].rolling( 103 | min_periods=1, 104 | window=window, 105 | center=False).std() 106 | df['bbtop'].fillna(df['bbtop'].mean(), inplace=True) 107 | df['bbbottom'] = df['sma'] - stddev * df[targetcol].rolling( 108 | min_periods=1, 109 | window=window, 110 | center=False).std() 111 | df['bbbottom'].fillna(df['bbbottom'].mean(), inplace=True) 112 | df['bbrange'] = df['bbtop'] - df['bbbottom'] 113 | df['bbpercent'] = ((df[targetcol] - df['bbbottom']) / df['bbrange']) - 0.5 114 | return df 115 | 116 | 117 | def plotRSI(p, df, plotwidth=800, upcolor='green', downcolor='red'): 118 | # create y axis for rsi 119 | p.extra_y_ranges = {"rsi": Range1d(start=0, end=100)} 120 | p.add_layout(LinearAxis(y_range_name="rsi"), 'right') 121 | 122 | # create rsi 'zone' (30-70) 123 | p.patch(np.append(df['date'].values, df['date'].values[::-1]), 124 | np.append([30 for i in df['rsi'].values], 125 | [70 for i in df['rsi'].values[::-1]]), 126 | color='olive', 127 | fill_alpha=0.2, 128 | legend="rsi", 129 | y_range_name="rsi") 130 | 131 | candleWidth = (df.iloc[2]['date'].timestamp() - 132 | df.iloc[1]['date'].timestamp()) * plotwidth 133 | # plot green bars 134 | inc = df.rsi >= 50 135 | p.vbar(x=df.date[inc], 136 | width=candleWidth, 137 | top=df.rsi[inc], 138 | bottom=50, 139 | fill_color=upcolor, 140 | line_color=upcolor, 141 | alpha=0.5, 142 | y_range_name="rsi") 143 | # Plot red bars 144 | dec = df.rsi <= 50 145 | p.vbar(x=df.date[dec], 146 | width=candleWidth, 147 | top=50, 148 | bottom=df.rsi[dec], 149 | fill_color=downcolor, 150 | line_color=downcolor, 151 | alpha=0.5, 152 | y_range_name="rsi") 153 | 154 | 155 | def plotMACD(p, df, color='blue'): 156 | # plot macd 157 | p.line(df['date'], df['macd'], line_width=4, 158 | color=color, alpha=0.8, legend="macd") 159 | p.yaxis[0].formatter = NumeralTickFormatter(format='0.00000000') 160 | 161 | 162 | def plotCandlesticks(p, df, plotwidth=750, upcolor='green', downcolor='red'): 163 | candleWidth = (df.iloc[2]['date'].timestamp() - 164 | df.iloc[1]['date'].timestamp()) * plotwidth 165 | # Plot candle 'shadows'/wicks 166 | p.segment(x0=df.date, 167 | y0=df.high, 168 | x1=df.date, 169 | y1=df.low, 170 | color="black", 171 | line_width=2) 172 | # Plot green candles 173 | inc = df.close > df.open 174 | p.vbar(x=df.date[inc], 175 | width=candleWidth, 176 | top=df.open[inc], 177 | bottom=df.close[inc], 178 | fill_color=upcolor, 179 | line_width=0.5, 180 | line_color='black') 181 | # Plot red candles 182 | dec = df.open > df.close 183 | p.vbar(x=df.date[dec], 184 | width=candleWidth, 185 | top=df.open[dec], 186 | bottom=df.close[dec], 187 | fill_color=downcolor, 188 | line_width=0.5, 189 | line_color='black') 190 | # format price labels 191 | p.yaxis[0].formatter = NumeralTickFormatter(format='0.00000000') 192 | 193 | 194 | def plotVolume(p, df, plotwidth=800, upcolor='green', downcolor='red'): 195 | candleWidth = (df.iloc[2]['date'].timestamp() - 196 | df.iloc[1]['date'].timestamp()) * plotwidth 197 | # create new y axis for volume 198 | p.extra_y_ranges = {"volume": Range1d(start=min(df['volume'].values), 199 | end=max(df['volume'].values))} 200 | p.add_layout(LinearAxis(y_range_name="volume"), 'right') 201 | # Plot green candles 202 | inc = df.close > df.open 203 | p.vbar(x=df.date[inc], 204 | width=candleWidth, 205 | top=df.volume[inc], 206 | bottom=0, 207 | alpha=0.1, 208 | fill_color=upcolor, 209 | line_color=upcolor, 210 | y_range_name="volume") 211 | 212 | # Plot red candles 213 | dec = df.open > df.close 214 | p.vbar(x=df.date[dec], 215 | width=candleWidth, 216 | top=df.volume[dec], 217 | bottom=0, 218 | alpha=0.1, 219 | fill_color=downcolor, 220 | line_color=downcolor, 221 | y_range_name="volume") 222 | 223 | 224 | def plotBBands(p, df, color='navy'): 225 | # Plot bbands 226 | p.patch(np.append(df['date'].values, df['date'].values[::-1]), 227 | np.append(df['bbbottom'].values, df['bbtop'].values[::-1]), 228 | color=color, 229 | fill_alpha=0.1, 230 | legend="bband") 231 | # plot sma 232 | p.line(df['date'], df['sma'], color=color, alpha=0.9, legend="sma") 233 | 234 | 235 | def plotMovingAverages(p, df): 236 | # Plot moving averages 237 | p.line(df['date'], df['emaslow'], 238 | color='orange', alpha=0.9, legend="emaslow") 239 | p.line(df['date'], df['emafast'], 240 | color='red', alpha=0.9, legend="emafast") 241 | 242 | 243 | class Charter(object): 244 | """ Retrieves 5min candlestick data for a market and saves it in a mongo 245 | db collection. Can display data in a dataframe or bokeh plot.""" 246 | 247 | def __init__(self, api): 248 | """ 249 | api = poloniex api object 250 | """ 251 | self.api = api 252 | 253 | def __call__(self, pair, frame=False): 254 | """ returns raw chart data from the mongo database, updates/fills the 255 | data if needed, the date column is the '_id' of each candle entry, and 256 | the date column has been removed. Use 'frame' to restrict the amount 257 | of data returned. 258 | Example: 'frame=api.YEAR' will return last years data 259 | """ 260 | # use last pair and period if not specified 261 | if not frame: 262 | frame = self.api.YEAR * 10 263 | dbcolName = pair + 'chart' 264 | # get db connection 265 | db = MongoClient()['poloniex'][dbcolName] 266 | # get last candle 267 | try: 268 | last = sorted( 269 | list(db.find({"_id": {"$gt": time() - 60 * 20}})), 270 | key=itemgetter('_id'))[-1] 271 | except: 272 | last = False 273 | # no entrys found, get all 5min data from poloniex 274 | if not last: 275 | logger.warning('%s collection is empty!', dbcolName) 276 | new = self.api.returnChartData(pair, 277 | period=60 * 5, 278 | start=time() - self.api.YEAR * 13) 279 | else: 280 | new = self.api.returnChartData(pair, 281 | period=60 * 5, 282 | start=int(last['_id'])) 283 | # add new candles 284 | updateSize = len(new) 285 | logger.info('Updating %s with %s new entrys!', 286 | dbcolName, str(updateSize)) 287 | 288 | # show the progess 289 | for i in range(updateSize): 290 | print("\r%s/%s" % (str(i + 1), str(updateSize)), end=" complete ") 291 | date = new[i]['date'] 292 | del new[i]['date'] 293 | db.update_one({'_id': date}, {"$set": new[i]}, upsert=True) 294 | print('') 295 | 296 | logger.debug('Getting chart data from db') 297 | # return data from db (sorted just in case...) 298 | return sorted( 299 | list(db.find({"_id": {"$gt": time() - frame}})), 300 | key=itemgetter('_id')) 301 | 302 | def dataFrame(self, pair, frame=False, zoom=False, window=120): 303 | """ returns pandas DataFrame from raw db data with indicators. 304 | zoom = passed as the resample(rule) argument to 'merge' candles into a 305 | different timeframe 306 | window = number of candles to use when calculating indicators 307 | """ 308 | data = self.__call__(pair, frame) 309 | # make dataframe 310 | df = pd.DataFrame(data) 311 | # set date column 312 | df['date'] = pd.to_datetime(df["_id"], unit='s') 313 | if zoom: 314 | df.set_index('date', inplace=True) 315 | df = df.resample(rule=zoom, 316 | closed='left', 317 | label='left').apply({'open': 'first', 318 | 'high': 'max', 319 | 'low': 'min', 320 | 'close': 'last', 321 | 'quoteVolume': 'sum', 322 | 'volume': 'sum', 323 | 'weightedAverage': 'mean'}) 324 | df.reset_index(inplace=True) 325 | 326 | # calculate/add sma and bbands 327 | df = bbands(df, window) 328 | # add slow ema 329 | df = ema(df, window, colname='emaslow') 330 | # add fast ema 331 | df = ema(df, int(window // 3.5), colname='emafast') 332 | # add macd 333 | df = macd(df) 334 | # add rsi 335 | df = rsi(df, window // 5) 336 | # add candle body and shadow size 337 | df['bodysize'] = df['close'] - df['open'] 338 | df['shadowsize'] = df['high'] - df['low'] 339 | df['percentChange'] = df['close'].pct_change() 340 | df.dropna(inplace=True) 341 | return df 342 | 343 | def graph(self, pair, frame=False, zoom=False, 344 | window=120, plot_width=1000, min_y_border=40, 345 | border_color="whitesmoke", background_color="white", 346 | background_alpha=0.4, legend_location="top_left", 347 | tools="pan,wheel_zoom,reset"): 348 | """ 349 | Plots market data using bokeh and returns a 2D array for gridplot 350 | """ 351 | df = self.dataFrame(pair, frame, zoom, window) 352 | # 353 | # Start Candlestick Plot ------------------------------------------- 354 | # create figure 355 | candlePlot = figure( 356 | x_axis_type=None, 357 | y_range=(min(df['low'].values) - (min(df['low'].values) * 0.2), 358 | max(df['high'].values) * 1.2), 359 | x_range=(df.tail(int(len(df) // 10)).date.min().timestamp() * 1000, 360 | df.date.max().timestamp() * 1000), 361 | tools=tools, 362 | title=pair, 363 | plot_width=plot_width, 364 | plot_height=int(plot_width // 2.7), 365 | toolbar_location="above") 366 | # add plots 367 | # plot volume 368 | plotVolume(candlePlot, df) 369 | # plot candlesticks 370 | plotCandlesticks(candlePlot, df) 371 | # plot bbands 372 | plotBBands(candlePlot, df) 373 | # plot moving aves 374 | plotMovingAverages(candlePlot, df) 375 | # set legend location 376 | candlePlot.legend.location = legend_location 377 | # set background color 378 | candlePlot.background_fill_color = background_color 379 | candlePlot.background_fill_alpha = background_alpha 380 | # set border color and size 381 | candlePlot.border_fill_color = border_color 382 | candlePlot.min_border_left = min_y_border 383 | candlePlot.min_border_right = candlePlot.min_border_left 384 | # 385 | # Start RSI/MACD Plot ------------------------------------------- 386 | # create a new plot and share x range with candlestick plot 387 | rsiPlot = figure(plot_height=int(candlePlot.plot_height // 2.5), 388 | x_axis_type="datetime", 389 | y_range=(-(max(df['macd'].values) * 2), 390 | max(df['macd'].values) * 2), 391 | x_range=candlePlot.x_range, 392 | plot_width=candlePlot.plot_width, 393 | title=None, 394 | toolbar_location=None) 395 | # plot macd 396 | plotMACD(rsiPlot, df) 397 | # plot rsi 398 | plotRSI(rsiPlot, df) 399 | # set background color 400 | rsiPlot.background_fill_color = candlePlot.background_fill_color 401 | rsiPlot.background_fill_alpha = candlePlot.background_fill_alpha 402 | # set border color and size 403 | rsiPlot.border_fill_color = candlePlot.border_fill_color 404 | rsiPlot.min_border_left = candlePlot.min_border_left 405 | rsiPlot.min_border_right = candlePlot.min_border_right 406 | rsiPlot.min_border_bottom = 20 407 | # orient x labels 408 | rsiPlot.xaxis.major_label_orientation = pi / 4 409 | # set legend 410 | rsiPlot.legend.location = legend_location 411 | # set dataframe 'date' as index 412 | df.set_index('date', inplace=True) 413 | # return layout and df 414 | return [[candlePlot], [rsiPlot]], df 415 | 416 | 417 | if __name__ == '__main__': 418 | from poloniex import Poloniex 419 | from bokeh.layouts import gridplot 420 | 421 | logging.basicConfig(level=logging.DEBUG) 422 | logging.getLogger("poloniex").setLevel(logging.INFO) 423 | logging.getLogger('requests').setLevel(logging.ERROR) 424 | 425 | api = Poloniex(jsonNums=float) 426 | 427 | layout, df = Charter(api).graph('USDT_BTC', window=90, 428 | frame=api.YEAR * 12, zoom='1W') 429 | print(df.tail()) 430 | p = gridplot(layout) 431 | show(p) --------------------------------------------------------------------------------