├── .gitignore ├── LICENSE ├── README.md ├── configs ├── collector.xml ├── tg_bot.xml ├── tg_client.xml └── trader.xml ├── requirements.txt ├── src ├── __init__.py ├── bot.py ├── client.py ├── collector.py ├── constants.py ├── predictor.py ├── scribe.py ├── trader.py └── util.py └── start.py /.gitignore: -------------------------------------------------------------------------------- 1 | # PyCharm files 2 | .idea/ 3 | 4 | # Virtual environment 5 | venv/ 6 | 7 | # Generic stuff 8 | data/ 9 | 10 | # ML stuff 11 | *.ipynb 12 | *.docx 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Ivan Modin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | NOTICE: The CryptoPing company seems to be closed so this project is no longer relevant 2 | 3 | Trading system for altcoins which is able to: 4 | 5 | - Collect signals from [CryptoPing](https://cryptoping.tech) Telegram bot 6 | - Match signals from this bot to signals from [CryptoPing](https://cryptoping.tech) site 7 | - Learn to predict expected signal profit (RandomForest) 8 | - Listen for incoming signals in Telegram 9 | - Buy altcoins which have good predictions on 7 crypto exchanges: Bittrex, Poloniex, YoBit, HitBTC, Tidex, Binance, Bitfinex 10 | - Give reports about prediction model performance, exchanges balances, current trades and finished trades in separate Telegram bot 11 | 12 | Every setting that can be tuned placed in `/src/constants.py`. All api keys, session tokens, and other sensitive values can be filled in configs placed in `/configs`. 13 | 14 | Full dataset collected starting from August 2017 can be found on [Kaggle](https://www.kaggle.com/reddelexc/crypto-assets-signals/activity), feel free to experiment with it. 15 | -------------------------------------------------------------------------------- /configs/collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | _frontend_session 4 | https://cryptoping.tech/backend/notifications?page= 5 | 6 | 7 | 8 | 2017-08-31 05:11:43 9 | 2017-08-31 00:36:19 10 | 2017-08-31 00:20:17 11 | 2017-08-30 22:54:08 12 | 2017-08-30 22:36:28 13 | 2017-08-30 22:30:57 14 | 2017-08-30 21:51:23 15 | 2017-08-30 21:22:08 16 | 2017-08-30 21:16:50 17 | 2017-08-30 21:15:48 18 | 2017-08-30 20:31:15 19 | 2017-08-30 20:14:17 20 | 2017-08-30 19:11:49 21 | 2017-08-30 19:08:58 22 | 2017-08-30 18:48:45 23 | 2017-08-30 18:16:33 24 | 2017-08-30 18:12:11 25 | 2017-08-30 18:05:18 26 | 2017-08-30 17:35:22 27 | 2017-08-30 17:21:37 28 | 2017-08-30 17:13:29 29 | 2017-08-30 17:05:17 30 | 2017-08-30 17:04:14 31 | 2017-08-30 16:47:43 32 | 2017-08-30 16:44:27 33 | 2017-08-30 16:39:05 34 | 2017-08-30 16:36:46 35 | 2017-08-30 16:29:34 36 | 2017-08-30 16:26:08 37 | 2017-08-13 14:54:19 38 | 2017-08-13 14:39:58 39 | 2018-07-29 13:11:04 40 | 2018-10-26 19:36:54 41 | 42 | -------------------------------------------------------------------------------- /configs/tg_bot.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /configs/tg_client.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /configs/trader.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Bittrex 4 | 5 | 6 | 7 | 8 | Poloniex 9 | 10 | 11 | 12 | 13 | YoBit 14 | 15 | 16 | 17 | 18 | HitBTC 19 | 20 | 21 | 22 | 23 | Tidex 24 | 25 | 26 | 27 | 28 | Binance 29 | 30 | 31 | 32 | 33 | Bitfinex 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | python-telegram-bot 2 | PySocks 3 | Telethon==0.19.1.6 4 | requests 5 | pandas 6 | numpy 7 | scikit-learn 8 | ccxt 9 | joblib -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | from .bot import Bot 2 | from .client import Client 3 | from .collector import Collector 4 | from .predictor import Predictor, PredictorLearnThread 5 | from .scribe import Scribe 6 | from .trader import Trader, TraderThreadCleaner 7 | from .util import GarbageCleanerThread 8 | -------------------------------------------------------------------------------- /src/bot.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import Updater, CommandHandler 2 | from xml.etree import ElementTree 3 | 4 | from .util import PoolObject, form_traceback, get_btc_price 5 | from .constants import proxy_host, proxy_port, tg_bot_config 6 | 7 | 8 | class Bot(PoolObject): 9 | def __init__(self, use_proxy=False): 10 | PoolObject.__init__(self) 11 | 12 | self.meta = self.parse_xml() 13 | if use_proxy: 14 | self.updater = Updater( 15 | self.meta['bot_token'], 16 | use_context=False, 17 | request_kwargs={'proxy_url': 'socks5://{0}:{1}'.format(proxy_host, proxy_port)} 18 | ) 19 | else: 20 | self.updater = Updater(self.meta['bot_token']) 21 | self.dispatcher = self.updater.dispatcher 22 | self.add_handlers() 23 | self.updater.start_polling() 24 | self.available = True 25 | 26 | print('bot: started') 27 | 28 | @staticmethod 29 | def parse_xml(): 30 | tree = ElementTree.parse(tg_bot_config) 31 | root = tree.getroot() 32 | meta = { 33 | 'owner_id': root[0].text, 34 | 'bot_token': root[1].text 35 | } 36 | return meta 37 | 38 | def auth(self, chat_id): 39 | return str(chat_id) == self.meta['owner_id'] 40 | 41 | def send(self, tokens): 42 | try: 43 | tokens = [str(token) for token in tokens] 44 | self.updater.bot.send_message(self.meta['owner_id'], '\n'.join(tokens)) 45 | except Exception as exc: 46 | print('failed to send message: {0}, through bot: {1}'.format(tokens, exc)) 47 | self.updater.bot.send_message(self.meta['owner_id'], '\n'.join(tokens)) 48 | 49 | def add_handlers(self): 50 | self.dispatcher.add_handler(CommandHandler('say_hi', self.say_hi)) 51 | self.dispatcher.add_handler(CommandHandler('get_predictor_report', self.get_predictor_report)) 52 | self.dispatcher.add_handler(CommandHandler('get_scribe_report', self.get_scribe_report)) 53 | self.dispatcher.add_handler(CommandHandler('get_trader_report', self.get_trader_report)) 54 | self.dispatcher.add_handler(CommandHandler('start_listener', self.start_listener)) 55 | self.dispatcher.add_handler(CommandHandler('stop_listener', self.stop_listener)) 56 | self.dispatcher.add_handler(CommandHandler('cur_listener_status', self.cur_listener_status)) 57 | self.dispatcher.add_handler(CommandHandler('update_dataset', self.update_dataset)) 58 | self.dispatcher.add_handler(CommandHandler('rewrite_dataset', self.rewrite_dataset)) 59 | self.dispatcher.add_handler(CommandHandler('complete_dataset', self.complete_dataset)) 60 | self.dispatcher.add_handler(CommandHandler('cur_dataset_size', self.cur_dataset_size)) 61 | self.dispatcher.add_handler(CommandHandler('cur_btc_price', self.cur_btc_price)) 62 | self.dispatcher.add_handler(CommandHandler('restore_threads', self.restore_threads)) 63 | 64 | def say_hi(self, _, update): 65 | if not self.auth(update.message.chat_id): 66 | return 67 | update.message.reply_text('hi') 68 | 69 | def get_predictor_report(self, _, update): 70 | if not self.auth(update.message.chat_id): 71 | return 72 | try: 73 | update.message.reply_text('Predictor:\n' + self.pool['predictor'].get_report()) 74 | except Exception as exc: 75 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 76 | 77 | def get_scribe_report(self, _, update): 78 | if not self.auth(update.message.chat_id): 79 | return 80 | try: 81 | update.message.reply_text('Scribe:\n' + self.pool['scribe'].get_report()) 82 | except Exception as exc: 83 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 84 | 85 | def get_trader_report(self, _, update): 86 | if not self.auth(update.message.chat_id): 87 | return 88 | try: 89 | update.message.reply_text('Trader:\n' + self.pool['trader'].get_report()) 90 | except Exception as exc: 91 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 92 | 93 | def start_listener(self, _, update): 94 | if not self.auth(update.message.chat_id): 95 | return 96 | try: 97 | self.pool['client'].start_listener() 98 | update.message.reply_text('Started listener') 99 | except Exception as exc: 100 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 101 | 102 | def stop_listener(self, _, update): 103 | if not self.auth(update.message.chat_id): 104 | return 105 | try: 106 | self.pool['client'].stop_listener() 107 | update.message.reply_text('Stopped listener') 108 | except Exception as exc: 109 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 110 | 111 | def cur_listener_status(self, _, update): 112 | if not self.auth(update.message.chat_id): 113 | return 114 | try: 115 | if self.pool['client'].cur_listener_status(): 116 | update.message.reply_text('Listener is active') 117 | else: 118 | update.message.reply_text('Listener is inactive') 119 | except Exception as exc: 120 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 121 | 122 | def update_dataset(self, _, update): 123 | if not self.auth(update.message.chat_id): 124 | return 125 | update.message.reply_text('Updating dataset...') 126 | try: 127 | self.pool['client'].update_dataset() 128 | update.message.reply_text('Dataset updated') 129 | except Exception as exc: 130 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 131 | 132 | def rewrite_dataset(self, _, update): 133 | if not self.auth(update.message.chat_id): 134 | return 135 | update.message.reply_text('Rewriting dataset...') 136 | try: 137 | self.pool['client'].rewrite_dataset() 138 | update.message.reply_text('Dataset rewritten') 139 | except Exception as exc: 140 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 141 | 142 | def complete_dataset(self, _, update): 143 | if not self.auth(update.message.chat_id): 144 | return 145 | update.message.reply_text('Completing dataset...') 146 | try: 147 | self.pool['client'].complete_dataset() 148 | update.message.reply_text('Dataset completed') 149 | except Exception as exc: 150 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 151 | 152 | def cur_dataset_size(self, _, update): 153 | if not self.auth(update.message.chat_id): 154 | return 155 | try: 156 | size = self.pool['client'].cur_dataset_size() 157 | update.message.reply_text('Dataset size is ' + str(size)) 158 | except Exception as exc: 159 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 160 | 161 | def cur_btc_price(self, _, update): 162 | if not self.auth(update.message.chat_id): 163 | return 164 | try: 165 | price = get_btc_price() 166 | update.message.reply_text('Current Bitcoin price is {0:.0f}$'.format(price)) 167 | except Exception as exc: 168 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 169 | 170 | def restore_threads(self, _, update): 171 | if not self.auth(update.message.chat_id): 172 | return 173 | try: 174 | result = self.pool['trader'].restore_threads() 175 | if result: 176 | update.message.reply_text('Trader threads restored') 177 | else: 178 | update.message.reply_text('Not trader threads to restore') 179 | 180 | except Exception as exc: 181 | update.message.reply_text('Something wrong happened:\n' + form_traceback(exc)) 182 | -------------------------------------------------------------------------------- /src/client.py: -------------------------------------------------------------------------------- 1 | import socks 2 | import os 3 | 4 | from xml.etree import ElementTree 5 | from telethon import TelegramClient, events, utils 6 | from telethon.errors import SessionPasswordNeededError 7 | 8 | from .util import PoolObject, form_traceback 9 | from .constants import data, client_tg_session, proxy_host, proxy_port, tg_client_config 10 | 11 | 12 | class Client(TelegramClient, PoolObject): 13 | def __init__(self, use_proxy=False): 14 | PoolObject.__init__(self) 15 | 16 | self.meta = self.parse_xml() 17 | self.listener_status = False 18 | if not os.path.exists(data): 19 | os.makedirs(data) 20 | if use_proxy: 21 | super().__init__( 22 | client_tg_session, 23 | self.meta['api_id'], 24 | self.meta['api_hash'], 25 | proxy=(socks.SOCKS5, proxy_host, proxy_port), 26 | ) 27 | else: 28 | super().__init__( 29 | client_tg_session, 30 | self.meta['api_id'], 31 | self.meta['api_hash'], 32 | ) 33 | try: 34 | self.connect() 35 | except ConnectionError: 36 | print('client: 1st connect failed, 2nd connect...') 37 | self.connect() 38 | 39 | if not self.is_user_authorized(): 40 | print('client: sending code to authorize...') 41 | self.sign_in(self.meta['owner_phone']) 42 | self_user = None 43 | while self_user is None: 44 | code = input('client: enter the code: ') 45 | try: 46 | self_user = self.sign_in(code=code) 47 | except SessionPasswordNeededError: 48 | pw = input('client: enter 2fa pass: ') 49 | self_user = self.sign_in(password=pw) 50 | self.available = True 51 | print('client: started') 52 | 53 | @staticmethod 54 | def parse_xml(): 55 | tree = ElementTree.parse(tg_client_config) 56 | root = tree.getroot() 57 | meta = { 58 | 'owner_phone': root[0].text, 59 | 'api_id': root[1].text, 60 | 'api_hash': root[2].text, 61 | 'cryptoping_bot_id': root[3].text 62 | } 63 | return meta 64 | 65 | def update_handler(self, update): 66 | update = update.original_update 67 | if str(update.user_id) == self.meta['cryptoping_bot_id'] and update.message.startswith('💎'): 68 | try: 69 | self.pool['collector'].process_signal(update) 70 | except Exception as exc: 71 | self.pool['bot'].send(['Something wrong happened:', form_traceback(exc)]) 72 | 73 | def start_listener(self): 74 | if self.listener_status: 75 | return 76 | self.listener_status = True 77 | self.add_event_handler(self.update_handler, events.NewMessage) 78 | 79 | def stop_listener(self): 80 | if not self.listener_status: 81 | return 82 | self.listener_status = False 83 | self.remove_event_handler(self.update_handler, events.NewMessage) 84 | 85 | def cur_listener_status(self): 86 | return self.listener_status 87 | 88 | def get_cryptoping_entity(self): 89 | dialogs = self.get_dialogs() 90 | for dialog in dialogs: 91 | if utils.get_display_name(dialog.entity) == 'CryptoPing': 92 | return dialog.entity 93 | return None 94 | 95 | def update_dataset(self): 96 | cryptoping_dialog_entity = self.get_cryptoping_entity() 97 | messages = [] 98 | for msg in reversed(self.get_messages( 99 | cryptoping_dialog_entity, 100 | limit=None, 101 | min_id=self.pool['collector'].meta['last_signal_id'])): 102 | if msg.message.startswith('💎'): 103 | messages.append(msg) 104 | self.pool['collector'].update_dataset(messages) 105 | 106 | def rewrite_dataset(self): 107 | cryptoping_dialog_entity = self.get_cryptoping_entity() 108 | messages = [] 109 | for msg in reversed(self.get_messages(cryptoping_dialog_entity, limit=None)): 110 | if msg.message.startswith('💎'): 111 | messages.append(msg) 112 | self.pool['collector'].update_dataset(messages, rewrite=True) 113 | 114 | def complete_dataset(self): 115 | self.pool['collector'].complete_dataset() 116 | 117 | def cur_dataset_size(self): 118 | return self.pool['collector'].meta['dataset_size'] 119 | -------------------------------------------------------------------------------- /src/collector.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import re 3 | import time 4 | import requests 5 | 6 | from xml.etree import ElementTree 7 | from datetime import datetime 8 | from threading import RLock 9 | 10 | from .util import PoolObject, get_btc_price 11 | from .constants import collector_config, allowed_exchanges, volume_threshold, predictor_dataset 12 | 13 | lock = RLock() 14 | 15 | 16 | class Collector(PoolObject): 17 | def __init__(self): 18 | PoolObject.__init__(self) 19 | 20 | self.meta = self.parse_xml() 21 | self.available = True 22 | 23 | print('collector: started') 24 | 25 | @staticmethod 26 | def parse_xml(): 27 | tree = ElementTree.parse(collector_config) 28 | root = tree.getroot() 29 | meta = { 30 | 'cryptoping_session': root[0].text, 31 | 'cryptoping_session_name': root[1].text, 32 | 'cryptoping_url': root[2].text, 33 | 'dataset_size': int(root[3].text), 34 | 'last_signal_id': int(root[4].text), 35 | } 36 | signal_exceptions = [] 37 | for item in root[5]: 38 | signal_exceptions.append(item.text) 39 | meta['signal_exceptions'] = signal_exceptions 40 | return meta 41 | 42 | def update_xml(self): 43 | tree = ElementTree.parse(collector_config) 44 | root = tree.getroot() 45 | root[3].text = str(self.meta['dataset_size']) 46 | root[4].text = str(self.meta['last_signal_id']) 47 | tree.write(collector_config) 48 | 49 | def process_signal(self, msg): 50 | signal = Collector.parse_message(msg) 51 | signal['buy_vol_per'] = float(signal['buy_vol_per']) 52 | signal['buy_vol_btc'] = float(signal['buy_vol_btc']) 53 | signal['price_per'] = float(signal['price_per']) 54 | signal['price_btc'] = float(signal['price_btc']) 55 | signal['week_signals'] = int(signal['week_signals']) 56 | signal['cap'] = None if signal['cap'] == '' else float(signal['cap']) 57 | signal['1h_max'] = 0.0 58 | signal['6h_max'] = 0.0 59 | signal['24h_max'] = 0.0 60 | signal['48h_max'] = 0.0 61 | signal['7d_max'] = 0.0 62 | signal['bpi'] = get_btc_price() 63 | signal['volume'] = signal['buy_vol_btc'] / signal['buy_vol_per'] * 100 * 24 64 | 65 | waiting_time = 0 66 | while True: 67 | with lock: 68 | available = self.pool['predictor'].available 69 | if available: 70 | break 71 | time.sleep(1) 72 | waiting_time += 1 73 | if waiting_time >= 60: 74 | self.pool['bot'].send(['Collector:', 'Predictor is not available, waiting to process signal...']) 75 | waiting_time = 0 76 | 77 | pred = self.pool['predictor'].predict(signal) 78 | metrics = self.pool['predictor'].metrics 79 | signal['estimated_profit'] = pred 80 | ignore_reason = None 81 | 82 | if signal['exchange'] in allowed_exchanges: 83 | with lock: 84 | locked = self.pool['trader'].locks[signal['exchange']] 85 | if locked: 86 | ignore_reason = 'exchange balance locked' 87 | elif float(signal['volume']) * float(signal['bpi']) < volume_threshold: 88 | ignore_reason = 'low volume' 89 | elif pred < metrics['preds_75_percentile']: 90 | ignore_reason = 'low estimated profit' 91 | else: 92 | ignore_reason = 'not allowed exchange' 93 | 94 | if ignore_reason is None: 95 | self.pool['bot'].send([ 96 | '[APPROVED]', 97 | ' - Ticker: ' + signal['ticker'], 98 | ' - Exchange: ' + signal['exchange'], 99 | ' - Signal price: {0:.8f}'.format(signal['price_btc']), 100 | ' - Estimated profit: ' + str(signal['estimated_profit'])]) 101 | with lock: 102 | self.pool['scribe'].approved(signal) 103 | self.pool['trader'].make_trade(signal) 104 | else: 105 | signal['ignore_reason'] = ignore_reason 106 | self.pool['bot'].send([ 107 | '[IGNORED]', 108 | ' - Ticker: ' + signal['ticker'], 109 | ' - Exchange: ' + signal['exchange'], 110 | ' - Signal price: {0:.8f}'.format(signal['price_btc']), 111 | ' - Estimated profit: ' + str(signal['estimated_profit']), 112 | ' - Ignore reason: ' + signal['ignore_reason']]) 113 | with lock: 114 | self.pool['scribe'].ignored(signal) 115 | 116 | @staticmethod 117 | def parse_message(msg): 118 | message_tokens = re.split('\n|, ', msg.message) 119 | message = { 120 | 'id': msg.id, 121 | 'date': str(msg.date).split('+')[0], 122 | 'ticker': message_tokens[0][3:], 123 | 'exchange': message_tokens[1].split()[3], 124 | 'buy_vol_per': message_tokens[2][1:-1], 125 | 'buy_vol_btc': message_tokens[3].split()[4], 126 | 'price_per': message_tokens[4][1:-1], 127 | 'price_btc': message_tokens[5].split()[1], 128 | 'week_signals': message_tokens[6].split()[1][:-3], 129 | 'cap': message_tokens[7].split()[2][1:].replace(',', ''), 130 | '1h_max': '', 131 | '6h_max': '', 132 | '24h_max': '', 133 | '48h_max': '', 134 | '7d_max': '', 135 | 'bpi': '' 136 | } 137 | return message 138 | 139 | @staticmethod 140 | def parse_page(resp): 141 | start_index = resp.find('') 142 | end_index = resp.find('') 143 | regexp = re.compile('<.*?>') 144 | signals_tokens = re.sub(regexp, ' ', resp[start_index:end_index]).split() 145 | signals_tokens = [st for st in signals_tokens if '/7d' not in st] 146 | parsed_signals = [] 147 | i = 0 148 | while i < len(signals_tokens): 149 | signal = { 150 | 'ticker': signals_tokens[i], 151 | 'date': signals_tokens[i + 2] + ' ' + signals_tokens[i + 3], 152 | 'price': signals_tokens[i + 4], 153 | '1h_max': signals_tokens[i + 5], 154 | '6h_max': signals_tokens[i + 7], 155 | '24h_max': signals_tokens[i + 9], 156 | '48h_max': signals_tokens[i + 11], 157 | '7d_max': signals_tokens[i + 13], 158 | 'exchange': signals_tokens[i + 15] 159 | } 160 | parsed_signals.append(signal) 161 | i += 16 162 | return parsed_signals 163 | 164 | def update_dataset(self, new_items, rewrite=False): 165 | messages = [] 166 | for item in new_items: 167 | parsed_message = self.parse_message(item) 168 | if str(parsed_message['date']) not in self.meta['signal_exceptions']: 169 | messages.append(self.parse_message(item)) 170 | if len(messages) == 0: 171 | return 172 | columns = messages[0].keys() 173 | write_mode = 'w' if rewrite else 'a' 174 | with open(predictor_dataset, write_mode, newline='', encoding='utf-8') as file: 175 | writer = csv.DictWriter(file, fieldnames=columns) 176 | if rewrite: 177 | writer.writeheader() 178 | writer.writerows(messages) 179 | if rewrite: 180 | self.meta['dataset_size'] = len(messages) 181 | else: 182 | self.meta['dataset_size'] += len(messages) 183 | self.meta['last_signal_id'] = messages[-1]['id'] 184 | self.update_xml() 185 | 186 | def complete_dataset(self): 187 | completed_samples = [] 188 | samples_to_complete = [] 189 | recent_samples = [] 190 | now_utc = datetime.utcnow() 191 | with open(predictor_dataset, 'r', newline='') as file: 192 | reader = csv.DictReader(file) 193 | for row in reader: 194 | if row['7d_max'] != '': 195 | completed_samples.append(row) 196 | else: 197 | row_date = datetime.strptime(row['date'], '%Y-%m-%d %H:%M:%S') 198 | if (now_utc - row_date).days >= 7: 199 | samples_to_complete.append(row) 200 | else: 201 | recent_samples.append(row) 202 | samples_to_complete = list(reversed(samples_to_complete)) 203 | 204 | if len(samples_to_complete) == 0: 205 | return 206 | 207 | page = 0 208 | i = 0 209 | not_completed = True 210 | while not_completed: 211 | page += 1 212 | cookie = {self.meta['cryptoping_session_name']: self.meta['cryptoping_session']} 213 | url = str(self.meta['cryptoping_url']) + str(page) 214 | resp = requests.get(url, cookies=cookie) 215 | parsed_signals = self.parse_page(resp.text) 216 | resp.close() 217 | j = 0 218 | 219 | prev_date = None 220 | cur_bpi = None 221 | 222 | while j < len(parsed_signals): 223 | row_date = datetime.strptime(samples_to_complete[i]['date'], '%Y-%m-%d %H:%M:%S') 224 | signal_date = datetime.strptime(parsed_signals[j]['date'], '%Y-%m-%d %H:%M') 225 | if row_date < signal_date: 226 | signal_date, row_date = row_date, signal_date 227 | if samples_to_complete[i]['ticker'] == parsed_signals[j]['ticker'] and \ 228 | samples_to_complete[i]['price_btc'] == parsed_signals[j]['price'] and \ 229 | samples_to_complete[i]['exchange'].lower() == parsed_signals[j]['exchange'].lower() and \ 230 | (row_date - signal_date).seconds < 7200: 231 | samples_to_complete[i]['1h_max'] = parsed_signals[j]['1h_max'] 232 | samples_to_complete[i]['6h_max'] = parsed_signals[j]['6h_max'] 233 | samples_to_complete[i]['24h_max'] = parsed_signals[j]['24h_max'] 234 | samples_to_complete[i]['48h_max'] = parsed_signals[j]['48h_max'] 235 | samples_to_complete[i]['7d_max'] = parsed_signals[j]['7d_max'] 236 | 237 | cur_date = str(samples_to_complete[i]['date'].split()[0]) 238 | if prev_date is None or cur_date != prev_date: 239 | cur_bpi = get_btc_price(cur_date) 240 | prev_date = cur_date 241 | time.sleep(0.1) 242 | samples_to_complete[i]['bpi'] = cur_bpi 243 | # print('fulfilled at index', i, ': ', str(samples_to_complete[i])) 244 | i += 1 245 | if i == len(samples_to_complete): 246 | not_completed = False 247 | break 248 | else: 249 | # print('stumbled at index ', i, ': ', str(samples_to_complete[i])) 250 | pass 251 | j += 1 252 | time.sleep(1) 253 | columns = completed_samples[0].keys() 254 | with open(predictor_dataset, 'w', newline='', encoding='utf-8') as file: 255 | writer = csv.DictWriter(file, fieldnames=columns) 256 | writer.writeheader() 257 | writer.writerows(completed_samples) 258 | writer.writerows(reversed(samples_to_complete)) 259 | writer.writerows(recent_samples) 260 | -------------------------------------------------------------------------------- /src/constants.py: -------------------------------------------------------------------------------- 1 | proxy_host = '127.0.0.1' 2 | proxy_port = 1080 3 | 4 | collector_config = 'configs/collector.xml' 5 | trader_config = 'configs/trader.xml' 6 | tg_bot_config = 'configs/tg_bot.xml' 7 | tg_client_config = 'configs/tg_client.xml' 8 | 9 | data = 'data/' 10 | client_tg_session = 'data/tg_client' 11 | predictor_dataset = 'data/dataset.csv' 12 | scribe_ignored_signals = 'data/scribe_ignored.csv' 13 | scribe_approved_signals = 'data/scribe_approved.csv' 14 | scribe_finished_trades = 'data/scribe_trades.csv' 15 | trained_model = 'data/trained_model/' 16 | trader_dumps = 'data/trader_dumps/' 17 | 18 | predictor_target_col = '24h_per' 19 | 20 | predictor_main_cols = [ 21 | 'ticker', 22 | 'exchange', 23 | 'buy_vol_per', 24 | 'buy_vol_btc', 25 | 'price_per', 26 | 'week_signals', 27 | 'hour', 28 | 'bpi', 29 | '24h_per' 30 | ] 31 | 32 | predictor_dummy_cols = [ 33 | 'ticker', 34 | 'exchange' 35 | ] 36 | 37 | allowed_exchanges = [ 38 | 'Poloniex' 39 | # 'Bittrex', 40 | 'YoBit', 41 | 'HitBTC', 42 | 'Binance', 43 | 'Bitfinex' 44 | # 'Tidex' 45 | ] 46 | 47 | necessary_exchange_methods = [ 48 | 'cancelOrder', 49 | 'createOrder', 50 | 'createLimitOrder', 51 | 'editOrder', 52 | 'fetchBalance', 53 | 'fetchClosedOrders', 54 | 'fetchL2OrderBook', 55 | 'fetchOpenOrders', 56 | 'fetchOrder', 57 | 'fetchTicker', 58 | 'fetchTrades', 59 | 'withdraw' 60 | ] 61 | 62 | report_cols = [ 63 | 'date', 64 | 'symbol', 65 | 'exchange', 66 | 'signal_price', 67 | 'bpi', 68 | 'trade_amount_per_thread', 69 | 'time_to_trade_secs', 70 | 'estimated_profit', 71 | 'real_profit', 72 | 'buy_price', 73 | 'sell_price', 74 | 'cancel_reason', 75 | 'sell_reason' 76 | ] 77 | 78 | # in percent 79 | exchanges_fees = { 80 | 'YoBit': 0.002, 81 | 'Cryptopia': 0.002, 82 | 'Bittrex': 0.0025, 83 | 'HitBTC': 0.002, 84 | 'Binance': 0.001 85 | } 86 | 87 | # dollars 88 | volume_threshold = 2000 89 | trade_amount_per_thread = 10 90 | 91 | # percent 92 | max_price_decrease = -5 93 | 94 | # seconds 95 | learning_period = 86400 96 | thread_cleaning_period = 43200 97 | trade_time_period = 86400 98 | garbage_cleaning_period = 10800 99 | pending_order_time = 20 100 | max_tries_to_call_api = 10 101 | -------------------------------------------------------------------------------- /src/predictor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import gc 3 | import time 4 | import joblib 5 | import pandas as pd 6 | import numpy as np 7 | 8 | from sklearn.ensemble import RandomForestRegressor 9 | from sklearn.metrics import mean_absolute_error, median_absolute_error 10 | from datetime import datetime 11 | from threading import RLock, Thread 12 | 13 | from .util import PoolObject, yobit_err, form_traceback 14 | from .constants import predictor_main_cols, predictor_target_col, predictor_dataset, \ 15 | predictor_dummy_cols, trained_model, learning_period 16 | 17 | lock = RLock() 18 | 19 | 20 | class Predictor(PoolObject): 21 | def __init__(self): 22 | PoolObject.__init__(self) 23 | 24 | self.dummies = None 25 | self.model = None 26 | self.model_date = None 27 | self.metrics = None 28 | self.load_stuff() 29 | 30 | self.data = None 31 | self.train_data = None 32 | self.val_data = None 33 | self.available = True 34 | 35 | print('predictor: started') 36 | 37 | def get_report(self): 38 | report = ' - Model last training date:\n' 39 | report += ' * {0}\n'.format(self.model_date) 40 | report += ' - Model metrics:\n' 41 | if self.metrics is None: 42 | report += ' * None\n' 43 | return report 44 | for k, v in self.metrics.items(): 45 | if k == 'cols': 46 | report += ' * {0}:\n'.format(k) 47 | for token in v.split(','): 48 | report += ' > {0}\n'.format(token) 49 | else: 50 | report += ' * {0}: {1:.2f}\n'.format(k, v) 51 | return report 52 | 53 | def learn(self): 54 | self.read_and_prepare_data() 55 | self.train() 56 | 57 | def predict(self, signal): 58 | self.data = pd.DataFrame(signal, index=[0]) 59 | 60 | if self.model is None: 61 | self.load_stuff() 62 | self.read_and_prepare_data(to_predict=True) 63 | 64 | data_use_cols = self.data[predictor_main_cols] 65 | data_dummied = data_use_cols.reindex(columns=self.dummies, fill_value=0) 66 | data_dummied.pop(predictor_target_col) 67 | x = data_dummied 68 | 69 | preds = self.model.predict(x) 70 | return preds[0] 71 | 72 | def read_and_prepare_data(self, to_predict=False): 73 | if not to_predict: 74 | self.data = pd.read_csv(predictor_dataset) 75 | self.data = self.data[self.data['1h_max'].notnull()] 76 | train_size = int(self.data.shape[0] * 0.75) 77 | self.data = self.data.iloc[-train_size:].reset_index(drop=True) 78 | 79 | self.data['date'] = pd.to_datetime(self.data['date'], format='%Y-%m-%d %H:%M:%S') 80 | self.data['year'] = self.data['date'].apply(lambda d: d.year) 81 | self.data['month'] = self.data['date'].apply(lambda d: d.month) 82 | self.data['day'] = self.data['date'].apply(lambda d: d.day) 83 | self.data['hour'] = self.data['date'].apply(lambda d: d.hour) 84 | self.data['minute'] = self.data['date'].apply(lambda d: d.minute) 85 | self.data['exchange'] = self.data['exchange'].apply(yobit_err) 86 | self.data['1h_per'] = (self.data['1h_max'] / self.data['price_btc'] - 1) * 100 87 | self.data['6h_per'] = (self.data['6h_max'] / self.data['price_btc'] - 1) * 100 88 | self.data['24h_per'] = (self.data['24h_max'] / self.data['price_btc'] - 1) * 100 89 | self.data['48h_per'] = (self.data['48h_max'] / self.data['price_btc'] - 1) * 100 90 | self.data['7d_per'] = (self.data['7d_max'] / self.data['price_btc'] - 1) * 100 91 | 92 | if not to_predict: 93 | last_index = self.data.shape[0] - 1 94 | last_day = self.data.iloc[-1]['day'] 95 | 96 | while self.data.iloc[last_index]['day'] == last_day: 97 | last_index -= 1 98 | 99 | val_end_index = last_index + 1 100 | last_day = self.data.iloc[last_index]['day'] 101 | 102 | while self.data.iloc[last_index]['day'] == last_day: 103 | last_index -= 1 104 | 105 | val_start_index = last_index + 1 106 | 107 | self.train_data = self.data.iloc[:val_start_index].reset_index(drop=True) 108 | self.val_data = self.data.iloc[val_start_index:val_end_index].reset_index(drop=True) 109 | 110 | def train(self): 111 | train_data_use_cols = self.train_data[predictor_main_cols] 112 | val_data_use_cols = self.val_data[predictor_main_cols] 113 | 114 | train_data_dummied = pd.get_dummies(train_data_use_cols, columns=predictor_dummy_cols) 115 | val_data_dummied = val_data_use_cols.reindex(columns=train_data_dummied.columns, fill_value=0) 116 | 117 | train_y = train_data_dummied.pop(predictor_target_col) 118 | train_x = train_data_dummied 119 | 120 | test_y = val_data_dummied.pop(predictor_target_col) 121 | test_x = val_data_dummied 122 | 123 | self.pool['bot'].send(['Predictor: started training for metrics']) 124 | val_model = RandomForestRegressor(n_estimators=100, random_state=100) 125 | val_model.fit(train_x, train_y) 126 | self.metrics = Predictor.get_metrics(predictor_main_cols, test_y, val_model.predict(test_x)) 127 | self.dump_metrics() 128 | 129 | self.train_data = None 130 | self.val_data = None 131 | gc.collect() 132 | 133 | self.pool['bot'].send(['Predictor: finished training for metrics']) 134 | 135 | data_use_cols = self.data[predictor_main_cols] 136 | data_dummied = pd.get_dummies(data_use_cols, columns=predictor_dummy_cols) 137 | self.dummies = data_dummied.columns 138 | 139 | train_y = data_dummied.pop(predictor_target_col) 140 | train_x = data_dummied 141 | 142 | self.pool['bot'].send(['Predictor: started training for real']) 143 | model = RandomForestRegressor(n_estimators=100, random_state=100) 144 | model.fit(train_x, train_y) 145 | self.model = model 146 | self.model_date = datetime.utcnow() 147 | self.dump_stuff() 148 | 149 | self.model = None 150 | self.dummies = None 151 | self.data = None 152 | gc.collect() 153 | 154 | self.pool['bot'].send(['Predictor: finished training for real']) 155 | 156 | def dump_stuff(self): 157 | with lock: 158 | self.available = False 159 | if not os.path.exists(trained_model): 160 | os.makedirs(trained_model) 161 | joblib.dump(self.dummies, os.path.join(trained_model, 'dummies')) 162 | joblib.dump(self.model, os.path.join(trained_model, 'model')) 163 | joblib.dump(self.model_date, os.path.join(trained_model, 'model_date')) 164 | with lock: 165 | self.available = True 166 | 167 | def dump_metrics(self): 168 | with lock: 169 | self.available = False 170 | if not os.path.exists(trained_model): 171 | os.makedirs(trained_model) 172 | joblib.dump(self.metrics, os.path.join(trained_model, 'metrics')) 173 | with lock: 174 | self.available = True 175 | 176 | def load_stuff(self): 177 | if not os.path.exists(trained_model): 178 | return 179 | self.dummies = joblib.load(os.path.join(trained_model, 'dummies')) 180 | self.model = joblib.load(os.path.join(trained_model, 'model')) 181 | self.model_date = joblib.load(os.path.join(trained_model, 'model_date')) 182 | self.metrics = joblib.load(os.path.join(trained_model, 'metrics')) 183 | 184 | @staticmethod 185 | def get_metrics(cols, real, preds): 186 | dev_1 = 0 187 | dev_5 = 0 188 | dev_10 = 0 189 | less_pred = 0 190 | more_pred = 0 191 | length = len(real) 192 | 193 | real = real.values 194 | for i in range(len(real)): 195 | if preds[i] >= real[i]: 196 | more_pred += 1 197 | if preds[i] < real[i]: 198 | less_pred += 1 199 | if abs(real[i] - preds[i]) <= 1: 200 | dev_1 += 1 201 | if abs(real[i] - preds[i]) <= 5: 202 | dev_5 += 1 203 | if abs(real[i] - preds[i]) <= 10: 204 | dev_10 += 1 205 | 206 | metrics = { 207 | 'cols': ', '.join(cols), 208 | 'real_mean': real.mean(), 209 | 'real_median': np.median(real), 210 | 'real_75_percentile': np.percentile(real, 75), 211 | 'preds_mean': preds.mean(), 212 | 'preds_median': np.median(preds), 213 | 'preds_75_percentile': np.percentile(preds, 75), 214 | 'mean_deviation': mean_absolute_error(real, preds), 215 | 'median_deviation': median_absolute_error(real, preds), 216 | 'deviation <= 1%': dev_1 / length, 217 | 'deviation <= 5%': dev_5 / length, 218 | 'deviation <= 10%': dev_10 / length, 219 | 'pred < real': less_pred / length, 220 | 'pred >= real': more_pred / length 221 | } 222 | 223 | return metrics 224 | 225 | 226 | class PredictorLearnThread(Thread): 227 | def __init__(self, predictor, client, bot): 228 | Thread.__init__(self) 229 | self.predictor = predictor 230 | self.client = client 231 | self.bot = bot 232 | 233 | def run(self): 234 | while True: 235 | try: 236 | self.bot.send(['Updating dataset...']) 237 | self.client.update_dataset() 238 | self.bot.send(['Dataset updated']) 239 | 240 | self.bot.send(['Completing dataset...']) 241 | self.client.complete_dataset() 242 | self.bot.send(['Dataset completed']) 243 | 244 | self.predictor.learn() 245 | except Exception as exc: 246 | self.bot.send(['Something wrong happened:', form_traceback(exc)]) 247 | time.sleep(learning_period) 248 | -------------------------------------------------------------------------------- /src/scribe.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os.path 3 | 4 | from .util import PoolObject 5 | from .constants import scribe_finished_trades, report_cols, scribe_ignored_signals, scribe_approved_signals 6 | 7 | 8 | class Scribe(PoolObject): 9 | def __init__(self): 10 | PoolObject.__init__(self) 11 | 12 | self.available = True 13 | 14 | print('scribe: started') 15 | 16 | @staticmethod 17 | def get_report(): 18 | count_of_signals = 3 19 | trade_signals = Scribe.read_from_csv(scribe_finished_trades, count_of_signals) 20 | report = ' - Last {0} trade signals\n'.format(count_of_signals) 21 | if len(trade_signals) == 0: 22 | report += ' * None\n' 23 | return report 24 | for signal in trade_signals: 25 | for k, v in signal.items(): 26 | if k not in report_cols or v is None: 27 | continue 28 | if str(v) != '': 29 | if k == 'signal_price' or k == 'buy_price' or k == 'sell_price' or k == 'estimated_profit': 30 | report += ' * {0}: {1:.8f}\n'.format(k, float(v)) 31 | else: 32 | report += ' * {0}: {1}\n'.format(k, v) 33 | report += '\n' 34 | return report 35 | 36 | @staticmethod 37 | def ignored(signal): 38 | Scribe.write_to_csv(scribe_ignored_signals, signal) 39 | 40 | @staticmethod 41 | def approved(signal): 42 | Scribe.write_to_csv(scribe_approved_signals, signal) 43 | 44 | @staticmethod 45 | def trade(signal): 46 | Scribe.write_to_csv(scribe_finished_trades, signal) 47 | 48 | @staticmethod 49 | def write_to_csv(filename, signal): 50 | write_header = not os.path.isfile(filename) 51 | with open(filename, 'w' if write_header else 'a', newline='') as file: 52 | columns = signal.keys() 53 | writer = csv.DictWriter(file, fieldnames=columns) 54 | if write_header: 55 | writer.writeheader() 56 | writer.writerow(signal) 57 | 58 | @staticmethod 59 | def read_from_csv(filename, count_of_signals): 60 | signals = [] 61 | if os.path.isfile(filename): 62 | with open(filename, 'r', newline='') as file: 63 | rows = list(csv.DictReader(file))[-count_of_signals:] 64 | for row in rows: 65 | signals.append(row) 66 | return signals 67 | -------------------------------------------------------------------------------- /src/trader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import time 4 | import ccxt 5 | 6 | from xml.etree import ElementTree 7 | from threading import RLock, Thread 8 | from os.path import join 9 | 10 | from .util import PoolObject, get_btc_price, form_traceback 11 | from .constants import allowed_exchanges, trader_config, report_cols, trader_dumps, \ 12 | trade_amount_per_thread, trade_time_period, exchanges_fees, max_tries_to_call_api, \ 13 | pending_order_time, max_price_decrease, thread_cleaning_period 14 | 15 | lock = RLock() 16 | 17 | 18 | class Trader(PoolObject): 19 | def __init__(self): 20 | PoolObject.__init__(self) 21 | 22 | self.exchanges = {} 23 | self.threads = [] 24 | self.free_balances = {} 25 | self.used_balances = {} 26 | self.locks = {} 27 | for ex in allowed_exchanges: 28 | self.locks[ex] = False 29 | 30 | self.meta = self.parse_xml() 31 | self.setup_clients() 32 | self.available = True 33 | 34 | print('trader: started') 35 | 36 | @staticmethod 37 | def parse_xml(): 38 | tree = ElementTree.parse(trader_config) 39 | root = tree.getroot() 40 | meta = {} 41 | for key in root: 42 | meta[key[0].text] = { 43 | 'public': key[1].text, 44 | 'secret': key[2].text 45 | } 46 | return meta 47 | 48 | def setup_clients(self): 49 | self.exchanges['Bittrex'] = ccxt.bittrex({ 50 | 'apiKey': self.meta['Bittrex']['public'], 51 | 'secret': self.meta['Bittrex']['secret'], 52 | }) 53 | self.exchanges['Poloniex'] = ccxt.poloniex({ 54 | 'apiKey': self.meta['Poloniex']['public'], 55 | 'secret': self.meta['Poloniex']['secret'], 56 | }) 57 | self.exchanges['YoBit'] = ccxt.yobit({ 58 | 'apiKey': self.meta['YoBit']['public'], 59 | 'secret': self.meta['YoBit']['secret'], 60 | }) 61 | self.exchanges['HitBTC'] = ccxt.hitbtc2({ 62 | 'apiKey': self.meta['HitBTC']['public'], 63 | 'secret': self.meta['HitBTC']['secret'], 64 | }) 65 | self.exchanges['Tidex'] = ccxt.tidex({ 66 | 'apiKey': self.meta['Tidex']['public'], 67 | 'secret': self.meta['Tidex']['secret'], 68 | }) 69 | self.exchanges['Binance'] = ccxt.binance({ 70 | 'options': {'adjustForTimeDifference': True}, 71 | 'apiKey': self.meta['Binance']['public'], 72 | 'secret': self.meta['Binance']['secret'], 73 | }) 74 | self.exchanges['Bitfinex'] = ccxt.bitfinex({ 75 | 'apiKey': self.meta['Bitfinex']['public'], 76 | 'secret': self.meta['Bitfinex']['secret'], 77 | }) 78 | 79 | def fetch_balances(self): 80 | for exchange, client in self.exchanges.items(): 81 | try: 82 | balances = client.fetch_balance() 83 | self.free_balances[exchange] = balances['free'] 84 | self.used_balances[exchange] = balances['used'] 85 | tickers_to_remove = [] 86 | for ticker, quantity in self.free_balances[exchange].items(): 87 | if float(quantity) == 0 and ticker.lower() != 'btc': 88 | tickers_to_remove.append(ticker) 89 | for ticker in tickers_to_remove: 90 | self.free_balances[exchange].pop(ticker, None) 91 | tickers_to_remove = [] 92 | 93 | for ticker, quantity in self.used_balances[exchange].items(): 94 | if float(quantity) == 0 and ticker.lower() != 'btc': 95 | tickers_to_remove.append(ticker) 96 | for ticker in tickers_to_remove: 97 | self.used_balances[exchange].pop(ticker, None) 98 | except Exception as exc: 99 | self.pool['bot'].send(['Something wrong happened during fetching ' + exchange + ' balance:', str(exc)]) 100 | 101 | def get_report(self): 102 | self.fetch_balances() 103 | report = ' - Free balances:\n' 104 | for exchange, balance in self.free_balances.items(): 105 | report += ' * {0}:\n'.format(exchange) 106 | for ticker, quantity in balance.items(): 107 | price = get_btc_price() 108 | dollars = float(quantity) * price 109 | if ticker != 'BTC': 110 | try: 111 | ticker_price = self.exchanges[exchange].fetch_ticker(ticker + '/BTC')['last'] 112 | except Exception: 113 | continue 114 | dollars *= ticker_price 115 | if dollars < 0.01: 116 | continue 117 | report += ' > {0}: {1:.8f} ({2:.2f}$)\n'.format(ticker, quantity, dollars) 118 | 119 | report += ' - Used balances:\n' 120 | for exchange, balance in self.used_balances.items(): 121 | report += ' * {0}:\n'.format(exchange) 122 | for ticker, quantity in balance.items(): 123 | price = get_btc_price() 124 | dollars = float(quantity) * price 125 | if ticker != 'BTC': 126 | try: 127 | ticker_price = self.exchanges[exchange].fetch_ticker(ticker + '/BTC')['last'] 128 | except Exception: 129 | continue 130 | dollars *= ticker_price 131 | if dollars < 0.01: 132 | continue 133 | report += ' > {0}: {1:.8f} ({2:.2f}$)\n'.format(ticker, quantity, dollars) 134 | 135 | report += ' - Current trades:\n' 136 | for thread in self.threads: 137 | if not thread.is_alive(): 138 | continue 139 | for k, v in thread.report.items(): 140 | if k not in report_cols or v is None: 141 | continue 142 | if k == 'signal_price' or ((k == 'buy_price' or k == 'sell_price') and v is not None): 143 | report += ' * {0}: {1:.8f}\n'.format(k, v) 144 | else: 145 | report += ' * {0}: {1}\n'.format(k, v) 146 | report += '\n' 147 | return report 148 | 149 | def make_trade(self, signal=None, report=None): 150 | if signal is None: 151 | exchange = report['exchange'] 152 | else: 153 | exchange = signal['exchange'] 154 | trader_thread = TraderThread( 155 | self, 156 | self.pool['bot'], 157 | self.pool['scribe'], 158 | self.exchanges[exchange], 159 | signal, 160 | report) 161 | trader_thread.setDaemon(True) 162 | trader_thread.start() 163 | self.threads.append(trader_thread) 164 | 165 | def restore_threads(self): 166 | if not os.path.exists(trader_dumps): 167 | os.makedirs(trader_dumps) 168 | dumps_filenames = os.listdir(trader_dumps) 169 | if len(dumps_filenames) == 0: 170 | return False 171 | for filename in dumps_filenames: 172 | with open(join(trader_dumps, filename), 'r') as file: 173 | report = json.load(file) 174 | self.make_trade(None, report) 175 | return True 176 | 177 | def dump_thread(self, report): 178 | if not os.path.exists(trader_dumps): 179 | os.makedirs(trader_dumps) 180 | try: 181 | filename = report['date'][:-9] + '_' + report['symbol'][:-4] + '_' + report['exchange'] 182 | with open(join(trader_dumps, filename), 'w+') as file: 183 | json.dump(report, file) 184 | except Exception as exc: 185 | self.pool['bot'].send(['Something wrong happened during dumping thread:', str(exc)]) 186 | 187 | def remove_thread_dump(self, report): 188 | try: 189 | filename = report['date'][:-9] + '_' + report['symbol'][:-4] + '_' + report['exchange'] 190 | os.remove(join(trader_dumps, filename)) 191 | except Exception as exc: 192 | self.pool['bot'].send(['Something wrong happened during removing the dump:', str(exc)]) 193 | 194 | 195 | class TraderThread(Thread): 196 | def __init__(self, trader, bot, scribe, client, signal=None, report=None): 197 | Thread.__init__(self) 198 | self.trader = trader 199 | self.bot = bot 200 | self.scribe = scribe 201 | self.client = client 202 | if report is None: 203 | self.report = { 204 | 'id': signal['id'], 205 | 'date': signal['date'], 206 | 'symbol': signal['ticker'] + '/BTC', 207 | 'exchange': signal['exchange'], 208 | 'signal_price': signal['price_btc'], 209 | 'bpi': signal['bpi'], 210 | 'trade_amount_per_thread': trade_amount_per_thread, 211 | 'time_to_trade_secs': trade_time_period, 212 | 'estimated_profit': signal['estimated_profit'], 213 | 'buy_price': None, 214 | 'sell_price': None, 215 | 'real_profit': None, 216 | 'buy_order_id': None, 217 | 'sell_order_id': None, 218 | 'cancel_reason': None, 219 | 'sell_reason': None, 220 | 'work_time_secs': 0, 221 | 'iteration_time_secs': client.rateLimit / 1000.0 + 0.1, 222 | 'placed_buy_order': False, 223 | 'bought': False, 224 | 'placed_sell_order': False, 225 | 'sold': False, 226 | 'order_open_time': 0, 227 | 'tries_to_call_api': 0 228 | } 229 | else: 230 | self.report = report 231 | 232 | def place_order(self, side, quantity, price): 233 | exception = None 234 | order = None 235 | while True: 236 | if self.report['tries_to_call_api'] > max_tries_to_call_api: 237 | self.bot.send(['Number of attempts to place {0} order exceeded: {1}'.format(side, str(exception))]) 238 | self.report['tries_to_call_api'] = 0 239 | return 240 | try: 241 | time.sleep(self.report['iteration_time_secs']) 242 | self.report['work_time_secs'] += self.report['iteration_time_secs'] 243 | order = self.client.create_order( 244 | symbol=self.report['symbol'], 245 | type='limit', 246 | side=side, 247 | amount=quantity, 248 | price=price) 249 | except Exception as exc: 250 | exception = exc 251 | self.report['tries_to_call_api'] += 1 252 | continue 253 | self.report['tries_to_call_api'] = 0 254 | break 255 | if side == 'buy': 256 | self.report['buy_order_id'] = order['id'] 257 | self.report['placed_buy_order'] = True 258 | else: 259 | self.report['sell_order_id'] = order['id'] 260 | self.report['placed_sell_order'] = True 261 | 262 | def cancel_order(self, side): 263 | exception = None 264 | while True: 265 | if self.report['tries_to_call_api'] > max_tries_to_call_api: 266 | self.bot.send(['Number of attempts to cancel {0} order exceeded: {1}'.format(side, str(exception))]) 267 | self.report['tries_to_call_api'] = 0 268 | return 269 | try: 270 | time.sleep(self.report['iteration_time_secs']) 271 | self.report['work_time_secs'] += self.report['iteration_time_secs'] 272 | self.client.cancel_order( 273 | self.report['buy_order_id'] if side == 'buy' else self.report['sell_order_id'], 274 | self.report['symbol']) 275 | except Exception as exc: 276 | exception = exc 277 | self.report['tries_to_call_api'] += 1 278 | continue 279 | self.report['tries_to_call_api'] = 0 280 | break 281 | if side == 'buy': 282 | self.report['buy_order_id'] = None 283 | self.report['placed_buy_order'] = False 284 | else: 285 | self.report['sell_order_id'] = None 286 | self.report['placed_sell_order'] = False 287 | 288 | def fetch_order(self, side): 289 | exception = None 290 | while True: 291 | if self.report['tries_to_call_api'] > max_tries_to_call_api: 292 | self.bot.send(['Number of attempts to fetch {0} order info exceeded: {1}'.format(side, str(exception))]) 293 | self.report['tries_to_call_api'] = 0 294 | return None 295 | try: 296 | time.sleep(self.report['iteration_time_secs']) 297 | self.report['work_time_secs'] += self.report['iteration_time_secs'] 298 | self.report['order_open_time'] += self.report['iteration_time_secs'] 299 | order_info = self.client.fetch_order( 300 | self.report['buy_order_id'] if side == 'buy' else self.report['sell_order_id'], 301 | self.report['symbol']) 302 | except Exception as exc: 303 | exception = exc 304 | self.report['tries_to_call_api'] += 1 305 | continue 306 | self.report['tries_to_call_api'] = 0 307 | return order_info 308 | 309 | def fetch_balance(self, category, ticker): 310 | exception = None 311 | while True: 312 | if self.report['tries_to_call_api'] > max_tries_to_call_api: 313 | self.bot.send(['Number of attempts to fetch {0} {1} balance exceeded: {2}'.format( 314 | category, 315 | ticker, 316 | str(exception))]) 317 | self.report['tries_to_call_api'] = 0 318 | return None 319 | try: 320 | time.sleep(self.report['iteration_time_secs']) 321 | self.report['work_time_secs'] += self.report['iteration_time_secs'] 322 | balance = float(self.client.fetch_balance()[category][ticker]) 323 | except Exception as exc: 324 | exception = exc 325 | self.report['tries_to_call_api'] += 1 326 | continue 327 | self.report['tries_to_call_api'] = 0 328 | return balance 329 | 330 | def fetch_token(self, when): 331 | exception = None 332 | while True: 333 | if self.report['tries_to_call_api'] > max_tries_to_call_api: 334 | self.bot.send(['Number of attempts to fetch ticker info ({0}) exceeded: {1}'.format( 335 | when, 336 | str(exception))]) 337 | self.report['tries_to_call_api'] = 0 338 | return None 339 | try: 340 | time.sleep(self.report['iteration_time_secs']) 341 | self.report['work_time_secs'] += self.report['iteration_time_secs'] 342 | ticker_stats = self.client.fetch_ticker(self.report['symbol']) 343 | except Exception as exc: 344 | exception = exc 345 | self.report['tries_to_call_api'] += 1 346 | continue 347 | self.report['tries_to_call_api'] = 0 348 | return ticker_stats 349 | 350 | def run(self): 351 | try: 352 | with lock: 353 | self.trader.locks[self.report['exchange']] = True 354 | 355 | while True: 356 | self.trader.dump_thread(self.report) 357 | 358 | if not self.report['placed_buy_order']: 359 | btc_balance = self.fetch_balance('free', 'BTC') 360 | if btc_balance is None: 361 | self.report['cancel_reason'] = 'unable to fetch BTC balance before placing buy order' 362 | break 363 | 364 | needed_btc_balance = trade_amount_per_thread / float(self.report['bpi']) 365 | 366 | ticker_stats = self.fetch_token('place buy') 367 | if ticker_stats is None: 368 | self.report['cancel_reason'] = 'unable to fetch token info before placing buy order' 369 | break 370 | 371 | last_price = ticker_stats['last'] 372 | pref_buy_price = ticker_stats['ask'] 373 | if btc_balance < needed_btc_balance: 374 | needed_btc_balance = btc_balance 375 | price_increase = (last_price / self.report['signal_price'] - 1) * 100 376 | if price_increase > self.report['estimated_profit']: 377 | self.report['cancel_reason'] = 'price increase is already higher than estimated profit' 378 | break 379 | self.report['estimated_profit'] -= price_increase 380 | 381 | self.trader.dump_thread(self.report) 382 | 383 | commission = exchanges_fees[self.report['exchange']] * needed_btc_balance 384 | quantity_to_buy = self.client.amount_to_precision( 385 | self.report['symbol'], 386 | float((needed_btc_balance - commission) / pref_buy_price)) 387 | self.place_order('buy', quantity_to_buy, pref_buy_price) 388 | if not self.report['placed_buy_order']: 389 | self.report['cancel_reason'] = 'unable to place buy order' 390 | break 391 | self.trader.dump_thread(self.report) 392 | 393 | if self.report['placed_buy_order'] and not self.report['bought']: 394 | order_info = self.fetch_order('buy') 395 | if order_info is None: 396 | self.report['cancel_reason'] = 'unable to fetch order info after placing buy order' 397 | break 398 | 399 | if self.report['order_open_time'] > pending_order_time: 400 | self.report['order_open_time'] = 0 401 | if order_info['status'] == 'open': 402 | self.cancel_order('buy') 403 | if self.report['placed_buy_order']: 404 | self.report['cancel_reason'] = 'unable to cancel buy order' 405 | break 406 | continue 407 | else: 408 | self.report['bought'] = True 409 | self.report['buy_price'] = order_info['price'] 410 | self.bot.send(['Trader:', 'Bought {0} on {1}'.format(self.report['symbol'], 411 | self.report['exchange'])]) 412 | else: 413 | if order_info['status'] == 'open': 414 | continue 415 | else: 416 | self.report['bought'] = True 417 | self.report['buy_price'] = order_info['price'] 418 | self.report['order_open_time'] = 0 419 | self.bot.send(['Trader:', 'Bought {0} on {1}'.format(self.report['symbol'], 420 | self.report['exchange'])]) 421 | 422 | self.trader.dump_thread(self.report) 423 | 424 | if self.report['bought'] and not self.report['placed_sell_order']: 425 | ticker_stats = self.fetch_token('place sell') 426 | if ticker_stats is None: 427 | self.report['cancel_reason'] = 'unable to fetch token info before placing sell order' 428 | break 429 | 430 | pref_sell_price = ticker_stats['bid'] 431 | price_change = (pref_sell_price / self.report['buy_price'] - 1) * 100 432 | 433 | if price_change < max_price_decrease: 434 | self.report['sell_reason'] = 'price decreased more than on {0}%'.format(max_price_decrease) 435 | elif price_change >= self.report['estimated_profit']: 436 | self.report['sell_reason'] = 'price reached estimated value' 437 | elif self.report['work_time_secs'] > trade_time_period: 438 | self.report['sell_reason'] = 'trade time exceeded' 439 | else: 440 | continue 441 | 442 | ticker_balance = self.fetch_balance('free', self.report['symbol'][:-4]) 443 | if ticker_balance is None: 444 | self.report['cancel_reason'] = 'unable to to fetch token balance before placing sell order' 445 | break 446 | 447 | self.place_order('sell', ticker_balance, pref_sell_price) 448 | if not self.report['placed_sell_order']: 449 | self.report['cancel_reason'] = 'unable to place sell order' 450 | break 451 | self.trader.dump_thread(self.report) 452 | 453 | if self.report['placed_sell_order'] and not self.report['sold']: 454 | order_info = self.fetch_order('sell') 455 | if order_info is None: 456 | self.report['cancel_reason'] = 'unable to fetch order info after placing sell order' 457 | break 458 | 459 | if self.report['order_open_time'] > pending_order_time: 460 | self.report['order_open_time'] = 0 461 | if order_info['status'] == 'open': 462 | self.cancel_order('sell') 463 | if self.report['placed_sell_order']: 464 | self.report['cancel_reason'] = 'unable to cancel sell order' 465 | break 466 | continue 467 | else: 468 | self.report['sold'] = True 469 | self.report['sell_price'] = order_info['price'] 470 | self.bot.send(['Trader:', 'Sold {0} on {1}'.format(self.report['symbol'], 471 | self.report['exchange'])]) 472 | else: 473 | if order_info['status'] == 'open': 474 | continue 475 | else: 476 | self.report['sold'] = True 477 | self.report['sell_price'] = order_info['price'] 478 | self.report['order_open_time'] = 0 479 | self.bot.send(['Trader:', 'Sold {0} on {1}'.format(self.report['symbol'], 480 | self.report['exchange'])]) 481 | 482 | self.trader.dump_thread(self.report) 483 | 484 | if self.report['sold']: 485 | self.report['real_profit'] = (self.report['sell_price'] / self.report['buy_price'] - 1) * 100 486 | break 487 | 488 | self.trader.dump_thread(self.report) 489 | 490 | with lock: 491 | self.trader.locks[self.report['exchange']] = False 492 | self.scribe.trade(self.report) 493 | self.trader.remove_thread_dump(self.report) 494 | except Exception as exc: 495 | self.bot.send(['Something wrong happened:', form_traceback(exc)]) 496 | 497 | 498 | class TraderThreadCleaner(Thread): 499 | def __init__(self, trader, bot): 500 | Thread.__init__(self) 501 | self.trader = trader 502 | self.bot = bot 503 | 504 | def run(self): 505 | while True: 506 | time.sleep(thread_cleaning_period) 507 | try: 508 | self.trader.threads = [t for t in self.trader.threads if t.is_alive()] 509 | except Exception as exc: 510 | self.bot.send(['Something wrong happened:', form_traceback(exc)]) 511 | -------------------------------------------------------------------------------- /src/util.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | import requests 3 | import json 4 | import time 5 | import gc 6 | 7 | from threading import Thread 8 | 9 | from .constants import garbage_cleaning_period 10 | 11 | 12 | class PoolObject: 13 | def __init__(self): 14 | self.pool = None 15 | self.available = False 16 | 17 | def set_pool(self, pool): 18 | self.pool = pool 19 | 20 | 21 | class GarbageCleanerThread(Thread): 22 | def __init__(self, bot): 23 | Thread.__init__(self) 24 | self.bot = bot 25 | 26 | def run(self): 27 | while True: 28 | gc.collect() 29 | time.sleep(garbage_cleaning_period) 30 | 31 | 32 | def yobit_err(value): 33 | if value == 'Yobit': 34 | return 'YoBit' 35 | else: 36 | return value 37 | 38 | 39 | def form_traceback(exc): 40 | trace = [] 41 | for line in traceback.extract_tb(exc.__traceback__, limit=4096): 42 | trace.append(str(line)) 43 | trace.append(str(exc)) 44 | trace_str = '\n'.join(trace) 45 | return trace_str 46 | 47 | 48 | def get_btc_price(date=None): 49 | if date is not None: 50 | date = str(date) 51 | url = 'https://api.coindesk.com/v1/bpi/historical/close.json?start=' + date + '&end=' + date 52 | resp = requests.get(url) 53 | price = json.loads(resp.text)['bpi'][date] 54 | resp.close() 55 | return price 56 | else: 57 | url = 'https://api.coindesk.com/v1/bpi/currentprice.json' 58 | resp = requests.get(url) 59 | price = json.loads(resp.text)['bpi']['USD']['rate_float'] 60 | resp.close() 61 | return price 62 | -------------------------------------------------------------------------------- /start.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from src import Bot, Client, Collector, \ 4 | Predictor, PredictorLearnThread, Scribe, \ 5 | Trader, TraderThreadCleaner, GarbageCleanerThread 6 | 7 | if __name__ == '__main__': 8 | use_proxy = len(sys.argv) > 1 and sys.argv[1] == '-p' 9 | pool = { 10 | 'client': Client(use_proxy), 11 | 'bot': Bot(use_proxy), 12 | 'collector': Collector(), 13 | 'predictor': Predictor(), 14 | 'scribe': Scribe(), 15 | 'trader': Trader(), 16 | } 17 | 18 | for _, entity in pool.items(): 19 | entity.set_pool(pool) 20 | 21 | garbage_cleaning_thread = GarbageCleanerThread(pool['bot']) 22 | garbage_cleaning_thread.setDaemon(True) 23 | garbage_cleaning_thread.start() 24 | 25 | predictor_learn_thread = PredictorLearnThread(pool['predictor'], pool['client'], pool['bot']) 26 | predictor_learn_thread.setDaemon(True) 27 | predictor_learn_thread.start() 28 | 29 | trader_thread_cleaner = TraderThreadCleaner(pool['trader'], pool['bot']) 30 | trader_thread_cleaner.setDaemon(True) 31 | trader_thread_cleaner.start() 32 | --------------------------------------------------------------------------------