├── tickers └── empty ├── exchangeAPI ├── __init__.py ├── bitmex.py ├── poloniex.py ├── bitfinex.py ├── gdax.py └── bittrex.py ├── README.md ├── .gitignore ├── requirements.txt ├── download.py ├── parallel.py ├── utility.py ├── database.py └── __main__.py /tickers/empty: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /exchangeAPI/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ExchangeStat 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | *.pyc 3 | **__pycache__ 4 | **.json 5 | **.csv 6 | 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aws-shell==0.1.1 2 | awscli==1.11.132 3 | boto3==1.4.5 4 | botocore==1.5.95 5 | certifi==2017.7.27.1 6 | chardet==3.0.4 7 | colorama==0.3.7 8 | configobj==5.0.6 9 | docutils==0.14 10 | idna==2.5 11 | jmespath==0.9.3 12 | prompt-toolkit==1.0.15 13 | pyasn1==0.3.2 14 | Pygments==2.2.0 15 | pymongo==3.5.0 16 | python-dateutil==2.6.1 17 | PyYAML==3.12 18 | requests==2.18.3 19 | rsa==3.4.2 20 | s3transfer==0.1.10 21 | six==1.10.0 22 | urllib3==1.22 23 | wcwidth==0.1.7 24 | -------------------------------------------------------------------------------- /exchangeAPI/bitmex.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | class Bitmex(object): 5 | 6 | allowed_pairs = { 7 | 'USD_BTC' : 'XBT', 8 | 'YEN_BTC' : 'XBJ', 9 | 'BTC_DASH' : 'DASH', 10 | 'BTC_ETH' : 'ETH', 11 | 'BTC_ETC' : 'ETC', 12 | 'BTC_LTC' : 'LTC', 13 | 'BTC_QTUM' : 'QTUM', 14 | 'BTC_XMR' : 'XMR', 15 | 'BTC_XRP' : 'XRP', 16 | 'BTC_XTZ' : 'XTZ', 17 | 'BTX_ZEC' : 'ZEC' 18 | } 19 | 20 | @staticmethod 21 | def check_pair(pair): 22 | if pair in Bitmex.allowed_pairs.keys(): 23 | return [True, 'OK'] 24 | return [False, 'This instrument is not available on this exchange.'] 25 | 26 | @staticmethod 27 | def get_quotation(pair): 28 | try: 29 | ret = requests.get('https://www.bitmex.com/api/v1/quote?symbol=' + Bitmex.allowed_pairs[pair] + '&count=1&reverse=true', timeout=0.8) 30 | if ret.status_code != 200: 31 | return 0 32 | ret = ret.json()[0] 33 | return [ret['bidPrice'], ret['askPrice']] 34 | except: 35 | return 0 36 | -------------------------------------------------------------------------------- /download.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from database import Data 3 | import sys 4 | from pandas.io.json import json_normalize 5 | 6 | db = Data() 7 | coursor = db.get_tickers() 8 | res = [] 9 | first = True 10 | 11 | for ticker in coursor: 12 | if first: 13 | _from = ticker['time'] 14 | first = False 15 | last = ticker['time'] 16 | ticker.pop('_id') 17 | res.append(ticker) 18 | if len(res) == 0: 19 | print ('No data to download in database.') 20 | sys.exit(0) 21 | 22 | data_name = _from[:10] + ' - ' + last[:10] + '.csv' 23 | tmp = 'tickers/' + data_name 24 | 25 | res = json_normalize(res) 26 | res.to_csv(tmp) 27 | 28 | client = boto3.client('s3', region_name='eu-central-1') 29 | response = client.list_buckets() 30 | buckets = [bucket['Name'] for bucket in response['Buckets']] 31 | 32 | bucket_name = 'exchangestatistics' 33 | if bucket_name not in buckets: 34 | client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={ 35 | 'LocationConstraint': 'eu-central-1' 36 | }) 37 | 38 | client.upload_file(tmp, bucket_name, data_name) 39 | 40 | presigned_url = client.generate_presigned_url('get_object', Params = {'Bucket': bucket_name, 'Key': data_name}) 41 | print ('Your link for downloading: ') 42 | print (presigned_url) 43 | 44 | db.del_tickers() 45 | -------------------------------------------------------------------------------- /exchangeAPI/poloniex.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | class Poloniex(object): 5 | def retry(func): 6 | def wrapper(*args, **kwargs): 7 | for i in range(3): 8 | try: 9 | res = func(*args) 10 | if res: 11 | return [res, 'OK'] 12 | else: 13 | return [res, 'This instrument is not available on this exchange.'] 14 | except: 15 | continue 16 | return [False, 'Poloniex: problems with network.'] 17 | return wrapper 18 | 19 | @staticmethod 20 | @retry 21 | def check_pair(pair): 22 | pair = pair.replace('USD', 'USDT') 23 | ret = requests.get('https://poloniex.com/public?command=returnTicker', timeout=2) 24 | if pair not in ret.json().keys(): 25 | return False 26 | return True 27 | 28 | @staticmethod 29 | def get_quotation(pair): 30 | pair = pair.replace('USD', 'USDT') 31 | try: 32 | ret = requests.get('https://poloniex.com/public?command=returnTicker', timeout=0.8) 33 | if ret.status_code != 200: 34 | return 0 35 | ret = ret.json()[pair] 36 | return [float(ret['highestBid']), float(ret['lowestAsk'])] 37 | except: 38 | return 0 39 | -------------------------------------------------------------------------------- /exchangeAPI/bitfinex.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | class Bitfinex(object): 5 | 6 | def retry(func): 7 | def wrapper(*args, **kwargs): 8 | for i in range(3): 9 | try: 10 | res = func(*args) 11 | if res: 12 | return [res, 'OK'] 13 | else: 14 | return [res, 'This instrument is not available on this exchange.'] 15 | except: 16 | continue 17 | return [False, 'Bitfinex: problems with network.'] 18 | return wrapper 19 | 20 | @staticmethod 21 | @retry 22 | def check_pair(pair): 23 | pair = pair[pair.find('_') + 1:] + pair[:pair.find('_')] 24 | pair = pair.lower() 25 | ret = requests.get('https://api.bitfinex.com/v1/symbols', timeout=2) 26 | if pair not in ret.json(): 27 | return False 28 | return True 29 | 30 | def get_quotation(pair): 31 | pair = pair[pair.find('_') + 1:] + pair[:pair.find('_')] 32 | try: 33 | ret = requests.get('https://api.bitfinex.com/v1/pubticker/' + pair, timeout=0.8) 34 | if ret.status_code != 200: 35 | return 0 36 | ret = ret.json() 37 | return [float(ret['bid']), float(ret['ask'])] 38 | except: 39 | return 0 40 | -------------------------------------------------------------------------------- /parallel.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process 2 | from multiprocessing.pool import ThreadPool 3 | from database import Data 4 | from utility import template2 5 | import time 6 | 7 | def do_func(arg_list): 8 | exchange = template2[arg_list[0]] 9 | pairs = arg_list[1] 10 | if len(pairs) == 0: 11 | return 0 12 | pool = ThreadPool(len(pairs)) 13 | ret = pool.map(exchange.get_quotation, pairs) 14 | pool.terminate() 15 | pool.join() 16 | return ret 17 | 18 | def start_collecting(): 19 | db = Data() 20 | while True: 21 | result_dict = {} 22 | _from = time.time() 23 | result_dict['time'] = time.strftime("%Y.%m.%d - %H:%M:%S") 24 | cur_setup = db.get_cur_setup() 25 | list_ = [] 26 | for key, item in cur_setup.items(): 27 | list_.append([key, item]) 28 | pool = ThreadPool(5) 29 | ret = pool.map(do_func, list_) 30 | pool.terminate() 31 | pool.join() 32 | for jdx, item in enumerate(list_): 33 | tmp_dict = {} 34 | for idx, pair in enumerate(item[1]): 35 | tmp_dict[pair] = ret[jdx][idx] 36 | result_dict[item[0]] = tmp_dict 37 | 38 | #print (result_dict) 39 | db.insert_tick(result_dict) 40 | end_time = time.time() - _from 41 | if end_time < 1: 42 | time.sleep(1 - end_time) 43 | 44 | def stop_collecting(process): 45 | process.terminate() 46 | -------------------------------------------------------------------------------- /exchangeAPI/gdax.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | class Gdax(object): 5 | 6 | def retry(func): 7 | def wrapper(*args, **kwargs): 8 | for i in range(3): 9 | try: 10 | res = func(*args) 11 | if res: 12 | return [res, 'OK'] 13 | else: 14 | return [res, 'This instrument is not available on this exchange.'] 15 | except: 16 | continue 17 | return [False, 'Gdax: problems with network.'] 18 | return wrapper 19 | 20 | @staticmethod 21 | @retry 22 | def check_pair(pair): 23 | pair = pair[pair.find('_') + 1:] + '-' + pair[:pair.find('_')] 24 | ret = requests.get('https://api.gdax.com/products', timeout=2) 25 | ret = ret.json() 26 | for cur_pair in ret: 27 | if pair == cur_pair["id"]: 28 | return True 29 | return False 30 | 31 | @staticmethod 32 | def get_quotation(pair): 33 | pair = pair[pair.find('_') + 1:] + '-' + pair[:pair.find('_')] 34 | try: 35 | ret = requests.get('https://api.gdax.com/products/' + pair + '/book', timeout=0.8) 36 | if ret.status_code != 200: 37 | return 0 38 | ret = ret.json() 39 | return [float(ret["bids"][0][0]), float(ret["asks"][0][0])] 40 | except: 41 | return 0 42 | -------------------------------------------------------------------------------- /exchangeAPI/bittrex.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | class Bittrex(object): 5 | 6 | def retry(func): 7 | def wrapper(*args, **kwargs): 8 | for i in range(3): 9 | try: 10 | res = func(*args) 11 | if res: 12 | return [res, 'OK'] 13 | else: 14 | return [res, 'This instrument is not available on this exchange.'] 15 | except: 16 | continue 17 | return [False, 'Bittrex: problems with network.'] 18 | return wrapper 19 | 20 | @staticmethod 21 | @retry 22 | def check_pair(pair): 23 | pair = pair.replace('USD', 'USDT') 24 | pair = pair.replace('_', '-') 25 | ret = requests.get('https://bittrex.com/api/v1.1/public/getmarkets', timeout=2) 26 | ret = ret.json()['result'] 27 | for cur_pair in ret: 28 | if pair == cur_pair['MarketName']: 29 | return True 30 | return False 31 | 32 | @staticmethod 33 | def get_quotation(pair): 34 | pair = pair.replace('USD', 'USDT') 35 | pair = pair.replace('_', '-') 36 | try: 37 | ret = requests.get('https://bittrex.com/api/v1.1/public/getticker?market=' + pair, timeout=0.8) 38 | if ret.status_code != 200: 39 | return 0 40 | ret = ret.json()['result'] 41 | return [ret['Bid'], ret['Ask']] 42 | except: 43 | return 0 44 | -------------------------------------------------------------------------------- /utility.py: -------------------------------------------------------------------------------- 1 | from exchangeAPI.poloniex import Poloniex 2 | from exchangeAPI.bitmex import Bitmex 3 | from exchangeAPI.bitfinex import Bitfinex 4 | from exchangeAPI.bittrex import Bittrex 5 | from exchangeAPI.gdax import Gdax 6 | 7 | template = { 8 | 'poloniex' : [], 9 | 'bittrex' : [], 10 | 'gdax' : [], 11 | 'bitfinex' : [], 12 | 'bitmex' : [] 13 | } 14 | 15 | template2 = { 16 | 'poloniex' : Poloniex, 17 | 'bittrex' : Bittrex, 18 | 'gdax' : Gdax, 19 | 'bitfinex' : Bitfinex, 20 | 'bitmex' : Bitmex 21 | } 22 | 23 | def check_exchanges(list_): 24 | error = False 25 | for exch in list_: 26 | if exch not in template.keys(): 27 | print ('There is no exchange named: ' + exch) 28 | error = True 29 | if error: 30 | print ('Invalid command. Please, try again.') 31 | return True 32 | return False 33 | 34 | def check_cur_pair(pair, exchanges): 35 | error = False 36 | for exch in exchanges: 37 | res = template2[exch].check_pair(pair) 38 | if res[0] == False: 39 | error = True 40 | print (exch + ': ' + res[1]) 41 | return error 42 | 43 | def wtf(): 44 | er_text = '.\n.\n.' 45 | print (er_text) 46 | 47 | help_message = ('Available commands:\n' 48 | ' /add CUR1_CUR2 exchg1 exchg2 ... exchg5 - add instrument for collecting on given exchanges' 49 | ' if it is available there. Can be used with bot turned on.\n' 50 | ' /remove CUR1_CUR2 exchg1 exchg2 ... exchg5 - remove instrument from collecting on given' 51 | ' exchanges.\n' 52 | ' /info - show information about current bot settings (what instruments are being collected' 53 | ' at the moment).\n' 54 | ' /on - turn the bot on and start collecting.\n' 55 | ' /off - turn the bot off and stop collecting.') 56 | -------------------------------------------------------------------------------- /database.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | from utility import template 3 | 4 | class Data(object): 5 | def __init__(self): 6 | self.database = MongoClient()['ExchangeStat'] 7 | 8 | def setup(self, setup_dict): 9 | cur_setup = self.database['setup'].find_one() 10 | if cur_setup != None: 11 | for key in setup_dict.keys(): 12 | if len(setup_dict[key]) != 0: 13 | if setup_dict[key][0] not in cur_setup[key]: 14 | cur_setup[key].extend(setup_dict[key]) 15 | self.database['setup'].update_one( 16 | {}, 17 | { 18 | '$set' : cur_setup 19 | } 20 | ) 21 | else: 22 | self.database['setup'].insert_one( 23 | setup_dict 24 | ) 25 | 26 | def get_cur_setup(self): 27 | cur_setup = self.database['setup'].find_one() 28 | if cur_setup == None: 29 | return template 30 | else: 31 | cur_setup.pop('_id') 32 | return cur_setup 33 | 34 | def remove(self, pair, exchanges): 35 | cur_setup = self.database['setup'].find_one() 36 | if cur_setup == None: 37 | return 38 | else: 39 | for cur in exchanges: 40 | try: 41 | cur_setup[cur].remove(pair) 42 | except: 43 | pass 44 | self.database['setup'].update_one( 45 | {}, 46 | { 47 | '$set' : cur_setup 48 | } 49 | ) 50 | 51 | def insert_tick(self, tick): 52 | self.database['tickers'].insert_one( 53 | tick 54 | ) 55 | 56 | def get_tickers(self): 57 | tickers = self.database['tickers'].find() 58 | return tickers 59 | 60 | def del_tickers(self): 61 | self.database['tickers'].delete_many({}) 62 | 63 | def clean(self): 64 | cur_setup = self.database['setup'].find_one() 65 | if cur_setup != None: 66 | self.database['setup'].delete_one( 67 | cur_setup 68 | ) 69 | -------------------------------------------------------------------------------- /__main__.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | from exchangeAPI.poloniex import Poloniex 3 | from exchangeAPI.bitmex import Bitmex 4 | from exchangeAPI.bitfinex import Bitfinex 5 | from exchangeAPI.bittrex import Bittrex 6 | from exchangeAPI.gdax import Gdax 7 | from multiprocessing import Process 8 | from database import Data 9 | import parallel 10 | import utility 11 | import sys 12 | import time 13 | import copy 14 | 15 | proc = [] 16 | 17 | def command_handler(command): 18 | command = command.split(' ') 19 | if command[0] == '/help': 20 | print (utility.help_message) 21 | utility.wtf() 22 | return 23 | 24 | if command[0] == '/add': 25 | exch_check_er = utility.check_exchanges(command[2:]) 26 | if exch_check_er: 27 | utility.wtf() 28 | return 29 | pair_er = utility.check_cur_pair(command[1], command[2:]) 30 | if pair_er: 31 | utility.wtf() 32 | return 33 | setup_dict = copy.deepcopy(utility.template) 34 | if len(command[2:]) == 0: 35 | print ('Enter exchanges.') 36 | utility.wtf() 37 | return 38 | for exchange in command[2:]: 39 | setup_dict[exchange].append(command[1]) 40 | db = Data() 41 | db.setup(setup_dict) 42 | print ('OK, instrument added to collector. Use /on to start gathering information.') 43 | utility.wtf() 44 | return 45 | 46 | if command[0] == '/remove': 47 | exch_check_er = utility.check_exchanges(command[2:]) 48 | if exch_check_er: 49 | utility.wtf() 50 | return 51 | db = Data() 52 | db.remove(command[1], command[2:]) 53 | utility.wtf() 54 | return 55 | 56 | if command[0] == '/info': 57 | db = Data() 58 | cur_setup = db.get_cur_setup() 59 | for key, item in cur_setup.items(): 60 | print (key, ':', item) 61 | utility.wtf() 62 | return 63 | 64 | global proc 65 | if command[0] == '/off': 66 | if len(proc) != 0: 67 | parallel.stop_collecting(proc[0]) 68 | print ('Collecting stopped.') 69 | proc = [] 70 | utility.wtf() 71 | return 72 | 73 | if command[0] == '/on': 74 | proc.append(Process(target=parallel.start_collecting, args=())) 75 | proc[0].start() 76 | print ('Collecting started.') 77 | utility.wtf() 78 | return 79 | 80 | raise BaseException 81 | 82 | if __name__ == '__main__': 83 | print ('You are using exchange statistics collector.\nEnter command to start gathering information.\n' 84 | 'Enter /help to get list of available commands.') 85 | while True: 86 | try: 87 | command = input().strip() 88 | command_handler(command) 89 | except KeyboardInterrupt: 90 | print ('Turning bot off...') 91 | db = Data() 92 | db.clean() 93 | if len(proc) != 0: 94 | parallel.stop_collecting(proc[0]) 95 | sys.exit(0) 96 | except: 97 | print ('Unrecognized command. Please, try again.') 98 | utility.wtf() 99 | --------------------------------------------------------------------------------