├── BitmexOrderBookSaver.py ├── Generators.py ├── LICENSE ├── OrderBook.py ├── OrderBookContainer.py ├── README.md ├── Signal.py ├── TurexNetwork.py ├── my_utils.py └── requirements.txt /BitmexOrderBookSaver.py: -------------------------------------------------------------------------------- 1 | import websocket 2 | import threading 3 | import traceback 4 | import datetime 5 | from time import sleep 6 | import json 7 | import urllib 8 | import math 9 | from my_utils import * 10 | from OrderBook import OrderBook 11 | 12 | #included in bitmex_websocket package 13 | from util.api_key import generate_nonce, generate_signature 14 | 15 | 16 | #the sources with changes taken from https://github.com/BitMEX/api-connectors/tree/master/official-ws/python 17 | class BitmexOrderBookSaver: 18 | 19 | def __init__(self, api_key, api_secret, folder): 20 | 21 | self.out_file = None 22 | self.cur_YYYYMMDD = 0 23 | self.folder = folder 24 | 25 | self.L2 = {} 26 | self.L2['Buy'] = {} 27 | self.L2['Sell'] = {} 28 | 29 | self.api_key = api_key 30 | self.api_secret = api_secret 31 | 32 | self.data = {} 33 | self.keys = {} 34 | 35 | self.is_connected = self._connect('wss://www.bitmex.com/realtime?subscribe=orderBookL2:XBTUSD') 36 | 37 | if self.is_connected == False: 38 | return None 39 | 40 | self._wait_for_account() 41 | 42 | def exit(self): 43 | self.ws.close() 44 | sleep(0.1) 45 | 46 | def _connect(self, wsURL): 47 | self.ws = websocket.WebSocketApp(wsURL, 48 | on_message=self._on_message, 49 | on_close=self._on_close, 50 | on_open=self._on_open, 51 | on_error=self._on_error, 52 | header=self._get_auth()) 53 | 54 | self.wst = threading.Thread(target=lambda: self.ws.run_forever()) 55 | self.wst.daemon = True 56 | self.wst.start() 57 | 58 | conn_timeout = 10 59 | while not self.ws.sock or not self.ws.sock.connected and conn_timeout: 60 | sleep(1) 61 | conn_timeout -= 1 62 | if not conn_timeout: 63 | self.exit() 64 | raise websocket.WebSocketTimeoutException('Couldn\'t connect to WS! Exiting.') 65 | 66 | return True 67 | 68 | def _get_auth(self): 69 | nonce = generate_nonce() 70 | return [ 71 | f"api-nonce: {nonce}", 72 | f"api-signature: {generate_signature(self.api_secret, 'GET', '/realtime', nonce, '')}", 73 | f"api-key:{self.api_key}" 74 | ] 75 | 76 | def _wait_for_account(self): 77 | while not {'orderBookL2'} <= set(self.data): 78 | sleep(0.1) 79 | 80 | def _send_command(self, command, args=None): 81 | if args is None: 82 | args = [] 83 | self.ws.send(json.dumps({"op": command, "args": args})) 84 | 85 | def get_orderbook(self): 86 | orderbook = {} 87 | 88 | cur_UTC = datetime.datetime.utcnow() 89 | 90 | orderbook['date'] = get_YYYYMMDD(cur_UTC) 91 | orderbook['time'] = get_HHMMSSmmm(cur_UTC) 92 | 93 | orderbook['prices'] = [] 94 | orderbook['volumes'] = [] 95 | 96 | #only even values are applicable 97 | half_size = 50 98 | 99 | sell_side = sorted(self.L2['Sell'].items()) 100 | buy_side = sorted(self.L2['Buy'].items(), reverse=True) 101 | 102 | buy_side_len = len(buy_side) 103 | sell_side_len = len(sell_side) 104 | 105 | if buy_side_len < half_size or sell_side_len < half_size: 106 | return None 107 | 108 | for idx in reversed(range(0, half_size)): 109 | orderbook['prices'].append(buy_side[idx][0]) 110 | orderbook['volumes'].append(buy_side[idx][1]) 111 | 112 | for idx in range(0, half_size): 113 | orderbook['prices'].append(sell_side[idx][0]) 114 | orderbook['volumes'].append(sell_side[idx][1]) 115 | 116 | return OrderBook(orderbook) 117 | 118 | def _save_orderbook(self): 119 | 120 | orderbook = self.get_orderbook() 121 | 122 | if orderbook is not None: 123 | if self.out_file == None or self.cur_YYYYMMDD != orderbook.date: 124 | self.cur_YYYYMMDD = orderbook.date 125 | f_path = self.folder + '\\' + 'XBTUSD_' + str(self.cur_YYYYMMDD) + '.txt' 126 | self.out_file = open(f_path, 'a+') 127 | 128 | orderbook.write(self.out_file) 129 | 130 | def _on_message(self, ws, message): 131 | message = json.loads(message) 132 | table, action = message.get('table'), message.get('action') 133 | 134 | try: 135 | if action: 136 | if table not in self.data: 137 | self.data[table] = [] 138 | 139 | if table == 'orderBookL2': 140 | 141 | if action in ('partial', 'insert'): 142 | for item in message['data']: 143 | self.L2[item["side"]][item['price']] = item['size'] 144 | 145 | elif action == 'update': 146 | for item in message['data']: 147 | pr = get_price_from_ID(item['id']) 148 | 149 | self.L2[item["side"]][pr] = item['size'] 150 | 151 | elif action == 'delete': 152 | for item in message['data']: 153 | pr = get_price_from_ID(item['id']) 154 | del self.L2[item["side"]][pr] 155 | 156 | self._save_orderbook(); 157 | except: 158 | print(traceback.format_exc()) 159 | 160 | 161 | def _on_error(self, ws, error): 162 | self.ws.close() 163 | 164 | def _on_open(self, ws): 165 | pass 166 | 167 | def _on_close(self, ws): 168 | pass 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /Generators.py: -------------------------------------------------------------------------------- 1 | from Signal import Signal 2 | from TurexNetwork import * 3 | 4 | 5 | def sample_generator(volumes, threshold): 6 | 7 | height = len(volumes) 8 | 9 | h1 = sum(volumes[0:int(height / 2) + 1]) 10 | h2 = sum(volumes[int(height / 2):height + 1]) 11 | 12 | ind = (h2-h1) / (h2+h1) 13 | 14 | if ind > threshold: 15 | return Signal.SELL 16 | elif ind < -threshold: 17 | return Signal.BUY 18 | else: 19 | return Signal.WAIT 20 | 21 | 22 | def sample_neural_generator(turex_network, volumes, threshold): 23 | 24 | prediction = turex_network.predict(volumes) 25 | 26 | ind = prediction[1] - prediction[0] 27 | 28 | if ind > threshold: 29 | return Signal.SELL 30 | elif ind < -threshold: 31 | return Signal.BUY 32 | else: 33 | return Signal.WAIT 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 sturex 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /OrderBook.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | class OrderBook: 4 | def __init__(self, data): 5 | 6 | self._date = data['date'] 7 | self._time = data['time'] 8 | self._volumes = data['volumes'] 9 | self._prices = data['prices'] 10 | 11 | self._height = len(self._prices) 12 | 13 | @property 14 | def time(self): 15 | return self._time 16 | 17 | @property 18 | def date(self): 19 | return self._date 20 | 21 | @property 22 | def volumes(self): 23 | return self._volumes 24 | 25 | @property 26 | def best_prices(self): 27 | return { 'sell_price': self._prices[int(self._height / 2)], 'buy_price': self._prices[int(self._height / 2) - 1], 'time': self._time} 28 | 29 | def write(self, json_file): 30 | json.dump({'date': self._date, 'time': self._time, 'prices': self._prices, 'volumes': self._volumes}, json_file) 31 | json_file.write('\n') 32 | 33 | -------------------------------------------------------------------------------- /OrderBookContainer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from contextlib import suppress 4 | from OrderBook import * 5 | from Signal import Signal 6 | 7 | 8 | class OrderBookContainer: 9 | def __init__(self, path_to_file): 10 | 11 | self.order_books = [] 12 | self.trades = [] 13 | self.cur_directory = os.path.dirname(path_to_file) 14 | self.f_name = os.path.split(path_to_file)[1] 15 | 16 | with open(path_to_file, 'r') as infile: 17 | for line in infile: 18 | ob = json.loads(line) 19 | self.order_books.append(OrderBook(ob)) 20 | 21 | def create_training_dataset(self): 22 | if not self.order_books: 23 | return 24 | 25 | output_dir = os.path.join(self.cur_directory, 'Datasets') 26 | 27 | with suppress(OSError): 28 | os.mkdir(output_dir) 29 | 30 | dataset_file_path = os.path.splitext(os.path.join(output_dir, self.f_name))[0] + '.ds' 31 | 32 | best_prices = self.order_books[0].best_prices 33 | mid_price = (best_prices['buy_price'] + best_prices['sell_price']) / 2 34 | 35 | with open(dataset_file_path, 'w') as json_file: 36 | for idx, ob in enumerate(self.order_books[0:-1]): 37 | 38 | next_best_prices = self.order_books[idx + 1].best_prices 39 | next_mid_price = (next_best_prices['buy_price'] + next_best_prices['sell_price']) / 2 40 | 41 | if mid_price != next_mid_price: 42 | direction = 0 if mid_price > next_mid_price else 1 43 | json.dump({'volumes': ob.volumes, 'direction': direction}, json_file) 44 | json_file.write('\n') 45 | 46 | mid_price = next_mid_price 47 | 48 | 49 | def _open_position(self, best_prices, signal): 50 | self.trades.append({}) 51 | 52 | self.trades[-1]['direction'] = signal 53 | self.trades[-1]['open_time'] = best_prices['time']; 54 | 55 | if signal == Signal.BUY: 56 | self.trades[-1]['open_price'] = best_prices['buy_price']; 57 | elif signal == Signal.SELL: 58 | self.trades[-1]['open_price'] = best_prices['sell_price']; 59 | 60 | def _close_position(self, best_prices): 61 | self.trades[-1]['close_time'] = best_prices['time']; 62 | 63 | if self.trades[-1]['direction'] == Signal.BUY: 64 | self.trades[-1]['close_price'] = best_prices['sell_price']; 65 | elif self.trades[-1]['direction'] == Signal.SELL: 66 | self.trades[-1]['close_price'] = best_prices['buy_price']; 67 | 68 | def _reverse_position(self, best_prices, signal): 69 | self._close_position(best_prices) 70 | self._open_position(best_prices, signal) 71 | 72 | def backtest(self, generator, threshold): 73 | self.trades = [] 74 | 75 | for ob in self.order_books[0:-1]: 76 | best_prices = ob.best_prices 77 | signal = generator(ob.volumes, threshold) 78 | 79 | if not self.trades and signal != Signal.WAIT: 80 | self._open_position(best_prices, signal) 81 | elif signal != self.trades[-1]['direction'] and signal != Signal.WAIT: 82 | self._reverse_position(best_prices, signal) 83 | 84 | if not self.trades: 85 | best_prices = self.order_books[-1].best_prices 86 | self._close_position(best_prices) 87 | 88 | return self.trades 89 | 90 | 91 | def backtest_n(self, generator, ffnn, threshold): 92 | self.trades = [] 93 | 94 | for ob in self.order_books[0:-1]: 95 | best_prices = ob.best_prices 96 | signal = generator(ffnn, ob.volumes, threshold) 97 | 98 | if not self.trades and signal != Signal.WAIT: 99 | self._open_position(best_prices, signal) 100 | elif signal != self.trades[-1]['direction'] and signal != Signal.WAIT: 101 | self._reverse_position(best_prices, signal) 102 | 103 | if not self.trades: 104 | best_prices = self.order_books[-1].best_prices 105 | self._close_position(best_prices) 106 | 107 | return self.trades 108 | 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## With this sample framework you`ll be able to: 2 | * get order book (Level II, market depth) from [BitMEX](https://www.bitmex.com) and save as json structures in day-lenght files 3 | * generate trading signals [BUY, SELL, WAIT] on orderbooks (not only BitMEX\`s) with or without help of sequential neural network 4 | * create training dataset for neural network from presaved orderbooks 5 | * create, train, save and restore neural network with input data of orderbooks (not only BitMEX`s) 6 | * perform backtest trading with output of trades 7 | 8 | ## Additional external info 9 | The closely related ideas are also exposed in "One Way to Trading over Orderbook Analisys" [article on Medium](https://medium.com/@sturex/one-way-to-trading-over-orderbook-analisys-689475ae839f). 10 | Read the article for better understanding. 11 | 12 | ### Limitations 13 | * [BitMEX](https://www.bitmex.com) only and [XBTUSD](https://www.bitmex.com/app/contract/XBTUSD) data fetching only are provided. The source connection code taken from [Python Adapter for BitMEX Realtime Data](https://github.com/BitMEX/api-connectors/tree/master/official-ws/python) 14 | * Code works correctly only for 100-depth order books. 15 | * Neural network has voluntaristic rigid architecture where you can operate only number of layers and neurons quantity. Input layer must contain 100 neurons, output layer 2 neurons. 16 | * No commissions, fees etc. are calculated while backtesting. Result trades must be extra analyzed. 17 | 18 | 19 | ## Installation 20 | ``` 21 | pip install -r requirements.txt 22 | ``` 23 | 24 | ## Use cases 25 | #### Retrieving order books ([Bitmex](https://www.bitmex.com) only) 26 | Just run code below with your [API-key credentials to BitMEX](https://www.bitmex.com/app/apiKeysUsage). 27 | On every update of market 100-depth order book is writing to disk. 28 | [Bid-ask spread](https://en.wikipedia.org/wiki/Bid-ask_spread) is in the middle of order book. New trading day starts with new file. 29 | 30 | ```python 31 | from BitmexOrderBookSaver import * 32 | api_key = '' 33 | api_secret = '' 34 | save_folder = '' 35 | bitmex = BitmexOrderBookSaver(api_key, api_secret, save_folder) 36 | print('Retrieving orderbooks market data. Press any key to stop') 37 | input() 38 | bitmex.exit() 39 | ``` 40 | 41 | 42 | #### Dataset creation for neural network training 43 | 44 | ```python 45 | from OrderBookContainer import * 46 | 47 | folder='' 48 | input_files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))] 49 | for in_file in input_files: 50 | obc = OrderBookContainer(os.path.join(folder, in_file)) 51 | obc.create_training_dataset() 52 | ``` 53 | As a result the script will create Datasets subfolder with \*.ds files. 54 | 55 | 56 | #### Neural network creation, training and saving for next time use 57 | 58 | My goal is just to show that neural networks work without price movement analysis but only on current market timestamp (== order book) analysis. 59 | 60 | So, network gets only order book volumes as input and generates floating point value as output. **Really, there are no prices in input data!** 61 | 62 | - Is it possible to predict price movements without price analysis? 63 | - Yes! 64 | 65 | The code below will create three-layered feed-forward sequential network. I use [Keras framework](https://keras.io/). 66 | 67 | I use sigmoid activation function for all layers except for last one where softmax is used. 68 | The first layer consists of 100 neurons, one for each line in order book. 69 | The last layer must contain of 2 neurons because of two variants are possible - BUY and SELL. 70 | 71 | ```python 72 | import TurexNetwork 73 | 74 | nwk = TurexNetwork.TurexNetwork() 75 | nwk.create_model((100, 50, 2)) 76 | datasets_folder='' 77 | nwk.train(datasets_folder) 78 | nwk.save_model('full_path_to_file.h5') 79 | ``` 80 | 81 | #### Trading signal generation 82 | 83 | You can generate trading signal with possible values of [BUY, SELL, WAIT] with order book analysis only. 84 | On every *orderbook* you get from exchange or read from file signal can be generated with code below. 85 | *threshold* is floating point value in range [0, 1]. The less the value the more signals you get. 86 | ###### Neural generator 87 | 88 | ```python 89 | from Generators import sample_generator_n 90 | nwk = TurexNetwork.TurexNetwork() 91 | nwk.load_model('model_from_code_above.h5') 92 | signal = sample_generator_n(nwk, orderbook.volumes, threshold) 93 | ``` 94 | ###### Simple generator 95 | ```python 96 | from Generators import sample_generator 97 | signal = sample_generator(orderbook.volumes, threshold) 98 | ``` 99 | 100 | 101 | #### Backtesting 102 | The mean of *threshold* is described above. 103 | 104 | ```python 105 | import TurexNetwork 106 | import Generators 107 | from OrderBookContainer import * 108 | 109 | obc = OrderBookContainer('path_to_file') 110 | 111 | nwk = TurexNetwork.TurexNetwork() 112 | nwk.load_model('path_to_file.h5') 113 | threshold = 0.0 114 | trades = obc.backtest_n(Generators.sample_neural_generator, nwk, threshold) 115 | #trades = obc.backtest(Generators.sample_generator, threshold) 116 | for trade in trades: 117 | print(trade) 118 | ``` 119 | -------------------------------------------------------------------------------- /Signal.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | class Signal(Enum): 4 | BUY = 1 5 | SELL = -1 6 | WAIT = 0 7 | -------------------------------------------------------------------------------- /TurexNetwork.py: -------------------------------------------------------------------------------- 1 | import os 2 | from keras import models 3 | from keras import layers 4 | from keras.utils import to_categorical 5 | from keras import initializers 6 | from keras import optimizers 7 | import numpy 8 | import json 9 | 10 | 11 | class TurexNetwork: 12 | 13 | def __init__(self): 14 | self.model = None 15 | 16 | def create_model(self, network_structure): 17 | if len(network_structure) <= 2: 18 | raise Exception('The network must contain at least two layers') 19 | 20 | self.input_size = network_structure[0] 21 | self.output_size = network_structure[-1] 22 | 23 | self.model = models.Sequential() 24 | self.model.add(layers.Dense(self.input_size, activation = 'sigmoid')) 25 | for neurons_count in network_structure[1: -1]: 26 | self.model.add(layers.Dense(neurons_count, activation = 'sigmoid')) 27 | self.model.add(layers.Dense(self.output_size, activation = 'softmax')) 28 | self.model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) 29 | 30 | def load_model(self, path_to_file): 31 | self.model = models.load_model(path_to_file) 32 | 33 | layers = self.model.layers 34 | 35 | if len(layers) <= 2: 36 | raise Exception('The network must contain at least two layers') 37 | 38 | self.input_size = len(layers[0].get_weights()[1]) 39 | self.output_size = len(layers[-1].get_weights()[1]) 40 | 41 | def save_model(self, path_to_file): 42 | models.save_model(self.model, path_to_file) 43 | 44 | def train(self, train_folder): 45 | 46 | if self.model == None: 47 | return 48 | 49 | input_files = [f for f in os.listdir(train_folder) if os.path.isfile(os.path.join(train_folder, f))] 50 | 51 | for in_file in input_files: 52 | with open(os.path.join(train_folder, in_file), 'r') as f: 53 | input_data = [] 54 | output_data = [] 55 | for line in f.readlines(): 56 | json_line = json.loads(line) 57 | 58 | input_data.append(json_line['volumes']) 59 | output_data.append(json_line['direction']) 60 | 61 | input_dataset = numpy.asarray(input_data) 62 | output_dataset = to_categorical(numpy.asarray(output_data)) 63 | self.model.fit(input_dataset, output_dataset) 64 | 65 | 66 | def predict(self, _volumes): 67 | if len(_volumes) != self.input_size: 68 | return None 69 | input_dataset = numpy.asarray([_volumes]) 70 | return self.model.predict(input_dataset)[0] 71 | 72 | -------------------------------------------------------------------------------- /my_utils.py: -------------------------------------------------------------------------------- 1 | 2 | #works only for XBTUSD 3 | #for other symbols see https://www.bitmex.com/app/restAPI 4 | def get_price_from_ID(id): 5 | return 1000000 - (id % 100000000) / 100 6 | 7 | 8 | def get_YYYYMMDD(datetime_UTC): 9 | return datetime_UTC.year * 10000 + datetime_UTC.month * 100 + datetime_UTC.day 10 | 11 | def get_HHMMSSmmm(datetime_UTC): 12 | return datetime_UTC.hour * 10000000 + datetime_UTC.minute * 100000 + datetime_UTC.second * 1000 + int(datetime_UTC.microsecond / 1000) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.1 2 | astor==0.8.0 3 | bitmex-ws==0.3.1 4 | certifi==2019.6.16 5 | gast==0.2.2 6 | grpcio==1.16.1 7 | h5py==2.9.0 8 | Keras==2.2.4 9 | Keras-Applications==1.0.8 10 | Keras-Preprocessing==1.1.0 11 | Markdown==3.1.1 12 | mkl-fft==1.0.14 13 | mkl-random==1.0.2 14 | mkl-service==2.0.2 15 | numpy==1.16.4 16 | pip==19.2.2 17 | protobuf==3.8.0 18 | pydot==1.4.1 19 | pyparsing==2.4.2 20 | pyreadline==2.1 21 | PyYAML==5.1.2 22 | scipy==1.3.1 23 | setuptools==41.0.1 24 | six==1.12.0 25 | tensorboard==1.14.0 26 | tensorflow==1.15.2 27 | tensorflow-estimator==1.14.0 28 | termcolor==1.1.0 29 | websocket-client==0.46.0 30 | Werkzeug==0.15.5 31 | wheel==0.33.4 32 | wincertstore==0.2 33 | wrapt==1.11.2 34 | --------------------------------------------------------------------------------