├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST ├── README.md ├── befh ├── __init__.py ├── api_socket.py ├── bitcoinexchangefh.py ├── clients │ ├── csv.py │ ├── database.py │ ├── kafka.py │ ├── kdbplus.py │ ├── mysql.py │ ├── sql.py │ ├── sql_template.py │ ├── sqlite.py │ └── zmq.py ├── exchanges │ ├── aex.py │ ├── bcex.py │ ├── bibox.py │ ├── bigone.py │ ├── binance.py │ ├── bitfinex.py │ ├── bitflyer.py │ ├── bitmex.py │ ├── bitstamp.py │ ├── bittrex.py │ ├── btcc.py │ ├── coincheck.py │ ├── coinone.py │ ├── cryptopia.py │ ├── gatecoin.py │ ├── gateio.py │ ├── gateway.py │ ├── gdax.py │ ├── huobi.py │ ├── kkex.py │ ├── kraken.py │ ├── liqui.py │ ├── luno.py │ ├── okcoin.py │ ├── okex_future.py │ ├── okex_spot.py │ ├── poloniex.py │ ├── quoine.py │ ├── restful_template.py │ ├── wex.py │ ├── ws_template.py │ └── yunbi.py ├── instrument.py ├── market_data.py ├── restful_api_socket.py ├── subscription_manager.py ├── test │ ├── __init__.py │ ├── test_file_client.py │ ├── test_kdbplus_client.py │ ├── test_sqlite_client.py │ ├── test_subscription_manager.py │ ├── test_subscriptions.ini │ ├── test_ws_server.py │ └── test_zmqfeed.py ├── util.py └── ws_api_socket.py ├── doc ├── icon.jpg ├── sample.jpg └── sample2.jpg ├── requirement.txt ├── setup-env.sh ├── setup.py ├── subscription.ini ├── subscriptions.ini └── third-party ├── q └── q.exe /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | __pycache__/ 3 | data/* 4 | dist/* 5 | *.pyc 6 | bitcoinexchangefh.tar.gz 7 | _windows/* 8 | *.xml 9 | inspection/* 10 | .env/* 11 | *.egg-info/* 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.2" 4 | - "3.3" 5 | - "3.4" 6 | - "3.5" 7 | - "3.5-dev" # 3.5 development branch 8 | - "3.6-dev" # 3.6 development branch 9 | - "nightly" # currently points to 3.7-dev 10 | # command to install dependencies 11 | install: "pip install -r python/requirements.txt" 12 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | setup.py 3 | befh/__init__.py 4 | befh/api_socket.py 5 | befh/bitcoinexchangefh.py 6 | befh/database_client.py 7 | befh/exch_binance.py 8 | befh/exch_bitfinex.py 9 | befh/exch_bitmex.py 10 | befh/exch_bitstamp.py 11 | befh/exch_bittrex.py 12 | befh/exch_btcc.py 13 | befh/exch_cryptopia.py 14 | befh/exch_gatecoin.py 15 | befh/exch_gdax.py 16 | befh/exch_kraken.py 17 | befh/exch_liqui.py 18 | befh/exch_luno.py 19 | befh/exch_okcoin.py 20 | befh/exch_poloniex.py 21 | befh/exch_quoine.py 22 | befh/exch_restful_template.py 23 | befh/exch_ws_template.py 24 | befh/exch_yunbi.py 25 | befh/exchange.py 26 | befh/file_client.py 27 | befh/instrument.py 28 | befh/kdbplus_client.py 29 | befh/market_data.py 30 | befh/mysql_client.py 31 | befh/restful_api_socket.py 32 | befh/sql_client.py 33 | befh/sql_client_template.py 34 | befh/sqlite_client.py 35 | befh/subscription_manager.py 36 | befh/util.py 37 | befh/ws_api_socket.py 38 | befh/zmq_client.py 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | # BitcoinExchangeFH - Bitcoin exchange market data feed handler 6 | 7 | BitcoinExchangeFH is a slim application to record the price depth and trades in various exchanges. You can set it up quickly and record the all the exchange data in a few minutes! 8 | 9 | Users can 10 | 11 | 1. Streaming market data to a target application (via ZeroMQ or Kafka) 12 | 2. Recording market data for backtesting and analysis. 13 | 3. Recording market data to a in-memory database and other applications can quickly access to it. 14 | 4. Customize the project for trading use. 15 | 16 | ### MySQL 17 | 18 |

19 | 20 |

21 | 22 | ### Kdb+ 23 | 24 |

25 | 26 |

27 | 28 | ## Supported exchanges 29 | 30 | - Binance (RESTful) 31 | - Bitflyer (RESTful) 32 | - Bitfinex (Websocket) 33 | - BitMEX (Websocket) 34 | - Bitstamp (Websocket) 35 | - Bittrex (RESTful) 36 | - BTCC (RESTful) 37 | - Cryptopia (RESTful) 38 | - Coincheck (RESTful) 39 | - Gatecoin (RESTful) 40 | - GDAX (Websocket) 41 | - HuoBi (Websocket) 42 | - Kraken (RESTful) 43 | - Liqui (RESTful) 44 | - Luno (Websocket) 45 | - Poloniex (RESTful) 46 | - OkCoin (Websocket) 47 | - Okex (Websocket) 48 | - Quoine (RESTful) 49 | - Yunbi (RESTful) 50 | - Wex (Restful) 51 | - Kkex (RESTful) 52 | - Bibox (RESTful) 53 | - Okex (RESTful) 54 | - Aex (RESTful) 55 | 56 | Currently the support of other exchanges is still under development. 57 | 58 | ## Supported database/channel 59 | 60 | - Kafka 61 | - ZeroMQ 62 | - Kdb+ 63 | - MySQL 64 | - Sqlite 65 | - CSV 66 | 67 | ## Getting started 68 | 69 | It is highly recommended to use pip for installing python dependence. 70 | 71 | ``` 72 | pip install bitcoinexchangefh 73 | ``` 74 | 75 | If your operation system has pythons with version 2 and 3, please specify 76 | pip3 for python 3 installation. 77 | 78 | ``` 79 | pip3 install bitcoinexchangefh 80 | ``` 81 | 82 | ### Destination 83 | 84 | #### Applications 85 | 86 | You can write your application to receive the market data via ZeroMQ/Kafka socket. 87 | 88 | BitcoinExchangeFH acts as a publisher in the 89 | [Publish/Subscibe](http://learning-0mq-with-pyzmq.readthedocs.io/en/latest/pyzmq/patterns/pubsub.html) model. 90 | You can open a TCP or inter-process traffic. 91 | 92 | For example, if you decide the data feed is subscribed at 127.0.0.1 at port 6001. 93 | 94 | ``` 95 | bitcoinexchangefh -zmq -zmqdest "tcp://127.0.0.1:6001" -instmts subscription.ini 96 | ``` 97 | 98 | According to [zmq-tcp](http://api.zeromq.org/2-1:zmq-tcp), please provide "127.0.0.1" 99 | instead of "localhost" as the local machine destination. 100 | 101 | If the data feed is subscribed via inter-process shared memory with address "bitcoin". 102 | 103 | ``` 104 | bitcoinexchangefh -zmq -zmqdest "ipc://bitcoin" -instmts subscription.ini 105 | ``` 106 | 107 | #### Sqlite 108 | 109 | No further setup is required. Just define the output sqlite file. 110 | 111 | For example, to record the data to default sqlite file "bitcoinexchange.raw", run the command 112 | 113 | ``` 114 | bitcoinexchangefh -sqlite -sqlitepath bitcoinexchangefh.sqlite -instmts subscription.ini 115 | ``` 116 | 117 | #### Kdb+ 118 | 119 | First, start your Kdb+ database. You can either choose your own binary or the binary in the [third-party](https://github.com/gavincyi/BitcoinExchangeFH/tree/master/third-party) folder. 120 | 121 | ``` 122 | q -p 5000 123 | ``` 124 | 125 | Then connect to the database with dedicated port (for example 5000 in the example). 126 | 127 | For example connecting to localhost at port 5000, run the command 128 | 129 | ``` 130 | bitcoinexchangefh -kdb -kdbdest "localhost:5000" -instmts subscription.ini 131 | ``` 132 | 133 | #### MySQL 134 | 135 | To store the market data to MySQL database, please install [mysql-server](https://dev.mysql.com/downloads/mysql/) first. Then enable the following user privileges on your target schema 136 | 137 | ``` 138 | CREATE 139 | UPDATE 140 | INSERT 141 | SELECT 142 | ``` 143 | 144 | For example connecting to localhost with user "bitcoin", password "bitcoin" and schema "bcex", run the command 145 | 146 | ``` 147 | bitcoinexchangefh -mysql -mysqldest "bitcoin:bitcoin@localhost:3306" -mysqlschema bcex -instmts subscription.ini 148 | ``` 149 | 150 | #### CSV 151 | 152 | No further setup is required. Just define the output folder path. 153 | 154 | For example to a folder named "data", you can run the following command. 155 | 156 | ``` 157 | bitcoinexchangefh -csv -csvpath data/ -instmts subscription.ini 158 | ``` 159 | 160 | #### Kafka 161 | 162 | Please install Kafka firstly. 163 | 164 | ``` 165 | bitcoinexchangefh -kafka -kafkadest "127.0.0.1:9092" -instmts subscription.ini 166 | ``` 167 | or 168 | 169 | ``` 170 | source setup-env.sh 171 | python ./befh/bitcoinexchangefh.py -kafka -kafkadest "127.0.0.1:9092" -instmts subscription.ini -output log.txt 172 | ``` 173 | 174 | 175 | ### Multiple destination 176 | 177 | Bitcoinexchangefh supports multiple destinations. 178 | 179 | For example, if you store the market data into the database and, at the same time, publish the data through ZeroMQ publisher, you can run the command 180 | 181 | ``` 182 | bitcoinexchangefh -zmq -zmqdest "tcp://localhost:6001" -kdb -kdbdest "localhost:5000" -instmts subscription.ini 183 | ``` 184 | 185 | ### Arguments 186 | 187 | |Argument|Description| 188 | |---|---| 189 | |mode|Please refer to [Mode](#mode)| 190 | |instmts|Instrument subscription file.| 191 | |exchtime|Use exchange timestamp if possible.| 192 | |zmq|Streamed with ZeroMQ sockets.| 193 | |zmqdest|ZeroMQ destination. Formatted as "type://address(:port)", e.g. "tcp://127.0.0.1:6001".| 194 | |kdb|Use Kdb+ database.| 195 | |kdbdest|Kdb+ database destination. Formatted as "address:port", e.g. "127.0.0.1:5000".| 196 | |sqlite|Use SQLite database.| 197 | |sqlitepath|SQLite database file path, e.g. "bitcoinexchangefh.sqlite".| 198 | |mysql|Use MySQL.| 199 | |mysqldest|MySQL database destination. Formatted as "username:password@address:host", e.g. "peter:Password123@127.0.0.1:3306".| 200 | |csv|Use CSV file as database.| 201 | |csvpath|CSV file directory, e.g. "data/"| 202 | |output|Verbose output file path.| 203 | 204 | ### Subscription 205 | All the instrument subscription are mentioned in the configuration file [subscriptions.ini](subscriptions.ini). For supported exchanges, you can include its instruments as a block of subscription. 206 | 207 | |Argument|Description| 208 | |---|---| 209 | |(block name)|Unique subscription ID| 210 | |exchange|Exchange name.| 211 | |instmt_name|Instrument name. Used in application, e.g. database table name| 212 | |instmt_code|Exchange instrument code. Used in exchange API| 213 | |enabled|Indicate whether to subscribe it| 214 | 215 | ### Market Data 216 | 217 | All market data are stored in the dedicated database. For each instrument, there are two tables, order book and trades. The order book is the price depth at top five levels. They are recorded under the table names of 218 | 219 | ``` 220 | exch___snapshot 221 | ``` 222 | 223 | ### Output 224 | 225 | Each record (in any output format e.g. CSV/SQLite/KDB+/etc) indicates either a new trade or a change in the order book. 226 | 227 | The column definition is as follows: 228 | 229 | |Name|Description| 230 | |---|---| 231 | |trade_px|Last trade price| 232 | |trade_volume|Last trade volume| 233 | |b\, a\|Best bid and ask prices, where n is between 1 and 5| 234 | |bq\, aq\|Best bid and ask volumes, where n is between 1 and 5| 235 | |update_type|Update type. 1 indicates price depth update, and 2 indicates trade update| 236 | |order_date_time, trade_date_time|Last update time for the price depth and the trades| 237 | 238 | ### Library 239 | 240 | If you do not like the console application and would like to write your own, you can use it as 241 | a library. 242 | 243 | ``` 244 | from befh.exch_bittrex import ExchGwApiBittrex as Feed 245 | from befh.instrument import Instrument 246 | instmt = Instrument(exchange_name="Bittrex", 247 | instmt_name="LTC/BTC", 248 | instmt_code="BTC-LTC") 249 | 250 | # Get the order book depth 251 | depth = Feed.get_order_book(instmt) 252 | 253 | # Get the trades 254 | trades = Feed.get_trades(instmt) 255 | ``` 256 | 257 | where parameter `instmt_code` is the exchange API instrument code. 258 | 259 | ## Inquiries 260 | 261 | You can first look up to the page [FAQ](https://github.com/gavincyi/BitcoinExchangeFH/wiki/FAQ). For more inquiries, you can either leave it in issues or drop me an email. I will get you back as soon as possible. 262 | 263 | ## Compatibility 264 | The application is compatible with version higher or equal to python 3.0. 265 | 266 | ## Contributions 267 | Always welcome for any contribution. Please fork the project, make the changes, and submit the merge request. :) 268 | 269 | For any questions and comment, please feel free to contact me through email (gavincyi at gmail) 270 | 271 | Your comment will be a huge contribution to the project! 272 | 273 | ## Continuity 274 | If you are not satisified with python performance, you can contact me to discuss migrating the project into other languages, e.g. C++. 275 | -------------------------------------------------------------------------------- /befh/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/befh/__init__.py -------------------------------------------------------------------------------- /befh/api_socket.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | class ApiSocket: 4 | """ 5 | API socket 6 | """ 7 | def __init__(self): 8 | pass 9 | 10 | @classmethod 11 | def parse_l2_depth(cls, instmt, raw): 12 | """ 13 | Parse raw data to L2 depth 14 | :param instmt: Instrument 15 | :param raw: Raw data in JSON 16 | """ 17 | return None 18 | 19 | @classmethod 20 | def parse_trade(cls, instmt, raw): 21 | """ 22 | :param instmt: Instrument 23 | :param raw: Raw data in JSON 24 | :return: 25 | """ 26 | return None 27 | 28 | def get_order_book(self, instmt): 29 | """ 30 | Get order book 31 | :param instmt: Instrument 32 | :return: Object L2Depth 33 | """ 34 | return None 35 | 36 | def get_trades(self, instmt, trade_id): 37 | """ 38 | Get trades 39 | :param instmt: Instrument 40 | :param trade_id: Trade id 41 | :return: List of trades 42 | """ 43 | return None 44 | -------------------------------------------------------------------------------- /befh/bitcoinexchangefh.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | import argparse 4 | import sys 5 | 6 | 7 | from befh.exchanges.gateway import ExchangeGateway 8 | from befh.exchanges.bitmex import ExchGwBitmex 9 | from befh.exchanges.btcc import ExchGwBtccSpot, ExchGwBtccFuture 10 | from befh.exchanges.bitfinex import ExchGwBitfinex 11 | from befh.exchanges.okcoin import ExchGwOkCoin 12 | from befh.exchanges.kraken import ExchGwKraken 13 | from befh.exchanges.gdax import ExchGwGdax 14 | from befh.exchanges.bitstamp import ExchGwBitstamp 15 | from befh.exchanges.huobi import ExchGwHuoBi 16 | from befh.exchanges.coincheck import ExchGwCoincheck 17 | from befh.exchanges.gatecoin import ExchGwGatecoin 18 | from befh.exchanges.quoine import ExchGwQuoine 19 | from befh.exchanges.poloniex import ExchGwPoloniex 20 | from befh.exchanges.bittrex import ExchGwBittrex 21 | from befh.exchanges.yunbi import ExchGwYunbi 22 | from befh.exchanges.liqui import ExchGwLiqui 23 | from befh.exchanges.binance import ExchGwBinance 24 | from befh.exchanges.cryptopia import ExchGwCryptopia 25 | from befh.exchanges.okex_spot import ExchGwOkexSpot 26 | from befh.exchanges.okex_future import ExchGwOkexFuture 27 | from befh.exchanges.wex import ExchGwWex 28 | from befh.exchanges.bitflyer import ExchGwBitflyer 29 | from befh.exchanges.coinone import ExchGwCoinOne 30 | from befh.exchanges.kkex import ExchGwKkex 31 | from befh.exchanges.bibox import ExchGwBibox 32 | from befh.exchanges.aex import ExchGwAex 33 | from befh.exchanges.bigone import ExchGwBigone 34 | from befh.exchanges.gateio import ExchGwGateio 35 | 36 | from befh.clients.kdbplus import KdbPlusClient 37 | from befh.clients.mysql import MysqlClient 38 | # from befh.clients.sqlite import SqliteClient 39 | from befh.clients.csv import FileClient 40 | from befh.clients.zmq import ZmqClient 41 | from befh.clients.kafka import KafkaClient 42 | from befh.subscription_manager import SubscriptionManager 43 | from befh.util import Logger 44 | 45 | 46 | def main(): 47 | parser = argparse.ArgumentParser(description='Bitcoin exchange market data feed handler.') 48 | parser.add_argument('-instmts', action='store', help='Instrument subscription file.', default='subscriptions.ini') 49 | parser.add_argument('-exchtime', action='store_true', help='Use exchange timestamp.') 50 | parser.add_argument('-kdb', action='store_true', help='Use Kdb+ as database.') 51 | parser.add_argument('-csv', action='store_true', help='Use csv file as database.') 52 | parser.add_argument('-sqlite', action='store_true', help='Use SQLite database.') 53 | parser.add_argument('-mysql', action='store_true', help='Use MySQL.') 54 | parser.add_argument('-zmq', action='store_true', help='Use zmq publisher.') 55 | parser.add_argument('-kafka', action='store_true', help='Use kafka publisher.') 56 | parser.add_argument('-mysqldest', action='store', dest='mysqldest', 57 | help='MySQL destination. Formatted as ', 58 | default='') 59 | parser.add_argument('-mysqlschema', action='store', dest='mysqlschema', 60 | help='MySQL schema.', 61 | default='') 62 | parser.add_argument('-kdbdest', action='store', dest='kdbdest', 63 | help='Kdb+ destination. Formatted as ', 64 | default='') 65 | parser.add_argument('-zmqdest', action='store', dest='zmqdest', 66 | help='Zmq destination. For example \"tcp://127.0.0.1:3306\"', 67 | default='') 68 | parser.add_argument('-kafkadest', action='store', dest='kafkadest', 69 | help='Kafka destination. For example \"127.0.0.1:9092\"', 70 | default='') 71 | parser.add_argument('-sqlitepath', action='store', dest='sqlitepath', 72 | help='SQLite database path', 73 | default='') 74 | parser.add_argument('-csvpath', action='store', dest='csvpath', 75 | help='Csv file path', 76 | default='') 77 | parser.add_argument('-output', action='store', dest='output', 78 | help='Verbose output file path') 79 | args = parser.parse_args() 80 | 81 | Logger.init_log(args.output) 82 | 83 | db_clients = [] 84 | is_database_defined = False 85 | # if args.sqlite: 86 | # db_client = SqliteClient() 87 | # db_client.connect(path=args.sqlitepath) 88 | # db_clients.append(db_client) 89 | # is_database_defined = True 90 | if args.mysql: 91 | db_client = MysqlClient() 92 | mysqldest = args.mysqldest 93 | logon_credential = mysqldest.split('@')[0] 94 | connection = mysqldest.split('@')[1] 95 | db_client.connect(host=connection.split(':')[0], 96 | port=int(connection.split(':')[1]), 97 | user=logon_credential.split(':')[0], 98 | pwd=logon_credential.split(':')[1], 99 | schema=args.mysqlschema) 100 | db_clients.append(db_client) 101 | is_database_defined = True 102 | if args.csv: 103 | if args.csvpath != '': 104 | db_client = FileClient(dir=args.csvpath) 105 | else: 106 | db_client = FileClient() 107 | db_clients.append(db_client) 108 | is_database_defined = True 109 | if args.kdb: 110 | db_client = KdbPlusClient() 111 | db_client.connect(host=args.kdbdest.split(':')[0], port=int(args.kdbdest.split(':')[1])) 112 | db_clients.append(db_client) 113 | is_database_defined = True 114 | if args.zmq: 115 | db_client = ZmqClient() 116 | db_client.connect(addr=args.zmqdest) 117 | db_clients.append(db_client) 118 | is_database_defined = True 119 | if args.kafka: 120 | db_client = KafkaClient() 121 | db_client.connect(addr=args.kafkadest) 122 | db_clients.append(db_client) 123 | is_database_defined = True 124 | 125 | if not is_database_defined: 126 | print('Error: Please define which database is used.') 127 | parser.print_help() 128 | sys.exit(1) 129 | 130 | # Subscription instruments 131 | if args.instmts is None or len(args.instmts) == 0: 132 | print('Error: Please define the instrument subscription list. You can refer to subscriptions.ini.') 133 | parser.print_help() 134 | sys.exit(1) 135 | 136 | # Use exchange timestamp rather than local timestamp 137 | if args.exchtime: 138 | ExchangeGateway.is_local_timestamp = False 139 | 140 | # Initialize subscriptions 141 | subscription_instmts = SubscriptionManager(args.instmts).get_subscriptions() 142 | if len(subscription_instmts) == 0: 143 | print('Error: No instrument is found in the subscription file. ' + 144 | 'Please check the file path and the content of the subscription file.') 145 | parser.print_help() 146 | sys.exit(1) 147 | 148 | # Initialize snapshot destination 149 | ExchangeGateway.init_snapshot_table(db_clients) 150 | 151 | Logger.info('[main]', 'Subscription file = %s' % args.instmts) 152 | log_str = 'Exchange/Instrument/InstrumentCode:\n' 153 | for instmt in subscription_instmts: 154 | log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name, instmt.instmt_code) 155 | Logger.info('[main]', log_str) 156 | 157 | exch_gws = [] 158 | exch_gws.append(ExchGwBtccSpot(db_clients)) 159 | exch_gws.append(ExchGwBtccFuture(db_clients)) 160 | exch_gws.append(ExchGwBitmex(db_clients)) 161 | exch_gws.append(ExchGwBitfinex(db_clients)) 162 | exch_gws.append(ExchGwOkexSpot(db_clients)) 163 | exch_gws.append(ExchGwOkexFuture(db_clients)) 164 | exch_gws.append(ExchGwKraken(db_clients)) 165 | exch_gws.append(ExchGwGdax(db_clients)) 166 | exch_gws.append(ExchGwBitstamp(db_clients)) 167 | exch_gws.append(ExchGwBitflyer(db_clients)) 168 | exch_gws.append(ExchGwHuoBi(db_clients)) 169 | exch_gws.append(ExchGwCoincheck(db_clients)) 170 | exch_gws.append(ExchGwCoinOne(db_clients)) 171 | exch_gws.append(ExchGwGatecoin(db_clients)) 172 | exch_gws.append(ExchGwQuoine(db_clients)) 173 | exch_gws.append(ExchGwPoloniex(db_clients)) 174 | exch_gws.append(ExchGwBittrex(db_clients)) 175 | exch_gws.append(ExchGwLiqui(db_clients)) 176 | exch_gws.append(ExchGwBinance(db_clients)) 177 | exch_gws.append(ExchGwCryptopia(db_clients)) 178 | exch_gws.append(ExchGwWex(db_clients)) 179 | exch_gws.append(ExchGwKkex(db_clients)) 180 | exch_gws.append(ExchGwBibox(db_clients)) 181 | exch_gws.append(ExchGwAex(db_clients)) 182 | exch_gws.append(ExchGwBigone(db_clients)) 183 | exch_gws.append(ExchGwGateio(db_clients)) 184 | 185 | threads = [] 186 | for exch in exch_gws: 187 | for instmt in subscription_instmts: 188 | if instmt.get_exchange_name() == exch.get_exchange_name(): 189 | Logger.info("[main]", "Starting instrument %s-%s..." % \ 190 | (instmt.get_exchange_name(), instmt.get_instmt_name())) 191 | threads += exch.start(instmt) 192 | 193 | if __name__ == '__main__': 194 | main() 195 | -------------------------------------------------------------------------------- /befh/clients/csv.py: -------------------------------------------------------------------------------- 1 | from befh.clients.database import DatabaseClient 2 | from befh.util import Logger 3 | import threading 4 | import os 5 | import csv 6 | 7 | 8 | class FileClient(DatabaseClient): 9 | """ 10 | File client 11 | """ 12 | 13 | class Operator: 14 | UNKNOWN = 0 15 | EQUAL = 1 16 | NOT_EQUAL = 2 17 | GREATER = 3 18 | GREATER_OR_EQUAL = 4 19 | SMALLER = 5 20 | SMALLER_OR_EQUAL = 6 21 | 22 | def __init__(self, dir=os.getcwd()): 23 | """ 24 | Constructor 25 | """ 26 | DatabaseClient.__init__(self) 27 | self.lock = threading.Lock() 28 | self.file_mapping = dict() 29 | 30 | if dir is None or dir == '': 31 | raise Exception("FileClient does not accept empty directory.") 32 | 33 | self.file_directory = dir 34 | 35 | @staticmethod 36 | def convert_to(from_str, to_type): 37 | """ 38 | Convert the element to the given type 39 | """ 40 | if to_type is int: 41 | return int(from_str) 42 | elif to_type is float: 43 | return float(from_str) 44 | else: 45 | return from_str 46 | 47 | 48 | def create(self, table, columns, types, primary_key_index=(), is_ifnotexists=True): 49 | """ 50 | Create table in the database 51 | :param table: Table name 52 | :param columns: Column array 53 | :param types: Type array 54 | :param is_ifnotexists: Create table if not exists keyword 55 | """ 56 | file_path = os.path.join(self.file_directory, table + ".csv") 57 | columns = [e.split(' ')[0] for e in columns] 58 | if len(columns) != len(types): 59 | return False 60 | 61 | self.lock.acquire() 62 | if os.path.isfile(file_path): 63 | Logger.info(self.__class__.__name__, "File (%s) has been created already." % file_path) 64 | else: 65 | with open(file_path, 'w+') as csvfile: 66 | csvfile.write(','.join(["\"" + e + "\"" for e in columns])+'\n') 67 | 68 | self.lock.release() 69 | return True 70 | 71 | def insert(self, table, columns, types, values, primary_key_index=(), is_orreplace=False, is_commit=True): 72 | """ 73 | Insert into the table 74 | :param table: Table name 75 | :param columns: Column array 76 | :param types: Type array 77 | :param values: Value array 78 | :param primary_key_index: An array of indices of primary keys in columns, 79 | e.g. [0] means the first column is the primary key 80 | :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" 81 | """ 82 | ret = True 83 | file_path = os.path.join(self.file_directory, table + ".csv") 84 | if len(columns) != len(values): 85 | return False 86 | 87 | self.lock.acquire() 88 | if not os.path.isfile(file_path): 89 | ret = False 90 | else: 91 | with open(file_path, "a+") as csvfile: 92 | writer = csv.writer(csvfile, lineterminator='\n', quotechar='\"', quoting=csv.QUOTE_NONNUMERIC) 93 | writer.writerow(values) 94 | self.lock.release() 95 | 96 | if not ret: 97 | raise Exception("File (%s) has not been created.") 98 | 99 | return True 100 | 101 | def select(self, table, columns=['*'], condition='', orderby='', limit=0, isFetchAll=True): 102 | """ 103 | Select rows from the table. 104 | Currently the method only processes the one column ordering and condition 105 | :param table: Table name 106 | :param columns: Selected columns 107 | :param condition: Where condition 108 | :param orderby: Order by condition 109 | :param limit: Rows limit 110 | :param isFetchAll: Indicator of fetching all 111 | :return Result rows 112 | """ 113 | file_path = self.file_directory + table + ".csv" 114 | is_all_columns = len(columns) == 1 and columns[0] == '*' 115 | csv_field_names = [] 116 | columns = [e.split(' ')[0] for e in columns] 117 | ret = [] 118 | is_error = False 119 | 120 | # Preparing condition 121 | if condition != '': 122 | condition = condition.replace('=', '==') 123 | condition = condition.replace('!==', '!=') 124 | condition = condition.replace('>==', '>=') 125 | condition = condition.replace('<==', '<=') 126 | 127 | self.lock.acquire() 128 | if not os.path.isfile(file_path): 129 | is_error = True 130 | else: 131 | with open(file_path, "r") as csvfile: 132 | reader = csv.reader(csvfile, lineterminator='\n', quotechar='\"', quoting=csv.QUOTE_NONNUMERIC) 133 | csv_field_names = next(reader, None) 134 | for col in columns: 135 | if not is_all_columns and col not in csv_field_names: 136 | raise Exception("Field (%s) is not in the table." % col) 137 | 138 | for csv_row in reader: 139 | # Filter by condition statement 140 | is_selected = True 141 | if condition != '': 142 | condition_eval = condition 143 | for i in range(0, len(csv_field_names)): 144 | key = csv_field_names[i] 145 | value = csv_row[i] 146 | if condition_eval.find(key) > -1: 147 | condition_eval = condition_eval.replace(key, str(value)) 148 | is_selected = eval(condition_eval) 149 | 150 | if is_selected: 151 | ret.append(list(csv_row)) 152 | 153 | self.lock.release() 154 | 155 | if is_error: 156 | raise Exception("File (%s) has not been created.") 157 | 158 | if orderby != '': 159 | # Sort the result 160 | field = orderby.split(' ')[0].strip() 161 | asc_val = orderby.split(' ')[1].strip() if len(orderby.split(' ')) > 1 else 'asc' 162 | if asc_val != 'asc' and asc_val != 'desc': 163 | raise Exception("Incorrect orderby in select statement (%s)." % orderby) 164 | elif field not in csv_field_names: 165 | raise Exception("Field (%s) is not in the table." % col) 166 | 167 | field_index = csv_field_names.index(field) 168 | ret = sorted(ret, key=lambda x:x[field_index], reverse=(asc_val == 'desc')) 169 | 170 | if limit > 0: 171 | # Trim the result by the limit 172 | ret = ret[:limit] 173 | 174 | if not is_all_columns: 175 | field_index = [csv_field_names.index(x) for x in columns] 176 | ret = [[row[i] for i in field_index] for row in ret] 177 | 178 | return ret 179 | 180 | def delete(self, table, condition='1==1'): 181 | """ 182 | Delete rows from the table 183 | :param table: Table name 184 | :param condition: Where condition 185 | """ 186 | raise Exception("Deletion is not supported in file client.") 187 | 188 | -------------------------------------------------------------------------------- /befh/clients/database.py: -------------------------------------------------------------------------------- 1 | class DatabaseClient: 2 | """ 3 | Base database client 4 | """ 5 | def __init__(self): 6 | """ 7 | Constructor 8 | """ 9 | pass 10 | 11 | @classmethod 12 | def convert_str(cls, val): 13 | """ 14 | Convert the value to string 15 | :param val: Can be string, int or float 16 | :return: 17 | """ 18 | if isinstance(val, str): 19 | return "'" + val + "'" 20 | elif isinstance(val, bytes): 21 | return "'" + str(val) + "'" 22 | elif isinstance(val, int): 23 | return str(val) 24 | elif isinstance(val, float): 25 | return "%.8f" % val 26 | else: 27 | raise Exception("Cannot convert value (%s)<%s> to string. Value is not a string, an integer nor a float" %\ 28 | (val, type(val))) 29 | 30 | def connect(self, **args): 31 | """ 32 | Connect 33 | :return True if it is connected 34 | """ 35 | return True 36 | 37 | def create(self, table, columns, types, primary_key_index=(), is_ifnotexists=True): 38 | """ 39 | Create table in the database 40 | :param table: Table name 41 | :param columns: Column array 42 | :param types: Type array 43 | :param primary_key_index: An array of indices of primary keys in columns, 44 | e.g. [0] means the first column is the primary key 45 | :param is_ifnotexists: Create table if not exists keyword 46 | """ 47 | return True 48 | 49 | def insert(self, table, columns, types, values, primary_key_index=(), is_orreplace=False, is_commit=True): 50 | """ 51 | Insert into the table 52 | :param table: Table name 53 | :param columns: Column array 54 | :param types: Type array 55 | :param values: Value array 56 | :param primary_key_index: An array of indices of primary keys in columns, 57 | e.g. [0] means the first column is the primary key 58 | :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" 59 | :param is_commit: Indicate if the query is committed (in sql command database mostly) 60 | """ 61 | return True 62 | 63 | def select(self, table, columns=['*'], condition='', orderby='', limit=0, isFetchAll=True): 64 | """ 65 | Select rows from the table 66 | :param table: Table name 67 | :param columns: Selected columns 68 | :param condition: Where condition 69 | :param orderby: Order by condition 70 | :param limit: Rows limit 71 | :param isFetchAll: Indicator of fetching all 72 | :return Result rows 73 | """ 74 | 75 | def close(self): 76 | """ 77 | Close connection 78 | :return: 79 | """ 80 | return True 81 | -------------------------------------------------------------------------------- /befh/clients/kafka.py: -------------------------------------------------------------------------------- 1 | from befh.clients.sql import SqlClient 2 | from befh.util import Logger 3 | import threading 4 | import re 5 | import time 6 | import traceback 7 | import json 8 | from kafka import KafkaProducer 9 | from kafka import KafkaConsumer 10 | from kafka.errors import KafkaError 11 | 12 | class KafkaClient(SqlClient): 13 | """ 14 | Kafka Client 15 | """ 16 | 17 | def __init__(self): 18 | """ 19 | Constructor 20 | """ 21 | SqlClient.__init__(self) 22 | self.conn = None 23 | self.lock = threading.Lock() 24 | 25 | def connect(self, **kwargs): 26 | """ 27 | Connect 28 | :param path: sqlite file to connect 29 | """ 30 | addr = kwargs['addr'] 31 | Logger.info(self.__class__.__name__, 32 | 'Kafka client is connecting to %s' % addr) 33 | 34 | self.conn = KafkaProducer(bootstrap_servers=addr, 35 | # key_serializer=str.encode, 36 | value_serializer=lambda v: json.dumps(v).encode('utf-8')) 37 | 38 | 39 | return self.conn is not None 40 | 41 | def execute(self, sql): 42 | """ 43 | Execute the sql command 44 | :param sql: SQL command 45 | """ 46 | return True 47 | 48 | def commit(self): 49 | """ 50 | Commit 51 | """ 52 | return True 53 | 54 | def fetchone(self): 55 | """ 56 | Fetch one record 57 | :return Record 58 | """ 59 | return [] 60 | 61 | def fetchall(self): 62 | """ 63 | Fetch all records 64 | :return Record 65 | """ 66 | return [] 67 | 68 | def create(self, table, columns, types, primary_key_index=[], is_ifnotexists=True): 69 | """ 70 | Create table in the database. 71 | Caveat - Assign the first few column as the keys!!! 72 | :param table: Table name 73 | :param columns: Column array 74 | :param types: Type array 75 | :param is_ifnotexists: Create table if not exists keyword 76 | """ 77 | return True 78 | 79 | def insert(self, table, columns, types, values, primary_key_index=[], is_orreplace=False, is_commit=True): 80 | """ 81 | Insert into the table 82 | :param table: Table name 83 | :param columns: Column array 84 | :param types: Type array 85 | :param values: Value array 86 | :param primary_key_index: An array of indices of primary keys in columns, 87 | e.g. [0] means the first column is the primary key 88 | :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" 89 | """ 90 | 91 | ret = dict(zip(columns, values)) 92 | ret['table'] = table 93 | self.lock.acquire() 94 | 95 | # print(ret) 96 | # print('columns:', columns) 97 | # print('values:', values) 98 | 99 | future = self.conn.send(table, value=ret) 100 | 101 | result = True 102 | # Block for 'synchronous' sends 103 | try: 104 | record_metadata = future.get(timeout=60) 105 | # print(record_metadata) 106 | Logger.info(self.__class__.__name__, "topic: %s, offset: %s" % (record_metadata.topic, record_metadata.offset)) 107 | except Exception as ex: 108 | Logger.error(self.__class__.__name__, "exception in producer:%s" % ex) 109 | # traceback.print_exc() 110 | result = False 111 | # raise Exception("kafka send failed.") 112 | finally: 113 | self.lock.release() 114 | 115 | return result 116 | 117 | def select(self, table, columns=['*'], condition='', orderby='', limit=0, isFetchAll=True): 118 | """ 119 | Select rows from the table 120 | :param table: Table name 121 | :param columns: Selected columns 122 | :param condition: Where condition 123 | :param orderby: Order by condition 124 | :param limit: Rows limit 125 | :param isFetchAll: Indicator of fetching all 126 | :return Result rows 127 | """ 128 | return [] 129 | 130 | def delete(self, table, condition='1==1'): 131 | """ 132 | Delete rows from the table 133 | :param table: Table name 134 | :param condition: Where condition 135 | """ 136 | return True 137 | 138 | 139 | if __name__ == '__main__': 140 | Logger.init_log() 141 | db_client = KafkaClient() 142 | db_client.connect(addr='localhost:9092') 143 | for i in range(1, 100): 144 | db_client.insert('df-depth-replicated', ['c1', 'c2', 'c3', 'c4'], [], [ 145 | 'abc', i, 1.1, 5]) 146 | time.sleep(1) 147 | -------------------------------------------------------------------------------- /befh/clients/mysql.py: -------------------------------------------------------------------------------- 1 | from befh.clients.sql import SqlClient 2 | import pymysql 3 | 4 | 5 | class MysqlClient(SqlClient): 6 | """ 7 | Sqlite client 8 | """ 9 | def __init__(self): 10 | """ 11 | Constructor 12 | """ 13 | SqlClient.__init__(self) 14 | 15 | def connect(self, **kwargs): 16 | """ 17 | Connect 18 | :param path: sqlite file to connect 19 | """ 20 | host = kwargs['host'] 21 | port = kwargs['port'] 22 | user = kwargs['user'] 23 | pwd = kwargs['pwd'] 24 | schema = kwargs['schema'] 25 | self.conn = pymysql.connect(host=host, 26 | port=port, 27 | user=user, 28 | password=pwd, 29 | db=schema, 30 | charset='utf8mb4', 31 | cursorclass=pymysql.cursors.DictCursor) 32 | self.cursor = self.conn.cursor() 33 | return self.conn is not None and self.cursor is not None 34 | 35 | def execute(self, sql): 36 | """ 37 | Execute the sql command 38 | :param sql: SQL command 39 | """ 40 | return self.cursor.execute(sql) 41 | 42 | def commit(self): 43 | """ 44 | Commit 45 | """ 46 | self.conn.commit() 47 | 48 | def fetchone(self): 49 | """ 50 | Fetch one record 51 | :return Record 52 | """ 53 | return self.cursor.fetchone() 54 | 55 | def fetchall(self): 56 | """ 57 | Fetch all records 58 | :return Record 59 | """ 60 | return self.cursor.fetchall() 61 | 62 | def select(self, table, columns=['*'], condition='', orderby='', limit=0, isFetchAll=True): 63 | """ 64 | Select rows from the table 65 | :param table: Table name 66 | :param columns: Selected columns 67 | :param condition: Where condition 68 | :param orderby: Order by condition 69 | :param limit: Rows limit 70 | :param isFetchAll: Indicator of fetching all 71 | :return Result rows 72 | """ 73 | select = SqlClient.select(self, table, columns, condition, orderby, limit, isFetchAll) 74 | if len(select) > 0: 75 | if columns[0] != '*': 76 | ret = [] 77 | for ele in select: 78 | row = [] 79 | for column in columns: 80 | row.append(ele[column]) 81 | 82 | ret.append(row) 83 | else: 84 | ret = [list(e.values()) for e in select] 85 | 86 | return ret 87 | else: 88 | return select 89 | -------------------------------------------------------------------------------- /befh/clients/sql.py: -------------------------------------------------------------------------------- 1 | from befh.clients.database import DatabaseClient 2 | from befh.util import Logger 3 | import threading 4 | 5 | 6 | class SqlClient(DatabaseClient): 7 | """ 8 | Sql client 9 | """ 10 | @classmethod 11 | def replace_keyword(cls): 12 | return 'replace into' 13 | 14 | def __init__(self): 15 | """ 16 | Constructor 17 | """ 18 | DatabaseClient.__init__(self) 19 | self.conn = None 20 | self.cursor = None 21 | self.lock = threading.Lock() 22 | 23 | def execute(self, sql): 24 | """ 25 | Execute the sql command 26 | :param sql: SQL command 27 | """ 28 | return True 29 | 30 | def commit(self): 31 | """ 32 | Commit 33 | """ 34 | return True 35 | 36 | def fetchone(self): 37 | """ 38 | Fetch one record 39 | :return Record 40 | """ 41 | return [] 42 | 43 | def fetchall(self): 44 | """ 45 | Fetch all records 46 | :return Record 47 | """ 48 | return [] 49 | 50 | def create(self, table, columns, types, primary_key_index=(), is_ifnotexists=True): 51 | """ 52 | Create table in the database 53 | :param table: Table name 54 | :param columns: Column array 55 | :param types: Type array 56 | :param is_ifnotexists: Create table if not exists keyword 57 | """ 58 | if len(columns) != len(types): 59 | raise Exception("Incorrect create statement. Number of columns and that of types are different.\n%s\n%s" % \ 60 | (columns, types)) 61 | 62 | column_names = '' 63 | for i in range(0, len(columns)): 64 | column_names += '%s %s,' % (columns[i], types[i]) 65 | 66 | if len(primary_key_index) > 0: 67 | column_names += 'PRIMARY KEY (%s)' % (",".join([columns[e] for e in primary_key_index])) 68 | else: 69 | column_names = column_names[0:len(column_names)-1] 70 | 71 | if is_ifnotexists: 72 | sql = "create table if not exists %s (%s)" % (table, column_names) 73 | else: 74 | sql = "create table %s (%s)" % (table, column_names) 75 | 76 | self.lock.acquire() 77 | 78 | try: 79 | self.execute(sql) 80 | except Exception as e: 81 | raise Exception("Error in create statement (%s).\nError: %s\n" % (sql, e)) 82 | 83 | self.commit() 84 | self.lock.release() 85 | return True 86 | 87 | def insert(self, table, columns, types, values, primary_key_index=(), is_orreplace=False, is_commit=True): 88 | """ 89 | Insert into the table 90 | :param table: Table name 91 | :param columns: Column array 92 | :param types: Type array 93 | :param values: Value array 94 | :param primary_key_index: An array of indices of primary keys in columns, 95 | e.g. [0] means the first column is the primary key 96 | :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" 97 | """ 98 | if len(columns) != len(values): 99 | return False 100 | 101 | column_names = ','.join(columns) 102 | value_string = ','.join([SqlClient.convert_str(e) for e in values]) 103 | if is_orreplace: 104 | sql = "%s %s (%s) values (%s)" % (self.replace_keyword(), table, column_names, value_string) 105 | else: 106 | sql = "insert into %s (%s) values (%s)" % (table, column_names, value_string) 107 | 108 | self.lock.acquire() 109 | try: 110 | self.execute(sql) 111 | if is_commit: 112 | self.commit() 113 | except Exception as e: 114 | Logger.info(self.__class__.__name__, "SQL error: %s\nSQL: %s" % (e, sql)) 115 | self.lock.release() 116 | return True 117 | 118 | def select(self, table, columns=['*'], condition='', orderby='', limit=0, isFetchAll=True): 119 | """ 120 | Select rows from the table 121 | :param table: Table name 122 | :param columns: Selected columns 123 | :param condition: Where condition 124 | :param orderby: Order by condition 125 | :param limit: Rows limit 126 | :param isFetchAll: Indicator of fetching all 127 | :return Result rows 128 | """ 129 | sql = "select %s from %s" % (','.join(columns), table) 130 | if len(condition) > 0: 131 | sql += " where %s" % condition 132 | 133 | if len(orderby) > 0: 134 | sql += " order by %s" % orderby 135 | 136 | if limit > 0: 137 | sql += " limit %d" % limit 138 | 139 | self.lock.acquire() 140 | self.execute(sql) 141 | if isFetchAll: 142 | ret = self.fetchall() 143 | self.lock.release() 144 | return ret 145 | else: 146 | ret = self.fetchone() 147 | self.lock.release() 148 | return ret 149 | 150 | def delete(self, table, condition='1==1'): 151 | """ 152 | Delete rows from the table 153 | :param table: Table name 154 | :param condition: Where condition 155 | """ 156 | sql = "delete from %s" % table 157 | if len(condition) > 0: 158 | sql += " where %s" % condition 159 | 160 | self.lock.acquire() 161 | self.execute(sql) 162 | self.commit() 163 | self.lock.release() 164 | return True 165 | -------------------------------------------------------------------------------- /befh/clients/sql_template.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | from befh.clients.sql import SqlClient 4 | from befh.util import Logger 5 | 6 | 7 | class SqlClientTemplate(SqlClient): 8 | """ 9 | Sql client template 10 | """ 11 | def __init__(self): 12 | """ 13 | Constructor 14 | """ 15 | SqlClient.__init__(self) 16 | 17 | def connect(self, **kwargs): 18 | """ 19 | Connect 20 | """ 21 | return True 22 | 23 | def execute(self, sql): 24 | """ 25 | Execute the sql command 26 | :param sql: SQL command 27 | """ 28 | Logger.info(self.__class__.__name__, "Execute command = %s" % sql) 29 | 30 | def commit(self): 31 | """ 32 | Commit 33 | """ 34 | pass 35 | 36 | def fetchone(self): 37 | """ 38 | Fetch one record 39 | :return Record 40 | """ 41 | return [] 42 | 43 | def fetchall(self): 44 | """ 45 | Fetch all records 46 | :return Record 47 | """ 48 | return [] 49 | 50 | -------------------------------------------------------------------------------- /befh/clients/sqlite.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | from befh.clients.sql import SqlClient 4 | import sqlite3 5 | 6 | 7 | class SqliteClient(SqlClient): 8 | """ 9 | Sqlite client 10 | """ 11 | @classmethod 12 | def replace_keyword(cls): 13 | return 'insert or replace into' 14 | 15 | def __init__(self): 16 | """ 17 | Constructor 18 | """ 19 | SqlClient.__init__(self) 20 | 21 | def connect(self, **kwargs): 22 | """ 23 | Connect 24 | :param path: sqlite file to connect 25 | """ 26 | path = kwargs['path'] 27 | self.conn = sqlite3.connect(path, check_same_thread=False) 28 | self.cursor = self.conn.cursor() 29 | return self.conn is not None and self.cursor is not None 30 | 31 | def execute(self, sql): 32 | """ 33 | Execute the sql command 34 | :param sql: SQL command 35 | """ 36 | return self.cursor.execute(sql) 37 | 38 | def commit(self): 39 | """ 40 | Commit 41 | """ 42 | self.conn.commit() 43 | 44 | def fetchone(self): 45 | """ 46 | Fetch one record 47 | :return Record 48 | """ 49 | return self.cursor.fetchone() 50 | 51 | def fetchall(self): 52 | """ 53 | Fetch all records 54 | :return Record 55 | """ 56 | return self.cursor.fetchall() 57 | 58 | -------------------------------------------------------------------------------- /befh/clients/zmq.py: -------------------------------------------------------------------------------- 1 | from befh.clients.database import DatabaseClient 2 | from befh.util import Logger 3 | import threading 4 | import re 5 | import zmq 6 | import time 7 | 8 | 9 | class ZmqClient(DatabaseClient): 10 | """ 11 | Zmq Client 12 | """ 13 | def __init__(self): 14 | """ 15 | Constructor 16 | """ 17 | DatabaseClient.__init__(self) 18 | self.context = zmq.Context() 19 | self.conn = self.context.socket(zmq.PUB) 20 | self.lock = threading.Lock() 21 | 22 | def connect(self, **kwargs): 23 | """ 24 | Connect 25 | :param path: sqlite file to connect 26 | """ 27 | addr = kwargs['addr'] 28 | Logger.info(self.__class__.__name__, 'Zmq client is connecting to %s' % addr) 29 | self.conn.bind(addr) 30 | return self.conn is not None 31 | 32 | 33 | def execute(self, sql): 34 | """ 35 | Execute the sql command 36 | :param sql: SQL command 37 | """ 38 | return True 39 | 40 | def commit(self): 41 | """ 42 | Commit 43 | """ 44 | return True 45 | 46 | def fetchone(self): 47 | """ 48 | Fetch one record 49 | :return Record 50 | """ 51 | return [] 52 | 53 | def fetchall(self): 54 | """ 55 | Fetch all records 56 | :return Record 57 | """ 58 | return [] 59 | 60 | def create(self, table, columns, types, primary_key_index=(), is_ifnotexists=True): 61 | """ 62 | Create table in the database. 63 | Caveat - Assign the first few column as the keys!!! 64 | :param table: Table name 65 | :param columns: Column array 66 | :param types: Type array 67 | :param is_ifnotexists: Create table if not exists keyword 68 | """ 69 | return True 70 | 71 | def insert(self, table, columns, types, values, primary_key_index=(), is_orreplace=False, is_commit=True): 72 | """ 73 | Insert into the table 74 | :param table: Table name 75 | :param columns: Column array 76 | :param types: Type array 77 | :param values: Value array 78 | :param primary_key_index: An array of indices of primary keys in columns, 79 | e.g. [0] means the first column is the primary key 80 | :param is_orreplace: Indicate if the query is "INSERT OR REPLACE" 81 | """ 82 | ret = dict(zip(columns, values)) 83 | ret['table'] = table 84 | self.lock.acquire() 85 | self.conn.send_json(ret) 86 | self.lock.release() 87 | return True 88 | 89 | def select(self, table, columns=['*'], condition='', orderby='', limit=0, isFetchAll=True): 90 | """ 91 | Select rows from the table 92 | :param table: Table name 93 | :param columns: Selected columns 94 | :param condition: Where condition 95 | :param orderby: Order by condition 96 | :param limit: Rows limit 97 | :param isFetchAll: Indicator of fetching all 98 | :return Result rows 99 | """ 100 | return [] 101 | 102 | def delete(self, table, condition='1==1'): 103 | """ 104 | Delete rows from the table 105 | :param table: Table name 106 | :param condition: Where condition 107 | """ 108 | return True 109 | 110 | if __name__ == '__main__': 111 | Logger.init_log() 112 | db_client = ZmqClient() 113 | db_client.connect(addr='ipc://test') 114 | for i in range(1, 100): 115 | db_client.insert('test', ['c1', 'c2', 'c3', 'c4'], [], ['abc', i, 1.1, 5]) 116 | time.sleep(1) 117 | 118 | -------------------------------------------------------------------------------- /befh/exchanges/bigone.py: -------------------------------------------------------------------------------- 1 | from befh.restful_api_socket import RESTfulApiSocket 2 | from befh.exchanges.gateway import ExchangeGateway 3 | from befh.market_data import L2Depth, Trade 4 | from befh.util import Logger 5 | from befh.instrument import Instrument 6 | from befh.clients.sql_template import SqlClientTemplate 7 | from functools import partial 8 | from datetime import datetime 9 | import threading 10 | import time 11 | 12 | 13 | class ExchGwApiBigone(RESTfulApiSocket): 14 | """ 15 | Exchange gateway RESTfulApi 16 | """ 17 | def __init__(self): 18 | RESTfulApiSocket.__init__(self) 19 | 20 | @classmethod 21 | def get_timestamp_offset(cls): 22 | return 1 23 | 24 | @classmethod 25 | def get_order_book_timestamp_field_name(cls): 26 | return None 27 | 28 | @classmethod 29 | def get_trades_timestamp_field_name(cls): 30 | return 'created_at' 31 | 32 | @classmethod 33 | def get_bids_field_name(cls): 34 | return 'bids' 35 | 36 | @classmethod 37 | def get_asks_field_name(cls): 38 | return 'asks' 39 | 40 | @classmethod 41 | def get_trade_side_field_name(cls): 42 | return 'trade_side' 43 | 44 | @classmethod 45 | def get_trade_id_field_name(cls): 46 | return 'trade_id' 47 | 48 | @classmethod 49 | def get_trade_price_field_name(cls): 50 | return 'price' 51 | 52 | @classmethod 53 | def get_trade_volume_field_name(cls): 54 | return 'amount' 55 | 56 | @classmethod 57 | def get_order_book_link(cls, instmt): 58 | return "https://api.big.one/markets/%s/book" % instmt.get_instmt_code() 59 | 60 | @classmethod 61 | def get_trades_link(cls, instmt): 62 | return "https://api.big.one/markets/%s/trades" % instmt.get_instmt_code() 63 | 64 | @classmethod 65 | def parse_l2_depth(cls, instmt, raw): 66 | """ 67 | Parse raw data to L2 depth 68 | :param instmt: Instrument 69 | :param raw: Raw data in JSON 70 | """ 71 | l2_depth = L2Depth() 72 | keys = list(raw.keys()) 73 | if cls.get_bids_field_name() in keys and \ 74 | cls.get_asks_field_name() in keys: 75 | 76 | # No Date time information, has update id only 77 | l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 78 | 79 | # Bids 80 | bids = raw[cls.get_bids_field_name()] 81 | bids = sorted(bids, key=lambda x: x['price'], reverse=True) 82 | max_bid_len = min(len(bids), 5) 83 | for i in range(0, max_bid_len): 84 | l2_depth.bids[i].price = float(bids[i]['price']) if type(bids[i]['price']) != float else bids[i]['price'] 85 | l2_depth.bids[i].volume = float(bids[i]['amount']) if type(bids[i]['amount']) != float else bids[i]['amount'] 86 | 87 | # Asks 88 | asks = raw[cls.get_asks_field_name()] 89 | asks = sorted(asks, key=lambda x: x['price']) 90 | max_ask_len = min(len(asks), 5) 91 | for i in range(0, max_ask_len): 92 | l2_depth.asks[i].price = float(asks[i]['price']) if type(asks[i]['price']) != float else asks[i]['price'] 93 | l2_depth.asks[i].volume = float(asks[i]['amount']) if type(asks[i]['amount']) != float else asks[i]['amount'] 94 | else: 95 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 96 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 97 | raw)) 98 | 99 | return l2_depth 100 | 101 | @classmethod 102 | def parse_trade(cls, instmt, raw): 103 | """ 104 | :param instmt: Instrument 105 | :param raw: Raw data in JSON 106 | :return: 107 | """ 108 | trade = Trade() 109 | keys = list(raw.keys()) 110 | 111 | # print(raw) 112 | if cls.get_trade_id_field_name() in keys and \ 113 | cls.get_trade_price_field_name() in keys and \ 114 | cls.get_trade_volume_field_name() in keys: 115 | 116 | # Date time 117 | trade.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 118 | 119 | # Trade side 120 | trade.trade_side = Trade.parse_side(str(raw[cls.get_trade_side_field_name()])) 121 | 122 | # Trade id 123 | trade.trade_id = str(int(time.time()*1000)) 124 | 125 | # Trade price 126 | trade.trade_price = float(str(raw[cls.get_trade_price_field_name()])) 127 | 128 | # Trade volume 129 | trade.trade_volume = float(str(raw[cls.get_trade_volume_field_name()])) 130 | else: 131 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 132 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 133 | raw)) 134 | 135 | return trade 136 | 137 | @classmethod 138 | def get_order_book(cls, instmt): 139 | """ 140 | Get order book 141 | :param instmt: Instrument 142 | :return: Object L2Depth 143 | """ 144 | res = cls.request(cls.get_order_book_link(instmt)) 145 | if res['data']: 146 | return cls.parse_l2_depth(instmt=instmt, 147 | raw=res['data']) 148 | else: 149 | return None 150 | 151 | @classmethod 152 | def get_trades(cls, instmt): 153 | """ 154 | Get trades 155 | :param instmt: Instrument 156 | :param trade_id: Trade id 157 | :return: List of trades 158 | """ 159 | link = cls.get_trades_link(instmt) 160 | res = cls.request(link) 161 | 162 | trades = [] 163 | if len(res['data']) > 0: 164 | for t in res['data']: 165 | trade = cls.parse_trade(instmt=instmt, 166 | raw=t) 167 | trades.append(trade) 168 | 169 | return trades 170 | 171 | 172 | class ExchGwBigone(ExchangeGateway): 173 | """ 174 | Exchange gateway 175 | """ 176 | def __init__(self, db_clients): 177 | """ 178 | Constructor 179 | :param db_client: Database client 180 | """ 181 | ExchangeGateway.__init__(self, ExchGwApiBigone(), db_clients) 182 | 183 | @classmethod 184 | def get_exchange_name(cls): 185 | """ 186 | Get exchange name 187 | :return: Exchange name string 188 | """ 189 | return 'Bigone' 190 | 191 | def get_order_book_worker(self, instmt): 192 | """ 193 | Get order book worker 194 | :param instmt: Instrument 195 | """ 196 | while True: 197 | try: 198 | l2_depth = self.api_socket.get_order_book(instmt) 199 | # print(l2_depth) 200 | # if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): 201 | if l2_depth is not None: 202 | instmt.set_prev_l2_depth(instmt.get_l2_depth()) 203 | instmt.set_l2_depth(l2_depth) 204 | instmt.incr_order_book_id() 205 | self.insert_order_book(instmt) 206 | except Exception as e: 207 | Logger.error(self.__class__.__name__, "Error in order book: %s" % e) 208 | time.sleep(5) 209 | 210 | time.sleep(3) 211 | 212 | def get_trades_worker(self, instmt): 213 | """ 214 | Get order book worker thread 215 | :param instmt: Instrument name 216 | """ 217 | while True: 218 | try: 219 | ret = self.api_socket.get_trades(instmt) 220 | if ret is None or len(ret) == 0: 221 | time.sleep(5) 222 | continue 223 | except Exception as e: 224 | Logger.error(self.__class__.__name__, "Error in trades: %s" % e) 225 | time.sleep(5) 226 | continue 227 | 228 | # print(ret) 229 | for trade in ret: 230 | assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) 231 | assert isinstance(instmt.get_exch_trade_id(), str), \ 232 | "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) 233 | if int(trade.trade_id) > int(instmt.get_exch_trade_id()): 234 | instmt.set_exch_trade_id(trade.trade_id) 235 | instmt.incr_trade_id() 236 | self.insert_trade(instmt, trade) 237 | 238 | # After the first time of getting the trade, indicate the instrument 239 | # is recovered 240 | if not instmt.get_recovered(): 241 | instmt.set_recovered(True) 242 | 243 | time.sleep(3) 244 | 245 | def start(self, instmt): 246 | """ 247 | Start the exchange gateway 248 | :param instmt: Instrument 249 | :return List of threads 250 | """ 251 | instmt.set_l2_depth(L2Depth(5)) 252 | instmt.set_prev_l2_depth(L2Depth(5)) 253 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 254 | instmt.get_instmt_name())) 255 | self.init_instmt_snapshot_table(instmt) 256 | instmt.set_recovered(False) 257 | t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt)) 258 | t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) 259 | t1.start() 260 | t2.start() 261 | return [t1, t2] 262 | 263 | 264 | if __name__ == '__main__': 265 | Logger.init_log() 266 | exchange_name = 'Bigone' 267 | instmt_name = 'IDTBTC' 268 | instmt_code = 'IDT-BTC' 269 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 270 | db_client = SqlClientTemplate() 271 | exch = ExchGwBigone([db_client]) 272 | instmt.set_l2_depth(L2Depth(5)) 273 | instmt.set_prev_l2_depth(L2Depth(5)) 274 | instmt.set_recovered(False) 275 | # exch.get_order_book_worker(instmt) 276 | exch.get_trades_worker(instmt) 277 | -------------------------------------------------------------------------------- /befh/exchanges/bitstamp.py: -------------------------------------------------------------------------------- 1 | from befh.ws_api_socket import WebSocketApiClient 2 | from befh.market_data import L2Depth, Trade 3 | from befh.exchanges.gateway import ExchangeGateway 4 | from befh.instrument import Instrument 5 | from befh.clients.sql_template import SqlClientTemplate 6 | from befh.util import Logger 7 | import time 8 | import threading 9 | import json 10 | from functools import partial 11 | from datetime import datetime 12 | 13 | 14 | class ExchGwApiBitstamp(WebSocketApiClient): 15 | """ 16 | Exchange socket 17 | """ 18 | def __init__(self): 19 | """ 20 | Constructor 21 | """ 22 | WebSocketApiClient.__init__(self, 'Bitstamp') 23 | 24 | @classmethod 25 | def get_trades_timestamp_field_name(cls): 26 | return 'timestamp' 27 | 28 | @classmethod 29 | def get_bids_field_name(cls): 30 | return 'bids' 31 | 32 | @classmethod 33 | def get_asks_field_name(cls): 34 | return 'asks' 35 | 36 | @classmethod 37 | def get_trade_side_field_name(cls): 38 | return 'type' 39 | 40 | @classmethod 41 | def get_trade_id_field_name(cls): 42 | return 'id' 43 | 44 | @classmethod 45 | def get_trade_price_field_name(cls): 46 | return 'price' 47 | 48 | @classmethod 49 | def get_trade_volume_field_name(cls): 50 | return 'amount' 51 | 52 | @classmethod 53 | def get_link(cls): 54 | return 'ws://ws.pusherapp.com/app/de504dc5763aeef9ff52?protocol=7' 55 | 56 | @classmethod 57 | def get_order_book_subscription_string(cls, instmt): 58 | if cls.is_default_instmt(instmt): 59 | return json.dumps({"event":"pusher:subscribe","data":{"channel":"order_book"}}) 60 | else: 61 | return json.dumps({"event":"pusher:subscribe","data":{"channel":"order_book_%s" % instmt.get_instmt_code()}}) 62 | 63 | @classmethod 64 | def get_trades_subscription_string(cls, instmt): 65 | if cls.is_default_instmt(instmt): 66 | return json.dumps({"event":"pusher:subscribe","data":{"channel":"live_trades"}}) 67 | else: 68 | return json.dumps({"event":"pusher:subscribe","data":{"channel":"live_trades_%s" % instmt.get_instmt_code()}}) 69 | 70 | @classmethod 71 | def is_default_instmt(cls, instmt): 72 | return instmt.get_instmt_code() == "\"\"" or instmt.get_instmt_code() == "" or instmt.get_instmt_code() == "''" 73 | 74 | @classmethod 75 | def parse_l2_depth(cls, instmt, raw): 76 | """ 77 | Parse raw data to L2 depth 78 | :param instmt: Instrument 79 | :param raw: Raw data in JSON 80 | """ 81 | l2_depth = instmt.get_l2_depth() 82 | keys = list(raw.keys()) 83 | if cls.get_bids_field_name() in keys and \ 84 | cls.get_asks_field_name() in keys: 85 | 86 | # Date time 87 | l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 88 | 89 | # Bids 90 | bids = raw[cls.get_bids_field_name()] 91 | bids_len = min(l2_depth.depth, len(bids)) 92 | for i in range(0, bids_len): 93 | l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0] 94 | l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1] 95 | 96 | # Asks 97 | asks = raw[cls.get_asks_field_name()] 98 | asks_len = min(l2_depth.depth, len(asks)) 99 | for i in range(0, asks_len): 100 | l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0] 101 | l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1] 102 | else: 103 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 104 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 105 | raw)) 106 | 107 | return l2_depth 108 | 109 | @classmethod 110 | def parse_trade(cls, instmt, raw): 111 | """ 112 | :param instmt: Instrument 113 | :param raw: Raw data in JSON 114 | :return: 115 | """ 116 | trade = Trade() 117 | keys = list(raw.keys()) 118 | 119 | if cls.get_trades_timestamp_field_name() in keys and \ 120 | cls.get_trade_id_field_name() in keys and \ 121 | cls.get_trade_side_field_name() in keys and \ 122 | cls.get_trade_price_field_name() in keys and \ 123 | cls.get_trade_volume_field_name() in keys: 124 | 125 | # Date time 126 | date_time = float(raw[cls.get_trades_timestamp_field_name()]) 127 | trade.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f") 128 | 129 | # Trade side 130 | # Buy = 0 131 | # Side = 1 132 | trade.trade_side = Trade.parse_side(raw[cls.get_trade_side_field_name()] + 1) 133 | 134 | # Trade id 135 | trade.trade_id = str(raw[cls.get_trade_id_field_name()]) 136 | 137 | # Trade price 138 | trade.trade_price = raw[cls.get_trade_price_field_name()] 139 | 140 | # Trade volume 141 | trade.trade_volume = raw[cls.get_trade_volume_field_name()] 142 | else: 143 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 144 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 145 | raw)) 146 | 147 | return trade 148 | 149 | 150 | class ExchGwBitstamp(ExchangeGateway): 151 | """ 152 | Exchange gateway 153 | """ 154 | def __init__(self, db_clients): 155 | """ 156 | Constructor 157 | :param db_client: Database client 158 | """ 159 | ExchangeGateway.__init__(self, ExchGwApiBitstamp(), db_clients) 160 | 161 | @classmethod 162 | def get_exchange_name(cls): 163 | """ 164 | Get exchange name 165 | :return: Exchange name string 166 | """ 167 | return 'Bitstamp' 168 | 169 | def on_open_handler(self, instmt, ws): 170 | """ 171 | Socket on open handler 172 | :param instmt: Instrument 173 | :param ws: Web socket 174 | """ 175 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 176 | (instmt.get_instmt_name(), instmt.get_exchange_name())) 177 | if not instmt.get_subscribed(): 178 | ws.send(self.api_socket.get_order_book_subscription_string(instmt)) 179 | ws.send(self.api_socket.get_trades_subscription_string(instmt)) 180 | instmt.set_subscribed(True) 181 | 182 | def on_close_handler(self, instmt, ws): 183 | """ 184 | Socket on close handler 185 | :param instmt: Instrument 186 | :param ws: Web socket 187 | """ 188 | Logger.info(self.__class__.__name__, "Instrument %s is unsubscribed in channel %s" % \ 189 | (instmt.get_instmt_name(), instmt.get_exchange_name())) 190 | instmt.set_subscribed(False) 191 | 192 | def on_message_handler(self, instmt, message): 193 | """ 194 | Incoming message handler 195 | :param instmt: Instrument 196 | :param message: Message 197 | """ 198 | keys = message.keys() 199 | if 'event' in keys and message['event'] in ['data', 'trade'] and 'channel' in keys and 'data' in keys: 200 | channel_name = message['channel'] 201 | if (self.api_socket.is_default_instmt(instmt) and channel_name == "order_book") or \ 202 | (not self.api_socket.is_default_instmt(instmt) and channel_name == "order_book_%s" % instmt.get_instmt_code()): 203 | instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) 204 | self.api_socket.parse_l2_depth(instmt, json.loads(message['data'])) 205 | if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): 206 | instmt.incr_order_book_id() 207 | self.insert_order_book(instmt) 208 | elif (self.api_socket.is_default_instmt(instmt) and channel_name == "live_trades") or \ 209 | (not self.api_socket.is_default_instmt(instmt) and channel_name == "live_trades_%s" % instmt.get_instmt_code()): 210 | trade = self.api_socket.parse_trade(instmt, json.loads(message['data'])) 211 | if trade.trade_id != instmt.get_exch_trade_id(): 212 | instmt.incr_trade_id() 213 | instmt.set_exch_trade_id(trade.trade_id) 214 | self.insert_trade(instmt, trade) 215 | 216 | def start(self, instmt): 217 | """ 218 | Start the exchange gateway 219 | :param instmt: Instrument 220 | :return List of threads 221 | """ 222 | instmt.set_l2_depth(L2Depth(20)) 223 | instmt.set_prev_l2_depth(L2Depth(20)) 224 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 225 | instmt.get_instmt_name())) 226 | self.init_instmt_snapshot_table(instmt) 227 | return [self.api_socket.connect(self.api_socket.get_link(), 228 | on_message_handler=partial(self.on_message_handler, instmt), 229 | on_open_handler=partial(self.on_open_handler, instmt), 230 | on_close_handler=partial(self.on_close_handler, instmt))] 231 | 232 | 233 | if __name__ == '__main__': 234 | exchange_name = 'Bitstamp' 235 | instmt_name = 'BTCUSD' 236 | instmt_code = '' 237 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 238 | db_client = SqlClientTemplate() 239 | Logger.init_log() 240 | exch = ExchGwBitstamp([db_client]) 241 | td = exch.start(instmt) 242 | 243 | -------------------------------------------------------------------------------- /befh/exchanges/bittrex.py: -------------------------------------------------------------------------------- 1 | from befh.restful_api_socket import RESTfulApiSocket 2 | from befh.exchanges.gateway import ExchangeGateway 3 | from befh.market_data import L2Depth, Trade 4 | from befh.util import Logger 5 | from befh.instrument import Instrument 6 | from befh.clients.sql_template import SqlClientTemplate 7 | from functools import partial 8 | from datetime import datetime 9 | import threading 10 | import time 11 | 12 | 13 | class ExchGwApiBittrex(RESTfulApiSocket): 14 | """ 15 | Exchange gateway RESTfulApi 16 | """ 17 | def __init__(self): 18 | RESTfulApiSocket.__init__(self) 19 | 20 | @classmethod 21 | def get_trades_timestamp_field_name(cls): 22 | return 'TimeStamp' 23 | 24 | @classmethod 25 | def get_trades_timestamp_format(cls): 26 | return '%Y-%m-%dT%H:%M:%S.%f' 27 | 28 | @classmethod 29 | def get_bids_field_name(cls): 30 | return 'buy' 31 | 32 | @classmethod 33 | def get_asks_field_name(cls): 34 | return 'sell' 35 | 36 | @classmethod 37 | def get_price_field_name(cls): 38 | return "Rate" 39 | 40 | @classmethod 41 | def get_volume_field_name(cls): 42 | return "Quantity" 43 | 44 | @classmethod 45 | def get_trade_price_field_name(cls): 46 | return 'Price' 47 | 48 | @classmethod 49 | def get_trade_volume_field_name(cls): 50 | return 'Quantity' 51 | 52 | @classmethod 53 | def get_trade_side_field_name(cls): 54 | return 'OrderType' 55 | 56 | @classmethod 57 | def get_trade_id_field_name(cls): 58 | return 'Id' 59 | 60 | @classmethod 61 | def get_order_book_link(cls, instmt): 62 | return "https://bittrex.com/api/v1.1/public/getorderbook?market=%s&type=both&depth=5" % instmt.get_instmt_code() 63 | 64 | @classmethod 65 | def get_trades_link(cls, instmt): 66 | return "https://bittrex.com/api/v1.1/public/getmarkethistory?market=%s" % instmt.get_instmt_code() 67 | 68 | @classmethod 69 | def parse_l2_depth(cls, instmt, raw): 70 | """ 71 | Parse raw data to L2 depth 72 | :param instmt: Instrument 73 | :param raw: Raw data in JSON 74 | """ 75 | l2_depth = L2Depth() 76 | raw = raw["result"] 77 | keys = list(raw.keys()) 78 | if cls.get_bids_field_name() in keys and \ 79 | cls.get_asks_field_name() in keys: 80 | 81 | # Date time 82 | l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 83 | 84 | # Bids 85 | bids = raw[cls.get_bids_field_name()] 86 | max_bid_len = min(len(bids), 5) 87 | for i in range(0, max_bid_len): 88 | l2_depth.bids[i].price = bids[i][cls.get_price_field_name()] 89 | l2_depth.bids[i].volume = bids[i][cls.get_volume_field_name()] 90 | 91 | # Asks 92 | asks = raw[cls.get_asks_field_name()] 93 | max_ask_len = min(len(asks), 5) 94 | for i in range(0, max_ask_len): 95 | l2_depth.asks[i].price = asks[i][cls.get_price_field_name()] 96 | l2_depth.asks[i].volume = asks[i][cls.get_volume_field_name()] 97 | else: 98 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 99 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 100 | raw)) 101 | 102 | return l2_depth 103 | 104 | @classmethod 105 | def parse_trade(cls, instmt, raw): 106 | """ 107 | :param instmt: Instrument 108 | :param raw: Raw data in JSON 109 | :return: 110 | """ 111 | trade = Trade() 112 | keys = list(raw.keys()) 113 | 114 | if cls.get_trades_timestamp_field_name() in keys and \ 115 | cls.get_trade_id_field_name() in keys and \ 116 | cls.get_trade_side_field_name() in keys and \ 117 | cls.get_trade_price_field_name() in keys and \ 118 | cls.get_trade_volume_field_name() in keys: 119 | 120 | # Date time 121 | date_time = raw[cls.get_trades_timestamp_field_name()] 122 | if len(date_time) == 19: 123 | date_time += '.' 124 | date_time += '0' * (26 - len(date_time)) 125 | date_time = datetime.strptime(date_time, cls.get_trades_timestamp_format()) 126 | trade.date_time = date_time.strftime("%Y%m%d %H:%M:%S.%f") 127 | 128 | # Trade side 129 | trade.trade_side = 1 if raw[cls.get_trade_side_field_name()] == 'BUY' else 2 130 | 131 | # Trade id 132 | trade.trade_id = str(raw[cls.get_trade_id_field_name()]) 133 | 134 | # Trade price 135 | trade.trade_price = float(str(raw[cls.get_trade_price_field_name()])) 136 | 137 | # Trade volume 138 | trade.trade_volume = float(str(raw[cls.get_trade_volume_field_name()])) 139 | else: 140 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 141 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 142 | raw)) 143 | 144 | return trade 145 | 146 | @classmethod 147 | def get_order_book(cls, instmt): 148 | """ 149 | Get order book 150 | :param instmt: Instrument 151 | :return: Object L2Depth 152 | """ 153 | res = cls.request(cls.get_order_book_link(instmt)) 154 | if len(res) > 0: 155 | return cls.parse_l2_depth(instmt=instmt, 156 | raw=res) 157 | else: 158 | return None 159 | 160 | @classmethod 161 | def get_trades(cls, instmt): 162 | """ 163 | Get trades 164 | :param instmt: Instrument 165 | :param trade_id: Trade id 166 | :return: List of trades 167 | """ 168 | link = cls.get_trades_link(instmt) 169 | res = cls.request(link) 170 | res = res["result"] 171 | trades = [] 172 | if len(res) > 0: 173 | for i in range(len(res)-1, -1, -1): 174 | trade = cls.parse_trade(instmt=instmt, 175 | raw=res[i]) 176 | trades.append(trade) 177 | 178 | return trades 179 | 180 | 181 | class ExchGwBittrex(ExchangeGateway): 182 | """ 183 | Exchange gateway Bittrex 184 | """ 185 | def __init__(self, db_clients): 186 | """ 187 | Constructor 188 | :param db_client: Database client 189 | """ 190 | ExchangeGateway.__init__(self, ExchGwApiBittrex(), db_clients) 191 | 192 | @classmethod 193 | def get_exchange_name(cls): 194 | """ 195 | Get exchange name 196 | :return: Exchange name string 197 | """ 198 | return 'Bittrex' 199 | 200 | def get_order_book_worker(self, instmt): 201 | """ 202 | Get order book worker 203 | :param instmt: Instrument 204 | """ 205 | while True: 206 | try: 207 | l2_depth = self.api_socket.get_order_book(instmt) 208 | if l2_depth is not None: 209 | instmt.set_prev_l2_depth(instmt.get_l2_depth()) 210 | instmt.set_l2_depth(l2_depth) 211 | instmt.incr_order_book_id() 212 | self.insert_order_book(instmt) 213 | except Exception as e: 214 | Logger.error(self.__class__.__name__, "Error in order book: %s" % e) 215 | time.sleep(2) 216 | 217 | time.sleep(3) 218 | 219 | def get_trades_worker(self, instmt): 220 | """ 221 | Get order book worker thread 222 | :param instmt: Instrument name 223 | """ 224 | while True: 225 | try: 226 | ret = self.api_socket.get_trades(instmt) 227 | if ret is None or len(ret) == 0: 228 | time.sleep(5) 229 | continue 230 | except Exception as e: 231 | Logger.error(self.__class__.__name__, "Error in trades: %s" % e) 232 | time.sleep(5) 233 | continue 234 | 235 | for trade in ret: 236 | assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) 237 | assert isinstance(instmt.get_exch_trade_id(), str), \ 238 | "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) 239 | if int(trade.trade_id) > int(instmt.get_exch_trade_id()): 240 | instmt.set_exch_trade_id(trade.trade_id) 241 | instmt.incr_trade_id() 242 | self.insert_trade(instmt, trade) 243 | 244 | # After the first time of getting the trade, indicate the instrument 245 | # is recovered 246 | if not instmt.get_recovered(): 247 | instmt.set_recovered(True) 248 | 249 | time.sleep(3) 250 | 251 | def start(self, instmt): 252 | """ 253 | Start the exchange gateway 254 | :param instmt: Instrument 255 | :return List of threads 256 | """ 257 | instmt.set_l2_depth(L2Depth(5)) 258 | instmt.set_prev_l2_depth(L2Depth(5)) 259 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 260 | instmt.get_instmt_name())) 261 | self.init_instmt_snapshot_table(instmt) 262 | instmt.set_recovered(False) 263 | t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt)) 264 | t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) 265 | t1.start() 266 | t2.start() 267 | return [t1, t2] 268 | 269 | 270 | if __name__ == '__main__': 271 | Logger.init_log() 272 | exchange_name = 'Bittrex' 273 | instmt_name = 'GBYTE' 274 | instmt_code = 'BTC-GBYTE' 275 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 276 | db_client = SqlClientTemplate() 277 | exch = ExchGwBittrex([db_client]) 278 | instmt.set_l2_depth(L2Depth(5)) 279 | instmt.set_prev_l2_depth(L2Depth(5)) 280 | instmt.set_recovered(False) 281 | exch.get_order_book_worker(instmt) 282 | # exch.get_trades_worker(instmt) 283 | -------------------------------------------------------------------------------- /befh/exchanges/coinone.py: -------------------------------------------------------------------------------- 1 | from befh.restful_api_socket import RESTfulApiSocket 2 | from befh.exchanges.gateway import ExchangeGateway 3 | from befh.market_data import L2Depth, Trade 4 | from befh.util import Logger 5 | from befh.instrument import Instrument 6 | from befh.clients.sql_template import SqlClientTemplate 7 | from functools import partial 8 | from datetime import datetime 9 | import threading 10 | import time 11 | 12 | 13 | class ExchGwApiCoineOne(RESTfulApiSocket): 14 | """ 15 | Exchange gateway RESTfulApi 16 | """ 17 | def __init__(self): 18 | RESTfulApiSocket.__init__(self) 19 | 20 | @classmethod 21 | def get_timestamp_offset(cls): 22 | return 1000 23 | 24 | @classmethod 25 | def get_order_book_timestamp_field_name(cls): 26 | return 'date' 27 | 28 | @classmethod 29 | def get_trades_timestamp_field_name(cls): 30 | return 'timestamp' 31 | 32 | @classmethod 33 | def get_bids_field_name(cls): 34 | return 'bid' 35 | 36 | @classmethod 37 | def get_asks_field_name(cls): 38 | return 'ask' 39 | 40 | @classmethod 41 | def get_trade_side_field_name(cls): 42 | return 'side' 43 | 44 | @classmethod 45 | def get_trade_id_field_name(cls): 46 | return 'timestamp' 47 | 48 | @classmethod 49 | def get_trade_price_field_name(cls): 50 | return 'price' 51 | 52 | @classmethod 53 | def get_trade_volume_field_name(cls): 54 | return 'qty' 55 | 56 | @classmethod 57 | def get_order_book_link(cls, instmt): 58 | return 'https://api.coinone.co.kr/orderbook?currency={}'.format(instmt.instmt_code) 59 | 60 | @classmethod 61 | def get_trades_link(cls, instmt): 62 | return 'https://api.coinone.co.kr/trades?currency={}&period=hour'.format(instmt.instmt_code) 63 | 64 | @classmethod 65 | def parse_l2_depth(cls, instmt, raw): 66 | """ 67 | Parse raw data to L2 depth 68 | :param instmt: Instrument 69 | :param raw: Raw data in JSON 70 | """ 71 | l2_depth = L2Depth() 72 | keys = list(raw.keys()) 73 | if cls.get_bids_field_name() in keys and \ 74 | cls.get_asks_field_name() in keys: 75 | 76 | # No Date time information, has update id only 77 | l2_depth.date_time = datetime.now().strftime("%Y%m%d %H:%M:%S.%f") 78 | 79 | # Bids 80 | bids = raw[cls.get_bids_field_name()] 81 | bids = sorted(bids, key=lambda x: x['price'], reverse=True) 82 | for i in range(0, 5): 83 | l2_depth.bids[i].price = float(bids[i]['price']) if type(bids[i]['price']) != float else bids[i]['price'] 84 | l2_depth.bids[i].volume = float(bids[i]['qty']) if type(bids[i]['qty']) != float else bids[i]['qty'] 85 | 86 | # Asks 87 | asks = raw[cls.get_asks_field_name()] 88 | asks = sorted(asks, key=lambda x: x['price']) 89 | for i in range(0, 5): 90 | l2_depth.asks[i].price = float(asks[i]['price']) if type(asks[i]['price']) != float else asks[i]['price'] 91 | l2_depth.asks[i].volume = float(asks[i]['qty']) if type(asks[i]['qty']) != float else asks[i]['qty'] 92 | else: 93 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 94 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 95 | raw)) 96 | 97 | return l2_depth 98 | 99 | @classmethod 100 | def parse_trade(cls, instmt, raw): 101 | """ 102 | :param instmt: Instrument 103 | :param raw: Raw data in JSON 104 | :return: 105 | """ 106 | trade = Trade() 107 | keys = list(raw.keys()) 108 | 109 | if cls.get_trades_timestamp_field_name() in keys and \ 110 | cls.get_trade_id_field_name() in keys and \ 111 | cls.get_trade_price_field_name() in keys and \ 112 | cls.get_trade_volume_field_name() in keys: 113 | 114 | # Date time 115 | date_time = float(raw[cls.get_trades_timestamp_field_name()]) 116 | #trade.date_time = datetime.strptime(date_time, '%Y-%m-%dT%H:%M:%S.%f') 117 | trade.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f") 118 | # Trade side 119 | trade.trade_side = 1 120 | # Trade id 121 | trade.trade_id = str(raw[cls.get_trade_id_field_name()]) 122 | 123 | # Trade price 124 | trade.trade_price = float(str(raw[cls.get_trade_price_field_name()])) 125 | 126 | # Trade volume 127 | trade.trade_volume = float(str(raw[cls.get_trade_volume_field_name()])) 128 | else: 129 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 130 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 131 | raw)) 132 | 133 | return trade 134 | 135 | @classmethod 136 | def get_order_book(cls, instmt): 137 | """ 138 | Get order book 139 | :param instmt: Instrument 140 | :return: Object L2Depth 141 | """ 142 | # If verify cert, got 143 | res = cls.request(cls.get_order_book_link(instmt), verify_cert=False) 144 | if len(res) > 0: 145 | return cls.parse_l2_depth(instmt=instmt, 146 | raw=res) 147 | else: 148 | return None 149 | 150 | @classmethod 151 | def get_trades(cls, instmt): 152 | """ 153 | Get trades 154 | :param instmt: Instrument 155 | :param trade_id: Trade id 156 | :return: List of trades 157 | """ 158 | link = cls.get_trades_link(instmt) 159 | print(link) 160 | # If verify cert, got 161 | res = cls.request(link, verify_cert=False) 162 | trades = [] 163 | if len(res['completeOrders']) > 0: 164 | for t in res['completeOrders']: 165 | trade = cls.parse_trade(instmt=instmt, 166 | raw=t) 167 | trades.append(trade) 168 | 169 | return trades 170 | 171 | 172 | class ExchGwCoinOne(ExchangeGateway): 173 | """ 174 | Exchange gateway 175 | """ 176 | def __init__(self, db_clients): 177 | """ 178 | Constructor 179 | :param db_client: Database client 180 | """ 181 | ExchangeGateway.__init__(self, ExchGwApiCoineOne(), db_clients) 182 | 183 | @classmethod 184 | def get_exchange_name(cls): 185 | """ 186 | Get exchange name 187 | :return: Exchange name string 188 | """ 189 | return 'CoinOne' 190 | 191 | def get_order_book_worker(self, instmt): 192 | """ 193 | Get order book worker 194 | :param instmt: Instrument 195 | """ 196 | while True: 197 | try: 198 | l2_depth = self.api_socket.get_order_book(instmt) 199 | if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): 200 | instmt.set_prev_l2_depth(instmt.get_l2_depth()) 201 | instmt.set_l2_depth(l2_depth) 202 | instmt.incr_order_book_id() 203 | self.insert_order_book(instmt) 204 | except Exception as e: 205 | Logger.error(self.__class__.__name__, "Error in order book: %s" % e) 206 | time.sleep(1) 207 | 208 | def get_trades_worker(self, instmt): 209 | """ 210 | Get order book worker thread 211 | :param instmt: Instrument name 212 | """ 213 | while True: 214 | try: 215 | ret = self.api_socket.get_trades(instmt) 216 | if ret is None or len(ret) == 0: 217 | time.sleep(1) 218 | continue 219 | except Exception as e: 220 | Logger.error(self.__class__.__name__, "Error in trades: %s" % e) 221 | time.sleep(1) 222 | continue 223 | 224 | for trade in ret: 225 | assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) 226 | assert isinstance(instmt.get_exch_trade_id(), str), \ 227 | "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) 228 | if trade.trade_id > instmt.get_exch_trade_id(): 229 | instmt.set_exch_trade_id(trade.trade_id) 230 | instmt.incr_trade_id() 231 | self.insert_trade(instmt, trade) 232 | 233 | # After the first time of getting the trade, indicate the instrument 234 | # is recovered 235 | if not instmt.get_recovered(): 236 | instmt.set_recovered(True) 237 | 238 | time.sleep(1) 239 | 240 | def start(self, instmt): 241 | """ 242 | Start the exchange gateway 243 | :param instmt: Instrument 244 | :return List of threads 245 | """ 246 | instmt.set_l2_depth(L2Depth(5)) 247 | instmt.set_prev_l2_depth(L2Depth(5)) 248 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 249 | instmt.get_instmt_name())) 250 | self.init_instmt_snapshot_table(instmt) 251 | instmt.set_recovered(False) 252 | t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt)) 253 | t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) 254 | t1.start() 255 | t2.start() 256 | return [t1, t2] 257 | 258 | 259 | if __name__ == '__main__': 260 | exchange_name = 'CoinOne' 261 | instmt_name = 'btc' 262 | instmt_code = 'btc' 263 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 264 | Logger.init_log() 265 | db_client = SqlClientTemplate() 266 | exch = ExchGwCoinOne([db_client]) 267 | instmt.set_l2_depth(L2Depth(5)) 268 | instmt.set_prev_l2_depth(L2Depth(5)) 269 | instmt.set_recovered(False) 270 | exch.start(instmt) 271 | #exch.get_order_book_worker(instmt) 272 | #exch.get_trades_worker(instmt) 273 | -------------------------------------------------------------------------------- /befh/exchanges/gateway.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | from befh.clients.zmq import ZmqClient 3 | from befh.clients.csv import FileClient 4 | from befh.clients.mysql import MysqlClient 5 | # from befh.clients.sqlite import SqliteClient 6 | from befh.clients.kafka import KafkaClient 7 | from befh.market_data import L2Depth, Trade, Snapshot 8 | from datetime import datetime 9 | from threading import Lock 10 | import time 11 | 12 | class ExchangeGateway: 13 | ############################################################################ 14 | # Static variable 15 | # Applied on all gateways whether to record the timestamp in local machine, 16 | # rather than exchange timestamp given by the API 17 | is_local_timestamp = False 18 | ############################################################################ 19 | 20 | """ 21 | Exchange gateway 22 | """ 23 | def __init__(self, api_socket, db_clients=[]): 24 | """ 25 | Constructor 26 | :param exchange_name: Exchange name 27 | :param exchange_api: Exchange API 28 | :param db_client: Database client 29 | """ 30 | self.db_clients = db_clients 31 | self.api_socket = api_socket 32 | self.lock = Lock() 33 | self.exch_snapshot_id = 0 34 | self.date_time = datetime.utcnow().date() 35 | self.last_tick = 0 36 | self.tick_wait = 1 37 | 38 | def rate_limit(self): 39 | current_time = time.time() 40 | if current_time - self.last_tick < self.tick_wait: 41 | # print('.') 42 | return True 43 | 44 | self.last_tick = current_time 45 | return False 46 | 47 | @classmethod 48 | def get_exchange_name(cls): 49 | """ 50 | Get exchange name 51 | :return: Exchange name string 52 | """ 53 | return '' 54 | 55 | def get_instmt_snapshot_table_name(self, exchange, instmt_name): 56 | """ 57 | Get instmt snapshot 58 | :param exchange: Exchange name 59 | :param instmt_name: Instrument name 60 | """ 61 | #return 'exch_' + exchange.lower() + '_' + instmt_name.lower() + \ 62 | # '_snapshot_' + datetime.utcnow().strftime("%Y%m%d") 63 | return 'exch_' + exchange.lower() + '_' + instmt_name.lower() + \ 64 | '_snapshot_' + self.date_time.strftime("%Y%m%d") 65 | 66 | @classmethod 67 | def get_snapshot_table_name(cls): 68 | return 'exchanges_snapshot' 69 | 70 | @classmethod 71 | def is_allowed_snapshot(cls, db_client): 72 | return not isinstance(db_client, FileClient) 73 | 74 | @classmethod 75 | def is_allowed_instmt_record(cls, db_client): 76 | return not isinstance(db_client, ZmqClient) and not isinstance(db_client, KafkaClient) 77 | 78 | @classmethod 79 | def init_snapshot_table(cls, db_clients): 80 | for db_client in db_clients: 81 | db_client.create(cls.get_snapshot_table_name(), 82 | Snapshot.columns(), 83 | Snapshot.types(), 84 | [0,1], is_ifnotexists=True) 85 | 86 | def init_instmt_snapshot_table(self, instmt): 87 | table_name = self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 88 | instmt.get_instmt_name()) 89 | 90 | instmt.set_instmt_snapshot_table_name(table_name) 91 | 92 | for db_client in self.db_clients: 93 | db_client.create(table_name, 94 | ['id'] + Snapshot.columns(False), 95 | ['int'] + Snapshot.types(False), 96 | [0], is_ifnotexists=True) 97 | 98 | # if isinstance(db_client, (MysqlClient, SqliteClient)): 99 | if isinstance(db_client, (MysqlClient)): 100 | with self.lock: 101 | r = db_client.execute('select max(id) from {};'.format(table_name)) 102 | db_client.conn.commit() 103 | if r: 104 | res = db_client.cursor.fetchone() 105 | max_id = res['max(id)'] if isinstance(db_client, MysqlClient) else res[0] 106 | if max_id: 107 | self.exch_snapshot_id = max_id 108 | else: 109 | self.exch_snapshot_id = 0 110 | 111 | def start(self, instmt): 112 | """ 113 | Start the exchange gateway 114 | :param instmt: Instrument 115 | :return List of threads 116 | """ 117 | return [] 118 | 119 | def get_instmt_snapshot_id(self, instmt): 120 | with self.lock: 121 | self.exch_snapshot_id += 1 122 | 123 | return self.exch_snapshot_id 124 | 125 | def insert_order_book(self, instmt): 126 | """ 127 | Insert order book row into the database client 128 | :param instmt: Instrument 129 | """ 130 | # If local timestamp indicator is on, assign the local timestamp again 131 | if self.is_local_timestamp: 132 | instmt.get_l2_depth().date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 133 | 134 | # Update the snapshot 135 | if instmt.get_l2_depth() is not None: 136 | l2_depth = instmt.get_l2_depth() 137 | assert(len(l2_depth.asks) >= 5) 138 | assert(len(l2_depth.bids) >= 5) 139 | 140 | id = self.get_instmt_snapshot_id(instmt) 141 | for db_client in self.db_clients: 142 | if self.is_allowed_snapshot(db_client): 143 | db_client.insert(table=self.get_snapshot_table_name(), 144 | columns=Snapshot.columns(), 145 | types=Snapshot.types(), 146 | values=Snapshot.values(instmt.get_exchange_name(), 147 | instmt.get_instmt_name(), 148 | instmt.get_l2_depth(), 149 | Trade() if instmt.get_last_trade() is None else instmt.get_last_trade(), 150 | Snapshot.UpdateType.ORDER_BOOK), 151 | primary_key_index=[0,1], 152 | is_orreplace=True, 153 | is_commit=True) 154 | 155 | if self.is_allowed_instmt_record(db_client): 156 | db_client.insert(table=instmt.get_instmt_snapshot_table_name(), 157 | columns=['id'] + Snapshot.columns(False), 158 | types=['int'] + Snapshot.types(False), 159 | values=[id] + 160 | Snapshot.values('', 161 | '', 162 | instmt.get_l2_depth(), 163 | Trade() if instmt.get_last_trade() is None else instmt.get_last_trade(), 164 | Snapshot.UpdateType.ORDER_BOOK), 165 | is_commit=True) 166 | 167 | def insert_trade(self, instmt, trade): 168 | """ 169 | Insert trade row into the database client 170 | :param instmt: Instrument 171 | """ 172 | # If the instrument is not recovered, skip inserting into the table 173 | if not instmt.get_recovered(): 174 | return 175 | 176 | # If local timestamp indicator is on, assign the local timestamp again 177 | if self.is_local_timestamp: 178 | trade.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 179 | 180 | date_time = datetime.strptime(trade.date_time, "%Y%m%d %H:%M:%S.%f").date() 181 | if date_time != self.date_time: 182 | self.date_time = date_time 183 | self.init_instmt_snapshot_table(instmt) 184 | 185 | # Set the last trade to the current one 186 | instmt.set_last_trade(trade) 187 | 188 | # Update the snapshot 189 | if instmt.get_l2_depth() is not None and \ 190 | instmt.get_last_trade() is not None: 191 | id = self.get_instmt_snapshot_id(instmt) 192 | for db_client in self.db_clients: 193 | is_allowed_snapshot = self.is_allowed_snapshot(db_client) 194 | is_allowed_instmt_record = self.is_allowed_instmt_record(db_client) 195 | if is_allowed_snapshot: 196 | db_client.insert(table=self.get_snapshot_table_name(), 197 | columns=Snapshot.columns(), 198 | values=Snapshot.values(instmt.get_exchange_name(), 199 | instmt.get_instmt_name(), 200 | instmt.get_l2_depth(), 201 | instmt.get_last_trade(), 202 | Snapshot.UpdateType.TRADES), 203 | types=Snapshot.types(), 204 | primary_key_index=[0,1], 205 | is_orreplace=True, 206 | is_commit=not is_allowed_instmt_record) 207 | 208 | if is_allowed_instmt_record: 209 | db_client.insert(table=instmt.get_instmt_snapshot_table_name(), 210 | columns=['id'] + Snapshot.columns(False), 211 | types=['int'] + Snapshot.types(False), 212 | values=[id] + 213 | Snapshot.values('', 214 | '', 215 | instmt.get_l2_depth(), 216 | instmt.get_last_trade(), 217 | Snapshot.UpdateType.TRADES), 218 | is_commit=True) 219 | -------------------------------------------------------------------------------- /befh/exchanges/huobi.py: -------------------------------------------------------------------------------- 1 | from befh.ws_api_socket import WebSocketApiClient 2 | from befh.market_data import L2Depth, Trade 3 | from befh.exchanges.gateway import ExchangeGateway 4 | from befh.instrument import Instrument 5 | from befh.util import Logger 6 | from befh.clients.sql_template import SqlClientTemplate 7 | import time 8 | import threading 9 | import json 10 | from functools import partial 11 | from datetime import datetime 12 | 13 | 14 | class ExchGwApiHuoBiWs(WebSocketApiClient): 15 | """ 16 | Exchange Socket 17 | """ 18 | 19 | Client_Id = int(time.mktime(datetime.now().timetuple())) 20 | 21 | def __init__(self): 22 | """ 23 | Constructor 24 | """ 25 | WebSocketApiClient.__init__(self, 'ExchApiHuoBi', received_data_compressed=True) 26 | 27 | @classmethod 28 | def get_order_book_timestamp_field_name(cls): 29 | return 'ts' 30 | 31 | @classmethod 32 | def get_trades_timestamp_field_name(cls): 33 | return 'ts' 34 | 35 | @classmethod 36 | def get_bids_field_name(cls): 37 | return 'bids' 38 | 39 | @classmethod 40 | def get_asks_field_name(cls): 41 | return 'asks' 42 | 43 | @classmethod 44 | def get_trade_side_field_name(cls): 45 | return 'direction' 46 | 47 | @classmethod 48 | def get_trade_id_field_name(cls): 49 | return 'id' 50 | 51 | @classmethod 52 | def get_trade_price_field_name(cls): 53 | return 'price' 54 | 55 | @classmethod 56 | def get_trade_volume_field_name(cls): 57 | return 'amount' 58 | 59 | @classmethod 60 | def get_link(cls): 61 | return 'wss://api.huobipro.com/ws' 62 | 63 | @classmethod 64 | def get_order_book_subscription_string(cls, instmt): 65 | return json.dumps({"sub": "market.{}.depth.step2".format(instmt.instmt_code), "id": "id{}".format(cls.Client_Id)}) 66 | 67 | @classmethod 68 | def get_trades_subscription_string(cls, instmt): 69 | return json.dumps({"sub": "market.{}.trade.detail".format(instmt.instmt_code), "id": "id{}".format(cls.Client_Id)}) 70 | 71 | @classmethod 72 | def parse_l2_depth(cls, instmt, raw): 73 | """ 74 | Parse raw data to L2 depth 75 | :param instmt: Instrument 76 | :param raw: Raw data in JSON 77 | """ 78 | l2_depth = instmt.get_l2_depth() 79 | keys = list(raw.keys()) 80 | if cls.get_bids_field_name() in keys and \ 81 | cls.get_asks_field_name() in keys: 82 | 83 | # Date time 84 | timestamp = raw['ts'] 85 | l2_depth.date_time = datetime.utcfromtimestamp(timestamp/1000.0).strftime("%Y%m%d %H:%M:%S.%f") 86 | 87 | # Bids 88 | bids = raw[cls.get_bids_field_name()] 89 | bids_len = min(l2_depth.depth, len(bids)) 90 | for i in range(0, bids_len): 91 | l2_depth.bids[i].price = float(bids[i][0]) if type(bids[i][0]) != float else bids[i][0] 92 | l2_depth.bids[i].volume = float(bids[i][1]) if type(bids[i][1]) != float else bids[i][1] 93 | 94 | # Asks 95 | asks = raw[cls.get_asks_field_name()] 96 | asks_len = min(l2_depth.depth, len(asks)) 97 | for i in range(0, asks_len): 98 | l2_depth.asks[i].price = float(asks[i][0]) if type(asks[i][0]) != float else asks[i][0] 99 | l2_depth.asks[i].volume = float(asks[i][1]) if type(asks[i][1]) != float else asks[i][1] 100 | else: 101 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 102 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 103 | raw)) 104 | 105 | return l2_depth 106 | 107 | @classmethod 108 | def parse_trade(cls, instmt, raws): 109 | """ 110 | :param instmt: Instrument 111 | :param raw: Raw data in JSON 112 | :return: 113 | """ 114 | 115 | trades = [] 116 | for raw in raws: 117 | trade = Trade() 118 | keys = list(raw.keys()) 119 | 120 | if cls.get_trades_timestamp_field_name() in keys and \ 121 | cls.get_trade_id_field_name() in keys and \ 122 | cls.get_trade_side_field_name() in keys and \ 123 | cls.get_trade_price_field_name() in keys and \ 124 | cls.get_trade_volume_field_name() in keys: 125 | 126 | # Date time 127 | date_time = float(raw[cls.get_trades_timestamp_field_name()]) 128 | trade.date_time = datetime.utcfromtimestamp(date_time/1000.0).strftime("%Y%m%d %H:%M:%S.%f") 129 | 130 | # Trade side 131 | # Buy = 0 132 | # Side = 1 133 | trade.trade_side = Trade.parse_side(raw[cls.get_trade_side_field_name()]) 134 | 135 | # Trade id 136 | trade.trade_id = str(raw[cls.get_trade_id_field_name()]) 137 | 138 | # Trade price 139 | trade.trade_price = raw[cls.get_trade_price_field_name()] 140 | 141 | # Trade volume 142 | trade.trade_volume = raw[cls.get_trade_volume_field_name()] 143 | else: 144 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 145 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 146 | raw)) 147 | trades.append(trade) 148 | return trades 149 | 150 | 151 | class ExchGwHuoBi(ExchangeGateway): 152 | """ 153 | Exchange gateway 154 | """ 155 | def __init__(self, db_clients): 156 | """ 157 | Constructor 158 | :param db_client: Database client 159 | """ 160 | ExchangeGateway.__init__(self, ExchGwApiHuoBiWs(), db_clients) 161 | 162 | @classmethod 163 | def get_exchange_name(cls): 164 | """ 165 | Get exchange name 166 | :return: Exchange name string 167 | """ 168 | return 'HuoBi' 169 | 170 | def on_open_handler(self, instmt, ws): 171 | """ 172 | Socket on open handler 173 | :param instmt: Instrument 174 | :param ws: Web socket 175 | """ 176 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 177 | (instmt.get_instmt_name(), instmt.get_exchange_name())) 178 | if not instmt.get_subscribed(): 179 | ws.send(self.api_socket.get_order_book_subscription_string(instmt)) 180 | ws.send(self.api_socket.get_trades_subscription_string(instmt)) 181 | instmt.set_subscribed(True) 182 | 183 | def on_close_handler(self, instmt, ws): 184 | """ 185 | Socket on close handler 186 | :param instmt: Instrument 187 | :param ws: Web socket 188 | """ 189 | Logger.info(self.__class__.__name__, "Instrument %s is unsubscribed in channel %s" % \ 190 | (instmt.get_instmt_name(), instmt.get_exchange_name())) 191 | instmt.set_subscribed(False) 192 | 193 | def on_message_handler(self, instmt, message): 194 | """ 195 | Incoming message handler 196 | :param instmt: Instrument 197 | :param message: Message 198 | """ 199 | if 'ping' in message: 200 | #handle ping response 201 | ts = message['ping'] 202 | self.api_socket.send(json.dumps({'pong': ts})) 203 | elif 'ch' in message: 204 | if 'trade.detail' in message['ch']: 205 | trades = self.api_socket.parse_trade(instmt, message['tick']['data']) 206 | for trade in trades: 207 | if trade.trade_id != instmt.get_exch_trade_id(): 208 | instmt.incr_trade_id() 209 | instmt.set_exch_trade_id(trade.trade_id) 210 | self.insert_trade(instmt, trade) 211 | elif 'depth.step' in message['ch']: 212 | instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) 213 | self.api_socket.parse_l2_depth(instmt, message['tick']) 214 | if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): 215 | instmt.incr_order_book_id() 216 | self.insert_order_book(instmt) 217 | else: 218 | Logger.error(self.__class__.__name__, 'Not Trade or Market') 219 | else: 220 | Logger.info(self.__class__.__name__, 'Nothing to do!!') 221 | 222 | def start(self, instmt): 223 | """ 224 | Start the exchange gateway 225 | :param instmt: Instrument 226 | :return List of threads 227 | """ 228 | instmt.set_l2_depth(L2Depth(20)) 229 | instmt.set_prev_l2_depth(L2Depth(20)) 230 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 231 | instmt.get_instmt_name())) 232 | self.init_instmt_snapshot_table(instmt) 233 | return [self.api_socket.connect(self.api_socket.get_link(), 234 | on_message_handler=partial(self.on_message_handler, instmt), 235 | on_open_handler=partial(self.on_open_handler, instmt), 236 | on_close_handler=partial(self.on_close_handler, instmt))] 237 | 238 | if __name__ == '__main__': 239 | import logging 240 | import websocket 241 | websocket.enableTrace(True) 242 | logging.basicConfig() 243 | Logger.init_log() 244 | exchange_name = 'HuoBi' 245 | instmt_name = 'BTCUSDT' 246 | instmt_code = 'btcusdt' 247 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 248 | db_client = SqlClientTemplate() 249 | exch = ExchGwHuoBi([db_client]) 250 | td = exch.start(instmt) 251 | pass 252 | -------------------------------------------------------------------------------- /befh/exchanges/kraken.py: -------------------------------------------------------------------------------- 1 | from befh.restful_api_socket import RESTfulApiSocket 2 | from befh.exchanges.gateway import ExchangeGateway 3 | from befh.market_data import L2Depth, Trade 4 | from befh.instrument import Instrument 5 | from befh.util import Logger 6 | import time 7 | import threading 8 | from functools import partial 9 | from datetime import datetime 10 | 11 | 12 | class ExchGwKrakenRestfulApi(RESTfulApiSocket): 13 | """ 14 | Exchange socket 15 | """ 16 | def __init__(self): 17 | RESTfulApiSocket.__init__(self) 18 | 19 | @classmethod 20 | def get_bids_field_name(cls): 21 | return 'bids' 22 | 23 | @classmethod 24 | def get_asks_field_name(cls): 25 | return 'asks' 26 | 27 | @classmethod 28 | def get_order_book_link(cls, instmt): 29 | return 'https://api.kraken.com/0/public/Depth?pair=%s&count=5' % instmt.get_instmt_code() 30 | 31 | @classmethod 32 | def get_trades_link(cls, instmt): 33 | if instmt.get_exch_trade_id() != '' and instmt.get_exch_trade_id() != '0': 34 | return 'https://api.kraken.com/0/public/Trades?pair=%s&since=%s' % \ 35 | (instmt.get_instmt_code(), instmt.get_exch_trade_id()) 36 | else: 37 | return 'https://api.kraken.com/0/public/Trades?pair=%s' % instmt.get_instmt_code() 38 | 39 | @classmethod 40 | def parse_l2_depth(cls, instmt, raw): 41 | """ 42 | Parse raw data to L2 depth 43 | :param instmt: Instrument 44 | :param raw: Raw data in JSON 45 | """ 46 | l2_depth = L2Depth() 47 | keys = list(raw.keys()) 48 | if cls.get_bids_field_name() in keys and \ 49 | cls.get_asks_field_name() in keys: 50 | # Bids 51 | bids = raw[cls.get_bids_field_name()] 52 | bids = sorted(bids, key=lambda x: x[0], reverse=True) 53 | for i in range(0, len(bids)): 54 | l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0] 55 | l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1] 56 | 57 | # Asks 58 | asks = raw[cls.get_asks_field_name()] 59 | asks = sorted(asks, key=lambda x: x[0]) 60 | for i in range(0, len(asks)): 61 | l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0] 62 | l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1] 63 | 64 | return l2_depth 65 | 66 | @classmethod 67 | def parse_trade(cls, instmt, raw): 68 | """ 69 | :param instmt: Instrument 70 | :param raw: Raw data in JSON 71 | :return: 72 | """ 73 | trade = Trade() 74 | 75 | # Trade price 76 | trade.trade_price = float(str(raw[0])) 77 | 78 | # Trade volume 79 | trade.trade_volume = float(str(raw[1])) 80 | 81 | # Timestamp 82 | date_time = float(raw[2]) 83 | trade.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f") 84 | 85 | # Trade side 86 | trade.trade_side = Trade.parse_side(raw[3]) 87 | 88 | # Trade id 89 | trade.trade_id = trade.date_time + '-' + str(instmt.get_exch_trade_id()) 90 | 91 | return trade 92 | 93 | @classmethod 94 | def get_order_book(cls, instmt): 95 | """ 96 | Get order book 97 | :param instmt: Instrument 98 | :return: Object L2Depth 99 | """ 100 | res = cls.request(cls.get_order_book_link(instmt)) 101 | if len(res) > 0 and 'error' in res and len(res['error']) == 0: 102 | res = list(res['result'].values())[0] 103 | return cls.parse_l2_depth(instmt=instmt, 104 | raw=res) 105 | else: 106 | Logger.error(cls.__name__, "Cannot parse the order book. Return:\n%s" % res) 107 | return None 108 | 109 | @classmethod 110 | def get_trades(cls, instmt): 111 | """ 112 | Get trades 113 | :param instmt: Instrument 114 | :param trade_id: Trade id 115 | :return: List of trades 116 | """ 117 | res = cls.request(cls.get_trades_link(instmt)) 118 | 119 | trades = [] 120 | if len(res) > 0 and 'error' in res and len(res['error']) == 0: 121 | res = res['result'] 122 | if 'last' in res.keys(): 123 | instmt.set_exch_trade_id(res['last']) 124 | del res['last'] 125 | 126 | res = list(res.values())[0] 127 | 128 | for t in res: 129 | trade = cls.parse_trade(instmt=instmt, 130 | raw=t) 131 | trades.append(trade) 132 | 133 | return trades 134 | 135 | 136 | class ExchGwKraken(ExchangeGateway): 137 | """ 138 | Exchange gateway 139 | """ 140 | def __init__(self, db_clients): 141 | """ 142 | Constructor 143 | :param db_client: Database client 144 | """ 145 | ExchangeGateway.__init__(self, ExchGwKrakenRestfulApi(), db_clients) 146 | 147 | @classmethod 148 | def get_exchange_name(cls): 149 | """ 150 | Get exchange name 151 | :return: Exchange name string 152 | """ 153 | return 'Kraken' 154 | 155 | def get_order_book_worker(self, instmt): 156 | """ 157 | Get order book worker 158 | :param instmt: Instrument 159 | """ 160 | while True: 161 | try: 162 | l2_depth = self.api_socket.get_order_book(instmt) 163 | if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): 164 | instmt.set_prev_l2_depth(instmt.l2_depth.copy()) 165 | instmt.set_l2_depth(l2_depth) 166 | instmt.incr_order_book_id() 167 | self.insert_order_book(instmt) 168 | except Exception as e: 169 | Logger.error(self.__class__.__name__, 170 | "Error in order book: %s" % e) 171 | time.sleep(0.5) 172 | 173 | def get_trades_worker(self, instmt): 174 | """ 175 | Get order book worker thread 176 | :param instmt: Instrument name 177 | """ 178 | instmt.set_recovered(False) 179 | 180 | while True: 181 | try: 182 | ret = self.api_socket.get_trades(instmt) 183 | for trade in ret: 184 | instmt.incr_trade_id() 185 | self.insert_trade(instmt, trade) 186 | 187 | # After the first time of getting the trade, indicate the instrument 188 | # is recovered 189 | if not instmt.get_recovered(): 190 | instmt.set_recovered(True) 191 | 192 | except Exception as e: 193 | Logger.error(self.__class__.__name__, 194 | "Error in trades: %s\nReturn: %s" % (e, ret)) 195 | time.sleep(0.5) 196 | 197 | def start(self, instmt): 198 | """ 199 | Start the exchange gateway 200 | :param instmt: Instrument 201 | :return List of threads 202 | """ 203 | instmt.set_prev_l2_depth(L2Depth(5)) 204 | instmt.set_l2_depth(L2Depth(5)) 205 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 206 | instmt.get_instmt_name())) 207 | self.init_instmt_snapshot_table(instmt) 208 | t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt)) 209 | t1.start() 210 | t2 = threading.Thread(target=partial(self.get_trades_worker, instmt)) 211 | t2.start() 212 | return [t1, t2] 213 | 214 | 215 | 216 | -------------------------------------------------------------------------------- /befh/exchanges/liqui.py: -------------------------------------------------------------------------------- 1 | from befh.restful_api_socket import RESTfulApiSocket 2 | from befh.exchanges.gateway import ExchangeGateway 3 | from befh.market_data import L2Depth, Trade 4 | from befh.util import Logger 5 | from befh.instrument import Instrument 6 | from befh.clients.sql_template import SqlClientTemplate 7 | from functools import partial 8 | from datetime import datetime 9 | from multiprocessing import Process 10 | import time 11 | 12 | 13 | class ExchGwApiLiqui(RESTfulApiSocket): 14 | """ 15 | Exchange gateway RESTfulApi 16 | """ 17 | def __init__(self): 18 | RESTfulApiSocket.__init__(self) 19 | 20 | @classmethod 21 | def get_timestamp_offset(cls): 22 | return 1 23 | 24 | @classmethod 25 | def get_trades_timestamp_field_name(cls): 26 | return 'timestamp' 27 | 28 | @classmethod 29 | def get_bids_field_name(cls): 30 | return 'bids' 31 | 32 | @classmethod 33 | def get_asks_field_name(cls): 34 | return 'asks' 35 | 36 | @classmethod 37 | def get_trade_side_field_name(cls): 38 | return 'type' 39 | 40 | @classmethod 41 | def get_trade_id_field_name(cls): 42 | return 'tid' 43 | 44 | @classmethod 45 | def get_trade_price_field_name(cls): 46 | return 'price' 47 | 48 | @classmethod 49 | def get_trade_volume_field_name(cls): 50 | return 'amount' 51 | 52 | @classmethod 53 | def get_order_book_link(cls, instmt): 54 | return "https://api.liqui.io/api/3/depth/{0}".format( 55 | instmt.get_instmt_code()) 56 | 57 | @classmethod 58 | def get_trades_link(cls, instmt): 59 | return "https://api.liqui.io/api/3/trades/{0}?limit=20".format( 60 | (instmt.get_instmt_code())) 61 | 62 | @classmethod 63 | def parse_l2_depth(cls, instmt, raw): 64 | """ 65 | Parse raw data to L2 depth 66 | :param instmt: Instrument 67 | :param raw: Raw data in JSON 68 | """ 69 | l2_depth = L2Depth() 70 | raw = raw[instmt.instmt_code] 71 | keys = list(raw.keys()) 72 | if (cls.get_bids_field_name() in keys and 73 | cls.get_asks_field_name() in keys): 74 | # Date time 75 | l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f") 76 | 77 | # Bids 78 | bids = raw[cls.get_bids_field_name()] 79 | for i in range(0, 5): 80 | l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0] 81 | l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1] 82 | 83 | # Asks 84 | asks = raw[cls.get_asks_field_name()] 85 | for i in range(0, 5): 86 | l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0] 87 | l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1] 88 | else: 89 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 90 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 91 | raw)) 92 | 93 | return l2_depth 94 | 95 | @classmethod 96 | def parse_trade(cls, instmt, raw): 97 | """ 98 | :param instmt: Instrument 99 | :param raw: Raw data in JSON 100 | :return: 101 | """ 102 | trade = Trade() 103 | keys = list(raw.keys()) 104 | 105 | if cls.get_trades_timestamp_field_name() in keys and \ 106 | cls.get_trade_id_field_name() in keys and \ 107 | cls.get_trade_price_field_name() in keys and \ 108 | cls.get_trade_volume_field_name() in keys: 109 | 110 | # Date time 111 | date_time = float(raw[cls.get_trades_timestamp_field_name()]) 112 | date_time = date_time / cls.get_timestamp_offset() 113 | trade.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f") 114 | 115 | # Trade side 116 | trade.trade_side = 1 117 | 118 | # Trade id 119 | trade.trade_id = str(raw[cls.get_trade_id_field_name()]) 120 | 121 | # Trade price 122 | trade.trade_price = float(str(raw[cls.get_trade_price_field_name()])) 123 | 124 | # Trade volume 125 | trade.trade_volume = float(str(raw[cls.get_trade_volume_field_name()])) 126 | else: 127 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 128 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 129 | raw)) 130 | 131 | return trade 132 | 133 | @classmethod 134 | def get_order_book(cls, instmt): 135 | """ 136 | Get order book 137 | :param instmt: Instrument 138 | :return: Object L2Depth 139 | """ 140 | res = cls.request(cls.get_order_book_link(instmt)) 141 | if len(res) > 0: 142 | return cls.parse_l2_depth(instmt=instmt, 143 | raw=res) 144 | else: 145 | return None 146 | 147 | @classmethod 148 | def get_trades(cls, instmt): 149 | """ 150 | Get trades 151 | :param instmt: Instrument 152 | :param trade_id: Trade id 153 | :return: List of trades 154 | """ 155 | link = cls.get_trades_link(instmt) 156 | res = cls.request(link) 157 | trades = [] 158 | if len(res) > 0: 159 | res = res[instmt.instmt_code] 160 | for i in range(0, len(res)): 161 | t = res[len(res) - 1 - i] 162 | trade = cls.parse_trade(instmt=instmt, 163 | raw=t) 164 | trades.append(trade) 165 | 166 | return trades 167 | 168 | 169 | class ExchGwLiqui(ExchangeGateway): 170 | """ 171 | Exchange gateway 172 | """ 173 | def __init__(self, db_clients): 174 | """ 175 | Constructor 176 | :param db_client: Database client 177 | """ 178 | ExchangeGateway.__init__(self, ExchGwApiLiqui(), db_clients) 179 | 180 | @classmethod 181 | def get_exchange_name(cls): 182 | """ 183 | Get exchange name 184 | :return: Exchange name string 185 | """ 186 | return 'Liqui' 187 | 188 | def get_order_book_worker(self, instmt): 189 | """ 190 | Get order book worker 191 | :param instmt: Instrument 192 | """ 193 | while True: 194 | try: 195 | l2_depth = self.api_socket.get_order_book(instmt) 196 | if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()): 197 | instmt.set_prev_l2_depth(instmt.get_l2_depth()) 198 | instmt.set_l2_depth(l2_depth) 199 | instmt.incr_order_book_id() 200 | self.insert_order_book(instmt) 201 | except Exception as e: 202 | Logger.error(self.__class__.__name__, "Error in order book: %s" % e) 203 | time.sleep(1) 204 | 205 | def get_trades_worker(self, instmt): 206 | """ 207 | Get order book worker thread 208 | :param instmt: Instrument name 209 | """ 210 | while True: 211 | try: 212 | ret = self.api_socket.get_trades(instmt) 213 | if ret is None or len(ret) == 0: 214 | time.sleep(1) 215 | continue 216 | except Exception as e: 217 | Logger.error(self.__class__.__name__, "Error in trades: %s" % e) 218 | time.sleep(1) 219 | continue 220 | 221 | for trade in ret: 222 | assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id) 223 | assert isinstance(instmt.get_exch_trade_id(), str), \ 224 | "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id()) 225 | if int(trade.trade_id) > int(instmt.get_exch_trade_id()): 226 | instmt.set_exch_trade_id(trade.trade_id) 227 | instmt.incr_trade_id() 228 | self.insert_trade(instmt, trade) 229 | 230 | # After the first time of getting the trade, indicate the instrument 231 | # is recovered 232 | if not instmt.get_recovered(): 233 | instmt.set_recovered(True) 234 | 235 | time.sleep(1) 236 | 237 | def start(self, instmt): 238 | """ 239 | Start the exchange gateway 240 | :param instmt: Instrument 241 | :return List of threads 242 | """ 243 | instmt.set_l2_depth(L2Depth(5)) 244 | instmt.set_prev_l2_depth(L2Depth(5)) 245 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 246 | instmt.get_instmt_name())) 247 | self.init_instmt_snapshot_table(instmt) 248 | instmt.set_recovered(False) 249 | t1 = Process(target=partial(self.get_order_book_worker, instmt)) 250 | t2 = Process(target=partial(self.get_trades_worker, instmt)) 251 | t1.start() 252 | t2.start() 253 | return [t1, t2] 254 | 255 | 256 | if __name__ == '__main__': 257 | Logger.init_log() 258 | exchange_name = 'Liqui' 259 | instmt_name = 'ETHBTC' 260 | instmt_code = 'eth_btc' 261 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 262 | db_client = SqlClientTemplate() 263 | exch = ExchGwLiqui([db_client]) 264 | instmt.set_l2_depth(L2Depth(5)) 265 | instmt.set_prev_l2_depth(L2Depth(5)) 266 | instmt.set_recovered(False) 267 | # exch.get_order_book_worker(instmt) 268 | exch.get_trades_worker(instmt) 269 | -------------------------------------------------------------------------------- /befh/exchanges/okcoin.py: -------------------------------------------------------------------------------- 1 | from befh.ws_api_socket import WebSocketApiClient 2 | from befh.market_data import L2Depth, Trade 3 | from befh.exchanges.gateway import ExchangeGateway 4 | from befh.instrument import Instrument 5 | from befh.util import Logger 6 | import time 7 | import threading 8 | import json 9 | from functools import partial 10 | from datetime import datetime 11 | 12 | 13 | class ExchGwOkCoinWs(WebSocketApiClient): 14 | """ 15 | Exchange socket 16 | """ 17 | def __init__(self): 18 | """ 19 | Constructor 20 | """ 21 | WebSocketApiClient.__init__(self, 'ExchGwOkCoin') 22 | 23 | @classmethod 24 | def get_order_book_timestamp_field_name(cls): 25 | return 'timestamp' 26 | 27 | @classmethod 28 | def get_bids_field_name(cls): 29 | return 'bids' 30 | 31 | @classmethod 32 | def get_asks_field_name(cls): 33 | return 'asks' 34 | 35 | @classmethod 36 | def get_link(cls): 37 | return 'wss://real.okcoin.com:10440/websocket/okcoinapi' 38 | 39 | @classmethod 40 | def get_order_book_subscription_string(cls, instmt): 41 | return json.dumps({"event":"addChannel", "channel": instmt.get_order_book_channel_id()}) 42 | 43 | @classmethod 44 | def get_trades_subscription_string(cls, instmt): 45 | return json.dumps({"event":"addChannel", "channel": instmt.get_trades_channel_id()}) 46 | 47 | @classmethod 48 | def parse_l2_depth(cls, instmt, raw): 49 | """ 50 | Parse raw data to L2 depth 51 | :param instmt: Instrument 52 | :param raw: Raw data in JSON 53 | """ 54 | l2_depth = instmt.get_l2_depth() 55 | keys = list(raw.keys()) 56 | if cls.get_order_book_timestamp_field_name() in keys and \ 57 | cls.get_bids_field_name() in keys and \ 58 | cls.get_asks_field_name() in keys: 59 | 60 | # Date time 61 | timestamp = float(raw[cls.get_order_book_timestamp_field_name()])/1000.0 62 | l2_depth.date_time = datetime.utcfromtimestamp(timestamp).strftime("%Y%m%d %H:%M:%S.%f") 63 | 64 | # Bids 65 | bids = raw[cls.get_bids_field_name()] 66 | bids = sorted(bids, key=lambda x: x[0], reverse=True) 67 | for i in range(0, len(bids)): 68 | l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0] 69 | l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1] 70 | 71 | # Asks 72 | asks = raw[cls.get_asks_field_name()] 73 | asks = sorted(asks, key=lambda x: x[0]) 74 | for i in range(0, len(asks)): 75 | l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0] 76 | l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1] 77 | else: 78 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 79 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 80 | raw)) 81 | 82 | return l2_depth 83 | 84 | @classmethod 85 | def parse_trade(cls, instmt, raw): 86 | """ 87 | :param instmt: Instrument 88 | :param raw: Raw data in JSON 89 | :return: 90 | """ 91 | trade = Trade() 92 | trade_id = raw[0] 93 | trade_price = float(raw[1]) 94 | trade_volume = float(raw[2]) 95 | timestamp = raw[3] 96 | trade_side = raw[4] 97 | 98 | trade.trade_id = trade_id + timestamp 99 | trade.trade_price = trade_price 100 | trade.trade_volume = trade_volume 101 | trade.trade_side = Trade.parse_side(trade_side) 102 | 103 | return trade 104 | 105 | 106 | class ExchGwOkCoin(ExchangeGateway): 107 | """ 108 | Exchange gateway 109 | """ 110 | def __init__(self, db_clients): 111 | """ 112 | Constructor 113 | :param db_client: Database client 114 | """ 115 | ExchangeGateway.__init__(self, ExchGwOkCoinWs(), db_clients) 116 | 117 | @classmethod 118 | def get_exchange_name(cls): 119 | """ 120 | Get exchange name 121 | :return: Exchange name string 122 | """ 123 | return 'OkCoin' 124 | 125 | def on_open_handler(self, instmt, ws): 126 | """ 127 | Socket on open handler 128 | :param instmt: Instrument 129 | :param ws: Web socket 130 | """ 131 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 132 | (instmt.get_instmt_code(), instmt.get_exchange_name())) 133 | if not instmt.get_subscribed(): 134 | instmt_code_split = instmt.get_instmt_code().split('_') 135 | if len(instmt_code_split) == 3: 136 | # Future instruments 137 | instmt.set_order_book_channel_id("ok_sub_%s_%s_depth_%s_20" % \ 138 | (instmt_code_split[0], 139 | instmt_code_split[1], 140 | instmt_code_split[2])) 141 | instmt.set_trades_channel_id("ok_sub_%s_%s_trade_%s" % \ 142 | (instmt_code_split[0], 143 | instmt_code_split[1], 144 | instmt_code_split[2])) 145 | else: 146 | # Spot instruments 147 | instmt.set_order_book_channel_id("ok_sub_%s_depth_20" % instmt.get_instmt_code()) 148 | instmt.set_trades_channel_id("ok_sub_%s_trades" % instmt.get_instmt_code()) 149 | 150 | ws.send(self.api_socket.get_order_book_subscription_string(instmt)) 151 | ws.send(self.api_socket.get_trades_subscription_string(instmt)) 152 | instmt.set_subscribed(True) 153 | 154 | def on_close_handler(self, instmt, ws): 155 | """ 156 | Socket on close handler 157 | :param instmt: Instrument 158 | :param ws: Web socket 159 | """ 160 | Logger.info(self.__class__.__name__, "Instrument %s is unsubscribed in channel %s" % \ 161 | (instmt.get_instmt_code(), instmt.get_exchange_name())) 162 | instmt.set_subscribed(False) 163 | 164 | def on_message_handler(self, instmt, messages): 165 | """ 166 | Incoming message handler 167 | :param instmt: Instrument 168 | :param message: Message 169 | """ 170 | for message in messages: 171 | keys = message.keys() 172 | if 'channel' in keys: 173 | if 'data' in keys: 174 | if message['channel'] == instmt.get_order_book_channel_id(): 175 | data = message['data'] 176 | instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) 177 | self.api_socket.parse_l2_depth(instmt, data) 178 | 179 | # Insert only if the first 5 levels are different 180 | if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): 181 | instmt.incr_order_book_id() 182 | self.insert_order_book(instmt) 183 | 184 | elif message['channel'] == instmt.get_trades_channel_id(): 185 | for trade_raw in message['data']: 186 | trade = self.api_socket.parse_trade(instmt, trade_raw) 187 | if trade.trade_id != instmt.get_exch_trade_id(): 188 | instmt.incr_trade_id() 189 | instmt.set_exch_trade_id(trade.trade_id) 190 | self.insert_trade(instmt, trade) 191 | 192 | elif 'success' in keys: 193 | Logger.info(self.__class__.__name__, "Subscription to channel %s is %s" \ 194 | % (message['channel'], message['success'])) 195 | else: 196 | Logger.info(self.__class__.__name__, ' - ' + json.dumps(message)) 197 | 198 | def start(self, instmt): 199 | """ 200 | Start the exchange gateway 201 | :param instmt: Instrument 202 | :return List of threads 203 | """ 204 | instmt.set_prev_l2_depth(L2Depth(20)) 205 | instmt.set_l2_depth(L2Depth(20)) 206 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 207 | instmt.get_instmt_name())) 208 | self.init_instmt_snapshot_table(instmt) 209 | return [self.api_socket.connect(self.api_socket.get_link(), 210 | on_message_handler=partial(self.on_message_handler, instmt), 211 | on_open_handler=partial(self.on_open_handler, instmt), 212 | on_close_handler=partial(self.on_close_handler, instmt))] 213 | 214 | -------------------------------------------------------------------------------- /befh/exchanges/okex_future.py: -------------------------------------------------------------------------------- 1 | from befh.ws_api_socket import WebSocketApiClient 2 | from befh.market_data import L2Depth, Trade 3 | from befh.exchanges.gateway import ExchangeGateway 4 | from befh.instrument import Instrument 5 | from befh.util import Logger 6 | from befh.clients.sql_template import SqlClientTemplate 7 | import time 8 | import threading 9 | import json 10 | from functools import partial 11 | from datetime import datetime 12 | import pytz 13 | import re 14 | from tzlocal import get_localzone 15 | 16 | 17 | class ExchGwApiOkexFutureWs(WebSocketApiClient): 18 | """ 19 | Exchange Socket 20 | """ 21 | 22 | Client_Id = int(time.mktime(datetime.now().timetuple())) 23 | 24 | def __init__(self): 25 | """ 26 | Constructor 27 | """ 28 | WebSocketApiClient.__init__(self, 'ExchApiOkexFuture') 29 | 30 | @classmethod 31 | def get_bids_field_name(cls): 32 | return 'bids' 33 | 34 | @classmethod 35 | def get_asks_field_name(cls): 36 | return 'asks' 37 | 38 | @classmethod 39 | def get_link(cls): 40 | return 'wss://real.okex.com:10440/websocket/okexapi' 41 | 42 | @classmethod 43 | def get_order_book_subscription_string(cls, instmt): 44 | return json.dumps({'event':'addChannel','channel':'ok_sub_futureusd_{}_depth_this_week'.format(instmt.instmt_code)}) 45 | 46 | @classmethod 47 | def get_trades_subscription_string(cls, instmt): 48 | return json.dumps({'event':'addChannel','channel':'ok_sub_futureusd_{}_trade_this_week'.format(instmt.instmt_code)}) 49 | 50 | @classmethod 51 | def parse_l2_depth(cls, instmt, raw): 52 | """ 53 | Parse raw data to L2 depth 54 | :param instmt: Instrument 55 | :param raw: Raw data in JSON 56 | """ 57 | l2_depth = instmt.get_l2_depth() 58 | keys = list(raw.keys()) 59 | if cls.get_bids_field_name() in keys and \ 60 | cls.get_asks_field_name() in keys: 61 | 62 | # Date time 63 | timestamp = raw['timestamp'] 64 | l2_depth.date_time = datetime.utcfromtimestamp(timestamp/1000.0).strftime("%Y%m%d %H:%M:%S.%f") 65 | 66 | # Bids 67 | bids = raw[cls.get_bids_field_name()] 68 | bids_len = min(l2_depth.depth, len(bids)) 69 | for i in range(0, bids_len): 70 | l2_depth.bids[i].price = float(bids[i][0]) if type(bids[i][0]) != float else bids[i][0] 71 | l2_depth.bids[i].volume = float(bids[i][1]) if type(bids[i][1]) != float else bids[i][1] 72 | 73 | # Asks 74 | asks = raw[cls.get_asks_field_name()] 75 | asks_len = min(l2_depth.depth, len(asks)) 76 | for i in range(0, asks_len): 77 | l2_depth.asks[i].price = float(asks[i][0]) if type(asks[i][0]) != float else asks[i][0] 78 | l2_depth.asks[i].volume = float(asks[i][1]) if type(asks[i][1]) != float else asks[i][1] 79 | else: 80 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 81 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 82 | raw)) 83 | return l2_depth 84 | 85 | @classmethod 86 | def parse_trade(cls, instmt, raws): 87 | """ 88 | :param instmt: Instrument 89 | :param raw: Raw data in JSON 90 | :return: 91 | """ 92 | 93 | trades = [] 94 | for item in raws: 95 | trade = Trade() 96 | today = datetime.today().date() 97 | time = item[3] 98 | #trade.date_time = datetime.utcfromtimestamp(date_time/1000.0).strftime("%Y%m%d %H:%M:%S.%f") 99 | #Convert local time as to UTC. 100 | date_time = datetime(today.year, today.month, today.day, 101 | *list(map(lambda x: int(x), time.split(':'))), 102 | tzinfo = get_localzone() 103 | ) 104 | trade.date_time = date_time.astimezone(pytz.utc).strftime('%Y%m%d %H:%M:%S.%f') 105 | # Trade side 106 | # Buy = 0 107 | # Side = 1 108 | trade.trade_side = Trade.parse_side(item[4]) 109 | # Trade id 110 | trade.trade_id = str(item[0]) 111 | # Trade price 112 | trade.trade_price = item[1] 113 | # Trade volume 114 | trade.trade_volume = item[2] 115 | trades.append(trade) 116 | return trades 117 | 118 | 119 | class ExchGwOkexFuture(ExchangeGateway): 120 | """ 121 | Exchange gateway 122 | """ 123 | def __init__(self, db_clients): 124 | """ 125 | Constructor 126 | :param db_client: Database client 127 | """ 128 | ExchangeGateway.__init__(self, ExchGwApiOkexFutureWs(), db_clients) 129 | 130 | @classmethod 131 | def get_exchange_name(cls): 132 | """ 133 | Get exchange name 134 | :return: Exchange name string 135 | """ 136 | return 'OkexFuture' 137 | 138 | def on_open_handler(self, instmt, ws): 139 | """ 140 | Socket on open handler 141 | :param instmt: Instrument 142 | :param ws: Web socket 143 | """ 144 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 145 | (instmt.get_instmt_name(), instmt.get_exchange_name())) 146 | if not instmt.get_subscribed(): 147 | Logger.info(self.__class__.__name__, 'order book string:{}'.format(self.api_socket.get_order_book_subscription_string(instmt))) 148 | Logger.info(self.__class__.__name__, 'trade string:{}'.format(self.api_socket.get_trades_subscription_string(instmt))) 149 | ws.send(self.api_socket.get_order_book_subscription_string(instmt)) 150 | ws.send(self.api_socket.get_trades_subscription_string(instmt)) 151 | instmt.set_subscribed(True) 152 | 153 | def on_close_handler(self, instmt, ws): 154 | """ 155 | Socket on close handler 156 | :param instmt: Instrument 157 | :param ws: Web socket 158 | """ 159 | Logger.info(self.__class__.__name__, "Instrument %s is unsubscribed in channel %s" % \ 160 | (instmt.get_instmt_name(), instmt.get_exchange_name())) 161 | instmt.set_subscribed(False) 162 | 163 | def on_message_handler(self, instmt, message): 164 | """ 165 | Incoming message handler 166 | :param instmt: Instrument 167 | :param message: Message 168 | """ 169 | for item in message: 170 | if 'channel' in item: 171 | if re.search(r'ok_sub_futureusd_(.*)_depth_this_week', item['channel']): 172 | instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) 173 | self.api_socket.parse_l2_depth(instmt, item['data']) 174 | if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): 175 | instmt.incr_order_book_id() 176 | self.insert_order_book(instmt) 177 | elif re.search(r'ok_sub_futureusd_(.*)_trade_this_week', item['channel']): 178 | trades = self.api_socket.parse_trade(instmt, item['data']) 179 | for trade in trades: 180 | if trade.trade_id != instmt.get_exch_trade_id(): 181 | instmt.incr_trade_id() 182 | instmt.set_exch_trade_id(trade.trade_id) 183 | self.insert_trade(instmt, trade) 184 | else: 185 | Logger.info(self.__class__.__name__, 'Nothing to do!!') 186 | 187 | def start(self, instmt): 188 | """ 189 | Start the exchange gateway 190 | :param instmt: Instrument 191 | :return List of threads 192 | """ 193 | instmt.set_l2_depth(L2Depth(20)) 194 | instmt.set_prev_l2_depth(L2Depth(20)) 195 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 196 | instmt.get_instmt_name())) 197 | self.init_instmt_snapshot_table(instmt) 198 | Logger.info(self.__class__.__name__, 'instmt snapshot table: {}'.format(instmt.get_instmt_snapshot_table_name())) 199 | return [self.api_socket.connect(self.api_socket.get_link(), 200 | on_message_handler=partial(self.on_message_handler, instmt), 201 | on_open_handler=partial(self.on_open_handler, instmt), 202 | on_close_handler=partial(self.on_close_handler, instmt))] 203 | 204 | if __name__ == '__main__': 205 | import logging 206 | import websocket 207 | websocket.enableTrace(True) 208 | logging.basicConfig() 209 | Logger.init_log() 210 | exchange_name = 'OkexFuture' 211 | instmt_name = 'BTC' 212 | instmt_code = 'btc' 213 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 214 | db_client = SqlClientTemplate() 215 | exch = ExchGwOkexFuture([db_client]) 216 | td = exch.start(instmt) 217 | pass 218 | -------------------------------------------------------------------------------- /befh/exchanges/okex_spot.py: -------------------------------------------------------------------------------- 1 | from befh.ws_api_socket import WebSocketApiClient 2 | from befh.market_data import L2Depth, Trade 3 | from befh.exchanges.gateway import ExchangeGateway 4 | from befh.instrument import Instrument 5 | from befh.util import Logger 6 | from befh.clients.sql_template import SqlClientTemplate 7 | import time 8 | import threading 9 | import json 10 | from functools import partial 11 | from datetime import datetime 12 | 13 | 14 | class ExchGwApiOkexSpotWs(WebSocketApiClient): 15 | """ 16 | Exchange socket 17 | """ 18 | def __init__(self): 19 | """ 20 | Constructor 21 | """ 22 | WebSocketApiClient.__init__(self, 'ExchGwOkexSpot') 23 | 24 | @classmethod 25 | def get_timestamp_offset(cls): 26 | return 1000 27 | 28 | @classmethod 29 | def get_order_book_timestamp_field_name(cls): 30 | return 'timestamp' 31 | 32 | @classmethod 33 | def get_bids_field_name(cls): 34 | return 'bids' 35 | 36 | @classmethod 37 | def get_asks_field_name(cls): 38 | return 'asks' 39 | 40 | @classmethod 41 | def get_link(cls): 42 | return 'wss://real.okex.com:10441/websocket' 43 | 44 | @classmethod 45 | def get_order_book_subscription_string(cls, instmt): 46 | return json.dumps({"event":"addChannel", "channel": instmt.get_order_book_channel_id()}) 47 | 48 | @classmethod 49 | def get_trades_subscription_string(cls, instmt): 50 | return json.dumps({"event":"addChannel", "channel": instmt.get_trades_channel_id()}) 51 | 52 | @classmethod 53 | def parse_l2_depth(cls, instmt, raw): 54 | """ 55 | Parse raw data to L2 depth 56 | :param instmt: Instrument 57 | :param raw: Raw data in JSON 58 | """ 59 | # l2_depth = instmt.get_l2_depth() 60 | l2_depth = L2Depth() 61 | 62 | keys = list(raw.keys()) 63 | if cls.get_order_book_timestamp_field_name() in keys and \ 64 | cls.get_bids_field_name() in keys and \ 65 | cls.get_asks_field_name() in keys: 66 | 67 | # Date time 68 | timestamp = float(raw[cls.get_order_book_timestamp_field_name()])/cls.get_timestamp_offset() 69 | l2_depth.date_time = datetime.utcfromtimestamp(timestamp).strftime("%Y%m%d %H:%M:%S.%f") 70 | 71 | # Bids 72 | bids = raw[cls.get_bids_field_name()] 73 | bids = sorted(bids, key=lambda x: x[0], reverse=True) 74 | max_bid_len = min(len(bids), 5) 75 | 76 | for i in range(0, max_bid_len): 77 | l2_depth.bids[i].price = float(bids[i][0]) if type(bids[i][0]) != float else bids[i][0] 78 | l2_depth.bids[i].volume = float(bids[i][1]) if type(bids[i][1]) != float else bids[i][1] 79 | 80 | # Asks 81 | asks = raw[cls.get_asks_field_name()] 82 | asks = sorted(asks, key=lambda x: x[0]) 83 | max_ask_len = min(len(asks), 5) 84 | 85 | for i in range(0, max_ask_len): 86 | l2_depth.asks[i].price = float(asks[i][0]) if type(asks[i][0]) != float else asks[i][0] 87 | l2_depth.asks[i].volume = float(asks[i][1]) if type(asks[i][1]) != float else asks[i][1] 88 | else: 89 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 90 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 91 | raw)) 92 | 93 | return l2_depth 94 | 95 | @classmethod 96 | def parse_trade(cls, instmt, raw): 97 | """ 98 | :param instmt: Instrument 99 | :param raw: Raw data in JSON 100 | :return: 101 | """ 102 | trade = Trade() 103 | trade_id = raw[0] 104 | trade_price = float(raw[1]) 105 | trade_volume = float(raw[2]) 106 | date_time = raw[3] 107 | trade_side = raw[4] 108 | 109 | # trade.date_time = date_time 110 | trade.trade_id = str(trade_id) 111 | trade.trade_price = trade_price 112 | trade.trade_volume = trade_volume 113 | trade.trade_side = Trade.parse_side(trade_side) 114 | 115 | return trade 116 | 117 | 118 | class ExchGwOkexSpot(ExchangeGateway): 119 | """ 120 | Exchange gateway 121 | """ 122 | def __init__(self, db_clients): 123 | """ 124 | Constructor 125 | :param db_client: Database client 126 | """ 127 | ExchangeGateway.__init__(self, ExchGwApiOkexSpotWs(), db_clients) 128 | 129 | @classmethod 130 | def get_exchange_name(cls): 131 | """ 132 | Get exchange name 133 | :return: Exchange name string 134 | """ 135 | return 'Okex' 136 | 137 | def on_open_handler(self, instmt, ws): 138 | """ 139 | Socket on open handler 140 | :param instmt: Instrument 141 | :param ws: Web socket 142 | """ 143 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 144 | (instmt.get_instmt_code(), instmt.get_exchange_name())) 145 | if not instmt.get_subscribed(): 146 | instmt_code_split = instmt.get_instmt_code().split('_') 147 | if len(instmt_code_split) == 2: 148 | # Future instruments 149 | instmt.set_order_book_channel_id("ok_sub_spot_%s_%s_depth_5" % \ 150 | (instmt_code_split[0].lower(), 151 | instmt_code_split[1].lower())) 152 | instmt.set_trades_channel_id("ok_sub_spot_%s_%s_deals" % \ 153 | (instmt_code_split[0].lower(), 154 | instmt_code_split[1].lower())) 155 | else: 156 | # Spot instruments 157 | instmt.set_order_book_channel_id("ok_sub_spot_%s_depth_5" % instmt.get_instmt_code().lower()) 158 | instmt.set_trades_channel_id("ok_sub_spot_%s_deals" % instmt.get_instmt_code().lower()) 159 | 160 | ws.send(self.api_socket.get_order_book_subscription_string(instmt)) 161 | # ws.send(self.api_socket.get_trades_subscription_string(instmt)) 162 | instmt.set_subscribed(True) 163 | 164 | def on_close_handler(self, instmt, ws): 165 | """ 166 | Socket on close handler 167 | :param instmt: Instrument 168 | :param ws: Web socket 169 | """ 170 | Logger.info(self.__class__.__name__, "Instrument %s is unsubscribed in channel %s" % \ 171 | (instmt.get_instmt_code(), instmt.get_exchange_name())) 172 | instmt.set_subscribed(False) 173 | 174 | def on_message_handler(self, instmt, messages): 175 | """ 176 | Incoming message handler 177 | :param instmt: Instrument 178 | :param message: Message 179 | """ 180 | for message in messages: 181 | keys = message.keys() 182 | # print(keys) 183 | if 'channel' in keys: 184 | if 'data' in keys: 185 | if message['channel'] == instmt.get_order_book_channel_id(): 186 | data = message['data'] 187 | l2_depth = self.api_socket.parse_l2_depth(instmt, data) 188 | if l2_depth is not None: 189 | # Insert only if the first 5 levels are different 190 | # if l2_depth is not None and instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): 191 | instmt.set_prev_l2_depth(instmt.get_l2_depth()) 192 | instmt.set_l2_depth(l2_depth) 193 | instmt.incr_order_book_id() 194 | self.insert_order_book(instmt) 195 | 196 | elif message['channel'] == instmt.get_trades_channel_id(): 197 | for trade_raw in message['data']: 198 | trade = self.api_socket.parse_trade(instmt, trade_raw) 199 | if trade.trade_id != instmt.get_exch_trade_id(): 200 | instmt.incr_trade_id() 201 | instmt.set_exch_trade_id(trade.trade_id) 202 | self.insert_trade(instmt, trade) 203 | 204 | elif 'success' in keys: 205 | Logger.info(self.__class__.__name__, "Subscription to channel %s is %s" \ 206 | % (message['channel'], message['success'])) 207 | else: 208 | Logger.info(self.__class__.__name__, ' - ' + json.dumps(message)) 209 | 210 | def start(self, instmt): 211 | """ 212 | Start the exchange gateway 213 | :param instmt: Instrument 214 | :return List of threads 215 | """ 216 | instmt.set_prev_l2_depth(L2Depth(20)) 217 | instmt.set_l2_depth(L2Depth(20)) 218 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 219 | instmt.get_instmt_name())) 220 | self.init_instmt_snapshot_table(instmt) 221 | return [self.api_socket.connect(self.api_socket.get_link(), 222 | on_message_handler=partial(self.on_message_handler, instmt), 223 | on_open_handler=partial(self.on_open_handler, instmt), 224 | on_close_handler=partial(self.on_close_handler, instmt))] 225 | 226 | 227 | if __name__ == '__main__': 228 | exchange_name = 'Okex' 229 | instmt_name = 'BCHBTC' 230 | instmt_code = 'BCH_BTC' 231 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 232 | db_client = SqlClientTemplate() 233 | Logger.init_log() 234 | exch = ExchGwOkexSpot([db_client]) 235 | td = exch.start(instmt) 236 | -------------------------------------------------------------------------------- /befh/exchanges/ws_template.py: -------------------------------------------------------------------------------- 1 | from befh.ws_api_socket import WebSocketApiClient 2 | from befh.market_data import L2Depth, Trade 3 | from befh.exchanges.gateway import ExchangeGateway 4 | from befh.instrument import Instrument 5 | from befh.clients.sql_template import SqlClientTemplate 6 | from befh.util import Logger 7 | import time 8 | import threading 9 | import json 10 | from functools import partial 11 | from datetime import datetime 12 | 13 | 14 | class ExchGwApiTemplate(WebSocketApiClient): 15 | """ 16 | Exchange socket 17 | """ 18 | def __init__(self): 19 | """ 20 | Constructor 21 | """ 22 | WebSocketApiClient.__init__(self, 'Template') 23 | 24 | @classmethod 25 | def get_order_book_timestamp_field_name(cls): 26 | return 'timestamp' 27 | 28 | @classmethod 29 | def get_trades_timestamp_field_name(cls): 30 | return 'timestamp' 31 | 32 | @classmethod 33 | def get_bids_field_name(cls): 34 | return 'bids' 35 | 36 | @classmethod 37 | def get_asks_field_name(cls): 38 | return 'asks' 39 | 40 | @classmethod 41 | def get_trade_side_field_name(cls): 42 | return 'side' 43 | 44 | @classmethod 45 | def get_trade_id_field_name(cls): 46 | return 'trdMatchID' 47 | 48 | @classmethod 49 | def get_trade_price_field_name(cls): 50 | return 'price' 51 | 52 | @classmethod 53 | def get_trade_volume_field_name(cls): 54 | return 'size' 55 | 56 | @classmethod 57 | def get_link(cls): 58 | return 'wss://www.bitmex.com/realtime' 59 | 60 | @classmethod 61 | def get_order_book_subscription_string(cls, instmt): 62 | return json.dumps({"op":"subscribe", "args": ["orderBook10:%s" % instmt.get_instmt_code()]}) 63 | 64 | @classmethod 65 | def get_trades_subscription_string(cls, instmt): 66 | return json.dumps({"op":"subscribe", "args": ["trade:%s" % instmt.get_instmt_code()]}) 67 | 68 | @classmethod 69 | def parse_l2_depth(cls, instmt, raw): 70 | """ 71 | Parse raw data to L2 depth 72 | :param instmt: Instrument 73 | :param raw: Raw data in JSON 74 | """ 75 | l2_depth = instmt.get_l2_depth() 76 | keys = list(raw.keys()) 77 | if cls.get_order_book_timestamp_field_name() in keys and \ 78 | cls.get_bids_field_name() in keys and \ 79 | cls.get_asks_field_name() in keys: 80 | 81 | # Date time 82 | timestamp = raw[cls.get_order_book_timestamp_field_name()] 83 | timestamp = timestamp.replace('T', ' ').replace('Z', '').replace('-' , '') 84 | l2_depth.date_time = timestamp 85 | 86 | # Bids 87 | bids = raw[cls.get_bids_field_name()] 88 | bids = sorted(bids, key=lambda x: x[0], reverse=True) 89 | for i in range(0, len(bids)): 90 | l2_depth.bids[i].price = float(bids[i][0]) if not isinstance(bids[i][0], float) else bids[i][0] 91 | l2_depth.bids[i].volume = float(bids[i][1]) if not isinstance(bids[i][1], float) else bids[i][1] 92 | 93 | # Asks 94 | asks = raw[cls.get_asks_field_name()] 95 | asks = sorted(asks, key=lambda x: x[0]) 96 | for i in range(0, len(asks)): 97 | l2_depth.asks[i].price = float(asks[i][0]) if not isinstance(asks[i][0], float) else asks[i][0] 98 | l2_depth.asks[i].volume = float(asks[i][1]) if not isinstance(asks[i][1], float) else asks[i][1] 99 | else: 100 | raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \ 101 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 102 | raw)) 103 | 104 | return l2_depth 105 | 106 | @classmethod 107 | def parse_trade(cls, instmt, raw): 108 | """ 109 | :param instmt: Instrument 110 | :param raw: Raw data in JSON 111 | :return: 112 | """ 113 | trade = Trade() 114 | keys = list(raw.keys()) 115 | 116 | if cls.get_trades_timestamp_field_name() in keys and \ 117 | cls.get_trade_id_field_name() in keys and \ 118 | cls.get_trade_side_field_name() in keys and \ 119 | cls.get_trade_price_field_name() in keys and \ 120 | cls.get_trade_volume_field_name() in keys: 121 | 122 | # Date time 123 | timestamp = raw[cls.get_trades_timestamp_field_name()] 124 | timestamp = timestamp.replace('T', ' ').replace('Z', '').replace('-' , '') 125 | trade.date_time = timestamp 126 | 127 | # Trade side 128 | trade.trade_side = Trade.parse_side(raw[cls.get_trade_side_field_name()]) 129 | 130 | # Trade id 131 | trade.trade_id = raw[cls.get_trade_id_field_name()] 132 | 133 | # Trade price 134 | trade.trade_price = raw[cls.get_trade_price_field_name()] 135 | 136 | # Trade volume 137 | trade.trade_volume = raw[cls.get_trade_volume_field_name()] 138 | else: 139 | raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \ 140 | (instmt.get_exchange_name(), instmt.get_instmt_name(), \ 141 | raw)) 142 | 143 | return trade 144 | 145 | 146 | class ExchGwTemplate(ExchangeGateway): 147 | """ 148 | Exchange gateway 149 | """ 150 | def __init__(self, db_clients): 151 | """ 152 | Constructor 153 | :param db_client: Database client 154 | """ 155 | ExchangeGateway.__init__(self, ExchGwApiTemplate(), db_clients) 156 | 157 | @classmethod 158 | def get_exchange_name(cls): 159 | """ 160 | Get exchange name 161 | :return: Exchange name string 162 | """ 163 | return 'Template' 164 | 165 | def on_open_handler(self, instmt, ws): 166 | """ 167 | Socket on open handler 168 | :param instmt: Instrument 169 | :param ws: Web socket 170 | """ 171 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 172 | (instmt.get_instmt_code(), instmt.get_exchange_name())) 173 | if not instmt.get_subscribed(): 174 | ws.send(self.api_socket.get_order_book_subscription_string(instmt)) 175 | ws.send(self.api_socket.get_trades_subscription_string(instmt)) 176 | instmt.set_subscribed(True) 177 | 178 | def on_close_handler(self, instmt, ws): 179 | """ 180 | Socket on close handler 181 | :param instmt: Instrument 182 | :param ws: Web socket 183 | """ 184 | Logger.info(self.__class__.__name__, "Instrument %s is subscribed in channel %s" % \ 185 | (instmt.get_instmt_code(), instmt.get_exchange_name())) 186 | instmt.set_subscribed(False) 187 | 188 | def on_message_handler(self, instmt, message): 189 | """ 190 | Incoming message handler 191 | :param instmt: Instrument 192 | :param message: Message 193 | """ 194 | keys = message.keys() 195 | if 'info' in keys: 196 | Logger.info(self.__class__.__name__, message['info']) 197 | elif 'subscribe' in keys: 198 | Logger.info(self.__class__.__name__, 'Subscription of %s is %s' % \ 199 | (message['request']['args'], \ 200 | 'successful' if message['success'] else 'failed')) 201 | elif 'table' in keys: 202 | if message['table'] == 'trade': 203 | for trade_raw in message['data']: 204 | if trade_raw["symbol"] == instmt.get_instmt_code(): 205 | # Filter out the initial subscriptions 206 | trade = self.api_socket.parse_trade(instmt, trade_raw) 207 | if trade.trade_id != instmt.get_exch_trade_id(): 208 | instmt.incr_trade_id() 209 | instmt.set_exch_trade_id(trade.trade_id) 210 | self.insert_trade(instmt, trade) 211 | elif message['table'] == 'orderBook10': 212 | for data in message['data']: 213 | if data["symbol"] == instmt.get_instmt_code(): 214 | instmt.set_prev_l2_depth(instmt.get_l2_depth().copy()) 215 | self.api_socket.parse_l2_depth(instmt, data) 216 | if instmt.get_l2_depth().is_diff(instmt.get_prev_l2_depth()): 217 | instmt.incr_order_book_id() 218 | self.insert_order_book(instmt) 219 | else: 220 | Logger.info(self.__class__.__name__, json.dumps(message,indent=2)) 221 | else: 222 | Logger.error(self.__class__.__name__, "Unrecognised message:\n" + json.dumps(message)) 223 | 224 | def start(self, instmt): 225 | """ 226 | Start the exchange gateway 227 | :param instmt: Instrument 228 | :return List of threads 229 | """ 230 | instmt.set_l2_depth(L2Depth(10)) 231 | instmt.set_prev_l2_depth(L2Depth(10)) 232 | instmt.set_instmt_snapshot_table_name(self.get_instmt_snapshot_table_name(instmt.get_exchange_name(), 233 | instmt.get_instmt_name())) 234 | self.init_instmt_snapshot_table(instmt) 235 | return [self.api_socket.connect(self.api_socket.get_link(), 236 | on_message_handler=partial(self.on_message_handler, instmt), 237 | on_open_handler=partial(self.on_open_handler, instmt), 238 | on_close_handler=partial(self.on_close_handler, instmt))] 239 | 240 | 241 | if __name__ == '__main__': 242 | exchange_name = 'Template' 243 | instmt_name = 'XBTUSD' 244 | instmt_code = 'XBTH17' 245 | instmt = Instrument(exchange_name, instmt_name, instmt_code) 246 | db_client = SqlClientTemplate() 247 | Logger.init_log() 248 | exch = ExchGwTemplate([db_client]) 249 | td = exch.start(instmt) 250 | 251 | -------------------------------------------------------------------------------- /befh/instrument.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | 4 | class Instrument(object): 5 | def __init__(self, 6 | exchange_name, 7 | instmt_name, 8 | instmt_code, 9 | **param): 10 | """ 11 | Constructor 12 | :param exchange: Exchange name 13 | :param instmt_code: Instrument code 14 | :param param: Options parameters, e.g. restful_order_book_link 15 | :return: 16 | """ 17 | self.exchange_name = exchange_name 18 | self.instmt_name = instmt_name 19 | self.instmt_code = instmt_code 20 | self.instmt_snapshot_table_name = '' 21 | self.order_book_id = 0 22 | self.trade_id = 0 23 | self.exch_trade_id = '0' 24 | self.subscribed = False 25 | self.recovered = True 26 | self.l2_depth = None 27 | self.prev_l2_depth = None 28 | self.last_trade = None 29 | self.order_book_channel_id = '' 30 | self.trades_channel_id = '' 31 | self.realtime_order_book_prices = [{}, {}] # Only for BitMEX to use 32 | self.realtime_order_book_ids = [{}, {}] # Only for BitMEX to use 33 | 34 | def copy(self, obj): 35 | """ 36 | Copy constructor 37 | """ 38 | self.exchange_name = obj.exchange_name 39 | self.instmt_name = obj.instmt_name 40 | self.instmt_code = obj.instmt_code 41 | self.instmt_snapshot_table_name = obj.instmt_snapshot_table_name 42 | self.order_book_id = obj.order_book_id 43 | self.trade_id = obj.trade_id 44 | self.exch_trade_id = obj.exch_trade_id 45 | self.subscribed = obj.subscribed 46 | self.recovered = obj.recovered 47 | self.l2_depth = obj.l2_depth 48 | self.prev_l2_depth = obj.prev_l2_depth 49 | self.last_trade = obj.last_trade 50 | self.order_book_channel_id = obj.order_book_channel_id 51 | self.trades_channel_id = obj.trades_channel_id 52 | self.realtime_order_book_prices = copy.deepcopy( 53 | obj.realtime_order_book_prices) 54 | self.realtime_order_book_ids = copy.deepcopy( 55 | obj.realtime_order_book_ids) 56 | 57 | def get_exchange_name(self): 58 | return self.exchange_name 59 | 60 | def get_instmt_name(self): 61 | return self.instmt_name 62 | 63 | def get_instmt_code(self): 64 | return self.instmt_code 65 | 66 | def get_instmt_snapshot_table_name(self): 67 | return self.instmt_snapshot_table_name 68 | 69 | def get_order_book_id(self): 70 | return self.order_book_id 71 | 72 | def get_trade_id(self): 73 | return self.trade_id 74 | 75 | def get_exch_trade_id(self): 76 | return self.exch_trade_id 77 | 78 | def get_subscribed(self): 79 | return self.subscribed 80 | 81 | def get_recovered(self): 82 | return self.recovered 83 | 84 | def get_l2_depth(self): 85 | return self.l2_depth 86 | 87 | def get_prev_l2_depth(self): 88 | return self.prev_l2_depth 89 | 90 | def get_last_trade(self): 91 | return self.last_trade 92 | 93 | def get_order_book_channel_id(self): 94 | return self.order_book_channel_id 95 | 96 | def get_trades_channel_id(self): 97 | return self.trades_channel_id 98 | 99 | def set_trade_id(self, trade_id): 100 | self.trade_id = trade_id 101 | 102 | def set_instmt_snapshot_table_name(self, instmt_snapshot_table_name): 103 | self.instmt_snapshot_table_name = instmt_snapshot_table_name 104 | 105 | def set_trades_channel_id(self, trades_channel_id): 106 | self.trades_channel_id = trades_channel_id 107 | 108 | def set_order_book_id(self, order_book_id): 109 | self.order_book_id = order_book_id 110 | 111 | def set_exch_trade_id(self, exch_trade_id): 112 | assert isinstance(exch_trade_id, str), \ 113 | "exch_trade_id (%s) = %s" % (type(exch_trade_id), exch_trade_id) 114 | self.exch_trade_id = exch_trade_id 115 | 116 | def set_subscribed(self, subscribed): 117 | self.subscribed = subscribed 118 | 119 | def set_recovered(self, recovered): 120 | self.recovered = recovered 121 | 122 | def set_l2_depth(self, l2_depth): 123 | self.l2_depth = l2_depth 124 | 125 | def set_prev_l2_depth(self, prev_l2_depth): 126 | self.prev_l2_depth = prev_l2_depth 127 | 128 | def set_last_trade(self, trade): 129 | self.last_trade = trade 130 | 131 | def set_order_book_channel_id(self, order_book_channel_id): 132 | self.order_book_channel_id = order_book_channel_id 133 | 134 | def incr_order_book_id(self): 135 | assert isinstance(self.order_book_id, int), "Type(self.order_book_id) = %s" % type(self.order_book_id) 136 | self.order_book_id += 1 137 | 138 | def incr_trade_id(self): 139 | assert isinstance(self.trade_id, int), "Type(self.trade_id) = %s" % type(self.trade_id) 140 | self.trade_id += 1 141 | 142 | -------------------------------------------------------------------------------- /befh/market_data.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | from pprint import pformat 3 | from datetime import datetime 4 | import copy 5 | 6 | 7 | class MarketDataBase: 8 | """ 9 | Abstract class of a market data 10 | """ 11 | class Side: 12 | NONE = 0 13 | BUY = 1 14 | SELL = 2 15 | 16 | class Depth(object): 17 | def __init__(self, price=0.0, count=0, volume=0.0): 18 | """ 19 | Constructor 20 | """ 21 | self.price = price 22 | self.count = count 23 | self.volume = volume 24 | 25 | def copy(self): 26 | return copy.deepcopy(self) 27 | 28 | def __repr__(self): 29 | return pformat(vars(self)) 30 | 31 | @staticmethod 32 | def parse_side(value): 33 | """ 34 | Decode the value to Side (BUY/SELL) 35 | :param value: Integer or string 36 | :return: Side (NONE, BUY, SELL) 37 | """ 38 | if type(value) != int: 39 | value = value.lower() 40 | if value == 'buy' or value == 'bid' or value == 'b': 41 | return MarketDataBase.Side.BUY 42 | elif value == 'sell' or value == 'ask' or value == 's': 43 | return MarketDataBase.Side.SELL 44 | else: 45 | return MarketDataBase.Side.NONE 46 | 47 | if value == 1: 48 | return MarketDataBase.Side.BUY 49 | elif value == 2: 50 | return MarketDataBase.Side.SELL 51 | else: 52 | raise Exception("Cannot parse the side (%s)" % value) 53 | 54 | def __init__(self): 55 | """ 56 | Constructor 57 | """ 58 | pass 59 | 60 | class L2Depth(MarketDataBase): 61 | """ 62 | L2 price depth. Container of date, time, bid and ask up to 5 levels 63 | """ 64 | def __init__(self, depth=5): 65 | """ 66 | Constructor 67 | :param depth: Number of depth 68 | """ 69 | MarketDataBase.__init__(self) 70 | self.date_time = datetime(2000, 1, 1, 0, 0, 0).strftime("%Y%m%d %H:%M:%S.%f") 71 | self.depth = depth 72 | self.bids = [MarketDataBase.Depth() for i in range(0, self.depth)] 73 | self.asks = [MarketDataBase.Depth() for i in range(0, self.depth)] 74 | 75 | @staticmethod 76 | def columns(): 77 | """ 78 | Return static columns names 79 | """ 80 | return ['date_time', 81 | 'b1', 'b2', 'b3', 'b4', 'b5', 82 | 'a1', 'a2', 'a3', 'a4', 'a5', 83 | 'bq1', 'bq2', 'bq3', 'bq4', 'bq5', 84 | 'aq1', 'aq2', 'aq3', 'aq4', 'aq5'] 85 | 86 | @staticmethod 87 | def types(): 88 | """ 89 | Return static column types 90 | """ 91 | return ['varchar(25)'] + \ 92 | ['decimal(10,5)'] * 10 + \ 93 | ['decimal(20,8)'] * 10 94 | 95 | def values(self): 96 | """ 97 | Return values in a list 98 | """ 99 | if self.depth == 5: 100 | return [self.date_time] + \ 101 | [b.price for b in self.bids] + \ 102 | [a.price for a in self.asks] + \ 103 | [b.volume for b in self.bids] + \ 104 | [a.volume for a in self.asks] 105 | else: 106 | return [self.date_time] + \ 107 | [b.price for b in self.bids[0:5]] + \ 108 | [a.price for a in self.asks[0:5]] + \ 109 | [b.volume for b in self.bids[0:5]] + \ 110 | [a.volume for a in self.asks[0:5]] 111 | 112 | def sort_bids(self): 113 | """ 114 | Sorting bids 115 | :return: 116 | """ 117 | self.bids.sort(key=lambda x:x.price, reverse=True) 118 | if len(self.bids) > self.depth: 119 | self.bids = self.bids[0:self.depth] 120 | 121 | def sort_asks(self): 122 | """ 123 | Sorting bids 124 | :return: 125 | """ 126 | self.asks.sort(key=lambda x:x.price) 127 | if len(self.asks) > self.depth: 128 | self.asks = self.asks[0:self.depth] 129 | 130 | def copy(self): 131 | """ 132 | Copy 133 | """ 134 | ret = L2Depth(depth=self.depth) 135 | ret.date_time = self.date_time 136 | ret.bids = [e.copy() for e in self.bids] 137 | ret.asks = [e.copy() for e in self.asks] 138 | return ret 139 | 140 | def is_diff(self, l2_depth): 141 | """ 142 | Compare the first 5 price depth 143 | :param l2_depth: Another L2Depth object 144 | :return: True if they are different 145 | """ 146 | # return True 147 | for i in range(0, 5): 148 | if abs(self.bids[i].price - l2_depth.bids[i].price) > 1e-09 or \ 149 | abs(self.bids[i].volume - l2_depth.bids[i].volume) > 1e-09: 150 | return True 151 | elif abs(self.asks[i].price - l2_depth.asks[i].price) > 1e-09 or \ 152 | abs(self.asks[i].volume - l2_depth.asks[i].volume) > 1e-09: 153 | return True 154 | return False 155 | 156 | def __repr__(self): 157 | return pformat(vars(self)) 158 | 159 | class Trade(MarketDataBase): 160 | """ 161 | Trade. Container of date, time, trade price, volume and side. 162 | """ 163 | def __init__(self): 164 | """ 165 | Constructor 166 | :param exch: Exchange name 167 | :param instmt: Instrument name 168 | :param default_format: Default date time format 169 | """ 170 | MarketDataBase.__init__(self) 171 | self.date_time = datetime(2000, 1, 1, 0, 0, 0).strftime("%Y%m%d %H:%M:%S.%f") 172 | self.trade_id = '' 173 | self.trade_price = 0.0 174 | self.trade_volume = 0.0 175 | self.trade_side = MarketDataBase.Side.NONE 176 | self.update_date_time = datetime.utcnow() 177 | 178 | 179 | @staticmethod 180 | def columns(): 181 | """ 182 | Return static columns names 183 | """ 184 | return ['date_time', 'trade_id', 'trade_price', 'trade_volume', 'trade_side'] 185 | 186 | @staticmethod 187 | def types(): 188 | """ 189 | Return static column types 190 | """ 191 | return ['varchar(25)', 'text', 'decimal(10,5)', 'decimal(20,8)', 'int'] 192 | 193 | def values(self): 194 | """ 195 | Return values in a list 196 | """ 197 | return [self.date_time] + \ 198 | [self.trade_id] + [self.trade_price] + [self.trade_volume] + [self.trade_side] 199 | 200 | def __repr__(self): 201 | return pformat(vars(self)) 202 | 203 | class Snapshot(MarketDataBase): 204 | """ 205 | Market price snapshot 206 | """ 207 | class UpdateType: 208 | NONE = 0 209 | ORDER_BOOK = 1 210 | TRADES = 2 211 | 212 | def __init__(self, exchange, instmt_name): 213 | """ 214 | Constructor 215 | :param exch: Exchange name 216 | :param instmt: Instrument name 217 | :param default_format: Default date time format 218 | """ 219 | MarketDataBase.__init__(self) 220 | 221 | @staticmethod 222 | def columns(is_name=True): 223 | """ 224 | Return static columns names 225 | """ 226 | if is_name: 227 | return ['exchange', 'instmt', 228 | 'trade_px', 'trade_volume', 229 | 'b1', 'b2', 'b3', 'b4', 'b5', 230 | 'a1', 'a2', 'a3', 'a4', 'a5', 231 | 'bq1', 'bq2', 'bq3', 'bq4', 'bq5', 232 | 'aq1', 'aq2', 'aq3', 'aq4', 'aq5', 233 | 'order_date_time', 'trades_date_time', 'update_type'] 234 | else: 235 | return ['trade_px', 'trade_volume', 236 | 'b1', 'b2', 'b3', 'b4', 'b5', 237 | 'a1', 'a2', 'a3', 'a4', 'a5', 238 | 'bq1', 'bq2', 'bq3', 'bq4', 'bq5', 239 | 'aq1', 'aq2', 'aq3', 'aq4', 'aq5', 240 | 'order_date_time', 'trades_date_time', 'update_type'] 241 | 242 | @staticmethod 243 | def types(is_name=True): 244 | """ 245 | Return static column types 246 | """ 247 | if is_name: 248 | return ['varchar(20)', 'varchar(20)', 'decimal(20,8)', 'decimal(20,8)'] + \ 249 | ['decimal(20,8)'] * 10 + \ 250 | ['decimal(20,8)'] * 10 + \ 251 | ['varchar(25)', 'varchar(25)', 'int'] 252 | else: 253 | return ['decimal(20,8)', 'decimal(20,8)'] + \ 254 | ['decimal(20,8)'] * 10 + \ 255 | ['decimal(20,8)'] * 10 + \ 256 | ['varchar(25)', 'varchar(25)', 'int'] 257 | 258 | 259 | @staticmethod 260 | def values(exchange_name='', instmt_name='', l2_depth=None, last_trade=None, update_type=UpdateType.NONE): 261 | """ 262 | Return values in a list 263 | """ 264 | assert l2_depth is not None and last_trade is not None, "L2 depth and last trade must not be none." 265 | return ([exchange_name] if exchange_name else []) + \ 266 | ([instmt_name] if instmt_name else []) + \ 267 | [last_trade.trade_price, last_trade.trade_volume] + \ 268 | [b.price for b in l2_depth.bids[0:5]] + \ 269 | [a.price for a in l2_depth.asks[0:5]] + \ 270 | [b.volume for b in l2_depth.bids[0:5]] + \ 271 | [a.volume for a in l2_depth.asks[0:5]] + \ 272 | [l2_depth.date_time, last_trade.date_time, update_type] 273 | 274 | -------------------------------------------------------------------------------- /befh/restful_api_socket.py: -------------------------------------------------------------------------------- 1 | from befh.api_socket import ApiSocket 2 | import requests 3 | 4 | # try: 5 | # import urllib.request as urlrequest 6 | # except ImportError: 7 | # import urllib as urlrequest 8 | 9 | # import json 10 | # import ssl 11 | 12 | class RESTfulApiSocket(ApiSocket): 13 | """ 14 | Generic REST API call 15 | """ 16 | DEFAULT_URLOPEN_TIMEOUT = 10 17 | 18 | def __init__(self): 19 | """ 20 | Constructor 21 | """ 22 | ApiSocket.__init__(self) 23 | 24 | @classmethod 25 | def request(cls, url, verify_cert=True): 26 | """ 27 | Web request 28 | :param: url: The url link 29 | :return JSON object 30 | """ 31 | # try: 32 | res = requests.request("GET", url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=RESTfulApiSocket.DEFAULT_URLOPEN_TIMEOUT) 33 | if res.status_code >= 400: 34 | raise Exception('request failed! {} {}'.format(res.status_code, url)) 35 | else: 36 | res = res.json() 37 | return res 38 | # except expression as identifier: 39 | # return {} 40 | 41 | 42 | # req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) 43 | # # res = urlrequest.urlopen(url) 44 | 45 | # if verify_cert: 46 | # res = urlrequest.urlopen( 47 | # req, 48 | # timeout=RESTfulApiSocket.DEFAULT_URLOPEN_TIMEOUT) 49 | # else: 50 | # res = urlrequest.urlopen( 51 | # req, 52 | # context=ssl._create_unverified_context(), 53 | # timeout=RESTfulApiSocket.DEFAULT_URLOPEN_TIMEOUT) 54 | # try: 55 | # res = json.loads(res.read().decode('utf8')) 56 | # return res 57 | # except: 58 | # return {} 59 | 60 | @classmethod 61 | def parse_l2_depth(cls, instmt, raw): 62 | """ 63 | Parse raw data to L2 depth 64 | :param instmt: Instrument 65 | :param raw: Raw data in JSON 66 | """ 67 | return None 68 | 69 | @classmethod 70 | def parse_trade(cls, instmt, raw): 71 | """ 72 | :param instmt: Instrument 73 | :param raw: Raw data in JSON 74 | :return: 75 | """ 76 | return None 77 | 78 | @classmethod 79 | def get_order_book(cls, instmt): 80 | """ 81 | Get order book 82 | :param instmt: Instrument 83 | :return: Object L2Depth 84 | """ 85 | return None 86 | 87 | @classmethod 88 | def get_trades(cls, instmt, trade_id): 89 | """ 90 | Get trades 91 | :param instmt: Instrument 92 | :param trade_id: Trade id 93 | :return: List of trades 94 | """ 95 | return None 96 | 97 | -------------------------------------------------------------------------------- /befh/subscription_manager.py: -------------------------------------------------------------------------------- 1 | from befh.instrument import Instrument 2 | try: 3 | import ConfigParser 4 | except ImportError: 5 | import configparser as ConfigParser 6 | 7 | class SubscriptionManager: 8 | def __init__(self, config_path): 9 | """ 10 | Constructor 11 | """ 12 | self.config = ConfigParser.ConfigParser() 13 | self.config.read(config_path) 14 | 15 | def get_instmt_ids(self): 16 | """ 17 | Return all the instrument ids 18 | """ 19 | return self.config.sections() 20 | 21 | def get_instrument(self, instmt_id): 22 | """ 23 | Return the instrument object by instrument id 24 | :param instmt_id: Instrument ID 25 | :return Instrument object 26 | """ 27 | exchange_name = self.config.get(instmt_id, 'exchange') 28 | instmt_name = self.config.get(instmt_id, 'instmt_name') 29 | instmt_code = self.config.get(instmt_id, 'instmt_code') 30 | enabled = int(self.config.get(instmt_id, 'enabled')) 31 | params = dict(self.config.items(instmt_id)) 32 | del params['exchange'] 33 | del params['instmt_name'] 34 | del params['instmt_code'] 35 | del params['enabled'] 36 | 37 | if enabled == 1: 38 | return Instrument(exchange_name, instmt_name, instmt_code, **params) 39 | else: 40 | return None 41 | 42 | def get_subscriptions(self): 43 | """ 44 | Get all the subscriptions from the configuration file 45 | :return List of instrument objects 46 | """ 47 | instmts = [self.get_instrument(inst) for inst in self.get_instmt_ids()] 48 | return [instmt for instmt in instmts if instmt is not None] 49 | 50 | -------------------------------------------------------------------------------- /befh/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/befh/test/__init__.py -------------------------------------------------------------------------------- /befh/test/test_file_client.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | import unittest 4 | import os 5 | from file_client import FileClient 6 | from util import Logger 7 | 8 | path = 'test\\' 9 | table_name = 'test_query' 10 | columns = ['date', 'time', 'k', 'v', 'v2'] 11 | types = ['text', 'text', 'int PRIMARY KEY', 'text', 'decimal(10,5)'] 12 | 13 | class SqliteClientTest(unittest.TestCase): 14 | @classmethod 15 | def setUpClass(cls): 16 | Logger.init_log() 17 | cls.db_client = FileClient(dir=path) 18 | cls.db_client.connect() 19 | 20 | @classmethod 21 | def tearDownClass(cls): 22 | cls.db_client.close() 23 | os.remove(path + table_name + ".csv") 24 | pass 25 | 26 | def test_query(self): 27 | # Check table creation 28 | self.assertTrue(self.db_client.create(table_name, columns, types)) 29 | 30 | # Check table insertion 31 | self.assertTrue(self.db_client.insert( 32 | table_name, 33 | columns, 34 | ['20161026','10:00:00.000000',1,'AbC',10.3])) 35 | self.assertTrue(self.db_client.insert( 36 | table_name, 37 | columns, 38 | ['20161026','10:00:01.000000',2,'AbCD',10.4])) 39 | self.assertTrue(self.db_client.insert( 40 | table_name, 41 | columns, 42 | ['20161026','10:00:02.000000',3,'Efgh',10.5])) 43 | 44 | # # Fetch the whole table 45 | row = self.db_client.select(table=table_name) 46 | self.assertEqual(len(row), 3) 47 | self.assertEqual(row[0][0], "20161026") 48 | self.assertEqual(row[0][1], "10:00:00.000000") 49 | self.assertEqual(row[0][2], 1) 50 | self.assertEqual(row[0][3], 'AbC') 51 | self.assertEqual(row[0][4], 10.3) 52 | self.assertEqual(row[1][0], "20161026") 53 | self.assertEqual(row[1][1], "10:00:01.000000") 54 | self.assertEqual(row[1][2], 2) 55 | self.assertEqual(row[1][4], 10.4) 56 | self.assertEqual(row[1][3], 'AbCD') 57 | self.assertEqual(row[2][0], "20161026") 58 | self.assertEqual(row[2][1], "10:00:02.000000") 59 | self.assertEqual(row[2][2], 3) 60 | self.assertEqual(row[2][3], 'Efgh') 61 | self.assertEqual(row[2][4], 10.5) 62 | 63 | # Fetch with condition 64 | row = self.db_client.select(table=table_name, condition="k=2") 65 | self.assertEqual(len(row), 1) 66 | self.assertEqual(row[0][0], "20161026") 67 | self.assertEqual(row[0][1], "10:00:01.000000") 68 | self.assertEqual(row[0][2], 2) 69 | self.assertEqual(row[0][3], 'AbCD') 70 | self.assertEqual(row[0][4], 10.4) 71 | 72 | # # Fetch with ordering 73 | row = self.db_client.select(table=table_name, orderby="k desc") 74 | self.assertEqual(len(row), 3) 75 | self.assertEqual(row[2][0], "20161026") 76 | self.assertEqual(row[2][1], "10:00:00.000000") 77 | self.assertEqual(row[2][2], 1) 78 | self.assertEqual(row[2][3], 'AbC') 79 | self.assertEqual(row[2][4], 10.3) 80 | self.assertEqual(row[1][0], "20161026") 81 | self.assertEqual(row[1][1], "10:00:01.000000") 82 | self.assertEqual(row[1][2], 2) 83 | self.assertEqual(row[1][3], 'AbCD') 84 | self.assertEqual(row[1][4], 10.4) 85 | self.assertEqual(row[0][0], "20161026") 86 | self.assertEqual(row[0][1], "10:00:02.000000") 87 | self.assertEqual(row[0][2], 3) 88 | self.assertEqual(row[0][3], 'Efgh') 89 | self.assertEqual(row[0][4], 10.5) 90 | 91 | # Fetch with limit 92 | row = self.db_client.select(table=table_name, limit=1) 93 | self.assertEqual(len(row), 1) 94 | self.assertEqual(row[0][0], "20161026") 95 | self.assertEqual(row[0][1], "10:00:00.000000") 96 | self.assertEqual(row[0][2], 1) 97 | self.assertEqual(row[0][3], 'AbC') 98 | self.assertEqual(row[0][4], 10.3) 99 | 100 | # Select with columns 101 | row = self.db_client.select(table=table_name, columns=['k', 'v']) 102 | self.assertEqual(len(row), 3) 103 | self.assertEqual(row[0][0], 1) 104 | self.assertEqual(row[0][1], 'AbC') 105 | self.assertEqual(row[1][0], 2) 106 | self.assertEqual(row[1][1], 'AbCD') 107 | self.assertEqual(row[2][0], 3) 108 | self.assertEqual(row[2][1], 'Efgh') 109 | 110 | # # Negative case 111 | self.assertTrue(not self.db_client.create(table_name, columns[1::], types)) 112 | self.assertTrue(not self.db_client.insert(table_name, columns, [])) 113 | 114 | if __name__ == '__main__': 115 | unittest.main() 116 | 117 | -------------------------------------------------------------------------------- /befh/test/test_kdbplus_client.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | import unittest 4 | from util import Logger 5 | from kdbplus_client import KdbPlusClient 6 | 7 | class SqliteClientTest(unittest.TestCase): 8 | @classmethod 9 | def setUpClass(cls): 10 | cls.db_client = KdbPlusClient() 11 | cls.db_client.connect(host='localhost', port=5000) 12 | 13 | @classmethod 14 | def tearDownClass(cls): 15 | pass 16 | 17 | def test_query(self): 18 | table_name = 'test_query' 19 | columns = ['k', 'date', 'time', 'v', 'v2'] 20 | types = ['int', 'text', 'text', 'text', 'decimal(10,5)'] 21 | 22 | # Check table creation 23 | self.assertTrue(self.db_client.create(table_name, columns, types, [0], is_ifnotexists=False)) 24 | 25 | # Check table insertion 26 | self.assertTrue(self.db_client.insert( 27 | table_name, 28 | columns, 29 | [1,'20161026','10:00:00.000000','AbC',10.3])) 30 | self.assertTrue(self.db_client.insert( 31 | table_name, 32 | columns, 33 | [2,'20161026','10:00:01.000000','AbCD',10.4])) 34 | self.assertTrue(self.db_client.insert( 35 | table_name, 36 | columns, 37 | [3,'20161026','10:00:02.000000','Efgh',10.5])) 38 | 39 | # Check table "IF NOT EXISTS" condition 40 | self.assertTrue(self.db_client.create(table_name, columns, types)) 41 | 42 | # Fetch the whole table 43 | row = self.db_client.select(table=table_name) 44 | self.assertEqual(len(row), 3) 45 | self.assertEqual(row[0][0], 1) 46 | self.assertEqual(row[0][1], "20161026") 47 | self.assertEqual(row[0][2], "10:00:00.000000") 48 | self.assertEqual(row[0][3], 'AbC') 49 | self.assertEqual(row[0][4], 10.3) 50 | self.assertEqual(row[1][0], 2) 51 | self.assertEqual(row[1][1], "20161026") 52 | self.assertEqual(row[1][2], "10:00:01.000000") 53 | self.assertEqual(row[1][3], 'AbCD') 54 | self.assertEqual(row[1][4], 10.4) 55 | self.assertEqual(row[2][0], 3) 56 | self.assertEqual(row[2][1], "20161026") 57 | self.assertEqual(row[2][2], "10:00:02.000000") 58 | self.assertEqual(row[2][3], 'Efgh') 59 | self.assertEqual(row[2][4], 10.5) 60 | 61 | # Fetch with condition 62 | row = self.db_client.select(table=table_name, condition="k=2") 63 | self.assertEqual(len(row), 1) 64 | self.assertEqual(row[0][0], 2) 65 | self.assertEqual(row[0][1], "20161026") 66 | self.assertEqual(row[0][2], "10:00:01.000000") 67 | self.assertEqual(row[0][3], 'AbCD') 68 | self.assertEqual(row[0][4], 10.4) 69 | 70 | # Fetch with ordering 71 | row = self.db_client.select(table=table_name, orderby="k desc") 72 | self.assertEqual(len(row), 3) 73 | self.assertEqual(row[2][0], 1) 74 | self.assertEqual(row[2][1], "20161026") 75 | self.assertEqual(row[2][2], "10:00:00.000000") 76 | self.assertEqual(row[2][3], 'AbC') 77 | self.assertEqual(row[2][4], 10.3) 78 | self.assertEqual(row[1][0], 2) 79 | self.assertEqual(row[1][1], "20161026") 80 | self.assertEqual(row[1][2], "10:00:01.000000") 81 | self.assertEqual(row[1][3], 'AbCD') 82 | self.assertEqual(row[1][4], 10.4) 83 | self.assertEqual(row[0][0], 3) 84 | self.assertEqual(row[0][1], "20161026") 85 | self.assertEqual(row[0][2], "10:00:02.000000") 86 | self.assertEqual(row[0][3], 'Efgh') 87 | self.assertEqual(row[0][4], 10.5) 88 | 89 | # Fetch with limit 90 | row = self.db_client.select(table=table_name, limit=1) 91 | self.assertEqual(len(row), 1) 92 | self.assertEqual(row[0][0], 1) 93 | self.assertEqual(row[0][1], "20161026") 94 | self.assertEqual(row[0][2], "10:00:00.000000") 95 | self.assertEqual(row[0][3], 'AbC') 96 | self.assertEqual(row[0][4], 10.3) 97 | 98 | # Check table insertion or replacement 99 | self.assertTrue(self.db_client.insert( 100 | table_name, 101 | columns, 102 | [2,'20161026','10:00:04.000000','NoNoNo',10.5], 103 | True)) 104 | 105 | # Fetch the whole table 106 | row = self.db_client.select(table=table_name) 107 | self.assertEqual(len(row), 3) 108 | self.assertEqual(row[0][0], 1) 109 | self.assertEqual(row[0][1], "20161026") 110 | self.assertEqual(row[0][2], "10:00:00.000000") 111 | self.assertEqual(row[0][3], 'AbC') 112 | self.assertEqual(row[0][4], 10.3) 113 | self.assertEqual(row[1][0], 2) 114 | self.assertEqual(row[1][1], "20161026") 115 | self.assertEqual(row[1][2], "10:00:04.000000") 116 | self.assertEqual(row[1][3], 'NoNoNo') 117 | self.assertEqual(row[1][4], 10.5) 118 | self.assertEqual(row[2][0], 3) 119 | self.assertEqual(row[2][1], "20161026") 120 | self.assertEqual(row[2][2], "10:00:02.000000") 121 | self.assertEqual(row[2][3], 'Efgh') 122 | self.assertEqual(row[2][4], 10.5) 123 | 124 | # Fetch the whole table for some columns 125 | row = self.db_client.select(table=table_name, columns=[columns[0]]+[columns[1]]) 126 | self.assertEqual(len(row), 3) 127 | self.assertEqual(row[0][0], 1) 128 | self.assertEqual(row[0][1], "20161026") 129 | self.assertEqual(row[1][0], 2) 130 | self.assertEqual(row[1][1], "20161026") 131 | self.assertEqual(row[2][0], 3) 132 | self.assertEqual(row[2][1], "20161026") 133 | 134 | # Delete a row from the table 135 | self.db_client.delete( 136 | table_name, 137 | "k=2") 138 | 139 | # Fetch the whole table 140 | row = self.db_client.select(table=table_name) 141 | self.assertEqual(len(row), 2) 142 | self.assertEqual(row[0][0], 1) 143 | self.assertEqual(row[0][1], "20161026") 144 | self.assertEqual(row[0][2], "10:00:00.000000") 145 | self.assertEqual(row[0][3], 'AbC') 146 | self.assertEqual(row[0][4], 10.3) 147 | self.assertEqual(row[1][0], 3) 148 | self.assertEqual(row[1][1], "20161026") 149 | self.assertEqual(row[1][2], "10:00:02.000000") 150 | self.assertEqual(row[1][3], 'Efgh') 151 | self.assertEqual(row[1][4], 10.5) 152 | 153 | # Delete remaining rows from the table 154 | self.db_client.delete(table_name) 155 | # Fetch the whole table 156 | row = self.db_client.select(table=table_name) 157 | self.assertEqual(len(row), 0) 158 | 159 | if __name__ == '__main__': 160 | Logger.init_log() 161 | unittest.main() 162 | 163 | -------------------------------------------------------------------------------- /befh/test/test_sqlite_client.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | import unittest 4 | import os 5 | from sqlite_client import SqliteClient 6 | 7 | file_name = 'sqliteclienttest.sqlite' 8 | 9 | class SqliteClientTest(unittest.TestCase): 10 | @classmethod 11 | def setUpClass(cls): 12 | cls.db_client = SqliteClient() 13 | cls.db_client.connect(path=file_name) 14 | 15 | @classmethod 16 | def tearDownClass(cls): 17 | os.remove(file_name) 18 | pass 19 | 20 | def test_query(self): 21 | table_name = 'test_query' 22 | columns = ['date', 'time', 'k', 'v', 'v2'] 23 | types = ['text', 'text', 'int PRIMARY KEY', 'text', 'decimal(10,5)'] 24 | 25 | # Check table creation 26 | self.assertTrue(self.db_client.create(table_name, columns, types)) 27 | 28 | # Check table insertion 29 | self.assertTrue(self.db_client.insert( 30 | table_name, 31 | columns, 32 | ['20161026','10:00:00.000000',1,'AbC',10.3])) 33 | self.assertTrue(self.db_client.insert( 34 | table_name, 35 | columns, 36 | ['20161026','10:00:01.000000',2,'AbCD',10.4])) 37 | self.assertTrue(self.db_client.insert( 38 | table_name, 39 | columns, 40 | ['20161026','10:00:02.000000',3,'Efgh',10.5])) 41 | 42 | # Check table "IF NOT EXISTS" condition 43 | self.assertTrue(self.db_client.create(table_name, columns, types)) 44 | 45 | # Fetch the whole table 46 | row = self.db_client.select(table=table_name) 47 | self.assertEqual(len(row), 3) 48 | self.assertEqual(row[0][0], "20161026") 49 | self.assertEqual(row[0][1], "10:00:00.000000") 50 | self.assertEqual(row[0][2], 1) 51 | self.assertEqual(row[0][3], 'AbC') 52 | self.assertEqual(row[0][4], 10.3) 53 | self.assertEqual(row[1][0], "20161026") 54 | self.assertEqual(row[1][1], "10:00:01.000000") 55 | self.assertEqual(row[1][2], 2) 56 | self.assertEqual(row[1][4], 10.4) 57 | self.assertEqual(row[1][3], 'AbCD') 58 | self.assertEqual(row[2][0], "20161026") 59 | self.assertEqual(row[2][1], "10:00:02.000000") 60 | self.assertEqual(row[2][2], 3) 61 | self.assertEqual(row[2][3], 'Efgh') 62 | self.assertEqual(row[2][4], 10.5) 63 | 64 | # Fetch with condition 65 | row = self.db_client.select(table=table_name, condition="k=2") 66 | self.assertEqual(len(row), 1) 67 | self.assertEqual(row[0][0], "20161026") 68 | self.assertEqual(row[0][1], "10:00:01.000000") 69 | self.assertEqual(row[0][2], 2) 70 | self.assertEqual(row[0][3], 'AbCD') 71 | self.assertEqual(row[0][4], 10.4) 72 | 73 | # Fetch with ordering 74 | row = self.db_client.select(table=table_name, orderby="k desc") 75 | self.assertEqual(len(row), 3) 76 | self.assertEqual(row[2][0], "20161026") 77 | self.assertEqual(row[2][1], "10:00:00.000000") 78 | self.assertEqual(row[2][2], 1) 79 | self.assertEqual(row[2][3], 'AbC') 80 | self.assertEqual(row[2][4], 10.3) 81 | self.assertEqual(row[1][0], "20161026") 82 | self.assertEqual(row[1][1], "10:00:01.000000") 83 | self.assertEqual(row[1][2], 2) 84 | self.assertEqual(row[1][3], 'AbCD') 85 | self.assertEqual(row[1][4], 10.4) 86 | self.assertEqual(row[0][0], "20161026") 87 | self.assertEqual(row[0][1], "10:00:02.000000") 88 | self.assertEqual(row[0][2], 3) 89 | self.assertEqual(row[0][3], 'Efgh') 90 | self.assertEqual(row[0][4], 10.5) 91 | 92 | # Fetch with limit 93 | row = self.db_client.select(table=table_name, limit=1) 94 | self.assertEqual(len(row), 1) 95 | self.assertEqual(row[0][0], "20161026") 96 | self.assertEqual(row[0][1], "10:00:00.000000") 97 | self.assertEqual(row[0][2], 1) 98 | self.assertEqual(row[0][3], 'AbC') 99 | self.assertEqual(row[0][4], 10.3) 100 | 101 | # Check table insertion or replacement 102 | self.assertTrue(self.db_client.insert( 103 | table_name, 104 | columns, 105 | ['20161026','10:00:04.000000',2,'NoNoNo',10.5], 106 | True)) 107 | 108 | # Fetch the whole table 109 | row = self.db_client.select(table=table_name) 110 | self.assertEqual(len(row), 3) 111 | self.assertEqual(row[0][0], "20161026") 112 | self.assertEqual(row[0][1], "10:00:00.000000") 113 | self.assertEqual(row[0][2], 1) 114 | self.assertEqual(row[0][3], 'AbC') 115 | self.assertEqual(row[0][4], 10.3) 116 | self.assertEqual(row[2][0], "20161026") 117 | self.assertEqual(row[2][1], "10:00:04.000000") 118 | self.assertEqual(row[2][2], 2) 119 | self.assertEqual(row[2][3], 'NoNoNo') 120 | self.assertEqual(row[2][4], 10.5) 121 | self.assertEqual(row[1][0], "20161026") 122 | self.assertEqual(row[1][1], "10:00:02.000000") 123 | self.assertEqual(row[1][2], 3) 124 | self.assertEqual(row[1][3], 'Efgh') 125 | self.assertEqual(row[1][4], 10.5) 126 | 127 | # Fetch the whole table for some columns 128 | row = self.db_client.select(table=table_name, columns=[columns[0]]+[columns[2]]) 129 | self.assertEqual(len(row), 3) 130 | self.assertEqual(row[0][0], "20161026") 131 | self.assertEqual(row[0][1], 1) 132 | self.assertEqual(row[2][0], "20161026") 133 | self.assertEqual(row[2][1], 2) 134 | self.assertEqual(row[1][0], "20161026") 135 | self.assertEqual(row[1][1], 3) 136 | 137 | # Delete a row from the table 138 | self.db_client.delete( 139 | table_name, 140 | "k=2") 141 | 142 | # Fetch the whole table 143 | row = self.db_client.select(table=table_name) 144 | self.assertEqual(len(row), 2) 145 | self.assertEqual(row[0][0], "20161026") 146 | self.assertEqual(row[0][1], "10:00:00.000000") 147 | self.assertEqual(row[0][2], 1) 148 | self.assertEqual(row[0][3], 'AbC') 149 | self.assertEqual(row[0][4], 10.3) 150 | self.assertEqual(row[1][0], "20161026") 151 | self.assertEqual(row[1][1], "10:00:02.000000") 152 | self.assertEqual(row[1][2], 3) 153 | self.assertEqual(row[1][3], 'Efgh') 154 | self.assertEqual(row[1][4], 10.5) 155 | 156 | # Delete remaining rows from the table 157 | self.db_client.delete( 158 | table_name) 159 | # Fetch the whole table 160 | row = self.db_client.select(table=table_name) 161 | self.assertEqual(len(row), 0) 162 | 163 | # Negative case 164 | self.assertTrue(not self.db_client.create(table_name, columns[1::], types)) 165 | self.assertTrue(not self.db_client.insert(table_name, columns, [])) 166 | 167 | if __name__ == '__main__': 168 | unittest.main() 169 | 170 | -------------------------------------------------------------------------------- /befh/test/test_subscription_manager.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | import unittest 4 | import os 5 | from subscription_manager import SubscriptionManager 6 | import json 7 | 8 | file_name = 'test/test_subscriptions.ini' 9 | 10 | class SubscriptionManagerTest(unittest.TestCase): 11 | def test_get_instrument(self): 12 | config = SubscriptionManager(file_name) 13 | instmts = dict() 14 | for instmt_id in config.get_instmt_ids(): 15 | instmts[instmt_id] = config.get_instrument(instmt_id) 16 | 17 | # BTCC-BTCCNY 18 | name = 'BTCC-BTCCNY-Restful' 19 | self.assertEqual(instmts[name].get_exchange_name(), 'BTCC') 20 | self.assertEqual(instmts[name].get_instmt_name(), 'BTCCNY') 21 | self.assertEqual(instmts[name].get_instmt_code(), 'btccny') 22 | self.assertEqual(instmts[name].get_order_book_link(), 23 | 'https://data.btcchina.com/data/orderbook?limit=5&market=btccny') 24 | self.assertEqual(instmts[name].get_trades_link(), 25 | 'https://data.btcchina.com/data/historydata?limit=1000&since=&market=btccny') 26 | m = instmts[name].get_order_book_fields_mapping() 27 | self.assertEqual(m['date'], 'TIMESTAMP') 28 | self.assertEqual(m['bids'], 'BIDS') 29 | self.assertEqual(m['asks'], 'ASKS') 30 | m = instmts[name].get_trades_fields_mapping() 31 | self.assertEqual(m['date'], 'TIMESTAMP') 32 | self.assertEqual(m['type'], 'TRADE_SIDE') 33 | self.assertEqual(m['tid'], 'TRADE_ID') 34 | self.assertEqual(m['price'], 'TRADE_PRICE') 35 | self.assertEqual(m['amount'], 'TRADE_VOLUME') 36 | 37 | # BTCC-XBTCNY 38 | name = 'BTCC-XBTCNY-Restful' 39 | self.assertEqual(instmts[name].get_exchange_name(), 'BTCC') 40 | self.assertEqual(instmts[name].get_instmt_name(), 'XBTCNY') 41 | self.assertEqual(instmts[name].get_instmt_code(), 'xbtcny') 42 | self.assertEqual(instmts[name].get_order_book_link(), 43 | 'https://pro-data.btcc.com/data/pro/orderbook?limit=5&symbol=xbtcny') 44 | self.assertEqual(instmts[name].get_trades_link(), 45 | 'https://pro-data.btcc.com/data/pro/historydata?limit=1000&symbol=xbtcny') 46 | m = instmts[name].get_order_book_fields_mapping() 47 | self.assertEqual(m['date'], 'TIMESTAMP') 48 | self.assertEqual(m['bids'], 'BIDS') 49 | self.assertEqual(m['asks'], 'ASKS') 50 | self.assertEqual(m['TIMESTAMP_OFFSET'], 1000) 51 | m = instmts[name].get_trades_fields_mapping() 52 | self.assertEqual(m['Timestamp'], 'TIMESTAMP') 53 | self.assertEqual(m['Side'], 'TRADE_SIDE') 54 | self.assertEqual(m['Id'], 'TRADE_ID') 55 | self.assertEqual(m['Price'], 'TRADE_PRICE') 56 | self.assertEqual(m['Quantity'], 'TRADE_VOLUME') 57 | 58 | def test_get_subscriptions(self): 59 | instmts = SubscriptionManager(file_name).get_subscriptions() 60 | 61 | self.assertEqual(instmts[0].get_exchange_name(), 'BTCC') 62 | self.assertEqual(instmts[0].get_instmt_name(), 'BTCCNY') 63 | self.assertEqual(instmts[0].get_instmt_code(), 'btccny') 64 | self.assertEqual(instmts[0].get_order_book_link(), 65 | 'https://data.btcchina.com/data/orderbook?limit=5&market=btccny') 66 | self.assertEqual(instmts[0].get_trades_link(), 67 | 'https://data.btcchina.com/data/historydata?limit=1000&since=&market=btccny') 68 | m = instmts[0].get_order_book_fields_mapping() 69 | self.assertEqual(m['date'], 'TIMESTAMP') 70 | self.assertEqual(m['bids'], 'BIDS') 71 | self.assertEqual(m['asks'], 'ASKS') 72 | m = instmts[0].get_trades_fields_mapping() 73 | self.assertEqual(m['date'], 'TIMESTAMP') 74 | self.assertEqual(m['type'], 'TRADE_SIDE') 75 | self.assertEqual(m['tid'], 'TRADE_ID') 76 | self.assertEqual(m['price'], 'TRADE_PRICE') 77 | self.assertEqual(m['amount'], 'TRADE_VOLUME') 78 | 79 | self.assertEqual(instmts[1].get_exchange_name(), 'BTCC') 80 | self.assertEqual(instmts[1].get_instmt_name(), 'XBTCNY') 81 | self.assertEqual(instmts[1].get_instmt_code(), 'xbtcny') 82 | self.assertEqual(instmts[1].get_order_book_link(), 83 | 'https://pro-data.btcc.com/data/pro/orderbook?limit=5&symbol=xbtcny') 84 | self.assertEqual(instmts[1].get_trades_link(), 85 | 'https://pro-data.btcc.com/data/pro/historydata?limit=1000&symbol=xbtcny') 86 | m = instmts[1].get_order_book_fields_mapping() 87 | self.assertEqual(m['date'], 'TIMESTAMP') 88 | self.assertEqual(m['bids'], 'BIDS') 89 | self.assertEqual(m['asks'], 'ASKS') 90 | self.assertEqual(m['TIMESTAMP_OFFSET'], 1000) 91 | m = instmts[1].get_trades_fields_mapping() 92 | self.assertEqual(m['Timestamp'], 'TIMESTAMP') 93 | self.assertEqual(m['Side'], 'TRADE_SIDE') 94 | self.assertEqual(m['Id'], 'TRADE_ID') 95 | self.assertEqual(m['Price'], 'TRADE_PRICE') 96 | self.assertEqual(m['Quantity'], 'TRADE_VOLUME') 97 | 98 | 99 | if __name__ == '__main__': 100 | unittest.main() 101 | -------------------------------------------------------------------------------- /befh/test/test_subscriptions.ini: -------------------------------------------------------------------------------- 1 | [BTCC-Spot-BTCCNY-Restful] 2 | exchange = BTCC_Spot 3 | instmt_name = BTCCNY 4 | instmt_code = btccny 5 | enabled = 1 6 | 7 | [BTCC-Spot-XBTCNY-Restful] 8 | exchange = BTCC_Future 9 | instmt_name = XBTCNY 10 | instmt_code = xbtcny 11 | enabled = 1 12 | 13 | [BitMEX-XBTUSD-WS] 14 | exchange = BitMEX 15 | instmt_name = XBTUSD 16 | instmt_code = XBTUSD 17 | enabled = 1 18 | 19 | [BitMEX-XBTZ16-WS] 20 | exchange = BitMEX 21 | instmt_name = XBTH17 22 | instmt_code = XBTH17 23 | enabled = 1 -------------------------------------------------------------------------------- /befh/test/test_ws_server.py: -------------------------------------------------------------------------------- 1 | from websocket_server import WebsocketServer 2 | import json 3 | 4 | # Called for every client connecting (after handshake) 5 | def new_client(client, server): 6 | print("New client connected and was given id %d" % client['id']) 7 | server.send_message_to_all(json.dumps({"new_client": client['id']})) 8 | 9 | 10 | # Called for every client disconnecting 11 | def client_left(client, server): 12 | print("Client(%d) disconnected" % client['id']) 13 | 14 | 15 | # Called when a client sends a message 16 | def message_received(client, server, message): 17 | if len(message) > 200: 18 | message = message[:200]+'..' 19 | print("Client(%d) said: %s" % (client['id'], message)) 20 | 21 | 22 | PORT=80 23 | server = WebsocketServer(PORT) 24 | server.set_fn_new_client(new_client) 25 | server.set_fn_client_left(client_left) 26 | server.set_fn_message_received(message_received) 27 | server.run_forever() -------------------------------------------------------------------------------- /befh/test/test_zmqfeed.py: -------------------------------------------------------------------------------- 1 | import zmq 2 | 3 | market_feed_name = "marketfeed" 4 | context = zmq.Context() 5 | sock = context.socket(zmq.SUB) 6 | # sock.connect("ipc://%s" % market_feed_name) 7 | sock.connect("tcp://127.0.0.1:6001") 8 | sock.setsockopt_string(zmq.SUBSCRIBE, '') 9 | 10 | print("Started...") 11 | while True: 12 | ret = sock.recv_pyobj() 13 | print(ret) -------------------------------------------------------------------------------- /befh/util.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import logging 3 | 4 | 5 | class Logger: 6 | logger = None 7 | 8 | @staticmethod 9 | def init_log(output=None): 10 | """ 11 | Initialise the logger 12 | """ 13 | logging.basicConfig() 14 | 15 | Logger.logger = logging.getLogger('BitcoinExchangeFH') 16 | # Logger.logger.setLevel(logging.INFO) 17 | Logger.logger.setLevel(logging.WARNING) 18 | 19 | formatter = logging.Formatter('%(asctime)s - %(levelname)s \n%(message)s\n') 20 | if output is None: 21 | slogger = logging.StreamHandler() 22 | slogger.setFormatter(formatter) 23 | Logger.logger.addHandler(slogger) 24 | else: 25 | flogger = logging.FileHandler(output) 26 | flogger.setFormatter(formatter) 27 | Logger.logger.addHandler(flogger) 28 | 29 | logging.getLogger("websocket").setLevel(logging.WARNING) 30 | 31 | @staticmethod 32 | def info(method, str): 33 | """ 34 | Write info log 35 | :param method: Method name 36 | :param str: Log message 37 | """ 38 | Logger.logger.info('[%s]\n%s\n' % (method, str)) 39 | 40 | @staticmethod 41 | def error(method, str): 42 | """ 43 | Write info log 44 | :param method: Method name 45 | :param str: Log message 46 | """ 47 | Logger.logger.error('[%s]\n%s\n' % (method, str)) 48 | -------------------------------------------------------------------------------- /befh/ws_api_socket.py: -------------------------------------------------------------------------------- 1 | from befh.api_socket import ApiSocket 2 | from befh.util import Logger 3 | import websocket 4 | import threading 5 | import json 6 | import time 7 | import zlib 8 | 9 | class WebSocketApiClient(ApiSocket): 10 | """ 11 | Generic REST API call 12 | """ 13 | def __init__(self, id, received_data_compressed=False): 14 | """ 15 | Constructor 16 | :param id: Socket id 17 | """ 18 | ApiSocket.__init__(self) 19 | self.ws = None # Web socket 20 | self.id = id 21 | self.wst = None # Web socket thread 22 | self._connecting = False 23 | self._connected = False 24 | self._received_data_compressed = received_data_compressed 25 | self.on_message_handlers = [] 26 | self.on_open_handlers = [] 27 | self.on_close_handlers = [] 28 | self.on_error_handlers = [] 29 | 30 | def connect(self, url, 31 | on_message_handler=None, 32 | on_open_handler=None, 33 | on_close_handler=None, 34 | on_error_handler=None, 35 | reconnect_interval=10): 36 | """ 37 | :param url: Url link 38 | :param on_message_handler: Message handler which take the message as 39 | the first argument 40 | :param on_open_handler: Socket open handler which take the socket as 41 | the first argument 42 | :param on_close_handler: Socket close handler which take the socket as 43 | the first argument 44 | :param on_error_handler: Socket error handler which take the socket as 45 | the first argument and the error as the second 46 | argument 47 | :param reconnect_interval: The time interval for reconnection 48 | """ 49 | Logger.info(self.__class__.__name__, "Connecting to socket <%s> <%s>..." % (self.id, url)) 50 | if on_message_handler is not None: 51 | self.on_message_handlers.append(on_message_handler) 52 | if on_open_handler is not None: 53 | self.on_open_handlers.append(on_open_handler) 54 | if on_close_handler is not None: 55 | self.on_close_handlers.append(on_close_handler) 56 | if on_error_handler is not None: 57 | self.on_error_handlers.append(on_error_handler) 58 | 59 | if not self._connecting and not self._connected: 60 | self._connecting = True 61 | self.ws = websocket.WebSocketApp(url, 62 | on_message=self.__on_message, 63 | on_close=self.__on_close, 64 | on_open=self.__on_open, 65 | on_error=self.__on_error) 66 | self.wst = threading.Thread(target=lambda: self.__start(reconnect_interval=reconnect_interval)) 67 | self.wst.start() 68 | 69 | return self.wst 70 | 71 | def send(self, msg): 72 | """ 73 | Send message 74 | :param msg: Message 75 | :return: 76 | """ 77 | self.ws.send(msg) 78 | 79 | def __start(self, reconnect_interval=10): 80 | while True: 81 | self.ws.run_forever() 82 | Logger.info(self.__class__.__name__, "Socket <%s> is going to reconnect..." % self.id) 83 | time.sleep(reconnect_interval) 84 | 85 | def __on_message(self, ws, m): 86 | if self._received_data_compressed is True: 87 | data = zlib.decompress(m, zlib.MAX_WBITS|16).decode('UTF-8') 88 | m = json.loads(data) 89 | else: 90 | m = json.loads(m) 91 | if len(self.on_message_handlers) > 0: 92 | for handler in self.on_message_handlers: 93 | handler(m) 94 | 95 | def __on_open(self, ws): 96 | Logger.info(self.__class__.__name__, "Socket <%s> is opened." % self.id) 97 | self._connected = True 98 | if len(self.on_open_handlers) > 0: 99 | for handler in self.on_open_handlers: 100 | handler(ws) 101 | 102 | def __on_close(self, ws): 103 | Logger.info(self.__class__.__name__, "Socket <%s> is closed." % self.id) 104 | self._connecting = False 105 | self._connected = False 106 | if len(self.on_close_handlers) > 0: 107 | for handler in self.on_close_handlers: 108 | handler(ws) 109 | 110 | def __on_error(self, ws, error): 111 | Logger.error(self.__class__.__name__, "Socket <%s> error:\n %s" % (self.id, error)) 112 | if len(self.on_error_handlers) > 0: 113 | for handler in self.on_error_handlers: 114 | handler(ws, error) 115 | 116 | if __name__ == '__main__': 117 | Logger.init_log() 118 | socket = WebSocketApiClient('test') 119 | socket.connect('ws://localhost', reconnect_interval=1) 120 | time.sleep(10) 121 | -------------------------------------------------------------------------------- /doc/icon.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/doc/icon.jpg -------------------------------------------------------------------------------- /doc/sample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/doc/sample.jpg -------------------------------------------------------------------------------- /doc/sample2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/doc/sample2.jpg -------------------------------------------------------------------------------- /requirement.txt: -------------------------------------------------------------------------------- 1 | pymysql 2 | websocket-client 3 | numpy 4 | qpython 5 | pyzmq 6 | tzlocal 7 | requests 8 | pandas 9 | kafka-python 10 | -------------------------------------------------------------------------------- /setup-env.sh: -------------------------------------------------------------------------------- 1 | export PYTHONPATH="$PWD" 2 | 3 | 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='BitcoinExchangeFH', 5 | version='0.2.4', 6 | author='Gavin Chan', 7 | author_email='gavincyi@gmail.com', 8 | packages=['befh'], 9 | url='http://pypi.python.org/pypi/BitcoinExchangeFH/', 10 | license='LICENSE.txt', 11 | description='Cryptocurrency exchange market data feed handler.', 12 | entry_points={ 13 | 'console_scripts': ['bitcoinexchangefh=befh.bitcoinexchangefh:main'] 14 | }, 15 | install_requires=[ 16 | 'pymysql', 17 | 'websocket-client', 18 | 'numpy', 19 | 'qpython', 20 | 'pyzmq', 21 | 'kafka-python', 22 | 'requests' 23 | ] 24 | ) 25 | -------------------------------------------------------------------------------- /subscriptions.ini: -------------------------------------------------------------------------------- 1 | [BTCC-Spot-BTCCNY-Restful] 2 | exchange = BTCC_Spot 3 | instmt_name = BTCCNY 4 | instmt_code = btccny 5 | enabled = 1 6 | 7 | [BTCC-Spot-XBTCNY-Restful] 8 | exchange = BTCC_Future 9 | instmt_name = XBTCNY 10 | instmt_code = xbtcny 11 | enabled = 1 12 | 13 | [BitMEX-XBTUSD-WS] 14 | exchange = BitMEX 15 | instmt_name = XBTUSD 16 | instmt_code = XBTUSD 17 | enabled = 1 18 | 19 | [Bitfinex-BTCUSD-WS] 20 | exchange = Bitfinex 21 | instmt_name = BTCUSD 22 | instmt_code = BTCUSD 23 | enabled = 1 24 | 25 | [OkCoin-SPOTBTCUSD-WS] 26 | exchange = OkCoin 27 | instmt_name = SPOT_BTCUSD 28 | instmt_code = spotusd_btc 29 | enabled = 1 30 | 31 | [OkCoin-FUTBTCUSD_QUARTER-WS] 32 | exchange = OkCoin 33 | instmt_name = FUT_BTCUSD 34 | instmt_code = futureusd_btc_quarter 35 | enabled = 1 36 | 37 | [Kraken-XBTEUR-Restful] 38 | exchange = Kraken 39 | instmt_name = XBTEUR 40 | instmt_code = xbteur 41 | enabled = 1 42 | 43 | [Kraken-XBTUSD-Restful] 44 | exchange = Kraken 45 | instmt_name = XBTUSD 46 | instmt_code = xbtusd 47 | enabled = 1 48 | 49 | [Gdax-BTCUSD-RestfulAndWs] 50 | exchange = Gdax 51 | instmt_name = BTCUSD 52 | instmt_code = BTC-USD 53 | enabled = 1 54 | 55 | [Bitstamp-BTCUSD-Ws] 56 | exchange = Bitstamp 57 | instmt_name = BTCUSD 58 | instmt_code = "" 59 | enabled = 1 60 | 61 | [Gatecoin-BTCHKD-Restful] 62 | exchange = Gatecoin 63 | instmt_name = BTCHKD 64 | instmt_code = BTCHKD 65 | enabled = 1 66 | 67 | [Quoine-BTCUSD-Restful] 68 | exchange = Quoine 69 | instmt_name = BTCUSD 70 | instmt_code = 1 71 | enabled = 1 72 | 73 | [Quoine-BTCHKD-Restful] 74 | exchange = Quoine 75 | instmt_name = BTCHKD 76 | instmt_code = 9 77 | enabled = 1 78 | 79 | [Poloniex-BTCETH-Restful] 80 | exchange = Poloniex 81 | instmt_name = BTCETH 82 | instmt_code = BTC_ETH 83 | enabled = 1 84 | 85 | [Bittrex-BTCETH-Restful] 86 | exchange = Bittrex 87 | instmt_name = BTCETH 88 | instmt_code = BTC-ETH 89 | enabled = 1 90 | 91 | [HuoBi-BTCUSDT-Ws] 92 | exchange = HuoBi 93 | instmt_name = BTCUSDT 94 | instmt_code = btcusdt 95 | enabled = 1 96 | 97 | [Okex-BTC-Ws] 98 | exchange = Okex 99 | instmt_name = BTC 100 | instmt_code = btc 101 | enabled = 1 102 | 103 | [Wex-BTCUSD-Restful] 104 | exchange = Wex 105 | instmt_name = BTCUSD 106 | instmt_code = btc_usd 107 | enabled = 1 108 | 109 | [Bitflyer-BTCJPY-Restful] 110 | exchange = Bitflyer 111 | instmt_name = BTCJPY 112 | instmt_code = BTC_JPY 113 | enabled = 1 114 | 115 | [Coinone-BTC-Restful] 116 | exchange = CoinOne 117 | instmt_name = btc 118 | instmt_code = btc 119 | enabled = 1 120 | 121 | [Coincheck-BTC-Restful] 122 | exchange = Coincheck 123 | instmt_name = btc_jpy 124 | instmt_code = btc_jpy 125 | enabled = 1 126 | -------------------------------------------------------------------------------- /third-party/q: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/third-party/q -------------------------------------------------------------------------------- /third-party/q.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philsong/BitcoinExchangeFH/3c45d4be2ea2a258f132d982f62f69d649e0b083/third-party/q.exe --------------------------------------------------------------------------------