├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── Dockerfile ├── LICENSE ├── README.md ├── arbcharm ├── __main__.py ├── charm.py ├── errors.py ├── exchange_api │ ├── __init__.py │ ├── binance.py │ ├── bitfinex.py │ └── huobipro.py ├── json_logger.py ├── models.py ├── settings.py └── tools.py ├── build.sh ├── requirements.txt ├── setup.py └── start.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # pycharm 107 | .idea/ 108 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: https://github.com/RunningToTheEdgeOfTheWorld/pre-commit.git 2 | sha: 411c305fadd02f19e543cc10d3ecd5baeab438e0 3 | hooks: 4 | - id: python-pylint 5 | - id: python-isort 6 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | [MESSAGES CONTROL] 4 | disable=missing-docstring, 5 | too-few-public-methods, 6 | invalid-name, 7 | cyclic-import, 8 | too-many-ancestors, 9 | duplicate-code, 10 | import-error, 11 | too-many-instance-attributes, 12 | no-member, 13 | protected-access, 14 | fixme 15 | 16 | [REPORTS] 17 | reports=no 18 | 19 | # Do not modify score 20 | score=no 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pypy:3-6 2 | 3 | # base 4 | RUN cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ 5 | && echo 'Asia/Shanghai' >/etc/timezone \ 6 | && apt-get update -y \ 7 | && pip install --upgrade pip 8 | 9 | # project 10 | COPY . /opt/project 11 | 12 | WORKDIR /opt/project 13 | 14 | RUN pip install /opt/project 15 | 16 | CMD ["pypy3", "-m", "arbcharm"] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 ZJ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # arbcharm （已停止更新，有需求可以联系我） 2 | 3 | ## 简介 4 | 套利的艺术 5 | * 使用python3协程接入主流数字货币交易所websocket, 只需在配置文件./settings.py中配置密钥即可。 6 | * 系统响应时延小于6ms 7 | * 目前支持的主流数字货币交易所包括： 8 | * bitfinex 9 | * binance 10 | * huobipro 11 | 12 | ## 使用 13 | 项目启动需要依赖以下组件： 14 | 15 | * docker 16 | 17 | 启动步骤： 18 | 19 | 1. 执行./build.sh构建镜像。 20 | 2. 执行./start.sh启动服务。 21 | -------------------------------------------------------------------------------- /arbcharm/__main__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import asyncio 6 | 7 | print('server start') 8 | 9 | main_loop = asyncio.get_event_loop() 10 | 11 | 12 | def main(): 13 | start_arbtrage_task() 14 | 15 | 16 | def start_arbtrage_task(): 17 | from arbcharm import settings 18 | from arbcharm.charm import ArbCharm 19 | from arbcharm.models import get_exchange 20 | 21 | tasks = [] 22 | for sym, exc_dict in settings.ARB_CONF.items(): 23 | excs = [get_exchange(e, config) for e, config in exc_dict.items()] 24 | tasks.append(ArbCharm(sym, excs).start()) 25 | 26 | cortasks = asyncio.gather(*tasks) 27 | main_loop.run_until_complete(cortasks) 28 | 29 | 30 | main() 31 | -------------------------------------------------------------------------------- /arbcharm/charm.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import asyncio 6 | import time 7 | from typing import List 8 | 9 | from arbcharm import settings 10 | from arbcharm.models import BaseExchange, Order, OrderBook, Trade, get_exchange 11 | from arbcharm.tools import get_logger, rate_limit_generator 12 | from ccxt.base import errors as ccxt_errors 13 | 14 | 15 | class ArbCharm: 16 | def __init__(self, symbol, exchanges: List[BaseExchange]): 17 | self.symbol = symbol 18 | self.exchanges = exchanges 19 | self.name = 'ArbCharm-{}'.format(self.symbol) 20 | self.trade_limit = rate_limit_generator() 21 | self.trade_limit.send(None) 22 | self.logger = get_logger(self.name) 23 | self.is_running = False 24 | 25 | async def start(self): 26 | self.is_running = True 27 | self.logger.info(event='arbcharm_start') 28 | 29 | for e in self.exchanges: 30 | asyncio.ensure_future(e.set_orderbook_d(self.symbol)) 31 | 32 | alerter_exc = get_exchange('binance', {}) 33 | while self.is_running: 34 | await alerter_exc.wait_book_update(self.symbol) 35 | await self.arbitrage() 36 | alerter_exc.reset_book_update(self.symbol) 37 | 38 | # TODO cancel all orders 39 | self.logger.info(event='arbcharm_exit') 40 | 41 | def close(self): 42 | self.is_running = False 43 | 44 | async def arbitrage(self): 45 | now = time.time() 46 | ob_l = self.get_valide_ob_l() 47 | if len(ob_l) < 2: 48 | return 49 | 50 | self.logger.debug( 51 | event='arbitrage', 52 | market_price={ob.exchange.name: (ob.asks[0].price+ob.bids[0].price)/2 for ob in ob_l}, 53 | time_delay={ob.exchange.name: now-ob.timestamp for ob in ob_l}, 54 | ) 55 | trades = auto_match(ob_l) 56 | opportunity = self.find_opportunity_from_trade(trades) 57 | 58 | if opportunity: 59 | self.logger.info( 60 | event='found_opportunity', 61 | opportunity=[o.to_dict() for o in opportunity], 62 | ) 63 | await self.catch_opportunity(opportunity) 64 | await asyncio.sleep(3) 65 | 66 | def get_valide_ob_l(self): 67 | now = time.time() 68 | ob_l = [] 69 | for e in self.exchanges: 70 | book = e.get_book(self.symbol) 71 | if book and now - book.timestamp < 1: 72 | ob_l.append(book) 73 | return ob_l 74 | 75 | def find_opportunity_from_trade(self, trades: List[Trade]) -> List[Order]: 76 | rate = settings.ARBITRAGE_OPPORTUNITY_RATE 77 | orders = {} 78 | 79 | for t in trades: 80 | if (t.bid_price-t.ask_price)/t.ask_price > rate: 81 | od = orders.get( 82 | t.ask_exc, 83 | Order( 84 | exc=t.ask_exc, 85 | symbol=self.symbol, 86 | price=t.ask_price, 87 | amount=0, 88 | side=Order.SIDE_BUY, 89 | ) 90 | ) 91 | assert od.side == Order.SIDE_BUY 92 | od.amount += t.amount 93 | od.price = max(od.price, t.ask_price) 94 | orders[od.exc] = od 95 | 96 | od = orders.get( 97 | t.bid_exc, 98 | Order( 99 | exc=t.bid_exc, 100 | symbol=self.symbol, 101 | price=t.bid_price, 102 | amount=0, 103 | side=Order.SIDE_SELL, 104 | ) 105 | ) 106 | assert od.side == Order.SIDE_SELL 107 | od.amount += t.amount 108 | od.price = min(od.price, t.bid_price) 109 | orders[od.exc] = od 110 | 111 | return list(orders.values()) 112 | 113 | async def catch_opportunity(self, opportunity: List[Order]): 114 | order_task = [] 115 | for o in opportunity: 116 | fixd_am = self.fix_amount(o.amount) 117 | if o.side == Order.SIDE_BUY: 118 | order_task.append( 119 | self.deal_buy(o.exc, fixd_am, o.price) 120 | ) 121 | if o.side == Order.SIDE_SELL: 122 | order_task.append( 123 | self.deal_sell(o.exc, fixd_am, o.price) 124 | ) 125 | 126 | await asyncio.wait(order_task) 127 | 128 | def fix_amount(self, amount): 129 | amount_mul = settings.ARBCHARM_AMOUNT_MULTIPLIER 130 | config_min_amount = max([e.ccxt_exchange.min_amount for e in self.exchanges]) 131 | return max(amount*amount_mul, config_min_amount) 132 | 133 | async def deal_buy(self, exchange: BaseExchange, amount, price): 134 | if settings.MODE != 'prd': 135 | return 136 | try: 137 | res = await exchange.ccxt_exchange.create_limit_buy_order( 138 | self.symbol, 139 | amount, 140 | price, 141 | ) 142 | except ccxt_errors.BaseError: 143 | self.logger.exception() 144 | return 145 | 146 | oid = res['id'] 147 | await asyncio.sleep(3) 148 | await self.deal_cancel(exchange, oid) 149 | await self.save_order_and_trade(exchange, oid) 150 | 151 | async def deal_sell(self, exchange: BaseExchange, amount, price): 152 | if settings.MODE != 'prd': 153 | return 154 | try: 155 | res = await exchange.ccxt_exchange.create_limit_sell_order( 156 | self.symbol, 157 | amount, 158 | price, 159 | ) 160 | except ccxt_errors.BaseError: 161 | self.logger.exception() 162 | return 163 | 164 | oid = res['id'] 165 | await asyncio.sleep(10) 166 | await self.deal_cancel(exchange, oid) 167 | await self.save_order_and_trade(exchange, oid) 168 | 169 | async def deal_cancel(self, exchange: BaseExchange, oid): 170 | cancel_success = False 171 | while not cancel_success: 172 | try: 173 | await exchange.ccxt_exchange.cancel_order(oid, symbol=self.symbol) 174 | cancel_success = True 175 | self.logger.info(event='cancel_order_success', oid=oid, exchange=exchange.name) 176 | except ccxt_errors.OrderNotFound: 177 | cancel_success = True 178 | except ccxt_errors.BaseError as e: 179 | self.logger.exception(error_class=e.__class__.__name__) 180 | await asyncio.sleep(3) 181 | return oid 182 | 183 | async def save_order_and_trade(self, exchange: BaseExchange, oid): 184 | pass 185 | 186 | def __str__(self): 187 | return self.name 188 | 189 | 190 | def auto_match(ob_l: List[OrderBook]) -> List[Trade]: 191 | """ 192 | orderbook成交函数 193 | :param ob_l: 多个交易所和当前的orderbook 194 | :return: 成交列表 195 | """ 196 | bid_row_l = [] 197 | for ob in ob_l[::-1]: # 相同条件下, 放在前面的交易所得到优先成交 198 | bid_row_l.extend([[ob.exchange, ow] for ow in ob.bids]) 199 | bid_row_l = sorted(bid_row_l, key=lambda i: i[1].price) # best order row at last index 200 | 201 | ask_row_l = [] 202 | for ob in ob_l[::-1]: 203 | ask_row_l.extend([[ob.exchange, ow] for ow in ob.asks]) 204 | ask_row_l = sorted(ask_row_l, key=lambda i: i[1].price, reverse=True) 205 | 206 | trade_l = [] 207 | bid_retain_row = None 208 | ask_retain_row = None 209 | while bid_row_l and ask_row_l: 210 | bid_exc, bid_ow = bid_retain_row if bid_retain_row else bid_row_l.pop() 211 | ask_exc, ask_ow = ask_retain_row if ask_retain_row else ask_row_l.pop() 212 | if bid_ow.price < ask_ow.price: 213 | break 214 | 215 | if bid_ow.amount < ask_ow.amount: 216 | amount = bid_ow.amount 217 | ask_ow.amount = ask_ow.amount - amount 218 | bid_retain_row, ask_retain_row = None, [ask_exc, ask_ow] 219 | elif bid_ow.amount == ask_ow.amount: 220 | amount = bid_ow.amount 221 | bid_retain_row, ask_retain_row = None, None 222 | else: 223 | amount = ask_ow.amount 224 | bid_ow.amount = bid_ow.amount - amount 225 | bid_retain_row, ask_retain_row = [bid_exc, bid_ow], None 226 | 227 | trade_l.append( 228 | Trade( 229 | bid_exc=bid_exc, 230 | ask_exc=ask_exc, 231 | bid_price=bid_ow.price, 232 | ask_price=ask_ow.price, 233 | amount=amount, 234 | ) 235 | ) 236 | return trade_l 237 | 238 | 239 | if __name__ == "__main__": 240 | lp = asyncio.get_event_loop() 241 | ac = ArbCharm('BTC/USDT', [get_exchange(e, {}) for e in ('binance', 'bitfinex', 'huobipro')]) 242 | lp.run_until_complete(ac.start()) 243 | -------------------------------------------------------------------------------- /arbcharm/errors.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/15' 4 | 5 | 6 | class CharmError(Exception): 7 | pass 8 | 9 | 10 | class ExchangeError(CharmError): 11 | pass 12 | -------------------------------------------------------------------------------- /arbcharm/exchange_api/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | -------------------------------------------------------------------------------- /arbcharm/exchange_api/binance.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import asyncio 6 | import json 7 | import time 8 | 9 | import websockets 10 | 11 | from arbcharm.models import BaseExchange, OrderBook, OrderRow 12 | 13 | 14 | class Binance(BaseExchange): 15 | """ 16 | binance websockets doc: 17 | https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md 18 | """ 19 | async def set_orderbook_d(self, symbol): 20 | """ 21 | bitfinex websockets doc: https://docs.bitfinex.com/docs/ws-general 22 | """ 23 | if symbol in self._orderbook_d: 24 | return 25 | self.clear_book(symbol) 26 | 27 | while True: 28 | try: 29 | await self._cache_book_ws(symbol) 30 | except websockets.exceptions.ConnectionClosed as e: 31 | self.logger.exception(error_class=e.__class__.__name__) # TODO fix connection error 32 | await asyncio.sleep(10) 33 | except: # pylint: disable=bare-except 34 | self.logger.exception() 35 | await asyncio.sleep(10) 36 | 37 | async def _cache_book_ws(self, symbol): 38 | __sym = symbol.replace('/', '').lower() 39 | stm_book = '{}@depth20'.format(__sym) 40 | stm_trade = '{}@aggTrade'.format(__sym) 41 | url = 'wss://stream.binance.com:9443/stream?streams={}/{}'.format(stm_book, stm_trade) 42 | 43 | async with websockets.connect(url) as ws: 44 | while True: 45 | msg = json.loads(await asyncio.wait_for(ws.recv(), timeout=30)) 46 | event = msg.get('stream') 47 | if event == stm_book: 48 | self.deal_book_event(symbol, msg['data']) 49 | elif event == stm_trade: 50 | self.deal_trade_event(symbol, msg['data']) 51 | 52 | def deal_book_event(self, symbol, data): 53 | """ 54 | data: 55 | { 56 | 'lastUpdateId': 265429175, 57 | 'bids': [['6749.48000000', '0.14953300', []],...], 58 | 'asks': [['6749.48000000', '0.14953300', []],...], 59 | } 60 | """ 61 | ob = OrderBook( 62 | exchange=self, 63 | symbol=symbol, 64 | asks=[OrderRow(price=float(p), amount=float(a), count=1) for p, a, _ in data['asks']], 65 | bids=[OrderRow(price=float(p), amount=float(a), count=1) for p, a, _ in data['bids']], 66 | timestamp=time.time(), 67 | ) 68 | self.set_book(ob) 69 | 70 | def deal_trade_event(self, symbol, data): 71 | """ 72 | data: 73 | { 74 | 'e': 'aggTrade', 75 | 'E': 1539683812143, 76 | 's': 'BTCUSDT', 77 | 'a': 67071898, 78 | 'p': '6754.35000000', 79 | 'q': '0.22700000', 80 | 'f': 75240975, 81 | 'l': 75240975, 82 | 'T': 1539683812141, 83 | 'm': False, 84 | 'M': True 85 | } 86 | """ 87 | ob = self.get_book(symbol) 88 | if ob: 89 | ob.remove_bad_price(float(data['p'])) 90 | 91 | async def cancel_all(self): 92 | pass 93 | 94 | 95 | if __name__ == '__main__': 96 | ba = Binance('binance', {}) 97 | lp = asyncio.get_event_loop() 98 | lp.run_until_complete(ba.set_orderbook_d('BTC/USDT')) 99 | -------------------------------------------------------------------------------- /arbcharm/exchange_api/bitfinex.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import asyncio 6 | import json 7 | import time 8 | 9 | import websockets 10 | 11 | from arbcharm import errors, settings 12 | from arbcharm.models import BaseExchange, OrderBook, OrderRow 13 | from arbcharm.tools import bisect_right, rate_limit_generator 14 | 15 | 16 | class Bitfinex(BaseExchange): 17 | """ 18 | bitfinex websockets doc: https://docs.bitfinex.com/docs/ws-general 19 | """ 20 | async def set_orderbook_d(self, symbol): 21 | if symbol in self._orderbook_d: 22 | return 23 | self.clear_book(symbol) 24 | 25 | while True: 26 | try: 27 | await self._cache_book_ws(symbol) 28 | except websockets.exceptions.ConnectionClosed as e: 29 | self.logger.exception(error_class=e.__class__.__name__) # TODO fix connection error 30 | await asyncio.sleep(10) 31 | except: # pylint: disable=bare-except 32 | self.logger.exception() 33 | await asyncio.sleep(10) 34 | 35 | async def _cache_book_ws(self, symbol): 36 | # pylint: disable=too-many-locals, too-many-branches, too-many-statements 37 | __sym = symbol.replace('/', '').replace('USDT', 'USD').upper() 38 | d1 = { 39 | "event": "subscribe", 40 | "channel": "book", 41 | "pair": __sym, 42 | "prec": "P0", 43 | "freq": "F0", 44 | "length": str(settings.CACHE_ORDER_ROW_LENGTH), 45 | } 46 | # d2 = { 47 | # "event": "subscribe", 48 | # "channel": "trades", 49 | # "pair": __sym 50 | # } 51 | book_channel_id = None 52 | # trades_channel_id = None 53 | ping_rate_limit = rate_limit_generator() 54 | ping_rate_limit.send(None) 55 | async with websockets.connect("wss://api.bitfinex.com/ws") as ws: 56 | await ws.send(json.dumps(d1)) 57 | # await ws.send(json.dumps(d2)) 58 | asks = [] 59 | bids = [] 60 | 61 | while True: 62 | data = json.loads(await asyncio.wait_for(ws.recv(), timeout=30)) 63 | if isinstance(data, dict): 64 | if data.get('event') == 'info': 65 | self.logger.info(event='get_bitfinex_event', data=data) 66 | if data.get('code') == 20051: # reconn signal 67 | return 68 | if data.get('code') == 20060: # pause signal 69 | await asyncio.sleep(15) 70 | return 71 | continue 72 | elif data.get('event') == 'error': 73 | raise errors.ExchangeError(data) 74 | elif data.get('event') == 'subscribed': 75 | if data.get('channel') == 'book': 76 | book_channel_id = data['chanId'] 77 | # if data.get('channel') == 'trades': 78 | # trades_channel_id = data['chanId'] 79 | elif isinstance(data, list) and book_channel_id == data[0]: 80 | await self._handle_ws_book(data, asks, bids, symbol) 81 | # elif isinstance(data, list) and trades_channel_id == data[0]: 82 | # await self._handle_ws_trades(data) 83 | 84 | if ping_rate_limit.send(('{}.ping'.format(self.name), 5)): 85 | await ws.send(json.dumps({"event": "ping"})) 86 | 87 | async def _handle_ws_book(self, data, asks, bids, symbol): 88 | # pylint: disable=too-many-locals 89 | recv_time = time.time() 90 | if len(data) == 2: # get book snapshot 91 | if data[1] == 'hb': 92 | return 93 | _, ows = data 94 | asks.clear() 95 | bids.clear() 96 | for p, c, a in ows: 97 | if a >= 0: 98 | bids.append(OrderRow(price=p, count=c, amount=a)) 99 | else: 100 | asks.append(OrderRow(price=p, count=c, amount=-a)) 101 | elif len(data) == 4: # get change 102 | _conn_id, price, count, amount = data 103 | side_ows = bids if amount >= 0 else asks 104 | amount = abs(amount) 105 | ow = OrderRow(price=price, count=count, amount=amount) 106 | # Binary search [O(logn)] 107 | ind = bisect_right( 108 | side_ows, ow, key=lambda o: o.price, rv=side_ows is bids 109 | ) 110 | l_ind = ind-1 111 | if l_ind >= 0 and side_ows[l_ind].price == ow.price: 112 | if count == 0: 113 | side_ows.pop(l_ind) 114 | else: 115 | side_ows[l_ind] = ow 116 | else: 117 | side_ows.insert(ind, ow) 118 | ob = OrderBook(exchange=self, 119 | symbol=symbol, 120 | bids=bids, 121 | asks=asks, 122 | timestamp=recv_time) 123 | self.set_book(ob) 124 | 125 | async def _handle_ws_trades(self): 126 | pass 127 | 128 | async def cancel_all(self): 129 | pass 130 | 131 | 132 | if __name__ == '__main__': 133 | lp = asyncio.get_event_loop() 134 | bf = Bitfinex('bitfinex', {}) 135 | lp.run_until_complete(bf.set_orderbook_d('BTC/USDT')) 136 | -------------------------------------------------------------------------------- /arbcharm/exchange_api/huobipro.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import asyncio 6 | import gzip 7 | import json 8 | import time 9 | 10 | import websockets 11 | 12 | from arbcharm.models import BaseExchange, OrderBook, OrderRow 13 | 14 | 15 | class HuoBiPro(BaseExchange): 16 | async def set_orderbook_d(self, symbol): 17 | """ 18 | huobipro websockets doc: https://github.com/huobiapi/API_Docs/wiki/WS_request 19 | """ 20 | if symbol in self._orderbook_d: 21 | return 22 | self.clear_book(symbol) 23 | 24 | while True: 25 | try: 26 | await self._cache_book_ws(symbol) 27 | except websockets.exceptions.ConnectionClosed as e: 28 | self.logger.exception(error_class=e.__class__.__name__) # TODO fix connection error 29 | await asyncio.sleep(10) 30 | except: # pylint: disable=bare-except 31 | self.logger.exception() 32 | await asyncio.sleep(10) 33 | 34 | 35 | async def _cache_book_ws(self, symbol): 36 | url = 'wss://api.huobi.pro/ws' 37 | __sym = symbol.replace('/', '').lower() 38 | topic = 'market.{}.depth.step0'.format(__sym) 39 | async with websockets.connect(url) as ws: 40 | await self.send_data(ws, {'sub': topic, "freq-ms": 0}) 41 | while True: 42 | data = self.decompress_msg(await asyncio.wait_for(ws.recv(), timeout=30)) 43 | await self.data_handler(ws, symbol, data) 44 | 45 | async def data_handler(self, ws, symbol, data): 46 | if 'ping' in data: 47 | await self.send_data(ws, {'pong': int(time.time()*1000)}) 48 | return 49 | elif 'status' in data: 50 | if data['status'] != 'ok': 51 | self.logger.error(data) 52 | return 53 | elif 'ch' in data: 54 | self.handle_book(symbol, data) 55 | else: 56 | self.logger.error(event='unhandle_data', data=data) 57 | 58 | def handle_book(self, symbol, data): 59 | asks = [OrderRow(price=p, amount=a, count=1) for p, a in data['tick']['asks']] 60 | bids = [OrderRow(price=p, amount=a, count=1) for p, a in data['tick']['bids']] 61 | timestamp = data['tick']['ts']/1000 62 | ob = OrderBook(exchange=self, symbol=symbol, asks=asks, bids=bids, timestamp=timestamp) 63 | self.set_book(ob) 64 | 65 | @staticmethod 66 | async def send_data(ws, data): 67 | await ws.send(json.dumps(data)) 68 | 69 | @staticmethod 70 | def decompress_msg(msg): 71 | return json.loads(gzip.decompress(msg).decode()) 72 | 73 | @staticmethod 74 | def compress_msg(msg): 75 | return gzip.compress(json.dumps(msg).encode()) 76 | 77 | async def cancel_all(self): 78 | pass 79 | 80 | 81 | if __name__ == '__main__': 82 | ba = HuoBiPro('huobipro', {}) 83 | lp = asyncio.get_event_loop() 84 | lp.run_until_complete(ba.set_orderbook_d('BTC/USDT')) 85 | -------------------------------------------------------------------------------- /arbcharm/json_logger.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/7/10' 4 | 5 | # pylint: disable=arguments-differ 6 | 7 | import json 8 | import logging 9 | import traceback 10 | 11 | 12 | class JsonLogger(logging.Logger): 13 | def __init__(self, *args, **kwargs): 14 | self.__formater = None 15 | super().__init__(*args, **kwargs) 16 | self.propagate = False 17 | 18 | def debug(self, msg=None, **kwargs): 19 | msg, _args, _kwargs = self._parse_arge(msg, **kwargs) 20 | return super().debug(msg, *_args, **_kwargs) 21 | 22 | def info(self, msg=None, **kwargs): 23 | msg, _args, _kwargs = self._parse_arge(msg, **kwargs) 24 | return super().info(msg, *_args, **_kwargs) 25 | 26 | def warning(self, msg=None, **kwargs): 27 | msg, _args, _kwargs = self._parse_arge(msg, **kwargs) 28 | return super().warning(msg, *_args, **_kwargs) 29 | 30 | def error(self, msg=None, **kwargs): 31 | msg, _args, _kwargs = self._parse_arge(msg, **kwargs) 32 | return super().error(msg, *_args, **_kwargs) 33 | 34 | def exception(self, msg=None, **kwargs): 35 | kwargs['traceback'] = traceback.format_exc().splitlines() 36 | msg, _args, _kwargs = self._parse_arge(msg, **kwargs) 37 | return super().error(msg, *_args, **_kwargs) 38 | 39 | def critical(self, msg=None, **kwargs): 40 | msg, _args, _kwargs = self._parse_arge(msg, **kwargs) 41 | return super().critical(msg, *_args, **_kwargs) 42 | 43 | @staticmethod 44 | def _parse_arge(msg, **kwargs): 45 | if 'msg' in kwargs: 46 | raise ValueError('msg is not allowed in kwargs') 47 | _args = kwargs.pop('_args', []) 48 | _kwargs = kwargs.pop('_kwargs', {}) 49 | if msg is not None: 50 | kwargs['msg'] = msg 51 | msg = format_msg(kwargs) 52 | return msg, _args, _kwargs 53 | 54 | def addHandler(self, hdlr): 55 | if hasattr(self, 'formaater') and not isinstance(hdlr.formatter, JsonFormatter): 56 | raise ValueError('Please use JsonFormatter!') 57 | super().addHandler(hdlr) 58 | 59 | 60 | root = JsonLogger(logging.INFO) 61 | json_manager = logging.Manager(root) 62 | json_manager.loggerClass = JsonLogger 63 | 64 | JsonLogger.manager = json_manager 65 | JsonLogger.root = root 66 | 67 | 68 | def get_json_logger(name=None): 69 | if name: 70 | return JsonLogger.manager.getLogger(name) 71 | return root 72 | 73 | 74 | def format_msg(msg): 75 | """ 76 | transfor msg dict jsonable dict 77 | :param: msg 78 | :return: json 79 | """ 80 | def _format_msg(_m): 81 | if isinstance(_m, (str, int, float)): 82 | return _m 83 | elif isinstance(_m, bytes): 84 | return _m.decode() 85 | elif isinstance(_m, dict): 86 | return {_format_msg(k): _format_msg(v) for k, v in _m.items()} 87 | elif isinstance(_m, (tuple, list)): 88 | return [_format_msg(i) for i in _m] 89 | else: 90 | try: 91 | return str(_m) 92 | except: 93 | raise ValueError("Not allowed object, object must have __str__ method!") 94 | 95 | return json.dumps(_format_msg(msg)) 96 | 97 | 98 | class JsonFormatter(logging.Formatter): 99 | def __init__(self, fmt_dict: dict = None, datefmt=None, style='%'): 100 | if not fmt_dict: 101 | fmt_dict = { 102 | 'asctime': '%(asctime)s', 103 | 'level': '%(levelname)s', 104 | 'message': '%(message)s', 105 | } 106 | if not isinstance(fmt_dict, dict): 107 | raise ValueError('fmt_dict must be a dict') 108 | 109 | fmt = format_msg(fmt_dict).replace('"%(message)s"', '%(message)s') 110 | super().__init__(fmt, datefmt, style) 111 | 112 | 113 | _default_formatter = JsonFormatter() 114 | _default_handler = logging.StreamHandler() 115 | _default_handler.setLevel(logging.INFO) 116 | _default_handler.formatter = _default_formatter 117 | root.addHandler(_default_handler) 118 | 119 | 120 | if __name__ == '__main__': 121 | logging.basicConfig(level=logging.DEBUG) 122 | logger = get_json_logger('test') 123 | logger.addHandler(_default_handler) 124 | logger.info(word='Hello world') 125 | try: 126 | raise RuntimeError() 127 | except RuntimeError: 128 | logger.exception('Hello world') 129 | -------------------------------------------------------------------------------- /arbcharm/models.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import asyncio 6 | import time 7 | from typing import Dict, List 8 | 9 | import ccxt.async_support as ccxt 10 | from arbcharm.tools import get_logger 11 | 12 | 13 | class BaseExchange: 14 | def __init__(self, name, config: Dict): 15 | self.name = name 16 | self.logger = get_logger(self.name) 17 | self.ccxt_exchange = getattr(ccxt, self.name)(config) 18 | self._orderbook_d = {} 19 | self.alert_event_dict = {} # sym: event 20 | 21 | def set_book(self, ob: "OrderBook"): 22 | if ob.symbol in self.alert_event_dict: 23 | self.alert_event_dict[ob.symbol].set() 24 | self._orderbook_d[ob.symbol] = ob 25 | 26 | def get_book(self, symbol) -> "OrderBook": 27 | return self._orderbook_d.get(symbol) 28 | 29 | def clear_book(self, symbol): 30 | self._orderbook_d[symbol] = None 31 | 32 | async def wait_book_update(self, symbol): 33 | event = self.alert_event_dict.get(symbol) 34 | if not event: 35 | self.alert_event_dict[symbol] = asyncio.Event() 36 | await self.alert_event_dict[symbol].wait() 37 | 38 | def reset_book_update(self, symbol): 39 | self.alert_event_dict[symbol].clear() 40 | 41 | async def set_orderbook_d(self, symbol): 42 | raise NotImplementedError() 43 | 44 | async def cancel_all(self): 45 | raise NotImplementedError() 46 | 47 | def __str__(self): 48 | return self.name 49 | 50 | 51 | def _get_exchange_factory(): 52 | 53 | single_instance = {} 54 | 55 | def _get_exchange(name, config) -> BaseExchange: 56 | if name not in single_instance: 57 | if name == 'bitfinex': 58 | from arbcharm.exchange_api.bitfinex import Bitfinex 59 | exc_cls = Bitfinex 60 | elif name == 'huobipro': 61 | from arbcharm.exchange_api.huobipro import HuoBiPro 62 | exc_cls = HuoBiPro 63 | elif name == 'binance': 64 | from arbcharm.exchange_api.binance import Binance 65 | exc_cls = Binance 66 | else: 67 | exc_cls = BaseExchange 68 | single_instance[name] = exc_cls(name, config) 69 | return single_instance[name] 70 | 71 | return _get_exchange 72 | 73 | 74 | get_exchange = _get_exchange_factory() 75 | 76 | 77 | class Trade: 78 | def __init__( 79 | self, 80 | *, 81 | bid_exc: BaseExchange, 82 | ask_exc: BaseExchange, 83 | bid_price, 84 | ask_price, 85 | amount 86 | ): 87 | """ 88 | :param bid_exc: bid_price所属的交易所 89 | :param ask_exc: ask_price所属的交易所 90 | :param bid_price: 成交的orderbook买档价格 91 | :param ask_price: 成交的orderbook卖档价格 92 | :param amount: 成交量 93 | """ 94 | assert bid_price >= ask_price 95 | self.bid_exc = bid_exc 96 | self.ask_exc = ask_exc 97 | self.bid_price = bid_price 98 | self.ask_price = ask_price 99 | self.amount = amount 100 | 101 | def __str__(self): 102 | return 'Trade:[{}]'.format(self.__dict__) 103 | 104 | def to_dict(self): 105 | return self.__dict__ 106 | 107 | 108 | class Order: 109 | SIDE_BUY = 'buy' 110 | SIDE_SELL = 'sell' 111 | 112 | def __init__(self, *, exc: BaseExchange, symbol, price, amount, side, otype='limit'): 113 | self.exc = exc 114 | assert side in (self.SIDE_SELL, self.SIDE_BUY) 115 | self.symbol = symbol 116 | self.side = side 117 | self.price = price 118 | self.amount = amount 119 | self.otype = otype 120 | 121 | def to_dict(self): 122 | return self.__dict__ 123 | 124 | def __str__(self): 125 | return 'Order:[{}]'.format(self.to_dict()) 126 | 127 | 128 | class OrderRow: 129 | def __init__(self, *, price: float, amount: float, count: int): 130 | self.price = price 131 | self.count = count 132 | self.amount = amount 133 | 134 | 135 | class OrderBook: 136 | def __init__( 137 | self, 138 | *, 139 | exchange: BaseExchange, 140 | symbol: str, 141 | asks: List[OrderRow], 142 | bids: List[OrderRow], 143 | timestamp: float 144 | ): 145 | self.exchange = exchange 146 | self.symbol = symbol 147 | self.asks = asks 148 | self.bids = bids 149 | self.timestamp = timestamp if timestamp else time.time() 150 | 151 | def remove_bad_price(self, price): 152 | self.asks = [row for row in self.asks if row.price >= price] 153 | self.bids = [row for row in self.bids if row.price <= price] 154 | -------------------------------------------------------------------------------- /arbcharm/settings.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import os 6 | 7 | MODE = os.getenv('ARBCHARM_MODE', 'test') # or prd 8 | 9 | ARBITRAGE_OPPORTUNITY_RATE = 0.004 10 | 11 | ARBCHARM_AMOUNT_MULTIPLIER = 0.01 12 | 13 | CACHE_ORDER_ROW_LENGTH = 20 14 | 15 | ARB_CONF = { 16 | 'BTC/USDT': { 17 | 'binance': { 18 | 'apiKey': os.getenv('binance_apiKey', '请替换我'), 19 | 'secret': os.getenv('binance_secret', '请替换我'), 20 | 'min_amount': 0.002 21 | }, 22 | 'huobipro': { 23 | 'apiKey': os.getenv('huobipro_apiKey', '请替换我'), 24 | 'secret': os.getenv('huobipro_secret', '请替换我'), 25 | 'min_amount': 0.002 26 | }, 27 | 'bitfinex': { 28 | 'apiKey': os.getenv('binance_apiKey', '请替换我'), 29 | 'secret': os.getenv('binance_secret', '请替换我'), 30 | 'min_amount': 0.002 31 | }, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /arbcharm/tools.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/12' 4 | 5 | import logging 6 | import sys 7 | import time 8 | 9 | from arbcharm.json_logger import JsonFormatter, get_json_logger 10 | 11 | 12 | def bisect_right(a, x, key=None, lo=0, hi=None, rv=False): # pylint: disable=too-many-arguments 13 | """ 14 | binary search 15 | :param a: a list 16 | :param x: target value 17 | :param key: sort_attr 18 | :param lo: start_index 19 | :param hi: end_index 20 | :param rv: reverse 21 | :return: index of insert 22 | """ 23 | key = key if key else lambda i: i 24 | if lo < 0: 25 | raise ValueError('lo must be non-negative') 26 | if hi is None: 27 | hi = len(a) 28 | while lo < hi: 29 | mid = (lo+hi)//2 30 | if rv: 31 | if key(x) > key(a[mid]): 32 | hi = mid 33 | else: 34 | lo = mid+1 35 | else: 36 | if key(x) < key(a[mid]): 37 | hi = mid 38 | else: 39 | lo = mid+1 40 | return lo 41 | 42 | 43 | def get_logger(name='root'): 44 | class LevelFilter(object): 45 | def __init__(self, level): 46 | self.__level = level 47 | 48 | def filter(self, log_record): 49 | return log_record.levelno <= self.__level 50 | 51 | logger = get_json_logger(name) 52 | logger.propagate = False 53 | if not logger.handlers: 54 | formatter = JsonFormatter( 55 | { 56 | 'logger': name, 57 | 'asctime': '%(asctime)s', 58 | 'level': '%(levelname)s', 59 | 'message': '%(message)s', 60 | } 61 | ) 62 | 63 | sh_i = logging.StreamHandler(sys.stdout) 64 | sh_i.setLevel(logging.INFO) 65 | sh_i.setFormatter(formatter) 66 | sh_i.addFilter(LevelFilter(logging.INFO)) 67 | 68 | sh_e = logging.StreamHandler(sys.stderr) 69 | sh_e.setFormatter(formatter) 70 | sh_e.setLevel(logging.ERROR) 71 | 72 | logger.addHandler(sh_i) 73 | logger.addHandler(sh_e) 74 | return logger 75 | 76 | 77 | def rate_limit_generator(): 78 | """ 79 | usage: 80 | r = rate_limit_generator() 81 | r.send(None) # start generator 82 | r.send(('place_one', 10)) # input key and time interval 83 | yield True while allow to access 84 | """ 85 | loc_time_map = {} # {key: timestamp} 86 | key, min_time_interval = yield True 87 | while True: 88 | early_time = loc_time_map.get(key) 89 | now = time.time() 90 | if early_time: 91 | if (now - early_time) > min_time_interval: 92 | access = True 93 | loc_time_map[key] = time.time() 94 | else: 95 | access = False 96 | else: 97 | loc_time_map[key] = time.time() 98 | access = True 99 | key, min_time_interval = yield access 100 | 101 | 102 | def print_cost_time(func): 103 | def _func(*args, **kwargs): 104 | t1 = time.time() 105 | res = func(*args, **kwargs) 106 | print(time.time() - t1) 107 | return res 108 | return _func() 109 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker rmi arbcharm || echo 4 | docker build -t arbcharm . -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ccxt==1.17.390 2 | aiohttp==3.4.4 3 | websockets==6.0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | __author__ = 'Rick Zhang' 3 | __time__ = '2018/10/14' 4 | 5 | import sys 6 | from codecs import open as openc 7 | from os import path 8 | 9 | from setuptools import find_packages, setup 10 | 11 | here = path.abspath(path.dirname(__file__)) 12 | 13 | 14 | py_version = sys.version_info 15 | assert py_version.major == 3 16 | assert py_version.minor >= 5 17 | 18 | with openc(path.join(here, 'README.md'), encoding='utf-8') as f: 19 | long_description = f.read() 20 | 21 | with openc('requirements.txt', 'r') as f: 22 | require_list = [line.strip() for line in f] 23 | 24 | setup( 25 | name='arbcharm', 26 | version='1.0.0', 27 | description='Charm Of Arbitrage', 28 | long_description=long_description, 29 | url='https://github.com/RunningToTheEdgeOfTheWorld/arbcharm', 30 | author='Rick Zhang', 31 | author_email='rickzhangjie@gmail.com', 32 | license='MIT', 33 | classifiers=[ 34 | 'Development Status :: 5 - Production/Stable', 35 | 'Intended Audience :: Developers', 36 | 'Topic :: Software Development :: Build Tools', 37 | 'License :: OSI Approved :: MIT License', 38 | 'Programming Language :: Python :: 3', 39 | ], 40 | keywords='', 41 | packages=find_packages(), 42 | install_requires=require_list, 43 | extras_require={ 44 | 'dev': [ 45 | ], 46 | 'test': ['coverage'], 47 | }, 48 | package_data={ 49 | '': ['*.*'], 50 | }, 51 | ) 52 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker stop arbcharm || echo 4 | docker rm -v arbcharm || echo 5 | docker run -dt --name arbcharm -e ARBCHARM_MODE=prd arbcharm --------------------------------------------------------------------------------