├── bt_ig ├── __init__.py ├── igdata.py ├── igbroker.py └── igstore.py ├── .gitignore ├── LICENSE.md ├── historical-sample.py ├── sample.py └── README.md /bt_ig/__init__.py: -------------------------------------------------------------------------------- 1 | from .igstore import * 2 | from .igdata import * 3 | from .igbroker import * 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | 47 | # Translations 48 | *.mo 49 | *.pot 50 | 51 | # Django stuff: 52 | *.log 53 | 54 | # Sphinx documentation 55 | docs/_build/ 56 | 57 | # PyBuilder 58 | target/ 59 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2017 Dave Vallance 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /historical-sample.py: -------------------------------------------------------------------------------- 1 | import backtrader as bt 2 | from datetime import datetime 3 | import logging 4 | from bt_ig import IGStore 5 | from bt_ig import IGData 6 | 7 | 8 | api_key = 'INSERT YOUR API KEY' 9 | usr = 'INSERT YOUR USERNAME' 10 | pwd = 'INSERT YOU PASSWORD' 11 | acc = "INSERT YOUR ACC NUM" 12 | 13 | class IGTest(bt.Strategy): 14 | ''' 15 | Simple strat to test IGStore. 16 | ''' 17 | 18 | def __init__(self): 19 | pass 20 | 21 | def next(self): 22 | dt = self.datetime.datetime() 23 | bar = len(self) 24 | print('{}: O: {} H: {} L: {} C:{}'.format(dt, self.data.open[0], 25 | self.data.high[0],self.data.low[0],self.data.close[0])) 26 | 27 | 28 | ## NOTIFICATIONS 29 | def notify_order(self,order): 30 | if order.status == order.Rejected: 31 | print('Order Rejected') 32 | 33 | def notify_data(self, data, status, *args, **kwargs): 34 | print('DATA NOTIF: {}: {}'.format(data._getstatusname(status), ','.join(args))) 35 | 36 | def notify_store(self, msg, *args, **kwargs): 37 | print('STORE NOTIF: {}'.format(msg)) 38 | 39 | #Logging - Uncomment to see ig_trading library logs 40 | #logging.basicConfig(level=logging.DEBUG) 41 | 42 | tframes = dict( 43 | seconds = bt.TimeFrame.Seconds, 44 | minutes=bt.TimeFrame.Minutes, 45 | daily=bt.TimeFrame.Days, 46 | weekly=bt.TimeFrame.Weeks, 47 | monthly=bt.TimeFrame.Months) 48 | 49 | #Create an instance of cerebro 50 | cerebro = bt.Cerebro() 51 | 52 | #Setup IG 53 | igs = IGStore(usr=usr, pwd=pwd, token=api_key, account=sbet) 54 | broker = igs.getbroker() 55 | cerebro.setbroker(broker) 56 | 57 | data = igs.getdata(dataname='CS.D.GBPUSD.TODAY.IP', historical=True, 58 | timeframe=tframes['minutes'], compression=60, 59 | fromdate=datetime(2017,11,1,8,0), todate=datetime(2017,11,1,12,0)) #Get 10 bars for testing. 60 | #Replay the data in forward test envirnoment so we can act quicker 61 | cerebro.adddata(data, name='GBP_USD') 62 | 63 | #Add our strategy 64 | cerebro.addstrategy(IGTest) 65 | 66 | # Run over everything 67 | cerebro.run() 68 | -------------------------------------------------------------------------------- /sample.py: -------------------------------------------------------------------------------- 1 | import backtrader as bt 2 | from datetime import datetime 3 | import logging 4 | from bt_ig import IGStore 5 | from bt_ig import IGData 6 | 7 | 8 | api_key = 'INSERT YOUR API KEY' 9 | usr = 'INSERT YOUR USERNAME' 10 | pwd = 'INSERT YOU PASSWORD' 11 | acc = "INSERT YOUR ACC NUM" 12 | 13 | class IGTest(bt.Strategy): 14 | ''' 15 | Simple strat to test IGStore. 16 | ''' 17 | def __init__(self): 18 | pass 19 | 20 | def next(self): 21 | dt = self.datetime.datetime() 22 | bar = len(self) 23 | print('{}: O: {} H: {} L: {} C:{}'.format(dt, self.data.open[0], 24 | self.data.high[0],self.data.low[0],self.data.close[0])) 25 | 26 | 27 | if bar == 1: 28 | print('Testing Get Cash!') 29 | cash = self.broker.getcash() 30 | print("Current Cash: {}".format(cash)) 31 | 32 | print('Testing Get Value!') 33 | value = self.broker.getvalue() 34 | print("Current Value: {}".format(value)) 35 | 36 | if bar == 2: 37 | print('Testing Simple Order!') 38 | pos = self.broker.getposition(self.data) 39 | self.buy(size=5) 40 | if bar == 3: 41 | print('Closing Order') 42 | pos = self.broker.getposition(self.data) 43 | print('Open Position Size = {}'.format(pos.size)) 44 | cOrd = self.close() 45 | print('Closing Order Size = {}'.format(cOrd.size)) 46 | 47 | # CHECK CASH AND EQUITY ARE AUTMATICALLY BEING UPDATED 48 | cash = self.broker.getcash() 49 | value = self.broker.getvalue() 50 | print("Current Cash: {}".format(cash)) 51 | print("Current Value: {}".format(value)) 52 | 53 | if bar == 4: 54 | print('Testing Limit Order') 55 | limit_price = self.data.close[0] * 0.9 #Buy better price 10% lower 56 | self.limit_ord = self.buy(exectype=bt.Order.Limit, price=limit_price, size=5) 57 | 58 | if bar == 5: 59 | print('Cancelling Limit Order') 60 | self.cancel(self.limit_ord) 61 | 62 | if bar ==6: 63 | print('Testing Stop Order') 64 | stop_price = self.data.close[0] * 0.9 #buy at a worse price 10% lower 65 | self.stop_ord = self.buy(exectype=bt.Order.Limit, price=stop_price, size=5) 66 | 67 | if bar == 7: 68 | print('Cancelling Stop Order') 69 | self.cancel(self.stop_ord) 70 | 71 | if bar == 8: 72 | print("Test Finished") 73 | self.env.runstop() 74 | 75 | 76 | ## NOTIFICATIONS 77 | def notify_order(self,order): 78 | if order.status == order.Rejected: 79 | print('Order Rejected') 80 | 81 | def notify_data(self, data, status, *args, **kwargs): 82 | print('DATA NOTIF: {}: {}'.format(data._getstatusname(status), ','.join(args))) 83 | 84 | def notify_store(self, msg, *args, **kwargs): 85 | print('STORE NOTIF: {}'.format(msg)) 86 | 87 | 88 | #Logging - Uncomment to see ig_trading library logs 89 | #logging.basicConfig(level=logging.DEBUG) 90 | 91 | tframes = dict( 92 | seconds = bt.TimeFrame.Seconds, 93 | minutes=bt.TimeFrame.Minutes, 94 | daily=bt.TimeFrame.Days, 95 | weekly=bt.TimeFrame.Weeks, 96 | monthly=bt.TimeFrame.Months) 97 | 98 | #Create an instance of cerebro 99 | cerebro = bt.Cerebro() 100 | 101 | #Setup IG 102 | igs = IGStore(usr=usr, pwd=pwd, token=api_key, account=sbet) 103 | broker = igs.getbroker() 104 | cerebro.setbroker(broker) 105 | 106 | 107 | data = igs.getdata(dataname='CS.D.GBPUSD.TODAY.IP') 108 | #Replay the data in forward test envirnoment so we can act quicker 109 | cerebro.resampledata(data, timeframe=tframes['seconds'], compression=15, name='GBP_USD') 110 | 111 | #Add our strategy 112 | cerebro.addstrategy(IGTest) 113 | 114 | # Run over everything 115 | cerebro.run() 116 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bt-ig-store 2 | 3 | IG markets store for Backtrader 4 | 5 | ## Warning UNDER Development 6 | Currently integration is very limited. Only streaming is working. 7 | 8 | __Requires__ 9 | 10 | 1 - trading_ig 11 | 12 | A lightweight Python library that can be used to get live data from IG Markets REST and STREAM API 13 | 14 | - https://github.com/ig-python/ig-markets-api-python-library 15 | 16 | 2 - Backtrader 17 | 18 | Python Backtesting library for trading strategies 19 | - https://github.com/mementum/backtrader 20 | - https://www.backtrader.com 21 | 22 | 3 - Pandas. 23 | 24 | - http://pandas.pydata.org/ 25 | 26 | ## Current Functionality 27 | 28 | - Basic instrument streaming 29 | - Performs the open position check when initialized to track existing positions 30 | - Opening and closing of simple Market orders using the self.buy() and self.close() is now supported. 31 | - Set IG currency code as a store initialization parameter (Default GBP). 32 | - Stop order creation and cancellation supported. 33 | - Limit order creation and cancellation supported. 34 | - expiry, guaranteed_stop, time_in_force, and good_till_date parameters 35 | - Improved streamer setup. Can use the same streamer for multiple get_instruments rather than creating multiple streamers 36 | - Manual pull of cash and value 37 | - Account cash and value live streaming 38 | - __*FIX:*__ Level Set during order creation caused MARKET Orders to be rejected can now all be passed as key word arguments during order creation and handled appropriately. Defaults are used where no kwarg is passed. 39 | - Historical data download for backtesting 40 | - Printing of remaining allowance (will later be updated to be a store notification, once notifications are working) 41 | - Sample script for historical data download testing. 42 | - Store and Data notifications tested and working 43 | - Samples updated to print Store and Data notifications 44 | - Updated granualarity check to only make the check if Backfilling, Historical or Backfill_Start are required. This allows you to work with any timeframe using tick data for live trading only. 45 | - __*New:*__ Backfilling from start now supported 46 | - __*New:*__ Parameter `backfill_bars` allows you to set how many bars to backfill from the start. Since the IG api places restrictions on the number of historical data points a user can download per week, this parameter allows a user to limit the amount of historical data downloaded to only the required amount. 47 | 48 | NOTE: Backfilling is not yet supported. The check mentioned above is in preparation for those features being supported. 49 | 50 | ### Important! 51 | To use historical data, you will need to use the forked trading_ig API from my profile. 52 | 53 | If you do not, you will receive a `ValueError` when requesting historical data. A 54 | pull request has been submitted to the official project and this warning will be 55 | removed if/when it is accepted. 56 | 57 | ## Known Issues 58 | 59 | See the issues tab: https://github.com/Dave-Vallance/bt-ig-store/issues 60 | 61 | __Value for header {Version: 2} must be of type str or bytes, not __ 62 | 63 | This issue is documented in the issues tab. If you see it, you should make sure you 64 | have the latest trading_ig module code. 65 | 66 | ## Streaming Example 67 | 68 | ```python 69 | import backtrader as bt 70 | from datetime import datetime 71 | import logging 72 | from bt_ig import IGStore 73 | from bt_ig import IGData 74 | 75 | 76 | api_key = 'INSERT YOUR API KEY' 77 | usr = 'INSERT YOUR USERNAME' 78 | pwd = 'INSERT YOU PASSWORD' 79 | acc = "INSERT YOUR ACC NUM" 80 | 81 | class IGTest(bt.Strategy): 82 | ''' 83 | Simple strat to test IGStore. 84 | ''' 85 | 86 | def __init__(self): 87 | self._live = False #Track whether we have live data to avoid entering on backfill 88 | self._last_hist_bar = None #To track the last delivered historical bar 89 | 90 | def next(self): 91 | dt = self.datetime.datetime() 92 | bar = len(self) 93 | lhst = self._last_hist_bar 94 | print('{}: O: {} H: {} L: {} C:{}'.format(dt, self.data.open[0], 95 | self.data.high[0],self.data.low[0],self.data.close[0])) 96 | if self._live: 97 | if bar == lhst + 1: 98 | print('Testing Get Cash!') 99 | cash = self.broker.getcash() 100 | print("Current Cash: {}".format(cash)) 101 | 102 | print('Testing Get Value!') 103 | value = self.broker.getvalue() 104 | print("Current Value: {}".format(value)) 105 | 106 | if bar == lhst + 2: 107 | print('Testing Simple Order!') 108 | pos = self.broker.getposition(self.data) 109 | self.buy(size=5) 110 | if bar == lhst + 3: 111 | print('Closing Order') 112 | pos = self.broker.getposition(self.data) 113 | print('Open Position Size = {}'.format(pos.size)) 114 | cOrd = self.close() 115 | print('Closing Order Size = {}'.format(cOrd.size)) 116 | 117 | # CHECK CASH AND EQUITY ARE AUTMATICALLY BEING UPDATED 118 | cash = self.broker.getcash() 119 | value = self.broker.getvalue() 120 | print("Current Cash: {}".format(cash)) 121 | print("Current Value: {}".format(value)) 122 | 123 | if bar == lhst +4: 124 | print('Testing Limit Order') 125 | limit_price = self.data.close[0] * 0.9 #Buy better price 10% lower 126 | self.limit_ord = self.buy(exectype=bt.Order.Limit, price=limit_price, size=5) 127 | 128 | if bar == lhst + 5: 129 | print('Cancelling Limit Order') 130 | self.cancel(self.limit_ord) 131 | 132 | if bar == lhst + 6: 133 | print('Testing Stop Order') 134 | stop_price = self.data.close[0] * 0.9 #buy at a worse price 10% lower 135 | self.stop_ord = self.buy(exectype=bt.Order.Limit, price=stop_price, size=5) 136 | 137 | if bar == lhst + 7: 138 | print('Cancelling Stop Order') 139 | self.cancel(self.stop_ord) 140 | 141 | if bar == lhst + 8: 142 | print("Test Finished") 143 | self.env.runstop() 144 | 145 | ## NOTIFICATIONS 146 | def notify_order(self,order): 147 | if order.status == order.Rejected: 148 | print('ORDER NOTIF: Order Rejected') 149 | 150 | def notify_data(self, data, status, *args, **kwargs): 151 | print('DATA NOTIF: {}: {}'.format(data._getstatusname(status), ','.join(args))) 152 | if status == data.LIVE: 153 | self._live = True 154 | self._last_hist_bar = len(self) 155 | 156 | def notify_store(self, msg, *args, **kwargs): 157 | print('STORE NOTIF: {}'.format(msg)) 158 | 159 | #Logging - Uncomment to see ig_trading library logs 160 | #logging.basicConfig(level=logging.DEBUG) 161 | 162 | tframes = dict( 163 | seconds = bt.TimeFrame.Seconds, 164 | minutes=bt.TimeFrame.Minutes, 165 | daily=bt.TimeFrame.Days, 166 | weekly=bt.TimeFrame.Weeks, 167 | monthly=bt.TimeFrame.Months) 168 | 169 | #Create an instance of cerebro 170 | cerebro = bt.Cerebro() 171 | 172 | #Setup IG 173 | igs = IGStore(usr=usr, pwd=pwd, token=api_key, account=sbet) 174 | broker = igs.getbroker() 175 | cerebro.setbroker(broker) 176 | 177 | 178 | data = igs.getdata(dataname='CS.D.GBPUSD.TODAY.IP', backfill_start=True, backfill_bars=50) 179 | #Replay the data in forward test envirnoment so we can act quicker 180 | #cerebro.resampledata(data, timeframe=tframes['seconds'], compression=15, name='GBP_USD') 181 | cerebro.resampledata(data, timeframe=tframes['minutes'], compression=1, name='GBP_USD') 182 | 183 | #Add our strategy 184 | cerebro.addstrategy(IGTest) 185 | 186 | # Run over everything 187 | cerebro.run() 188 | 189 | ``` 190 | 191 | ## Historical Data Example 192 | 193 | ```python 194 | import backtrader as bt 195 | from datetime import datetime 196 | import logging 197 | from bt_ig import IGStore 198 | from bt_ig import IGData 199 | 200 | 201 | api_key = 'INSERT YOUR API KEY' 202 | usr = 'INSERT YOUR USERNAME' 203 | pwd = 'INSERT YOU PASSWORD' 204 | acc = "INSERT YOUR ACC NUM" 205 | 206 | class IGTest(bt.Strategy): 207 | ''' 208 | Simple strat to test IGStore. 209 | ''' 210 | 211 | def __init__(self): 212 | pass 213 | 214 | def next(self): 215 | dt = self.datetime.datetime() 216 | bar = len(self) 217 | print('{}: O: {} H: {} L: {} C:{}'.format(dt, self.data.open[0], 218 | self.data.high[0],self.data.low[0],self.data.close[0])) 219 | 220 | 221 | ## NOTIFICATIONS 222 | def notify_order(self,order): 223 | if order.status == order.Rejected: 224 | print('Order Rejected') 225 | 226 | def notify_data(self, data, status, *args, **kwargs): 227 | print('DATA NOTIF: {}: {}'.format(data._getstatusname(status), ','.join(args))) 228 | 229 | def notify_store(self, msg, *args, **kwargs): 230 | print('STORE NOTIF: {}'.format(msg)) 231 | 232 | #Logging - Uncomment to see ig_trading library logs 233 | #logging.basicConfig(level=logging.DEBUG) 234 | 235 | tframes = dict( 236 | seconds = bt.TimeFrame.Seconds, 237 | minutes=bt.TimeFrame.Minutes, 238 | daily=bt.TimeFrame.Days, 239 | weekly=bt.TimeFrame.Weeks, 240 | monthly=bt.TimeFrame.Months) 241 | 242 | #Create an instance of cerebro 243 | cerebro = bt.Cerebro() 244 | 245 | #Setup IG 246 | igs = IGStore(usr=usr, pwd=pwd, token=api_key, account=sbet) 247 | broker = igs.getbroker() 248 | cerebro.setbroker(broker) 249 | 250 | data = igs.getdata(dataname='CS.D.GBPUSD.TODAY.IP', historical=True, 251 | timeframe=tframes['minutes'], compression=60, 252 | fromdate=datetime(2017,11,1,8,0), todate=datetime(2017,11,1,12,0)) #Get 10 bars for testing. 253 | #Replay the data in forward test envirnoment so we can act quicker 254 | cerebro.adddata(data, name='GBP_USD') 255 | 256 | #Add our strategy 257 | cerebro.addstrategy(IGTest) 258 | 259 | # Run over everything 260 | cerebro.run() 261 | 262 | ``` 263 | -------------------------------------------------------------------------------- /bt_ig/igdata.py: -------------------------------------------------------------------------------- 1 | from __future__ import (absolute_import, division, print_function, 2 | unicode_literals) 3 | 4 | from datetime import datetime, timedelta 5 | import pytz, logging 6 | from backtrader.feed import DataBase 7 | from backtrader import TimeFrame, date2num, num2date 8 | from backtrader.utils.py3 import (integer_types, queue, string_types, 9 | with_metaclass) 10 | from backtrader.metabase import MetaParams 11 | from . import igstore 12 | 13 | 14 | class MetaIGData(DataBase.__class__): 15 | def __init__(cls, name, bases, dct): 16 | '''Class has already been created ... register''' 17 | # Initialize the class 18 | super(MetaIGData, cls).__init__(name, bases, dct) 19 | 20 | # Register with the store 21 | igstore.IGStore.DataCls = cls 22 | 23 | class IGData(with_metaclass(MetaIGData, DataBase)): 24 | ''' 25 | params: 26 | 27 | ''' 28 | #TODO insert params 29 | params = ( 30 | ('historical', False), 31 | ('useask', False), 32 | ('bidask', True), 33 | ('backfill_start', False), # do backfilling at the start 34 | ('backfill_bars', 0), #number of bars to backfill to avoid using too much allowance 35 | ('backfill', False), # do backfilling when reconnecting 36 | ('reconnections', -1), 37 | ('qcheck', 5) 38 | ) 39 | 40 | # States for the Finite State Machine in _load 41 | _ST_FROM, _ST_START, _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(5) 42 | 43 | _store = igstore.IGStore 44 | 45 | def islive(self): 46 | '''Returns ``True`` to notify ``Cerebro`` that preloading and runonce 47 | should be deactivated''' 48 | return True 49 | 50 | def __init__(self, **kwargs): 51 | self.o = self._store(**kwargs) 52 | 53 | 54 | def setenvironment(self, env): 55 | '''Receives an environment (cerebro) and passes it over to the store it 56 | belongs to''' 57 | super(IGData, self).setenvironment(env) 58 | env.addstore(self.o) 59 | 60 | def start(self): 61 | '''Starts the IG connecction and gets the real contract and 62 | contractdetails if it exists''' 63 | super(IGData, self).start() 64 | 65 | # Create attributes as soon as possible 66 | self._statelivereconn = False # if reconnecting in live state 67 | self._storedmsg = dict() # keep pending live message (under None) 68 | self.qlive = queue.Queue() 69 | self._state = self._ST_OVER 70 | 71 | # Kickstart store and get queue to wait on 72 | self.o.start(data=self) 73 | 74 | # check if the granularity is supported 75 | if self.p.historical or self.p.backfill or self.p.backfill_start: 76 | otf = self.o.get_granularity(self._timeframe, self._compression) 77 | if otf is None: 78 | self.put_notification(self.NOTSUPPORTED_TF) 79 | self._state = self._ST_OVER 80 | return 81 | 82 | self._start_finish() 83 | self._state = self._ST_START # initial state for _load 84 | self._st_start() 85 | 86 | self._reconns = 0 87 | 88 | def _st_start(self, instart=True, tmout=None): 89 | if self.p.historical: 90 | self.put_notification(self.DELAYED) 91 | dtend = None 92 | if self.todate < float('inf'): 93 | dtend = self.todate 94 | 95 | dtbegin = None 96 | if self.fromdate > float('-inf'): 97 | dtbegin = self.fromdate 98 | 99 | self.qhist = self.o.candles( 100 | self.p.dataname, dtbegin, dtend, 101 | self._timeframe, self._compression) 102 | 103 | self._state = self._ST_HISTORBACK 104 | return True 105 | 106 | # streaming prices returns the same queue the streamer is using. 107 | self.qlive = self.o.streaming_prices(self.p.dataname, tmout=tmout) 108 | 109 | if instart: 110 | self._statelivereconn = self.p.backfill_start 111 | else: 112 | self._statelivereconn = self.p.backfill 113 | 114 | if self._statelivereconn: 115 | self.put_notification(self.DELAYED) 116 | 117 | self._state = self._ST_LIVE 118 | if instart: 119 | self._reconns = self.p.reconnections 120 | 121 | return True # no return before - implicit continue 122 | 123 | def stop(self): 124 | '''Stops and tells the store to stop''' 125 | super(IGData, self).stop() 126 | self.o.stop() 127 | 128 | def haslivedata(self): 129 | return bool(self._storedmsg or self.qlive) # do not return the objs 130 | 131 | 132 | def _load(self): 133 | ''' 134 | steps 135 | 136 | 1 - check if we status live. If so process message 137 | - Check for error codes in message and change status appropriately 138 | - Process the message as long as the status is not trying to reconnect 139 | - Setup a backfill if data is missing. 140 | 2 - If not, is the status set to perform a backfill? 141 | 142 | ''' 143 | 144 | if self._state == self._ST_OVER: 145 | return False 146 | 147 | while True: 148 | if self._state == self._ST_LIVE: 149 | try: 150 | msg = (self._storedmsg.pop(None, None) or 151 | self.qlive.get(timeout=self._qcheck)) 152 | except queue.Empty: 153 | return None # indicate timeout situation 154 | 155 | if msg is None: # Conn broken during historical/backfilling 156 | self.put_notification(self.CONNBROKEN) 157 | self.put_notification(self.DISCONNECTED) 158 | self._state = self._ST_OVER 159 | return False # failed 160 | 161 | #TODO handle error messages in feed 162 | 163 | 164 | #Check for empty data. Sometimes all the fields return None... 165 | if msg['UTM'] is None: 166 | return None 167 | 168 | #self._reconns = self.p.reconnections 169 | 170 | # Process the message according to expected return type 171 | if not self._statelivereconn: 172 | if self._laststatus != self.LIVE: 173 | if self.qlive.qsize() <= 1: # very short live queue 174 | self.put_notification(self.LIVE) 175 | 176 | ret = self._load_tick(msg) 177 | if ret: 178 | return True 179 | 180 | 181 | # could not load bar ... go and get new one 182 | continue 183 | 184 | dtend = None 185 | if len(self) > 1: 186 | # len == 1 ... forwarded for the 1st time 187 | dtbegin = self.datetime.datetime(-1) 188 | elif self.fromdate > float('-inf'): 189 | dtbegin = num2date(self.fromdate) 190 | else: # 1st bar and no begin set 191 | # passing None to fetch max possible in 1 request 192 | dtbegin = None 193 | 194 | if dtbegin: 195 | dtend = datetime.utcfromtimestamp(int(msg['time']) / 10 ** 6) 196 | 197 | self.qhist = self.o.candles( 198 | self.p.dataname, dtbegin, dtend, 199 | self._timeframe, self._compression) 200 | else: 201 | self.qhist = self.o.candles(self.p.dataname, dtbegin, dtend, 202 | self._timeframe, self._compression, numpoints=True, bars=self.p.backfill_bars) 203 | 204 | self._state = self._ST_HISTORBACK 205 | self._statelivereconn = False # no longer in live 206 | 207 | continue 208 | 209 | 210 | 211 | elif self._state == self._ST_HISTORBACK: 212 | msg = self.qhist.get() 213 | if msg is None: # Conn broken during historical/backfilling 214 | # Situation not managed. Simply bail out 215 | self.put_notification(self.DISCONNECTED) 216 | self._state = self._ST_OVER 217 | return False # error management cancelled the queue 218 | 219 | if msg: 220 | if self._load_history(msg): 221 | return True # loading worked 222 | 223 | continue # not loaded ... date may have been seen 224 | else: 225 | # End of histdata 226 | if self.p.historical: # only historical 227 | self.put_notification(self.DISCONNECTED) 228 | self._state = self._ST_OVER 229 | return False # end of historical 230 | 231 | # Live is also wished - go for it 232 | self._state = self._ST_LIVE 233 | continue 234 | 235 | elif self._state == self._ST_START: 236 | if not self._st_start(instart=False): 237 | self._state = self._ST_OVER 238 | return False 239 | #TODO 240 | # - Check for delays in feed 241 | # - put a self.put_notification(self.DELAYED) 242 | # - Attempt to fill in missing data 243 | # - Setup a backfill of some sort when starting a feed. 244 | # - Set Dissonnected status where appropriate. 245 | 246 | 247 | def _load_tick(self, msg): 248 | #print('MSG = {}'.format(msg)) 249 | #print(msg['UTM']) 250 | dtobj = datetime.utcfromtimestamp(int(msg['UTM']) / 1000) 251 | dt = date2num(dtobj) 252 | 253 | try: 254 | vol = int(msg['LTV']) 255 | except TypeError: 256 | vol = 0 257 | 258 | #Check for missing Bid quote (Happens sometimes) 259 | if msg['BID'] == None and msg['OFR']: 260 | bid = float(msg['OFR']) 261 | ofr = float(msg['OFR']) 262 | #Check for missing offer quote (Happens sometimes) 263 | elif msg['OFR'] == None and msg['BID']: 264 | bid = float(msg['BID']) 265 | ofr = float(msg['BID']) 266 | else: 267 | bid = float(msg['BID']) 268 | ofr = float(msg['OFR']) 269 | 270 | if dt <= self.lines.datetime[-1]: 271 | return False # time already seen 272 | 273 | # Common fields 274 | self.lines.datetime[0] = dt 275 | self.lines.volume[0] = vol 276 | self.lines.openinterest[0] = 0.0 277 | 278 | # Put the prices into the bar 279 | #SOMETIME tick can be missing BID or OFFER.... Need to fallback 280 | 281 | tick = ofr if self.p.useask else bid 282 | 283 | self.lines.open[0] = tick 284 | self.lines.high[0] = tick 285 | self.lines.low[0] = tick 286 | self.lines.close[0] = tick 287 | self.lines.volume[0] = vol 288 | self.lines.openinterest[0] = 0.0 289 | return True 290 | 291 | 292 | def _load_history(self, msg): 293 | #TODO 294 | #print(msg) 295 | logging.debug(msg) 296 | local_tz = pytz.timezone('Europe/London') 297 | dtobj = datetime.strptime(msg['snapshotTime'], '%Y:%m:%d-%H:%M:%S') 298 | dtobj = local_tz.localize(dtobj).astimezone(pytz.utc) 299 | dt = date2num(dtobj) 300 | if dt <= self.lines.datetime[-1]: 301 | return False # time already seen 302 | 303 | # Common fields 304 | self.lines.datetime[0] = dt 305 | self.lines.volume[0] = float(msg['lastTradedVolume']) 306 | self.lines.openinterest[0] = 0.0 307 | 308 | # Put the prices into the bar 309 | if self.p.bidask: 310 | if not self.p.useask: 311 | self.lines.open[0] = float(msg['openPrice']['bid']) 312 | self.lines.high[0] = float(msg['highPrice']['bid']) 313 | self.lines.low[0] = float(msg['lowPrice']['bid']) 314 | self.lines.close[0] = float(msg['closePrice']['bid']) 315 | else: 316 | self.lines.open[0] = float(msg['openPrice']['ask']) 317 | self.lines.high[0] = float(msg['highPrice']['ask']) 318 | self.lines.low[0] = float(msg['lowPrice']['ask']) 319 | self.lines.close[0] = float(msg['closePrice']['ask']) 320 | else: 321 | self.lines.open[0] = ((float(msg['openPrice']['ask']) + 322 | float(msg['openPrice']['bid'])) / 2) 323 | self.lines.high[0] = ((float(msg['highPrice']['ask']) + 324 | float(msg['highPrice']['bid'])) / 2) 325 | self.lines.low[0] = ((float(msg['lowPrice']['ask']) + 326 | float(msg['lowPrice']['bid'])) / 2) 327 | self.lines.close[0] = ((float(msg['closePrice']['ask']) + 328 | float(msg['closePrice']['bid'])) / 2) 329 | 330 | return True 331 | -------------------------------------------------------------------------------- /bt_ig/igbroker.py: -------------------------------------------------------------------------------- 1 | from __future__ import (absolute_import, division, print_function, 2 | unicode_literals) 3 | 4 | import collections 5 | from copy import copy 6 | from datetime import date, datetime, timedelta 7 | import threading 8 | 9 | from backtrader.feed import DataBase 10 | from backtrader import (TimeFrame, num2date, date2num, BrokerBase, 11 | Order, BuyOrder, SellOrder, OrderBase, OrderData) 12 | from backtrader.utils.py3 import bytes, with_metaclass, MAXFLOAT 13 | from backtrader.metabase import MetaParams 14 | from backtrader.comminfo import CommInfoBase 15 | from backtrader.position import Position 16 | from . import igstore 17 | from backtrader.utils import AutoDict, AutoOrderedDict 18 | from backtrader.comminfo import CommInfoBase 19 | 20 | class IGCommInfo(CommInfoBase): 21 | def getvaluesize(self, size, price): 22 | # In real life the margin approaches the price 23 | return abs(size) * price 24 | 25 | def getoperationcost(self, size, price): 26 | '''Returns the needed amount of cash an operation would cost''' 27 | # Same reasoning as above 28 | return abs(size) * price 29 | 30 | class MetaIGBroker(BrokerBase.__class__): 31 | def __init__(cls, name, bases, dct): 32 | '''Class has already been created ... register''' 33 | # Initialize the class 34 | super(MetaIGBroker, cls).__init__(name, bases, dct) 35 | igstore.IGStore.BrokerCls = cls 36 | 37 | class IGBroker(with_metaclass(MetaIGBroker, BrokerBase)): 38 | '''Broker implementation for IG. 39 | This class maps the orders/positions from IG to the 40 | internal API of ``backtrader``. 41 | Params: 42 | - ``use_positions`` (default:``True``): When connecting to the broker 43 | provider use the existing positions to kickstart the broker. 44 | Set to ``False`` during instantiation to disregard any existing 45 | position 46 | ''' 47 | params = ( 48 | ('use_positions', True), 49 | ) 50 | 51 | def __init__(self, **kwargs): 52 | super(IGBroker, self).__init__() 53 | 54 | self.o = igstore.IGStore(**kwargs) 55 | 56 | self.orders = collections.OrderedDict() # orders by order id 57 | self.notifs = collections.deque() # holds orders which are notified 58 | 59 | self.opending = collections.defaultdict(list) # pending transmission 60 | self.brackets = dict() # confirmed brackets 61 | 62 | self.startingcash = self.cash = 0.0 63 | self.startingvalue = self.value = 0.0 64 | self.positions = collections.defaultdict(Position) 65 | self.addcommissioninfo(self, IGCommInfo(mult=1.0, stocklike=False)) 66 | 67 | def start(self): 68 | super(IGBroker, self).start() 69 | self.addcommissioninfo(self, IGCommInfo(mult=1.0, stocklike=False)) 70 | self.o.start(broker=self) 71 | self.startingcash = self.cash = cash = self.o.get_cash() 72 | self.startingvalue = self.value = self.o.get_value() 73 | 74 | if self.p.use_positions: 75 | for p in self.o.get_positions(): 76 | # TODO This should be a store notification 77 | print('position for instrument:', p['market']['epic']) 78 | is_sell = p['position']['direction'] == 'SELL' 79 | size = p['position']['dealSize'] 80 | if is_sell: 81 | size = -size 82 | price = p['position']['openLevel'] 83 | self.positions[p['market']['epic']] = Position(size, price) 84 | 85 | def data_started(self, data): 86 | pos = self.getposition(data) 87 | 88 | if pos.size < 0: 89 | order = SellOrder(data=data, 90 | size=pos.size, price=pos.price, 91 | exectype=Order.Market, 92 | simulated=True) 93 | 94 | order.addcomminfo(self.getcommissioninfo(data)) 95 | order.execute(0, pos.size, pos.price, 96 | 0, 0.0, 0.0, 97 | pos.size, 0.0, 0.0, 98 | 0.0, 0.0, 99 | pos.size, pos.price) 100 | 101 | order.completed() 102 | self.notify(order) 103 | 104 | elif pos.size > 0: 105 | order = BuyOrder(data=data, 106 | size=pos.size, price=pos.price, 107 | exectype=Order.Market, 108 | simulated=True) 109 | 110 | order.addcomminfo(self.getcommissioninfo(data)) 111 | order.execute(0, pos.size, pos.price, 112 | 0, 0.0, 0.0, 113 | pos.size, 0.0, 0.0, 114 | 0.0, 0.0, 115 | pos.size, pos.price) 116 | 117 | order.completed() 118 | self.notify(order) 119 | 120 | def stop(self): 121 | super(IGBroker, self).stop() 122 | self.o.stop() 123 | 124 | def getcash(self): 125 | # This call cannot block if no answer is available from IG 126 | self.cash = cash = self.o.get_cash() 127 | return cash 128 | 129 | def getvalue(self, datas=None): 130 | self.value = self.o.get_value() 131 | return self.value 132 | 133 | def getposition(self, data, clone=True): 134 | # return self.o.getposition(data._dataname, clone=clone) 135 | pos = self.positions[data._dataname] 136 | if clone: 137 | pos = pos.clone() 138 | 139 | return pos 140 | 141 | def orderstatus(self, order): 142 | o = self.orders[order.ref] 143 | return o.status 144 | 145 | def _submit(self, oref): 146 | order = self.orders[oref] 147 | order.submit(self) 148 | self.notify(order) 149 | for o in self._bracketnotif(order): 150 | o.submit(self) 151 | self.notify(o) 152 | 153 | def _reject(self, oref): 154 | order = self.orders[oref] 155 | order.reject(self) 156 | self.notify(order) 157 | self._bracketize(order, cancel=True) 158 | 159 | def _accept(self, oref): 160 | order = self.orders[oref] 161 | order.accept() 162 | self.notify(order) 163 | for o in self._bracketnotif(order): 164 | o.accept(self) 165 | self.notify(o) 166 | 167 | def _cancel(self, oref): 168 | order = self.orders[oref] 169 | order.cancel() 170 | self.notify(order) 171 | self._bracketize(order, cancel=True) 172 | 173 | def _expire(self, oref): 174 | order = self.orders[oref] 175 | order.expire() 176 | self.notify(order) 177 | self._bracketize(order, cancel=True) 178 | 179 | def _bracketnotif(self, order): 180 | pref = getattr(order.parent, 'ref', order.ref) # parent ref or self 181 | br = self.brackets.get(pref, None) # to avoid recursion 182 | return br[-2:] if br is not None else [] 183 | 184 | def _bracketize(self, order, cancel=False): 185 | pref = getattr(order.parent, 'ref', order.ref) # parent ref or self 186 | br = self.brackets.pop(pref, None) # to avoid recursion 187 | if br is None: 188 | return 189 | 190 | if not cancel: 191 | if len(br) == 3: # all 3 orders in place, parent was filled 192 | br = br[1:] # discard index 0, parent 193 | for o in br: 194 | o.activate() # simulate activate for children 195 | self.brackets[pref] = br # not done - reinsert children 196 | 197 | elif len(br) == 2: # filling a children 198 | oidx = br.index(order) # find index to filled (0 or 1) 199 | self._cancel(br[1 - oidx].ref) # cancel remaining (1 - 0 -> 1) 200 | else: 201 | # Any cancellation cancel the others 202 | for o in br: 203 | if o.alive(): 204 | self._cancel(o.ref) 205 | 206 | def _fill(self, oref, size, price, ttype, **kwargs): 207 | order = self.orders[oref] 208 | 209 | if not order.alive(): # can be a bracket 210 | pref = getattr(order.parent, 'ref', order.ref) 211 | if pref not in self.brackets: 212 | msg = ('Order fill received for {}, with price {} and size {} ' 213 | 'but order is no longer alive and is not a bracket. ' 214 | 'Unknown situation') 215 | msg.format(order.ref, price, size) 216 | self.put_notification(msg, order, price, size) 217 | return 218 | 219 | # [main, stopside, takeside], neg idx to array are -3, -2, -1 220 | if ttype == 'STOP_LOSS_FILLED': 221 | order = self.brackets[pref][-2] 222 | elif ttype == 'TAKE_PROFIT_FILLED': 223 | order = self.brackets[pref][-1] 224 | else: 225 | msg = ('Order fill received for {}, with price {} and size {} ' 226 | 'but order is no longer alive and is a bracket. ' 227 | 'Unknown situation') 228 | msg.format(order.ref, price, size) 229 | self.put_notification(msg, order, price, size) 230 | return 231 | 232 | data = order.data 233 | pos = self.getposition(data, clone=False) 234 | psize, pprice, opened, closed = pos.update(size, price) 235 | 236 | comminfo = self.getcommissioninfo(data) 237 | 238 | closedvalue = closedcomm = 0.0 239 | openedvalue = openedcomm = 0.0 240 | margin = pnl = 0.0 241 | 242 | order.execute(data.datetime[0], size, price, 243 | closed, closedvalue, closedcomm, 244 | opened, openedvalue, openedcomm, 245 | margin, pnl, 246 | psize, pprice) 247 | 248 | if order.executed.remsize: 249 | order.partial() 250 | self.notify(order) 251 | else: 252 | order.completed() 253 | self.notify(order) 254 | self._bracketize(order) 255 | 256 | def _transmit(self, order): 257 | oref = order.ref 258 | pref = getattr(order.parent, 'ref', oref) # parent ref or self 259 | 260 | if order.transmit: 261 | if oref != pref: # children order 262 | # Put parent in orders dict, but add stopside and takeside 263 | # to order creation. Return the takeside order, to have 3s 264 | takeside = order # alias for clarity 265 | parent, stopside = self.opending.pop(pref) 266 | for o in parent, stopside, takeside: 267 | self.orders[o.ref] = o # write them down 268 | 269 | self.brackets[pref] = [parent, stopside, takeside] 270 | self.o.order_create(parent, stopside, takeside) 271 | return takeside # parent was already returned 272 | 273 | else: # Parent order, which is not being transmitted 274 | self.orders[order.ref] = order 275 | return self.o.order_create(order) 276 | 277 | # Not transmitting 278 | self.opending[pref].append(order) 279 | return order 280 | 281 | def buy(self, owner, data, 282 | size, price=None, plimit=None, 283 | exectype=None, valid=None, tradeid=0, oco=None, 284 | trailamount=None, trailpercent=None, 285 | parent=None, transmit=True, 286 | **kwargs): 287 | 288 | order = BuyOrder(owner=owner, data=data, 289 | size=size, price=price, pricelimit=plimit, 290 | exectype=exectype, valid=valid, tradeid=tradeid, 291 | trailamount=trailamount, trailpercent=trailpercent, 292 | parent=parent, transmit=transmit) 293 | 294 | order.addinfo(**kwargs) 295 | order.addcomminfo(self.getcommissioninfo(data)) 296 | return self._transmit(order) 297 | 298 | def sell(self, owner, data, 299 | size, price=None, plimit=None, 300 | exectype=None, valid=None, tradeid=0, oco=None, 301 | trailamount=None, trailpercent=None, 302 | parent=None, transmit=True, 303 | **kwargs): 304 | 305 | order = SellOrder(owner=owner, data=data, 306 | size=size, price=price, pricelimit=plimit, 307 | exectype=exectype, valid=valid, tradeid=tradeid, 308 | trailamount=trailamount, trailpercent=trailpercent, 309 | parent=parent, transmit=transmit) 310 | 311 | order.addinfo(**kwargs) 312 | order.addcomminfo(self.getcommissioninfo(data)) 313 | return self._transmit(order) 314 | 315 | def cancel(self, order): 316 | o = self.orders[order.ref] 317 | if order.status == Order.Cancelled: # already cancelled 318 | return 319 | 320 | return self.o.order_cancel(order) 321 | 322 | def notify(self, order): 323 | self.notifs.append(order.clone()) 324 | 325 | def get_notification(self): 326 | if not self.notifs: 327 | return None 328 | 329 | return self.notifs.popleft() 330 | 331 | def next(self): 332 | self.notifs.append(None) # mark notification boundary 333 | -------------------------------------------------------------------------------- /bt_ig/igstore.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Dreamworld 3 | 4 | #My question about store development 5 | https://community.backtrader.com/topic/459/store-development/2 6 | 7 | #Example for adding a data feed. Can use online sources 8 | https://www.backtrader.com/docu/datafeed-develop-general/datafeed-develop-general.html 9 | 10 | I need to implement 11 | 12 | 13 | 2) IG Broker - Look at bt/brokers/oandabroker.py for an Example 14 | 1) IG Store 15 | 1) IG Feed - Look at bt/feeds/oanda.py for an Example. It seems the feed 16 | imports and has many references to the store. 17 | 18 | ''' 19 | 20 | #Python Imports 21 | from __future__ import (absolute_import, division, print_function, 22 | unicode_literals) 23 | 24 | import collections 25 | from datetime import datetime, timedelta 26 | import time as _time 27 | import json 28 | import threading 29 | 30 | 31 | #Backtrader imports 32 | import backtrader as bt 33 | from backtrader import TimeFrame, date2num, num2date 34 | from backtrader.metabase import MetaParams 35 | from backtrader.utils.py3 import queue, with_metaclass 36 | from backtrader.utils import AutoDict 37 | 38 | 39 | #IG Imports 40 | from trading_ig import (IGService, IGStreamService) 41 | from trading_ig.lightstreamer import Subscription 42 | 43 | ''' 44 | #Dev IG Imports 45 | from ...dev.trading_ig import (IGService, IGStreamService) 46 | from ...dev.trading_ig.lightstreamer import Subscription 47 | ''' 48 | #TODO ADD Errors if appropriate 49 | 50 | 51 | class Streamer(IGStreamService): 52 | ''' 53 | TODO 54 | - Boatloads! 55 | - Add a listener for notifiactions 56 | - add methods to set queues for account and prices 57 | ''' 58 | def __init__(self, *args, **kwargs): 59 | super(Streamer, self).__init__(*args, **kwargs) 60 | self.price_q = dict() 61 | 62 | def run(self, params): 63 | 64 | self.connected = True 65 | #params = params or {} 66 | 67 | def set_price_q(self, q, epic): 68 | self.price_q[epic] = q 69 | 70 | def set_account_q(self, q): 71 | self.account_q = q 72 | 73 | def on_account_update(self,data): 74 | ''' 75 | Listener for account updates 76 | ''' 77 | self.account_q.put(data['values']) 78 | 79 | def on_prices_update(self, data): 80 | ''' 81 | Oandapy uses on_success from the api to extract the data feed. 82 | 83 | For IG, we register the on_prices_update listener. 84 | ''' 85 | name = data['name'] 86 | epic = name.replace('CHART:','') 87 | epic = epic.replace(':TICK','') 88 | self.price_q[epic].put(data['values']) 89 | 90 | def on_error(self,data): 91 | 92 | # Disconnecting 93 | ig_stream_service.disconnect() 94 | #TODO Add error message from data 95 | 96 | 97 | class MetaSingleton(MetaParams): 98 | '''Metaclass to make a metaclassed class a singleton 99 | 100 | MetaParams - Imported from backtrader framework 101 | ''' 102 | def __init__(cls, name, bases, dct): 103 | super(MetaSingleton, cls).__init__(name, bases, dct) 104 | cls._singleton = None 105 | 106 | def __call__(cls, *args, **kwargs): 107 | if cls._singleton is None: 108 | cls._singleton = ( 109 | super(MetaSingleton, cls).__call__(*args, **kwargs)) 110 | 111 | return cls._singleton 112 | 113 | 114 | class IGStore(with_metaclass(MetaSingleton, object)): 115 | ''' 116 | The IG store class should inherit from the the metaclass and add some 117 | extensions to it. 118 | ''' 119 | BrokerCls = None # broker class will autoregister 120 | DataCls = None # data class will auto register 121 | 122 | params = ( 123 | ('token', ''), 124 | ('account', ''), 125 | ('usr', ''), 126 | ('pwd', ''), 127 | ('currency_code', 'GBP'), #The currency code of the account 128 | ('practice', True), 129 | ('account_tmout', 10.0), # account balance refresh timeout 130 | ) 131 | 132 | _ENVPRACTICE = 'DEMO' 133 | _ENVLIVE = 'LIVE' 134 | 135 | _ORDEREXECS = { 136 | bt.Order.Market: 'MARKET', 137 | bt.Order.Limit: 'LIMIT', 138 | bt.Order.Stop: 'STOP', 139 | bt.Order.StopLimit: 'TODO', 140 | } 141 | 142 | _GRANULARITIES = { 143 | (bt.TimeFrame.Seconds, 1): 'SECOND', 144 | (bt.TimeFrame.Minutes, 1): 'MINUTE', 145 | (bt.TimeFrame.Minutes, 2): 'MINUTE_2', 146 | (bt.TimeFrame.Minutes, 3): 'MINUTE_3', 147 | (bt.TimeFrame.Minutes, 5): 'MINUTE_5', 148 | (bt.TimeFrame.Minutes, 10): 'MINUTE_10', 149 | (bt.TimeFrame.Minutes, 15): 'MINUTE_15', 150 | (bt.TimeFrame.Minutes, 30): 'MINUTE_30', 151 | (bt.TimeFrame.Minutes, 60): 'HOUR', 152 | (bt.TimeFrame.Minutes, 120): 'HOUR_2', 153 | (bt.TimeFrame.Minutes, 180): 'HOUR_3', 154 | (bt.TimeFrame.Minutes, 240): 'HOUR_4', 155 | (bt.TimeFrame.Days, 1): 'DAY', 156 | (bt.TimeFrame.Weeks, 1): 'WEEK', 157 | (bt.TimeFrame.Months, 1): 'MONTH', 158 | } 159 | 160 | _DT_FORMAT = '%Y-%m-%d %H:%M:%S' 161 | 162 | @classmethod 163 | def getdata(cls, *args, **kwargs): 164 | '''Returns ``DataCls`` with args, kwargs''' 165 | return cls.DataCls(*args, **kwargs) 166 | 167 | @classmethod 168 | def getbroker(cls, *args, **kwargs): 169 | '''Returns broker with *args, **kwargs from registered ``BrokerCls``''' 170 | return cls.BrokerCls(*args, **kwargs) 171 | 172 | 173 | def __init__(self): 174 | super(IGStore, self).__init__() 175 | 176 | self.notifs = collections.deque() # store notifications for cerebro 177 | self._env = None # reference to cerebro for general notifications 178 | self.broker = None # broker instance 179 | self.datas = list() # datas that have registered over start 180 | 181 | self._orders = collections.OrderedDict() # map order.ref to oid 182 | self._ordersrev = collections.OrderedDict() # map oid to order.ref 183 | self._transpend = collections.defaultdict(collections.deque) 184 | 185 | self._oenv = self._ENVPRACTICE if self.p.practice else self._ENVLIVE 186 | 187 | self.igapi = IGService(self.p.usr, self.p.pwd, self.p.token, self._oenv) 188 | self.igapi.create_session() 189 | 190 | self.igss = Streamer(ig_service=self.igapi) 191 | self.ig_session = self.igss.create_session() 192 | self.igss.connect(self.p.account) 193 | #Work with JSON rather than Pandas for better backtrader integration 194 | self.igapi.return_dataframe = False 195 | self._cash = 0.0 196 | self._value = 0.0 197 | self.pull_cash_and_value() 198 | self._evt_acct = threading.Event() 199 | 200 | def broker_threads(self): 201 | ''' 202 | Setting up threads and targets for broker related notifications. 203 | ''' 204 | 205 | self.q_account = queue.Queue() 206 | kwargs = {'q': self.q_account} 207 | self.q_account.put(True) # force an immediate update 208 | t = threading.Thread(target=self._t_account) 209 | t.daemon = True 210 | t.start() 211 | 212 | t = threading.Thread(target=self._t_account_events, kwargs=kwargs) 213 | t.daemon = True 214 | t.start() 215 | 216 | self.q_ordercreate = queue.Queue() 217 | t = threading.Thread(target=self._t_order_create) 218 | t.daemon = True 219 | t.start() 220 | 221 | self.q_orderclose = queue.Queue() 222 | t = threading.Thread(target=self._t_order_cancel) 223 | t.daemon = True 224 | t.start() 225 | 226 | # Wait once for the values to be set 227 | self._evt_acct.wait(self.p.account_tmout) 228 | 229 | def pull_cash_and_value(self): 230 | ''' 231 | Method to set the initial cash and value before streaming updates start. 232 | ''' 233 | accounts = self.igapi.fetch_accounts() 234 | for account in accounts['accounts']: 235 | if self.p.account == account['accountId']: 236 | self._cash = account['balance']['available'] 237 | self._value = account['balance']['balance'] 238 | 239 | 240 | def get_cash(self): 241 | return self._cash 242 | 243 | def get_notifications(self): 244 | '''Return the pending "store" notifications''' 245 | self.notifs.append(None) # put a mark / threads could still append 246 | return [x for x in iter(self.notifs.popleft, None)] 247 | 248 | def get_open_orders(self): 249 | #TODO Return all open orders and pass them to self.pending in the order list 250 | pass 251 | 252 | def get_positions(self): 253 | #TODO - Get postion info from returned object. 254 | positions = self.igapi.fetch_open_positions() 255 | return positions['positions'] 256 | 257 | def get_value(self): 258 | return self._value 259 | 260 | def put_notification(self, msg, *args, **kwargs): 261 | self.notifs.append((msg, args, kwargs)) 262 | 263 | def start(self, data=None, broker=None): 264 | 265 | # Datas require some processing to kickstart data reception 266 | if data is None and broker is None: 267 | self.cash = None 268 | return 269 | 270 | if data is not None: 271 | self._env = data._env 272 | # For datas simulate a queue with None to kickstart co 273 | self.datas.append(data) 274 | 275 | if self.broker is not None: 276 | self.broker.data_started(data) 277 | 278 | elif broker is not None: 279 | self.broker = broker 280 | self.streaming_events() 281 | self.broker_threads() 282 | 283 | def stop(self): 284 | # signal end of thread 285 | if self.broker is not None: 286 | self.q_ordercreate.put(None) 287 | self.q_orderclose.put(None) 288 | self.q_account.put(None) 289 | 290 | def get_granularity(self, timeframe, compression): 291 | return self._GRANULARITIES.get((timeframe, compression), None) 292 | 293 | def candles(self, dataname, dtbegin, dtend, timeframe, compression, numpoints=False, bars=None): 294 | 295 | kwargs = locals().copy() 296 | kwargs.pop('self') 297 | kwargs['q'] = q = queue.Queue() 298 | t = threading.Thread(target=self._t_candles, kwargs=kwargs) 299 | t.daemon = True 300 | t.start() 301 | return q 302 | 303 | def _t_candles(self, dataname, dtbegin, dtend, timeframe, compression, q, numpoints=False, bars=None): 304 | 305 | granularity = self.get_granularity(timeframe, compression) 306 | if granularity is None: 307 | raise ValueError('Unsupported granularity provided ' 308 | 'please check https://labs.ig.com/rest-trading-api-reference/service-detail?id=530 ' 309 | 'for a list of supported granularity') 310 | return 311 | 312 | dtkwargs = {} 313 | if dtbegin is not None: 314 | dtkwargs['start_date'] = datetime.strftime(num2date(dtbegin), format=self._DT_FORMAT) 315 | 316 | if dtend is not None: 317 | dtkwargs['end_date'] = datetime.strftime(num2date(dtend), format=self._DT_FORMAT) 318 | 319 | try: 320 | if numpoints: 321 | response = self.igapi.fetch_historical_prices_by_epic_and_num_points( 322 | epic=dataname, 323 | resolution=granularity, 324 | numpoints=bars) 325 | else: 326 | response = self.igapi.fetch_historical_prices_by_epic_and_date_range( 327 | epic=dataname, 328 | resolution=granularity, 329 | **dtkwargs) 330 | 331 | remaining = response['allowance']['remainingAllowance'] 332 | allowance = response['allowance']['totalAllowance'] 333 | next_ref = timedelta(seconds=response['allowance']['allowanceExpiry']) 334 | 335 | print("HISTORICAL ALLOWANCE: Total: {}, Remaining: {}, Next Refresh: {}".format( 336 | allowance, remaining, next_ref)) 337 | except Exception as e: 338 | self.put_notification('ERROR: {}'.format(e)) 339 | q.put(None) 340 | return 341 | 342 | for candle in response.get('prices', []): 343 | q.put(candle) 344 | 345 | q.put({}) # end of transmission 346 | 347 | 348 | ''' 349 | Loads of methods to add in-between 350 | ''' 351 | 352 | 353 | def _t_account(self): 354 | #TODO 355 | ''' 356 | This is a thread with a queue that will extract data as it comes in 357 | I need to pass the relavant account info here after subscribing to 358 | the account information through lightstreamer 359 | ''' 360 | while True: 361 | try: 362 | msg = self.q_account.get(timeout=self.p.account_tmout) 363 | if msg is None: 364 | break # end of thread 365 | elif type(msg) != bool: #Check it is not the true value at the start of the queue... TODO improve this 366 | try: 367 | self._cash = float(msg["AVAILABLE_CASH"]) 368 | self._value = float(msg["EQUITY"]) 369 | except KeyError: 370 | pass 371 | 372 | except queue.Empty: # tmout -> time to refresh 373 | pass 374 | 375 | self._evt_acct.set() 376 | 377 | def order_create(self, order, stopside=None, takeside=None, **kwargs): 378 | ''' 379 | additional kwargs 380 | 381 | expiry: Sting, default = 'DFB' Other examples could be 'DEC-14'. Check 382 | the instrument details through IG to find out the correct expiry. 383 | 384 | guaranteed_stop: Bool, default = False. Sets whether or not to use a 385 | guranteed stop. 386 | 387 | time_in_force: String. Must be either 'GOOD_TILL_CANCELLED' or "GOOD_TILL_DATE" 388 | 389 | good_till_date: Datetime object. Must be provided is "GOOD_TILL_DATE" is set. 390 | ''' 391 | okwargs = dict() 392 | okwargs['currency_code'] = self.p.currency_code 393 | #okwargs['dealReference'] = order.ref 394 | okwargs['epic'] = order.data._dataname 395 | #Size must be positive for both buy and sell orders 396 | okwargs['size'] = abs(order.created.size) 397 | okwargs['direction'] = 'BUY' if order.isbuy() else 'SELL' 398 | okwargs['order_type'] = self._ORDEREXECS[order.exectype] 399 | #TODO FILL_OR_KILL 400 | #okwargs['timeInForce'] = 'FILL_OR_KILL' 401 | okwargs['force_open']= "false" 402 | 403 | #Filler - required arguments can update later if Limit order is required 404 | okwargs['level'] = order.created.price 405 | okwargs['limit_level'] = None 406 | okwargs['limit_distance'] = None 407 | okwargs['stop_level'] = None 408 | okwargs['stop_distance'] = None 409 | #Allow users to set the expiry through kwargs 410 | if 'expiry' in kwargs: 411 | okwargs['expiry'] = kwargs["expiry"] 412 | else: 413 | okwargs['expiry'] = 'DFB' 414 | #Allow users to set the a guaranteed stop 415 | #Convert from boolean value to string. 416 | if 'guaranteed_stop' in kwargs: 417 | if kwargs['guaranteed_stop'] == True: 418 | okwargs['guaranteed_stop'] = "true" 419 | elif kwargs['guaranteed_stop'] == False: 420 | okwargs['guaranteed_stop'] = "false" 421 | else: 422 | raise ValueError('guaranteed_stop must be a boolean value: "{}" ' 423 | 'was entered'.format(kwargs['guaranteed_stop'])) 424 | else: 425 | okwargs['guaranteed_stop'] = "false" 426 | 427 | #Market orders use an 'order_type' keyword. Limit and stop orders use 'type' 428 | if order.exectype == bt.Order.Market: 429 | okwargs['quote_id'] = None 430 | okwargs['level'] = None #IG Does not allow a level to be set on market orders 431 | 432 | if order.exectype in [bt.Order.Stop, bt.Order.Limit]: 433 | 434 | #Allow passing of a timeInForce kwarg 435 | if 'time_in_force' in kwargs: 436 | okwargs['time_in_force'] = kwargs['time_in_force'] 437 | if kwargs['time_in_force'] == 'GOOD_TILL_DATE': 438 | if 'good_till_date' in kwargs: 439 | #Trading_IG will do a datetime conversion 440 | okwargs['good_till_date'] = kwargs['good_till_date'] 441 | else: 442 | raise ValueError('If timeInForce == GOOD_TILL_DATE, a ' 443 | 'goodTillDate datetime kwarg must be provided.') 444 | else: 445 | okwargs['time_in_force'] = 'GOOD_TILL_CANCELLED' 446 | 447 | if order.exectype == bt.Order.StopLimit: 448 | #TODO 449 | okwargs['lowerBound'] = order.created.pricelimit 450 | okwargs['upperBound'] = order.created.pricelimit 451 | 452 | if order.exectype == bt.Order.StopTrail: 453 | # TODO need to figure out how to get the stop distance and increment 454 | # from the trail amount. 455 | # print('order trail amount: {}'.format(order.trailamount)) 456 | okwargs['stop_distance'] = order.trailamount 457 | #okwargs['trailingStopIncrement'] = 'TODO!' 458 | 459 | if stopside is not None: 460 | okwargs['stop_level'] = stopside.price 461 | 462 | if takeside is not None: 463 | okwargs['limit_level'] = takeside.price 464 | 465 | okwargs.update(**kwargs) # anything from the user 466 | 467 | self.q_ordercreate.put((order.ref, okwargs,)) 468 | return order 469 | 470 | def order_cancel(self, order): 471 | self.q_orderclose.put(order.ref) 472 | return order 473 | 474 | def _t_order_cancel(self): 475 | while True: 476 | oref = self.q_orderclose.get() 477 | if oref is None: 478 | break 479 | 480 | oid = self._orders.get(oref, None) 481 | if oid is None: 482 | continue # the order is no longer there 483 | try: 484 | o = self.igapi.delete_working_order(oid) 485 | except Exception as e: 486 | self.put_notification(e) 487 | continue # not cancelled - FIXME: notify 488 | 489 | self.broker._cancel(oref) 490 | 491 | def _t_order_create(self): 492 | while True: 493 | msg = self.q_ordercreate.get() 494 | if msg is None: 495 | break 496 | oref, okwargs = msg 497 | # Check to see if it is a market order or working order. 498 | # Market orders have an 'order_type' kwarg. Working orders 499 | # use the 'type' kwarg for setting stop or limit 500 | if okwargs['order_type'] == 'MARKET': 501 | try: 502 | 503 | #NOTE The IG API will confirm the deal automatically with the 504 | #create_open_position call. Therefore if no error is returned here 505 | #Then it was accepted and open. 506 | o = self.igapi.create_open_position(**okwargs) 507 | except Exception as e: 508 | self.put_notification(e) 509 | self.broker._reject(oref) 510 | return 511 | else: 512 | # print('Creating Working Order') 513 | try: 514 | o = self.igapi.create_working_order(**okwargs) 515 | 516 | except Exception as e: 517 | #print(e) 518 | self.put_notification(e) 519 | self.broker._reject(oref) 520 | return 521 | 522 | # Ids are delivered in different fields and all must be fetched to 523 | # match them (as executions) to the order generated here 524 | _o = {'dealId': None} 525 | oids = list() 526 | 527 | oids.append(o['dealId']) 528 | 529 | #print('_t_order_create Deal ID = {}'.format(o['dealId'])) 530 | if o['dealStatus'] == 'REJECTED': 531 | self.broker._reject(oref) 532 | self.put_notification(o['reason']) 533 | 534 | if not oids: 535 | self.broker._reject(oref) 536 | return 537 | 538 | self._orders[oref] = oids[0] 539 | 540 | #Send the summission notification 541 | #TODO Shouldn't this come earlier???? 542 | self.broker._submit(oref) 543 | 544 | if okwargs['order_type'] == 'MARKET': 545 | self.broker._accept(oref) # taken immediately 546 | self.broker._fill(oref, o['size'], o['level'], okwargs['order_type']) 547 | for oid in oids: 548 | self._ordersrev[oid] = oref # maps ids to backtrader order 549 | 550 | 551 | def streaming_account(self, tmout=None): 552 | ''' 553 | Added by me to create a subscription to account information such as 554 | balance, equity funds, margin. 555 | ''' 556 | q = queue.Queue() 557 | kwargs = {'q': q, 'tmout': tmout} 558 | 559 | t = threading.Thread(target=self._t_account_listener, kwargs=kwargs) 560 | t.daemon = True 561 | t.start() 562 | 563 | t = threading.Thread(target=self._t_account_events, kwargs=kwargs) 564 | t.daemon = True 565 | t.start() 566 | return q 567 | 568 | 569 | def _t_account_events(self, q, tmout=None): 570 | ''' 571 | Thread to create the subscription to account events. 572 | 573 | Here we create a merge subscription for lightstreamer. 574 | ''' 575 | self.igss.set_account_q(q) 576 | # Making an other Subscription in MERGE mode 577 | subscription_account = Subscription( 578 | mode="MERGE", 579 | items=['ACCOUNT:'+self.p.account], 580 | fields=["AVAILABLE_CASH", "EQUITY"], 581 | ) 582 | # #adapter="QUOTE_ADAPTER") 583 | 584 | # Adding the "on_balance_update" function to Subscription 585 | subscription_account.addlistener(self.igss.on_account_update) 586 | 587 | # Registering the Subscription 588 | sub_key_account = self.igss.ls_client.subscribe(subscription_account) 589 | 590 | def streaming_events(self, tmout=None): 591 | pass 592 | 593 | def streaming_prices(self, dataname, tmout=None): 594 | q = queue.Queue() 595 | kwargs = {'q': q, 'dataname': dataname, 'tmout': tmout} 596 | t = threading.Thread(target=self._t_streaming_prices, kwargs=kwargs) 597 | t.daemon = True 598 | t.start() 599 | return q 600 | 601 | def _t_streaming_prices(self, dataname, q, tmout): 602 | ''' 603 | Target for the streaming prices thread. This will setup the streamer. 604 | ''' 605 | if tmout is not None: 606 | _time.sleep(tmout) 607 | 608 | self.igss.set_price_q(q, dataname) 609 | #igss = Streamer(q, ig_service=self.igapi) 610 | #ig_session = igss.create_session() 611 | #igss.connect(self.p.account) 612 | 613 | epic = 'CHART:'+dataname+':TICK' 614 | # Making a new Subscription in MERGE mode 615 | subcription_prices = Subscription( 616 | mode="DISTINCT", 617 | items=[epic], 618 | fields=["UTM", "BID", "OFR", "TTV","LTV"], 619 | ) 620 | #adapter="QUOTE_ADAPTER") 621 | 622 | # Adding the "on_price_update" function to Subscription 623 | subcription_prices.addlistener(self.igss.on_prices_update) 624 | 625 | sub_key_prices = self.igss.ls_client.subscribe(subcription_prices) 626 | --------------------------------------------------------------------------------