├── .gitignore ├── LICENSE ├── README.md ├── bootstrap.py ├── config ├── currencies.py ├── instruments.py ├── portfolios.py ├── settings.py.template ├── spots.py └── strategy.py.template ├── core ├── __init__.py ├── basestore.py ├── contract_store.py ├── currency.py ├── data_feed.py ├── hdfstore.py ├── ib_connection.py ├── instrument.py ├── logger.py ├── spot.py └── utility.py ├── data ├── __init__.py ├── data_provider.py ├── db_mongo.py ├── ib_provider.py ├── providers_factory.py └── quandl_provider.py ├── docs ├── Getting started with Interactive Brokers.ipynb ├── How our system works.ipynb ├── How to test new rules.ipynb ├── Introduction to Trend Following.ipynb ├── Rolling & Carry.ipynb └── Working with Prices.ipynb ├── download.py ├── download.sh ├── pytest.ini ├── pytest.sh ├── requirements.txt ├── scheduler.py ├── scripts └── jupyter_extensions.sh ├── tests └── test_portfolio.py ├── trade.sh ├── trading ├── __init__.py ├── account.py ├── accountcurve.py ├── bootstrap.py ├── bootstrap_portfolio.py ├── ibstate.py ├── portfolio.py ├── rules.py └── start.py └── validate.py /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | .** 3 | !/.gitignore 4 | *.pyc 5 | cache/* 6 | /*.ipynb 7 | config/settings.py 8 | config/strategy.py 9 | ignore/* 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Chris Muktar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyTrendFollow - Systematic Futures Trading using Trend Following 2 | 3 | ## Introduction 4 | 5 | This program trades futures using a systematic trend following strategy, similar to most managed 6 | futures hedge funds. It produces returns of around ~20% per year, based on a volatility of 25%. 7 | You can read more about trend following in the /docs folder. Start with [introduction to trend following](https://github.com/chrism2671/PyTrendFollow/blob/master/docs/Introduction%20to%20Trend%20Following.ipynb). If you just want to play with futures data, see [working with prices](https://github.com/chrism2671/PyTrendFollow/blob/master/docs/Working%20with%20Prices.ipynb). 8 | 9 | ## Features 10 | * Integration with Interactive Brokers for fully automated trading. 11 | * Automatic downloading of contract data from Quandl & Interactive Brokers. 12 | * Automatic rolling of futures contracts. 13 | * Trading strategies backtesting on historical data 14 | * Designed to use Jupyter notebook as an R&D environment. 15 | 16 | ## Installation 17 | 18 | ### Data sources 19 | 20 | The system supports downloading of price data from 21 | 1. [Quandl](https://www.quandl.com/) 22 | 1. [Interactive Brokers](https://www.interactivebrokers.com) (IB) 23 | 24 | It is recommended (though not required) to have data subscriptions for both Quandl and IB. 25 | Quandl has more historical contracts and works well for backtesting, 26 | while IB data is usually updated more frequently and is better for live trading. 27 | 28 | To use MySQL as the data storage backend (optional, default is HDF5), you'll need a configured 29 | server with privileges to create databases and tables. 30 | 31 | ### Trading 32 | 33 | For automated trading with Interactive Brokers, install the latest 34 | [TWS terminal](https://www.interactivebrokers.com/en/index.php?f=16040) 35 | or [Gateway](https://www.interactivebrokers.com/en/index.php?f=16457). You'll need to enable the API and set it to port 4001. 36 | 37 | ### Code 38 | 39 | 1. Python version 3.* is required 40 | 1. Get the code: 41 | 42 | `git clone https://github.com/chrism2671/PyTrendFollow.git` 43 | 44 | `cd PyTrendFollow` 45 | 1. Install requirements: 46 | * install python tkinter (for Linux it's usually present in a distribution repository, e.g. 47 | for Ubuntu: `apt-get install python3-tk`) if necessary. 48 | * To compile the binary version of [arch](https://pypi.org/project/arch/4.0/), you will need the 49 | development lirbary for your version of Python. e.g., for Python 3.5 on Ubuntu, use 50 | `apt-get install libpython3.5-dev`. 51 | * install Python requirements: `pip3 install -r requirements.txt` 52 | 1. `cp config/settings.py.template config/settings.py`, update the settings file with your API keys, 53 | data path, etc. If you don't use one of the data sources (IB or Quandl), comment the corresponding 54 | line in `data_sources`. 55 | 1. `cp config/strategy.py.template config/strategy.py`, review and update the strategy parameters if 56 | necessary 57 | 58 | ## Usage 59 | 60 | Before you start, run the IB TWS terminal or Gateway, go to `Edit -> Global configuration -> API -> 61 | Settings` and make sure that `Socket port` matches the value of `ib_port` in your local 62 | `config/settings.py` file (default value is 4001). 63 | 64 | * To download contracts data from IB and Quandl: 65 | 66 | `python download.py quandl --concurrent` 67 | 68 | `python download.py ib` 69 | 70 | Use the `--concurrent` flag only if you have the concurrent downloads feature enabled on Quandl, 71 | otherwise you'll hit API requests limit. 72 | 73 | * After the download has completed, make sure the data is valid: 74 | 75 | `python validate.py` 76 | 77 | The output of this script should be a table showing if the data for each instrument in the 78 | portfolio is not corrupted, is up to date and some other useful information. 79 | 80 | * To place your market orders now, update positions and quit: 81 | 82 | `python scheduler.py --now --quit` 83 | 84 | * To schedule portfolio update for a specific time every day: 85 | 86 | `python scheduler.py` 87 | 88 | For more details on how the system works and how to experiment with it, check out the `docs/` 89 | directory. 90 | 91 | ## Status, Disclaimers etc 92 | 93 | This project is dev status. Use at your own risk. We are looking for contributors and bug fixes. 94 | 95 | Only tested on Linux for now, but should work on Windows / MacOS. 96 | 97 | ## Acknowledgements 98 | 99 | This project is based heavily on the work of Rob Carver & the 100 | [PySystemTrade](https://github.com/robcarver17/pysystemtrade) project. 101 | -------------------------------------------------------------------------------- /bootstrap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import trading.portfolio as portfolio 3 | import config.portfolios 4 | 5 | 6 | if __name__ == '__main__': 7 | p_learn = portfolio.Portfolio(instruments=config.portfolios.p_learn) 8 | p_test = portfolio.Portfolio(instruments=config.portfolios.p_test) 9 | p_trade = portfolio.Portfolio(instruments=config.portfolios.p_trade) 10 | 11 | p_learn_weights = p_learn.bootstrap_pool() 12 | print(p_learn_weights.mean(axis=1)) 13 | 14 | p_test_weights = p_test.bootstrap_pool() 15 | print(p_test_weights.mean(axis=1)) 16 | 17 | p_trade_weights = p_trade.bootstrap_pool() 18 | print(p_trade_weights.mean(axis=1)) 19 | -------------------------------------------------------------------------------- /config/currencies.py: -------------------------------------------------------------------------------- 1 | from config.settings import base_currency 2 | 3 | """ 4 | This file defines currencies exchange rates. Loaded into core.currency.Currency object. 5 | Used for instrument's returns calculation and for some spot prices. 6 | 7 | { 8 | 'code': 'USD' + base_currency, # Currency pair unique code that we can refer by 9 | 'currency_data': ['ib', 'quandl'], # Data providers we should collect and merge data from 10 | 'quandl_database': 'CURRFX', # Database on Quandl providing the data 11 | 'quandl_symbol': 'USD' + base_currency, # Symbol on Quandl representing this currency pair 12 | 'quandl_rate': lambda x: x, # See 'ib_rate' 13 | 'ib_exchange': 'IDEALPRO', # Exchange identifier on Interactive Brokers. 14 | 'ib_symbol': base_currency, # Symbol on Interactive Brokers 15 | 'ib_currency': 'USD', # Denomination currency on Interactive Brokers 16 | 'ib_rate': lambda x: 1.0 / x, # Sometimes data provider may not have data for 17 | the currency pair we need, but have the reversed 18 | one (e.g. we need USD/GPB, but there is only 19 | GPB/USD). In such case we reverse the rate with a 20 | lambda expression to obtain the data we need. 21 | This can also be used to multiply the price by 22 | some factor or do any other transformation if 23 | necessary. 24 | } 25 | 26 | """ 27 | currencies_definitions = [ 28 | { 29 | # this dummy definition helps to avoid additional key-checks for the special case 30 | # when denomination == base_currency 31 | 'code': base_currency * 2, 32 | 'currency_data': [] 33 | }, 34 | { 35 | 'code': 'USD' + base_currency, 36 | 'currency_data': ['ib', 'quandl'], 37 | 'quandl_database': 'CURRFX', 38 | 'quandl_symbol': 'USD' + base_currency, 39 | 'quandl_rate': lambda x: x, 40 | 'ib_exchange': 'IDEALPRO', 41 | 'ib_symbol': base_currency, 42 | 'ib_currency': 'USD', 43 | 'ib_rate': lambda x: 1.0 / x, 44 | }, 45 | { 46 | 'code': 'CHF' + base_currency, 47 | 'currency_data': ['ib', 'quandl'], 48 | 'quandl_database': 'CURRFX', 49 | 'quandl_symbol': 'CHF' + base_currency, 50 | 'quandl_rate': lambda x: x, 51 | 'ib_exchange': 'IDEALPRO', 52 | 'ib_symbol': base_currency, 53 | 'ib_currency': 'CHF', 54 | 'ib_rate': lambda x: 1.0 / x, 55 | }, 56 | { 57 | 'code': 'HKD' + base_currency, 58 | 'currency_data': ['ib', 'quandl'], 59 | 'quandl_database': 'CURRFX', 60 | 'quandl_symbol': 'HKD' + base_currency, 61 | 'quandl_rate': lambda x: x, 62 | 'ib_exchange': 'IDEALPRO', 63 | 'ib_symbol': base_currency, 64 | 'ib_currency': 'HKD', 65 | 'ib_rate': lambda x: 1.0 / x, 66 | }, 67 | { 68 | 'code': 'EUR' + base_currency, 69 | 'currency_data': ['ib', 'quandl'], 70 | 'quandl_database': 'CURRFX', 71 | 'quandl_symbol': 'EUR' + base_currency, 72 | 'quandl_rate': lambda x: x, 73 | 'ib_exchange': 'IDEALPRO', 74 | 'ib_symbol': 'EUR', 75 | 'ib_currency': base_currency, 76 | 'ib_rate': lambda x: x, 77 | }, 78 | { 79 | 'code': 'BTC' + base_currency, 80 | 'currency_data': ['quandl'], 81 | 'quandl_database': 'BCHARTS', 82 | 'quandl_symbol': 'LOCALBTC' + base_currency, 83 | 'quandl_rate': lambda x: x, 84 | }, 85 | 86 | { 87 | 'code': 'BTCUSD', 88 | 'currency_data': ['quandl'], 89 | 'quandl_database': 'BCHARTS', 90 | 'quandl_symbol': 'LOCALBTCUSD', 91 | 'quandl_rate': lambda x: x, 92 | }, 93 | { 94 | 'code': 'MXNUSD', 95 | 'currency_data': ['ib', 'quandl'], 96 | 'quandl_database': 'CURRFX', 97 | 'quandl_symbol': 'MXNUSD', 98 | 'quandl_rate': lambda x: x, 99 | 'ib_exchange': 'IDEALPRO', 100 | 'ib_symbol': 'USD', 101 | 'ib_currency': 'MXN', 102 | 'ib_rate': lambda x: 1.0 / x, 103 | }, 104 | { 105 | 'code': 'AUDUSD', 106 | 'currency_data': ['ib', 'quandl'], 107 | 'quandl_database': 'CURRFX', 108 | 'quandl_symbol': 'AUDUSD', 109 | 'quandl_rate': lambda x: x, 110 | 'ib_exchange': 'IDEALPRO', 111 | 'ib_symbol': 'AUD', 112 | 'ib_currency': 'USD', 113 | 'ib_rate': lambda x: x, 114 | }, 115 | { 116 | 'code': 'EURUSD', 117 | 'currency_data': ['ib', 'quandl'], 118 | 'quandl_database': 'CURRFX', 119 | 'quandl_symbol': 'EURUSD', 120 | 'quandl_rate': lambda x: x, 121 | 'ib_exchange': 'IDEALPRO', 122 | 'ib_symbol': 'EUR', 123 | 'ib_currency': 'USD', 124 | 'ib_rate': lambda x: x, 125 | }, 126 | { 127 | 'code': 'GBPUSD', 128 | 'currency_data': ['ib', 'quandl'], 129 | 'quandl_database': 'CURRFX', 130 | 'quandl_symbol': 'GBPUSD', 131 | 'quandl_rate': lambda x: x, 132 | 'ib_exchange': 'IDEALPRO', 133 | 'ib_symbol': 'GBP', 134 | 'ib_currency': 'USD', 135 | 'ib_rate': lambda x: x, 136 | }, 137 | { 138 | 'code': 'NZDUSD', 139 | 'currency_data': ['ib', 'quandl'], 140 | 'quandl_database': 'CURRFX', 141 | 'quandl_symbol': 'NZDUSD', 142 | 'quandl_rate': lambda x: x, 143 | 'ib_exchange': 'IDEALPRO', 144 | 'ib_symbol': 'NZD', 145 | 'ib_currency': 'USD', 146 | 'ib_rate': lambda x: x, 147 | }, 148 | { 149 | 'code': 'JPYUSD', 150 | 'currency_data': ['ib', 'quandl'], 151 | 'quandl_database': 'CURRFX', 152 | 'quandl_symbol': 'JPYUSD', 153 | 'quandl_rate': lambda x: x, 154 | 'ib_exchange': 'IDEALPRO', 155 | 'ib_symbol': 'USD', 156 | 'ib_currency': 'JPY', 157 | 'ib_rate': lambda x: 1.0 / x, 158 | }, 159 | ] 160 | 161 | currencies_all = {x['code']: x for x in currencies_definitions} 162 | -------------------------------------------------------------------------------- /config/instruments.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This file defines the instruments the system has access to. 4 | 5 | These may change from time-to-time. If you make reasonable changes, please submit a pull request at https://github.com/chrism2671/PyTrendFollow so we can update it for everybody. 6 | 7 | See the defaults in core/instrument.py and extra parameters not shown here. 8 | 9 | Schema: 10 | 11 | { 12 | 'name': 'corn', # Unique shorthand name which we can refer to this instrument by 13 | 'contract_data': ['ib', 'quandl'], # Data providers we should collect and merge data from 14 | 'months_traded': [3, 5, 7, 9, 12], # Months that this contract is available for trading 15 | 'first_contract': 196003, # First month this contract traded, so we know when to download from (March 1960) 16 | 'backtest_from_contract': 199312, # Backtest from this month (December 1993) 17 | 'trade_only': [12], # Trade only these contract months (December - the 'Z' contract) 18 | 'quandl_symbol': 'C', # Symbol on Quandl representing this contract 19 | 'quandl_database': 'CME', # Database on Quandl providing contract data 20 | 'ib_code': 'ZC', # Symbol on Interactive Brokers representing this contract 21 | 'denomination' 'USD', # Currency of the contract. Default is 22 | 'roll_day': -20, # Day of month to roll to next contract. If it's negative, it's this day in the previous month. For example, if we are in the December contract, a roll_day of -20 means roll on November 20th. 23 | 'exchange': 'ECBOT', # Exchange identifier on Interactive Brokers. 24 | 'point_value': 50, # $ move per point move on the futures contract. Usually equal to the contract multiplier. 25 | 'spread': 0.25, # Estimated spread, for estimating costs 26 | 'commission': 2.81, # Interactive Brokers per contract commission in base currency 27 | }, 28 | 29 | 30 | When setting roll days, we can't let it get to close to the expiry dates. Interactive Brokers require us to be out of a contract well before expiry. The dates for each contract are on this page: https://www.interactivebrokers.co.uk/en/index.php?f=deliveryExerciseActions&p=physical 31 | 32 | """ 33 | 34 | from core.currency import Currency 35 | from core.spot import Spot 36 | 37 | MONTHLY = list(range(1, 13)) 38 | QUARTERLY = [3, 6, 9, 12] 39 | ODDS = [1, 3, 5, 7, 9, 11] 40 | EVENS = [2, 4, 6, 8, 10, 12] 41 | 42 | instruments_all = { 43 | ### Soft ### 44 | 45 | 'soft': [ 46 | { 47 | 'name': 'arabica', 48 | 'fullname': 'Arabica ICE Coffee C', 49 | 'contract_data': ['quandl'], 50 | 'quandl_database': 'ICE', 51 | 'quandl_data_factor': 100, 52 | 'quandl_symbol': 'KC', 53 | 'first_contract': 197312, 54 | 'months_traded': [3, 5, 7, 9, 12], 55 | 'trade_only': [3, 5, 7, 9, 12], 56 | 'denomination': 'USD', 57 | 'ib_code': 'KC', 58 | 'point_value': 37500, 59 | 'exchange': 'NYBOT', 60 | 'roll_day': -19, 61 | 'spread': 0.0005, 62 | }, { 63 | 'name': 'cattle', 64 | 'fullname': 'Live Cattle', 65 | 'contract_data': ['ib', 'quandl'], 66 | 'quandl_symbol': 'LC', 67 | 'quandl_database': 'CME', 68 | 'first_contract': 196512, 69 | 'months_traded': EVENS, 70 | 'trade_only': [4, 8, 12], 71 | 'denomination': 'USD', 72 | 'ib_code': 'LE', 73 | 'roll_day': -15, 74 | 'exchange': 'GLOBEX', 75 | 'point_value': 400, 76 | 'spread': 0.025, 77 | }, { 78 | 'name': 'cocoa', 79 | 'fullname': 'Cocoa (London)', 80 | 'contract_data': ['quandl'], 81 | 'quandl_database': 'LIFFE', 82 | 'quandl_symbol': 'C', 83 | 'first_contract': 199309, 84 | 'months_traded': [3, 9, 12], 85 | 'trade_only': [3, 9, 12], 86 | 'denomination': 'GBP', 87 | 'ib_code': 'C', 88 | 'point_value': 10, 89 | 'exchange': 'ICEEUSOFT', 90 | 'roll_day': 1, 91 | 'spread': 1, 92 | }, { 93 | 'name': 'cotton', 94 | 'fullname': 'Cotton No 2', 95 | 'contract_data': ['quandl'], 96 | 'quandl_database': 'ICE', 97 | 'quandl_symbol': 'CT', 98 | 'quandl_data_factor': 100, 99 | 'first_contract': 197203, 100 | 'months_traded': [3, 5, 7, 10, 12], 101 | 'trade_only': [12], 102 | 'denomination': 'USD', 103 | 'ib_code': 'CT', 104 | 'point_value': 50000, 105 | 'exchange': 'NYBOT', 106 | 'roll_day': -15, 107 | 'spread': 0.0003, 108 | }, { 109 | 'name': 'feeder', 110 | 'contract_data': ['ib', 'quandl'], 111 | 'quandl_symbol': 'FC', 112 | 'quandl_database': 'CME', 113 | 'first_contract': 197403, 114 | 'months_traded': [1, 3, 4, 5, 8, 9, 10, 11], 115 | 'trade_only': [1, 3, 4, 5, 8, 9, 10, 11], 116 | 'denomination': 'USD', 117 | 'ib_code':'GF', 118 | 'roll_day':-15, 119 | 'exchange': 'GLOBEX', 120 | 'point_value': 500, 121 | 'spread':0.1, 122 | }, { 123 | 'name': 'wheat', 124 | 'contract_data': ['ib', 'quandl'], 125 | 'quandl_database': 'CME', 126 | 'quandl_symbol': 'W', 127 | 'first_contract': 195912, 128 | 'months_traded': [3, 5, 7, 9, 12], 129 | 'trade_only': [3, 5, 7, 9, 12], 130 | 'roll_day': -20, 131 | 'ib_code': 'ZW', 132 | 'exchange': 'ECBOT', 133 | 'point_value': 50, 134 | 'spread': 0.25, 135 | 'commission': 2.81, 136 | }, { 137 | 'name': 'corn', 138 | 'contract_data': ['ib', 'quandl'], 139 | 'months_traded': [3, 5, 7, 9, 12], 140 | 'first_contract': 196003, 141 | 'backtest_from_contract': 199312, 142 | 'trade_only': [12], 143 | 'quandl_symbol': 'C', 144 | 'quandl_database': 'CME', 145 | 'ib_code': 'ZC', 146 | 'roll_day': -20, 147 | 'exchange': 'ECBOT', 148 | 'point_value': 50, 149 | 'spread': 0.25, 150 | 'commission': 2.81, 151 | }, { 152 | #Trade the front 2 contracts only 153 | 'name': 'leanhog', 154 | 'contract_data': ['ib', 'quandl'], 155 | 'quandl_database': 'CME', 156 | 'quandl_symbol': 'LN', 157 | 'first_contract': 197002, 158 | 'roll_day': 7, 159 | 'months_traded': [2, 4, 6, 8, 10, 12], 160 | 'trade_only': [2, 4, 6, 8, 10, 12], 161 | 'ib_code': 'HE', 162 | 'exchange': 'GLOBEX', 163 | 'point_value': 400, 164 | 'spread': 0.025, 165 | 'commission': 2.89, 166 | }, { 167 | 'name': 'oats', 168 | 'contract_data': ['ib', 'quandl'], 169 | 'fullname': 'Oats (CME)', 170 | 'quandl_symbol': 'O', 171 | 'quandl_database': 'CME', 172 | 'first_contract': 197003, 173 | 'months_traded': [3, 5, 7, 9, 12], 174 | 'trade_only': [7, 12], 175 | 'denomination': 'USD', 176 | 'ib_code':'ZO', 177 | 'roll_day':-15, 178 | 'exchange': 'ECBOT', 179 | 'point_value':50, 180 | 'spread':0.25, #guess 181 | }, { 182 | 'name': 'robusta', 183 | 'contract_data': ['quandl'], 184 | 'fullname': 'Robusta Coffee (LIFFE)', 185 | 'quandl_database': 'LIFFE', 186 | 'quandl_symbol': 'RC', 187 | 'first_contract': 199309, 188 | 'months_traded': ODDS, 189 | 'trade_only': ODDS, 190 | 'denomination': 'USD', 191 | 'ib_code': 'D', 192 | 'point_value': 10, 193 | 'exchange': 'ICEEUSOFT', 194 | 'roll_day': 1, 195 | 'spread': 5, 196 | }, { 197 | 'name':'soybean', 198 | 'contract_data': ['ib', 'quandl'], 199 | 'quandl_database': 'CME', 200 | 'quandl_symbol': 'S', 201 | 'first_contract': 196208, 202 | 'months_traded': [1, 3, 5, 7, 8, 9, 11], 203 | 'trade_only': [11], 204 | 'roll_day': 5, 205 | 'denomination': 'USD', 206 | 'ib_code': 'ZS', 207 | 'exchange': 'ECBOT', 208 | 'point_value': 50, 209 | 'spread': 0.25, 210 | }, { 211 | 'name': 'soymeal', 212 | 'fullname': 'Soybean Meal', 213 | 'contract_data': ['ib', 'quandl'], 214 | 'quandl_database': 'CME', 215 | 'quandl_symbol': 'SM', 216 | 'first_contract': 196401, 217 | 'months_traded': [1, 3, 5, 7, 8, 9, 10, 12], 218 | 'trade_only': [7, 12], 219 | 'denomination': 'USD', 220 | 'ib_code': 'ZM', 221 | 'point_value': 100, 222 | 'exchange': 'ECBOT', 223 | 'roll_day': -15, 224 | 'spread': 0.1, 225 | }, { 226 | 'name': 'soyoil', 227 | 'fullname': 'Soybean Oil', 228 | 'contract_data': ['ib', 'quandl'], 229 | 'quandl_database': 'CME', 230 | 'quandl_symbol': 'BO', 231 | 'first_contract': 196101, 232 | 'months_traded': [1, 3, 5, 7, 8, 9, 10, 12], 233 | 'trade_only': [7, 12], 234 | 'denomination': 'USD', 235 | 'ib_code': 'ZL', 236 | 'point_value': 600, 237 | 'exchange': 'ECBOT', 238 | 'roll_day': -15, 239 | 'spread': 0.01, 240 | }, { 241 | 'name': 'sugar', 242 | 'fullname': 'White Sugar (ICE)', 243 | 'contract_data': ['quandl'], 244 | 'quandl_database': 'LIFFE', 245 | 'quandl_symbol': 'W', 246 | 'first_contract': 199310, 247 | 'months_traded': [3, 5, 8, 10, 12], 248 | 'trade_only': [3, 5, 8, 10, 12], 249 | 'denomination': 'USD', 250 | 'ib_code': 'W', 251 | 'point_value': 50, 252 | 'exchange': 'ICEEUSOFT', 253 | 'expiration_month': -1, 254 | 'roll_day': 1, 255 | 'spread': 0.2, 256 | }, 257 | ], 258 | 259 | 260 | ### Currencies ### 261 | 262 | 'currency': [ 263 | { 264 | 'name': 'mxp', 265 | 'contract_data': ['ib', 'quandl'], 266 | 'quandl_database': 'CME', 267 | 'quandl_symbol': 'MP', 268 | 'first_contract': 199603, 269 | 'trade_only': [3, 6, 9, 12], 270 | 'ib_code': 'MXP', 271 | 'roll_day': 5, 272 | 'exchange': 'GLOBEX', 273 | 'quandl_data_factor': 1000000, 274 | 'point_value': 500000, 275 | 'denomination': 'USD', 276 | 'spread': .00001, 277 | 'spot': Currency('MXNUSD').rate, 278 | 'expiry': 18, 279 | }, { 280 | 'name': 'aud', 281 | 'contract_data': ['ib', 'quandl'], 282 | 'quandl_database': 'CME', 283 | 'quandl_symbol': 'AD', 284 | 'first_contract': 198703, 285 | 'months_traded': QUARTERLY, 286 | 'trade_only': QUARTERLY, 287 | 'roll_day': 10, 288 | 'denomination': 'USD', 289 | 'ib_code': 'AUD', 290 | 'exchange': 'GLOBEX', 291 | 'point_value': 100000, 292 | 'spread': 0.0001, 293 | 'spot': Currency('AUDUSD').rate, 294 | 'expiry': 18, 295 | }, { 296 | 'name': 'eur', 297 | 'contract_data': ['ib', 'quandl'], 298 | 'quandl_symbol': 'EC', 299 | 'quandl_database': 'CME', 300 | 'first_contract': 199903, 301 | 'months_traded': [3, 6, 9, 12], 302 | 'trade_only': [3, 6, 9, 12], 303 | 'denomination': 'USD', 304 | 'ib_code':'EUR', 305 | 'roll_day':5, 306 | 'exchange': 'GLOBEX', 307 | 'point_value': 125000, 308 | 'spread':0.00005, 309 | 'spot': Currency('EURUSD').rate, 310 | 'expiry': 18, 311 | }, { 312 | 'name': 'gbp', 313 | 'contract_data': ['ib', 'quandl'], 314 | 'quandl_database': 'CME', 315 | 'quandl_symbol': 'BP', 316 | 'first_contract': 197509, 317 | 'roll_day': 5, 318 | 'months_traded': [3, 6, 9, 12], 319 | 'trade_only': [3, 6, 9, 12], 320 | 'ib_code': 'GBP', 321 | 'exchange': 'GLOBEX', 322 | 'point_value': 62500, 323 | 'denomination': 'USD', 324 | 'spread': .0001, 325 | 'spot': Currency('GBPUSD').rate, 326 | 'expiry': 18, 327 | }, { 328 | 'name':'nzd', 329 | 'contract_data': ['ib', 'quandl'], 330 | 'quandl_symbol': 'NE', 331 | 'quandl_database': 'CME', 332 | 'first_contract': 200403, 333 | 'months_traded': [3, 6, 9, 12], 334 | 'trade_only': [3, 6, 9, 12], 335 | 'denomination': 'USD', 336 | 'ib_code':'NZD', 337 | 'roll_day': 8, 338 | 'exchange': 'GLOBEX', 339 | 'point_value': 1E6, 340 | 'spread':0.0001, 341 | 'spot': Currency('NZDUSD').rate, 342 | 'expiry': 18, 343 | }, { 344 | 'name': 'yen', 345 | 'contract_data': ['ib', 'quandl'], 346 | 'commission': 2.46, 347 | 'quandl_database': 'CME', 348 | 'quandl_symbol': 'JY', 349 | 'first_contract': 197712, 350 | 'months_traded': [3, 6, 9, 12], 351 | 'trade_only': [3, 6, 9, 12], 352 | 'denomination': 'USD', 353 | 'ib_code': 'JPY', 354 | 'exchange': 'GLOBEX', 355 | 'quandl_data_factor': 1000000, 356 | 'point_value': 12500000, 357 | 'spread': .0000005, 358 | 'spot': Currency('JPYUSD').rate, 359 | 'expiry': 18, 360 | }, { 361 | 'name': 'bitcoin', 362 | 'contract_data': ['ib'], 363 | 'point_value': 1, 364 | 'denomination': 'USD', 365 | 'months_traded': MONTHLY, 366 | 'trade_only': MONTHLY, 367 | 'first_contract': 201801, 368 | 'exchange': 'CFECRYPTO', 369 | 'ib_code': 'GXBT', 370 | 'roll_day': 10, 371 | 'expiry': 14, 372 | } 373 | ], 374 | 375 | 376 | ### Hard ### 377 | 378 | 'hard': [ 379 | { 380 | 'name': 'copper', 381 | 'contract_data': ['ib', 'quandl'], 382 | 'quandl_database': 'CME', 383 | 'quandl_symbol': 'HG', 384 | 'first_contract': 196001, 385 | 'trade_only': [7, 12], 386 | 'months_traded': [1, 3, 5, 7, 9, 10, 12], 387 | 'roll_day': -20, 388 | 'commission': 2.36, 389 | 'spread': 0.0005, 390 | 'ib_code': 'HG', 391 | 'exchange': 'NYMEX', 392 | 'point_value': 25000, 393 | 'denomination': 'USD', 394 | 'expiry': 27, 395 | 'spot': Spot('copper').get, 396 | }, { 397 | 'name': 'crude', 398 | 'contract_data': ['ib', 'quandl'], 399 | 'quandl_database': 'CME', 400 | 'quandl_symbol': 'CL', 401 | 'first_contract': 198306, 402 | 'trade_only': [12], 403 | 'roll_day': -15, 404 | 'expiration_month': -1, 405 | 'commission': 2.36, 406 | 'spread': 0.01, 407 | 'ib_code': 'CL', 408 | 'exchange': 'NYMEX', 409 | 'point_value': 1000, 410 | 'denomination': 'USD', 411 | }, { 412 | 'name': 'gas', 413 | 'contract_data': ['ib', 'quandl'], 414 | 'quandl_database': 'CME', 415 | 'quandl_symbol': 'NG', 416 | 'first_contract': 199101, 417 | 'trade_only': [12], 418 | 'spread': 0.001, 419 | 'commission': 2.36, 420 | 'ib_code': 'NG', 421 | 'expiration_month': -1, 422 | 'roll_day': 18, 423 | 'exchange': 'NYMEX', 424 | 'point_value': 10000, 425 | 'denomination': 'USD', 426 | }, { 427 | 'name': 'gold', 428 | 'contract_data': ['ib', 'quandl'], 429 | 'commission': 2.36, 430 | 'quandl_database': 'CME', 431 | 'quandl_symbol': 'GC', 432 | 'first_contract': 197512, 433 | 'months_traded': [2, 4, 8, 10, 12], 434 | 'trade_only': [12], 435 | # Cutoff is 30th of the month before (30 Jan for Feb contract) 436 | 'roll_day' : -15, 437 | 'denomination': 'USD', 438 | 'ib_code': 'GC', 439 | 'exchange': 'NYMEX', 440 | 'point_value': 100, 441 | 'spread': 0.10, 442 | 'spot': Spot('gold').get, 443 | 'expiry': 27, 444 | }, { 445 | 'name': 'pallad', 446 | 'contract_data': ['ib', 'quandl'], 447 | 'quandl_symbol':'PA', 448 | 'quandl_database': 'CME', 449 | 'first_contract':'197703', 450 | 'months_traded': [3,4,5,6,9,12], 451 | 'trade_only': [3,6,9,12], 452 | 'denomination': 'USD', 453 | 'roll_day':-24, 454 | 'ib_code': 'PA', 455 | 'exchange': 'NYMEX', 456 | 'point_value':100, 457 | 'spread':0.50, 458 | 'spot': Spot('pallad').get, 459 | 'expiry': 27, 460 | }, { 461 | 'name': 'platinum', 462 | 'contract_data': ['ib', 'quandl'], 463 | 'quandl_database': 'CME', 464 | 'quandl_symbol': 'PL', 465 | 'first_contract': 201101, 466 | 'roll_day': -10, 467 | 'months_traded': [1, 2, 3, 4, 7, 10, 12], 468 | 'trade_only': [1, 4, 10], 469 | 'commission': 2.36, 470 | 'spread': 0.1, 471 | 'ib_code': 'PL', 472 | 'exchange': 'NYMEX', 473 | 'point_value': 50, 474 | 'denomination': 'USD', 475 | 'expiry': 27, 476 | 'spot': Spot('platinum').get, 477 | }, { 478 | 'name': 'silver', 479 | 'fullname': 'Silver', 480 | 'contract_data': ['ib', 'quandl'], 481 | 'quandl_database': 'CME', 482 | 'quandl_symbol': 'SI', 483 | 'first_contract': 196403, 484 | 'months_traded': [1, 3, 5, 7, 9, 12], 485 | 'trade_only': [3, 7, 12], 486 | 'denomination': 'USD', 487 | 'ib_code': 'SI', 488 | 'point_value': 1000, 489 | 'ib_multiplier': 1000, 490 | 'exchange': 'NYMEX', 491 | 'roll_day': -15, 492 | 'spread': 0.005, #GUESS 493 | 'expiry': 27, 494 | 'spot': Spot('silver').get, 495 | }, 496 | ], 497 | 498 | 499 | ### Index ### 500 | 501 | 'index': [ 502 | { 503 | 'name': 'aex', 504 | 'fullname': 'Amsterdam Exchange Index', 505 | 'contract_data': ['quandl'], 506 | 'quandl_symbol': 'FTI', 507 | 'quandl_database': 'LIFFE', 508 | 'first_contract': 201310, 509 | 'months_traded': MONTHLY, 510 | 'trade_only': MONTHLY, 511 | 'denomination': 'EUR', 512 | 'ib_code': 'EOE', 513 | 'ib_multiplier': 200, 514 | 'exchange': 'FTA', 515 | 'point_value': 200, 516 | 'roll_day': 12, 517 | 'spread': 0.05, 518 | }, { 519 | 'name': 'cac', 520 | 'contract_data': ['quandl'], 521 | 'quandl_database': 'LIFFE', 522 | 'quandl_symbol': 'FCE', 523 | 'first_contract': 199903, 524 | 'trade_only': [3, 6, 9, 12], 525 | 'months_traded': [3, 6, 9, 12], #Not true, but others have dodgy data on Quandl 526 | 'roll_day': 1, 527 | 'ib_code': 'CAC40', 528 | 'exchange': 'MONEP', 529 | 'denomination': 'EUR', 530 | 'point_value': 10, 531 | 'ib_multiplier': 10, 532 | 'spread': 0.5, 533 | 'per_contract_cost': 2, 534 | # 'spot': Index('CAC').close, 535 | 'expiry': 15, 536 | }, { 537 | 'name': 'dax', 538 | 'fullname': 'DAX Index Future', 539 | 'contract_data': ['quandl'], 540 | 'quandl_database': 'EUREX', 541 | 'quandl_symbol': 'FDAX', 542 | 'first_contract': 199703, 543 | 'months_traded': QUARTERLY, 544 | 'trade_only': QUARTERLY, 545 | 'denomination': 'EUR', 546 | 'ib_code': 'DAX', 547 | 'ib_multiplier': 25, 548 | 'point_value': 25, 549 | 'exchange': 'DTB', 550 | 'roll_day': 1, 551 | 'spread': 1, 552 | # 'spot': Index('DAX').close, 553 | 'expiry': 15, 554 | }, { 555 | 'name': 'eurostoxx', 556 | 'contract_data': ['ib', 'quandl'], 557 | 'quandl_symbol': 'FESX', 558 | 'quandl_database': 'EUREX', 559 | 'first_contract': 199809, 560 | 'months_traded': [3, 6, 9, 12], 561 | 'trade_only': [3, 6, 9, 12], 562 | 'denomination': 'EUR', 563 | 'ib_code': 'ESTX50', 564 | 'exchange': 'DTB', 565 | 'point_value': 10, 566 | 'roll_day': 10, 567 | 'spread': 1, 568 | }, { 569 | 'name': 'ftse', 570 | 'fullname': 'FTSE 100 Index', 571 | 'contract_data': ['quandl'], 572 | 'quandl_database': 'LIFFE', 573 | 'quandl_symbol': 'Z', 574 | 'first_contract': 198406, 575 | 'months_traded': QUARTERLY, 576 | 'trade_only': QUARTERLY, 577 | 'denomination': 'GBP', 578 | 'ib_code': 'Z', 579 | 'point_value': 10, 580 | 'exchange': 'ICEEU', 581 | 'roll_day': 1, 582 | 'spread': 0.5, 583 | # 'spot': Index('FTSE').close, 584 | 'expiry': 15, 585 | }, { 586 | 'name': 'hsi', 587 | 'fullname': 'Hang Seng Index', 588 | 'contract_data': ['ib', 'quandl'], 589 | 'quandl_symbol': 'HSI', 590 | 'quandl_database': 'HKEX', 591 | 'quandl_rename_columns': {"Last Traded": 'Settle', "Close": 'Settle'}, 592 | 'first_contract': 199709, 593 | 'months_traded': MONTHLY, 594 | 'trade_only': MONTHLY, 595 | 'denomination': 'HKD', 596 | 'ib_code': 'HSI', 597 | 'point_value': 50, 598 | 'commission': 30, #Commission in HKD 599 | 'exchange': 'HKFE', 600 | 'roll_day': 1, 601 | 'spread': 2, 602 | 'expiry': 28, 603 | }, { 604 | #This is the e-mini version 605 | 'name': 'nasdaq', 606 | 'fullname': 'Nasdaq 100', 607 | 'contract_data': ['ib', 'quandl'], 608 | 'commission': 0.85, 609 | 'quandl_database': 'CME', 610 | 'quandl_symbol': 'NQ', 611 | 'first_contract': 199812, 612 | 'months_traded': [3, 6, 9, 12], 613 | 'trade_only': [3, 6, 9, 12], 614 | 'roll_day': 13, 615 | 'denomination': 'USD', 616 | 'ib_code': 'NQ', 617 | 'exchange': 'GLOBEX', 618 | 'point_value': 20, 619 | 'spread': 0.25, 620 | # 'spot': Index('IUXX').close, 621 | 'expiry': 15, 622 | }, { 623 | 'name': 'r2000', 624 | 'fullname': 'Russell 2000 Mini', 625 | 'contract_data': ['quandl'], 626 | 'quandl_database': 'ICE', 627 | 'quandl_symbol': 'TF', 628 | 'first_contract': 200703, 629 | 'months_traded': QUARTERLY, 630 | 'trade_only': QUARTERLY, 631 | 'denomination': 'USD', 632 | 'ib_code': 'TF', 633 | 'point_value': 50, 634 | 'exchange': 'NYBOT', 635 | 'roll_day': 1, 636 | 'spread': 0.1, 637 | # 'spot': Index('IUX').close, 638 | 'expiry': 15, 639 | }, { 640 | 'name': 'smi', 641 | 'contract_data': ['ib', 'quandl'], 642 | 'quandl_symbol': 'FSMI', 643 | 'quandl_database': 'EUREX', 644 | 'first_contract': 201309, 645 | 'months_traded': [3, 6, 9, 12], 646 | 'trade_only': [3, 6, 9, 12], 647 | 'denomination': 'CHF', 648 | 'ib_code':'SMI', 649 | 'roll_day': -26, #guess 650 | 'exchange': 'SOFFEX', 651 | 'point_value': 10, 652 | 'spread':2, 653 | 'expiry': 15, 654 | }, { 655 | 'name': 'sp500', #trade the emini version 656 | 'contract_data': ['quandl'], 657 | 'quandl_symbol': 'ES', 658 | 'quandl_database': 'CME', 659 | 'first_contract': 199709, 660 | 'months_traded': [3, 6, 9, 12], 661 | 'trade_only': [3, 6, 9, 12], 662 | 'denomination': 'USD', 663 | 'ib_code': 'ES', 664 | 'exchange': 'GLOBEX', 665 | 'point_value': 50, 666 | 'roll_day': 10, 667 | 'spread': 0.25, 668 | 'expiry': 15, 669 | # 'spot': Index('INX').close 670 | }, 671 | { 672 | 'name': 'vix', 673 | 'contract_data': ['ib', 'quandl'], 674 | 'commission': 2.40, 675 | 'quandl_database': 'CBOE', 676 | 'quandl_rename_columns': {"Trade Date": "Date", "Total Volume": "Volume", 677 | # Prevent column duplication after renaming both Close and Settle 678 | "Close": "_Close_"}, 679 | 'quandl_symbol': 'VX', 680 | 'first_contract': 200501, 681 | #Quarterly contracts run for 6 months, other contracts are shorter 682 | # 'spot': lambda: quandl.get("CBOE/VIX")['VIX Close'], 683 | # 'expiry': 15, 684 | 'trade_only': [2,5,8,11], 685 | 'rules': ('sell_and_hold',), 686 | # 'rules': ('ewmac', 'carry_prev',), 687 | 'roll_shift': -91, 688 | 'ib_code': 'VIX', 689 | 'exchange': 'CFE', 690 | 'ib_trading_class': 'VX', 691 | 'ib_multiplier': 1000, 692 | 'point_value': 1000, 693 | 'denomination': 'USD', 694 | 'spread': 0.01, 695 | }, { 696 | 'name': 'vstoxx', 697 | 'contract_data': ['quandl'], 698 | 'commission': 0.0211, 699 | 'quandl_database': 'EUREX', 700 | 'quandl_symbol': 'FVS', 701 | 'first_contract': 201401, 702 | 'roll_day': 15, 703 | # 'rules': ('ewmac', 'carry_prev',), 704 | 'rules': ('sell_and_hold',), 705 | 'roll_shift': -61, 706 | 'trade_only': [1, 3, 5, 7, 9, 11], 707 | 'denomination': 'EUR', 708 | 'ib_code': 'V2TX', 709 | 'exchange': 'DTB', 710 | 'point_value': 100, 711 | 'spread':0.05, 712 | }, 713 | ], 714 | 715 | ### Rate ### 716 | 717 | 'rate': [ 718 | { 719 | 'name': 'eurodollar', 720 | 'contract_data': ['ib', 'quandl'], 721 | 'quandl_symbol': 'ED', 722 | 'first_contract': 199603, 723 | 'months_traded': QUARTERLY, 724 | 'trade_only': [12], 725 | 'quandl_database': 'CME', 726 | 'ib_code': 'GE', 727 | 'exchange': 'GLOBEX', 728 | 'point_value': 2500, 729 | 'denomination': 'USD', 730 | 'spread': 0.005, 731 | 'roll_shift': -900, 732 | 'rules': ('ewmac', 'carry_prev'), 733 | 'commission': 2.11, 734 | }, { 735 | 'name': 'us2', 736 | 'contract_data': ['ib', 'quandl'], 737 | 'quandl_database': 'CME', 738 | 'quandl_symbol': 'TU', 739 | 'first_contract': 199303, 740 | 'roll_day': -20, 741 | 'months_traded': [3, 6, 9, 12], 742 | 'trade_only': [3, 6, 9, 12], 743 | 'spread': 0.0078125, 744 | 'commission': 1.46, 745 | 'ib_code': 'ZT', 746 | 'exchange': 'ECBOT', 747 | 'point_value': 2000, 748 | 'denomination': 'USD', 749 | }, { 750 | 'name': 'bobl', 751 | 'contract_data': ['quandl'], 752 | 'quandl_database': 'EUREX', 753 | 'quandl_symbol': 'FGBM', 754 | 'first_contract': 199903, 755 | 'roll_day': -25, 756 | 'months_traded': [3, 6, 9, 12], 757 | 'trade_only': [3, 6, 9, 12], 758 | 'denomination': 'EUR', 759 | 'ib_code': 'GBM', 760 | 'point_value': 1000, 761 | 'exchange': 'DTB', 762 | 'spread': 0.01, 763 | 'per_contract_cost': 2, 764 | 'rules': ('ewmac',), 765 | }, { 766 | 'name': 'bund', 767 | 'contract_data': ['quandl'], 768 | 'quandl_symbol': 'FGBL', 769 | 'quandl_database': 'EUREX', 770 | 'first_contract': 201303, 771 | 'months_traded': [3, 6, 9, 12], 772 | 'trade_only': [3, 6, 9, 12], 773 | 'denomination': 'EUR', 774 | 'ib_code':'GBL', 775 | 'roll_day':-25, 776 | 'exchange': 'DTB', 777 | 'point_value': 1000, 778 | 'spread':0.01, 779 | }, { 780 | 'name': 'us30', 781 | 'contract_data': ['ib', 'quandl'], 782 | 'quandl_symbol': 'US', 783 | 'quandl_database': 'CME', 784 | 'first_contract': 197712, 785 | 'months_traded': [3, 6, 9, 12], 786 | 'trade_only': [3, 6, 9, 12], 787 | 'denomination': 'USD', 788 | 'ib_code':'ZB', 789 | 'roll_day':-24, 790 | 'exchange': 'ECBOT', 791 | 'point_value':1000, 792 | 'spread':1/32, 793 | }, { 794 | 'name': 'longbtp', 795 | 'contract_data': ['quandl'], 796 | 'quandl_symbol': 'FBTP', 797 | 'quandl_database': 'EUREX', 798 | 'first_contract': 201303, 799 | 'months_traded': [3, 6, 9, 12], 800 | 'trade_only': [3, 6, 9, 12], 801 | 'denomination': 'EUR', 802 | 'ib_code': 'BTP', 803 | 'exchange': 'DTB', 804 | 'point_value': 1000, 805 | 'roll_day': -25, 806 | 'spread': 0.1, 807 | }, { 808 | 'name': 'eurooat', 809 | 'contract_data': ['quandl'], 810 | 'fullname': 'French government bond', 811 | 'quandl_symbol': 'FOAT', 812 | 'quandl_database': 'EUREX', 813 | 'first_contract': 201303, 814 | 'months_traded': QUARTERLY, 815 | 'trade_only': QUARTERLY, 816 | 'denomination': 'EUR', 817 | 'ib_code': 'OAT', 818 | 'ib_multiplier': 1000, 819 | 'point_value': 1000, 820 | 'exchange': 'DTB', 821 | 'roll_day': -25, 822 | 'spread': 0.01, 823 | }, 824 | ], 825 | 826 | } 827 | 828 | # Backward compatibility workaround (flattens instruments_all into a single 1D-list) 829 | instrument_definitions = [entry for group in instruments_all.values() for entry in group] 830 | 831 | 832 | ### Financials 833 | 834 | # { 835 | # 'name': 'us5', 836 | # 'quandl_database': 'CME', 837 | # 'quandl_symbol': 'FV', 838 | # 'first_contract': 198812, 839 | # 'roll_day': -20, 840 | # 'months_traded': [3, 6, 9, 12], 841 | # 'trade_only': [3, 6, 9, 12], 842 | # 'spread': 0.0078125, #TO BE CONFIRMED 843 | # 'commission': 1.51, 844 | # 'ib_code': 'ZF', 845 | # 'exchange': 'ECBOT', 846 | # 'point_value': 1000, 847 | # 'denomination': 'USD', 848 | #} 849 | # { 850 | # 'name': 'zec', 851 | # 'quandl_symbol': 'ZEC', 852 | # 'first_contract': 201703, 853 | # 'months_traded': [3, 6, 9, 12], 854 | # 'trade_only': [3, 6, 9, 12], 855 | # 'roll_day': 27, 856 | # 'denomination': 'XBT', 857 | # 'broker': 'bitmex', 858 | # 'contract_name_format': 'bitmex', 859 | # 'point_value': 1, 860 | # 'spread': 0.01, 861 | # 'rules': ('ewmac',), 862 | # # 'spot': core.currency.Currency('XBTUSD').rate, 863 | # }, { 864 | # 'name': 'etc', 865 | # 'quandl_symbol': 'ETC7D', 866 | # # 'first_contract': 201703, 867 | # # 'months_traded': [3, 6, 9, 12], 868 | # # 'trade_only': [3, 6, 9, 12], 869 | # # 'roll_day': 27, 870 | # 'denomination': 'XBT', 871 | # 'broker': 'bitmex', 872 | # 'contract_name_format': 'bitmex', 873 | # 'point_value': 1, 874 | # 'spread': 0.01, 875 | # 'rules': ('ewmac',), 876 | # }, { 877 | # 'name': 'us10', 878 | # 'quandl_symbol': 'TY', 879 | # 'quandl_database': 'CME', 880 | # 'first_contract': 198206, 881 | # 'months_traded': [3, 6, 9, 12], 882 | # 'trade_only': [3, 6, 9, 12], 883 | # 'denomination': 'USD', 884 | # 'ib_code':'ZN', 885 | # 'roll_day':-24, 886 | # 'exchange': 'ECBOT', 887 | # 'point_value':1000, 888 | # 'spread':1/64, 889 | # },{ 890 | # 'name': 'n225', 891 | # 'fullname': 'Nikkei 225 Index Futures', 892 | # 'quandl_database': 'OSE', 893 | # 'quandl_symbol': 'NK225', 894 | # 'quandl_rename_columns': {'Sett Price': 'Settle'}, 895 | # 'first_contract': 201612, 896 | # 'months_traded': QUARTERLY, 897 | # 'trade_only': QUARTERLY, 898 | # 'denomination': 'JPY', 899 | # 'ib_code': 'N225', 900 | # 'point_value': 1000, 901 | # 'exchange': 'OSE.JPN', 902 | # 'roll_day': -15, 903 | # 'spread': 10, 904 | 905 | # }, { 906 | # 'name': 'dow', 907 | # 'fullname': 'Dow Jones Industrial Industrial Mini', 908 | # 'quandl_database': 'CME', 909 | # 'quandl_symbol': 'DJ', 910 | # 'first_contract': 201203, 911 | # 'months_traded': QUARTERLY, 912 | # 'trade_only': QUARTERLY, 913 | # 'denomination': 'USD', 914 | # 'ib_code': 'YM', 915 | # 'point_value': 5, 916 | # 'exchange': 'ECBOT', 917 | # 'roll_day': 1, 918 | # 'spread': 1, 919 | 920 | -------------------------------------------------------------------------------- /config/portfolios.py: -------------------------------------------------------------------------------- 1 | import config.instruments 2 | import random 3 | 4 | p_all = [v['name'] for v in config.instruments.instrument_definitions] 5 | 6 | p_soft = [v['name'] for v in config.instruments.instruments_all['soft']] 7 | p_hard = [v['name'] for v in config.instruments.instruments_all['hard']] 8 | p_currency = [v['name'] for v in config.instruments.instruments_all['currency']] 9 | p_rate = [v['name'] for v in config.instruments.instruments_all['rate']] 10 | p_index = [v['name'] for v in config.instruments.instruments_all['index']] 11 | # p_bitmex = [v['name'] for v in config.instruments.instruments_all['bitmex']] 12 | 13 | p_trade = [i for i in p_all if i not in (['dax', 'cac', 'aex', 'sp500', 'r2000', 'ftse', 'bitcoin'])] 14 | 15 | # Randomly split our portfolio into learning and testing sets 16 | x = p_trade 17 | random.shuffle(x) 18 | l = int(len(x)/2) 19 | p_learn, p_test = x[:l], x[l:] 20 | -------------------------------------------------------------------------------- /config/settings.py.template: -------------------------------------------------------------------------------- 1 | import quandl 2 | import logging 3 | import os 4 | 5 | 6 | # The currency of your broker account 7 | base_currency = 'GBP' 8 | # List of data sources used to pull quotes 9 | # Comment a line to disable a data source globally 10 | data_sources = [ 11 | 'ib', # Interactive Brokers (interactivebrokers.com) 12 | 'quandl', # quandl.com 13 | ] 14 | 15 | # Backend to be used as a storage for downloaded quotes. Possible values: 'hdf5' 16 | # HDF5 storage only requires pandas 17 | quotes_storage = 'hdf5' 18 | 19 | quandl.ApiConfig.api_key = "QUANDL_API_KEY" 20 | 21 | # Port configured in IB TWS or Gateway terminal 22 | ib_port = 4001 23 | 24 | # Insert MongoDB connection string if you use it as IB logs storage (optional) 25 | iblog_host = "mongodb://[username]:[password]@[host]:[port]" 26 | hdf_path = os.path.join("price_data/", "quotes") 27 | 28 | # Define logging settings 29 | console_logger = { 30 | 'enabled': True, 31 | 'level': logging.INFO, 32 | } 33 | file_logger = { 34 | 'enabled': True, 35 | 'level': logging.DEBUG, 36 | 'file_name': os.path.join("logs/", "pytrendfollow.log") 37 | } 38 | -------------------------------------------------------------------------------- /config/spots.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This file defines spot prices for futures. Loaded into core.spot.Spot object. 4 | Spots are the underlying prices for future contracts, used for trading with "carry spot" rule. 5 | 6 | { 7 | 'name': 'copper', # Unique name, usually the same as the derivative instrument from config/instruments.py 8 | 'price_data': ['quandl'], # Data providers we should collect and merge data from 9 | 'quandl_database': 'LME', # Database on Quandl providing the data 10 | 'quandl_symbol': 'PR_CU', # Symbol on Quandl 11 | 'quandl_column': 'Cash Buyer', # Data column on Quandl to be used as a 'close' price 12 | 'multiplier': (1/2204.62) # A factor to multiply the close price by 13 | ... 14 | 'ib_symbol': 'HSI', # Symbol on Interactive Brokers 15 | 'ib_exchange': 'HKFE', # Exchange identifier on Interactive Brokers 16 | 'denomination': 'HKD', # Denomination currency on Interactive Brokers 17 | 'sec_type': 'IND', # Contract security type on Interactive Brokers 18 | ... (currently only 'IND' is used, which stands for 'Index') 19 | 'bitmex_symbol': '.STRXBT', # Symbol on Bitmex 20 | 'bitmex_column': 'close' # Data column on Bitmex to be used as a 'close' price 21 | }, 22 | 23 | """ 24 | 25 | spots_definitions = [ 26 | 27 | # Hard 28 | { 29 | 'name': 'copper', 30 | 'price_data': ['quandl'], 31 | 'quandl_database': 'LME', 32 | 'quandl_symbol': 'PR_CU', 33 | 'quandl_column': 'Cash Buyer', 34 | 'multiplier': (1/2204.62) 35 | }, 36 | { 37 | 'name': 'gold', 38 | 'price_data': ['quandl'], 39 | 'quandl_database': 'LBMA', 40 | 'quandl_symbol': 'GOLD', 41 | 'quandl_column': 'USD (AM)', 42 | }, 43 | { 44 | 'name': 'pallad', 45 | 'price_data': ['quandl'], 46 | 'quandl_database': 'LPPM', 47 | 'quandl_symbol': 'PALL', 48 | 'quandl_column': 'USD AM', 49 | }, 50 | { 51 | 'name': 'platinum', 52 | 'price_data': ['quandl'], 53 | 'quandl_database': 'LPPM', 54 | 'quandl_symbol': 'PLAT', 55 | 'quandl_column': 'USD AM', 56 | }, 57 | { 58 | 'name': 'silver', 59 | 'price_data': ['quandl'], 60 | 'quandl_database': 'LBMA', 61 | 'quandl_symbol': 'SILVER', 62 | 'quandl_column': 'USD', 63 | }, 64 | 65 | # Index 66 | { 67 | 'name': 'hsi', 68 | 'price_data': ['ib'], 69 | 'ib_symbol': 'HSI', 70 | 'ib_exchange': 'HKFE', 71 | 'denomination': 'HKD', 72 | 'sec_type': 'IND', 73 | }, 74 | { 75 | 'name': 'smi', 76 | 'price_data': ['ib'], 77 | 'ib_symbol': 'SMI', 78 | 'ib_exchange': 'SOFFEX', 79 | 'denomination': 'CHF', 80 | 'sec_type': 'IND', 81 | }, 82 | { 83 | 'name': 'eurostoxx', 84 | 'price_data': ['ib'], 85 | 'ib_symbol': 'ESTX50', 86 | 'ib_exchange': 'DTB', 87 | 'denomination': 'EUR', 88 | 'sec_type': 'IND', 89 | }, 90 | 91 | # Bitmex 92 | { 93 | 'name': 'xbj', 94 | 'price_data': ['bitmex'], 95 | 'bitmex_symbol': '.XBTJPY', 96 | 'bitmex_column': 'close' 97 | }, 98 | { 99 | 'name': 'bat', 100 | 'price_data': ['bitmex'], 101 | 'bitmex_symbol': '.BATXBT', 102 | 'bitmex_column': 'close' 103 | }, 104 | { 105 | 'name': 'dash', 106 | 'price_data': ['bitmex'], 107 | 'bitmex_symbol': '.DASHXBT', 108 | 'bitmex_column': 'close' 109 | }, 110 | { 111 | 'name': 'ethereum', 112 | 'price_data': ['bitmex'], 113 | 'bitmex_symbol': '.ETHXBT', 114 | 'bitmex_column': 'close' 115 | }, 116 | { 117 | 'name': 'litecoin', 118 | 'price_data': ['bitmex'], 119 | 'bitmex_symbol': '.LTCXBT', 120 | 'bitmex_column': 'close' 121 | }, 122 | { 123 | 'name': 'monero', 124 | 'price_data': ['bitmex'], 125 | 'bitmex_symbol': '.XMRXBT', 126 | 'bitmex_column': 'close' 127 | }, 128 | { 129 | 'name': 'stellar', 130 | 'price_data': ['bitmex'], 131 | 'bitmex_symbol': '.STRXBT', 132 | 'bitmex_column': 'close' 133 | }, 134 | { 135 | 'name': 'ripple', 136 | 'price_data': ['bitmex'], 137 | 'bitmex_symbol': '.XRPXBT', 138 | 'bitmex_column': 'close' 139 | }, 140 | { 141 | 'name': 'zcash', 142 | 'price_data': ['bitmex'], 143 | 'bitmex_symbol': '.ZECXBT', 144 | 'bitmex_column': 'close' 145 | }, 146 | ] 147 | 148 | spots_all = {x['name']: x for x in spots_definitions} -------------------------------------------------------------------------------- /config/strategy.py.template: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # The amount of annual fluctuation allowed for a portfolio 4 | annual_volatility_target = 0.125 5 | daily_volatility_target = annual_volatility_target / np.sqrt(252) 6 | # Assumed capital volume for position calculation (in the base currency) 7 | capital = 500000 8 | # Trading rules to use by default if not specified in the instrument definition 9 | default_rules = ('ewmac', 'carry', ) 10 | # Scheduled time for trading 11 | portfolio_sync_time = '07:00' 12 | 13 | # Bootstrapped weights for trading rules 14 | rule_weights = {'carry': 1.0761308295764749, 15 | 'ewmac16': 0.94565802638498464, 16 | 'ewmac32': 0.97404151565516006, 17 | 'ewmac64': 1.037272611390585, 18 | 'ewmac8': 0.96689706277925536, 19 | 'buy_and_hold': 1, 20 | 'sell_and_hold': 1, 21 | } 22 | 23 | # Bootstrapped weights for each instrument in the portfolio 24 | portfolio_weights = { 25 | 'arabica': 1.0764621086234525, 26 | 'aud': 0.74729659791597902, 27 | # 'bobl': 0.9814458316942235, 28 | 'bund': 1.1254544260545285, 29 | 'cattle': 0.64460711884422217, 30 | 'cocoa': 1.0488721487984123, 31 | 'copper': 1.0521931052722646, 32 | 'corn': 0.444453118917291, 33 | 'cotton': 1.3980293077519761, 34 | 'crude': 1.2392045757961074, 35 | 'eur': 0.99783156513366777, 36 | 'eurodollar': 0.86480269329493398, 37 | 'eurooat': 1.424673335923534, 38 | # 'eurostoxx': 0.48127295259975711, 39 | 'eurostoxx': 0, 40 | 'feeder': 0.8587566361752671, 41 | # 'ftse': 0.79082402930099094, 42 | 'gas': 1.6888186395658307, 43 | 'gbp': 0.50499944731386159, 44 | 'gold': 0.68655819637369997, 45 | # 'hsi': 1.1234630740851541, 46 | 'hsi': 0, 47 | 'leanhog': 1.3476820013938289, 48 | 'longbtp': 1.1655100531388629, 49 | 'mxp': 1.2502890117057595, 50 | 'nasdaq': 0.99988109957289095, 51 | 'nzd': 0.78928436932897361, 52 | 'oats': 0.96279417063621475, 53 | 'pallad': 1.788833065344186, 54 | 'platinum': 1.3626265675542621, 55 | # 'robusta': 0.99046437107570096, 56 | 'robusta': 0, 57 | 'silver': 0.37040956997717217, 58 | # 'smi': 1.0303104515173107, 59 | 'smi': 0, 60 | 'soybean': 0.44198400941934951, 61 | 'soymeal': 0.69622767460724611, 62 | 'soyoil': 0.60480786480989479, 63 | # 'sugar': 1.6122731268294883, 64 | 'sugar': 0, 65 | 'us2': 1.3762540456060453, 66 | 'us30': 0.59543924706233453, 67 | 'vix': 0, 68 | 'vstoxx': 0, 69 | # 'vix': 1.6757839160024335, 70 | # 'vstoxx': 1.0301670670359759, 71 | 'wheat': 0.82110319441126278, 72 | 'yen': 0.99016423742465798} 73 | -------------------------------------------------------------------------------- /core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrism2671/PyTrendFollow/439232aed4d67ccf339e782b7c8b96f2dd961d43/core/__init__.py -------------------------------------------------------------------------------- /core/basestore.py: -------------------------------------------------------------------------------- 1 | from config.settings import quotes_storage 2 | 3 | """ 4 | This file imports data read/write methods for a local storage depending on the user's choice. 5 | These methods are used in core.contract_store. 6 | """ 7 | 8 | if quotes_storage == 'hdf5': 9 | from core.hdfstore import read_symbol, read_contract, write_data, drop_symbol -------------------------------------------------------------------------------- /core/contract_store.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import core.basestore as basestore 3 | from enum import Enum 4 | 5 | # This list is kept separately from the global config.settings.data_sources, because data may 6 | # still be written for a data provider, while it is disabled and not used 7 | providers_list = ['ib', 'quandl'] 8 | 9 | 10 | # Types of data that can be stored. Each type corresponds to a MySQL table or 11 | # a separate directory if HDF storage is used 12 | class QuotesType(Enum): 13 | futures = 'futures' 14 | currency = 'currency' 15 | others = 'others' 16 | 17 | 18 | # Define column names mapping from data provider's API to a local storage schema 19 | columns_mapping = { 20 | ('quandl', 'futures'): {'Date': 'date', 'Trade Date': 'date', 'Open': 'open', 'High': 'high', 21 | 'Low': 'low', 'Settle': 'close', 'Last Traded': 'close', 22 | 'Close': 'close', 'Volume': 'volume', 'Total Volume': 'volume'}, 23 | ('quandl', 'currency'): {'Date': 'date', 'Rate': 'rate', 'High (est)': 'high', 24 | 'Low (est)': 'low'}, 25 | ('quandl', 'others'): {'Date': 'date'}, 26 | ('ib', 'futures'): {}, 27 | ('ib', 'others'): {}, 28 | ('ib', 'currency'): {'close': 'rate'}, 29 | } 30 | 31 | 32 | class Store(object): 33 | """ 34 | Class to write, read and delete data from local storage (either MySQL of HDF5) 35 | """ 36 | def __init__(self, provider, quotes_type, symbol): 37 | assert isinstance(quotes_type, QuotesType) 38 | assert provider in providers_list 39 | self.provider = provider 40 | self.key = symbol 41 | self.quotes_type = quotes_type 42 | self.symbol = symbol 43 | 44 | def update(self, new_data): 45 | """Write a DataFrame to the local storage. Data will be updated on index collision""" 46 | assert type(new_data) is pd.DataFrame 47 | basestore.write_data(new_data, self.symbol, self.quotes_type.value, self.provider) 48 | 49 | def get(self): 50 | """Read a symbol from the local storage""" 51 | return basestore.read_symbol(self.symbol, self.quotes_type.value, self.provider) 52 | 53 | def delete(self): 54 | """Delete a symbol from the local storage""" 55 | basestore.drop_symbol(self.symbol, self.quotes_type.value, self.provider) 56 | -------------------------------------------------------------------------------- /core/currency.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import pandas as pd 3 | import config.settings 4 | import config.currencies 5 | from core import data_feed 6 | from core.logger import get_logger 7 | 8 | logger = get_logger('currency') 9 | 10 | 11 | class Currency(object): 12 | """ 13 | Object representing currency exchange rate 14 | """ 15 | 16 | @classmethod 17 | def load_all(cls): 18 | """Load all currencies in the system into a dictionary""" 19 | return {v['code']: Currency(v['code']) for v in config.currencies.currencies_definitions} 20 | 21 | def __init__(self, code): 22 | """Initialise the currency with defaults, taking overrides from the config/currencies.py""" 23 | self.code = code 24 | self.ib_symbol = None 25 | self.quandl_symbol = None 26 | self.currency_data = ['ib', 'quandl'] 27 | kwargs = config.currencies.currencies_all[code] 28 | for key, value in kwargs.items(): 29 | setattr(self, key, value) 30 | 31 | def rate(self, nofx=False): 32 | """ 33 | :param nofx: used to disable currency rates in the AccountCurve 34 | :return: currency exchange rate as pd.Series, or 1 in special cases 35 | """ 36 | if nofx is True: 37 | return 1 38 | elif self.code == config.settings.base_currency * 2: 39 | return 1 40 | else: 41 | data = data_feed.get_currency(self) 42 | if data is None or data.empty: 43 | raise Exception("No price data for currency %s" % self.code) 44 | return data['rate'] 45 | 46 | def __repr__(self): 47 | return self.code 48 | 49 | def age(self): 50 | """ 51 | :return: age of the price data in days 52 | """ 53 | if self.rate() is 1: 54 | return 0 55 | else: 56 | return (pd.to_datetime(datetime.date.today()) - self.rate().tail(1).index[0]).days 57 | -------------------------------------------------------------------------------- /core/data_feed.py: -------------------------------------------------------------------------------- 1 | from core.contract_store import Store, QuotesType 2 | import core.logger 3 | from config.settings import data_sources 4 | logger = core.logger.get_logger('data_feed') 5 | 6 | """ 7 | This file defines functions to pull the contracts data from the local storage for multiple 8 | data providers and merge them into a single pd.DataFrame. On duplicate index, data providers are 9 | prioritized in the order they appear in the list in the instrument definition (config/instruments.py) 10 | """ 11 | 12 | 13 | def get_instrument(instrument): 14 | """ 15 | Get all contracts data for an instrument 16 | :param instrument: core.trading.Instrument object 17 | :return: price data as pd.DataFrame 18 | """ 19 | # choose data providers with respect to the global list 20 | providers = [x for x in instrument.contract_data if x in data_sources] 21 | data = None 22 | for p in providers: 23 | if p == 'ib': 24 | db = instrument.exchange 25 | symbol = instrument.ib_code 26 | elif p == 'quandl': 27 | db = instrument.quandl_database 28 | symbol = instrument.quandl_symbol 29 | else: 30 | raise Exception('Unknown data provider string: %s' % p) 31 | p_data = _get_data(p, QuotesType.futures, db, symbol) 32 | data = p_data if data is None else data.combine_first(p_data) 33 | return data 34 | 35 | 36 | def get_currency(currency): 37 | """ 38 | Get data for a currency 39 | :param currency: core.currency.Currency object 40 | :return: rate data as pd.DataFrame 41 | """ 42 | # choose data providers with respect to the global list 43 | providers = [x for x in currency.currency_data if x in data_sources] 44 | data = None 45 | for p in providers: 46 | if p == 'ib': 47 | db = currency.ib_exchange 48 | symbol = currency.ib_symbol + currency.ib_currency 49 | elif p == 'quandl': 50 | db = currency.quandl_database 51 | symbol = currency.quandl_symbol 52 | else: 53 | raise Exception('Unknown data provider string: %s' % p) 54 | p_data = _get_data(p, QuotesType.currency, db, symbol) 55 | data = p_data if data is None else data.combine_first(p_data) 56 | return data 57 | 58 | 59 | def get_spot(spot): 60 | """ 61 | Get data for a spot 62 | :param spot: core.spot.Spot object 63 | :return: price data as pd.DataFrame 64 | """ 65 | # choose data providers with respect to the global list 66 | providers = [x for x in spot.price_data if x in data_sources] 67 | data = None 68 | for p in providers: 69 | if p == 'ib': 70 | db = spot.ib_exchange 71 | symbol = spot.ib_symbol 72 | elif p == 'quandl': 73 | db = spot.quandl_database 74 | symbol = spot.quandl_symbol 75 | else: 76 | raise Exception('Unknown data provider string: %s' % p) 77 | p_data = _get_data(p, QuotesType.others, db, symbol) 78 | data = p_data if data is None else data.combine_first(p_data) 79 | return data 80 | 81 | 82 | def get_quotes(provider, **kwargs): 83 | """ 84 | General method to get any quotes of type "others" from a single data provider 85 | :param provider: data provider name string 86 | :return: price data as pd.DataFrame 87 | """ 88 | return _get_data(provider, QuotesType.others, kwargs['database'], kwargs['symbol']) 89 | 90 | 91 | def _get_data(library, q_type, database, symbol, **kwargs): 92 | """ 93 | General method to get quotes data from storage 94 | :param library: storage library name (usually corresponds to a data provider name) 95 | :param q_type: one of 'futures' | 'currency' | 'others' 96 | :param database: local storage database name 97 | :param symbol: local storage symbol name 98 | :return: pd.DataFrame or None in case of error 99 | """ 100 | try: 101 | return Store(library, q_type, database + '_' + symbol).get() 102 | except Exception as e: 103 | logger.warning("Something went wrong on symbol %s_%s request from storage: %s" % 104 | (database, symbol, e)) 105 | return None 106 | -------------------------------------------------------------------------------- /core/hdfstore.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from config.settings import hdf_path 3 | import os 4 | import threading 5 | 6 | """ 7 | Implementation of the local storage with HDF5 files as the backend. 8 | 9 | * Directories structure for HDF storage: "[settings.hdf_path]/[provider]/[q_type]/[symbol].h5" 10 | * For q_type == 'futures' returnes DataFrame with the multi-index ['contract', 'date'], 11 | for other quotes types index is ['date'] 12 | """ 13 | lock = threading.Lock() 14 | 15 | 16 | def fname(symbol, q_type, provider): 17 | """ 18 | Resolve the HDF file name for the given store symbol 19 | """ 20 | fname = os.path.join(hdf_path, provider, q_type, symbol + '.h5') 21 | # create directories 22 | if not os.path.exists(os.path.dirname(fname)): 23 | os.makedirs(os.path.dirname(fname)) 24 | return fname 25 | 26 | 27 | def read_contract(symbol, contract, provider): 28 | """ 29 | Read a single contract for a future instrument 30 | """ 31 | if os.path.exists(fname(symbol, 'futures', provider)): 32 | data = pd.read_hdf(fname(symbol, 'futures', provider), 'quotes') 33 | return data.loc[int(contract), :] 34 | else: 35 | c = ['contract', 'date'] 36 | return pd.DataFrame(columns=c).set_index(c) 37 | 38 | 39 | def read_symbol(symbol, q_type, provider): 40 | """ 41 | Read data from the corresponding HDF file 42 | """ 43 | if os.path.exists(fname(symbol, q_type, provider)): 44 | data = pd.read_hdf(fname(symbol, q_type, provider), 'quotes') 45 | return data 46 | else: # if symbol doesn't exist, an empty df is returned with only index columns 47 | c = ['contract', 'date'] if q_type == 'futures' else ['date'] 48 | return pd.DataFrame(columns=c).set_index(c) 49 | 50 | 51 | def write_data(data, symbol, q_type, provider): 52 | """ 53 | Writes a dataframe to HDF file. If the symbol exists, the existing data will be updated 54 | on dataframe keys, favoring new data. 55 | """ 56 | # Use a thread lock here to ensure the read+merge+write operation is atomic 57 | # and HDF files won't get corrupted on concurrent downloading 58 | lock.acquire() 59 | existing_data = read_symbol(symbol, q_type, provider) 60 | idx = ['contract', 'date'] if q_type == 'futures' else ['date'] 61 | lvl = 0 if q_type == 'futures' else None 62 | data = data.set_index(idx) 63 | new_data = data.combine_first(existing_data).sort_index(level=lvl) 64 | new_data.to_hdf(fname(symbol, q_type, provider), 'quotes', mode='w', format='f') 65 | lock.release() 66 | 67 | 68 | def drop_symbol(symbol, q_type, provider): 69 | """ 70 | Remove the corresponding HDF file 71 | """ 72 | os.remove(fname(symbol, q_type, provider)) 73 | -------------------------------------------------------------------------------- /core/ib_connection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | This file defines the global IB connection properties shared among all clients 4 | """ 5 | 6 | start_id = 100 7 | _client_id = None 8 | 9 | 10 | def get_next_id(): 11 | global _client_id 12 | if not _client_id: 13 | _client_id = start_id 14 | else: 15 | _client_id += 1 16 | return _client_id 17 | 18 | __all__ = ['get_next_id'] -------------------------------------------------------------------------------- /core/instrument.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from collections import deque 3 | from functools import lru_cache 4 | import sys 5 | try: 6 | import config.strategy 7 | except ImportError: 8 | print("You need to set up the strategy file at config/strategy.py.") 9 | sys.exit() 10 | 11 | from core.currency import Currency 12 | import numpy as np 13 | import pandas as pd 14 | from trading.accountcurve import accountCurve 15 | 16 | import config.instruments 17 | import config.settings 18 | import trading.rules 19 | from core import data_feed 20 | from core.logger import get_logger 21 | from core.utility import contract_to_tuple, cbot_month_code, generate_roll_progression, weight_forecast 22 | 23 | logger = get_logger('instrument') 24 | 25 | 26 | class Instrument(object): 27 | """ 28 | Object representing a future instrument 29 | """ 30 | 31 | @classmethod 32 | def load(self, instruments): 33 | """Loads all instruments in the system into a dict so that they can be easily worked with. 34 | 35 | For example: 36 | i = Instruments.load() # Load all the instrument data to memory 37 | i['corn'].contracts() # Show the corn contracts 38 | 39 | """ 40 | if instruments == None: 41 | return {v['name']: Instrument(**v) for v in config.instruments.instrument_definitions} 42 | else: 43 | j = {v['name']: v for v in config.instruments.instrument_definitions} 44 | return {k: Instrument(**j[k]) for k in instruments} 45 | 46 | def __repr__(self): 47 | return self.name 48 | 49 | def __init__(self, **kwargs): 50 | """Initialise the instrument object with defaults, taking overrides from the config/instruments.py file.""" 51 | self.quandl_data_factor = 1 # Sometimes Quandl data needs to be multiplied/transformed (e.g. MXP) 52 | self.backtest_from_contract = False # Back test from contract (not date) e.g. 198512. Doesn't have to be a real contract. 53 | self.backtest_from_year = 1960 54 | self.roll_day = 14 55 | self.roll_shift = 0 # Move the roll day forwards (nearer) by this many days if negative, and vice-versa 56 | self.expiration_month = 0 57 | self.expiry = 15 # Default expiry day. 58 | self.commission = 2.50 # Default broker commission. 59 | self.spread = 0 # Default spread. 60 | self.denomination = 'USD' # Default demonination. 61 | self.months_traded = tuple(range(1, 13)) # 12 months 62 | self.trade_only = tuple(range(1, 13)) # 12 months 63 | self.rules = config.strategy.default_rules 64 | self.weights = config.strategy.rule_weights 65 | self.broker = 'ib' # Default broker 66 | self.bootstrapped_weights = None # Only used to store results of bootstrapping. Not used for trading. 67 | self.first_contract = None 68 | self.contract_data = ['ib', 'quandl'] 69 | 70 | # assign properties from instrument definition 71 | for key, value in kwargs.items(): 72 | value = tuple(value) if (type(value) == list) else value 73 | setattr(self, key, value) 74 | 75 | self.weights = pd.DataFrame.from_dict(self.weights, orient='index').transpose().loc[0] 76 | self.currency = Currency(self.denomination + config.settings.base_currency) 77 | 78 | def calculate(self): 79 | """ 80 | Utility function that calculates the position we want to be in for this contract. 81 | 82 | Called from Portfolio using multiprocessing to speed things up. 83 | 84 | Returns a dict with the important things we need in to trade with. 85 | """ 86 | if self.currency.rate() is 1: 87 | rate = pd.Series(1, index=self.panama_prices().index) 88 | else: 89 | rate = self.currency.rate() 90 | return { 91 | 'panama_prices': self.panama_prices(), 92 | 'roll_progression': self.roll_progression(), 93 | 'market_price': self.market_price(), 94 | 'position': self.position(), 95 | 'rate': rate, 96 | } 97 | 98 | def latest_price_date(self): 99 | """Gets the date of the latest price we've got, to help us calculate how old our data is.""" 100 | current_contract = self.roll_progression().loc[datetime.date.today()] 101 | latest_price = self.contracts().loc[current_contract].tail(1) 102 | return latest_price.index[0] # latest price date 103 | 104 | def validate(self): 105 | """Validates that the instrument has up-to-date data and sane results.""" 106 | d = { 107 | 'is_valid': True, 108 | 'today': pd.to_datetime(datetime.date.today()), 109 | 'latest_price_date': None, 110 | 'carry_forecast': np.nan, 111 | 'weighted_forecast': np.nan, 112 | 'panama_date': None, 113 | 'vol': np.nan, 114 | 'price_age': None, 115 | 'currency_age': None, 116 | 'panama_age': None, 117 | } 118 | try: 119 | # Do we have today's price? 120 | try: 121 | lpd = self.latest_price_date() 122 | d['latest_price_date'] = lpd 123 | except KeyError: 124 | raise Exception("couldn't obtain the latest price date") 125 | 126 | # Check carry is not zero 127 | try: 128 | d['carry_forecast'] = self.forecasts()['carry'].tail(1)[0] 129 | except KeyError: 130 | pass 131 | 132 | d['weighted_forecast'] = self.weighted_forecast().tail(1)[0] 133 | 134 | # Check panama_price data is not cached 135 | d['panama_date'] = self.panama_prices().tail(1).index[0] 136 | 137 | # Check volatility is not zero 138 | try: 139 | d['vol'] = self.return_volatility().loc[lpd] 140 | except KeyError: 141 | logger.warning("Validation for instrument %s failed: no volatility data" % self.name) 142 | d['is_valid'] = False 143 | return d 144 | 145 | d['price_age'] = (d['today'] - d['latest_price_date']).days 146 | d['currency_age'] = self.currency.age() 147 | d['panama_age'] = (d['today'] - d['panama_date']).days 148 | except Exception as e: 149 | logger.warning("Validation for instrument %s failed: %s" % (self.name, str(e))) 150 | d['is_valid'] = False 151 | finally: 152 | return d 153 | 154 | ### Pricing 155 | 156 | def pp(self, **kw): 157 | """ 158 | Shorthand function for Instrument.panama_prices() 159 | """ 160 | return self.panama_prices(**kw) 161 | 162 | def rp(self, **kw): 163 | """ 164 | Shorthand function for Instrument.roll_progression() 165 | """ 166 | return self.roll_progression(**kw) 167 | 168 | @lru_cache(maxsize=1) 169 | def panama_prices(self): 170 | """ 171 | Returns a Series representing a 'continuous future' time series. 172 | Our system uses the simplest method - 'panama stitching'. 173 | Absolute prices won't make any sense, but daily returns and trends are preserved. Perfectly suitable for our purposes. 174 | """ 175 | return self.contracts()['close'].diff().to_frame().swaplevel().fillna(0).join( 176 | self.rp().to_frame().set_index('contract',append=True), how='inner').\ 177 | reset_index('contract',drop=True)['close'].cumsum().rename(self.name) 178 | 179 | @lru_cache(maxsize=2) 180 | def return_volatility(self, **kw): 181 | """ 182 | Returns a Series with the EWA volatility of returns for this instrument. 183 | """ 184 | return (self.panama_prices() * self.point_value).\ 185 | diff().ewm(span=36, min_periods=36).std() * self.currency.rate(**kw) 186 | 187 | def market_price(self): 188 | """ 189 | Returns a series of real market prices for this contract. 190 | """ 191 | return self.roll_progression().to_frame().set_index('contract', append=True).\ 192 | swaplevel().join(self.contracts())['close'].swaplevel().dropna() 193 | 194 | ### Forecast & Position 195 | 196 | def position(self, capital=config.strategy.capital, forecasts=None, nofx=False): 197 | """ 198 | The ideal position with the current capital, rules set and strategy 199 | """ 200 | if forecasts is None: 201 | forecasts = self.weighted_forecast() 202 | position = (forecasts * (config.strategy.daily_volatility_target * capital / 10))\ 203 | .divide(self.return_volatility(nofx=nofx)[forecasts.index], axis=0) 204 | return np.around(position) 205 | 206 | @lru_cache(maxsize=8) 207 | def forecasts(self, rules=None): 208 | """ 209 | Position forecasts for individual trading rules 210 | """ 211 | if rules is None: 212 | rules = self.rules 213 | return pd.concat(list(map(lambda x: getattr(trading.rules, str(x))(self), rules)), axis=1).dropna() 214 | 215 | def weighted_forecast(self, rules=None): 216 | """ 217 | Returns a Series representing the weighted forecast for this instrument. 218 | """ 219 | return weight_forecast(self.forecasts(rules=rules), self.weights) 220 | 221 | @lru_cache(maxsize=8) 222 | def forecast_returns(self, **kw): 223 | """ 224 | Estimated returns for individual trading rules 225 | """ 226 | f = self.forecasts(**kw) 227 | positions = self.position(forecasts=f).dropna() 228 | curves = positions.apply(lambda x: accountCurve([self], positions=x, 229 | panama_prices = self.panama_prices())) 230 | return curves.apply(lambda x: x.returns()[self.name]).transpose() 231 | 232 | @lru_cache(maxsize=8) 233 | def contract_format(self, contract): 234 | """ 235 | Convert the contract label to the broker's format 236 | """ 237 | year, month = contract_to_tuple(contract) 238 | return self.quandl_symbol + cbot_month_code(month) + str(year) 239 | 240 | @lru_cache(maxsize=8) 241 | def next_contract(self, contract, months=None, reverse=False): 242 | """ 243 | Given a contract, return the next contract in the sequence. 244 | """ 245 | if months == None: 246 | months = self.months_traded 247 | rot = 1 if reverse else -1 248 | m_idx = 0 if reverse else -1 249 | d = pd.to_datetime(str(contract), format='%Y%m') 250 | months_traded = deque(months) 251 | months_traded.rotate(rot) 252 | output_month = months_traded[months.index(d.month)] 253 | output_year = d.year + (d.month == months[m_idx]) * (-rot) 254 | return int(str(output_year) + str("%02d" % (output_month,))) 255 | 256 | @lru_cache(maxsize=8) 257 | def contracts(self, **kw_in): 258 | """ 259 | Returns the filtered contract data. 260 | 261 | Defaults: 262 | active_only = True # Only returns days with actual trading volume 263 | trade_only = True # Only return contracts we have defined as 'trade_only' in instruments.py 264 | recent_only = False # Only return contracts from the last 3 years until today. 265 | 266 | """ 267 | kw = dict(active_only=True, trade_only=True, recent_only=False) 268 | kw.update(kw_in) 269 | 270 | data = data_feed.get_instrument(self) 271 | 272 | if data is not None: 273 | year = data.index.to_frame()['contract'] // 100 274 | if self.backtest_from_contract: 275 | data = data.loc[self.backtest_from_contract:] 276 | if self.backtest_from_year: 277 | data = data[year >= self.backtest_from_year] 278 | if kw['recent_only']: 279 | data = data[year > datetime.datetime.now().year-3] 280 | if kw['trade_only']: 281 | month = data.index.to_frame()['contract'] % 100 282 | data = data[month.isin(self.trade_only)] 283 | if kw['active_only']: 284 | data = data[(data['open'] > 0) & (data['volume'] > 1)] 285 | return data 286 | 287 | ### Rolling 288 | 289 | def term_structure(self, date=None): 290 | if date is None: 291 | date = self.pp().tail(1).index[0] 292 | return self.contracts(active_only=False).xs(date, level=1) 293 | 294 | def contract_volumes(self): 295 | return self.contracts(active_only=False).groupby(level=0)['volume'].mean().plot.bar() 296 | 297 | @lru_cache(maxsize=1) 298 | def roll_progression(self): 299 | """ 300 | Lists the contracts the system wants to be in depending on the date 301 | """ 302 | return generate_roll_progression(self.roll_day, self.trade_only, 303 | self.roll_shift + self.expiration_month * 30) 304 | 305 | def expiry_date(self, contract): 306 | """ 307 | Returns the estimated expiry date for this contract. 308 | """ 309 | year, month = contract_to_tuple(contract) 310 | return datetime.date(year, month, self.expiry) 311 | 312 | def expiries(self): 313 | """ 314 | Returns a Series of expiry dates for every day in the roll progression. 315 | 316 | """ 317 | return pd.to_datetime(self.roll_progression().apply(self.expiry_date)) 318 | 319 | def time_to_expiry(self): 320 | """ 321 | Returns approximate number of days until the contract expires. 322 | Used to calculate carry when comparing a contract with the spot price underneath (e.g. a currency future and the underly exchange rate) 323 | """ 324 | return (self.expiries() - self.expiries().index).apply(lambda x: getattr(x, 'days')) 325 | 326 | def plot_contracts(self, start, finish, panama=True): 327 | """ Function to help visualise individual contract data and stitching. Not used for trading. """ 328 | if panama is False: 329 | self.roll_progression().to_frame().set_index('contract', append=True).swaplevel().\ 330 | join(self.contracts(active_only=False)).reset_index().\ 331 | pivot(index='date', columns='contract', values='close')[start:finish].\ 332 | dropna(how='all',axis=1).plot() 333 | else: 334 | df = self.panama_prices().to_frame() 335 | r = self.roll_progression().to_frame() 336 | df = df.join(r, how='inner') 337 | return df[start:finish].pivot(columns='contract', values=self.name).plot() 338 | 339 | def curve(self, **kw): 340 | return accountCurve([self], **kw) 341 | 342 | ### Bootstrap 343 | 344 | def bootstrap(self, **kw): 345 | """ 346 | Optimize rule weights using bootstrapping 347 | """ 348 | import trading.bootstrap 349 | print("Bootstrap", self.name, "starting") 350 | result = trading.bootstrap.bootstrap(self, **kw) 351 | self.bootstrapped_weights = result.mean() 352 | print(accountCurve([self], positions=self.position(forecasts=weight_forecast( 353 | self.forecasts(), result.mean())), panama_prices=self.panama_prices())) 354 | return result.mean() 355 | 356 | def cache_clear(self): 357 | self.forecasts.cache_clear() 358 | self.roll_progression.cache_clear() 359 | self.panama_prices.cache_clear() 360 | self.return_volatility.cache_clear() 361 | self.forecast_returns.cache_clear() 362 | self.contract_format.cache_clear() 363 | self.next_contract.cache_clear() 364 | -------------------------------------------------------------------------------- /core/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from config.settings import file_logger, console_logger 4 | 5 | logging.basicConfig() 6 | 7 | if file_logger['enabled']: 8 | log_dir = os.path.dirname(file_logger['file_name']) 9 | if not os.path.exists(log_dir): 10 | os.makedirs(log_dir) 11 | 12 | 13 | def get_logger(name): 14 | logger = logging.getLogger(name) 15 | logger.setLevel(logging.DEBUG) 16 | logger.handlers = [] 17 | logger.propagate = False 18 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 19 | if console_logger['enabled']: 20 | s_handler = logging.StreamHandler() 21 | s_handler.setLevel(console_logger['level']) 22 | s_handler.setFormatter(formatter) 23 | logger.addHandler(s_handler) 24 | if file_logger['enabled']: 25 | f_handler = logging.FileHandler(file_logger['file_name']) 26 | f_handler.setLevel(file_logger['level']) 27 | f_handler.setFormatter(formatter) 28 | logger.addHandler(f_handler) 29 | return logger -------------------------------------------------------------------------------- /core/spot.py: -------------------------------------------------------------------------------- 1 | from core import data_feed 2 | import config.spots 3 | 4 | 5 | class Spot(object): 6 | """ 7 | Object representing the underlying price for a future contract 8 | """ 9 | @classmethod 10 | def load_all(cls): 11 | """Load all spots in the system into a dictionary""" 12 | return {v['name']: Spot(v['name']) for v in config.spots.spots_definitions} 13 | 14 | def __init__(self, name): 15 | """Initialise the spot with defaults, taking overrides from the config/currencies.py""" 16 | self.name = name 17 | self.ib_symbol = None 18 | self.quandl_symbol = None 19 | self.price_data = ['ib', 'quandl'] 20 | self.multiplier = 1.0 21 | kwargs = config.spots.spots_all[name] 22 | for key, value in kwargs.items(): 23 | setattr(self, key, value) 24 | 25 | def __repr__(self): 26 | return self.name + ' (spot)' 27 | 28 | def get(self): 29 | """ 30 | :return: close price as pd.Series 31 | """ 32 | data = data_feed.get_spot(self) 33 | if data is None or data.empty: 34 | raise Exception("No price data for symbol: %s" % self) 35 | return data['close'] 36 | -------------------------------------------------------------------------------- /core/utility.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import logging 3 | import pandas as pd 4 | from collections import namedtuple 5 | import datetime 6 | import subprocess 7 | 8 | """ 9 | Miscellaneous utility functions 10 | """ 11 | 12 | 13 | class ConnectionException(Exception): 14 | """Exception subclass to handle errors with data providers connection""" 15 | pass 16 | 17 | 18 | def cbot_month_code(month): 19 | """Return the CBOT contract month code by the month's number""" 20 | cbot_month_codes = ['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z'] 21 | return cbot_month_codes[int(month)-1] 22 | 23 | 24 | def contract_to_tuple(contract): 25 | """Return the year and the month for the given contract label""" 26 | contract = str(contract) 27 | return (int(contract[0:4]),int(contract[4:6])) 28 | 29 | 30 | def filter_outliers(s, sigma=4): 31 | """Filter out samples with more than sigma std deviations away from the mean""" 32 | return s[~(s-s.mean()).abs() > sigma*s.std()] 33 | 34 | 35 | def dict_to_namedtuple(dictionary): 36 | return namedtuple('GenericDict', dictionary.keys())(**dictionary) 37 | 38 | 39 | def sharpe(p): 40 | """Sharpe ratio of the returns""" 41 | try: 42 | return p.mean()/p.std()*np.sqrt(252) 43 | except ZeroDivisionError: 44 | logging.error("Zero volatility, divide by zero in Sharpe ratio.") 45 | return np.inf 46 | 47 | 48 | def rolling_sharpe(p): 49 | """Mean sharpe ratio of the returns in a rolling window of the size 252""" 50 | p = np.trim_zeros(p) 51 | return p.rolling(252, min_periods=252).mean()/p.rolling(252, min_periods=252).std()*np.sqrt(252) 52 | 53 | 54 | def expanding_sharpe(p): 55 | return p.expanding(min_periods=252).mean()/p.expanding(min_periods=252).std()*np.sqrt(252) 56 | 57 | 58 | def chunk_trades(j): 59 | """Take a list of notional positions and filter so that trades are only greater 60 | than 10% of notional position""" 61 | return np.around(np.exp(np.around(np.log(np.abs(j)), decimals=1)).multiply(np.sign(j), axis=0)) 62 | 63 | 64 | def drawdown(x): 65 | maxx = x.rolling(len(x), min_periods=1).max() 66 | return x - maxx 67 | 68 | 69 | def norm_vol(df): 70 | """Arbitrarily normalize the volatility of a series. Useful for where different instruments 71 | have different price volatilities but we still want to feed them into the same regressor 72 | WARNING: Lookahead bias here""" 73 | return (df*10/bootstrap(df, lambda x: x.dropna().std())) 74 | # return (df*10/df.expanding(50).std()) 75 | 76 | 77 | def contract_from_date(expire_day, months, date): 78 | """Find the expiry date for a given contract. Currently this function is not used.""" 79 | date = pd.to_datetime(date) 80 | expiries = [datetime.datetime(date.year, k, expire_day) for k in months] 81 | expiries.extend([datetime.datetime(date.year+1, k, expire_day) for k in months]) 82 | expiries = pd.Series(expiries) 83 | 84 | try: 85 | expiry = expiries[(date > expiries).diff().fillna(False)].iloc[0] 86 | except IndexError: 87 | expiry = expiries.iloc[0] 88 | 89 | return date_to_contract(expiry) 90 | 91 | 92 | def generate_roll_progression(roll_day, months, roll_shift): 93 | """Fast version of contract_from_date, used to generate roll sequence quickly.""" 94 | rolls = [] 95 | for year in range(1960,datetime.datetime.now().year+5): 96 | rolls.extend([datetime.datetime(year, k, abs(roll_day)) for k in months]) 97 | 98 | rolls = pd.Series(rolls, index=rolls, name='rolls').shift(-1) 99 | 100 | dates = pd.date_range(start='1960-01-01', end=datetime.datetime.now()+pd.DateOffset(years=10)) 101 | a = pd.Series(0, index=dates).to_frame().join(rolls).ffill().dropna()['rolls'].to_frame() 102 | a['contract'] = a['rolls'].apply(date_to_contract) 103 | a.index = a.index.rename('date') 104 | b = a['contract'] 105 | 106 | if roll_day<1: 107 | b = b.shift(-30).dropna().apply(int) 108 | 109 | if roll_shift!=0: 110 | b = b.shift(roll_shift).dropna().apply(int) 111 | return b[:datetime.datetime.now()] 112 | 113 | 114 | def date_to_contract(date): 115 | """Return contract label for the given date""" 116 | return int(str(date.year)+str("%02d" % (date.month,))) 117 | 118 | 119 | def capital(): 120 | return 1000000 121 | 122 | 123 | def direction(action): 124 | if action == 'BUY': 125 | return 1 126 | if action == 'SELL': 127 | return -1 128 | 129 | 130 | def draw_sample(df, length=1): 131 | """Random sample from a dataframe of the given length. Used for bootstrapping""" 132 | end_of_sample_selection_space = df.index.get_loc(df[-1:].index[0])-length 133 | s = np.random.choice(range(0, end_of_sample_selection_space)) 134 | return df.iloc[s:s+length] 135 | 136 | 137 | def weight_forecast(forecasts, weights): 138 | f = (forecasts*weights).mean(axis=1) 139 | f = norm_forecast(f) 140 | return f.clip(-20,20) 141 | 142 | 143 | def norm_forecast(a): 144 | """Normalize a forecast, such that it has an absolute mean of 10. 145 | WARNING: Insample lookahead bias""" 146 | return (a*10/bootstrap(a, lambda x: x.dropna().abs().mean())).clip(-20,20) 147 | 148 | 149 | def ibcode_to_inst(ib_code): 150 | import config.instruments 151 | a = {k.ib_code: k for k in config.instruments.instrument_definitions} 152 | return a[ib_code] 153 | 154 | 155 | def notify_send(title, message): 156 | """Send a system notification using libnotify if it's installed""" 157 | try: 158 | subprocess.Popen(['notify-send', title, message]) 159 | except: 160 | pass 161 | 162 | 163 | def contract_format(symbol, expiry, format='cbot'): 164 | """Return contract label for the given symbol and expiry""" 165 | year, month = contract_to_tuple(expiry) 166 | if format == 'cbot': 167 | return symbol + cbot_month_code(month) + str(year) 168 | else: 169 | return False 170 | 171 | 172 | def generate_random_prices(length=100000): 173 | a = np.random.rand(1,length) 174 | return pd.Series(a[0]-0.5).cumsum().rename('random') 175 | 176 | 177 | def bootstrap(x, f): 178 | if len(x) == 0: 179 | return np.nan 180 | from arch.bootstrap import StationaryBootstrap 181 | bs = StationaryBootstrap(50, x) 182 | return bs.apply(f,100).mean() 183 | 184 | 185 | def sortino(x): 186 | if type(x) == pd.Series: 187 | x = x.to_frame() 188 | return np.trim_zeros(x.sum(axis=1)).mean()/np.std(losses(x))*np.sqrt(252) 189 | 190 | 191 | def losses(x): 192 | return [z for z in np.trim_zeros(x).sum(axis=1) if z<0] 193 | -------------------------------------------------------------------------------- /data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrism2671/PyTrendFollow/439232aed4d67ccf339e782b7c8b96f2dd961d43/data/__init__.py -------------------------------------------------------------------------------- /data/data_provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class DataProvider(object): 5 | 6 | def __init__(self): 7 | self.library = '' 8 | 9 | def connect(self): 10 | raise NotImplementedError 11 | 12 | def disconnect(self): 13 | raise NotImplementedError 14 | 15 | def download_instrument(self, instrument, **kwagrs): 16 | """ 17 | Download and store historical data for all contracts for the given instrument 18 | :param instrument: core.instrument.Instrument object 19 | :param kwagrs: some providers may accept additional args, like start/end date, etc. 20 | :return: 21 | """ 22 | raise NotImplementedError 23 | 24 | def download_contract(self, instrument, cont_name, **kwargs): 25 | """ 26 | Download and store historical data for the given instrument and expiry 27 | :param instrument: core.instrument.Instrument object 28 | :param cont_name: contract label string (usually defined as expiry in format YYYYMM) 29 | :param kwagrs: some providers may accept additional args, like start/end date, etc. 30 | :return: 31 | """ 32 | raise NotImplementedError 33 | 34 | def download_currency(self, currency, **kwargs): 35 | """ 36 | Download and store historical data for the currencies exchange rates 37 | :param currency: core.currency.Currency object 38 | :param kwagrs: some providers may accept additional args, like start/end date, etc. 39 | :return: 40 | """ 41 | raise NotImplementedError 42 | 43 | def download_table(self, **kwargs): 44 | """ 45 | General method to download data from provider 46 | :param kwargs: database and symbol for Quandl, contract for IB, in general can be anything depending on prov 47 | :return: 48 | """ 49 | raise NotImplementedError 50 | 51 | def download_spot(self, spot): 52 | """ 53 | Download historical data for spot prices 54 | :param spot: core.spot.Spot object 55 | :return: 56 | """ 57 | raise NotImplementedError 58 | 59 | def drop_symbol(self, **kwargs): 60 | """ 61 | Delete a symbol from the storage 62 | """ 63 | raise NotImplementedError 64 | 65 | def drop_instrument(self, instrument): 66 | """ 67 | Delete the price data for the given instrument from the storage 68 | :param instrument: core.trading.Instrument object 69 | :return: 70 | """ 71 | raise NotImplementedError 72 | 73 | def drop_currency(self, currency): 74 | """ 75 | Delete the exchange rate data for the given currency from the storage 76 | :param instrument: core.currency.Currency object 77 | :return: 78 | """ 79 | raise NotImplementedError -------------------------------------------------------------------------------- /data/db_mongo.py: -------------------------------------------------------------------------------- 1 | import pymongo 2 | from config.settings import iblog_host 3 | import pandas as pd 4 | 5 | db_name = 'iblog' 6 | order_col = 'order' 7 | exec_col = 'execution' 8 | cr_col = 'commission_report' 9 | sum_col = 'account_summary' 10 | err_col = 'error' 11 | 12 | 13 | # Writing 14 | def to_dict(obj): 15 | return dict((name, getattr(obj, name)) for name in dir(obj) 16 | if (not callable(getattr(obj, name))) and (not name.startswith('__'))) 17 | 18 | 19 | def get_db(): 20 | conn = pymongo.MongoClient(iblog_host) 21 | return conn[db_name] 22 | 23 | 24 | def insert_order(openOrder): 25 | orders = get_db()[order_col] 26 | data = openOrder.order.__dict__ 27 | if data.get('m_algoParams') is not None: 28 | data['m_algoParams'] = data['m_algoParams'][0].__dict__ 29 | data['contract'] = openOrder.contract.__dict__ 30 | orders.insert(data) 31 | 32 | 33 | def insert_execution(execution): 34 | execs = get_db()[exec_col] 35 | execs.insert(execution.__dict__) 36 | 37 | 38 | def insert_commission_report(cr): 39 | crs = get_db()[cr_col] 40 | crs.insert(cr.__dict__) 41 | 42 | 43 | def insert_account_summary(summary): 44 | sums = get_db()[sum_col] 45 | sums.insert(to_dict(summary)) 46 | 47 | 48 | def insert_error(error, acc=None): 49 | errs = get_db()[err_col] 50 | new_entry = to_dict(error) 51 | if acc is not None: 52 | new_entry['account'] = acc 53 | errs.insert(new_entry) 54 | 55 | # Reading 56 | 57 | def get_all(col_name): 58 | return pd.DataFrame(list(get_db()[col_name].find())) 59 | 60 | def get_orders(): 61 | return get_all(order_col) 62 | 63 | def get_account_summary(): 64 | return get_all(sum_col) 65 | 66 | def get_errors(): 67 | return get_all(err_col) 68 | 69 | def get_commission_report(): 70 | return get_all(cr_col) 71 | 72 | def get_executions(): 73 | return get_all(exec_col) 74 | -------------------------------------------------------------------------------- /data/ib_provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import datetime 3 | import os 4 | import threading 5 | from time import sleep 6 | from core.utility import ConnectionException 7 | 8 | import numpy as np 9 | import pandas as pd 10 | from ib.ext.Contract import Contract 11 | from ib.opt import ibConnection 12 | 13 | import config.settings 14 | from core.contract_store import Store, QuotesType, columns_mapping 15 | from data.data_provider import DataProvider 16 | from core.ib_connection import get_next_id 17 | from core.logger import get_logger 18 | 19 | logger = get_logger('ib_provider') 20 | 21 | ib_errors = { 22 | 200: 'Contract not found', 23 | 162: 'Historical market data Service error message', 24 | 354: 'Not subscribed to requested market data', 25 | 321: 'Error validating request', 26 | } 27 | 28 | 29 | class IBProvider(DataProvider): 30 | 31 | def __init__(self): 32 | super().__init__() 33 | self.library = 'ib' 34 | self.api_delay = 5 35 | self.api_timeout = 30 36 | self.clientId = get_next_id() 37 | self.port = getattr(config.settings, 'ib_port', 4001) 38 | if os.environ.get('D_HOST') is not None: 39 | self.host = os.environ.get('D_HOST') 40 | else: 41 | self.host = 'localhost' 42 | 43 | # an increasing id for handling async requests 44 | self.ticker_id = 1 45 | # temporary data structures to keep data returned by API 46 | self.historical_data_req_contract = {} 47 | self.historical_data = {} 48 | self.historical_data_result = {} 49 | self.contracts_data = {} 50 | # threading events 51 | self.historical_data_event = threading.Event() 52 | self.contract_details_event = threading.Event() 53 | self.connection = None 54 | 55 | def connect(self): 56 | self.connection = ibConnection(host=self.host, port=self.port, clientId=self.clientId) 57 | fail_count = 0 58 | fail_limit = 10 59 | # 'isConnected' attribute is added to self.connection only after the actual connection, 60 | # so we are checking both conditions here 61 | while not ((hasattr(self.connection, 'isConnected') and self.connection.isConnected()) 62 | or fail_count >= fail_limit): 63 | logger.info('Connecting with clientId %d on %s:%d' % (self.clientId, self.host, self.port)) 64 | r = self.connection.connect() 65 | if r: 66 | logger.info('Connection successful!') 67 | self._register() 68 | else: 69 | logger.warning("Couldn't establish connection, retrying in %d s." 70 | % (self.api_delay * 2)) 71 | fail_count += 1 72 | sleep(self.api_delay*2) 73 | return fail_count < fail_limit 74 | 75 | def disconnect(self): 76 | if hasattr(self.connection, 'isConnected') and self.connection.isConnected(): 77 | self.connection.close() 78 | 79 | def download_instrument(self, instrument, **kwagrs): 80 | """ 81 | :param instrument: core.Instrument object 82 | :return: bool 83 | """ 84 | ok = self.connect() 85 | if not ok: 86 | logger.warning("Download failed: couldn't connect to TWS") 87 | raise ConnectionException("Couldn't connect to TWS") 88 | contracts = self.get_contracts(instrument) 89 | logger.info('Downloading contracts for instrument: %s' % instrument.name) 90 | for c in contracts: 91 | # setting noconn=True to prevent reconnecting for each contract 92 | self.download_contract(instrument, c, noconn=True) 93 | self.disconnect() 94 | return True 95 | 96 | def get_contracts(self, instrument): 97 | contract = Contract() 98 | contract.m_symbol = instrument.ib_code 99 | contract.m_secType = 'FUT' 100 | contract.m_exchange = instrument.exchange 101 | contract.m_currency = instrument.denomination 102 | # define the multiplier if we know it 103 | if hasattr(instrument, 'ib_multiplier'): 104 | contract.m_multiplier = instrument.ib_multiplier 105 | 106 | self.connection.reqContractDetails(self.ticker_id, contract) 107 | self.contract_details_event.wait() 108 | self.contract_details_event.clear() 109 | sleep(self.api_delay) 110 | res = self.contracts_data[self.ticker_id] 111 | res.sort() 112 | self.ticker_id += 1 113 | return res 114 | 115 | def download_contract(self, instrument, cont_name, **kwargs): 116 | noconn = kwargs.get('noconn', False) 117 | contract = Contract() 118 | contract.m_symbol = instrument.ib_code 119 | contract.m_secType = 'FUT' 120 | # Contract.m_expiry is actually the contract label, not the expiration date 121 | contract.m_expiry = cont_name 122 | contract.m_exchange = instrument.exchange 123 | contract.m_currency = instrument.denomination 124 | # define the multiplier if we know it 125 | if hasattr(instrument, 'ib_multiplier'): 126 | contract.m_multiplier = instrument.ib_multiplier 127 | req_args = ["3 Y", "1 day", "TRADES", 1, 1] 128 | return self.download_table(contract, req_args, noconn=noconn) 129 | 130 | def download_currency(self, currency, **kwargs): 131 | """ 132 | :param currency: core.Currency object 133 | :return: bool 134 | """ 135 | if currency.ib_symbol == currency.ib_currency: 136 | return True 137 | contract = Contract() 138 | contract.m_symbol = currency.ib_symbol 139 | contract.m_currency = currency.ib_currency 140 | contract.m_secType = 'CASH' 141 | contract.m_exchange = currency.ib_exchange 142 | # save currency object to a custom attribute, as it is needed for data formatting 143 | setattr(contract, 'currency_object', currency) 144 | req_args = ["3 Y", "1 day", "BID_ASK", 0, 1] 145 | return self.download_table(contract, req_args) 146 | 147 | def download_spot(self, spot): 148 | """ 149 | :param spot: core.Spot object 150 | :return: bool 151 | """ 152 | contract = Contract() 153 | contract.m_secType = spot.sec_type 154 | contract.m_symbol = spot.ib_symbol 155 | contract.m_currency = spot.denomination 156 | contract.m_exchange = spot.ib_exchange 157 | # save spot object to a custom attribute, as it is needed for data formatting 158 | setattr(contract, 'spot_object', spot) 159 | req_args = ["3 Y", "1 day", "TRADES", 1, 1] 160 | return self.download_table(contract, req_args) 161 | 162 | def download_table(self, contract, req_args, **kwargs): 163 | """ 164 | General method to download data from IB. Contract and req_args are filled according to the 165 | specific data. 166 | noconn should be set to True only if the client connection is handled somewhere 167 | outside this method. 168 | """ 169 | if self.historical_data_event.is_set(): 170 | return False 171 | noconn = kwargs.get('noconn', False) 172 | if not noconn: 173 | ok = self.connect() 174 | if not ok: 175 | logger.warning("Download failed: couldn't connect to TWS") 176 | raise ConnectionException("Couldn't connect to TWS") 177 | end_datetime = datetime.datetime.now().strftime('%Y%m%d %H:%M:%S') 178 | self.historical_data_req_contract[self.ticker_id] = contract 179 | self.connection.reqHistoricalData(self.ticker_id, contract, end_datetime, *req_args) 180 | res = self.historical_data_event.wait(self.api_timeout) 181 | self.historical_data_event.clear() 182 | if res: 183 | if not noconn: 184 | self.disconnect() 185 | sleep(self.api_delay) 186 | res = self.historical_data_result[self.ticker_id] 187 | self._clear_requests_data() 188 | else: 189 | logger.warning('IB historical data request timed out') 190 | self.ticker_id += 1 191 | return res 192 | 193 | def drop_symbol(self, q_type, exchange, symbol, **kwargs): 194 | Store(self.library, q_type, exchange + '_' + symbol).delete() 195 | 196 | def drop_instrument(self, instrument): 197 | self.drop_symbol(QuotesType.futures, instrument.exchange, instrument.ib_code) 198 | 199 | def drop_currency(self, currency): 200 | self.drop_symbol(QuotesType.currency, currency.ib_exchange, 201 | currency.ib_symbol + currency.ib_currency) 202 | 203 | def _register(self): 204 | self.connection.register(self._error_handler, 'Error') 205 | self.connection.register(self._historical_data_handler, 'HistoricalData') 206 | self.connection.register(self._contract_details_handler, 'ContractDetails') 207 | self.connection.register(self._contract_details_end_handler, 'ContractDetailsEnd') 208 | 209 | def _historical_data_handler(self, msg): 210 | if int(msg.reqId) not in self.historical_data.keys(): 211 | self.historical_data[int(msg.reqId)] = [] 212 | 213 | msg_dict = dict(zip(msg.keys(), msg.values())) 214 | if msg.close > 0: 215 | # We've got an incoming history line, save it to a buffer 216 | self.historical_data[int(msg.reqId)].append(msg_dict) 217 | else: 218 | # If msg.close = 0 then we've reached the end of the history. IB sends a zero line to signify the end. 219 | data = pd.DataFrame(self.historical_data[int(msg.reqId)]) 220 | data['date'] = pd.to_datetime(data['date'], format="%Y%m%d") 221 | contract = self.historical_data_req_contract[int(msg.reqId)] 222 | 223 | dbg_message = 'Wrote data for symbol %s' % contract.m_symbol 224 | if contract.m_secType == 'FUT': # futures 225 | data = self._format_future(data, contract.m_expiry) 226 | dbg_message += ', contract %s' % contract.m_expiry 227 | store_symbol = '_'.join([contract.m_exchange, contract.m_symbol]) 228 | q_type = QuotesType.futures 229 | elif contract.m_secType == 'CASH': # currency 230 | data = self._format_currency(data, contract.currency_object) 231 | dbg_message += contract.m_currency 232 | store_symbol = '_'.join([contract.m_exchange, contract.m_symbol + contract.m_currency]) 233 | q_type = QuotesType.currency 234 | elif contract.m_secType == 'IND': # indices 235 | data = self._format_other(data, contract.spot_object) 236 | dbg_message += ' (index)' 237 | q_type = QuotesType.others 238 | store_symbol = '_'.join([contract.m_exchange, contract.m_symbol]) 239 | else: 240 | raise Exception('Attempt to download data of unsupported secType') 241 | 242 | Store(self.library, q_type, store_symbol).update(data) 243 | logger.debug(dbg_message) 244 | self.historical_data_result[int(msg.reqId)] = True 245 | self.historical_data_event.set() 246 | 247 | def _error_handler(self, msg): 248 | if (msg.id is None) or (msg.id < 1): # if msg doesn't belong to any ticker - skip it 249 | return 250 | 251 | if msg.errorCode in [200, 162, 354, 321]: 252 | c = self.historical_data_req_contract.get(int(msg.id)) 253 | if c is None: 254 | c_str = 'No contract data' 255 | else: 256 | c_str = '%s: %s/%s%s' % (c.m_secType, c.m_exchange, c.m_symbol, c.m_expiry) 257 | logger.warning('%d: %s (%s)' % (msg.errorCode, ib_errors[msg.errorCode], c_str)) 258 | # elif ... (more error codes can be added and handled here if needed) 259 | else: 260 | logger.warning(str(msg)) 261 | 262 | self.historical_data_result[int(msg.id)] = False 263 | self.historical_data_event.set() 264 | 265 | def _contract_details_handler(self, msg): 266 | if int(msg.reqId) not in self.contracts_data.keys(): 267 | self.contracts_data[int(msg.reqId)] = [] 268 | self.contracts_data[int(msg.reqId)].append(str(msg.contractDetails.m_contractMonth)) 269 | 270 | def _contract_details_end_handler(self, msg): 271 | self.contract_details_event.set() 272 | 273 | def _format_currency(self, data, currency): 274 | if data is None: 275 | return None 276 | data.reset_index(inplace=True) 277 | data.rename(columns=columns_mapping[('ib', QuotesType.currency.value)], inplace=True) 278 | data[['rate', 'high', 'low']] = currency.ib_rate(data[['rate', 'high', 'low']]) 279 | return data[['date', 'rate', 'high', 'low']].copy() 280 | 281 | def _format_future(self, data, contract): 282 | if data is None: 283 | return None 284 | data.reset_index(inplace=True) 285 | data.rename(columns=columns_mapping[('ib', QuotesType.futures.value)], inplace=True) 286 | data['contract'] = int(contract) 287 | return data[['date', 'contract', 'close', 'high', 'low', 'open', 'volume']].copy() 288 | 289 | def _format_other(self, data, spot=None): 290 | if data is None: 291 | return None 292 | data.reset_index(inplace=True) 293 | data.rename(columns=columns_mapping[('ib', QuotesType.others.value)], inplace=True) 294 | if spot is not None: 295 | data['close'] *= spot.multiplier 296 | return data[['date', 'close']].copy() 297 | 298 | def _clear_requests_data(self): 299 | # Clear temporary data returned by API to prevent dictionaries from getting too big 300 | remove_keys = np.arange(1, self.ticker_id - 100) # store only 100 last responses 301 | for k in remove_keys: 302 | self.historical_data_req_contract.pop(k, None) 303 | self.historical_data.pop(k, None) 304 | self.historical_data_result.pop(k, None) 305 | self.contracts_data.pop(k, None) 306 | 307 | def _expiry_to_contract(self, expiry, inst): 308 | """Some contracts like CL and NG expire the month before the contract name. 309 | This transforms the expiries returned from IB to the correct contract if that's the case""" 310 | expiry = int(expiry) 311 | date = datetime.date(expiry // 100, expiry % 100, inst.expiry) 312 | date = date + datetime.timedelta(weeks = (-4 * inst.expiration_month)) 313 | return int(str(date.year) + '%02d' % date.month) 314 | 315 | def _contract_to_expiry(self, cont_name, inst): 316 | cont_name = int(cont_name) 317 | date = datetime.date(cont_name // 100, cont_name % 100, inst.expiry) 318 | date = date + datetime.timedelta(weeks = (4 * inst.expiration_month)) 319 | return str(date.year) + '%02d' % date.month 320 | -------------------------------------------------------------------------------- /data/providers_factory.py: -------------------------------------------------------------------------------- 1 | 2 | def get_provider(name): 3 | if name == 'quandl': 4 | from data.quandl_provider import QuandlProvider 5 | return QuandlProvider() 6 | elif name == 'ib': 7 | from data.ib_provider import IBProvider 8 | return IBProvider() 9 | else: 10 | raise Exception('Unknown data provider name: %s' % name) 11 | -------------------------------------------------------------------------------- /data/quandl_provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from json import JSONDecodeError 3 | from time import sleep 4 | from functools import partial 5 | 6 | import pandas as pd 7 | import quandl 8 | import quandl.errors.quandl_error 9 | 10 | import core.utility 11 | from core.contract_store import Store, QuotesType, columns_mapping 12 | from data.data_provider import DataProvider 13 | from core.logger import get_logger 14 | 15 | logger = get_logger('quandl_provider') 16 | 17 | 18 | class QuandlProvider(DataProvider): 19 | 20 | def __init__(self): 21 | super().__init__() 22 | self.library = 'quandl' 23 | self.api_delay = 0 24 | self.quotes_formats = { QuotesType.futures: self._format_future, 25 | QuotesType.currency: self._format_currency, 26 | QuotesType.others: self._format_other} 27 | 28 | def connect(self): 29 | """ 30 | Quandl doesn't require connection. Api key is specified in config.settings 31 | """ 32 | pass 33 | 34 | def disconnect(self): 35 | pass 36 | 37 | def download_instrument(self, instrument, **kwagrs): 38 | recent = kwagrs.get('recent', False) 39 | if recent: 40 | c = instrument.roll_progression().tail(1).iloc[0] - 100 #-100 here rolls it back one year 41 | else: 42 | c = str(instrument.first_contract) # Download all contracts 43 | fail_count = 0 44 | # Since quandl doesn't seem to have an API function to list all available contracts, 45 | # we just loop and download them one by one until no data can be found for the next 46 | logger.info('Downloading contracts for instrument: %s' % instrument.name) 47 | while fail_count <= 12: 48 | # print(c) 49 | if self.download_contract(instrument, c): 50 | fail_count = 0 51 | else: 52 | fail_count += 1 53 | # Just try one more time in case of a network error 54 | self.download_contract(instrument, c) 55 | c = instrument.next_contract(c) 56 | logger.debug('More than 12 missing contracts in a row - ending the downloading' 57 | ' for the instrument %s' % instrument.name) 58 | return True 59 | 60 | def download_contract(self, instrument, cont_name, **kwagrs): 61 | api_symbol = core.utility.contract_format(instrument.quandl_symbol, cont_name) 62 | return self.download_table(QuotesType.futures, instrument.quandl_database, 63 | symbol=api_symbol, db_symbol=instrument.quandl_symbol, 64 | instrument=instrument, contract=cont_name) 65 | 66 | def download_currency(self, currency, **kwargs): 67 | if currency.quandl_symbol[0:3] == currency.quandl_symbol[3:6]: 68 | return True 69 | return self.download_table(QuotesType.currency, currency.quandl_database, 70 | currency.quandl_symbol, currency=currency) 71 | 72 | def download_spot(self, spot): 73 | return self.download_table(QuotesType.others, spot.quandl_database, 74 | spot.quandl_symbol, col=spot.quandl_column) 75 | 76 | def download_table(self, q_type, database, symbol, db_symbol=None, **kwargs): 77 | # symbol name for the DB storage may be different from what we send to quandl API (e.g. for futures) 78 | if db_symbol is None: 79 | db_symbol = symbol 80 | 81 | # specify the format function for the table (depends on quotes type) 82 | formnat_fn = self.quotes_formats[q_type] 83 | # for some spot prices the data column is specified explicitly in instruments.py 84 | # in such cases we pass this column to a format function and save it to database as 'close' 85 | if 'col' in kwargs.keys(): 86 | formnat_fn = partial(formnat_fn, column=kwargs.get('col')) 87 | # pass currency object to scale the rate values where needed 88 | if 'currency' in kwargs.keys(): 89 | formnat_fn = partial(formnat_fn, currency=kwargs.get('currency')) 90 | # pass spot object to apply multiplier on data format 91 | if 'spot' in kwargs.keys(): 92 | formnat_fn = partial(formnat_fn, spot=kwargs.get('spot')) 93 | # pass instrument and contract to format futures data 94 | if 'instrument' in kwargs.keys(): 95 | formnat_fn = partial(formnat_fn, instrument=kwargs.get('instrument'), 96 | contract=kwargs.get('contract')) 97 | 98 | try: 99 | data = quandl.get(database + '/' + symbol) 100 | Store(self.library, q_type, database + '_' + db_symbol).update(formnat_fn(data=data)) 101 | logger.debug('Wrote data for %s/%s' % (database, symbol)) 102 | sleep(self.api_delay) 103 | except JSONDecodeError: 104 | logger.warning("JSONDecodeError") 105 | return False 106 | except quandl.errors.quandl_error.NotFoundError: 107 | logger.debug('Symbol %s not found on database %s' % (symbol, database)) 108 | return False 109 | except quandl.errors.quandl_error.LimitExceededError: 110 | logger.warning('Quandl API limit exceeded!') 111 | return False 112 | except Exception as e: 113 | logger.warning('Unexpected error occured: %s' % e) 114 | return False 115 | return True 116 | 117 | def drop_symbol(self, q_type, database, symbol, **kwargs): 118 | Store(self.library, q_type, database + '_' + symbol).delete() 119 | 120 | def drop_instrument(self, instrument): 121 | self.drop_symbol(QuotesType.futures, instrument.quandl_database, instrument.quandl_symbol) 122 | 123 | def drop_currency(self, currency): 124 | self.drop_symbol(QuotesType.currency, currency.quandl_database, currency.quandl_symbol) 125 | 126 | def _format_future(self, data, instrument, contract): 127 | if data is None: 128 | return None 129 | if hasattr(instrument, 'quandl_rename_columns'): 130 | data.rename(columns=instrument.quandl_rename_columns, inplace=True) 131 | # Very important - Some futures, such as MXP or Cotton have a different factor on Quandl vs reality 132 | if instrument.quandl_data_factor != 1: 133 | data[['Settle']] = \ 134 | data[['Settle']] / instrument.quandl_data_factor 135 | # Sometimes Quandl doesn't return 'Date' for the index, so let's make sure it's set 136 | data.index = data.index.rename('Date') 137 | 138 | data.reset_index(inplace=True) 139 | data.rename(columns=columns_mapping[('quandl', QuotesType.futures.value)], inplace=True) 140 | data['contract'] = pd.Series(int(contract), index=data.index) 141 | # .to_datetime() doesn't work here if source datatype is 'M8[ns]' 142 | # data_out['date'] = data_out['date'].astype('datetime64[ns]') 143 | return data[['date', 'contract', 'close', 'high', 'low', 'open', 'volume']].copy() 144 | 145 | def _format_currency(self, data, currency): 146 | if data is None: 147 | return None 148 | data.reset_index(inplace=True) 149 | data.rename(columns=columns_mapping[('quandl', QuotesType.currency.value)], inplace=True) 150 | data[['rate', 'high', 'low']] = currency.quandl_rate(data[['rate', 'high', 'low']]) 151 | return data[['date', 'rate', 'high', 'low']].copy() 152 | 153 | def _format_other(self, column, data, spot=None): 154 | if data is None: 155 | return None 156 | data.reset_index(inplace=True) 157 | mapping = columns_mapping[('quandl', QuotesType.others.value)] 158 | mapping[column] = 'close' 159 | data.rename(columns=mapping, inplace=True) 160 | if spot is not None: 161 | data['close'] *= spot.multiplier 162 | return data[['date', 'close']].copy() 163 | 164 | def _format_btc(self, data_in): 165 | raise NotImplementedError 166 | -------------------------------------------------------------------------------- /docs/Getting started with Interactive Brokers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Getting started with Interactive Brokers\n", 8 | "\n", 9 | "Our project connects to Interactive Brokers to perform trades.\n", 10 | "\n", 11 | "Interactive Brokers has two ways of accessing the API:\n", 12 | "* **TWS** - this is the main desktop client. It's useful for seeing what's going on and testing things. Shuts down automatically after 24 hours.\n", 13 | "* **Gateway** - this is a lightweight API client. It's still java based and requires a GUI login (there is no 'headless' API), but it's suitable for running a live system.\n", 14 | "\n", 15 | "There are also three levels of login:\n", 16 | "* Demo\n", 17 | "* Simulated/paper trading\n", 18 | "* Live trading\n", 19 | "\n", 20 | "These environments can quite different in behaviour (in terms of what instruments are available, and especially what historical/live data is available).\n", 21 | "\n", 22 | "## Using the demo Interactive Brokers mode to test API connectivity\n", 23 | "\n", 24 | "Download and install the latest TWS onto your computer.\n", 25 | "\n", 26 | "Open it and log in with:\n", 27 | "* Username: edemo\n", 28 | "* Password: demouser\n", 29 | "\n", 30 | "Go into the settings and configure the API:\n", 31 | "* Enable the API on port 4003\n", 32 | "* Disable Read-only API\n", 33 | "\n", 34 | "Our system has a module at core.ibstate which handles the connection to IB. IB's API is realtime & event driven.\n", 35 | "\n", 36 | "The following code should connect to IB:" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": { 43 | "ExecuteTime": { 44 | "end_time": "2017-06-16T12:39:33.826949+01:00", 45 | "start_time": "2017-06-16T12:39:23.175952" 46 | }, 47 | "collapsed": false 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "%matplotlib inline\n", 52 | "%load_ext autoreload\n", 53 | "%autoreload 2\n", 54 | "import os\n", 55 | "os.chdir('..')\n", 56 | "import pandas as pd\n", 57 | "import numpy as np\n", 58 | "import trading.start\n", 59 | "import trading.portfolio as portfolio\n", 60 | "import config.settings\n", 61 | "from time import sleep\n", 62 | "from core.utility import *\n", 63 | "from trading.accountcurve import *\n", 64 | "import data.db_mongo as db\n", 65 | "import config.portfolios\n", 66 | "from pylab import rcParams\n", 67 | "rcParams['figure.figsize'] = 15, 10\n", 68 | "p = portfolio.Portfolio(instruments=config.portfolios.p_trade)\n", 69 | "i = p.instruments\n", 70 | "from trading.ibstate import *\n", 71 | "\n", 72 | "ib = IBstate()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "IB has recently introduced a Python API, but we don't use it yet (we currently use IbPy, which is a Python module that maps to the Java API https://github.com/blampe/IbPy). \n", 80 | "\n", 81 | "The official API docs are here:\n", 82 | "http://interactivebrokers.github.io/tws-api/\n", 83 | "\n", 84 | "The API consists of two main components:\n", 85 | "* Client\n", 86 | "* Wrapper\n", 87 | "\n", 88 | "We send instructions to the API with the Client, and receive responses back via Wrapper. We have to subscribe to various event streams to receive responses from the API. IBstate tries to collect all these realtime responses to maintain a 'stateful' respresentation of what is going on. For example, our account balance ('net liquidiation') will change throughout the day. IBstate listens for these updates and remembers them, so when we call:" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": { 95 | "collapsed": true 96 | }, 97 | "outputs": [], 98 | "source": [ 99 | "ib.net()" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "it returns a result from our local buffer.\n", 107 | "\n", 108 | "IBstate can also collect responses and join them together (for example, portfolios are returned line by line, IBstate joins them together into a Pandas dataframe, so we can work it more easily).\n", 109 | "\n", 110 | "IBState also allows us to trade. The magic command is:" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": { 117 | "collapsed": true 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "ib.sync_portfolio(p, trade = True)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "When you run this, you should start to see the trades go into TWS. It pauses for 60 seconds between each trade, so this will take several minutes to run.\n", 129 | "\n", 130 | "If you refresh the following command, you should see a mirror image of what you see in the TWS interface." 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": { 137 | "collapsed": true 138 | }, 139 | "outputs": [], 140 | "source": [ 141 | "ib.portfolio" 142 | ] 143 | } 144 | ], 145 | "metadata": { 146 | "anaconda-cloud": {}, 147 | "kernelspec": { 148 | "display_name": "Python [conda root]", 149 | "language": "python", 150 | "name": "conda-root-py" 151 | }, 152 | "language_info": { 153 | "codemirror_mode": { 154 | "name": "ipython", 155 | "version": 3 156 | }, 157 | "file_extension": ".py", 158 | "mimetype": "text/x-python", 159 | "name": "python", 160 | "nbconvert_exporter": "python", 161 | "pygments_lexer": "ipython3", 162 | "version": "3.5.2" 163 | } 164 | }, 165 | "nbformat": 4, 166 | "nbformat_minor": 1 167 | } 168 | -------------------------------------------------------------------------------- /download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from core.instrument import Instrument 3 | from core.currency import Currency 4 | from core.spot import Spot 5 | from data.providers_factory import get_provider 6 | from core.contract_store import QuotesType 7 | import config.settings 8 | import config.portfolios 9 | import sys, traceback 10 | import core.logger 11 | logger = core.logger.get_logger('download') 12 | import pyprind 13 | from functools import partial 14 | from core.utility import ConnectionException 15 | 16 | 17 | """ 18 | Script usage: python download.py [--recent] [--concurrent] 19 | should be either 'ib' (for Interactive Brokers) or 'quandl' 20 | --recent flag: if provided, will only download historical contracts for the last year 21 | (only applies to quandl) 22 | --concurrent flag: if provided, will download data in multiple CPU threads (only applies to quandl, 23 | multi-threaded downloads is a paid feature, so make sure you have a subscription) 24 | """ 25 | 26 | # download a single instrument's contracts 27 | def dl_inst(i, prov_name, recent): 28 | dp = get_provider(prov_name) 29 | try: 30 | if prov_name in i.contract_data: 31 | dp.download_instrument(i, recent=recent) 32 | except ConnectionException as e: 33 | raise e 34 | except Exception as e: 35 | logger.warning(e) 36 | logger.warning('Contract download error, ignoring') 37 | 38 | 39 | # download currency exchange rate 40 | def dl_cur(c, prov_name): 41 | dp = get_provider(prov_name) 42 | try: 43 | if prov_name in c.currency_data: 44 | dp.download_currency(c) 45 | except ConnectionException as e: 46 | raise e 47 | except Exception as e: 48 | logger.warning(e) 49 | logger.warning('Currency download error, ignoring') 50 | 51 | 52 | # download spot prices 53 | def dl_spot(s, prov_name): 54 | dp = get_provider(prov_name) 55 | try: 56 | if prov_name in s.price_data: 57 | dp.download_spot(s) 58 | except ConnectionException as e: 59 | raise(e) 60 | except Exception as e: 61 | logger.warning(e) 62 | logger.warning('Spot price download error, ignoring') 63 | 64 | 65 | def download_all(prov_name, qtype, recent, concurrent): 66 | if qtype == QuotesType.futures: 67 | instruments = Instrument.load(config.portfolios.p_all) 68 | dl_fn = partial(dl_inst, prov_name=prov_name, recent=recent) 69 | attr = 'name' 70 | title_name = 'contracts' 71 | elif qtype == QuotesType.currency: 72 | instruments = Currency.load_all() 73 | dl_fn = partial(dl_cur, prov_name=prov_name) 74 | attr = 'code' 75 | title_name = 'currencies' 76 | elif qtype == QuotesType.others: 77 | instruments = Spot.load_all() 78 | dl_fn = partial(dl_spot, prov_name=prov_name) 79 | attr = 'name' 80 | title_name = 'spot prices' 81 | else: 82 | raise Exception('Unknown quotes type') 83 | 84 | title = 'Downloading %s history for %s' % (title_name, prov_name) 85 | if concurrent: title += ' (parallel)' 86 | bar = pyprind.ProgBar(len(instruments.values()), title=title) 87 | 88 | if concurrent: 89 | import concurrent.futures 90 | with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: 91 | download_futures = {executor.submit(lambda v: dl_fn(v), x): 92 | getattr(x, attr) for x in instruments.values()} 93 | for future in concurrent.futures.as_completed(download_futures): 94 | bar.update(item_id=download_futures[future]) 95 | else: 96 | for i in instruments.values(): 97 | dl_fn(i) 98 | 99 | 100 | if __name__ == "__main__": 101 | try: 102 | if len(sys.argv) < 2 or sys.argv[1] not in ['quandl', 'ib']: 103 | print('Usage: download.py [quandl|ib] [--recent] [--concurrent]') 104 | sys.exit(1) 105 | provider = sys.argv[1] 106 | recent = '--recent' in sys.argv 107 | concurrent = '--concurrent' in sys.argv 108 | download_all(provider, QuotesType.futures, recent, concurrent) 109 | download_all(provider, QuotesType.currency, recent, concurrent) 110 | download_all(provider, QuotesType.others, recent, concurrent) 111 | except KeyboardInterrupt: 112 | print("Shutdown requested...exiting") 113 | except ConnectionException as e: 114 | print('Connection error:', e) 115 | print('exiting...') 116 | except Exception: 117 | traceback.print_exc(file=sys.stdout) 118 | sys.exit(0) 119 | -------------------------------------------------------------------------------- /download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./download.py quandl --concurrent 3 | ./download.py ib 4 | ./validate.py 5 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | -------------------------------------------------------------------------------- /pytest.sh: -------------------------------------------------------------------------------- 1 | python -m pytest 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cython 2 | tables 3 | numpy 4 | scipy 5 | matplotlib 6 | pandas 7 | quandl 8 | seaborn 9 | pymongo 10 | pyprind 11 | IbPy2 12 | schedule 13 | multiprocessing_on_dill 14 | bravado 15 | pytest 16 | arch 17 | -------------------------------------------------------------------------------- /scheduler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from trading.ibstate import IBstate 3 | import trading.portfolio 4 | import config.portfolios 5 | import config.strategy 6 | from trading.account import Account 7 | from core.utility import notify_send 8 | from time import sleep 9 | import schedule 10 | import sys 11 | import traceback 12 | from core.logger import get_logger 13 | from functools import partial 14 | logger = get_logger('scheduler') 15 | p = trading.portfolio.Portfolio(instruments=config.portfolios.p_trade) 16 | i = p.instruments 17 | 18 | 19 | def init_ib(): 20 | ib = IBstate() 21 | ib.connect() 22 | print("connecting") 23 | while not ib.is_ready(): 24 | print("Not ready") 25 | sleep(5) 26 | if len(ib.open_orders()>0): 27 | print('Open orders:', ib.open_orders()) 28 | print_net(ib.accounts) 29 | jobs = [] 30 | jobs.append(schedule.every(6).hours.do(partial(print_net, ib.accounts))) 31 | jobs.append(schedule.every(15).seconds.do(ib.connect)) 32 | jobs.append(schedule.every(15).seconds.do(ib.update_open_orders)) 33 | return ib, jobs 34 | 35 | 36 | def sync_trades(): 37 | ib, ib_jobs = init_ib() 38 | print(1) 39 | accs = [a for a in ib.accounts.values() if a.name.startswith(('U', 'DU'))] 40 | notify('Running Sync Trades for {0} accounts: {1}'.format(len(accs), [a.name for a in accs])) 41 | trade = False if "--dryrun" in sys.argv else True 42 | p.cache_clear() 43 | # print("Starting validation") 44 | # validate = p.validate()[['carry_forecast', 'currency_age', 'panama_age', 'price_age', 'weighted_forecast']] 45 | # logger.info('\n' + str(validate)) 46 | # do for all accounts 47 | for a in accs: 48 | print_net(a) 49 | notify('Running Sync Trades for account %s' % a.name) 50 | try: 51 | ib.sync_portfolio(p, acc=a, trade=trade) 52 | except AssertionError: 53 | notify('No sync for account %s, validation failed' % a.name, level='warning') 54 | continue 55 | notify('Portfolio synced') 56 | print_net(a) 57 | # cancel ib-specific scheduler jobs 58 | [schedule.cancel_job(j) for j in ib_jobs] 59 | 60 | 61 | def print_net(accs): 62 | """ 63 | :param accs: Account object, or dict, or None 64 | """ 65 | if isinstance(accs, Account): 66 | accs = {accs.name: accs} 67 | for a in accs.values(): 68 | notify('Net liquidation for account %s: %.2f %s ' % (a.name, a.net, a.base_currency)) 69 | 70 | 71 | def set_schedule(time): 72 | schedule.every().monday.at(time).do(sync_trades) 73 | schedule.every().tuesday.at(time).do(sync_trades) 74 | schedule.every().wednesday.at(time).do(sync_trades) 75 | schedule.every().thursday.at(time).do(sync_trades) 76 | schedule.every().friday.at(time).do(sync_trades) 77 | 78 | 79 | def main(): 80 | # width = pd.util.terminal.get_terminal_size() # find the width of the user's terminal window 81 | # rows, columns = os.popen('stty size', 'r').read().split() 82 | # pd.set_option('display.width', width[0]) # set that as the max width in Pandas 83 | # pd.set_option('display.width', int(columns)) # set that as the max width in Pandas 84 | 85 | set_schedule(config.strategy.portfolio_sync_time) 86 | 87 | if "--now" in sys.argv: 88 | sync_trades() 89 | 90 | if "--quit" not in sys.argv: 91 | while True: 92 | schedule.run_pending() 93 | sleep(1) 94 | 95 | 96 | def notify(msg, level='info'): 97 | notify_send(level, msg) 98 | try: 99 | getattr(logger, level)(msg) 100 | except: 101 | logger.info(msg) 102 | 103 | 104 | if __name__ == "__main__": 105 | try: 106 | main() 107 | except KeyboardInterrupt: 108 | print("Shutdown requested...exiting") 109 | except Exception as e: 110 | traceback.print_exc(file=sys.stdout) 111 | logger.exception(e, exc_info=True) 112 | sys.exit(0) 113 | -------------------------------------------------------------------------------- /scripts/jupyter_extensions.sh: -------------------------------------------------------------------------------- 1 | conda install -c conda-forge jupyter_contrib_nbextensions 2 | jupyter contrib nbextension install --user 3 | jupyter serverextension enable jupyter_nbextensions_configurator 4 | jupyter nbextension enable ExecuteTime 5 | 6 | conda install ipyparallel 7 | jupyter serverextension enable --py ipyparallel 8 | jupyter nbextension install ipyparallel --user --py 9 | jupyter nbextension enable ipyparallel --user --py -------------------------------------------------------------------------------- /tests/test_portfolio.py: -------------------------------------------------------------------------------- 1 | import trading.portfolio as portfolio 2 | import trading.accountcurve as accountcurve 3 | import config.portfolios 4 | import config.settings 5 | 6 | 7 | """ 8 | Set of basic tests for portfolio. 9 | Should pass only if historical data are downloaded and up to date. 10 | """ 11 | 12 | 13 | def test_curve(): 14 | p = portfolio.Portfolio(instruments=config.portfolios.p_trade) 15 | a = p.curve() 16 | assert type(a) is accountcurve.accountCurve 17 | assert a.sharpe() > 0 18 | 19 | 20 | def test_validate(): 21 | p = portfolio.Portfolio(instruments=config.portfolios.p_trade) 22 | v = p.validate() 23 | assert v['is_valid'].any() 24 | 25 | 26 | def test_frontier(): 27 | p = portfolio.Portfolio(instruments=config.portfolios.p_trade) 28 | f = p.frontier() 29 | assert not f.empty 30 | -------------------------------------------------------------------------------- /trade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./download.sh 3 | ./scheduler.py --now --quit 4 | -------------------------------------------------------------------------------- /trading/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrism2671/PyTrendFollow/439232aed4d67ccf339e782b7c8b96f2dd961d43/trading/__init__.py -------------------------------------------------------------------------------- /trading/account.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import config.settings 3 | 4 | 5 | class Account(object): 6 | """ 7 | Represents an account in IB multi-account system 8 | """ 9 | def __init__(self, name): 10 | self.name = name 11 | self.base_currency = None 12 | self.summary = None 13 | self.net = 0 14 | self.portfolio = pd.DataFrame() 15 | 16 | def is_valid(self): 17 | return (self.net > 0) and (self.base_currency == config.settings.base_currency) -------------------------------------------------------------------------------- /trading/accountcurve.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import matplotlib.pyplot as plt 4 | import pprint 5 | import config.settings 6 | import config.strategy 7 | from core.utility import chunk_trades, sharpe, drawdown 8 | from multiprocessing_on_dill import Pool #, Process, Manager 9 | from contextlib import closing 10 | 11 | 12 | class accountCurve(): 13 | """ 14 | Account curve object for Portfolio and Instrument. 15 | 16 | Calculates the positions we want to be in, based on the volatility target. 17 | """ 18 | def __init__(self, portfolio, capital=500000, positions=None, panama_prices=None, nofx=False, portfolio_weights = 1, **kw): 19 | self.portfolio = portfolio 20 | self.nofx = nofx 21 | self.weights = portfolio_weights 22 | self.multiproc = kw.get('multiproc', True) 23 | # If working on one instrument, put it in a list 24 | if not isinstance(portfolio, list): 25 | self.portfolio = [self.portfolio] 26 | 27 | if isinstance(positions, pd.Series): 28 | positions = positions.rename(self.portfolio[0].name) 29 | 30 | self.capital = capital 31 | self.panama = panama_prices 32 | 33 | if positions is None: 34 | self.positions = self.instrument_positions() 35 | self.positions = self.positions.multiply(self.weights) 36 | else: 37 | self.positions = pd.DataFrame(positions) 38 | 39 | # Reduce all our positions so that they fit inside our target volatility when combined. 40 | self.positions = self.positions.multiply(self.vol_norm(),axis=0) 41 | 42 | # If we run out of data (for example, if the data feed is stopped), hold position for 5 trading days and then close. 43 | # chunk_trades() is a function that is designed to reduce the amount of trading (and hence cost) 44 | 45 | self.positions = chunk_trades(self.positions).ffill(limit=5).fillna(0) 46 | 47 | def __repr__(self): 48 | """ 49 | Returns a formatted list of statistics about the account curve. 50 | """ 51 | return pprint.pformat(self.stats_list()) 52 | 53 | def inst_calc(self): 54 | """Calculate all the things we need on all the instruments and cache it.""" 55 | try: 56 | return self.memo_inst_calc 57 | except: 58 | if len(self.portfolio)>1 and self.multiproc: 59 | with closing(Pool()) as pool: 60 | self.memo_inst_calc = dict(pool.map(lambda x: (x.name, x.calculate()), self.portfolio)) 61 | else: 62 | self.memo_inst_calc = dict(map(lambda x: (x.name, x.calculate()), self.portfolio)) 63 | return self.memo_inst_calc 64 | 65 | def instrument_positions(self): 66 | """Position returned by the instrument objects, not the final position in the portfolio""" 67 | try: 68 | return self.memo_instrument_positions 69 | except: 70 | self.memo_instrument_positions = pd.DataFrame({k: v['position'] for k, v in self.inst_calc().items()}) 71 | return self.memo_instrument_positions 72 | 73 | def rates(self): 74 | """ 75 | Returns a Series or DataFrame of exchange rates. 76 | """ 77 | if self.nofx==True: 78 | return 1 79 | try: 80 | return self.memo_rates 81 | except: 82 | self.memo_rates = pd.DataFrame({k: v['rate'] for k, v in self.inst_calc().items()}) 83 | return self.memo_rates 84 | 85 | def stats_list(self): 86 | stats_list = ["sharpe", 87 | "gross_sharpe", 88 | "annual_vol", 89 | "sortino", 90 | "cap", 91 | "avg_drawdown", 92 | "worst_drawdown", 93 | "time_in_drawdown", 94 | "calmar", 95 | "avg_return_to_drawdown"] 96 | return {k: getattr(self, k)() for k in stats_list} 97 | 98 | def returns(self): 99 | """ 100 | Returns a Series/Frame of net returns after commissions, spreads and estimated slippage. 101 | """ 102 | return self.position_returns() + self.transaction_returns() + self.commissions() + self.spreads() 103 | 104 | def position_returns(self): 105 | """The returns from holding the portfolio we had yesterday""" 106 | # We shift back 2, as self.positions is the frontier - tomorrow's ideal position. 107 | return (self.positions.shift(2).multiply((self.panama_prices()).diff(), axis=0).fillna(0) * self.point_values()) * self.rates() 108 | 109 | def transaction_returns(self): 110 | """Estimated returns from transactions including slippage. Uses the average settlement price of the last two days""" 111 | # self.positions.diff().shift(1) = today's trades 112 | slippage_multiplier = .5 113 | return (self.positions.diff().shift(1).multiply((self.panama_prices()).diff()*slippage_multiplier, axis=0).fillna(0) * self.point_values()) * self.rates() 114 | 115 | def commissions(self): 116 | commissions = pd.Series({v.name: v.commission for v in self.portfolio}) 117 | return (self.positions.diff().shift(1).multiply(commissions)).fillna(0).abs()*-1 118 | 119 | def spreads(self): 120 | spreads = pd.Series({v.name: v.spread for v in self.portfolio}) 121 | return (self.positions.diff().shift(1).multiply(spreads * self.point_values() * self.rates())).fillna(0).abs()*-1 122 | 123 | def vol_norm(self): 124 | return (config.strategy.daily_volatility_target * self.capital / \ 125 | (self.returns().sum(axis=1).shift(2).ewm(span=50).std())).clip(0,1.5) 126 | 127 | def panama_prices(self): 128 | if self.panama is not None: 129 | return pd.DataFrame(self.panama) 130 | else: 131 | try: 132 | return self.memo_panama_prices 133 | except: 134 | self.memo_panama_prices = pd.DataFrame({k: v['panama_prices'] for k, v in self.inst_calc().items()}) 135 | return self.memo_panama_prices 136 | 137 | def point_values(self): 138 | return pd.Series({v.name: v.point_value for v in self.portfolio}) 139 | 140 | def gross_sharpe(self): 141 | return sharpe(np.trim_zeros((self.position_returns() - self.transaction_returns()).sum(axis=1))) 142 | 143 | def sharpe(self): 144 | return sharpe(np.trim_zeros(self.returns().sum(axis=1))) 145 | 146 | def losses(self): 147 | return [z for z in np.trim_zeros(self.returns()).sum(axis=1) if z<0] 148 | 149 | def sortino(self): 150 | return np.trim_zeros(self.returns().sum(axis=1)).mean()/np.std(self.losses())*np.sqrt(252) 151 | 152 | def annual_vol(self): 153 | return "{0:,.4f}".format(np.trim_zeros(self.returns()).sum(axis=1).std() * np.sqrt(252)/self.capital) 154 | 155 | def plot(self): 156 | fig, axes = plt.subplots(nrows=1, ncols=1) 157 | # self.returns.cumsum().plot(ax=axes[0]) 158 | ar = self.annual_returns() 159 | ar.plot.bar(ax=axes, figsize=(12,1.5)) 160 | axes.set_xlabel("") 161 | axes.set_xticklabels([dt.strftime('%Y') for dt in ar.index.to_pydatetime()]) 162 | 163 | def annual_returns(self): 164 | return np.trim_zeros(self.returns().sum(axis=1).resample(rule='A').sum()/self.capital * 100) 165 | 166 | def annual_sharpes(self): 167 | return self.returns().sum(axis=1).resample(rule='A').sum()/(self.returns().sum(axis=1).resample(rule='A').std() * np.sqrt(252)) 168 | 169 | def drawdown(self): 170 | return drawdown(self.returns().sum(axis=1).cumsum()) 171 | 172 | def avg_drawdown(self): 173 | dd = self.drawdown() 174 | # return "{0:,.0f}".format(np.nanmean(dd.values)) 175 | return np.nanmean(dd.values)/self.capital 176 | 177 | def worst_drawdown(self): 178 | dd = self.drawdown() 179 | # return "{0:,.0f}".format(np.nanmin(dd.values)) 180 | return np.nanmin(dd.values)/self.capital 181 | def cap(self): 182 | return self.capital 183 | 184 | def time_in_drawdown(self): 185 | dd = self.drawdown() 186 | dd = [z for z in dd.values if not np.isnan(z)] 187 | in_dd = float(len([z for z in dd if z < 0])) 188 | return "{0:,.4f}".format(in_dd / float(len(dd))) 189 | 190 | def instrument_count(self): 191 | return np.maximum.accumulate((~np.isnan(self.panama_prices())).sum(axis=1)).plot() 192 | 193 | def underwater(self): 194 | r = self.returns().sum(axis=1) 195 | u = (r.cumsum() - r.cumsum().cummax())/self.capital 196 | return np.trim_zeros(u).plot() 197 | 198 | def cumcapital(self): 199 | return np.trim_zeros((self.returns().sum(axis=1)/self.capital)+1).cumprod() 200 | 201 | def calmar(self): 202 | return self.annual_returns().mean() * 0.01 / -self.worst_drawdown() 203 | 204 | def avg_return_to_drawdown(self): 205 | return self.annual_returns().mean() * 0.01 / -self.avg_drawdown() 206 | -------------------------------------------------------------------------------- /trading/bootstrap.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from multiprocessing import cpu_count 4 | from functools import partial 5 | from scipy.optimize import minimize 6 | from trading.accountcurve import accountCurve 7 | from core.utility import draw_sample, weight_forecast 8 | from multiprocessing_on_dill import Pool 9 | from contextlib import closing 10 | 11 | 12 | """ Bootstrap.py - find the best weights for forecasts on a single instrument. """ 13 | 14 | def optimize_weights(instrument, sample): 15 | """Optimize the weights on a particular sample""" 16 | guess = [1.0] * sample.shape[1] 17 | bounds = [(0.0,5.0)] * sample.shape[1] 18 | def function(w, instrument, sample): 19 | """This is the function that is minimized iteratively using scipy.optimize.minimize to find the best weights (w)""" 20 | wf = weight_forecast(sample, w) 21 | # We introduce a capital term, as certain currencies like HKD are very 'numerate', which means we need millions of HKD to get a 22 | # significant position 23 | position = instrument.position(forecasts = wf, nofx=True, capital=10E7).rename(instrument.name).to_frame().dropna() 24 | # position = instrument.position(forecasts = wf, nofx=True).rename(instrument.name).to_frame().dropna() 25 | l = accountCurve([instrument], positions = position, panama_prices=instrument.panama_prices().dropna(), nofx=True) 26 | s = l.sortino() 27 | try: 28 | assert np.isnan(s) == False 29 | except: 30 | print(sample, position) 31 | raise 32 | return -s 33 | 34 | result = minimize(function, guess, (instrument, sample),\ 35 | method = 'SLSQP',\ 36 | bounds = bounds,\ 37 | tol = 0.01,\ 38 | constraints = {'type': 'eq', 'fun': lambda x: sample.shape[1] - sum(x)},\ 39 | options = {'eps': .1}, 40 | ) 41 | return result.x 42 | 43 | def mp_optimize_weights(samples, instrument, **kw): 44 | """Calls the Optimize function, on different CPU cores""" 45 | with closing(Pool()) as pool: 46 | return pool.map(partial(optimize_weights, instrument), samples) 47 | 48 | def bootstrap(instrument, n=(cpu_count() * 4), **kw): 49 | """Use bootstrapping to optimize the weights for forecasts on a particular instrument. Sets up the samples and gets it going.""" 50 | forecasts = instrument.forecasts(**kw).dropna() 51 | weights_buffer = pd.DataFrame() 52 | sample_length = 200 53 | t = 0 54 | while t < 1: 55 | samples = [draw_sample(forecasts, length=sample_length) for k in range(0,n)] 56 | # This is a one hit for the whole price series 57 | # samples = [slice(prices.index[0:][0],prices.index[-1:][0])] 58 | weights = pd.DataFrame(list(mp_optimize_weights(samples, instrument, **kw))) 59 | weights_buffer = weights_buffer.append(weights).reset_index(drop=True) 60 | n=cpu_count() 61 | t = (weights_buffer.expanding(min_periods=21).mean().pct_change().abs()<0.05).product(axis=1).sum() 62 | weights_buffer.columns = forecasts.columns 63 | return weights_buffer 64 | -------------------------------------------------------------------------------- /trading/bootstrap_portfolio.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from concurrent.futures import ProcessPoolExecutor 3 | from functools import partial 4 | from scipy.optimize import minimize 5 | from core.utility import draw_sample, sortino 6 | 7 | """ Find the best weights of instruments in a portfolio """ 8 | 9 | def bootstrap(portfolio, n=1500, costs=True, **kw): 10 | data = portfolio.curve(portfolio_weights=1,capital=10E7).returns() 11 | sample_length = 300 12 | samples = [draw_sample(data, length=sample_length).index for k in range(0,n)] 13 | weights = list(mp_optimize_weights(samples, data, **kw)) 14 | weights_buffer = pd.DataFrame([x for x in weights if type(x) == pd.Series]) 15 | print(len(weights_buffer),"samples") 16 | weights_buffer.mean().plot.bar() 17 | return weights_buffer 18 | 19 | def mp_optimize_weights(samples, data, **kw): 20 | return ProcessPoolExecutor().map(partial(optimize_weights, data), samples) 21 | 22 | def optimize_weights(data, sample): 23 | data = data.loc[sample].dropna(axis=1, how='all') 24 | if data.shape[1] < 2: 25 | return 26 | guess = [1] * data.shape[1] 27 | bounds = [(0.0, 10.0)] * data.shape[1] 28 | 29 | def function(w, data, sample): 30 | wr = (w*data.loc[sample]).sum(axis=1) 31 | return -sortino(wr) 32 | 33 | result = minimize(function, guess, (data, sample),\ 34 | method = 'SLSQP',\ 35 | bounds = bounds,\ 36 | tol = 0.0001,\ 37 | constraints = {'type': 'eq', 'fun': lambda x: data.shape[1] - sum(x)},\ 38 | options = {'eps': 1e-1}, 39 | ) 40 | return pd.Series(result.x, index=data.columns) 41 | -------------------------------------------------------------------------------- /trading/ibstate.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import threading 4 | from time import sleep 5 | import pandas as pd 6 | from ib.ext.Contract import Contract 7 | from ib.ext.ExecutionFilter import ExecutionFilter 8 | from ib.ext.Order import Order 9 | from ib.ext.TagValue import TagValue 10 | from ib.opt import ibConnection 11 | import config.instruments 12 | import config.portfolios 13 | import config.settings 14 | import data.db_mongo as db 15 | import core.utility 16 | from core.ib_connection import get_next_id 17 | from core.logger import get_logger 18 | from trading.account import Account 19 | 20 | logger = get_logger('ibstate') 21 | 22 | 23 | class IBstate(object): 24 | """ 25 | Stateful object to help us interact with Interactive Brokers. 26 | """ 27 | def __init__(self): 28 | self.orders_cache = {} # market orders placed during the current trading session 29 | self.open_orders_raw = [] # list of orders that are placed, but not yet executed 30 | self.order_status_raw = [] 31 | self.clientId = get_next_id() # client ID fot TWS terminal 32 | self.port = getattr(config.settings, 'ib_port', 4001) # socket port for TWS terminal 33 | if os.environ.get('D_HOST') is not None: # host for TWS terminal 34 | self.host = os.environ.get('D_HOST') 35 | else: 36 | self.host = 'localhost' 37 | 38 | self.api_delay = 5 # Minimal interval between IB API requests in seconds 39 | self.last_error = None # Last API error code, used for error handling 40 | self.last_account = None # Last traded account, used for error handling 41 | # Flags used to determine if we're ready for trading 42 | self.accounts_loaded = False 43 | self.positions_loaded = False 44 | self.accounts = {} # List of IB accounts in a multi-account system 45 | # Events for thread synchronization 46 | self.next_id_event = threading.Event() 47 | self.open_orders_event = threading.Event() 48 | # IB connection objects used to send all API requests 49 | self.connection = ibConnection(host=self.host, port=self.port, clientId=self.clientId) 50 | 51 | def is_ready(self): 52 | """ 53 | :return: True when IBstate is ready for trading 54 | """ 55 | return self.accounts_loaded and self.positions_loaded 56 | 57 | def connect(self): 58 | """ 59 | Establishes a connection to TWS terminal. Automatically retries if an error occurs. 60 | """ 61 | while not (hasattr(self.connection, 'isConnected') and self.connection.isConnected()): 62 | self.accounts_loaded = False 63 | self.positions_loaded = False 64 | logger.info('Connecting with clientId %d on %s:%d' % (self.clientId, self.host, self.port)) 65 | r = self.connection.connect() 66 | if r: 67 | logger.info('Connection successful!') 68 | self._subscribe() 69 | self.update_open_orders() 70 | else: 71 | logger.warning("Couldn't establish connection, retrying in %d s." % (self.api_delay)) 72 | sleep(self.api_delay) 73 | 74 | def open_orders(self): 75 | """ 76 | :return: A pd.Series of orders that were placed on the exchange, 77 | but haven't yet been executed 78 | """ 79 | self.open_orders_event.wait() 80 | if len(self.open_orders_raw) > 0: 81 | return pd.Series( 82 | {(v.contract.m_symbol, v.contract.m_expiry[0:6], v.order.m_account): \ 83 | v.order.m_totalQuantity * core.utility.direction(v.order.m_action) \ 84 | for v in self.open_orders_raw}).fillna(0).rename('open'). \ 85 | rename_axis(['instrument', 'contract', 'account']) 86 | else: 87 | return pd.Series() 88 | 89 | def update_open_orders(self): 90 | """ 91 | Send API request to pull the list of open orders for all accounts. 92 | Callbacks: self._open_order_handler() and self._open_order_end_handler() 93 | """ 94 | self.open_orders_raw.clear() 95 | self.open_orders_event.clear() 96 | self.connection.reqAllOpenOrders() 97 | self.open_orders_event.wait() 98 | 99 | def place_order(self, instrument, expiry, quantity, acc=None): 100 | """ 101 | Send API request to place an order on the exchange. 102 | :param instrument: core.instrument.Instrument object 103 | :param expiry: contract label 104 | :param quantity: order size as a signed integer (quantity > 0 means 'BUY' 105 | and quantity < 0 means 'SELL') 106 | :param acc: IB account to place order from, if None - the default account will be used 107 | """ 108 | contract = Contract() 109 | contract.m_symbol = instrument.ib_code 110 | contract.m_secType = 'FUT' 111 | # place_order expects the contract label here, not the actual expiration date 112 | contract.m_expiry = expiry 113 | contract.m_exchange = instrument.exchange 114 | contract.m_currency = instrument.denomination 115 | if hasattr(instrument, 'ib_trading_class'): 116 | contract.m_tradingClass = instrument.ib_trading_class 117 | if hasattr(instrument, 'ib_multiplier'): 118 | contract.m_multiplier = instrument.ib_multiplier 119 | 120 | order = Order() 121 | order.m_orderType = 'MKT' 122 | order.m_algoStrategy = 'Adaptive' 123 | order.m_algoParams = [TagValue('adaptivePriority', 'Patient')] 124 | order.m_totalQuantity = int(abs(quantity)) 125 | order.m_action = quantity > 0 and 'BUY' or 'SELL' 126 | if acc is not None: 127 | order.m_account = acc.name 128 | self.last_account = acc 129 | logger.warning( 130 | ' '.join(['Order:', str(self.order_id), contract.m_symbol, contract.m_expiry, \ 131 | order.m_action, str(order.m_totalQuantity)])) 132 | self.connection.placeOrder(self.order_id, contract, order) 133 | self.orders_cache[self.order_id] = {'contract': contract, 134 | 'order': order} 135 | # order_id may not update just after the order is submitted so we save the previous one and 136 | # keep requesting until it's updated or we hit the time/iterations limit 137 | prev_id = self.order_id 138 | i = 0 139 | while prev_id >= self.order_id: 140 | sleep(self.api_delay) 141 | i += 1 142 | logger.debug('Requesting next order_id..') 143 | self.connection.reqIds(1) 144 | self.next_id_event.wait(timeout=(self.api_delay * 30)) 145 | self.next_id_event.clear() 146 | if i > 60: 147 | logger.warning("Couldn't obtain next valid order id. Next orders may not be" 148 | "submitted correctly!") 149 | return 150 | 151 | def sync_portfolio(self, portfolio, acc=None, trade=True): 152 | """ 153 | Calculate the ideal positions for a portfolio and place orders on the market 154 | :param portfolio: trading.portfolio.Portfolio object 155 | :param acc: trading.account.Account object 156 | :param trade: bool, if False - will print the positions but won't actually trade 157 | """ 158 | 159 | if acc is None: 160 | acc = list(self.accounts.values())[0] 161 | 162 | assert acc.is_valid() 163 | 164 | # if trade is True: 165 | # self.connection.reqGlobalCancel() 166 | 167 | frontier = portfolio.frontier(capital=acc.net) 168 | 169 | positions = acc.portfolio 170 | if positions.empty: 171 | trades = frontier 172 | trades['position'] = pd.Series(0, index=trades.index) 173 | logger.info('No position') 174 | else: 175 | logger.info('\n' + str(positions['pos'])) 176 | positions.rename(columns={'pos': 'position'}, inplace=True) 177 | trades = positions.join(frontier, how='outer').fillna(0)[['position', 'frontier']] 178 | 179 | oord = self.open_orders() 180 | # check if we have open orders for specific account 181 | hasorders = (len(oord) > 0) and ( 182 | acc.name in oord.index.levels[oord.index.names.index('account')]) 183 | if hasorders: 184 | trades = trades.join(oord[:, :, acc.name], how='outer').fillna(0) 185 | logger.error("Account {} has open orders, not trading".format(acc.name)) 186 | return 187 | else: 188 | trades['open'] = pd.Series(0, index=trades.index) 189 | 190 | trades['trade'] = trades['frontier'] - trades['position'] - trades['open'] 191 | trades['inst'] = trades.index.get_level_values(0) 192 | trades['inst'] = trades['inst'].apply(portfolio.ibcode_to_inst) 193 | trades = trades[trades['trade'].abs() > 0] 194 | # trades_close = trades[trades.isnull()['inst']] 195 | trades = trades[~trades.isnull()['inst']] 196 | logger.info('\n' + str(trades)) 197 | # pprint.pprint(trades) 198 | sleep(self.api_delay * 10) 199 | # # close any opened positions for instruments not in the portfolio 200 | # p_temp = Portfolio(instruments=config.portfolios.p_all) 201 | # trades_close['inst'] = trades_close.index.to_frame()['instrument'].apply(p_temp.ibcode_to_inst) 202 | # trades_close['trade'] = -(trades_close['position'] + trades_close['open']) 203 | if trade: 204 | # [self.place_order(k.inst, k.Index[1], k.trade, acc=acc) for k in trades_close.itertuples()] 205 | [self.place_order(k.inst, k.Index[1], k.trade, acc=acc) for k in 206 | trades.itertuples()] 207 | else: 208 | logger.info("Dry run, not actually trading.") 209 | return trades 210 | 211 | 212 | """ ===== API Events subscription and handlers ===== """ 213 | 214 | 215 | def _register(self): 216 | self.connection.register(self._error_handler, 'Error') 217 | self.connection.register(self._account_summary_handler, 'AccountSummary') 218 | self.connection.register(self._account_summary_end_handler, 'AccountSummaryEnd') 219 | self.connection.register(self._next_valid_id_handler, 'NextValidId') 220 | self.connection.register(self._execution_handler, 'ExecDetails') 221 | self.connection.register(self._commission_report_handler, 'CommissionReport') 222 | self.connection.register(self._open_order_handler, 'OpenOrder') 223 | self.connection.register(self._open_order_end_handler, 'OpenOrderEnd') 224 | self.connection.register(self._order_status_handler, 'OrderStatus') 225 | self.connection.register(self._managed_accounts_handler, 'ManagedAccounts') 226 | self.connection.register(self._positions_handler, 'Position') 227 | self.connection.register(self._positions_end_handler, 'PositionEnd') 228 | 229 | def _subscribe(self): 230 | self._register() 231 | self.connection.reqExecutions(2, ExecutionFilter()) 232 | self.connection.reqAccountSummary(1, 'All', 'NetLiquidation') 233 | self.connection.reqIds(1) 234 | # in theory this should help to avoid errors on connection break/restore in the middle of 235 | # trading, but needs some testing 236 | logger.debug('Connection: waiting for next valid ID..') 237 | self.next_id_event.wait() 238 | logger.debug('Obtained next ID, connection is completed') 239 | self.next_id_event.clear() 240 | 241 | def _order_status_handler(self, msg): 242 | self.order_status_raw.append(msg) 243 | 244 | def _open_order_handler(self, msg): 245 | if msg.contract.m_secType == 'FUT': 246 | self.open_orders_raw.append(msg) 247 | db.insert_order(msg) 248 | 249 | def _open_order_end_handler(self, msg): 250 | self.open_orders_event.set() 251 | 252 | def _managed_accounts_handler(self, msg): 253 | logger.info('Managed accounts: %s' % msg.accountsList) 254 | acc_names = msg.accountsList.split(',') 255 | try: # there may be an empty element due to leading comma 256 | acc_names.remove('') 257 | except ValueError: 258 | pass 259 | self.accounts = {x: Account(x) for x in acc_names} 260 | # subscribe to all accounts updates 261 | self.connection.reqPositions() 262 | 263 | def _positions_handler(self, msg): 264 | i = {v['ib_code']: v for v in config.instruments.instrument_definitions 265 | if v.get('ib_code') is not None} 266 | try: 267 | expiration_month = i[msg.contract.m_symbol]['expiration_month'] 268 | except KeyError: 269 | expiration_month = 0 270 | 271 | if msg.contract.m_secType == 'FUT': 272 | # m_expiry represents the actual expiration date here, so need to convert 273 | # that to a contract label 274 | date = datetime.datetime.strptime(msg.contract.m_expiry, '%Y%m%d') +\ 275 | datetime.timedelta(weeks = (-4 * expiration_month)) 276 | contract = datetime.datetime.strftime(date, '%Y%m') 277 | contract_index = pd.MultiIndex.from_tuples([(str(msg.contract.m_symbol), contract)],\ 278 | names=['instrument','contract']) 279 | pd_msg = pd.DataFrame(dict(msg.items()), index=contract_index) 280 | self.pd_msg = pd_msg 281 | 282 | if msg.account in self.accounts: 283 | acc = self.accounts[msg.account] 284 | if contract_index[0] in acc.portfolio.index: 285 | acc.portfolio.update(pd_msg) 286 | else: 287 | acc.portfolio = acc.portfolio.append(pd_msg) 288 | 289 | def _positions_end_handler(self, msg): 290 | self.positions_loaded = True 291 | logger.debug('Accounts positions loaded') 292 | 293 | def _account_summary_handler(self, msg): 294 | acc = self.accounts[msg.account] 295 | acc.summary = dict(zip(msg.keys(), msg.values())) 296 | acc.net = float(msg.value) 297 | acc.base_currency = msg.currency 298 | try: 299 | assert msg.currency == config.settings.base_currency 300 | except AssertionError: 301 | logger.error("IB currency %s, system currency %s", msg.currency, config.settings.base_currency) 302 | db.insert_account_summary(msg) 303 | 304 | def _account_summary_end_handler(self, msg): 305 | self.accounts_loaded = True 306 | logger.debug('Accounts summary loaded') 307 | 308 | def _next_valid_id_handler(self, msg): 309 | self.order_id = msg.orderId 310 | logger.debug('Next valid id = %d' % msg.orderId) 311 | self.next_id_event.set() 312 | 313 | def _error_handler(self, msg): 314 | if type(msg.errorMsg) == ConnectionResetError: 315 | msg.errorMsg = 'Connection Reset Error' 316 | self.clientId = get_next_id() 317 | logger.warning("Trying clientId %s", self.clientId) 318 | 319 | if (msg.id != -1) and (msg.id != None): 320 | if (self.last_account is None) and (len(self.accounts) > 0): 321 | self.last_account = list(self.accounts.values())[0] 322 | db.insert_error(msg, self.last_account.name) 323 | 324 | try: 325 | # Stop repeated disconnection errors 326 | if self.last_error != msg.errorCode and msg.errorCode != 1100: 327 | logger.error(' '.join([str(msg.id), str(msg.errorCode), msg.errorMsg])) 328 | except: 329 | logger.error('Null error - TWS gateway has probably closed.') 330 | 331 | self.last_error = msg.errorCode 332 | 333 | # Client Id in Use 334 | if msg.id == -1 and msg.errorCode == 326: 335 | self.clientId = get_next_id() 336 | logger.warning("Updated clientId %s", self.clientId) 337 | 338 | # Connection restored 339 | if msg.id == -1 and msg.errorCode == 1102: 340 | self._subscribe() 341 | 342 | # Not connected 343 | if msg.id == -1 and msg.errorCode == 504: 344 | self.connection.connect() 345 | self._subscribe() 346 | 347 | # Could not connect 348 | if msg.id == -1 and msg.errorCode == 502: 349 | sleep(self.api_delay*6) 350 | 351 | def _execution_handler(self, msg): 352 | print("Execution:", msg.execution.m_acctNumber, msg.execution.m_orderId, 353 | msg.execution.m_side, msg.execution.m_shares, msg.execution.m_price) 354 | db.insert_execution(msg.execution) 355 | 356 | def _commission_report_handler(self, msg): 357 | db.insert_commission_report(msg.commissionReport) 358 | -------------------------------------------------------------------------------- /trading/portfolio.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from functools import lru_cache 4 | import sys 5 | 6 | try: 7 | import config.strategy 8 | except ImportError: 9 | print("You need to set up the strategy file at config/strategy.py.") 10 | sys.exit() 11 | 12 | try: 13 | import config.settings 14 | except ImportError: 15 | print("You need to set up the settings file at config/settings.py.") 16 | sys.exit() 17 | 18 | from core.instrument import Instrument 19 | from core.utility import draw_sample, sharpe 20 | from trading.accountcurve import accountCurve 21 | import trading.bootstrap_portfolio as bp 22 | import seaborn 23 | import pyprind 24 | from multiprocessing_on_dill import Pool 25 | from contextlib import closing 26 | from core.logger import get_logger 27 | logger = get_logger('portfolio') 28 | 29 | 30 | class Portfolio(object): 31 | """ 32 | Portfolio is an object that is a group of instruments, with calculated positions based on the weighting and volatility target. 33 | """ 34 | 35 | def __init__(self, weights=1, instruments=None): 36 | self.instruments = Instrument.load(instruments) 37 | self.weights = pd.Series(config.strategy.portfolio_weights) 38 | # remove weights for instruments that aren't in the portfolio 39 | self.weights = self.weights[self.weights.index.isin(instruments)] 40 | # instruments blacklisted due to validation errors 41 | self.inst_blacklist = [] 42 | 43 | def __repr__(self): 44 | return str(len(self.valid_instruments())) + " instruments" 45 | 46 | @lru_cache(maxsize=8) 47 | def curve(self, **kw): 48 | """ 49 | Returns an AccountCurve for this Portfolio. 50 | """ 51 | kw2={'portfolio_weights': self.valid_weights()} 52 | kw2.update(kw) 53 | return accountCurve(list(self.valid_instruments().values()), **kw2) 54 | 55 | def valid_instruments(self): 56 | return dict([i for i in self.instruments.items() if i[1].name not in self.inst_blacklist]) 57 | 58 | def valid_weights(self): 59 | return self.weights[~self.weights.index.isin(self.inst_blacklist)] 60 | 61 | ### Utility Functions ################################################################### 62 | 63 | def validate(self): 64 | """ 65 | Runs Instrument.validate for every Instrument in the Portfolio and returns a DataFrame. Used for trading. 66 | """ 67 | import concurrent.futures 68 | bar = pyprind.ProgBar(len(self.instruments.values()), title='Validating instruments') 69 | with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: 70 | dl = {executor.submit(x.validate): x.name for x in self.instruments.values()} 71 | d = {} 72 | for fut in concurrent.futures.as_completed(dl): 73 | bar.update(item_id=dl[fut]) 74 | d[dl[fut]] = fut.result() 75 | d = pd.DataFrame(d).transpose() 76 | # blacklist instruments that didn't pass validation 77 | self.inst_blacklist = d[d['is_valid'] == False].index.tolist() 78 | return d 79 | 80 | def instrument_stats(self): 81 | """ 82 | Returns individual metrics for every Instrument in the Portfolio. Not used for trading, just for research. 83 | """ 84 | with closing(Pool()) as pool: 85 | df = pd.DataFrame(dict(pool.map(lambda x: (x.name, x.curve().stats_list()), 86 | self.valid_instruments().values()))).transpose() 87 | return df 88 | 89 | @lru_cache(maxsize=1) 90 | def inst_calc(self): 91 | """ 92 | Calculate the base positions for every instrument, before applying portfolio-wide weighting and volatility scaling. 93 | """ 94 | with closing(Pool()) as pool: 95 | d = dict(pool.map(lambda x: (x.name, x.calculate()), self.valid_instruments().values())) 96 | return d 97 | 98 | @lru_cache(maxsize=1) 99 | def panama_prices(self): 100 | """ 101 | Returns a dataframe with the panama prices of every instrument. Not used for trading. 102 | """ 103 | return pd.DataFrame({k: v['panama_prices'] for k, v in self.inst_calc().items()}) 104 | 105 | def point_values(self): 106 | """ 107 | Returns a series with the point values of every instrument. Not used for trading. 108 | """ 109 | return pd.Series({k: v.point_value for k, v in self.valid_instruments().items()}) 110 | 111 | def corr(self): 112 | """ 113 | Returns a correlation matrix of the all the instruments with trading rules applied, with returns bagged by week. 114 | 115 | Not used for trading. Intended to be used in Jupyter. 116 | """ 117 | df = self.curve().returns().resample('W').sum().replace(0, np.nan).corr() 118 | cm = seaborn.dark_palette("green", as_cmap=True) 119 | s = df.style.background_gradient(cmap=cm) 120 | return s 121 | 122 | def corr_pp(self): 123 | """ 124 | Returns a correlation matrix of all the instruments panama prices, with returns bagged by week. 125 | 126 | Not used for trading. Intended to be used in Jupyter. 127 | """ 128 | 129 | df = self.panama_prices().diff().resample('W').sum().corr() 130 | cm = seaborn.dark_palette("green", as_cmap=True) 131 | s = df.style.background_gradient(cmap=cm) 132 | return s 133 | 134 | def cov(self): 135 | """ 136 | Returns the covariance matrix of all the instruments with trading rules applied. 137 | 138 | Not used for trading. Intended to be used in Jupyter. 139 | """ 140 | return self.curve().returns().resample('W').sum().cov() 141 | 142 | def plot(self): 143 | """ 144 | Returns a plot of the cumulative returns of the Portfolio 145 | """ 146 | return self.returns().sum(axis=1).cumsum().plot() 147 | 148 | def instrument_count(self): 149 | """ 150 | The number of instruments being traded by date. 151 | """ 152 | return np.maximum.accumulate((~np.isnan(self.panama_prices())).sum(axis=1)).plot() 153 | 154 | def bootstrap_pool(self, **kw): 155 | """ 156 | Bootstrap forecast weights using all the data in the portfolio. 157 | """ 158 | # m = ProcessPoolExecutor().map(lambda x: x.bootstrap(), self.instruments.values()) 159 | bs = {k: v.bootstrap(**kw) for k, v in self.valid_instruments().items()} 160 | self.bs = pd.DataFrame.from_dict(bs) 161 | self.bs.mean(axis=1).plot.bar(yerr=self.bs.std(axis=1)) 162 | return self.bs 163 | 164 | def bootstrap_portfolio(self, **kw): 165 | """ 166 | Bootstrap the instrument weights to work out the 'best' combination of instruments to use. 167 | 168 | Strongly advise against using this and to handcraft instrument weights yourself as this method may lead to overfitting. 169 | """ 170 | self.bp_weights = bp.bootstrap(self, **kw) 171 | return self.bp_weights 172 | 173 | def bootstrap_rules(self, n=10000, **kw): 174 | z = self.forecast_returns(**kw) 175 | a = pd.Series({k: v.shape[0] for k, v in z.items()}) 176 | b=(a/a.sum()) 177 | sharpes = [] 178 | corrs = {} 179 | for k, v in b.iteritems(): 180 | for x in range(0,int(round(v*n))): 181 | sample = draw_sample(z[k], 252) 182 | sharpes.append(sharpe(sample).rename(k)) 183 | corrs[(k, x)] = (sample.resample('W').sum().corr()) 184 | return pd.DataFrame(sharpes), pd.Panel(corrs).mean(axis=0) 185 | 186 | @lru_cache(maxsize=1) 187 | def forecast_returns(self, **kw): 188 | """Get the returns for individual forecasts for each instrument, useful for bootstrapping 189 | forecast Sharpe ratios""" 190 | with closing(Pool()) as pool: 191 | d = dict(pool.map(lambda x: (x.name, x.forecast_returns(**kw).dropna()), 192 | self.valid_instruments().values())) 193 | return d 194 | 195 | @lru_cache(maxsize=1) 196 | def forecasts(self, **kw): 197 | """ 198 | Returns a dict of forecasts for every Instrument in the Portfolio. 199 | """ 200 | with closing(Pool()) as pool: 201 | d = dict(pool.map(lambda x: (x.name, x.forecasts(**kw)), 202 | self.valid_instruments().values())) 203 | return d 204 | 205 | @lru_cache(maxsize=1) 206 | def weighted_forecasts(self, **kw): 207 | """ 208 | Returns a dict of weighted forecasts for every Instrument in the Portfolio. 209 | """ 210 | with closing(Pool()) as pool: 211 | d = dict(pool.map(lambda x: (x.name, x.weighted_forecast(**kw)), 212 | self.valid_instruments().values())) 213 | return d 214 | 215 | @lru_cache(maxsize=1) 216 | def market_prices(self): 217 | """ 218 | Returns the market prices of the instruments, for the currently traded contract. 219 | """ 220 | mp = pd.DataFrame({k: v['market_price'] for k, v in self.inst_calc().items()}).ffill() 221 | return pd.DataFrame({k: mp[k].loc[self.inst_calc()[k]['roll_progression']\ 222 | .to_frame().set_index('contract', append=True).index].reset_index( 223 | 'contract', drop=True) for k in self.valid_instruments().keys()}) 224 | 225 | def ibcode_to_inst(self, ib_code): 226 | """ 227 | Take an IB symbol and return the Instrument. 228 | """ 229 | a = {k.ib_code: k for k in self.instruments.values()} 230 | try: 231 | return a[ib_code] 232 | except KeyError: 233 | logger.warn('Ignoring mystery instrument in IB portfolio ' + ib_code) 234 | return None 235 | 236 | def frontier(self, capital=500000): 237 | """ 238 | Returns a DataFrame of positions we want for a given capital. The last line represents today's trade. 239 | """ 240 | c = self.curve(capital=capital) 241 | f = c.positions.tail(1).iloc[0] 242 | f.index = pd.MultiIndex.from_tuples([(k, str(c.inst_calc()[k]['roll_progression']. 243 | tail(1)[0])) for k in f.index]) 244 | f.rename('frontier', inplace=True) 245 | f = f.to_frame() 246 | f.index = pd.MultiIndex.from_tuples(f.index.map( 247 | lambda x: (self.valid_instruments()[x[0]].ib_code, x[1]))) 248 | f.index = f.index.rename(['instrument', 'contract']) 249 | return f 250 | 251 | def cache_clear(self): 252 | """ 253 | Clear the functools.lru_cache 254 | """ 255 | self.inst_calc.cache_clear() 256 | self.panama_prices.cache_clear() 257 | self.forecasts.cache_clear() 258 | self.market_prices.cache_clear() 259 | self.forecast_returns.cache_clear() 260 | self.curve.cache_clear() 261 | [v.cache_clear() for v in self.instruments.values()] 262 | logger.info("Portfolio LRU Cache cleared") 263 | -------------------------------------------------------------------------------- /trading/rules.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from functools import partial 4 | from core.utility import norm_forecast, norm_vol 5 | 6 | def pickleable_ewmac(d, x): 7 | """ 8 | Returns an exponentially weighted moving average crossover for a single series and period. 9 | """ 10 | return d.ewm(span = x, min_periods = x*4).mean() - d.ewm(span = x*4, min_periods=x*4).mean() 11 | 12 | def ewmac(inst, **kw): 13 | """ 14 | Exponentially weighted moving average crossover. 15 | 16 | Returns a DataFrame with four different periods. 17 | """ 18 | d = norm_vol(inst.panama_prices(**kw)) 19 | columns = [8, 16, 32, 64] 20 | f = map(partial(pickleable_ewmac, d), columns) 21 | f = pd.DataFrame(list(f)).transpose() 22 | f.columns = pd.Series(columns).map(lambda x: "ewmac"+str(x)) 23 | return norm_forecast(f) 24 | 25 | def mr(inst, **kw): 26 | """ 27 | Mean reversion. 28 | 29 | Never managed to make this profitable on futures, however it is presented here if you want to try it. 30 | """ 31 | d = norm_vol(inst.panama_prices(**kw)) 32 | columns = [2, 4, 8, 16, 32, 64] 33 | f = map(partial(pickleable_ewmac, d), columns) 34 | f = pd.DataFrame(list(f)).transpose() * -1 35 | f = f*10/f.abs().mean() 36 | f.columns = pd.Series(columns).map(lambda x: "mr"+str(x)) 37 | return f.clip(-20,20) 38 | 39 | def carry(inst, **kw): 40 | """ 41 | Generic carry function. 42 | 43 | If the Instrument has a spot price series, use that for carry (carry_spot), otherwise use the next contract (carry_next) 44 | """ 45 | if hasattr(inst, 'spot'): 46 | return carry_spot(inst, **kw).rename('carry') 47 | else: 48 | return carry_next(inst, **kw).rename('carry') 49 | 50 | def carry_spot(inst, **kw): 51 | """ 52 | Calculates the carry between the current future price and the underlying spot price. 53 | """ 54 | f = inst.spot() - inst.market_price().reset_index('contract', drop=True) 55 | f = f * 365 / inst.time_to_expiry() 56 | return norm_forecast(f.ewm(90).mean()).rename('carry_spot') 57 | 58 | def carry_next(inst, debug=False, **kw): 59 | """ 60 | Calculates the carry between the current future price and the next contract we are going to roll to. 61 | """ 62 | # If not trading nearest contract, Nearer contract price minus current contract price, divided by the time difference 63 | # If trading nearest contract, Current contract price minus next contract price, divided by the time difference 64 | 65 | current_contract = inst.roll_progression().to_frame().set_index('contract', append=True).swaplevel() 66 | next_contract = inst.roll_progression().apply(inst.next_contract, months=inst.trade_only).to_frame().set_index('contract', append=True).swaplevel() 67 | 68 | current_prices = inst.contracts(active_only=True).join(current_contract, how='inner')['close'].reset_index('contract') 69 | 70 | next_prices = inst.contracts(active_only=True).join(next_contract, how='inner')['close'].reset_index('contract') 71 | 72 | #Replace zeros with nan 73 | next_prices[next_prices==0] = np.nan 74 | 75 | # Apply a ffill for low volume contracts 76 | # next_prices.ffill(inplace=True) 77 | 78 | current_prices['contract'] = current_prices['contract'].apply(_get_month) 79 | next_prices['contract'] = next_prices['contract'].apply(_get_month) 80 | td = next_prices['contract'] - current_prices['contract'] 81 | td.loc[td<=0] = td.loc[td<=0] + 12 82 | td = td/12 83 | # Apply a 5 day mean to prices to stabilise signal 84 | carry = (current_prices['close'] - next_prices['close']) 85 | 86 | carry = carry.rolling(window=5).mean()/td 87 | 88 | f = norm_forecast(carry).ffill(limit=3) 89 | 90 | # if f.isnull().values.any(): 91 | # print(inst.name, "has some missing carry values") 92 | 93 | if f.sum() == 0: 94 | print(inst.name + ' carry is zero') 95 | if debug==True: 96 | return f.rename('carry_next'), current_prices, next_prices, td 97 | # return f.mean().rename('carry_next') 98 | return f.interpolate().rolling(window=90).mean().rename('carry_next') 99 | 100 | def carry_prev(inst, **kw): 101 | """ 102 | Analogue of carry_next() but looks at the previous contract. Useful when not trading the nearest contract but one further one. Typically you'd do this for instruments where the near contract has deathly skew - e.g. Eurodollar or VIX. 103 | """ 104 | 105 | current_contract = inst.roll_progression().to_frame().set_index('contract', append=True).swaplevel() 106 | prev_contract = inst.roll_progression().apply(inst.next_contract, months=inst.trade_only, 107 | reverse=True).to_frame().set_index('contract', append=True).swaplevel() 108 | current_prices = inst.contracts(active_only=True).join(current_contract, how='inner')['close'].reset_index('contract') 109 | prev_prices = inst.contracts(active_only=True).join(prev_contract, how='inner')['close'].reset_index('contract') 110 | # Replace zeros with nan 111 | prev_prices[prev_prices == 0] = np.nan 112 | 113 | current_prices['contract'] = current_prices['contract'].apply(_get_month) 114 | prev_prices['contract'] = prev_prices['contract'].apply(_get_month) 115 | td = current_prices['contract'] - prev_prices['contract'] 116 | td.loc[td <= 0] = td.loc[td <= 0] + 12 117 | td = td / 12 118 | # Apply a 5 day mean to prices to stabilise signal 119 | carry = prev_prices['close'] - current_prices['close'] 120 | carry = carry.rolling(window=5).mean() / td 121 | f = norm_forecast(carry).ffill(limit=3) 122 | if f.sum() == 0: 123 | print(inst.name + ' carry is zero') 124 | return f.interpolate().rolling(window=90).mean().rename('carry') 125 | 126 | 127 | def open_close(inst, **kw): 128 | """ 129 | Read this one in a book, and it was easy to test. Consistently unprofitable in its current form. 130 | """ 131 | #yesterdays open-close, divided by std deviation of returns 132 | inst.panama_prices() 133 | a = inst.rp().to_frame().set_index('contract', append=True).swaplevel().join(inst.contracts(), how='inner') 134 | f = ((a['close']-a['open'])/inst.return_volatility).dropna().reset_index('contract', drop=True) 135 | return norm_forecast(f).rename('open_close') 136 | 137 | def weather_rule(inst, **kw): 138 | """ 139 | They say the most likely weather for tomorrow is today's weather. If today's return was +ve, 140 | returns +10 for tomorrow and vice versa. 141 | """ 142 | r = inst.panama_prices().diff() 143 | #where today's return is 0, just use yesterday's forecast 144 | r[r==0]=np.nan 145 | r = np.sign(r) * 10 146 | r.ffill() 147 | return r.rename('weather_rule') 148 | 149 | 150 | def buy_and_hold(inst, **kw): 151 | """ 152 | Returns a fixed forecast of 10. 153 | """ 154 | return pd.Series(10, index=inst.pp().index).rename('buy_and_hold') 155 | 156 | def sell_and_hold(inst, **kw): 157 | """ 158 | Returns a fixed forecast of -10. 159 | """ 160 | return pd.Series(-10, index=inst.pp().index).rename('sell_and_hold') 161 | 162 | def breakout_fn(data, lookback, smooth=None): 163 | """ 164 | :param data: prices DataFrame 165 | :param lookback: lookback window in days 166 | :param smooth: moving average window for the forecast in days 167 | :return: forecast DataFrame, range [-1, 1] (unnormed) 168 | """ 169 | if smooth is None: 170 | smooth = max(int(lookback / 4.0), 1) 171 | price_roll = data.rolling(lookback, min_periods=int(min(len(data), np.ceil(lookback / 2.0)))) 172 | roll_min = price_roll.min() 173 | roll_max = price_roll.max() 174 | roll_mean = (roll_max + roll_min) / 2.0 175 | b = (data - roll_mean) / (roll_max - roll_min) 176 | bsmooth = b.ewm(span=smooth, min_periods=np.ceil(smooth / 2.0)).mean() 177 | return bsmooth 178 | 179 | def breakout(inst, **kw): 180 | """ 181 | Returns a DataFrame of breakout forecasts based on Rob Carver's work here: 182 | https://qoppac.blogspot.com.es/2016/05/a-simple-breakout-trading-rule.html 183 | """ 184 | prices = inst.panama_prices(**kw) 185 | lookbacks = [40, 80, 160, 320] 186 | res = map(partial(breakout_fn, prices), lookbacks) 187 | res = pd.DataFrame(list(res)).transpose() 188 | res.columns = pd.Series(lookbacks).map(lambda x: "brk%d" % x) 189 | return norm_forecast(res) 190 | 191 | def _get_month(a): 192 | return int(str(a)[4:6]) 193 | 194 | -------------------------------------------------------------------------------- /trading/start.py: -------------------------------------------------------------------------------- 1 | # import logging 2 | # logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 3 | # formatter = logging.Formatter(fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 4 | # root_logger = logging.getLogger() 5 | 6 | # create path if not exists 7 | # path = os.path.join(config.local.data_dir, 'log') 8 | # if not os.path.exists(path): 9 | # os.makedirs(path) 10 | # hdlr = logging.FileHandler(os.path.join(path, config.local.environment + '.log')) 11 | # hdlr.setFormatter(formatter) 12 | # root_logger.addHandler(hdlr) 13 | # root_logger.setLevel(logging.DEBUG) 14 | 15 | import pandas as pd 16 | from pylab import rcParams 17 | rcParams['figure.figsize'] = 15, 10 18 | 19 | idx=pd.IndexSlice 20 | 21 | 22 | # suppress numpy warnings explicitly, as Pandas doesn't do this in other threads 23 | import warnings 24 | warnings.filterwarnings('ignore') 25 | -------------------------------------------------------------------------------- /validate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import trading.portfolio 3 | import config.portfolios 4 | import logging 5 | import traceback 6 | import sys 7 | logger = logging.getLogger('validate') 8 | 9 | 10 | if __name__ == "__main__": 11 | p = trading.portfolio.Portfolio(instruments=config.portfolios.p_trade) 12 | try: 13 | validate = p.validate()[['carry_forecast', 'currency_age', 'panama_age', 'price_age', 'weighted_forecast']] 14 | print(validate) 15 | except KeyboardInterrupt: 16 | print("Shutdown requested...exiting") 17 | except Exception: 18 | traceback.print_exc(file=sys.stdout) 19 | sys.exit(0) 20 | 21 | --------------------------------------------------------------------------------