├── .gitignore ├── 0 - Preparing custom dataset for Zipline.ipynb ├── 1 - Hello World in Zipline backtester.ipynb ├── 2 - ARIMA time-series modelling.ipynb ├── 3 - Cointegration Mean Reversion Pairs Trading.ipynb ├── README.md ├── utils └── plotting.py └── zipline_extensions ├── .DS_Store ├── bundles └── fxcm.py └── calendar_fxcm.py /.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | backtest.pickle 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | -------------------------------------------------------------------------------- /0 - Preparing custom dataset for Zipline.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# How to setup Zipline backtester with custom data from FXCM " 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## 1. Install standard packages\n", 15 | "ou should be able to install zipline by using pip. If it doesn't work, there are other ways to install zipline that you can look up. \n", 16 | "\n", 17 | "```\n", 18 | "pip install zipline\n", 19 | "```\n", 20 | "I had to upgrade some packages\n", 21 | "```\n", 22 | "pip install -U pip\n", 23 | "pip install -U numpy\n", 24 | "```" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "## 2. Download custom forex calendar \n", 32 | "Zipline has made its calendars a standalone package available at https://github.com/quantopian/trading_calendars \n", 33 | "to make it more available for adding custom calendars, which is what you want because Zipline doesn't support the 24/7 forex calendar as default. \n", 34 | "\n", 35 | "Clone my fork of `trading_calendars` into any preferable location \n", 36 | "```\n", 37 | "git clone https://github.com/grananqvist/trading_calendars.git \n", 38 | "```\n", 39 | "This fork as a new calendar called `forex` " 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "## 3. Download my zipline utilities \n", 47 | "Necessary utilities are located in zipline_extensions in this repo, download them by cloning the repo into preferable location \n", 48 | "```\n", 49 | "git glone https://github.com/grananqvist/Machine-Learning-Trading-Strategies.git\n", 50 | "```" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "## 4. Add packages to PYTHONPATH\n", 58 | "For python to find the repos above, add them to your `$PYTHONPATH` \n", 59 | "```\n", 60 | "export PYTHONPATH=$PYTHONPATH::\n", 61 | "```\n", 62 | "In my case, my `$PYTHONPATH` looks like this \n", 63 | "`/Users/system/Github/Machine-Learning-Trading-Strategies/:/Users/system/Github/trading_calendars/` \n", 64 | "\n", 65 | "Test if packages are installed correctly (you will need to restart the jupyter notebook kernel): " 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 13, 71 | "metadata": { 72 | "collapsed": true 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "from zipline.utils.calendars import get_calendar\n", 77 | "get_calendar('forex')\n", 78 | "from zipline_extensions.bundles.fxcm import via_fxcm_csv_daily" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "## 5. Download FXCM data \n", 86 | "1. Clone my FXCM data downloader\n", 87 | "```\n", 88 | "git clone https://github.com/grananqvist/FXCM-Forex-Data-Downloader.git\n", 89 | "```\n", 90 | "2. Get an [API key](https://www.fxcm.com/uk/algorithmic-trading/api-trading/) from FXCM, works with a demo account \n", 91 | "3. Download data using the download script \n", 92 | "```\n", 93 | "python main.py -pe D1 -t -p \n", 94 | "```" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "## 6. Edit your extensions.py\n", 102 | "A file called `extensions.py` should be located in `~/.zipline/` \n", 103 | "Insert the code below into `extensions.py` to register fxcm_daily as a bundle for zipline \n", 104 | "\n", 105 | "**Note:** replace `DATA_PATH` with the path to your downloaded dataset " 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "import os\n", 117 | "from zipline.data.bundles import register\n", 118 | "from zipline_extensions.bundles.fxcm import via_fxcm_csv_daily\n", 119 | "DATA_PATH = os.path.join(\n", 120 | " os.environ['HOME'], 'Github/Machine-Learning-Trading-Strategies/data/D1/')\n", 121 | "\n", 122 | "register('fxcm_daily', via_fxcm_csv_daily(DATA_PATH), calendar_name='forex')" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "## 7. Ingest bundle" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 16, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "Making bundle with symbols: ('AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD', 'AUS200', 'Bund', 'CADCHF', 'CADJPY', 'CHFJPY', 'CHN50', 'Copper', 'ESP35', 'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', 'EURJPY', 'EURNOK', 'EURNZD', 'EURSEK', 'EURTRY', 'EURUSD', 'EUSTX50', 'FRA40', 'GBPAUD', 'GBPCAD', 'GBPCHF', 'GBPJPY', 'GBPNZD', 'GBPUSD', 'GER30', 'HKG33', 'JPN225', 'NAS100', 'NGAS', 'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD', 'SOYF', 'SPX500', 'TRYJPY', 'UK100', 'UKOil', 'US30', 'USDCAD', 'USDCHF', 'USDCNH', 'USDHKD', 'USDJPY', 'USDMXN', 'USDNOK', 'USDOLLAR', 'USDSEK', 'USDTRY', 'USDZAR', 'USOil', 'XAGUSD', 'XAUUSD', 'ZARJPY')\n", 142 | "symbol= AUDCAD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/AUDCAD_D1.csv\n", 143 | "--Preprocessing summary--\n", 144 | "candles before: 4826\n", 145 | "candles after: 4433\n", 146 | "symbol= AUDCHF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/AUDCHF_D1.csv\n", 147 | "--Preprocessing summary--\n", 148 | "candles before: 4770\n", 149 | "candles after: 4376\n", 150 | "symbol= AUDJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/AUDJPY_D1.csv\n", 151 | "--Preprocessing summary--\n", 152 | "candles before: 4834\n", 153 | "candles after: 4433\n", 154 | "symbol= AUDNZD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/AUDNZD_D1.csv\n", 155 | "--Preprocessing summary--\n", 156 | "candles before: 4783\n", 157 | "candles after: 4376\n", 158 | "symbol= AUDUSD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/AUDUSD_D1.csv\n", 159 | "--Preprocessing summary--\n", 160 | "candles before: 4830\n", 161 | "candles after: 4433\n", 162 | "symbol= AUS200 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/AUS200_D1.csv\n", 163 | "--Preprocessing summary--\n", 164 | "candles before: 4371\n", 165 | "candles after: 4432\n", 166 | "symbol= Bund file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/Bund_D1.csv\n", 167 | "--Preprocessing summary--\n", 168 | "candles before: 1587\n", 169 | "candles after: 1577\n", 170 | "symbol= CADCHF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/CADCHF_D1.csv\n", 171 | "--Preprocessing summary--\n", 172 | "candles before: 2850\n", 173 | "candles after: 2714\n", 174 | "symbol= CADJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/CADJPY_D1.csv\n", 175 | "--Preprocessing summary--\n", 176 | "candles before: 4774\n", 177 | "candles after: 4376\n", 178 | "symbol= CHFJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/CHFJPY_D1.csv\n", 179 | "--Preprocessing summary--\n", 180 | "candles before: 4830\n", 181 | "candles after: 4433\n", 182 | "symbol= CHN50 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/CHN50_D1.csv\n", 183 | "--Preprocessing summary--\n", 184 | "candles before: 3141\n", 185 | "candles after: 3374\n", 186 | "symbol= Copper file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/Copper_D1.csv\n", 187 | "--Preprocessing summary--\n", 188 | "candles before: 1865\n", 189 | "candles after: 1833\n", 190 | "symbol= ESP35 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/ESP35_D1.csv\n", 191 | "--Preprocessing summary--\n", 192 | "candles before: 4370\n", 193 | "candles after: 4432\n", 194 | "symbol= EURAUD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURAUD_D1.csv\n", 195 | "--Preprocessing summary--\n", 196 | "candles before: 4847\n", 197 | "candles after: 4433\n", 198 | "symbol= EURCAD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURCAD_D1.csv\n", 199 | "--Preprocessing summary--\n", 200 | "candles before: 4836\n", 201 | "candles after: 4433\n", 202 | "symbol= EURCHF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURCHF_D1.csv\n", 203 | "--Preprocessing summary--\n", 204 | "candles before: 4833\n", 205 | "candles after: 4433\n", 206 | "symbol= EURGBP file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURGBP_D1.csv\n", 207 | "--Preprocessing summary--\n", 208 | "candles before: 4837\n", 209 | "candles after: 4433\n", 210 | "symbol= EURJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURJPY_D1.csv\n", 211 | "--Preprocessing summary--\n", 212 | "candles before: 4835\n", 213 | "candles after: 4433\n", 214 | "symbol= EURNOK file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURNOK_D1.csv\n", 215 | "--Preprocessing summary--\n", 216 | "candles before: 3105\n", 217 | "candles after: 2857\n", 218 | "symbol= EURNZD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURNZD_D1.csv\n", 219 | "--Preprocessing summary--\n", 220 | "candles before: 4771\n", 221 | "candles after: 4376\n", 222 | "symbol= EURSEK file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURSEK_D1.csv\n", 223 | "--Preprocessing summary--\n", 224 | "candles before: 3115\n", 225 | "candles after: 2857\n", 226 | "symbol= EURTRY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURTRY_D1.csv\n", 227 | "--Preprocessing summary--\n", 228 | "candles before: 2933\n", 229 | "candles after: 2714\n", 230 | "symbol= EURUSD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EURUSD_D1.csv\n", 231 | "--Preprocessing summary--\n", 232 | "candles before: 4845\n", 233 | "candles after: 4433\n", 234 | "symbol= EUSTX50 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/EUSTX50_D1.csv\n", 235 | "--Preprocessing summary--\n", 236 | "candles before: 1581\n", 237 | "candles after: 1573\n", 238 | "symbol= FRA40 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/FRA40_D1.csv\n", 239 | "--Preprocessing summary--\n", 240 | "candles before: 4407\n", 241 | "candles after: 4432\n", 242 | "symbol= GBPAUD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GBPAUD_D1.csv\n", 243 | "--Preprocessing summary--\n", 244 | "candles before: 4770\n", 245 | "candles after: 4376\n", 246 | "symbol= GBPCAD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GBPCAD_D1.csv\n", 247 | "--Preprocessing summary--\n", 248 | "candles before: 4767\n", 249 | "candles after: 4376\n", 250 | "symbol= GBPCHF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GBPCHF_D1.csv\n", 251 | "--Preprocessing summary--\n", 252 | "candles before: 4827\n", 253 | "candles after: 4433\n", 254 | "symbol= GBPJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GBPJPY_D1.csv\n", 255 | "--Preprocessing summary--\n", 256 | "candles before: 4831\n", 257 | "candles after: 4433\n", 258 | "symbol= GBPNZD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GBPNZD_D1.csv\n", 259 | "--Preprocessing summary--\n", 260 | "candles before: 4761\n", 261 | "candles after: 4376\n", 262 | "symbol= GBPUSD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GBPUSD_D1.csv\n", 263 | "--Preprocessing summary--\n", 264 | "candles before: 4825\n", 265 | "candles after: 4433\n", 266 | "symbol= GER30 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/GER30_D1.csv\n", 267 | "--Preprocessing summary--\n", 268 | "candles before: 4380\n", 269 | "candles after: 4432\n", 270 | "symbol= HKG33 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/HKG33_D1.csv\n", 271 | "--Preprocessing summary--\n", 272 | "candles before: 4250\n", 273 | "candles after: 4432\n", 274 | "symbol= JPN225 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/JPN225_D1.csv\n", 275 | "--Preprocessing summary--\n", 276 | "candles before: 4312\n", 277 | "candles after: 4432\n", 278 | "symbol= NAS100 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/NAS100_D1.csv\n", 279 | "--Preprocessing summary--\n", 280 | "candles before: 4390\n", 281 | "candles after: 4432\n", 282 | "symbol= NGAS file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/NGAS_D1.csv\n", 283 | "--Preprocessing summary--\n", 284 | "candles before: 1618\n", 285 | "candles after: 1618\n", 286 | "symbol= NZDCAD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/NZDCAD_D1.csv\n", 287 | "--Preprocessing summary--\n", 288 | "candles before: 2859\n", 289 | "candles after: 2602\n", 290 | "symbol= NZDCHF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/NZDCHF_D1.csv\n", 291 | "--Preprocessing summary--\n", 292 | "candles before: 2869\n", 293 | "candles after: 2602\n", 294 | "symbol= NZDJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/NZDJPY_D1.csv\n", 295 | "--Preprocessing summary--\n", 296 | "candles before: 4771\n", 297 | "candles after: 4376\n", 298 | "symbol= NZDUSD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/NZDUSD_D1.csv\n", 299 | "--Preprocessing summary--\n", 300 | "candles before: 4819\n", 301 | "candles after: 4433\n", 302 | "symbol= SOYF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/SOYF_D1.csv\n", 303 | "--Preprocessing summary--\n", 304 | "candles before: 82\n", 305 | "candles after: 184\n", 306 | "symbol= SPX500 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/SPX500_D1.csv\n", 307 | "--Preprocessing summary--\n", 308 | "candles before: 4390\n", 309 | "candles after: 4432\n", 310 | "symbol= TRYJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/TRYJPY_D1.csv\n", 311 | "--Preprocessing summary--\n", 312 | "candles before: 2437\n", 313 | "candles after: 2173\n", 314 | "symbol= UK100 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/UK100_D1.csv\n", 315 | "--Preprocessing summary--\n", 316 | "candles before: 4358\n", 317 | "candles after: 4433\n", 318 | "symbol= UKOil file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/UKOil_D1.csv\n", 319 | "--Preprocessing summary--\n", 320 | "candles before: 4413\n", 321 | "candles after: 4432\n", 322 | "symbol= US30 file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/US30_D1.csv\n" 323 | ] 324 | }, 325 | { 326 | "name": "stdout", 327 | "output_type": "stream", 328 | "text": [ 329 | "--Preprocessing summary--\n", 330 | "candles before: 4392\n", 331 | "candles after: 4432\n", 332 | "symbol= USDCAD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDCAD_D1.csv\n", 333 | "--Preprocessing summary--\n", 334 | "candles before: 4829\n", 335 | "candles after: 4433\n", 336 | "symbol= USDCHF file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDCHF_D1.csv\n", 337 | "--Preprocessing summary--\n", 338 | "candles before: 4823\n", 339 | "candles after: 4433\n", 340 | "symbol= USDCNH file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDCNH_D1.csv\n", 341 | "--Preprocessing summary--\n", 342 | "candles before: 1935\n", 343 | "candles after: 1708\n", 344 | "symbol= USDHKD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDHKD_D1.csv\n", 345 | "--Preprocessing summary--\n", 346 | "candles before: 3271\n", 347 | "candles after: 3032\n", 348 | "symbol= USDJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDJPY_D1.csv\n", 349 | "--Preprocessing summary--\n", 350 | "candles before: 4833\n", 351 | "candles after: 4433\n", 352 | "symbol= USDMXN file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDMXN_D1.csv\n", 353 | "--Preprocessing summary--\n", 354 | "candles before: 2845\n", 355 | "candles after: 2714\n", 356 | "symbol= USDNOK file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDNOK_D1.csv\n", 357 | "--Preprocessing summary--\n", 358 | "candles before: 3051\n", 359 | "candles after: 3109\n", 360 | "symbol= USDOLLAR file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDOLLAR_D1.csv\n", 361 | "--Preprocessing summary--\n", 362 | "candles before: 2049\n", 363 | "candles after: 2002\n", 364 | "symbol= USDSEK file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDSEK_D1.csv\n", 365 | "--Preprocessing summary--\n", 366 | "candles before: 3069\n", 367 | "candles after: 3021\n", 368 | "symbol= USDTRY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDTRY_D1.csv\n", 369 | "--Preprocessing summary--\n", 370 | "candles before: 2920\n", 371 | "candles after: 2678\n", 372 | "symbol= USDZAR file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USDZAR_D1.csv\n", 373 | "--Preprocessing summary--\n", 374 | "candles before: 2994\n", 375 | "candles after: 3032\n", 376 | "symbol= USOil file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/USOil_D1.csv\n", 377 | "--Preprocessing summary--\n", 378 | "candles before: 4384\n", 379 | "candles after: 4432\n", 380 | "symbol= XAGUSD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/XAGUSD_D1.csv\n", 381 | "--Preprocessing summary--\n", 382 | "candles before: 4393\n", 383 | "candles after: 4433\n", 384 | "symbol= XAUUSD file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/XAUUSD_D1.csv\n", 385 | "--Preprocessing summary--\n", 386 | "candles before: 4412\n", 387 | "candles after: 4433\n", 388 | "symbol= ZARJPY file= /Users/system/Github/Machine-Learning-Trading-Strategies/data/D1/ZARJPY_D1.csv\n", 389 | "--Preprocessing summary--\n", 390 | "candles before: 2868\n", 391 | "candles after: 2608\n", 392 | "\u001b[?25lMerging daily equity files: [####################################] \u001b[?25h\n", 393 | "Daily data has been written\n", 394 | "Metadata:\n", 395 | " start_date end_date auto_close_date symbol exchange\n", 396 | "0 2001-09-17 2018-09-05 2018-09-06 AUDCAD FXCM\n", 397 | "1 2001-11-28 2018-09-05 2018-09-06 AUDCHF FXCM\n", 398 | "2 2001-09-17 2018-09-05 2018-09-06 AUDJPY FXCM\n", 399 | "3 2001-11-28 2018-09-05 2018-09-06 AUDNZD FXCM\n", 400 | "4 2001-09-17 2018-09-05 2018-09-06 AUDUSD FXCM\n", 401 | "5 2001-09-17 2018-09-05 2018-09-06 AUS200 FXCM\n", 402 | "6 2012-08-21 2018-09-05 2018-09-06 Bund FXCM\n", 403 | "7 2008-04-11 2018-09-05 2018-09-06 CADCHF FXCM\n", 404 | "8 2001-11-28 2018-09-05 2018-09-06 CADJPY FXCM\n", 405 | "9 2001-09-17 2018-09-05 2018-09-06 CHFJPY FXCM\n", 406 | "10 2005-09-30 2018-09-05 2018-09-06 CHN50 FXCM\n", 407 | "11 2011-08-29 2018-09-05 2018-09-06 Copper FXCM\n", 408 | "12 2001-09-17 2018-09-05 2018-09-06 ESP35 FXCM\n", 409 | "13 2001-09-17 2018-09-05 2018-09-06 EURAUD FXCM\n", 410 | "14 2001-09-17 2018-09-05 2018-09-06 EURCAD FXCM\n", 411 | "15 2001-09-17 2018-09-05 2018-09-06 EURCHF FXCM\n", 412 | "16 2001-09-17 2018-09-05 2018-09-06 EURGBP FXCM\n", 413 | "17 2001-09-17 2018-09-05 2018-09-06 EURJPY FXCM\n", 414 | "18 2007-09-25 2018-09-05 2018-09-06 EURNOK FXCM\n", 415 | "19 2001-11-28 2018-09-05 2018-09-06 EURNZD FXCM\n", 416 | "20 2007-09-25 2018-09-05 2018-09-06 EURSEK FXCM\n", 417 | "21 2008-04-11 2018-09-05 2018-09-06 EURTRY FXCM\n", 418 | "22 2001-09-17 2018-09-05 2018-09-06 EURUSD FXCM\n", 419 | "23 2012-08-27 2018-09-05 2018-09-06 EUSTX50 FXCM\n", 420 | "24 2001-09-17 2018-09-05 2018-09-06 FRA40 FXCM\n", 421 | "25 2001-11-28 2018-09-05 2018-09-06 GBPAUD FXCM\n", 422 | "26 2001-11-28 2018-09-05 2018-09-06 GBPCAD FXCM\n", 423 | "27 2001-09-17 2018-09-05 2018-09-06 GBPCHF FXCM\n", 424 | "28 2001-09-17 2018-09-05 2018-09-06 GBPJPY FXCM\n", 425 | "29 2001-11-28 2018-09-05 2018-09-06 GBPNZD FXCM\n", 426 | ".. ... ... ... ... ...\n", 427 | "31 2001-09-17 2018-09-05 2018-09-06 GER30 FXCM\n", 428 | "32 2001-09-17 2018-09-05 2018-09-06 HKG33 FXCM\n", 429 | "33 2001-09-17 2018-09-05 2018-09-06 JPN225 FXCM\n", 430 | "34 2001-09-17 2018-09-05 2018-09-06 NAS100 FXCM\n", 431 | "35 2012-06-25 2018-09-05 2018-09-06 NGAS FXCM\n", 432 | "36 2008-09-16 2018-09-05 2018-09-06 NZDCAD FXCM\n", 433 | "37 2008-09-16 2018-09-05 2018-09-06 NZDCHF FXCM\n", 434 | "38 2001-11-28 2018-09-05 2018-09-06 NZDJPY FXCM\n", 435 | "39 2001-09-17 2018-09-05 2018-09-06 NZDUSD FXCM\n", 436 | "40 2017-12-22 2018-09-05 2018-09-06 SOYF FXCM\n", 437 | "41 2001-09-17 2018-09-05 2018-09-06 SPX500 FXCM\n", 438 | "42 2010-05-10 2018-09-05 2018-09-06 TRYJPY FXCM\n", 439 | "43 2001-09-17 2018-09-05 2018-09-06 UK100 FXCM\n", 440 | "44 2001-09-17 2018-09-05 2018-09-06 UKOil FXCM\n", 441 | "45 2001-09-17 2018-09-05 2018-09-06 US30 FXCM\n", 442 | "46 2001-09-17 2018-09-05 2018-09-06 USDCAD FXCM\n", 443 | "47 2001-09-17 2018-09-05 2018-09-06 USDCHF FXCM\n", 444 | "48 2012-02-20 2018-09-05 2018-09-06 USDCNH FXCM\n", 445 | "49 2007-01-23 2018-09-05 2018-09-06 USDHKD FXCM\n", 446 | "50 2001-09-17 2018-09-05 2018-09-06 USDJPY FXCM\n", 447 | "51 2008-04-11 2018-09-05 2018-09-06 USDMXN FXCM\n", 448 | "52 2006-10-06 2018-09-05 2018-09-06 USDNOK FXCM\n", 449 | "53 2011-01-04 2018-09-05 2018-09-06 USDOLLAR FXCM\n", 450 | "54 2007-02-07 2018-09-05 2018-09-06 USDSEK FXCM\n", 451 | "55 2008-06-02 2018-09-05 2018-09-06 USDTRY FXCM\n", 452 | "56 2007-01-23 2018-09-05 2018-09-06 USDZAR FXCM\n", 453 | "57 2001-09-17 2018-09-05 2018-09-06 USOil FXCM\n", 454 | "58 2001-09-17 2018-09-05 2018-09-06 XAGUSD FXCM\n", 455 | "59 2001-09-17 2018-09-05 2018-09-06 XAUUSD FXCM\n", 456 | "60 2008-09-08 2018-09-05 2018-09-06 ZARJPY FXCM\n", 457 | "\n", 458 | "[61 rows x 5 columns]\n", 459 | "Metadata has been written\n" 460 | ] 461 | } 462 | ], 463 | "source": [ 464 | "#%load_ext zipline\n", 465 | "!zipline ingest -b fxcm_daily" 466 | ] 467 | } 468 | ], 469 | "metadata": { 470 | "kernelspec": { 471 | "display_name": "Python 3", 472 | "language": "python", 473 | "name": "python3" 474 | }, 475 | "language_info": { 476 | "codemirror_mode": { 477 | "name": "ipython", 478 | "version": 3 479 | }, 480 | "file_extension": ".py", 481 | "mimetype": "text/x-python", 482 | "name": "python", 483 | "nbconvert_exporter": "python", 484 | "pygments_lexer": "ipython3", 485 | "version": "3.5.2" 486 | } 487 | }, 488 | "nbformat": 4, 489 | "nbformat_minor": 2 490 | } 491 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine-Learning-Trading-Strategies 2 | 3 | ## This repo will be continously updated with my upcomming ML algo trading research 4 | -------------------------------------------------------------------------------- /utils/plotting.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from matplotlib.collections import PatchCollection 4 | from matplotlib.patches import Rectangle 5 | import statsmodels.api as sm 6 | 7 | 8 | def plot_acf(X, nlags=40, conf_interval=0.05, is_pacf=False): 9 | """ Plots the Autocorrelations for different lags 10 | 11 | Arguments: 12 | X: the time-series to plot ACF for 13 | nlags: number of lags to plot 14 | conf_interval: the confidence interval to plot for every autocorrelation 15 | is_pacf: whether or not to use PACF (ACF by default) 16 | 17 | """ 18 | acf_f = sm.tsa.pacf if is_pacf else sm.tsa.acf 19 | acf_title = 'PACF' if is_pacf else 'ACF' 20 | 21 | # The confidence intervals are returned by the functions as (lower, upper) 22 | # The plotting function needs them in the form (x-lower, upper-x) 23 | X_acf, X_acf_confs = acf_f(X, nlags=nlags, alpha=0.05) 24 | 25 | errorbars = np.ndarray((2, len(X_acf))) 26 | errorbars[0, :] = X_acf - X_acf_confs[:, 0] 27 | errorbars[1, :] = X_acf_confs[:, 1] - X_acf 28 | 29 | plt.plot(X_acf, 'ro') 30 | plt.errorbar( 31 | range(len(X_acf)), 32 | X_acf, 33 | yerr=errorbars, 34 | fmt='none', 35 | ecolor='gray', 36 | capthick=2) 37 | plt.xlabel('Lag') 38 | plt.ylabel('Autocorrelation') 39 | plt.title(acf_title) 40 | 41 | # 5% box 42 | box = Rectangle((1, -0.05), nlags, 0.10) 43 | # Create patch collection with specified colour/alpha 44 | pc = PatchCollection([box], facecolor='r', alpha=0.5) 45 | plt.gca().add_collection(pc) 46 | 47 | # print stats, skipping first one 48 | print('Number of autocorrelated lags: {0}'.format( 49 | sum(X_acf > 0.05) + sum(X_acf < -0.05) - 1)) 50 | print('Number of lags where 0 is not included in confidence interval: {0}'. 51 | format(sum(X_acf_confs[:, 0] > 0) + sum(X_acf_confs[:, 1] < 0) - 1)) 52 | -------------------------------------------------------------------------------- /zipline_extensions/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grananqvist/Machine-Learning-Trading-Strategies/4c9f16f9a788b86d8a393cf0533dd8ce1ce17b20/zipline_extensions/.DS_Store -------------------------------------------------------------------------------- /zipline_extensions/bundles/fxcm.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | from datetime import datetime 5 | from zipline.utils.cli import maybe_show_progress 6 | from zipline.utils.calendars import get_calendar 7 | 8 | 9 | def via_fxcm_csv_daily(path, calendar_name='forex', start=None, end=None): 10 | 11 | # TODO: use FXM downloader here. meanwhile, static start and end 12 | start = pd.Timestamp('2001-09-07') 13 | end = pd.Timestamp('2018-08-31') 14 | 15 | d1_path = os.path.join(path, 'D1/') 16 | m1_path = os.path.join(path, 'm1/') 17 | 18 | # get files to bundle 19 | _, _, file_names = list(os.walk(d1_path))[0] 20 | file_names = [f for f in file_names if f[-4:] == '.csv'] 21 | # TODO: DEBUG 22 | symbols = tuple([file_name.split('_')[0] for file_name in [file_names[0]]]) 23 | 24 | # there are inconsistent data before this date 25 | earliest_date = pd.Timestamp('2001-09-17 00:00:00', tz=None) 26 | 27 | # Define custom ingest function 28 | def ingest(environ, 29 | asset_db_writer, 30 | minute_bar_writer, 31 | daily_bar_writer, 32 | adjustment_writer, 33 | calendar, 34 | start_session, 35 | end_session, 36 | cache, 37 | show_progress, 38 | output_dir): 39 | 40 | print('Making bundle with symbols: ', symbols) 41 | print(start_session) 42 | print(end_session) 43 | 44 | start_session = start_session if start_session is not None else start 45 | end_session = end_session if end_session is not None else end 46 | 47 | df_metadata = pd.DataFrame( 48 | np.empty( 49 | len(symbols), 50 | dtype=[ 51 | ('start_date', 'datetime64[ns]'), 52 | ('end_date', 'datetime64[ns]'), 53 | ('first_traded', 'datetime64[ns]'), 54 | ('auto_close_date', 'datetime64[ns]'), 55 | ('symbol', 'object'), 56 | ])) 57 | 58 | # We need to feed something that is iterable - like a list or a generator - 59 | # that is a tuple with an integer for sid and a DataFrame for the data to daily_bar_writer 60 | data = [] 61 | data_intraday = [] 62 | sid = 0 63 | 64 | # preprocess every symbol 65 | for symbol in symbols: 66 | 67 | d1_filename = symbol + '_D1.csv' 68 | m1_filename = symbol + '_m1.csv' 69 | 70 | print("symbol=", symbol, "file=", d1_filename) 71 | 72 | df = preprocess_csv( 73 | os.path.join(d1_path, d1_filename), 74 | calendar_name, 75 | sample_period='1D') 76 | df = df[(df.index >= start) & (df.index <= end)] 77 | data.append((sid, df)) 78 | 79 | df = preprocess_csv( 80 | os.path.join(m1_path, m1_filename), 81 | calendar_name, 82 | sample_period='1min') 83 | df = df[(df.index >= start) & (df.index <= end)] 84 | data_intraday.append((sid, df)) 85 | 86 | # the start date is the date of the first trade and 87 | start_date = df.index[0] 88 | 89 | if start_date < earliest_date: 90 | start_date = earliest_date 91 | 92 | # the end date is the date of the last trade 93 | end_date = df.index[-1] 94 | 95 | # The auto_close date is the day after the last trade. 96 | ac_date = end_date + pd.Timedelta(days=1) 97 | 98 | # Update our meta data 99 | df_metadata.iloc[sid] = start_date, end_date, start_date, ac_date, symbol 100 | 101 | sid += 1 102 | 103 | daily_bar_writer.write(data, show_progress=True) 104 | print('Daily data has been written') 105 | 106 | minute_bar_writer.write(data_intraday, show_progress=True) 107 | print('Intraday data has been written') 108 | 109 | df_metadata['exchange'] = "FXCM" 110 | 111 | print('Metadata:') 112 | print(df_metadata) 113 | 114 | # Not sure why symbol_map is needed 115 | symbol_map = pd.Series(df_metadata.symbol.index, df_metadata.symbol) 116 | 117 | asset_db_writer.write(equities=df_metadata) 118 | adjustment_writer.write() 119 | print('Metadata has been written') 120 | 121 | return ingest 122 | 123 | 124 | def preprocess_csv(path, calendar_name, sample_period='1D'): 125 | 126 | df = pd.read_csv( 127 | path, 128 | index_col='date', 129 | parse_dates=True, 130 | ).sort_index() 131 | 132 | if sample_period == '1D': 133 | # change time to midnight. is 21:00 by default 134 | df.index = df.index + df.index.map( 135 | lambda x: pd.Timedelta(hours=24 - x.hour)) 136 | 137 | candles_before = len(df) 138 | 139 | # convert to a single OHLC number 140 | df['open'] = (df['bidopen'] + df['askopen']) / 2 141 | df['high'] = (df['bidhigh'] + df['askhigh']) / 2 142 | df['low'] = (df['bidlow'] + df['asklow']) / 2 143 | df['close'] = (df['bidclose'] + df['askclose']) / 2 144 | df = df[['open', 'high', 'low', 'close']] * 100 145 | 146 | # forward-fill any missing dates 147 | df = df.resample(sample_period).mean() 148 | df.fillna(method="ffill", inplace=True) 149 | 150 | # remove dates not included in current calendar 151 | calendar = get_calendar(calendar_name) 152 | df = calendar.filter_dates(df, intraday=sample_period != '1D') 153 | 154 | # dummy volume col 155 | df['volume'] = 0 156 | df['volume'] = df['volume'].astype(np.int32) 157 | 158 | candles_after = len(df) 159 | 160 | print('--Preprocessing summary--\ncandles before: {0}\ncandles after: {1}'. 161 | format(candles_before, candles_after)) 162 | 163 | return df 164 | -------------------------------------------------------------------------------- /zipline_extensions/calendar_fxcm.py: -------------------------------------------------------------------------------- 1 | import pytz 2 | import pandas as pd 3 | from datetime import time 4 | from zipline.utils.calendars import TradingCalendar 5 | from zipline.utils.memoize import lazyval 6 | 7 | 8 | class ForexCalendar(TradingCalendar): 9 | 10 | NYT_4PM = time(20) 11 | NYT_6PM = time(22) 12 | 13 | @lazyval 14 | def day(self): 15 | return pd.tseries.offsets.CustomBusinessDay( 16 | holidays=self.adhoc_holidays, 17 | calendar=self.regular_holidays, 18 | weekmask='Sun Mon Tue Wed Thu Fri' 19 | ) 20 | 21 | @property 22 | def name(self): 23 | return "forex" 24 | 25 | @property 26 | def tz(self): 27 | return pytz.UTC 28 | 29 | @property 30 | def open_time(self): 31 | return time(0, 0) 32 | 33 | @property 34 | def close_time(self): 35 | return time(23, 59) 36 | 37 | def _calculate_special_opens(self, start, end): 38 | return self._special_dates( 39 | self.special_opens, 40 | self.special_opens_adhoc(start, end), 41 | start, 42 | end, 43 | ) 44 | 45 | def _calculate_special_closes(self, start, end): 46 | return self._special_dates( 47 | self.special_closes, 48 | self.special_closes_adhoc(start, end), 49 | start, 50 | end, 51 | ) 52 | 53 | def special_opens_adhoc(self, start, end): 54 | return [ 55 | (self.NYT_6PM, self._sunday_dates(start, end)) 56 | ] 57 | 58 | def special_closes_adhoc(self, start, end): 59 | return [ 60 | (self.NYT_4PM, self._friday_dates(start, end)) 61 | ] 62 | 63 | def _friday_dates(self, start, end): 64 | return pd.date_range(start=start, 65 | end=end, 66 | freq='W-FRI') 67 | 68 | def _sunday_dates(self, start, end): 69 | return pd.date_range(start=start, 70 | end=end, 71 | freq='W-SUN') 72 | --------------------------------------------------------------------------------