├── 2 Getting Data.ipynb ├── 3 S&P 500 Webscrape.ipynb ├── 4 Full Dataset.ipynb ├── 5 Regressions.ipynb ├── 6 Machine Learning.ipynb ├── 7 Machine Learning Function.ipynb ├── 8 Visualize Data.ipynb └── README.md /2 Getting Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Requirement already satisfied: pandas-datareader in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages\n", 13 | "Requirement already satisfied: requests>=2.3.0 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas-datareader)\n", 14 | "Requirement already satisfied: requests-ftp in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas-datareader)\n", 15 | "Requirement already satisfied: requests-file in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas-datareader)\n", 16 | "Requirement already satisfied: pandas>=0.17.0 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas-datareader)\n", 17 | "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from requests>=2.3.0->pandas-datareader)\n", 18 | "Requirement already satisfied: certifi>=2017.4.17 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from requests>=2.3.0->pandas-datareader)\n", 19 | "Requirement already satisfied: idna<2.6,>=2.5 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from requests>=2.3.0->pandas-datareader)\n", 20 | "Requirement already satisfied: urllib3<1.22,>=1.21.1 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from requests>=2.3.0->pandas-datareader)\n", 21 | "Requirement already satisfied: six in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from requests-file->pandas-datareader)\n", 22 | "Requirement already satisfied: numpy>=1.7.0 in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas>=0.17.0->pandas-datareader)\n", 23 | "Requirement already satisfied: python-dateutil in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas>=0.17.0->pandas-datareader)\n", 24 | "Requirement already satisfied: pytz>=2011k in /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages (from pandas>=0.17.0->pandas-datareader)\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "!pip install pandas-datareader" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 2, 35 | "metadata": { 36 | "collapsed": true 37 | }, 38 | "outputs": [], 39 | "source": [ 40 | "import datetime\n", 41 | "startDate = datetime.datetime(2010, 1, 1)\n", 42 | "endDate = datetime.datetime(2017, 5, 1)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 3, 48 | "metadata": { 49 | "collapsed": true 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "import pandas_datareader.data as web\n", 54 | "stock = web.DataReader(\"F\", 'google', startDate, endDate)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 4, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | " Open High Low Close Volume\n", 67 | "Date \n", 68 | "2010-01-04 10.17 10.28 10.05 10.28 60855796\n", 69 | "2010-01-05 10.45 11.24 10.40 10.96 215620138\n", 70 | "2010-01-06 11.21 11.46 11.13 11.37 200070554\n", 71 | "2010-01-07 11.46 11.69 11.32 11.66 130201626\n", 72 | "2010-01-08 11.67 11.74 11.46 11.69 130462912\n", 73 | "2010-01-11 11.90 12.14 11.78 12.11 170626174\n", 74 | "2010-01-12 11.98 12.03 11.72 11.87 162995898\n", 75 | "2010-01-13 11.91 11.93 11.47 11.68 154527028\n", 76 | "2010-01-14 11.65 11.86 11.51 11.76 116572673\n", 77 | "2010-01-15 11.74 11.76 11.55 11.60 96174717\n", 78 | "2010-01-19 11.51 11.83 11.46 11.75 65933944\n", 79 | "2010-01-20 11.68 11.69 11.50 11.51 71649459\n", 80 | "2010-01-21 11.53 11.62 11.01 11.18 121451396\n", 81 | "2010-01-22 11.01 11.12 10.41 10.52 161530010\n", 82 | "2010-01-25 10.73 11.10 10.61 11.03 121621423\n", 83 | "2010-01-26 11.17 11.46 11.07 11.19 108250462\n", 84 | "2010-01-27 11.57 11.62 11.22 11.55 105091531\n", 85 | "2010-01-28 11.90 11.95 11.27 11.41 203466724\n", 86 | "2010-01-29 11.60 11.61 10.70 10.84 159741164\n", 87 | "2010-02-01 11.14 11.18 10.93 11.12 82748130\n", 88 | "2010-02-02 11.26 11.52 11.19 11.39 119785377\n", 89 | "2010-02-03 11.49 11.66 11.42 11.64 90125495\n", 90 | "2010-02-04 11.49 11.53 11.00 11.06 129792142\n", 91 | "2010-02-05 10.97 11.11 10.49 10.91 181535182\n", 92 | "2010-02-08 11.09 11.32 10.88 10.97 92031327\n", 93 | "2010-02-09 11.18 11.22 11.02 11.15 83207091\n", 94 | "2010-02-10 11.12 11.14 10.90 10.94 73395561\n", 95 | "2010-02-11 11.00 11.18 10.88 11.18 65116156\n", 96 | "2010-02-12 10.92 11.18 10.85 11.12 69465400\n", 97 | "2010-02-16 11.21 11.38 11.11 11.32 62537480\n", 98 | "... ... ... ... ... ...\n", 99 | "2017-03-20 12.48 12.49 12.25 12.28 45329650\n", 100 | "2017-03-21 12.30 12.32 11.70 11.72 101559879\n", 101 | "2017-03-22 11.78 11.83 11.62 11.77 71107861\n", 102 | "2017-03-23 11.57 11.75 11.50 11.67 71209260\n", 103 | "2017-03-24 11.66 11.76 11.55 11.62 43772442\n", 104 | "2017-03-27 11.51 11.55 11.41 11.46 57713419\n", 105 | "2017-03-28 11.51 11.80 11.48 11.65 57662806\n", 106 | "2017-03-29 11.73 11.76 11.64 11.68 35795737\n", 107 | "2017-03-30 11.67 11.74 11.61 11.68 28008241\n", 108 | "2017-03-31 11.66 11.68 11.60 11.64 26717520\n", 109 | "2017-04-03 11.64 11.64 11.28 11.44 65671114\n", 110 | "2017-04-04 11.38 11.48 11.28 11.37 40068175\n", 111 | "2017-04-05 11.48 11.50 11.25 11.26 43316140\n", 112 | "2017-04-06 11.27 11.36 11.22 11.27 36165281\n", 113 | "2017-04-07 11.26 11.31 11.21 11.23 28874909\n", 114 | "2017-04-10 11.26 11.35 11.24 11.25 25857108\n", 115 | "2017-04-11 11.27 11.28 11.17 11.28 36366316\n", 116 | "2017-04-12 11.27 11.30 11.20 11.23 56588425\n", 117 | "2017-04-13 11.22 11.25 11.11 11.11 32789905\n", 118 | "2017-04-17 11.22 11.30 11.14 11.28 44731735\n", 119 | "2017-04-18 11.15 11.18 11.08 11.14 35659989\n", 120 | "2017-04-19 11.19 11.28 11.17 11.19 37524445\n", 121 | "2017-04-20 11.25 11.50 11.24 11.47 49990191\n", 122 | "2017-04-21 11.46 11.48 11.32 11.34 26076186\n", 123 | "2017-04-24 11.48 11.50 11.36 11.43 41071326\n", 124 | "2017-04-25 11.47 11.50 11.43 11.48 35862521\n", 125 | "2017-04-26 11.47 11.70 11.45 11.60 51685768\n", 126 | "2017-04-27 11.66 11.68 11.34 11.47 50203741\n", 127 | "2017-04-28 11.51 11.51 11.40 11.47 41415145\n", 128 | "2017-05-01 11.49 11.54 11.40 11.42 32118379\n", 129 | "\n", 130 | "[1844 rows x 5 columns]\n" 131 | ] 132 | } 133 | ], 134 | "source": [ 135 | "print(stock)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 5, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "Date\n", 148 | "2010-01-04 10.28\n", 149 | "2010-01-05 10.96\n", 150 | "2010-01-06 11.37\n", 151 | "2010-01-07 11.66\n", 152 | "2010-01-08 11.69\n", 153 | "2010-01-11 12.11\n", 154 | "2010-01-12 11.87\n", 155 | "2010-01-13 11.68\n", 156 | "2010-01-14 11.76\n", 157 | "2010-01-15 11.60\n", 158 | "2010-01-19 11.75\n", 159 | "2010-01-20 11.51\n", 160 | "2010-01-21 11.18\n", 161 | "2010-01-22 10.52\n", 162 | "2010-01-25 11.03\n", 163 | "2010-01-26 11.19\n", 164 | "2010-01-27 11.55\n", 165 | "2010-01-28 11.41\n", 166 | "2010-01-29 10.84\n", 167 | "2010-02-01 11.12\n", 168 | "2010-02-02 11.39\n", 169 | "2010-02-03 11.64\n", 170 | "2010-02-04 11.06\n", 171 | "2010-02-05 10.91\n", 172 | "2010-02-08 10.97\n", 173 | "2010-02-09 11.15\n", 174 | "2010-02-10 10.94\n", 175 | "2010-02-11 11.18\n", 176 | "2010-02-12 11.12\n", 177 | "2010-02-16 11.32\n", 178 | " ... \n", 179 | "2017-03-20 12.28\n", 180 | "2017-03-21 11.72\n", 181 | "2017-03-22 11.77\n", 182 | "2017-03-23 11.67\n", 183 | "2017-03-24 11.62\n", 184 | "2017-03-27 11.46\n", 185 | "2017-03-28 11.65\n", 186 | "2017-03-29 11.68\n", 187 | "2017-03-30 11.68\n", 188 | "2017-03-31 11.64\n", 189 | "2017-04-03 11.44\n", 190 | "2017-04-04 11.37\n", 191 | "2017-04-05 11.26\n", 192 | "2017-04-06 11.27\n", 193 | "2017-04-07 11.23\n", 194 | "2017-04-10 11.25\n", 195 | "2017-04-11 11.28\n", 196 | "2017-04-12 11.23\n", 197 | "2017-04-13 11.11\n", 198 | "2017-04-17 11.28\n", 199 | "2017-04-18 11.14\n", 200 | "2017-04-19 11.19\n", 201 | "2017-04-20 11.47\n", 202 | "2017-04-21 11.34\n", 203 | "2017-04-24 11.43\n", 204 | "2017-04-25 11.48\n", 205 | "2017-04-26 11.60\n", 206 | "2017-04-27 11.47\n", 207 | "2017-04-28 11.47\n", 208 | "2017-05-01 11.42\n", 209 | "Name: Close, Length: 1844, dtype: float64\n" 210 | ] 211 | } 212 | ], 213 | "source": [ 214 | "print(stock[\"Close\"])" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": 6, 220 | "metadata": {}, 221 | "outputs": [ 222 | { 223 | "name": "stdout", 224 | "output_type": "stream", 225 | "text": [ 226 | " Close Open\n", 227 | "Date \n", 228 | "2010-01-04 10.28 10.17\n", 229 | "2010-01-05 10.96 10.45\n", 230 | "2010-01-06 11.37 11.21\n", 231 | "2010-01-07 11.66 11.46\n", 232 | "2010-01-08 11.69 11.67\n", 233 | "2010-01-11 12.11 11.90\n", 234 | "2010-01-12 11.87 11.98\n", 235 | "2010-01-13 11.68 11.91\n", 236 | "2010-01-14 11.76 11.65\n", 237 | "2010-01-15 11.60 11.74\n", 238 | "2010-01-19 11.75 11.51\n", 239 | "2010-01-20 11.51 11.68\n", 240 | "2010-01-21 11.18 11.53\n", 241 | "2010-01-22 10.52 11.01\n", 242 | "2010-01-25 11.03 10.73\n", 243 | "2010-01-26 11.19 11.17\n", 244 | "2010-01-27 11.55 11.57\n", 245 | "2010-01-28 11.41 11.90\n", 246 | "2010-01-29 10.84 11.60\n", 247 | "2010-02-01 11.12 11.14\n", 248 | "2010-02-02 11.39 11.26\n", 249 | "2010-02-03 11.64 11.49\n", 250 | "2010-02-04 11.06 11.49\n", 251 | "2010-02-05 10.91 10.97\n", 252 | "2010-02-08 10.97 11.09\n", 253 | "2010-02-09 11.15 11.18\n", 254 | "2010-02-10 10.94 11.12\n", 255 | "2010-02-11 11.18 11.00\n", 256 | "2010-02-12 11.12 10.92\n", 257 | "2010-02-16 11.32 11.21\n", 258 | "... ... ...\n", 259 | "2017-03-20 12.28 12.48\n", 260 | "2017-03-21 11.72 12.30\n", 261 | "2017-03-22 11.77 11.78\n", 262 | "2017-03-23 11.67 11.57\n", 263 | "2017-03-24 11.62 11.66\n", 264 | "2017-03-27 11.46 11.51\n", 265 | "2017-03-28 11.65 11.51\n", 266 | "2017-03-29 11.68 11.73\n", 267 | "2017-03-30 11.68 11.67\n", 268 | "2017-03-31 11.64 11.66\n", 269 | "2017-04-03 11.44 11.64\n", 270 | "2017-04-04 11.37 11.38\n", 271 | "2017-04-05 11.26 11.48\n", 272 | "2017-04-06 11.27 11.27\n", 273 | "2017-04-07 11.23 11.26\n", 274 | "2017-04-10 11.25 11.26\n", 275 | "2017-04-11 11.28 11.27\n", 276 | "2017-04-12 11.23 11.27\n", 277 | "2017-04-13 11.11 11.22\n", 278 | "2017-04-17 11.28 11.22\n", 279 | "2017-04-18 11.14 11.15\n", 280 | "2017-04-19 11.19 11.19\n", 281 | "2017-04-20 11.47 11.25\n", 282 | "2017-04-21 11.34 11.46\n", 283 | "2017-04-24 11.43 11.48\n", 284 | "2017-04-25 11.48 11.47\n", 285 | "2017-04-26 11.60 11.47\n", 286 | "2017-04-27 11.47 11.66\n", 287 | "2017-04-28 11.47 11.51\n", 288 | "2017-05-01 11.42 11.49\n", 289 | "\n", 290 | "[1844 rows x 2 columns]\n" 291 | ] 292 | } 293 | ], 294 | "source": [ 295 | "print(stock[[\"Close\",\"Open\"]])" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": 7, 301 | "metadata": {}, 302 | "outputs": [ 303 | { 304 | "name": "stdout", 305 | "output_type": "stream", 306 | "text": [ 307 | " Open High Low Close Volume\n", 308 | "Date \n", 309 | "2010-01-04 NaN NaN NaN NaN NaN\n", 310 | "2010-01-05 0.027532 0.093385 0.034826 0.066148 2.543132\n", 311 | "2010-01-06 0.072727 0.019573 0.070192 0.037409 -0.072116\n", 312 | "2010-01-07 0.022302 0.020070 0.017071 0.025506 -0.349221\n", 313 | "2010-01-08 0.018325 0.004277 0.012367 0.002573 0.002007\n", 314 | "2010-01-11 0.019709 0.034072 0.027923 0.035928 0.307852\n", 315 | "2010-01-12 0.006723 -0.009061 -0.005093 -0.019818 -0.044719\n", 316 | "2010-01-13 -0.005843 -0.008313 -0.021331 -0.016007 -0.051958\n", 317 | "2010-01-14 -0.021830 -0.005868 0.003487 0.006849 -0.245616\n", 318 | "2010-01-15 0.007725 -0.008432 0.003475 -0.013605 -0.174981\n", 319 | "2010-01-19 -0.019591 0.005952 -0.007792 0.012931 -0.314436\n", 320 | "2010-01-20 0.014770 -0.011834 0.003490 -0.020426 0.086685\n", 321 | "2010-01-21 -0.012842 -0.005988 -0.042609 -0.028671 0.695078\n", 322 | "2010-01-22 -0.045100 -0.043029 -0.054496 -0.059034 0.329997\n", 323 | "2010-01-25 -0.025431 -0.001799 0.019212 0.048479 -0.247066\n", 324 | "2010-01-26 0.041007 0.032432 0.043355 0.014506 -0.109939\n", 325 | "2010-01-27 0.035810 0.013962 0.013550 0.032172 -0.029182\n", 326 | "2010-01-28 0.028522 0.028399 0.004456 -0.012121 0.936091\n", 327 | "2010-01-29 -0.025210 -0.028452 -0.050577 -0.049956 -0.214903\n", 328 | "2010-02-01 -0.039655 -0.037037 0.021495 0.025830 -0.481986\n", 329 | "2010-02-02 0.010772 0.030411 0.023788 0.024281 0.447590\n", 330 | "2010-02-03 0.020426 0.012153 0.020554 0.021949 -0.247609\n", 331 | "2010-02-04 0.000000 -0.011149 -0.036778 -0.049828 0.440127\n", 332 | "2010-02-05 -0.045257 -0.036427 -0.046364 -0.013562 0.398661\n", 333 | "2010-02-08 0.010939 0.018902 0.037178 0.005500 -0.493039\n", 334 | "2010-02-09 0.008115 -0.008834 0.012868 0.016408 -0.095883\n", 335 | "2010-02-10 -0.005367 -0.007130 -0.010889 -0.018834 -0.117917\n", 336 | "2010-02-11 -0.010791 0.003591 -0.001835 0.021938 -0.112805\n", 337 | "2010-02-12 -0.007273 0.000000 -0.002757 -0.005367 0.066792\n", 338 | "2010-02-16 0.026557 0.017889 0.023963 0.017986 -0.099732\n", 339 | "... ... ... ... ... ...\n", 340 | "2017-03-20 -0.020408 -0.019623 -0.016064 -0.016026 -0.174761\n", 341 | "2017-03-21 -0.014423 -0.013611 -0.044898 -0.045603 1.240473\n", 342 | "2017-03-22 -0.042276 -0.039773 -0.006838 0.004266 -0.299843\n", 343 | "2017-03-23 -0.017827 -0.006762 -0.010327 -0.008496 0.001426\n", 344 | "2017-03-24 0.007779 0.000851 0.004348 -0.004284 -0.385298\n", 345 | "2017-03-27 -0.012864 -0.017857 -0.012121 -0.013769 0.318488\n", 346 | "2017-03-28 0.000000 0.021645 0.006135 0.016579 -0.000877\n", 347 | "2017-03-29 0.019114 -0.003390 0.013937 0.002575 -0.379223\n", 348 | "2017-03-30 -0.005115 -0.001701 -0.002577 0.000000 -0.217554\n", 349 | "2017-03-31 -0.000857 -0.005111 -0.000861 -0.003425 -0.046084\n", 350 | "2017-04-03 -0.001715 -0.003425 -0.027586 -0.017182 1.457979\n", 351 | "2017-04-04 -0.022337 -0.013746 0.000000 -0.006119 -0.389866\n", 352 | "2017-04-05 0.008787 0.001742 -0.002660 -0.009675 0.081061\n", 353 | "2017-04-06 -0.018293 -0.012174 -0.002667 0.000888 -0.165085\n", 354 | "2017-04-07 -0.000887 -0.004401 -0.000891 -0.003549 -0.201585\n", 355 | "2017-04-10 0.000000 0.003537 0.002676 0.001781 -0.104513\n", 356 | "2017-04-11 0.000888 -0.006167 -0.006228 0.002667 0.406434\n", 357 | "2017-04-12 0.000000 0.001773 0.002686 -0.004433 0.556067\n", 358 | "2017-04-13 -0.004437 -0.004425 -0.008036 -0.010686 -0.420555\n", 359 | "2017-04-17 0.000000 0.004444 0.002700 0.015302 0.364192\n", 360 | "2017-04-18 -0.006239 -0.010619 -0.005386 -0.012411 -0.202803\n", 361 | "2017-04-19 0.003587 0.008945 0.008123 0.004488 0.052284\n", 362 | "2017-04-20 0.005362 0.019504 0.006267 0.025022 0.332203\n", 363 | "2017-04-21 0.018667 -0.001739 0.007117 -0.011334 -0.478374\n", 364 | "2017-04-24 0.001745 0.001742 0.003534 0.007937 0.575051\n", 365 | "2017-04-25 -0.000871 0.000000 0.006162 0.004374 -0.126823\n", 366 | "2017-04-26 0.000000 0.017391 0.001750 0.010453 0.441220\n", 367 | "2017-04-27 0.016565 -0.001709 -0.009607 -0.011207 -0.028674\n", 368 | "2017-04-28 -0.012864 -0.014555 0.005291 0.000000 -0.175059\n", 369 | "2017-05-01 -0.001738 0.002606 0.000000 -0.004359 -0.224477\n", 370 | "\n", 371 | "[1844 rows x 5 columns]\n" 372 | ] 373 | } 374 | ], 375 | "source": [ 376 | "print(stock.pct_change())" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": 8, 382 | "metadata": {}, 383 | "outputs": [ 384 | { 385 | "name": "stdout", 386 | "output_type": "stream", 387 | "text": [ 388 | "Date\n", 389 | "2010-01-04 NaN\n", 390 | "2010-01-05 0.066148\n", 391 | "2010-01-06 0.037409\n", 392 | "2010-01-07 0.025506\n", 393 | "2010-01-08 0.002573\n", 394 | "2010-01-11 0.035928\n", 395 | "2010-01-12 -0.019818\n", 396 | "2010-01-13 -0.016007\n", 397 | "2010-01-14 0.006849\n", 398 | "2010-01-15 -0.013605\n", 399 | "2010-01-19 0.012931\n", 400 | "2010-01-20 -0.020426\n", 401 | "2010-01-21 -0.028671\n", 402 | "2010-01-22 -0.059034\n", 403 | "2010-01-25 0.048479\n", 404 | "2010-01-26 0.014506\n", 405 | "2010-01-27 0.032172\n", 406 | "2010-01-28 -0.012121\n", 407 | "2010-01-29 -0.049956\n", 408 | "2010-02-01 0.025830\n", 409 | "2010-02-02 0.024281\n", 410 | "2010-02-03 0.021949\n", 411 | "2010-02-04 -0.049828\n", 412 | "2010-02-05 -0.013562\n", 413 | "2010-02-08 0.005500\n", 414 | "2010-02-09 0.016408\n", 415 | "2010-02-10 -0.018834\n", 416 | "2010-02-11 0.021938\n", 417 | "2010-02-12 -0.005367\n", 418 | "2010-02-16 0.017986\n", 419 | " ... \n", 420 | "2017-03-20 -0.016026\n", 421 | "2017-03-21 -0.045603\n", 422 | "2017-03-22 0.004266\n", 423 | "2017-03-23 -0.008496\n", 424 | "2017-03-24 -0.004284\n", 425 | "2017-03-27 -0.013769\n", 426 | "2017-03-28 0.016579\n", 427 | "2017-03-29 0.002575\n", 428 | "2017-03-30 0.000000\n", 429 | "2017-03-31 -0.003425\n", 430 | "2017-04-03 -0.017182\n", 431 | "2017-04-04 -0.006119\n", 432 | "2017-04-05 -0.009675\n", 433 | "2017-04-06 0.000888\n", 434 | "2017-04-07 -0.003549\n", 435 | "2017-04-10 0.001781\n", 436 | "2017-04-11 0.002667\n", 437 | "2017-04-12 -0.004433\n", 438 | "2017-04-13 -0.010686\n", 439 | "2017-04-17 0.015302\n", 440 | "2017-04-18 -0.012411\n", 441 | "2017-04-19 0.004488\n", 442 | "2017-04-20 0.025022\n", 443 | "2017-04-21 -0.011334\n", 444 | "2017-04-24 0.007937\n", 445 | "2017-04-25 0.004374\n", 446 | "2017-04-26 0.010453\n", 447 | "2017-04-27 -0.011207\n", 448 | "2017-04-28 0.000000\n", 449 | "2017-05-01 -0.004359\n", 450 | "Name: Close, Length: 1844, dtype: float64\n" 451 | ] 452 | } 453 | ], 454 | "source": [ 455 | "df = stock.pct_change()[\"Close\"]\n", 456 | "print(df)" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": 9, 462 | "metadata": { 463 | "collapsed": true 464 | }, 465 | "outputs": [], 466 | "source": [ 467 | "df.dropna(inplace=True)" 468 | ] 469 | }, 470 | { 471 | "cell_type": "code", 472 | "execution_count": 10, 473 | "metadata": {}, 474 | "outputs": [ 475 | { 476 | "name": "stdout", 477 | "output_type": "stream", 478 | "text": [ 479 | " Open High Low Close Volume\n", 480 | "Date \n", 481 | "2010-01-04 112.37 113.39 111.51 113.33 118944541\n", 482 | "2010-01-05 113.26 113.68 112.85 113.63 111579866\n", 483 | "2010-01-06 113.52 113.99 113.43 113.71 116074402\n", 484 | "2010-01-07 113.50 114.33 113.18 114.19 131091048\n", 485 | "2010-01-08 113.89 114.62 113.66 114.57 126402764\n", 486 | "2010-01-11 115.08 115.13 114.24 114.73 106375678\n", 487 | "2010-01-12 113.97 114.21 113.22 113.66 163333432\n", 488 | "2010-01-13 113.95 114.94 113.37 114.62 161821927\n", 489 | "2010-01-14 114.49 115.14 114.42 114.93 115783759\n", 490 | "2010-01-15 NaN NaN NaN 113.64 0\n", 491 | "2010-01-19 113.62 115.13 113.59 115.06 139172635\n", 492 | "2010-01-20 114.28 114.45 112.98 113.89 216490121\n", 493 | "2010-01-21 113.92 114.27 111.56 111.70 344859506\n", 494 | "2010-01-22 111.20 111.74 109.09 109.21 345942402\n", 495 | "2010-01-25 110.21 110.41 109.41 109.77 186937468\n", 496 | "2010-01-26 109.34 110.47 109.04 109.31 211168719\n", 497 | "2010-01-27 109.17 110.08 108.33 109.83 271863580\n", 498 | "2010-01-28 110.19 110.25 107.91 108.57 316103993\n", 499 | "2010-01-29 109.04 109.80 107.22 107.39 310677592\n", 500 | "2010-02-01 108.15 109.07 107.50 109.06 187864905\n", 501 | "2010-02-02 109.26 110.59 108.88 110.38 216327851\n", 502 | "2010-02-03 109.88 110.48 109.51 109.83 172730603\n", 503 | "2010-02-04 108.98 109.03 106.42 106.44 356715612\n", 504 | "2010-02-05 106.56 106.88 104.58 106.66 493585758\n", 505 | "2010-02-08 106.74 107.33 105.81 105.89 224166833\n", 506 | "2010-02-09 107.13 108.15 106.27 107.22 337820408\n", 507 | "2010-02-10 107.05 107.60 106.11 107.01 240511495\n", 508 | "2010-02-11 106.87 108.25 106.25 108.13 223591548\n", 509 | "2010-02-12 NaN NaN NaN 108.04 0\n", 510 | "2010-02-16 108.86 109.85 107.82 109.74 159317418\n", 511 | "... ... ... ... ... ...\n", 512 | "2017-03-20 237.03 237.36 236.32 236.77 52536979\n", 513 | "2017-03-21 237.47 237.61 233.58 233.73 131809275\n", 514 | "2017-03-22 233.77 234.61 233.05 234.28 97569204\n", 515 | "2017-03-23 234.00 235.34 233.60 234.03 100410277\n", 516 | "2017-03-24 234.38 235.04 232.96 233.86 112504853\n", 517 | "2017-03-27 231.93 233.92 231.61 233.62 87454452\n", 518 | "2017-03-28 233.27 235.81 233.14 235.32 93483915\n", 519 | "2017-03-29 234.99 235.81 234.72 235.54 61950354\n", 520 | "2017-03-30 235.47 236.52 235.27 236.29 56737890\n", 521 | "2017-03-31 235.90 236.51 235.68 235.74 73733094\n", 522 | "2017-04-03 235.80 236.03 233.91 235.33 85546486\n", 523 | "2017-04-04 234.95 235.58 234.56 235.48 56466195\n", 524 | "2017-04-05 236.26 237.39 234.54 234.78 108800604\n", 525 | "2017-04-06 234.94 236.04 234.42 235.44 69135757\n", 526 | "2017-04-07 235.15 236.00 234.64 235.20 74412311\n", 527 | "2017-04-10 235.36 236.26 234.73 235.34 67615302\n", 528 | "2017-04-11 234.90 235.18 233.34 235.06 88045276\n", 529 | "2017-04-12 234.74 234.96 233.77 234.03 81864436\n", 530 | "2017-04-13 233.64 234.49 232.51 232.51 92880394\n", 531 | "2017-04-17 233.11 234.57 232.88 234.57 68405367\n", 532 | "2017-04-18 233.72 234.49 233.08 233.87 83225821\n", 533 | "2017-04-19 234.52 234.95 233.18 233.44 68699868\n", 534 | "2017-04-20 234.15 235.84 233.78 235.34 92572186\n", 535 | "2017-04-21 235.25 235.31 234.13 234.59 110389847\n", 536 | "2017-04-24 237.18 237.41 234.56 237.17 119209877\n", 537 | "2017-04-25 237.91 238.95 237.81 238.55 76698265\n", 538 | "2017-04-26 238.51 239.53 238.35 238.40 84702455\n", 539 | "2017-04-27 238.77 238.95 237.98 238.60 57410326\n", 540 | "2017-04-28 238.90 238.93 237.93 238.08 63532845\n", 541 | "2017-05-01 238.68 239.17 238.20 238.68 66882521\n", 542 | "\n", 543 | "[1843 rows x 5 columns]\n" 544 | ] 545 | } 546 | ], 547 | "source": [ 548 | "index = web.DataReader(\"SPY\", 'google', startDate, endDate)\n", 549 | "print(index)" 550 | ] 551 | }, 552 | { 553 | "cell_type": "code", 554 | "execution_count": 11, 555 | "metadata": {}, 556 | "outputs": [ 557 | { 558 | "name": "stdout", 559 | "output_type": "stream", 560 | "text": [ 561 | " DCOILWTICO\n", 562 | "DATE \n", 563 | "2010-01-01 NaN\n", 564 | "2010-01-04 81.52\n", 565 | "2010-01-05 81.74\n", 566 | "2010-01-06 83.12\n", 567 | "2010-01-07 82.60\n", 568 | "2010-01-08 82.74\n", 569 | "2010-01-11 82.54\n", 570 | "2010-01-12 80.79\n", 571 | "2010-01-13 79.66\n", 572 | "2010-01-14 79.35\n", 573 | "2010-01-15 77.96\n", 574 | "2010-01-18 NaN\n", 575 | "2010-01-19 78.98\n", 576 | "2010-01-20 77.42\n", 577 | "2010-01-21 75.84\n", 578 | "2010-01-22 74.25\n", 579 | "2010-01-25 74.90\n", 580 | "2010-01-26 74.67\n", 581 | "2010-01-27 73.64\n", 582 | "2010-01-28 73.62\n", 583 | "2010-01-29 72.85\n", 584 | "2010-02-01 74.41\n", 585 | "2010-02-02 77.21\n", 586 | "2010-02-03 76.96\n", 587 | "2010-02-04 73.13\n", 588 | "2010-02-05 71.15\n", 589 | "2010-02-08 71.87\n", 590 | "2010-02-09 73.71\n", 591 | "2010-02-10 74.48\n", 592 | "2010-02-11 75.23\n", 593 | "... ...\n", 594 | "2017-03-21 47.02\n", 595 | "2017-03-22 47.29\n", 596 | "2017-03-23 47.00\n", 597 | "2017-03-24 47.30\n", 598 | "2017-03-27 47.02\n", 599 | "2017-03-28 48.36\n", 600 | "2017-03-29 49.47\n", 601 | "2017-03-30 50.30\n", 602 | "2017-03-31 50.54\n", 603 | "2017-04-03 50.25\n", 604 | "2017-04-04 50.99\n", 605 | "2017-04-05 51.14\n", 606 | "2017-04-06 51.69\n", 607 | "2017-04-07 52.25\n", 608 | "2017-04-10 53.06\n", 609 | "2017-04-11 53.38\n", 610 | "2017-04-12 53.12\n", 611 | "2017-04-13 53.19\n", 612 | "2017-04-14 NaN\n", 613 | "2017-04-17 52.62\n", 614 | "2017-04-18 52.46\n", 615 | "2017-04-19 50.49\n", 616 | "2017-04-20 50.26\n", 617 | "2017-04-21 49.64\n", 618 | "2017-04-24 48.90\n", 619 | "2017-04-25 49.22\n", 620 | "2017-04-26 49.22\n", 621 | "2017-04-27 48.96\n", 622 | "2017-04-28 49.31\n", 623 | "2017-05-01 48.83\n", 624 | "\n", 625 | "[1912 rows x 1 columns]\n" 626 | ] 627 | } 628 | ], 629 | "source": [ 630 | "oil = web.DataReader(\"DCOILWTICO\", \"fred\", startDate, endDate)\n", 631 | "print(oil)" 632 | ] 633 | }, 634 | { 635 | "cell_type": "code", 636 | "execution_count": 12, 637 | "metadata": {}, 638 | "outputs": [ 639 | { 640 | "name": "stdout", 641 | "output_type": "stream", 642 | "text": [ 643 | " GOLDAMGBD228NLBM\n", 644 | "DATE \n", 645 | "2010-01-01 NaN\n", 646 | "2010-01-04 1113.00\n", 647 | "2010-01-05 1125.25\n", 648 | "2010-01-06 1125.00\n", 649 | "2010-01-07 1130.75\n", 650 | "2010-01-08 1121.75\n", 651 | "2010-01-11 1158.00\n", 652 | "2010-01-12 1152.75\n", 653 | "2010-01-13 1132.75\n", 654 | "2010-01-14 1137.50\n", 655 | "2010-01-15 1132.00\n", 656 | "2010-01-18 1135.75\n", 657 | "2010-01-19 1134.00\n", 658 | "2010-01-20 1129.00\n", 659 | "2010-01-21 1104.00\n", 660 | "2010-01-22 1096.50\n", 661 | "2010-01-25 1103.50\n", 662 | "2010-01-26 1090.75\n", 663 | "2010-01-27 1094.75\n", 664 | "2010-01-28 1091.75\n", 665 | "2010-01-29 1082.75\n", 666 | "2010-02-01 1082.00\n", 667 | "2010-02-02 1114.00\n", 668 | "2010-02-03 1118.50\n", 669 | "2010-02-04 1102.50\n", 670 | "2010-02-05 1052.25\n", 671 | "2010-02-08 1070.00\n", 672 | "2010-02-09 1068.00\n", 673 | "2010-02-10 1075.50\n", 674 | "2010-02-11 1079.50\n", 675 | "... ...\n", 676 | "2017-03-21 1232.05\n", 677 | "2017-03-22 1246.10\n", 678 | "2017-03-23 1247.90\n", 679 | "2017-03-24 1244.00\n", 680 | "2017-03-27 1256.90\n", 681 | "2017-03-28 1253.65\n", 682 | "2017-03-29 1252.90\n", 683 | "2017-03-30 1250.90\n", 684 | "2017-03-31 1241.70\n", 685 | "2017-04-03 1246.25\n", 686 | "2017-04-04 1258.65\n", 687 | "2017-04-05 1252.50\n", 688 | "2017-04-06 1253.75\n", 689 | "2017-04-07 1264.30\n", 690 | "2017-04-10 1253.60\n", 691 | "2017-04-11 1255.70\n", 692 | "2017-04-12 1272.30\n", 693 | "2017-04-13 1286.10\n", 694 | "2017-04-14 NaN\n", 695 | "2017-04-17 NaN\n", 696 | "2017-04-18 1285.00\n", 697 | "2017-04-19 1282.05\n", 698 | "2017-04-20 1279.90\n", 699 | "2017-04-21 1281.50\n", 700 | "2017-04-24 1271.80\n", 701 | "2017-04-25 1270.50\n", 702 | "2017-04-26 1264.95\n", 703 | "2017-04-27 1264.30\n", 704 | "2017-04-28 1265.55\n", 705 | "2017-05-01 NaN\n", 706 | "\n", 707 | "[1912 rows x 1 columns]\n" 708 | ] 709 | } 710 | ], 711 | "source": [ 712 | "gold = web.DataReader(\"GOLDAMGBD228NLBM\", \"fred\", startDate, endDate)\n", 713 | "print(gold)" 714 | ] 715 | }, 716 | { 717 | "cell_type": "code", 718 | "execution_count": 13, 719 | "metadata": {}, 720 | "outputs": [ 721 | { 722 | "name": "stdout", 723 | "output_type": "stream", 724 | "text": [ 725 | " DHHNGSP\n", 726 | "DATE \n", 727 | "2010-01-01 NaN\n", 728 | "2010-01-04 6.09\n", 729 | "2010-01-05 6.19\n", 730 | "2010-01-06 6.47\n", 731 | "2010-01-07 7.51\n", 732 | "2010-01-08 6.56\n", 733 | "2010-01-11 5.77\n", 734 | "2010-01-12 5.57\n", 735 | "2010-01-13 5.61\n", 736 | "2010-01-14 5.77\n", 737 | "2010-01-15 5.66\n", 738 | "2010-01-18 NaN\n", 739 | "2010-01-19 5.51\n", 740 | "2010-01-20 5.54\n", 741 | "2010-01-21 5.52\n", 742 | "2010-01-22 5.67\n", 743 | "2010-01-25 5.76\n", 744 | "2010-01-26 5.61\n", 745 | "2010-01-27 5.42\n", 746 | "2010-01-28 5.32\n", 747 | "2010-01-29 5.26\n", 748 | "2010-02-01 5.30\n", 749 | "2010-02-02 5.47\n", 750 | "2010-02-03 5.51\n", 751 | "2010-02-04 5.47\n", 752 | "2010-02-05 5.61\n", 753 | "2010-02-08 5.73\n", 754 | "2010-02-09 5.54\n", 755 | "2010-02-10 5.48\n", 756 | "2010-02-11 5.53\n", 757 | "... ...\n", 758 | "2017-03-21 3.09\n", 759 | "2017-03-22 3.02\n", 760 | "2017-03-23 2.93\n", 761 | "2017-03-24 2.97\n", 762 | "2017-03-27 2.94\n", 763 | "2017-03-28 2.95\n", 764 | "2017-03-29 3.03\n", 765 | "2017-03-30 3.08\n", 766 | "2017-03-31 3.13\n", 767 | "2017-04-03 3.13\n", 768 | "2017-04-04 3.06\n", 769 | "2017-04-05 3.22\n", 770 | "2017-04-06 3.23\n", 771 | "2017-04-07 3.23\n", 772 | "2017-04-10 3.15\n", 773 | "2017-04-11 3.11\n", 774 | "2017-04-12 2.97\n", 775 | "2017-04-13 3.00\n", 776 | "2017-04-14 3.00\n", 777 | "2017-04-17 3.09\n", 778 | "2017-04-18 3.11\n", 779 | "2017-04-19 3.11\n", 780 | "2017-04-20 3.13\n", 781 | "2017-04-21 3.06\n", 782 | "2017-04-24 3.06\n", 783 | "2017-04-25 2.99\n", 784 | "2017-04-26 3.06\n", 785 | "2017-04-27 3.13\n", 786 | "2017-04-28 3.20\n", 787 | "2017-05-01 3.20\n", 788 | "\n", 789 | "[1912 rows x 1 columns]\n" 790 | ] 791 | } 792 | ], 793 | "source": [ 794 | "naturalGas = web.DataReader(\"DHHNGSP\", \"fred\", startDate, endDate)\n", 795 | "print(naturalGas)" 796 | ] 797 | }, 798 | { 799 | "cell_type": "code", 800 | "execution_count": 14, 801 | "metadata": { 802 | "collapsed": true 803 | }, 804 | "outputs": [], 805 | "source": [ 806 | "index = index[\"Close\"].pct_change()\n", 807 | "oil = oil.pct_change()\n", 808 | "gold = gold.pct_change()\n", 809 | "naturalGas = naturalGas.pct_change()" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 15, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "name": "stdout", 819 | "output_type": "stream", 820 | "text": [ 821 | " Close DCOILWTICO GOLDAMGBD228NLBM DHHNGSP\n", 822 | "DATE \n", 823 | "2010-01-01 NaN NaN NaN NaN\n", 824 | "2010-01-04 NaN NaN NaN NaN\n", 825 | "2010-01-05 0.002647 0.002699 0.011006 0.016420\n", 826 | "2010-01-06 0.000704 0.016883 -0.000222 0.045234\n", 827 | "2010-01-07 0.004221 -0.006256 0.005111 0.160742\n", 828 | "2010-01-08 0.003328 0.001695 -0.007959 -0.126498\n", 829 | "2010-01-11 0.001397 -0.002417 0.032316 -0.120427\n", 830 | "2010-01-12 -0.009326 -0.021202 -0.004534 -0.034662\n", 831 | "2010-01-13 0.008446 -0.013987 -0.017350 0.007181\n", 832 | "2010-01-14 0.002705 -0.003892 0.004193 0.028520\n", 833 | "2010-01-15 -0.011224 -0.017517 -0.004835 -0.019064\n", 834 | "2010-01-18 NaN NaN 0.003313 NaN\n", 835 | "2010-01-19 0.012496 0.013084 -0.001541 -0.026502\n", 836 | "2010-01-20 -0.010169 -0.019752 -0.004409 0.005445\n", 837 | "2010-01-21 -0.019229 -0.020408 -0.022143 -0.003610\n", 838 | "2010-01-22 -0.022292 -0.020965 -0.006793 0.027174\n", 839 | "2010-01-25 0.005128 0.008754 0.006384 0.015873\n", 840 | "2010-01-26 -0.004191 -0.003071 -0.011554 -0.026042\n", 841 | "2010-01-27 0.004757 -0.013794 0.003667 -0.033868\n", 842 | "2010-01-28 -0.011472 -0.000272 -0.002740 -0.018450\n", 843 | "2010-01-29 -0.010869 -0.010459 -0.008244 -0.011278\n", 844 | "2010-02-01 0.015551 0.021414 -0.000693 0.007605\n", 845 | "2010-02-02 0.012103 0.037629 0.029575 0.032075\n", 846 | "2010-02-03 -0.004983 -0.003238 0.004039 0.007313\n", 847 | "2010-02-04 -0.030866 -0.049766 -0.014305 -0.007260\n", 848 | "2010-02-05 0.002067 -0.027075 -0.045578 0.025594\n", 849 | "2010-02-08 -0.007219 0.010119 0.016869 0.021390\n", 850 | "2010-02-09 0.012560 0.025602 -0.001869 -0.033159\n", 851 | "2010-02-10 -0.001959 0.010446 0.007022 -0.010830\n", 852 | "2010-02-11 0.010466 0.010070 0.003719 0.009124\n", 853 | "... ... ... ... ...\n", 854 | "2017-03-21 -0.012839 -0.016112 -0.000770 0.054608\n", 855 | "2017-03-22 0.002353 0.005742 0.011404 -0.022654\n", 856 | "2017-03-23 -0.001067 -0.006132 0.001445 -0.029801\n", 857 | "2017-03-24 -0.000726 0.006383 -0.003125 0.013652\n", 858 | "2017-03-27 -0.001026 -0.005920 0.010370 -0.010101\n", 859 | "2017-03-28 0.007277 0.028499 -0.002586 0.003401\n", 860 | "2017-03-29 0.000935 0.022953 -0.000598 0.027119\n", 861 | "2017-03-30 0.003184 0.016778 -0.001596 0.016502\n", 862 | "2017-03-31 -0.002328 0.004771 -0.007355 0.016234\n", 863 | "2017-04-03 -0.001739 -0.005738 0.003664 0.000000\n", 864 | "2017-04-04 0.000637 0.014726 0.009950 -0.022364\n", 865 | "2017-04-05 -0.002973 0.002942 -0.004886 0.052288\n", 866 | "2017-04-06 0.002811 0.010755 0.000998 0.003106\n", 867 | "2017-04-07 -0.001019 0.010834 0.008415 0.000000\n", 868 | "2017-04-10 0.000595 0.015502 -0.008463 -0.024768\n", 869 | "2017-04-11 -0.001190 0.006031 0.001675 -0.012698\n", 870 | "2017-04-12 -0.004382 -0.004871 0.013220 -0.045016\n", 871 | "2017-04-13 -0.006495 0.001318 0.010846 0.010101\n", 872 | "2017-04-14 NaN NaN NaN 0.000000\n", 873 | "2017-04-17 0.008860 -0.010716 NaN 0.030000\n", 874 | "2017-04-18 -0.002984 -0.003041 -0.000855 0.006472\n", 875 | "2017-04-19 -0.001839 -0.037552 -0.002296 0.000000\n", 876 | "2017-04-20 0.008139 -0.004555 -0.001677 0.006431\n", 877 | "2017-04-21 -0.003187 -0.012336 0.001250 -0.022364\n", 878 | "2017-04-24 0.010998 -0.014907 -0.007569 0.000000\n", 879 | "2017-04-25 0.005819 0.006544 -0.001022 -0.022876\n", 880 | "2017-04-26 -0.000629 0.000000 -0.004368 0.023411\n", 881 | "2017-04-27 0.000839 -0.005282 -0.000514 0.022876\n", 882 | "2017-04-28 -0.002179 0.007149 0.000989 0.022364\n", 883 | "2017-05-01 0.002520 -0.009734 NaN 0.000000\n", 884 | "\n", 885 | "[1912 rows x 4 columns]\n" 886 | ] 887 | } 888 | ], 889 | "source": [ 890 | "import pandas as pd\n", 891 | "df = pd.concat([index, oil,gold,naturalGas], axis=1)\n", 892 | "print(df)" 893 | ] 894 | }, 895 | { 896 | "cell_type": "code", 897 | "execution_count": 16, 898 | "metadata": {}, 899 | "outputs": [ 900 | { 901 | "name": "stdout", 902 | "output_type": "stream", 903 | "text": [ 904 | " Close DCOILWTICO GOLDAMGBD228NLBM DHHNGSP\n", 905 | "DATE \n", 906 | "2010-01-05 0.002647 0.002699 0.011006 0.016420\n", 907 | "2010-01-06 0.000704 0.016883 -0.000222 0.045234\n", 908 | "2010-01-07 0.004221 -0.006256 0.005111 0.160742\n", 909 | "2010-01-08 0.003328 0.001695 -0.007959 -0.126498\n", 910 | "2010-01-11 0.001397 -0.002417 0.032316 -0.120427\n", 911 | "2010-01-12 -0.009326 -0.021202 -0.004534 -0.034662\n", 912 | "2010-01-13 0.008446 -0.013987 -0.017350 0.007181\n", 913 | "2010-01-14 0.002705 -0.003892 0.004193 0.028520\n", 914 | "2010-01-15 -0.011224 -0.017517 -0.004835 -0.019064\n", 915 | "2010-01-19 0.012496 0.013084 -0.001541 -0.026502\n", 916 | "2010-01-20 -0.010169 -0.019752 -0.004409 0.005445\n", 917 | "2010-01-21 -0.019229 -0.020408 -0.022143 -0.003610\n", 918 | "2010-01-22 -0.022292 -0.020965 -0.006793 0.027174\n", 919 | "2010-01-25 0.005128 0.008754 0.006384 0.015873\n", 920 | "2010-01-26 -0.004191 -0.003071 -0.011554 -0.026042\n", 921 | "2010-01-27 0.004757 -0.013794 0.003667 -0.033868\n", 922 | "2010-01-28 -0.011472 -0.000272 -0.002740 -0.018450\n", 923 | "2010-01-29 -0.010869 -0.010459 -0.008244 -0.011278\n", 924 | "2010-02-01 0.015551 0.021414 -0.000693 0.007605\n", 925 | "2010-02-02 0.012103 0.037629 0.029575 0.032075\n", 926 | "2010-02-03 -0.004983 -0.003238 0.004039 0.007313\n", 927 | "2010-02-04 -0.030866 -0.049766 -0.014305 -0.007260\n", 928 | "2010-02-05 0.002067 -0.027075 -0.045578 0.025594\n", 929 | "2010-02-08 -0.007219 0.010119 0.016869 0.021390\n", 930 | "2010-02-09 0.012560 0.025602 -0.001869 -0.033159\n", 931 | "2010-02-10 -0.001959 0.010446 0.007022 -0.010830\n", 932 | "2010-02-11 0.010466 0.010070 0.003719 0.009124\n", 933 | "2010-02-12 -0.000832 -0.014888 -0.001158 -0.009042\n", 934 | "2010-02-16 0.015735 0.038726 0.014325 0.031022\n", 935 | "2010-02-17 0.004738 0.003767 0.002690 -0.031858\n", 936 | "... ... ... ... ...\n", 937 | "2017-03-16 -0.001967 -0.000827 0.019422 -0.059603\n", 938 | "2017-03-17 -0.006080 0.000828 0.002570 0.003521\n", 939 | "2017-03-20 -0.001097 -0.011378 0.003459 0.028070\n", 940 | "2017-03-21 -0.012839 -0.016112 -0.000770 0.054608\n", 941 | "2017-03-22 0.002353 0.005742 0.011404 -0.022654\n", 942 | "2017-03-23 -0.001067 -0.006132 0.001445 -0.029801\n", 943 | "2017-03-24 -0.000726 0.006383 -0.003125 0.013652\n", 944 | "2017-03-27 -0.001026 -0.005920 0.010370 -0.010101\n", 945 | "2017-03-28 0.007277 0.028499 -0.002586 0.003401\n", 946 | "2017-03-29 0.000935 0.022953 -0.000598 0.027119\n", 947 | "2017-03-30 0.003184 0.016778 -0.001596 0.016502\n", 948 | "2017-03-31 -0.002328 0.004771 -0.007355 0.016234\n", 949 | "2017-04-03 -0.001739 -0.005738 0.003664 0.000000\n", 950 | "2017-04-04 0.000637 0.014726 0.009950 -0.022364\n", 951 | "2017-04-05 -0.002973 0.002942 -0.004886 0.052288\n", 952 | "2017-04-06 0.002811 0.010755 0.000998 0.003106\n", 953 | "2017-04-07 -0.001019 0.010834 0.008415 0.000000\n", 954 | "2017-04-10 0.000595 0.015502 -0.008463 -0.024768\n", 955 | "2017-04-11 -0.001190 0.006031 0.001675 -0.012698\n", 956 | "2017-04-12 -0.004382 -0.004871 0.013220 -0.045016\n", 957 | "2017-04-13 -0.006495 0.001318 0.010846 0.010101\n", 958 | "2017-04-18 -0.002984 -0.003041 -0.000855 0.006472\n", 959 | "2017-04-19 -0.001839 -0.037552 -0.002296 0.000000\n", 960 | "2017-04-20 0.008139 -0.004555 -0.001677 0.006431\n", 961 | "2017-04-21 -0.003187 -0.012336 0.001250 -0.022364\n", 962 | "2017-04-24 0.010998 -0.014907 -0.007569 0.000000\n", 963 | "2017-04-25 0.005819 0.006544 -0.001022 -0.022876\n", 964 | "2017-04-26 -0.000629 0.000000 -0.004368 0.023411\n", 965 | "2017-04-27 0.000839 -0.005282 -0.000514 0.022876\n", 966 | "2017-04-28 -0.002179 0.007149 0.000989 0.022364\n", 967 | "\n", 968 | "[1807 rows x 4 columns]\n" 969 | ] 970 | } 971 | ], 972 | "source": [ 973 | "df.dropna(inplace=True)\n", 974 | "print(df)" 975 | ] 976 | }, 977 | { 978 | "cell_type": "code", 979 | "execution_count": 17, 980 | "metadata": { 981 | "collapsed": true 982 | }, 983 | "outputs": [], 984 | "source": [ 985 | "df.to_csv(\"Variables.csv\", encoding=\"UTF-8\")" 986 | ] 987 | } 988 | ], 989 | "metadata": { 990 | "anaconda-cloud": {}, 991 | "kernelspec": { 992 | "display_name": "Python 3", 993 | "language": "python", 994 | "name": "python3" 995 | }, 996 | "language_info": { 997 | "codemirror_mode": { 998 | "name": "ipython", 999 | "version": 3 1000 | }, 1001 | "file_extension": ".py", 1002 | "mimetype": "text/x-python", 1003 | "name": "python", 1004 | "nbconvert_exporter": "python", 1005 | "pygments_lexer": "ipython3", 1006 | "version": "3.6.1" 1007 | } 1008 | }, 1009 | "nbformat": 4, 1010 | "nbformat_minor": 1 1011 | } 1012 | -------------------------------------------------------------------------------- /3 S&P 500 Webscrape.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from lxml import html\n", 12 | "import requests" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "page = requests.get('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 3, 29 | "metadata": { 30 | "collapsed": true 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "tree = html.fromstring(page.content)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 4, 40 | "metadata": { 41 | "collapsed": true 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "table = tree.xpath('//*[@id=\"mw-content-text\"]/div/table[1]')" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 5, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "name": "stdout", 55 | "output_type": "stream", 56 | "text": [ 57 | "[]\n", 58 | "\n" 59 | ] 60 | } 61 | ], 62 | "source": [ 63 | "print(table)\n", 64 | "table = table[0]\n", 65 | "print(table)" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 6, 71 | "metadata": {}, 72 | "outputs": [ 73 | { 74 | "name": "stdout", 75 | "output_type": "stream", 76 | "text": [ 77 | "[, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ]\n" 78 | ] 79 | } 80 | ], 81 | "source": [ 82 | "rows = table.findall(\"tr\")\n", 83 | "print(rows)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 7, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "name": "stdout", 93 | "output_type": "stream", 94 | "text": [ 95 | "MMM\n", 96 | "ABT\n", 97 | "ABBV\n", 98 | "ACN\n", 99 | "ATVI\n", 100 | "AYI\n", 101 | "ADBE\n", 102 | "AMD\n", 103 | "AAP\n", 104 | "AES\n", 105 | "AET\n", 106 | "AMG\n", 107 | "AFL\n", 108 | "A\n", 109 | "APD\n", 110 | "AKAM\n", 111 | "ALK\n", 112 | "ALB\n", 113 | "ARE\n", 114 | "ALXN\n", 115 | "ALGN\n", 116 | "ALLE\n", 117 | "AGN\n", 118 | "ADS\n", 119 | "LNT\n", 120 | "ALL\n", 121 | "GOOGL\n", 122 | "GOOG\n", 123 | "MO\n", 124 | "AMZN\n", 125 | "AEE\n", 126 | "AAL\n", 127 | "AEP\n", 128 | "AXP\n", 129 | "AIG\n", 130 | "AMT\n", 131 | "AWK\n", 132 | "AMP\n", 133 | "ABC\n", 134 | "AME\n", 135 | "AMGN\n", 136 | "APH\n", 137 | "APC\n", 138 | "ADI\n", 139 | "ANSS\n", 140 | "ANTM\n", 141 | "AON\n", 142 | "APA\n", 143 | "AIV\n", 144 | "AAPL\n", 145 | "AMAT\n", 146 | "ADM\n", 147 | "ARNC\n", 148 | "AJG\n", 149 | "AIZ\n", 150 | "T\n", 151 | "ADSK\n", 152 | "ADP\n", 153 | "AN\n", 154 | "AZO\n", 155 | "AVB\n", 156 | "AVY\n", 157 | "BHGE\n", 158 | "BLL\n", 159 | "BAC\n", 160 | "BK\n", 161 | "BCR\n", 162 | "BAX\n", 163 | "BBT\n", 164 | "BDX\n", 165 | "BBBY\n", 166 | "BRK.B\n", 167 | "BBY\n", 168 | "BIIB\n", 169 | "BLK\n", 170 | "HRB\n", 171 | "BA\n", 172 | "BWA\n", 173 | "BXP\n", 174 | "BSX\n", 175 | "BMY\n", 176 | "AVGO\n", 177 | "BF.B\n", 178 | "CHRW\n", 179 | "CA\n", 180 | "COG\n", 181 | "CPB\n", 182 | "COF\n", 183 | "CAH\n", 184 | "CBOE\n", 185 | "KMX\n", 186 | "CCL\n", 187 | "CAT\n", 188 | "CBG\n", 189 | "CBS\n", 190 | "CELG\n", 191 | "CNC\n", 192 | "CNP\n", 193 | "CTL\n", 194 | "CERN\n", 195 | "CF\n", 196 | "SCHW\n", 197 | "CHTR\n", 198 | "CHK\n", 199 | "CVX\n", 200 | "CMG\n", 201 | "CB\n", 202 | "CHD\n", 203 | "CI\n", 204 | "XEC\n", 205 | "CINF\n", 206 | "CTAS\n", 207 | "CSCO\n", 208 | "C\n", 209 | "CFG\n", 210 | "CTXS\n", 211 | "CLX\n", 212 | "CME\n", 213 | "CMS\n", 214 | "COH\n", 215 | "KO\n", 216 | "CTSH\n", 217 | "CL\n", 218 | "CMCSA\n", 219 | "CMA\n", 220 | "CAG\n", 221 | "CXO\n", 222 | "COP\n", 223 | "ED\n", 224 | "STZ\n", 225 | "COO\n", 226 | "GLW\n", 227 | "COST\n", 228 | "COTY\n", 229 | "CCI\n", 230 | "CSRA\n", 231 | "CSX\n", 232 | "CMI\n", 233 | "CVS\n", 234 | "DHI\n", 235 | "DHR\n", 236 | "DRI\n", 237 | "DVA\n", 238 | "DE\n", 239 | "DLPH\n", 240 | "DAL\n", 241 | "XRAY\n", 242 | "DVN\n", 243 | "DLR\n", 244 | "DFS\n", 245 | "DISCA\n", 246 | "DISCK\n", 247 | "DISH\n", 248 | "DG\n", 249 | "DLTR\n", 250 | "D\n", 251 | "DOV\n", 252 | "DOW\n", 253 | "DPS\n", 254 | "DTE\n", 255 | "DD\n", 256 | "DUK\n", 257 | "DXC\n", 258 | "ETFC\n", 259 | "EMN\n", 260 | "ETN\n", 261 | "EBAY\n", 262 | "ECL\n", 263 | "EIX\n", 264 | "EW\n", 265 | "EA\n", 266 | "EMR\n", 267 | "ETR\n", 268 | "EVHC\n", 269 | "EOG\n", 270 | "EQT\n", 271 | "EFX\n", 272 | "EQIX\n", 273 | "EQR\n", 274 | "ESS\n", 275 | "EL\n", 276 | "ES\n", 277 | "RE\n", 278 | "EXC\n", 279 | "EXPE\n", 280 | "EXPD\n", 281 | "ESRX\n", 282 | "EXR\n", 283 | "XOM\n", 284 | "FFIV\n", 285 | "FB\n", 286 | "FAST\n", 287 | "FRT\n", 288 | "FDX\n", 289 | "FIS\n", 290 | "FITB\n", 291 | "FE\n", 292 | "FISV\n", 293 | "FLIR\n", 294 | "FLS\n", 295 | "FLR\n", 296 | "FMC\n", 297 | "FL\n", 298 | "F\n", 299 | "FTV\n", 300 | "FBHS\n", 301 | "BEN\n", 302 | "FCX\n", 303 | "GPS\n", 304 | "GRMN\n", 305 | "IT\n", 306 | "GD\n", 307 | "GE\n", 308 | "GGP\n", 309 | "GIS\n", 310 | "GM\n", 311 | "GPC\n", 312 | "GILD\n", 313 | "GPN\n", 314 | "GS\n", 315 | "GT\n", 316 | "GWW\n", 317 | "HAL\n", 318 | "HBI\n", 319 | "HOG\n", 320 | "HRS\n", 321 | "HIG\n", 322 | "HAS\n", 323 | "HCA\n", 324 | "HCP\n", 325 | "HP\n", 326 | "HSIC\n", 327 | "HSY\n", 328 | "HES\n", 329 | "HPE\n", 330 | "HLT\n", 331 | "HOLX\n", 332 | "HD\n", 333 | "HON\n", 334 | "HRL\n", 335 | "HST\n", 336 | "HPQ\n", 337 | "HUM\n", 338 | "HBAN\n", 339 | "IDXX\n", 340 | "INFO\n", 341 | "ITW\n", 342 | "ILMN\n", 343 | "IR\n", 344 | "INTC\n", 345 | "ICE\n", 346 | "IBM\n", 347 | "INCY\n", 348 | "IP\n", 349 | "IPG\n", 350 | "IFF\n", 351 | "INTU\n", 352 | "ISRG\n", 353 | "IVZ\n", 354 | "IRM\n", 355 | "JEC\n", 356 | "JBHT\n", 357 | "SJM\n", 358 | "JNJ\n", 359 | "JCI\n", 360 | "JPM\n", 361 | "JNPR\n", 362 | "KSU\n", 363 | "K\n", 364 | "KEY\n", 365 | "KMB\n", 366 | "KIM\n", 367 | "KMI\n", 368 | "KLAC\n", 369 | "KSS\n", 370 | "KHC\n", 371 | "KR\n", 372 | "LB\n", 373 | "LLL\n", 374 | "LH\n", 375 | "LRCX\n", 376 | "LEG\n", 377 | "LEN\n", 378 | "LVLT\n", 379 | "LUK\n", 380 | "LLY\n", 381 | "LNC\n", 382 | "LKQ\n", 383 | "LMT\n", 384 | "L\n", 385 | "LOW\n", 386 | "LYB\n", 387 | "MTB\n", 388 | "MAC\n", 389 | "M\n", 390 | "MNK\n", 391 | "MRO\n", 392 | "MPC\n", 393 | "MAR\n", 394 | "MMC\n", 395 | "MLM\n", 396 | "MAS\n", 397 | "MA\n", 398 | "MAT\n", 399 | "MKC\n", 400 | "MCD\n", 401 | "MCK\n", 402 | "MDT\n", 403 | "MRK\n", 404 | "MET\n", 405 | "MTD\n", 406 | "KORS\n", 407 | "MCHP\n", 408 | "MU\n", 409 | "MSFT\n", 410 | "MAA\n", 411 | "MHK\n", 412 | "TAP\n", 413 | "MDLZ\n", 414 | "MON\n", 415 | "MNST\n", 416 | "MCO\n", 417 | "MS\n", 418 | "MOS\n", 419 | "MSI\n", 420 | "MUR\n", 421 | "MYL\n", 422 | "NDAQ\n", 423 | "NOV\n", 424 | "NAVI\n", 425 | "NTAP\n", 426 | "NFLX\n", 427 | "NWL\n", 428 | "NFX\n", 429 | "NEM\n", 430 | "NWSA\n", 431 | "NWS\n", 432 | "NEE\n", 433 | "NLSN\n", 434 | "NKE\n", 435 | "NI\n", 436 | "NBL\n", 437 | "JWN\n", 438 | "NSC\n", 439 | "NTRS\n", 440 | "NOC\n", 441 | "NRG\n", 442 | "NUE\n", 443 | "NVDA\n", 444 | "ORLY\n", 445 | "OXY\n", 446 | "OMC\n", 447 | "OKE\n", 448 | "ORCL\n", 449 | "PCAR\n", 450 | "PH\n", 451 | "PDCO\n", 452 | "PAYX\n", 453 | "PYPL\n", 454 | "PNR\n", 455 | "PBCT\n", 456 | "PEP\n", 457 | "PKI\n", 458 | "PRGO\n", 459 | "PFE\n", 460 | "PCG\n", 461 | "PM\n", 462 | "PSX\n", 463 | "PNW\n", 464 | "PXD\n", 465 | "PNC\n", 466 | "RL\n", 467 | "PPG\n", 468 | "PPL\n", 469 | "PX\n", 470 | "PCLN\n", 471 | "PFG\n", 472 | "PG\n", 473 | "PGR\n", 474 | "PLD\n", 475 | "PRU\n", 476 | "PEG\n", 477 | "PSA\n", 478 | "PHM\n", 479 | "PVH\n", 480 | "QRVO\n", 481 | "PWR\n", 482 | "QCOM\n", 483 | "DGX\n", 484 | "RRC\n", 485 | "RJF\n", 486 | "RTN\n", 487 | "O\n", 488 | "RHT\n", 489 | "REG\n", 490 | "REGN\n", 491 | "RF\n", 492 | "RSG\n", 493 | "RAI\n", 494 | "RHI\n", 495 | "ROK\n", 496 | "COL\n", 497 | "ROP\n", 498 | "ROST\n", 499 | "RCL\n", 500 | "CRM\n", 501 | "SCG\n", 502 | "SLB\n", 503 | "SNI\n", 504 | "STX\n", 505 | "SEE\n", 506 | "SRE\n", 507 | "SHW\n", 508 | "SIG\n", 509 | "SPG\n", 510 | "SWKS\n", 511 | "SLG\n", 512 | "SNA\n", 513 | "SO\n", 514 | "LUV\n", 515 | "SPGI\n", 516 | "SWK\n", 517 | "SPLS\n", 518 | "SBUX\n", 519 | "STT\n", 520 | "SRCL\n", 521 | "SYK\n", 522 | "STI\n", 523 | "SYMC\n", 524 | "SYF\n", 525 | "SNPS\n", 526 | "SYY\n", 527 | "TROW\n", 528 | "TGT\n", 529 | "TEL\n", 530 | "FTI\n", 531 | "TSO\n", 532 | "TXN\n", 533 | "TXT\n", 534 | "TMO\n", 535 | "TIF\n", 536 | "TWX\n", 537 | "TJX\n", 538 | "TMK\n", 539 | "TSS\n", 540 | "TSCO\n", 541 | "TDG\n", 542 | "RIG\n", 543 | "TRV\n", 544 | "TRIP\n", 545 | "FOXA\n", 546 | "FOX\n", 547 | "TSN\n", 548 | "UDR\n", 549 | "ULTA\n", 550 | "USB\n", 551 | "UA\n", 552 | "UAA\n", 553 | "UNP\n", 554 | "UAL\n", 555 | "UNH\n", 556 | "UPS\n", 557 | "URI\n", 558 | "UTX\n", 559 | "UHS\n", 560 | "UNM\n", 561 | "VFC\n", 562 | "VLO\n", 563 | "VAR\n", 564 | "VTR\n", 565 | "VRSN\n", 566 | "VRSK\n", 567 | "VZ\n", 568 | "VRTX\n", 569 | "VIAB\n", 570 | "V\n", 571 | "VNO\n", 572 | "VMC\n", 573 | "WMT\n", 574 | "WBA\n", 575 | "DIS\n", 576 | "WM\n", 577 | "WAT\n", 578 | "WEC\n", 579 | "WFC\n", 580 | "HCN\n", 581 | "WDC\n", 582 | "WU\n", 583 | "WRK\n", 584 | "WY\n", 585 | "WHR\n", 586 | "WFM\n", 587 | "WMB\n", 588 | "WLTW\n", 589 | "WYN\n", 590 | "WYNN\n", 591 | "XEL\n", 592 | "XRX\n", 593 | "XLNX\n", 594 | "XL\n", 595 | "XYL\n", 596 | "YUM\n", 597 | "ZBH\n", 598 | "ZION\n", 599 | "ZTS\n" 600 | ] 601 | } 602 | ], 603 | "source": [ 604 | "for x in rows:\n", 605 | " firstCell = x.find(\"td\")\n", 606 | " if firstCell!=None:\n", 607 | " print(firstCell.text_content())" 608 | ] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "execution_count": 8, 613 | "metadata": {}, 614 | "outputs": [ 615 | { 616 | "name": "stdout", 617 | "output_type": "stream", 618 | "text": [ 619 | "['MMM', 'ABT', 'ABBV', 'ACN', 'ATVI', 'AYI', 'ADBE', 'AMD', 'AAP', 'AES', 'AET', 'AMG', 'AFL', 'A', 'APD', 'AKAM', 'ALK', 'ALB', 'ARE', 'ALXN', 'ALGN', 'ALLE', 'AGN', 'ADS', 'LNT', 'ALL', 'GOOGL', 'GOOG', 'MO', 'AMZN', 'AEE', 'AAL', 'AEP', 'AXP', 'AIG', 'AMT', 'AWK', 'AMP', 'ABC', 'AME', 'AMGN', 'APH', 'APC', 'ADI', 'ANSS', 'ANTM', 'AON', 'APA', 'AIV', 'AAPL', 'AMAT', 'ADM', 'ARNC', 'AJG', 'AIZ', 'T', 'ADSK', 'ADP', 'AN', 'AZO', 'AVB', 'AVY', 'BHGE', 'BLL', 'BAC', 'BK', 'BCR', 'BAX', 'BBT', 'BDX', 'BBBY', 'BRK.B', 'BBY', 'BIIB', 'BLK', 'HRB', 'BA', 'BWA', 'BXP', 'BSX', 'BMY', 'AVGO', 'BF.B', 'CHRW', 'CA', 'COG', 'CPB', 'COF', 'CAH', 'CBOE', 'KMX', 'CCL', 'CAT', 'CBG', 'CBS', 'CELG', 'CNC', 'CNP', 'CTL', 'CERN', 'CF', 'SCHW', 'CHTR', 'CHK', 'CVX', 'CMG', 'CB', 'CHD', 'CI', 'XEC', 'CINF', 'CTAS', 'CSCO', 'C', 'CFG', 'CTXS', 'CLX', 'CME', 'CMS', 'COH', 'KO', 'CTSH', 'CL', 'CMCSA', 'CMA', 'CAG', 'CXO', 'COP', 'ED', 'STZ', 'COO', 'GLW', 'COST', 'COTY', 'CCI', 'CSRA', 'CSX', 'CMI', 'CVS', 'DHI', 'DHR', 'DRI', 'DVA', 'DE', 'DLPH', 'DAL', 'XRAY', 'DVN', 'DLR', 'DFS', 'DISCA', 'DISCK', 'DISH', 'DG', 'DLTR', 'D', 'DOV', 'DOW', 'DPS', 'DTE', 'DD', 'DUK', 'DXC', 'ETFC', 'EMN', 'ETN', 'EBAY', 'ECL', 'EIX', 'EW', 'EA', 'EMR', 'ETR', 'EVHC', 'EOG', 'EQT', 'EFX', 'EQIX', 'EQR', 'ESS', 'EL', 'ES', 'RE', 'EXC', 'EXPE', 'EXPD', 'ESRX', 'EXR', 'XOM', 'FFIV', 'FB', 'FAST', 'FRT', 'FDX', 'FIS', 'FITB', 'FE', 'FISV', 'FLIR', 'FLS', 'FLR', 'FMC', 'FL', 'F', 'FTV', 'FBHS', 'BEN', 'FCX', 'GPS', 'GRMN', 'IT', 'GD', 'GE', 'GGP', 'GIS', 'GM', 'GPC', 'GILD', 'GPN', 'GS', 'GT', 'GWW', 'HAL', 'HBI', 'HOG', 'HRS', 'HIG', 'HAS', 'HCA', 'HCP', 'HP', 'HSIC', 'HSY', 'HES', 'HPE', 'HLT', 'HOLX', 'HD', 'HON', 'HRL', 'HST', 'HPQ', 'HUM', 'HBAN', 'IDXX', 'INFO', 'ITW', 'ILMN', 'IR', 'INTC', 'ICE', 'IBM', 'INCY', 'IP', 'IPG', 'IFF', 'INTU', 'ISRG', 'IVZ', 'IRM', 'JEC', 'JBHT', 'SJM', 'JNJ', 'JCI', 'JPM', 'JNPR', 'KSU', 'K', 'KEY', 'KMB', 'KIM', 'KMI', 'KLAC', 'KSS', 'KHC', 'KR', 'LB', 'LLL', 'LH', 'LRCX', 'LEG', 'LEN', 'LVLT', 'LUK', 'LLY', 'LNC', 'LKQ', 'LMT', 'L', 'LOW', 'LYB', 'MTB', 'MAC', 'M', 'MNK', 'MRO', 'MPC', 'MAR', 'MMC', 'MLM', 'MAS', 'MA', 'MAT', 'MKC', 'MCD', 'MCK', 'MDT', 'MRK', 'MET', 'MTD', 'KORS', 'MCHP', 'MU', 'MSFT', 'MAA', 'MHK', 'TAP', 'MDLZ', 'MON', 'MNST', 'MCO', 'MS', 'MOS', 'MSI', 'MUR', 'MYL', 'NDAQ', 'NOV', 'NAVI', 'NTAP', 'NFLX', 'NWL', 'NFX', 'NEM', 'NWSA', 'NWS', 'NEE', 'NLSN', 'NKE', 'NI', 'NBL', 'JWN', 'NSC', 'NTRS', 'NOC', 'NRG', 'NUE', 'NVDA', 'ORLY', 'OXY', 'OMC', 'OKE', 'ORCL', 'PCAR', 'PH', 'PDCO', 'PAYX', 'PYPL', 'PNR', 'PBCT', 'PEP', 'PKI', 'PRGO', 'PFE', 'PCG', 'PM', 'PSX', 'PNW', 'PXD', 'PNC', 'RL', 'PPG', 'PPL', 'PX', 'PCLN', 'PFG', 'PG', 'PGR', 'PLD', 'PRU', 'PEG', 'PSA', 'PHM', 'PVH', 'QRVO', 'PWR', 'QCOM', 'DGX', 'RRC', 'RJF', 'RTN', 'O', 'RHT', 'REG', 'REGN', 'RF', 'RSG', 'RAI', 'RHI', 'ROK', 'COL', 'ROP', 'ROST', 'RCL', 'CRM', 'SCG', 'SLB', 'SNI', 'STX', 'SEE', 'SRE', 'SHW', 'SIG', 'SPG', 'SWKS', 'SLG', 'SNA', 'SO', 'LUV', 'SPGI', 'SWK', 'SPLS', 'SBUX', 'STT', 'SRCL', 'SYK', 'STI', 'SYMC', 'SYF', 'SNPS', 'SYY', 'TROW', 'TGT', 'TEL', 'FTI', 'TSO', 'TXN', 'TXT', 'TMO', 'TIF', 'TWX', 'TJX', 'TMK', 'TSS', 'TSCO', 'TDG', 'RIG', 'TRV', 'TRIP', 'FOXA', 'FOX', 'TSN', 'UDR', 'ULTA', 'USB', 'UA', 'UAA', 'UNP', 'UAL', 'UNH', 'UPS', 'URI', 'UTX', 'UHS', 'UNM', 'VFC', 'VLO', 'VAR', 'VTR', 'VRSN', 'VRSK', 'VZ', 'VRTX', 'VIAB', 'V', 'VNO', 'VMC', 'WMT', 'WBA', 'DIS', 'WM', 'WAT', 'WEC', 'WFC', 'HCN', 'WDC', 'WU', 'WRK', 'WY', 'WHR', 'WFM', 'WMB', 'WLTW', 'WYN', 'WYNN', 'XEL', 'XRX', 'XLNX', 'XL', 'XYL', 'YUM', 'ZBH', 'ZION', 'ZTS']\n" 620 | ] 621 | } 622 | ], 623 | "source": [ 624 | "stocksArray = []\n", 625 | "for x in rows:\n", 626 | " firstCell = x.find(\"td\")\n", 627 | " if firstCell!=None:\n", 628 | " stocksArray.append(firstCell.text_content())\n", 629 | "print(stocksArray)" 630 | ] 631 | }, 632 | { 633 | "cell_type": "code", 634 | "execution_count": 9, 635 | "metadata": { 636 | "collapsed": true 637 | }, 638 | "outputs": [], 639 | "source": [ 640 | "import csv\n", 641 | "with open(\"stocksArray.csv\", \"wt\") as f:\n", 642 | " writer = csv.writer(f)\n", 643 | " writer.writerow(stocksArray)" 644 | ] 645 | }, 646 | { 647 | "cell_type": "code", 648 | "execution_count": 10, 649 | "metadata": { 650 | "collapsed": true 651 | }, 652 | "outputs": [], 653 | "source": [ 654 | "import pandas as pd\n", 655 | "table = tree.xpath('//*[@id=\"mw-content-text\"]/div/table[1]')\n", 656 | "table = table[0]\n", 657 | "rows = table.findall(\"tr\")\n", 658 | "rows = rows[1:]\n", 659 | "cellsAr = []\n", 660 | "for x in rows:\n", 661 | " cells = x.findall(\"td\")\n", 662 | " cells = [x.text_content() for x in cells]\n", 663 | " cellsAr.append(cells)\n", 664 | "df = pd.DataFrame(cellsAr)\n", 665 | "df.columns = [\"Ticker\",\"Security\",\"SEC Filings\",\"GICS Sector\",\"GICS Sub Industry\",\"Address\",\"Date Added\",\"CIK\"]\n", 666 | "df.to_csv(\"SP500.csv\",encoding=\"UTF-8\")" 667 | ] 668 | } 669 | ], 670 | "metadata": { 671 | "kernelspec": { 672 | "display_name": "Python 3", 673 | "language": "python", 674 | "name": "python3" 675 | }, 676 | "language_info": { 677 | "codemirror_mode": { 678 | "name": "ipython", 679 | "version": 3 680 | }, 681 | "file_extension": ".py", 682 | "mimetype": "text/x-python", 683 | "name": "python", 684 | "nbconvert_exporter": "python", 685 | "pygments_lexer": "ipython3", 686 | "version": "3.6.1" 687 | } 688 | }, 689 | "nbformat": 4, 690 | "nbformat_minor": 2 691 | } 692 | -------------------------------------------------------------------------------- /4 Full Dataset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd\n", 12 | "variables = pd.DataFrame.from_csv(\"Variables.csv\", encoding=\"UTF-8\")\n", 13 | "variables.columns = [\"Index\",\"Oil\",\"Gold\",\"NaturalGas\"]" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "metadata": { 20 | "collapsed": true 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "import csv\n", 25 | "with open(\"stocksArray.csv\", 'rt') as f:\n", 26 | " reader = csv.reader(f)\n", 27 | " for row in reader:\n", 28 | " stocksArray = row" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 3, 34 | "metadata": { 35 | "collapsed": true 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "temp = []\n", 40 | "for x in stocksArray:\n", 41 | " if x == \"LMT\":\n", 42 | " x = \"NYSE:LMT\"\n", 43 | " if x ==\"NWL\":\n", 44 | " x = \"NYSE:NWL\"\n", 45 | " if x==\"NBL\":\n", 46 | " x = \"NYSE:NBL\"\n", 47 | " temp.append(x)\n", 48 | "stocksArray = temp" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 4, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "MMM\n", 61 | "ABT\n", 62 | "ABBV\n", 63 | "ACN\n", 64 | "ATVI\n", 65 | "AYI\n", 66 | "ADBE\n", 67 | "AMD\n", 68 | "AAP\n", 69 | "AES\n", 70 | "AET\n", 71 | "AMG\n", 72 | "AFL\n", 73 | "A\n", 74 | "APD\n", 75 | "AKAM\n", 76 | "ALK\n", 77 | "ALB\n", 78 | "ARE\n", 79 | "ALXN\n", 80 | "ALGN\n", 81 | "ALLE\n", 82 | "AGN\n", 83 | "ADS\n", 84 | "LNT\n", 85 | "ALL\n", 86 | "GOOGL\n", 87 | "GOOG\n", 88 | "MO\n", 89 | "AMZN\n", 90 | "AEE\n", 91 | "AAL\n", 92 | "AEP\n", 93 | "AXP\n", 94 | "AIG\n", 95 | "AMT\n", 96 | "AWK\n", 97 | "AMP\n", 98 | "ABC\n", 99 | "AME\n", 100 | "AMGN\n", 101 | "APH\n", 102 | "APC\n", 103 | "ADI\n", 104 | "ANSS\n", 105 | "ANTM\n", 106 | "AON\n", 107 | "APA\n", 108 | "AIV\n", 109 | "AAPL\n", 110 | "AMAT\n", 111 | "ADM\n", 112 | "ARNC\n", 113 | "AJG\n", 114 | "AIZ\n", 115 | "T\n", 116 | "ADSK\n", 117 | "ADP\n", 118 | "AN\n", 119 | "AZO\n", 120 | "AVB\n", 121 | "AVY\n", 122 | "BHGE\n", 123 | "BLL\n", 124 | "BAC\n", 125 | "BK\n", 126 | "BCR\n", 127 | "BAX\n", 128 | "BBT\n", 129 | "BDX\n", 130 | "BBBY\n", 131 | "BRK.B\n", 132 | "BBY\n", 133 | "BIIB\n", 134 | "BLK\n", 135 | "HRB\n", 136 | "BA\n", 137 | "BWA\n", 138 | "BXP\n", 139 | "BSX\n", 140 | "BMY\n", 141 | "AVGO\n", 142 | "BF.B\n", 143 | "CHRW\n", 144 | "CA\n", 145 | "COG\n", 146 | "CPB\n", 147 | "COF\n", 148 | "CAH\n", 149 | "CBOE\n", 150 | "KMX\n", 151 | "CCL\n", 152 | "CAT\n", 153 | "CBG\n", 154 | "CBS\n", 155 | "CELG\n", 156 | "CNC\n", 157 | "CNP\n", 158 | "CTL\n", 159 | "CERN\n", 160 | "CF\n", 161 | "SCHW\n", 162 | "CHTR\n", 163 | "CHK\n", 164 | "CVX\n", 165 | "CMG\n", 166 | "CB\n", 167 | "CHD\n", 168 | "CI\n", 169 | "XEC\n", 170 | "CINF\n", 171 | "CTAS\n", 172 | "CSCO\n", 173 | "C\n", 174 | "CFG\n", 175 | "CTXS\n", 176 | "CLX\n", 177 | "CME\n", 178 | "CMS\n", 179 | "COH\n", 180 | "KO\n", 181 | "CTSH\n", 182 | "CL\n", 183 | "CMCSA\n", 184 | "CMA\n", 185 | "CAG\n", 186 | "CXO\n", 187 | "COP\n", 188 | "ED\n", 189 | "STZ\n", 190 | "COO\n", 191 | "GLW\n", 192 | "COST\n", 193 | "COTY\n", 194 | "CCI\n", 195 | "CSRA\n", 196 | "CSX\n", 197 | "CMI\n", 198 | "CVS\n", 199 | "DHI\n", 200 | "DHR\n", 201 | "DRI\n", 202 | "DVA\n", 203 | "DE\n", 204 | "DLPH\n", 205 | "DAL\n", 206 | "XRAY\n", 207 | "DVN\n", 208 | "DLR\n", 209 | "DFS\n", 210 | "DISCA\n", 211 | "DISCK\n", 212 | "DISH\n", 213 | "DG\n", 214 | "DLTR\n", 215 | "D\n", 216 | "DOV\n", 217 | "DOW\n", 218 | "DPS\n", 219 | "DTE\n", 220 | "DD\n", 221 | "DUK\n", 222 | "DXC\n", 223 | "ETFC\n", 224 | "EMN\n", 225 | "ETN\n", 226 | "EBAY\n", 227 | "ECL\n", 228 | "EIX\n", 229 | "EW\n", 230 | "EA\n", 231 | "EMR\n", 232 | "ETR\n", 233 | "EVHC\n", 234 | "EOG\n", 235 | "EQT\n", 236 | "EFX\n", 237 | "EQIX\n", 238 | "EQR\n", 239 | "ESS\n", 240 | "EL\n", 241 | "ES\n", 242 | "RE\n", 243 | "EXC\n", 244 | "EXPE\n", 245 | "EXPD\n", 246 | "ESRX\n", 247 | "EXR\n", 248 | "XOM\n", 249 | "FFIV\n", 250 | "FB\n", 251 | "FAST\n", 252 | "FRT\n", 253 | "FDX\n", 254 | "FIS\n", 255 | "FITB\n", 256 | "FE\n", 257 | "FISV\n", 258 | "FLIR\n", 259 | "FLS\n", 260 | "FLR\n", 261 | "FMC\n", 262 | "FL\n", 263 | "F\n", 264 | "FTV\n", 265 | "FBHS\n", 266 | "BEN\n", 267 | "FCX\n", 268 | "GPS\n", 269 | "GRMN\n", 270 | "IT\n", 271 | "GD\n", 272 | "GE\n", 273 | "GGP\n", 274 | "GIS\n", 275 | "GM\n", 276 | "GPC\n", 277 | "GILD\n", 278 | "GPN\n", 279 | "GS\n", 280 | "GT\n", 281 | "GWW\n", 282 | "HAL\n", 283 | "HBI\n", 284 | "HOG\n", 285 | "HRS\n", 286 | "HIG\n", 287 | "HAS\n", 288 | "HCA\n", 289 | "HCP\n", 290 | "HP\n", 291 | "HSIC\n", 292 | "HSY\n", 293 | "HES\n", 294 | "HPE\n", 295 | "HLT\n", 296 | "HOLX\n", 297 | "HD\n", 298 | "HON\n", 299 | "HRL\n", 300 | "HST\n", 301 | "HPQ\n", 302 | "HUM\n", 303 | "HBAN\n", 304 | "IDXX\n", 305 | "INFO\n", 306 | "ITW\n", 307 | "ILMN\n", 308 | "IR\n", 309 | "INTC\n", 310 | "ICE\n", 311 | "IBM\n", 312 | "INCY\n", 313 | "IP\n", 314 | "IPG\n", 315 | "IFF\n", 316 | "INTU\n", 317 | "ISRG\n", 318 | "IVZ\n", 319 | "IRM\n", 320 | "JEC\n", 321 | "JBHT\n", 322 | "SJM\n", 323 | "JNJ\n", 324 | "JCI\n", 325 | "JPM\n", 326 | "JNPR\n", 327 | "KSU\n", 328 | "K\n", 329 | "KEY\n", 330 | "KMB\n", 331 | "KIM\n", 332 | "KMI\n", 333 | "KLAC\n", 334 | "KSS\n", 335 | "KHC\n", 336 | "KR\n", 337 | "LB\n", 338 | "LLL\n", 339 | "LH\n", 340 | "LRCX\n", 341 | "LEG\n", 342 | "LEN\n", 343 | "LVLT\n", 344 | "LUK\n", 345 | "LLY\n", 346 | "LNC\n", 347 | "LKQ\n", 348 | "NYSE:LMT\n", 349 | "L\n", 350 | "LOW\n", 351 | "LYB\n", 352 | "MTB\n", 353 | "MAC\n", 354 | "M\n", 355 | "MNK\n", 356 | "MRO\n", 357 | "MPC\n", 358 | "MAR\n", 359 | "MMC\n", 360 | "MLM\n", 361 | "MAS\n", 362 | "MA\n", 363 | "MAT\n", 364 | "MKC\n", 365 | "MCD\n", 366 | "MCK\n", 367 | "MDT\n", 368 | "MRK\n", 369 | "MET\n", 370 | "MTD\n", 371 | "KORS\n", 372 | "MCHP\n", 373 | "MU\n", 374 | "MSFT\n", 375 | "MAA\n", 376 | "MHK\n", 377 | "TAP\n", 378 | "MDLZ\n", 379 | "MON\n", 380 | "MNST\n", 381 | "MCO\n", 382 | "MS\n", 383 | "MOS\n", 384 | "MSI\n", 385 | "MUR\n", 386 | "MYL\n", 387 | "NDAQ\n", 388 | "NOV\n", 389 | "NAVI\n", 390 | "NTAP\n", 391 | "NFLX\n", 392 | "NYSE:NWL\n", 393 | "NFX\n", 394 | "NEM\n", 395 | "NWSA\n", 396 | "NWS\n", 397 | "NEE\n", 398 | "NLSN\n", 399 | "NKE\n", 400 | "NI\n", 401 | "NYSE:NBL\n", 402 | "JWN\n", 403 | "NSC\n", 404 | "NTRS\n", 405 | "NOC\n", 406 | "NRG\n", 407 | "NUE\n", 408 | "NVDA\n", 409 | "ORLY\n", 410 | "OXY\n", 411 | "OMC\n", 412 | "OKE\n", 413 | "ORCL\n", 414 | "PCAR\n", 415 | "PH\n", 416 | "PDCO\n", 417 | "PAYX\n", 418 | "PYPL\n", 419 | "PNR\n", 420 | "PBCT\n", 421 | "PEP\n", 422 | "PKI\n", 423 | "PRGO\n", 424 | "PFE\n", 425 | "PCG\n", 426 | "PM\n", 427 | "PSX\n", 428 | "PNW\n", 429 | "PXD\n", 430 | "PNC\n", 431 | "RL\n", 432 | "PPG\n", 433 | "PPL\n", 434 | "PX\n", 435 | "PCLN\n", 436 | "PFG\n", 437 | "PG\n", 438 | "PGR\n", 439 | "PLD\n", 440 | "PRU\n", 441 | "PEG\n", 442 | "PSA\n", 443 | "PHM\n", 444 | "PVH\n", 445 | "QRVO\n", 446 | "PWR\n", 447 | "QCOM\n", 448 | "DGX\n", 449 | "RRC\n", 450 | "RJF\n", 451 | "RTN\n", 452 | "O\n", 453 | "RHT\n", 454 | "REG\n", 455 | "REGN\n", 456 | "RF\n", 457 | "RSG\n", 458 | "RAI\n", 459 | "RHI\n", 460 | "ROK\n", 461 | "COL\n", 462 | "ROP\n", 463 | "ROST\n", 464 | "RCL\n", 465 | "CRM\n", 466 | "SCG\n", 467 | "SLB\n", 468 | "SNI\n", 469 | "STX\n", 470 | "SEE\n", 471 | "SRE\n", 472 | "SHW\n", 473 | "SIG\n", 474 | "SPG\n", 475 | "SWKS\n", 476 | "SLG\n", 477 | "SNA\n", 478 | "SO\n", 479 | "LUV\n", 480 | "SPGI\n", 481 | "SWK\n", 482 | "SPLS\n", 483 | "SBUX\n", 484 | "STT\n", 485 | "SRCL\n", 486 | "SYK\n", 487 | "STI\n", 488 | "SYMC\n", 489 | "SYF\n", 490 | "SNPS\n", 491 | "SYY\n", 492 | "TROW\n", 493 | "TGT\n", 494 | "TEL\n", 495 | "FTI\n", 496 | "TSO\n", 497 | "TXN\n", 498 | "TXT\n", 499 | "TMO\n", 500 | "TIF\n", 501 | "TWX\n", 502 | "TJX\n", 503 | "TMK\n", 504 | "TSS\n", 505 | "TSCO\n", 506 | "TDG\n", 507 | "RIG\n", 508 | "TRV\n", 509 | "TRIP\n", 510 | "FOXA\n", 511 | "FOX\n", 512 | "TSN\n", 513 | "UDR\n", 514 | "ULTA\n", 515 | "USB\n", 516 | "UA\n", 517 | "UAA\n", 518 | "UNP\n", 519 | "UAL\n", 520 | "UNH\n", 521 | "UPS\n", 522 | "URI\n", 523 | "UTX\n", 524 | "UHS\n", 525 | "UNM\n", 526 | "VFC\n", 527 | "VLO\n", 528 | "VAR\n", 529 | "VTR\n", 530 | "VRSN\n", 531 | "VRSK\n", 532 | "VZ\n", 533 | "VRTX\n", 534 | "VIAB\n", 535 | "V\n", 536 | "VNO\n", 537 | "VMC\n", 538 | "WMT\n", 539 | "WBA\n", 540 | "DIS\n", 541 | "WM\n", 542 | "WAT\n", 543 | "WEC\n", 544 | "WFC\n", 545 | "HCN\n", 546 | "WDC\n", 547 | "WU\n", 548 | "WRK\n", 549 | "WY\n", 550 | "WHR\n", 551 | "WFM\n", 552 | "WMB\n", 553 | "WLTW\n", 554 | "WYN\n", 555 | "WYNN\n", 556 | "XEL\n", 557 | "XRX\n", 558 | "XLNX\n", 559 | "XL\n", 560 | "XYL\n", 561 | "YUM\n", 562 | "ZBH\n", 563 | "ZION\n", 564 | "ZTS\n", 565 | " Index Oil Gold NaturalGas MMM ABT \\\n", 566 | "Date \n", 567 | "2010-01-04 NaN NaN NaN NaN NaN NaN \n", 568 | "2010-01-05 0.002647 0.002699 0.011006 0.016420 -0.006264 -0.008079 \n", 569 | "2010-01-06 0.000704 0.016883 -0.000222 0.045234 0.014182 0.005553 \n", 570 | "2010-01-07 0.004221 -0.006256 0.005111 0.160742 0.000717 0.008100 \n", 571 | "2010-01-08 0.003328 0.001695 -0.007959 -0.126498 0.007046 0.005113 \n", 572 | "2010-01-11 0.001397 -0.002417 0.032316 -0.120427 -0.004032 0.005087 \n", 573 | "2010-01-12 -0.009326 -0.021202 -0.004534 -0.034662 0.000834 -0.002892 \n", 574 | "2010-01-13 0.008446 -0.013987 -0.017350 0.007181 -0.003212 0.002175 \n", 575 | "2010-01-14 0.002705 -0.003892 0.004193 0.028520 -0.003342 0.001447 \n", 576 | "2010-01-15 -0.011224 -0.017517 -0.004835 -0.019064 -0.001557 0.002168 \n", 577 | "2010-01-19 0.012496 0.013084 -0.001541 -0.026502 0.020991 0.012978 \n", 578 | "2010-01-20 -0.010169 -0.019752 -0.004409 0.005445 -0.004699 0.000712 \n", 579 | "2010-01-21 -0.019229 -0.020408 -0.022143 -0.003610 -0.023843 -0.012091 \n", 580 | "2010-01-22 -0.022292 -0.020965 -0.006793 0.027174 -0.014752 -0.018719 \n", 581 | "2010-01-25 0.005128 0.008754 0.006384 0.015873 0.004296 0.003302 \n", 582 | "2010-01-26 -0.004191 -0.003071 -0.011554 -0.026042 -0.002444 -0.004022 \n", 583 | "2010-01-27 0.004757 -0.013794 0.003667 -0.033868 0.008208 -0.010646 \n", 584 | "2010-01-28 -0.011472 -0.000272 -0.002740 -0.018450 -0.018834 0.001113 \n", 585 | "2010-01-29 -0.010869 -0.010459 -0.008244 -0.011278 -0.003220 -0.018903 \n", 586 | "2010-02-01 0.015551 0.021414 -0.000693 0.007605 -0.000621 0.007556 \n", 587 | "2010-02-02 0.012103 0.037629 0.029575 0.032075 -0.000124 0.020622 \n", 588 | "2010-02-03 -0.004983 -0.003238 0.004039 0.007313 0.013303 0.003674 \n", 589 | "2010-02-04 -0.030866 -0.049766 -0.014305 -0.007260 -0.028098 -0.015007 \n", 590 | "2010-02-05 0.002067 -0.027075 -0.045578 0.025594 -0.008459 0.002601 \n", 591 | "2010-02-08 -0.007219 0.010119 0.016869 0.021390 -0.012860 -0.011119 \n", 592 | "2010-02-09 0.012560 0.025602 -0.001869 -0.033159 0.015349 0.000000 \n", 593 | "2010-02-10 -0.001959 0.010446 0.007022 -0.010830 -0.001270 -0.001874 \n", 594 | "2010-02-11 0.010466 0.010070 0.003719 0.009124 0.020987 0.005257 \n", 595 | "2010-02-12 -0.000832 -0.014888 -0.001158 -0.009042 -0.013579 0.007097 \n", 596 | "2010-02-16 0.015735 0.038726 0.014325 0.031022 0.016292 0.012611 \n", 597 | "... ... ... ... ... ... ... \n", 598 | "2017-03-16 -0.001967 -0.000827 0.019422 -0.059603 NaN NaN \n", 599 | "2017-03-17 -0.006080 0.000828 0.002570 0.003521 NaN NaN \n", 600 | "2017-03-20 -0.001097 -0.011378 0.003459 0.028070 NaN NaN \n", 601 | "2017-03-21 -0.012839 -0.016112 -0.000770 0.054608 NaN NaN \n", 602 | "2017-03-22 0.002353 0.005742 0.011404 -0.022654 NaN NaN \n", 603 | "2017-03-23 -0.001067 -0.006132 0.001445 -0.029801 NaN NaN \n", 604 | "2017-03-24 -0.000726 0.006383 -0.003125 0.013652 NaN NaN \n", 605 | "2017-03-27 -0.001026 -0.005920 0.010370 -0.010101 NaN NaN \n", 606 | "2017-03-28 0.007277 0.028499 -0.002586 0.003401 NaN NaN \n", 607 | "2017-03-29 0.000935 0.022953 -0.000598 0.027119 NaN NaN \n", 608 | "2017-03-30 0.003184 0.016778 -0.001596 0.016502 NaN NaN \n", 609 | "2017-03-31 -0.002328 0.004771 -0.007355 0.016234 NaN NaN \n", 610 | "2017-04-03 -0.001739 -0.005738 0.003664 0.000000 NaN NaN \n", 611 | "2017-04-04 0.000637 0.014726 0.009950 -0.022364 NaN NaN \n", 612 | "2017-04-05 -0.002973 0.002942 -0.004886 0.052288 NaN NaN \n", 613 | "2017-04-06 0.002811 0.010755 0.000998 0.003106 NaN NaN \n", 614 | "2017-04-07 -0.001019 0.010834 0.008415 0.000000 NaN NaN \n", 615 | "2017-04-10 0.000595 0.015502 -0.008463 -0.024768 NaN NaN \n", 616 | "2017-04-11 -0.001190 0.006031 0.001675 -0.012698 NaN NaN \n", 617 | "2017-04-12 -0.004382 -0.004871 0.013220 -0.045016 NaN NaN \n", 618 | "2017-04-13 -0.006495 0.001318 0.010846 0.010101 NaN NaN \n", 619 | "2017-04-18 -0.002984 -0.003041 -0.000855 0.006472 NaN NaN \n", 620 | "2017-04-19 -0.001839 -0.037552 -0.002296 0.000000 NaN NaN \n", 621 | "2017-04-20 0.008139 -0.004555 -0.001677 0.006431 NaN NaN \n", 622 | "2017-04-21 -0.003187 -0.012336 0.001250 -0.022364 NaN NaN \n", 623 | "2017-04-24 0.010998 -0.014907 -0.007569 0.000000 NaN NaN \n", 624 | "2017-04-25 0.005819 0.006544 -0.001022 -0.022876 NaN NaN \n", 625 | "2017-04-26 -0.000629 0.000000 -0.004368 0.023411 NaN NaN \n", 626 | "2017-04-27 0.000839 -0.005282 -0.000514 0.022876 NaN NaN \n", 627 | "2017-04-28 -0.002179 0.007149 0.000989 0.022364 NaN NaN \n", 628 | "\n", 629 | " ABBV ACN ATVI AYI ... WYNN XEL \\\n", 630 | "Date ... \n", 631 | "2010-01-04 NaN NaN NaN NaN ... NaN NaN \n", 632 | "2010-01-05 NaN 0.006180 0.001770 0.008326 ... 0.060819 -0.011860 \n", 633 | "2010-01-06 NaN 0.010631 -0.005300 0.014313 ... -0.013117 0.001920 \n", 634 | "2010-01-07 NaN -0.000935 -0.023979 0.031208 ... 0.021356 -0.004312 \n", 635 | "2010-01-08 NaN -0.003978 -0.008189 0.003684 ... -0.007165 0.000481 \n", 636 | "2010-01-11 NaN -0.000940 -0.000917 0.003146 ... -0.003240 0.009620 \n", 637 | "2010-01-12 NaN -0.006348 -0.011019 0.006273 ... 0.004433 0.011910 \n", 638 | "2010-01-13 NaN 0.011358 0.012999 0.010909 ... 0.020888 0.015537 \n", 639 | "2010-01-14 NaN 0.008891 -0.004583 -0.005910 ... 0.039049 0.001854 \n", 640 | "2010-01-15 NaN -0.007189 0.002762 -0.018093 ... -0.022327 -0.009255 \n", 641 | "2010-01-19 NaN 0.021957 -0.004591 0.010003 ... 0.011631 0.008874 \n", 642 | "2010-01-20 NaN -0.005486 -0.001845 -0.010425 ... -0.044307 -0.002778 \n", 643 | "2010-01-21 NaN -0.009653 -0.021257 -0.012642 ... -0.040933 -0.007892 \n", 644 | "2010-01-22 NaN -0.021119 -0.021719 -0.017338 ... -0.021723 -0.024801 \n", 645 | "2010-01-25 NaN -0.007112 -0.023166 -0.000271 ... -0.005629 -0.002399 \n", 646 | "2010-01-26 NaN -0.001194 -0.010870 -0.009775 ... -0.011637 0.000481 \n", 647 | "2010-01-27 NaN 0.002391 0.034965 0.001097 ... -0.001432 0.006731 \n", 648 | "2010-01-28 NaN -0.015264 -0.010618 -0.012325 ... -0.015774 -0.003820 \n", 649 | "2010-01-29 NaN -0.007266 -0.008780 -0.007765 ... 0.001781 -0.003835 \n", 650 | "2010-02-01 NaN 0.004879 -0.007874 0.020123 ... 0.059147 0.003369 \n", 651 | "2010-02-02 NaN 0.010439 0.016865 0.010685 ... -0.010681 0.015348 \n", 652 | "2010-02-03 NaN -0.000481 0.020488 0.001626 ... -0.009254 -0.011809 \n", 653 | "2010-02-04 NaN -0.023798 -0.028681 -0.041678 ... -0.052927 -0.032983 \n", 654 | "2010-02-05 NaN -0.001724 0.004921 0.007343 ... 0.008711 -0.004943 \n", 655 | "2010-02-08 NaN -0.009127 -0.004897 -0.011775 ... -0.009940 -0.004471 \n", 656 | "2010-02-09 NaN 0.001743 -0.010827 0.015603 ... 0.014319 0.007984 \n", 657 | "2010-02-10 NaN -0.005467 0.004975 0.001955 ... -0.003732 -0.006931 \n", 658 | "2010-02-11 NaN 0.006247 0.096040 0.013939 ... 0.057980 0.011466 \n", 659 | "2010-02-12 NaN 0.005463 0.003613 0.009623 ... -0.029557 -0.002957 \n", 660 | "2010-02-16 NaN 0.008644 -0.009001 0.005719 ... 0.029664 0.018290 \n", 661 | "... ... ... ... ... ... ... ... \n", 662 | "2017-03-16 NaN NaN NaN NaN ... NaN NaN \n", 663 | "2017-03-17 NaN NaN NaN NaN ... NaN NaN \n", 664 | "2017-03-20 NaN NaN NaN NaN ... NaN NaN \n", 665 | "2017-03-21 NaN NaN NaN NaN ... NaN NaN \n", 666 | "2017-03-22 NaN NaN NaN NaN ... NaN NaN \n", 667 | "2017-03-23 NaN NaN NaN NaN ... NaN NaN \n", 668 | "2017-03-24 NaN NaN NaN NaN ... NaN NaN \n", 669 | "2017-03-27 NaN NaN NaN NaN ... NaN NaN \n", 670 | "2017-03-28 NaN NaN NaN NaN ... NaN NaN \n", 671 | "2017-03-29 NaN NaN NaN NaN ... NaN NaN \n", 672 | "2017-03-30 NaN NaN NaN NaN ... NaN NaN \n", 673 | "2017-03-31 NaN NaN NaN NaN ... NaN NaN \n", 674 | "2017-04-03 NaN NaN NaN NaN ... NaN NaN \n", 675 | "2017-04-04 NaN NaN NaN NaN ... NaN NaN \n", 676 | "2017-04-05 NaN NaN NaN NaN ... NaN NaN \n", 677 | "2017-04-06 NaN NaN NaN NaN ... NaN NaN \n", 678 | "2017-04-07 NaN NaN NaN NaN ... NaN NaN \n", 679 | "2017-04-10 NaN NaN NaN NaN ... NaN NaN \n", 680 | "2017-04-11 NaN NaN NaN NaN ... NaN NaN \n", 681 | "2017-04-12 NaN NaN NaN NaN ... NaN NaN \n", 682 | "2017-04-13 NaN NaN NaN NaN ... NaN NaN \n", 683 | "2017-04-18 NaN NaN NaN NaN ... NaN NaN \n", 684 | "2017-04-19 NaN NaN NaN NaN ... NaN NaN \n", 685 | "2017-04-20 NaN NaN NaN NaN ... NaN NaN \n", 686 | "2017-04-21 NaN NaN NaN NaN ... NaN NaN \n", 687 | "2017-04-24 NaN NaN NaN NaN ... NaN NaN \n", 688 | "2017-04-25 NaN NaN NaN NaN ... NaN NaN \n", 689 | "2017-04-26 NaN NaN NaN NaN ... NaN NaN \n", 690 | "2017-04-27 NaN NaN NaN NaN ... NaN NaN \n", 691 | "2017-04-28 NaN NaN NaN NaN ... NaN NaN \n", 692 | "\n", 693 | " XRX XLNX XL XYL YUM ZBH ZION ZTS \n", 694 | "Date \n", 695 | "2010-01-04 NaN NaN NaN NaN NaN NaN NaN NaN \n", 696 | "2010-01-05 0.001159 -0.012608 NaN NaN -0.003420 0.031656 0.035259 NaN \n", 697 | "2010-01-06 -0.009259 -0.006784 NaN NaN -0.007149 -0.000323 0.086957 NaN \n", 698 | "2010-01-07 0.004673 -0.010044 NaN NaN -0.000288 0.022940 0.112000 NaN \n", 699 | "2010-01-08 -0.003488 0.014610 NaN NaN 0.000288 -0.021004 -0.016187 NaN \n", 700 | "2010-01-11 0.024504 -0.010400 NaN NaN 0.017281 0.022100 0.006094 NaN \n", 701 | "2010-01-12 0.007973 -0.027890 NaN NaN 0.018969 -0.019255 -0.027862 NaN \n", 702 | "2010-01-13 0.011299 0.008732 NaN NaN -0.002779 0.006598 0.018692 NaN \n", 703 | "2010-01-14 -0.006704 -0.006183 NaN NaN 0.010867 0.002878 0.022018 NaN \n", 704 | "2010-01-15 -0.005624 -0.024471 NaN NaN -0.016814 -0.015623 -0.026930 NaN \n", 705 | "2010-01-19 0.020362 0.015306 NaN NaN -0.000561 0.031741 0.052276 NaN \n", 706 | "2010-01-20 -0.014412 0.002513 NaN NaN -0.014025 -0.018521 0.047341 NaN \n", 707 | "2010-01-21 0.043870 0.054720 NaN NaN -0.005121 -0.018871 0.024554 NaN \n", 708 | "2010-01-22 -0.033405 -0.055842 NaN NaN -0.019445 -0.012388 -0.038126 NaN \n", 709 | "2010-01-25 -0.008919 0.022651 NaN NaN -0.000292 0.004126 0.014723 NaN \n", 710 | "2010-01-26 -0.003375 -0.006563 NaN NaN 0.001459 -0.002959 0.064732 NaN \n", 711 | "2010-01-27 0.003386 0.010735 NaN NaN 0.013982 -0.012859 -0.007338 NaN \n", 712 | "2010-01-28 -0.007874 -0.019199 NaN NaN -0.013502 -0.050601 0.021119 NaN \n", 713 | "2010-01-29 -0.011338 -0.017909 NaN NaN -0.003786 -0.009323 -0.019131 NaN \n", 714 | "2010-02-01 0.028670 0.016964 NaN NaN 0.035370 0.012607 0.050079 NaN \n", 715 | "2010-02-02 0.017837 0.012093 NaN NaN 0.011293 0.022269 -0.000502 NaN \n", 716 | "2010-02-03 -0.015334 0.009889 NaN NaN -0.013400 -0.020069 -0.064792 NaN \n", 717 | "2010-02-04 -0.037820 -0.050184 NaN NaN -0.054895 -0.026431 -0.060687 NaN \n", 718 | "2010-02-05 -0.020809 0.014605 NaN NaN -0.012575 -0.003416 0.029160 NaN \n", 719 | "2010-02-08 -0.002361 -0.006774 NaN NaN -0.007884 -0.000180 -0.000556 NaN \n", 720 | "2010-02-09 -0.008284 0.014919 NaN NaN 0.010697 0.011909 0.012229 NaN \n", 721 | "2010-02-10 0.008353 0.005880 NaN NaN 0.002721 0.010699 0.017024 NaN \n", 722 | "2010-02-11 0.036686 0.021294 NaN NaN 0.006936 0.016231 -0.018898 NaN \n", 723 | "2010-02-12 0.001142 0.004088 NaN NaN -0.000898 -0.006250 -0.017611 NaN \n", 724 | "2010-02-16 0.010262 0.019544 NaN NaN 0.007194 0.019567 0.022969 NaN \n", 725 | "... ... ... .. ... ... ... ... ... \n", 726 | "2017-03-16 NaN NaN NaN NaN NaN NaN NaN NaN \n", 727 | "2017-03-17 NaN NaN NaN NaN NaN NaN NaN NaN \n", 728 | "2017-03-20 NaN NaN NaN NaN NaN NaN NaN NaN \n", 729 | "2017-03-21 NaN NaN NaN NaN NaN NaN NaN NaN \n", 730 | "2017-03-22 NaN NaN NaN NaN NaN NaN NaN NaN \n", 731 | "2017-03-23 NaN NaN NaN NaN NaN NaN NaN NaN \n", 732 | "2017-03-24 NaN NaN NaN NaN NaN NaN NaN NaN \n", 733 | "2017-03-27 NaN NaN NaN NaN NaN NaN NaN NaN \n", 734 | "2017-03-28 NaN NaN NaN NaN NaN NaN NaN NaN \n", 735 | "2017-03-29 NaN NaN NaN NaN NaN NaN NaN NaN \n", 736 | "2017-03-30 NaN NaN NaN NaN NaN NaN NaN NaN \n", 737 | "2017-03-31 NaN NaN NaN NaN NaN NaN NaN NaN \n", 738 | "2017-04-03 NaN NaN NaN NaN NaN NaN NaN NaN \n", 739 | "2017-04-04 NaN NaN NaN NaN NaN NaN NaN NaN \n", 740 | "2017-04-05 NaN NaN NaN NaN NaN NaN NaN NaN \n", 741 | "2017-04-06 NaN NaN NaN NaN NaN NaN NaN NaN \n", 742 | "2017-04-07 NaN NaN NaN NaN NaN NaN NaN NaN \n", 743 | "2017-04-10 NaN NaN NaN NaN NaN NaN NaN NaN \n", 744 | "2017-04-11 NaN NaN NaN NaN NaN NaN NaN NaN \n", 745 | "2017-04-12 NaN NaN NaN NaN NaN NaN NaN NaN \n", 746 | "2017-04-13 NaN NaN NaN NaN NaN NaN NaN NaN \n", 747 | "2017-04-18 NaN NaN NaN NaN NaN NaN NaN NaN \n", 748 | "2017-04-19 NaN NaN NaN NaN NaN NaN NaN NaN \n", 749 | "2017-04-20 NaN NaN NaN NaN NaN NaN NaN NaN \n", 750 | "2017-04-21 NaN NaN NaN NaN NaN NaN NaN NaN \n", 751 | "2017-04-24 NaN NaN NaN NaN NaN NaN NaN NaN \n", 752 | "2017-04-25 NaN NaN NaN NaN NaN NaN NaN NaN \n", 753 | "2017-04-26 NaN NaN NaN NaN NaN NaN NaN NaN \n", 754 | "2017-04-27 NaN NaN NaN NaN NaN NaN NaN NaN \n", 755 | "2017-04-28 NaN NaN NaN NaN NaN NaN NaN NaN \n", 756 | "\n", 757 | "[1842 rows x 509 columns]\n" 758 | ] 759 | } 760 | ], 761 | "source": [ 762 | "import pandas_datareader.data as web\n", 763 | "import datetime\n", 764 | "df = variables\n", 765 | "start = datetime.datetime(2010, 1, 1)\n", 766 | "end = datetime.datetime(2017, 1, 1)\n", 767 | "for x in stocksArray:\n", 768 | " stock = web.DataReader(x, 'google', start, end)\n", 769 | " stock = stock[\"Close\"].pct_change()\n", 770 | " stock.name = x\n", 771 | " df = pd.concat([df,stock],axis=1)\n", 772 | " print(x)\n", 773 | "print(df)" 774 | ] 775 | }, 776 | { 777 | "cell_type": "code", 778 | "execution_count": 5, 779 | "metadata": { 780 | "collapsed": true 781 | }, 782 | "outputs": [], 783 | "source": [ 784 | "df.to_csv(\"StockData.csv\", encoding=\"UTF-8\")" 785 | ] 786 | } 787 | ], 788 | "metadata": { 789 | "kernelspec": { 790 | "display_name": "Python 3", 791 | "language": "python", 792 | "name": "python3" 793 | }, 794 | "language_info": { 795 | "codemirror_mode": { 796 | "name": "ipython", 797 | "version": 3 798 | }, 799 | "file_extension": ".py", 800 | "mimetype": "text/x-python", 801 | "name": "python", 802 | "nbconvert_exporter": "python", 803 | "pygments_lexer": "ipython3", 804 | "version": "3.6.1" 805 | } 806 | }, 807 | "nbformat": 4, 808 | "nbformat_minor": 2 809 | } 810 | -------------------------------------------------------------------------------- /5 Regressions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd\n", 12 | "import statsmodels.formula.api as sm\n", 13 | "df = pd.DataFrame.from_csv(\"StockData.csv\", encoding=\"UTF-8\")\n", 14 | "model = sm.ols(formula=\"PXD ~ Index + Oil + Gold + NaturalGas\", data=df)" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "result = model.fit()" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 3, 31 | "metadata": {}, 32 | "outputs": [ 33 | { 34 | "name": "stdout", 35 | "output_type": "stream", 36 | "text": [ 37 | " OLS Regression Results \n", 38 | "==============================================================================\n", 39 | "Dep. Variable: PXD R-squared: 0.513\n", 40 | "Model: OLS Adj. R-squared: 0.512\n", 41 | "Method: Least Squares F-statistic: 453.1\n", 42 | "Date: Sat, 22 Jul 2017 Prob (F-statistic): 6.19e-267\n", 43 | "Time: 18:39:30 Log-Likelihood: 4621.8\n", 44 | "No. Observations: 1727 AIC: -9234.\n", 45 | "Df Residuals: 1722 BIC: -9206.\n", 46 | "Df Model: 4 \n", 47 | "Covariance Type: nonrobust \n", 48 | "==============================================================================\n", 49 | " coef std err t P>|t| [0.025 0.975]\n", 50 | "------------------------------------------------------------------------------\n", 51 | "Intercept 0.0005 0.000 1.168 0.243 -0.000 0.001\n", 52 | "Index 1.2220 0.044 27.529 0.000 1.135 1.309\n", 53 | "Oil 0.3862 0.020 19.313 0.000 0.347 0.425\n", 54 | "Gold 0.0527 0.038 1.392 0.164 -0.022 0.127\n", 55 | "NaturalGas 0.0054 0.011 0.500 0.617 -0.016 0.027\n", 56 | "==============================================================================\n", 57 | "Omnibus: 171.149 Durbin-Watson: 2.047\n", 58 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 723.613\n", 59 | "Skew: 0.391 Prob(JB): 7.40e-158\n", 60 | "Kurtosis: 6.073 Cond. No. 113.\n", 61 | "==============================================================================\n", 62 | "\n", 63 | "Warnings:\n", 64 | "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" 65 | ] 66 | } 67 | ], 68 | "source": [ 69 | "print(result.summary())" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 4, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | " OLS Regression Results \n", 82 | "==============================================================================\n", 83 | "Dep. Variable: F R-squared: 0.474\n", 84 | "Model: OLS Adj. R-squared: 0.473\n", 85 | "Method: Least Squares F-statistic: 388.7\n", 86 | "Date: Sat, 22 Jul 2017 Prob (F-statistic): 1.13e-238\n", 87 | "Time: 18:39:30 Log-Likelihood: 5015.0\n", 88 | "No. Observations: 1727 AIC: -1.002e+04\n", 89 | "Df Residuals: 1722 BIC: -9993.\n", 90 | "Df Model: 4 \n", 91 | "Covariance Type: nonrobust \n", 92 | "==============================================================================\n", 93 | " coef std err t P>|t| [0.025 0.975]\n", 94 | "------------------------------------------------------------------------------\n", 95 | "Intercept -0.0003 0.000 -0.969 0.333 -0.001 0.000\n", 96 | "Index 1.2920 0.035 36.549 0.000 1.223 1.361\n", 97 | "Oil -0.0077 0.016 -0.485 0.628 -0.039 0.024\n", 98 | "Gold 0.0338 0.030 1.122 0.262 -0.025 0.093\n", 99 | "NaturalGas 0.0045 0.009 0.524 0.601 -0.012 0.021\n", 100 | "==============================================================================\n", 101 | "Omnibus: 322.554 Durbin-Watson: 1.856\n", 102 | "Prob(Omnibus): 0.000 Jarque-Bera (JB): 4855.417\n", 103 | "Skew: -0.411 Prob(JB): 0.00\n", 104 | "Kurtosis: 11.173 Cond. No. 113.\n", 105 | "==============================================================================\n", 106 | "\n", 107 | "Warnings:\n", 108 | "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n" 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "def regress(stock):\n", 114 | " result = sm.ols(formula=stock+\" ~ Index + Oil + Gold + NaturalGas\", data=df).fit()\n", 115 | " print(result.summary())\n", 116 | "regress(\"F\")" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 5, 122 | "metadata": {}, 123 | "outputs": [ 124 | { 125 | "name": "stdout", 126 | "output_type": "stream", 127 | "text": [ 128 | "Intercept 1.946173e-01\n", 129 | "Index 4.033200e-133\n", 130 | "Oil 1.547053e-02\n", 131 | "Gold 7.595522e-02\n", 132 | "NaturalGas 4.721861e-03\n", 133 | "dtype: float64\n" 134 | ] 135 | } 136 | ], 137 | "source": [ 138 | "def regress(stock):\n", 139 | " result = sm.ols(formula=stock+\" ~ Index + Oil + Gold + NaturalGas\", data=df).fit()\n", 140 | " print(result.pvalues)\n", 141 | "regress(\"AAPL\")" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 6, 147 | "metadata": {}, 148 | "outputs": [ 149 | { 150 | "name": "stdout", 151 | "output_type": "stream", 152 | "text": [ 153 | "Intercept False\n", 154 | "Index True\n", 155 | "Oil True\n", 156 | "Gold False\n", 157 | "NaturalGas False\n", 158 | "dtype: bool\n" 159 | ] 160 | } 161 | ], 162 | "source": [ 163 | "def regress(stock):\n", 164 | " result = sm.ols(formula=stock+\" ~ Index + Oil + Gold + NaturalGas\", data=df).fit()\n", 165 | " print(result.pvalues<.05)\n", 166 | "regress(\"PXD\")" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 7, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "Intercept False\n", 179 | "Index True\n", 180 | "Oil True\n", 181 | "Gold False\n", 182 | "NaturalGas False\n", 183 | "dtype: bool\n", 184 | "Intercept 0.000469\n", 185 | "Index 1.222012\n", 186 | "Oil 0.386156\n", 187 | "Gold 0.052714\n", 188 | "NaturalGas 0.005414\n", 189 | "dtype: float64\n" 190 | ] 191 | } 192 | ], 193 | "source": [ 194 | "def regress(stock):\n", 195 | " result = sm.ols(formula=stock+\" ~ Index + Oil + Gold + NaturalGas\", data=df).fit()\n", 196 | " print(result.pvalues<.05)\n", 197 | " print(result.params)\n", 198 | "regress(\"PXD\")" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 8, 204 | "metadata": {}, 205 | "outputs": [ 206 | { 207 | "name": "stdout", 208 | "output_type": "stream", 209 | "text": [ 210 | "Index 1.222012\n", 211 | "Oil 0.386156\n", 212 | "dtype: float64\n" 213 | ] 214 | } 215 | ], 216 | "source": [ 217 | "def regress(stock):\n", 218 | " result = sm.ols(formula=stock+\" ~ Index + Oil + Gold + NaturalGas\", data=df).fit()\n", 219 | " truthSeries = result.pvalues<.05\n", 220 | " print(result.params[truthSeries])\n", 221 | "regress(\"PXD\")" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 9, 227 | "metadata": {}, 228 | "outputs": [ 229 | { 230 | "data": { 231 | "text/html": [ 232 | "
\n", 233 | "\n", 246 | "\n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | "
PXD
Index1.222012
Oil0.386156
\n", 264 | "
" 265 | ], 266 | "text/plain": [ 267 | " PXD\n", 268 | "Index 1.222012\n", 269 | "Oil 0.386156" 270 | ] 271 | }, 272 | "execution_count": 9, 273 | "metadata": {}, 274 | "output_type": "execute_result" 275 | } 276 | ], 277 | "source": [ 278 | "def regress(stock):\n", 279 | " result = sm.ols(formula=stock+\" ~ Index + Oil + Gold + NaturalGas\", data=df).fit()\n", 280 | " return pd.DataFrame(result.params[result.pvalues<.05],columns=[stock])\n", 281 | "regress(\"PXD\")" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": 10, 287 | "metadata": {}, 288 | "outputs": [ 289 | { 290 | "ename": "PatsyError", 291 | "evalue": "Error evaluating factor: NameError: name 'BRK' is not defined\n BRK.B ~ Index + Oil + Gold + NaturalGas\n ^^^^^", 292 | "output_type": "error", 293 | "traceback": [ 294 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 295 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 296 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/compat.py\u001b[0m in \u001b[0;36mcall_and_wrap_exc\u001b[0;34m(msg, origin, f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 297 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/eval.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, expr, source_name, inner_namespace)\u001b[0m\n\u001b[1;32m 165\u001b[0m return eval(code, {}, VarLookupDict([inner_namespace]\n\u001b[0;32m--> 166\u001b[0;31m + self._namespaces))\n\u001b[0m\u001b[1;32m 167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 298 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n", 299 | "\u001b[0;31mNameError\u001b[0m: name 'BRK' is not defined", 300 | "\nThe above exception was the direct cause of the following exception:\n", 301 | "\u001b[0;31mPatsyError\u001b[0m Traceback (most recent call last)", 302 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mregress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"BRK.B\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 303 | "\u001b[0;32m\u001b[0m in \u001b[0;36mregress\u001b[0;34m(stock)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstock\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mols\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformula\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstock\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\" ~ Index + Oil + Gold + NaturalGas\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpvalues\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0;36m.05\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstock\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mregress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"PXD\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 304 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/statsmodels/base/model.py\u001b[0m in \u001b[0;36mfrom_formula\u001b[0;34m(cls, formula, data, subset, drop_cols, *args, **kwargs)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 154\u001b[0m tmp = handle_formula_data(data, None, formula, depth=eval_env,\n\u001b[0;32m--> 155\u001b[0;31m missing=missing)\n\u001b[0m\u001b[1;32m 156\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mendog\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexog\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmissing_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesign_info\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 305 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/statsmodels/formula/formulatools.py\u001b[0m in \u001b[0;36mhandle_formula_data\u001b[0;34m(Y, X, formula, depth, missing)\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_using_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 64\u001b[0m result = dmatrices(formula, Y, depth, return_type='dataframe',\n\u001b[0;32m---> 65\u001b[0;31m NA_action=na_action)\n\u001b[0m\u001b[1;32m 66\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 67\u001b[0m result = dmatrices(formula, Y, depth, return_type='dataframe',\n", 306 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/highlevel.py\u001b[0m in \u001b[0;36mdmatrices\u001b[0;34m(formula_like, data, eval_env, NA_action, return_type)\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0meval_env\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEvalEnvironment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcapture\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval_env\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreference\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 309\u001b[0m (lhs, rhs) = _do_highlevel_design(formula_like, data, eval_env,\n\u001b[0;32m--> 310\u001b[0;31m NA_action, return_type)\n\u001b[0m\u001b[1;32m 311\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlhs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mPatsyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"model is missing required outcome variables\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 307 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/highlevel.py\u001b[0m in \u001b[0;36m_do_highlevel_design\u001b[0;34m(formula_like, data, eval_env, NA_action, return_type)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,\n\u001b[0;32m--> 165\u001b[0;31m NA_action)\n\u001b[0m\u001b[1;32m 166\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdesign_infos\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 167\u001b[0m return build_design_matrices(design_infos, data,\n", 308 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/highlevel.py\u001b[0m in \u001b[0;36m_try_incr_builders\u001b[0;34m(formula_like, data_iter_maker, eval_env, NA_action)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0mdata_iter_maker\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0meval_env\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m NA_action)\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 309 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/build.py\u001b[0m in \u001b[0;36mdesign_matrix_builders\u001b[0;34m(termlists, data_iter_maker, eval_env, NA_action)\u001b[0m\n\u001b[1;32m 694\u001b[0m \u001b[0mfactor_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 695\u001b[0m \u001b[0mdata_iter_maker\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 696\u001b[0;31m NA_action)\n\u001b[0m\u001b[1;32m 697\u001b[0m \u001b[0;31m# Now we need the factor infos, which encapsulate the knowledge of\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 698\u001b[0m \u001b[0;31m# how to turn any given factor into a chunk of data:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 310 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/build.py\u001b[0m in \u001b[0;36m_examine_factor_types\u001b[0;34m(factors, factor_states, data_iter_maker, NA_action)\u001b[0m\n\u001b[1;32m 441\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_iter_maker\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfactor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexamine_needed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 443\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfactor_states\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfactor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 444\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfactor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcat_sniffers\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mguess_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 445\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfactor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcat_sniffers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 311 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/eval.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, memorize_state, data)\u001b[0m\n\u001b[1;32m 564\u001b[0m return self._eval(memorize_state[\"eval_code\"],\n\u001b[1;32m 565\u001b[0m \u001b[0mmemorize_state\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 566\u001b[0;31m data)\n\u001b[0m\u001b[1;32m 567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 568\u001b[0m \u001b[0m__getstate__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mno_pickling\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 312 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/eval.py\u001b[0m in \u001b[0;36m_eval\u001b[0;34m(self, code, memorize_state, data)\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0mmemorize_state\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"eval_env\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 550\u001b[0m \u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 551\u001b[0;31m inner_namespace=inner_namespace)\n\u001b[0m\u001b[1;32m 552\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmemorize_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwhich_pass\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 313 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/compat.py\u001b[0m in \u001b[0;36mcall_and_wrap_exc\u001b[0;34m(msg, origin, f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 122\u001b[0m origin)\n\u001b[1;32m 123\u001b[0m \u001b[0;31m# Use 'exec' to hide this syntax from the Python 2 parser:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"raise new_exc from e\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;31m# In python 2, we just let the original exception escape -- better\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 314 | "\u001b[0;32m/Users/seanmcowen/anaconda/lib/python3.6/site-packages/patsy/compat.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n", 315 | "\u001b[0;31mPatsyError\u001b[0m: Error evaluating factor: NameError: name 'BRK' is not defined\n BRK.B ~ Index + Oil + Gold + NaturalGas\n ^^^^^" 316 | ] 317 | } 318 | ], 319 | "source": [ 320 | "regress(\"BRK.B\")" 321 | ] 322 | }, 323 | { 324 | "cell_type": "code", 325 | "execution_count": 11, 326 | "metadata": { 327 | "collapsed": true 328 | }, 329 | "outputs": [], 330 | "source": [ 331 | "cols = []\n", 332 | "for x in df.columns:\n", 333 | " if \".\" in x:\n", 334 | " x = x.replace(\".\",\"\")\n", 335 | " if \":\" in x:\n", 336 | " x = x.replace(\":\",\"\")\n", 337 | " cols.append(x)\n", 338 | "df.columns = cols" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 12, 344 | "metadata": { 345 | "collapsed": true 346 | }, 347 | "outputs": [], 348 | "source": [ 349 | "del df[\"FTI\"]\n", 350 | "del df[\"COL\"]\n", 351 | "del df['DXC']\n", 352 | "del df[\"BHGE\"]" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": 13, 358 | "metadata": {}, 359 | "outputs": [ 360 | { 361 | "name": "stdout", 362 | "output_type": "stream", 363 | "text": [ 364 | " MMM ABT ABBV ACN ATVI AYI \\\n", 365 | "Gold NaN NaN NaN NaN NaN 0.078922 \n", 366 | "Index 0.958352 0.769490 1.098795 0.988583 0.904080 1.317917 \n", 367 | "Intercept NaN NaN NaN NaN NaN NaN \n", 368 | "NaturalGas NaN NaN NaN NaN -0.022059 NaN \n", 369 | "Oil NaN -0.034686 NaN NaN -0.046367 NaN \n", 370 | "\n", 371 | " ADBE AMD AAP AES ... WYNN \\\n", 372 | "Gold NaN NaN NaN NaN ... 0.150433 \n", 373 | "Index 1.171635 1.559588 0.810991 1.121761 ... 1.446935 \n", 374 | "Intercept NaN NaN NaN NaN ... NaN \n", 375 | "NaturalGas NaN NaN NaN NaN ... NaN \n", 376 | "Oil NaN 0.079827 -0.060120 0.081796 ... 0.052993 \n", 377 | "\n", 378 | " XEL XRX XLNX XL XYL YUM \\\n", 379 | "Gold NaN -0.079305 NaN NaN NaN NaN \n", 380 | "Index 0.571913 1.288955 1.022487 0.935746 0.956918 0.960295 \n", 381 | "Intercept NaN NaN NaN NaN NaN NaN \n", 382 | "NaturalGas NaN NaN NaN NaN NaN NaN \n", 383 | "Oil -0.040756 NaN NaN NaN 0.065023 NaN \n", 384 | "\n", 385 | " ZBH ZION ZTS \n", 386 | "Gold NaN -0.123886 NaN \n", 387 | "Index 0.923686 1.450939 0.937724 \n", 388 | "Intercept NaN NaN NaN \n", 389 | "NaturalGas NaN NaN NaN \n", 390 | "Oil NaN 0.076283 -0.039546 \n", 391 | "\n", 392 | "[5 rows x 501 columns]\n" 393 | ] 394 | } 395 | ], 396 | "source": [ 397 | "df2 = pd.DataFrame()\n", 398 | "for stock in df.columns[4:]:\n", 399 | " frame = regress(stock)\n", 400 | " df2 = pd.concat([df2,frame], axis=1)\n", 401 | "print(df2)" 402 | ] 403 | }, 404 | { 405 | "cell_type": "code", 406 | "execution_count": 14, 407 | "metadata": { 408 | "collapsed": true 409 | }, 410 | "outputs": [], 411 | "source": [ 412 | "df2.to_csv(\"RegressionMatrix1.csv\",encoding=\"UTF-8\")" 413 | ] 414 | } 415 | ], 416 | "metadata": { 417 | "kernelspec": { 418 | "display_name": "Python 3", 419 | "language": "python", 420 | "name": "python3" 421 | }, 422 | "language_info": { 423 | "codemirror_mode": { 424 | "name": "ipython", 425 | "version": 3 426 | }, 427 | "file_extension": ".py", 428 | "mimetype": "text/x-python", 429 | "name": "python", 430 | "nbconvert_exporter": "python", 431 | "pygments_lexer": "ipython3", 432 | "version": "3.6.1" 433 | } 434 | }, 435 | "nbformat": 4, 436 | "nbformat_minor": 2 437 | } 438 | -------------------------------------------------------------------------------- /6 Machine Learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | " MMM ABT ABBV ACN ATVI AYI \\\n", 13 | "Gold 0.000000 0.000000 0.000000 0.000000 0.000000 0.078922 \n", 14 | "Index 0.958352 0.769490 1.098795 0.988583 0.904080 1.317917 \n", 15 | "Intercept 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", 16 | "NaturalGas 0.000000 0.000000 0.000000 0.000000 -0.022059 0.000000 \n", 17 | "Oil 0.000000 -0.034686 0.000000 0.000000 -0.046367 0.000000 \n", 18 | "\n", 19 | " ADBE AMD AAP AES ... WYNN \\\n", 20 | "Gold 0.000000 0.000000 0.000000 0.000000 ... 0.150433 \n", 21 | "Index 1.171635 1.559588 0.810991 1.121761 ... 1.446935 \n", 22 | "Intercept 0.000000 0.000000 0.000000 0.000000 ... 0.000000 \n", 23 | "NaturalGas 0.000000 0.000000 0.000000 0.000000 ... 0.000000 \n", 24 | "Oil 0.000000 0.079827 -0.060120 0.081796 ... 0.052993 \n", 25 | "\n", 26 | " XEL XRX XLNX XL XYL YUM \\\n", 27 | "Gold 0.000000 -0.079305 0.000000 0.000000 0.000000 0.000000 \n", 28 | "Index 0.571913 1.288955 1.022487 0.935746 0.956918 0.960295 \n", 29 | "Intercept 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", 30 | "NaturalGas 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", 31 | "Oil -0.040756 0.000000 0.000000 0.000000 0.065023 0.000000 \n", 32 | "\n", 33 | " ZBH ZION ZTS \n", 34 | "Gold 0.000000 -0.123886 0.000000 \n", 35 | "Index 0.923686 1.450939 0.937724 \n", 36 | "Intercept 0.000000 0.000000 0.000000 \n", 37 | "NaturalGas 0.000000 0.000000 0.000000 \n", 38 | "Oil 0.000000 0.076283 -0.039546 \n", 39 | "\n", 40 | "[5 rows x 501 columns]\n" 41 | ] 42 | } 43 | ], 44 | "source": [ 45 | "import pandas as pd\n", 46 | "df = pd.DataFrame.from_csv(\"RegressionMatrix1.csv\",encoding=\"UTF-8\")\n", 47 | "df = df.fillna(0)\n", 48 | "print(df)" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | " Gold Index Intercept NaturalGas Oil\n", 61 | "MMM 0.000000 0.958352 0.000000 0.000000 0.000000\n", 62 | "ABT 0.000000 0.769490 0.000000 0.000000 -0.034686\n", 63 | "ABBV 0.000000 1.098795 0.000000 0.000000 0.000000\n", 64 | "ACN 0.000000 0.988583 0.000000 0.000000 0.000000\n", 65 | "ATVI 0.000000 0.904080 0.000000 -0.022059 -0.046367\n", 66 | "AYI 0.078922 1.317917 0.000000 0.000000 0.000000\n", 67 | "ADBE 0.000000 1.171635 0.000000 0.000000 0.000000\n", 68 | "AMD 0.000000 1.559588 0.000000 0.000000 0.079827\n", 69 | "AAP 0.000000 0.810991 0.000000 0.000000 -0.060120\n", 70 | "AES 0.000000 1.121761 0.000000 0.000000 0.081796\n", 71 | "AET 0.000000 1.026593 0.000000 0.000000 -0.040546\n", 72 | "AMG 0.000000 1.608255 0.000000 0.000000 0.044590\n", 73 | "AFL 0.000000 1.311599 0.000000 0.000000 0.000000\n", 74 | "A 0.000000 1.429291 0.000000 0.000000 0.000000\n", 75 | "APD 0.000000 1.005965 0.000000 0.000000 0.000000\n", 76 | "AKAM 0.000000 1.333811 0.000000 0.000000 0.000000\n", 77 | "ALK 0.000000 1.377245 0.000923 0.000000 -0.195529\n", 78 | "ALB 0.069697 1.300341 0.000000 0.000000 0.104721\n", 79 | "ARE 0.000000 1.000648 0.000000 0.013783 0.000000\n", 80 | "ALXN 0.000000 1.141033 0.000000 0.000000 0.000000\n", 81 | "ALGN 0.000000 1.370945 0.000000 0.000000 -0.054796\n", 82 | "ALLE 0.000000 1.076024 0.000000 -0.017504 0.000000\n", 83 | "AGN 0.000000 1.201925 0.000000 0.000000 -0.063908\n", 84 | "ADS 0.000000 1.100154 0.000000 0.000000 0.000000\n", 85 | "LNT 0.000000 0.694075 0.000000 0.000000 -0.030208\n", 86 | "ALL 0.000000 0.976587 0.000000 0.000000 -0.026111\n", 87 | "GOOGL 0.000000 1.006173 0.000000 0.000000 -0.038185\n", 88 | "GOOG 0.000000 1.005614 0.000000 0.000000 -0.036680\n", 89 | "MO 0.053331 0.597372 0.000505 0.000000 -0.043492\n", 90 | "AMZN 0.000000 1.136331 0.000000 0.000000 0.000000\n", 91 | "... ... ... ... ... ...\n", 92 | "V 0.000000 1.022765 0.000000 0.000000 0.000000\n", 93 | "VNO 0.000000 1.083205 0.000000 0.000000 -0.025167\n", 94 | "VMC 0.000000 1.289977 0.000000 0.000000 0.000000\n", 95 | "WMT 0.000000 0.562514 0.000000 0.000000 -0.075675\n", 96 | "WBA 0.000000 1.061482 0.000000 0.000000 -0.072180\n", 97 | "DIS -0.047368 1.069473 0.000000 0.000000 -0.038195\n", 98 | "WM 0.000000 0.760590 0.000000 0.000000 -0.029146\n", 99 | "WAT 0.000000 1.124098 0.000000 0.000000 0.000000\n", 100 | "WEC 0.000000 0.592273 0.000000 0.000000 -0.034510\n", 101 | "WFC -0.099045 1.312262 0.000000 0.000000 0.000000\n", 102 | "HCN 0.000000 0.799773 0.000000 0.000000 -0.027817\n", 103 | "WDC 0.000000 1.357867 0.000000 0.000000 0.000000\n", 104 | "WU 0.000000 1.026143 0.000000 0.000000 0.000000\n", 105 | "WRK 0.000000 1.140511 0.000000 0.000000 0.135817\n", 106 | "WY 0.088626 1.165765 0.000000 0.000000 0.000000\n", 107 | "WHR 0.000000 1.370129 0.000000 0.000000 0.000000\n", 108 | "WFM 0.000000 0.960385 0.000000 0.000000 0.000000\n", 109 | "WMB 0.000000 1.261073 0.000000 0.000000 0.250112\n", 110 | "WLTW 0.000000 0.737797 0.000000 0.000000 0.000000\n", 111 | "WYN 0.000000 1.357646 0.000000 0.000000 0.000000\n", 112 | "WYNN 0.150433 1.446935 0.000000 0.000000 0.052993\n", 113 | "XEL 0.000000 0.571913 0.000000 0.000000 -0.040756\n", 114 | "XRX -0.079305 1.288955 0.000000 0.000000 0.000000\n", 115 | "XLNX 0.000000 1.022487 0.000000 0.000000 0.000000\n", 116 | "XL 0.000000 0.935746 0.000000 0.000000 0.000000\n", 117 | "XYL 0.000000 0.956918 0.000000 0.000000 0.065023\n", 118 | "YUM 0.000000 0.960295 0.000000 0.000000 0.000000\n", 119 | "ZBH 0.000000 0.923686 0.000000 0.000000 0.000000\n", 120 | "ZION -0.123886 1.450939 0.000000 0.000000 0.076283\n", 121 | "ZTS 0.000000 0.937724 0.000000 0.000000 -0.039546\n", 122 | "\n", 123 | "[501 rows x 5 columns]\n" 124 | ] 125 | } 126 | ], 127 | "source": [ 128 | "df = df.transpose()\n", 129 | "print(df)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 3, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | " Gold Index Intercept NaturalGas Oil\n", 142 | "MMM -0.037364 -0.353290 -0.212029 0.013207 -0.125598\n", 143 | "ABT -0.037364 -1.040657 -0.212029 0.013207 -0.482210\n", 144 | "ABBV -0.037364 0.157857 -0.212029 0.013207 -0.125598\n", 145 | "ACN -0.037364 -0.243264 -0.212029 0.013207 -0.125598\n", 146 | "ATVI -0.037364 -0.550813 -0.212029 -2.323145 -0.602309\n", 147 | "AYI 1.826425 0.955356 -0.212029 0.013207 -0.125598\n", 148 | "ADBE -0.037364 0.422960 -0.212029 0.013207 -0.125598\n", 149 | "AMD -0.037364 1.834926 -0.212029 0.013207 0.695126\n", 150 | "AAP -0.037364 -0.889615 -0.212029 0.013207 -0.743704\n", 151 | "AES -0.037364 0.241443 -0.212029 0.013207 0.715364\n", 152 | "AET -0.037364 -0.104926 -0.212029 0.013207 -0.542458\n", 153 | "AMG -0.037364 2.012051 -0.212029 0.013207 0.332843\n", 154 | "AFL -0.037364 0.932364 -0.212029 0.013207 -0.125598\n", 155 | "A -0.037364 1.360707 -0.212029 0.013207 -0.125598\n", 156 | "APD -0.037364 -0.180001 -0.212029 0.013207 -0.125598\n", 157 | "AKAM -0.037364 1.013205 -0.212029 0.013207 -0.125598\n", 158 | "ALK -0.037364 1.171283 4.895146 0.013207 -2.135875\n", 159 | "ALB 1.608568 0.891388 -0.212029 0.013207 0.951058\n", 160 | "ARE -0.037364 -0.199351 -0.212029 1.473058 -0.125598\n", 161 | "ALXN -0.037364 0.311582 -0.212029 0.013207 -0.125598\n", 162 | "ALGN -0.037364 1.148354 -0.212029 0.013207 -0.688967\n", 163 | "ALLE -0.037364 0.074981 -0.212029 -1.840692 -0.125598\n", 164 | "AGN -0.037364 0.533200 -0.212029 0.013207 -0.782655\n", 165 | "ADS -0.037364 0.162804 -0.212029 0.013207 -0.125598\n", 166 | "LNT -0.037364 -1.315133 -0.212029 0.013207 -0.436175\n", 167 | "ALL -0.037364 -0.286923 -0.212029 0.013207 -0.394047\n", 168 | "GOOGL -0.037364 -0.179244 -0.212029 0.013207 -0.518189\n", 169 | "GOOG -0.037364 -0.181276 -0.212029 0.013207 -0.502709\n", 170 | "MO 1.222066 -1.667088 2.581727 0.013207 -0.572746\n", 171 | "AMZN -0.037364 0.294471 -0.212029 0.013207 -0.125598\n", 172 | "... ... ... ... ... ...\n", 173 | "V -0.037364 -0.118855 -0.212029 0.013207 -0.125598\n", 174 | "VNO -0.037364 0.101117 -0.212029 0.013207 -0.384345\n", 175 | "VMC -0.037364 0.853668 -0.212029 0.013207 -0.125598\n", 176 | "WMT -0.037364 -1.793953 -0.212029 0.013207 -0.903630\n", 177 | "WBA -0.037364 0.022056 -0.212029 0.013207 -0.867695\n", 178 | "DIS -1.155992 0.051140 -0.212029 0.013207 -0.518293\n", 179 | "WM -0.037364 -1.073051 -0.212029 0.013207 -0.425257\n", 180 | "WAT -0.037364 0.249949 -0.212029 0.013207 -0.125598\n", 181 | "WEC -0.037364 -1.685645 -0.212029 0.013207 -0.480405\n", 182 | "WFC -2.376377 0.934776 -0.212029 0.013207 -0.125598\n", 183 | "HCN -0.037364 -0.930444 -0.212029 0.013207 -0.411589\n", 184 | "WDC -0.037364 1.100756 -0.212029 0.013207 -0.125598\n", 185 | "WU -0.037364 -0.106561 -0.212029 0.013207 -0.125598\n", 186 | "WRK -0.037364 0.309683 -0.212029 0.013207 1.270766\n", 187 | "WY 2.055581 0.401595 -0.212029 0.013207 -0.125598\n", 188 | "WHR -0.037364 1.145382 -0.212029 0.013207 -0.125598\n", 189 | "WFM -0.037364 -0.345890 -0.212029 0.013207 -0.125598\n", 190 | "WMB -0.037364 0.748471 -0.212029 0.013207 2.445858\n", 191 | "WLTW -0.037364 -1.156006 -0.212029 0.013207 -0.125598\n", 192 | "WYN -0.037364 1.099951 -0.212029 0.013207 -0.125598\n", 193 | "WYNN 3.515203 1.424921 -0.212029 0.013207 0.419235\n", 194 | "XEL -0.037364 -1.759744 -0.212029 0.013207 -0.544619\n", 195 | "XRX -1.910188 0.849949 -0.212029 0.013207 -0.125598\n", 196 | "XLNX -0.037364 -0.119870 -0.212029 0.013207 -0.125598\n", 197 | "XL -0.037364 -0.435564 -0.212029 0.013207 -0.125598\n", 198 | "XYL -0.037364 -0.358508 -0.212029 0.013207 0.542919\n", 199 | "YUM -0.037364 -0.346218 -0.212029 0.013207 -0.125598\n", 200 | "ZBH -0.037364 -0.479457 -0.212029 0.013207 -0.125598\n", 201 | "ZION -2.963010 1.439493 -0.212029 0.013207 0.658687\n", 202 | "ZTS -0.037364 -0.428367 -0.212029 0.013207 -0.532184\n", 203 | "\n", 204 | "[501 rows x 5 columns]\n" 205 | ] 206 | } 207 | ], 208 | "source": [ 209 | "for x in df.columns:\n", 210 | " df[x] = (df[x] - df[x].mean())/df[x].std(ddof=0)\n", 211 | "print(df)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 4, 217 | "metadata": { 218 | "collapsed": true 219 | }, 220 | "outputs": [], 221 | "source": [ 222 | "from sklearn.cluster import KMeans" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 5, 228 | "metadata": { 229 | "collapsed": true 230 | }, 231 | "outputs": [], 232 | "source": [ 233 | "model = KMeans(n_clusters=8)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": 6, 239 | "metadata": { 240 | "collapsed": true 241 | }, 242 | "outputs": [], 243 | "source": [ 244 | "model = model.fit(df)" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 7, 250 | "metadata": {}, 251 | "outputs": [ 252 | { 253 | "name": "stdout", 254 | "output_type": "stream", 255 | "text": [ 256 | "[[ 3.31739278e-03 -7.54338725e-01 -2.12028748e-01 1.00284569e-01\n", 257 | " -3.33014511e-01]\n", 258 | " [ 5.75671370e+00 -2.20405373e-01 -2.12028748e-01 1.32069067e-02\n", 259 | " 2.50427969e-01]\n", 260 | " [ 8.58775936e-02 8.08151844e-01 -2.12028748e-01 3.09619839e-02\n", 261 | " -1.06283716e-01]\n", 262 | " [ 1.98872084e-01 -5.37045059e-01 4.03702736e+00 1.32069067e-02\n", 263 | " -6.39311671e-01]\n", 264 | " [ -3.73639610e-02 2.84249054e-01 -2.12028748e-01 5.89892504e+00\n", 265 | " 2.31309014e+00]\n", 266 | " [ 1.85311017e-01 3.66244384e-01 -2.12028748e-01 -2.57757960e+00\n", 267 | " 1.71609465e-01]\n", 268 | " [ 2.56052390e-01 1.68478886e-01 -2.12028748e-01 -1.61875424e-01\n", 269 | " 3.43239666e+00]\n", 270 | " [ -2.41785622e+00 1.18468492e+00 -2.12028748e-01 1.32069067e-02\n", 271 | " -4.60022228e-02]]\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "print(model.cluster_centers_)" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 8, 282 | "metadata": {}, 283 | "outputs": [ 284 | { 285 | "name": "stdout", 286 | "output_type": "stream", 287 | "text": [ 288 | "[0 0 2 0 5 2 2 2 0 2 0 2 2 2 0 2 3 2 0 2 2 5 2 2 0 0 0 0 3 2 0 2 0 2 2 0 0\n", 289 | " 2 0 5 0 2 6 2 0 0 0 6 0 0 2 0 2 0 0 0 2 0 2 3 0 2 0 7 7 0 0 7 0 0 0 0 2 2\n", 290 | " 0 2 2 0 2 0 2 0 5 0 4 0 7 0 0 2 2 5 2 2 2 2 0 0 0 2 7 2 4 6 5 0 0 0 6 0 3\n", 291 | " 0 2 7 2 0 0 0 2 0 5 0 0 7 0 6 6 0 3 3 2 0 0 0 0 2 2 0 2 2 0 7 2 2 2 0 6 0\n", 292 | " 2 2 0 0 0 3 0 5 2 0 0 2 0 2 2 5 2 0 0 0 2 5 0 0 6 4 0 0 0 0 0 0 0 0 5 0 0\n", 293 | " 3 0 2 0 2 0 2 0 7 0 3 2 6 2 5 3 2 0 2 2 6 0 0 3 0 2 2 0 2 0 0 0 2 2 0 6 2\n", 294 | " 5 2 2 0 0 0 6 0 0 6 2 0 2 3 2 3 2 0 0 7 0 0 2 2 2 0 0 0 2 2 2 0 0 0 2 1 5\n", 295 | " 0 0 0 0 7 2 2 0 7 0 2 6 2 0 0 0 2 0 0 2 2 2 2 2 0 7 2 3 0 0 6 7 2 2 2 6 2\n", 296 | " 2 0 2 2 2 0 0 0 0 0 0 7 2 0 2 7 0 0 5 0 0 0 1 2 7 2 0 6 2 2 6 2 2 3 2 6 1\n", 297 | " 2 0 0 1 0 0 6 2 2 7 3 4 2 2 3 6 0 6 2 2 5 0 0 5 2 0 0 5 0 0 0 0 5 0 6 7 5\n", 298 | " 2 0 0 2 2 0 0 2 7 0 0 2 2 4 2 0 0 4 2 0 0 2 0 3 7 0 3 2 2 5 3 2 2 0 6 0 2\n", 299 | " 2 0 3 2 0 2 2 5 0 2 0 2 0 0 7 0 0 7 0 0 0 0 2 0 2 2 2 2 2 2 0 3 0 0 0 0 6\n", 300 | " 0 5 2 2 3 0 3 7 1 2 2 2 3 0 2 0 0 2 0 2 0 0 0 0 0 2 2 0 2 2 0 0 0 0 2 0 7\n", 301 | " 0 2 0 2 2 2 0 6 0 2 1 0 7 0 0 0 0 0 7 0]\n" 302 | ] 303 | } 304 | ], 305 | "source": [ 306 | "print(model.labels_)" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 9, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stdout", 316 | "output_type": "stream", 317 | "text": [ 318 | " Gold Index Intercept NaturalGas Oil Group\n", 319 | "MMM -0.037364 -0.353290 -0.212029 0.013207 -0.125598 0\n", 320 | "ABT -0.037364 -1.040657 -0.212029 0.013207 -0.482210 0\n", 321 | "ABBV -0.037364 0.157857 -0.212029 0.013207 -0.125598 2\n", 322 | "ACN -0.037364 -0.243264 -0.212029 0.013207 -0.125598 0\n", 323 | "ATVI -0.037364 -0.550813 -0.212029 -2.323145 -0.602309 5\n", 324 | "AYI 1.826425 0.955356 -0.212029 0.013207 -0.125598 2\n", 325 | "ADBE -0.037364 0.422960 -0.212029 0.013207 -0.125598 2\n", 326 | "AMD -0.037364 1.834926 -0.212029 0.013207 0.695126 2\n", 327 | "AAP -0.037364 -0.889615 -0.212029 0.013207 -0.743704 0\n", 328 | "AES -0.037364 0.241443 -0.212029 0.013207 0.715364 2\n", 329 | "AET -0.037364 -0.104926 -0.212029 0.013207 -0.542458 0\n", 330 | "AMG -0.037364 2.012051 -0.212029 0.013207 0.332843 2\n", 331 | "AFL -0.037364 0.932364 -0.212029 0.013207 -0.125598 2\n", 332 | "A -0.037364 1.360707 -0.212029 0.013207 -0.125598 2\n", 333 | "APD -0.037364 -0.180001 -0.212029 0.013207 -0.125598 0\n", 334 | "AKAM -0.037364 1.013205 -0.212029 0.013207 -0.125598 2\n", 335 | "ALK -0.037364 1.171283 4.895146 0.013207 -2.135875 3\n", 336 | "ALB 1.608568 0.891388 -0.212029 0.013207 0.951058 2\n", 337 | "ARE -0.037364 -0.199351 -0.212029 1.473058 -0.125598 0\n", 338 | "ALXN -0.037364 0.311582 -0.212029 0.013207 -0.125598 2\n", 339 | "ALGN -0.037364 1.148354 -0.212029 0.013207 -0.688967 2\n", 340 | "ALLE -0.037364 0.074981 -0.212029 -1.840692 -0.125598 5\n", 341 | "AGN -0.037364 0.533200 -0.212029 0.013207 -0.782655 2\n", 342 | "ADS -0.037364 0.162804 -0.212029 0.013207 -0.125598 2\n", 343 | "LNT -0.037364 -1.315133 -0.212029 0.013207 -0.436175 0\n", 344 | "ALL -0.037364 -0.286923 -0.212029 0.013207 -0.394047 0\n", 345 | "GOOGL -0.037364 -0.179244 -0.212029 0.013207 -0.518189 0\n", 346 | "GOOG -0.037364 -0.181276 -0.212029 0.013207 -0.502709 0\n", 347 | "MO 1.222066 -1.667088 2.581727 0.013207 -0.572746 3\n", 348 | "AMZN -0.037364 0.294471 -0.212029 0.013207 -0.125598 2\n", 349 | "... ... ... ... ... ... ...\n", 350 | "V -0.037364 -0.118855 -0.212029 0.013207 -0.125598 0\n", 351 | "VNO -0.037364 0.101117 -0.212029 0.013207 -0.384345 2\n", 352 | "VMC -0.037364 0.853668 -0.212029 0.013207 -0.125598 2\n", 353 | "WMT -0.037364 -1.793953 -0.212029 0.013207 -0.903630 0\n", 354 | "WBA -0.037364 0.022056 -0.212029 0.013207 -0.867695 0\n", 355 | "DIS -1.155992 0.051140 -0.212029 0.013207 -0.518293 0\n", 356 | "WM -0.037364 -1.073051 -0.212029 0.013207 -0.425257 0\n", 357 | "WAT -0.037364 0.249949 -0.212029 0.013207 -0.125598 2\n", 358 | "WEC -0.037364 -1.685645 -0.212029 0.013207 -0.480405 0\n", 359 | "WFC -2.376377 0.934776 -0.212029 0.013207 -0.125598 7\n", 360 | "HCN -0.037364 -0.930444 -0.212029 0.013207 -0.411589 0\n", 361 | "WDC -0.037364 1.100756 -0.212029 0.013207 -0.125598 2\n", 362 | "WU -0.037364 -0.106561 -0.212029 0.013207 -0.125598 0\n", 363 | "WRK -0.037364 0.309683 -0.212029 0.013207 1.270766 2\n", 364 | "WY 2.055581 0.401595 -0.212029 0.013207 -0.125598 2\n", 365 | "WHR -0.037364 1.145382 -0.212029 0.013207 -0.125598 2\n", 366 | "WFM -0.037364 -0.345890 -0.212029 0.013207 -0.125598 0\n", 367 | "WMB -0.037364 0.748471 -0.212029 0.013207 2.445858 6\n", 368 | "WLTW -0.037364 -1.156006 -0.212029 0.013207 -0.125598 0\n", 369 | "WYN -0.037364 1.099951 -0.212029 0.013207 -0.125598 2\n", 370 | "WYNN 3.515203 1.424921 -0.212029 0.013207 0.419235 1\n", 371 | "XEL -0.037364 -1.759744 -0.212029 0.013207 -0.544619 0\n", 372 | "XRX -1.910188 0.849949 -0.212029 0.013207 -0.125598 7\n", 373 | "XLNX -0.037364 -0.119870 -0.212029 0.013207 -0.125598 0\n", 374 | "XL -0.037364 -0.435564 -0.212029 0.013207 -0.125598 0\n", 375 | "XYL -0.037364 -0.358508 -0.212029 0.013207 0.542919 0\n", 376 | "YUM -0.037364 -0.346218 -0.212029 0.013207 -0.125598 0\n", 377 | "ZBH -0.037364 -0.479457 -0.212029 0.013207 -0.125598 0\n", 378 | "ZION -2.963010 1.439493 -0.212029 0.013207 0.658687 7\n", 379 | "ZTS -0.037364 -0.428367 -0.212029 0.013207 -0.532184 0\n", 380 | "\n", 381 | "[501 rows x 6 columns]\n" 382 | ] 383 | } 384 | ], 385 | "source": [ 386 | "df[\"Group\"] = model.labels_\n", 387 | "print(df)" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": 10, 393 | "metadata": {}, 394 | "outputs": [ 395 | { 396 | "name": "stdout", 397 | "output_type": "stream", 398 | "text": [ 399 | "MMM False\n", 400 | "ABT False\n", 401 | "ABBV False\n", 402 | "ACN False\n", 403 | "ATVI False\n", 404 | "AYI False\n", 405 | "ADBE False\n", 406 | "AMD False\n", 407 | "AAP False\n", 408 | "AES False\n", 409 | "AET False\n", 410 | "AMG False\n", 411 | "AFL False\n", 412 | "A False\n", 413 | "APD False\n", 414 | "AKAM False\n", 415 | "ALK False\n", 416 | "ALB False\n", 417 | "ARE False\n", 418 | "ALXN False\n", 419 | "ALGN False\n", 420 | "ALLE False\n", 421 | "AGN False\n", 422 | "ADS False\n", 423 | "LNT False\n", 424 | "ALL False\n", 425 | "GOOGL False\n", 426 | "GOOG False\n", 427 | "MO False\n", 428 | "AMZN False\n", 429 | " ... \n", 430 | "V False\n", 431 | "VNO False\n", 432 | "VMC False\n", 433 | "WMT False\n", 434 | "WBA False\n", 435 | "DIS False\n", 436 | "WM False\n", 437 | "WAT False\n", 438 | "WEC False\n", 439 | "WFC False\n", 440 | "HCN False\n", 441 | "WDC False\n", 442 | "WU False\n", 443 | "WRK False\n", 444 | "WY False\n", 445 | "WHR False\n", 446 | "WFM False\n", 447 | "WMB False\n", 448 | "WLTW False\n", 449 | "WYN False\n", 450 | "WYNN True\n", 451 | "XEL False\n", 452 | "XRX False\n", 453 | "XLNX False\n", 454 | "XL False\n", 455 | "XYL False\n", 456 | "YUM False\n", 457 | "ZBH False\n", 458 | "ZION False\n", 459 | "ZTS False\n", 460 | "Name: Group, Length: 501, dtype: bool\n" 461 | ] 462 | } 463 | ], 464 | "source": [ 465 | "print(df[\"Group\"]==1)" 466 | ] 467 | }, 468 | { 469 | "cell_type": "code", 470 | "execution_count": 11, 471 | "metadata": {}, 472 | "outputs": [ 473 | { 474 | "name": "stdout", 475 | "output_type": "stream", 476 | "text": [ 477 | " Gold Index Intercept NaturalGas Oil Group\n", 478 | "IRM 6.017777 -0.493073 -0.212029 0.013207 -0.125598 1\n", 479 | "MNST 4.549059 -0.778753 -0.212029 0.013207 -0.125598 1\n", 480 | "NEM 7.381807 -2.405000 -0.212029 0.013207 1.585726 1\n", 481 | "NLSN 3.698270 -0.308266 -0.212029 0.013207 -0.125598 1\n", 482 | "UA 9.378166 1.237738 -0.212029 0.013207 -0.125598 1\n", 483 | "WYNN 3.515203 1.424921 -0.212029 0.013207 0.419235 1\n" 484 | ] 485 | } 486 | ], 487 | "source": [ 488 | "print(df[df[\"Group\"]==1])" 489 | ] 490 | }, 491 | { 492 | "cell_type": "code", 493 | "execution_count": 12, 494 | "metadata": {}, 495 | "outputs": [ 496 | { 497 | "name": "stdout", 498 | "output_type": "stream", 499 | "text": [ 500 | " Gold Index Intercept NaturalGas Oil Group\n", 501 | "MMM -0.037364 -0.353290 -0.212029 0.013207 -0.125598 0\n", 502 | "ABT -0.037364 -1.040657 -0.212029 0.013207 -0.482210 0\n", 503 | "ACN -0.037364 -0.243264 -0.212029 0.013207 -0.125598 0\n", 504 | "AAP -0.037364 -0.889615 -0.212029 0.013207 -0.743704 0\n", 505 | "AET -0.037364 -0.104926 -0.212029 0.013207 -0.542458 0\n", 506 | "APD -0.037364 -0.180001 -0.212029 0.013207 -0.125598 0\n", 507 | "ARE -0.037364 -0.199351 -0.212029 1.473058 -0.125598 0\n", 508 | "LNT -0.037364 -1.315133 -0.212029 0.013207 -0.436175 0\n", 509 | "ALL -0.037364 -0.286923 -0.212029 0.013207 -0.394047 0\n", 510 | "GOOGL -0.037364 -0.179244 -0.212029 0.013207 -0.518189 0\n", 511 | "GOOG -0.037364 -0.181276 -0.212029 0.013207 -0.502709 0\n", 512 | "AEE -0.037364 -1.371433 -0.212029 0.013207 -0.536962 0\n", 513 | "AEP -0.037364 -1.630641 -0.212029 0.013207 -0.424683 0\n", 514 | "AMT -0.037364 -0.700968 -0.212029 0.013207 -0.125598 0\n", 515 | "AWK 1.051967 -1.644105 -0.212029 0.013207 -0.438131 0\n", 516 | "ABC -0.037364 -1.084857 -0.212029 0.013207 -0.649327 0\n", 517 | "AMGN -0.037364 -0.331923 -0.212029 0.013207 -0.794792 0\n", 518 | "ANSS -0.037364 -0.059983 -0.212029 0.013207 -0.125598 0\n", 519 | "ANTM -0.037364 -0.351870 -0.212029 0.013207 -0.597235 0\n", 520 | "AON -0.037364 -0.320682 -0.212029 0.013207 -0.125598 0\n", 521 | "AIV -0.037364 0.110871 -0.212029 2.010638 -0.489773 0\n", 522 | "AAPL -0.037364 -0.272928 -0.212029 2.679741 -0.535081 0\n", 523 | "ADM -0.037364 -0.265843 -0.212029 0.013207 0.474966 0\n", 524 | "AJG -0.037364 -0.842017 -0.212029 0.013207 -0.325530 0\n", 525 | "AIZ -0.037364 -0.192013 -0.212029 0.013207 -0.125598 0\n", 526 | "T -0.037364 -1.583243 -0.212029 0.013207 -0.125598 0\n", 527 | "ADP -0.037364 -0.441252 -0.212029 -1.128750 -0.469736 0\n", 528 | "AVB -0.037364 -0.644525 -0.212029 0.013207 -0.560700 0\n", 529 | "BLL -0.037364 -0.404363 -0.212029 0.013207 -0.429496 0\n", 530 | "BCR -0.037364 -1.030823 -0.212029 0.013207 -0.554157 0\n", 531 | "... ... ... ... ... ... ...\n", 532 | "TSCO -0.037364 0.096304 -0.212029 0.013207 -0.842755 0\n", 533 | "TDG -0.037364 -0.251038 -0.212029 1.944846 -0.125598 0\n", 534 | "TRV -1.006348 -0.717772 -0.212029 0.013207 -0.485628 0\n", 535 | "UDR -0.037364 -0.280197 -0.212029 1.575018 -0.554958 0\n", 536 | "UPS -0.037364 -0.698490 -0.212029 -1.005039 -0.411684 0\n", 537 | "UTX 0.938637 -0.183754 -0.212029 0.013207 -0.125598 0\n", 538 | "UHS -0.037364 -0.137052 -0.212029 0.013207 -0.125598 0\n", 539 | "VFC -0.037364 -0.324981 -0.212029 0.013207 -0.125598 0\n", 540 | "VAR -0.037364 -0.193959 -0.212029 0.013207 -0.125598 0\n", 541 | "VTR -0.037364 -0.613804 -0.212029 2.465387 -0.485161 0\n", 542 | "VRSN -0.037364 -0.497860 -0.212029 0.013207 -0.125598 0\n", 543 | "VRSK -0.037364 -1.450412 -0.212029 0.013207 -0.658188 0\n", 544 | "VZ -0.037364 -1.604861 -0.212029 0.013207 -0.125598 0\n", 545 | "V -0.037364 -0.118855 -0.212029 0.013207 -0.125598 0\n", 546 | "WMT -0.037364 -1.793953 -0.212029 0.013207 -0.903630 0\n", 547 | "WBA -0.037364 0.022056 -0.212029 0.013207 -0.867695 0\n", 548 | "DIS -1.155992 0.051140 -0.212029 0.013207 -0.518293 0\n", 549 | "WM -0.037364 -1.073051 -0.212029 0.013207 -0.425257 0\n", 550 | "WEC -0.037364 -1.685645 -0.212029 0.013207 -0.480405 0\n", 551 | "HCN -0.037364 -0.930444 -0.212029 0.013207 -0.411589 0\n", 552 | "WU -0.037364 -0.106561 -0.212029 0.013207 -0.125598 0\n", 553 | "WFM -0.037364 -0.345890 -0.212029 0.013207 -0.125598 0\n", 554 | "WLTW -0.037364 -1.156006 -0.212029 0.013207 -0.125598 0\n", 555 | "XEL -0.037364 -1.759744 -0.212029 0.013207 -0.544619 0\n", 556 | "XLNX -0.037364 -0.119870 -0.212029 0.013207 -0.125598 0\n", 557 | "XL -0.037364 -0.435564 -0.212029 0.013207 -0.125598 0\n", 558 | "XYL -0.037364 -0.358508 -0.212029 0.013207 0.542919 0\n", 559 | "YUM -0.037364 -0.346218 -0.212029 0.013207 -0.125598 0\n", 560 | "ZBH -0.037364 -0.479457 -0.212029 0.013207 -0.125598 0\n", 561 | "ZTS -0.037364 -0.428367 -0.212029 0.013207 -0.532184 0\n", 562 | "\n", 563 | "[221 rows x 6 columns]\n", 564 | " Gold Index Intercept NaturalGas Oil Group\n", 565 | "IRM 6.017777 -0.493073 -0.212029 0.013207 -0.125598 1\n", 566 | "MNST 4.549059 -0.778753 -0.212029 0.013207 -0.125598 1\n", 567 | "NEM 7.381807 -2.405000 -0.212029 0.013207 1.585726 1\n", 568 | "NLSN 3.698270 -0.308266 -0.212029 0.013207 -0.125598 1\n", 569 | "UA 9.378166 1.237738 -0.212029 0.013207 -0.125598 1\n", 570 | "WYNN 3.515203 1.424921 -0.212029 0.013207 0.419235 1\n", 571 | " Gold Index Intercept NaturalGas Oil Group\n", 572 | "ABBV -0.037364 0.157857 -0.212029 0.013207 -0.125598 2\n", 573 | "AYI 1.826425 0.955356 -0.212029 0.013207 -0.125598 2\n", 574 | "ADBE -0.037364 0.422960 -0.212029 0.013207 -0.125598 2\n", 575 | "AMD -0.037364 1.834926 -0.212029 0.013207 0.695126 2\n", 576 | "AES -0.037364 0.241443 -0.212029 0.013207 0.715364 2\n", 577 | "AMG -0.037364 2.012051 -0.212029 0.013207 0.332843 2\n", 578 | "AFL -0.037364 0.932364 -0.212029 0.013207 -0.125598 2\n", 579 | "A -0.037364 1.360707 -0.212029 0.013207 -0.125598 2\n", 580 | "AKAM -0.037364 1.013205 -0.212029 0.013207 -0.125598 2\n", 581 | "ALB 1.608568 0.891388 -0.212029 0.013207 0.951058 2\n", 582 | "ALXN -0.037364 0.311582 -0.212029 0.013207 -0.125598 2\n", 583 | "ALGN -0.037364 1.148354 -0.212029 0.013207 -0.688967 2\n", 584 | "AGN -0.037364 0.533200 -0.212029 0.013207 -0.782655 2\n", 585 | "ADS -0.037364 0.162804 -0.212029 0.013207 -0.125598 2\n", 586 | "AMZN -0.037364 0.294471 -0.212029 0.013207 -0.125598 2\n", 587 | "AAL -0.037364 1.769704 -0.212029 0.013207 -1.827080 2\n", 588 | "AXP -0.037364 0.348283 -0.212029 0.013207 -0.568699 2\n", 589 | "AIG -0.037364 1.346869 -0.212029 0.013207 -0.125598 2\n", 590 | "AMP -0.037364 1.800819 -0.212029 0.013207 0.158996 2\n", 591 | "APH -0.037364 0.581549 -0.212029 0.013207 -0.125598 2\n", 592 | "ADI -0.037364 0.635097 -0.212029 0.013207 -0.125598 2\n", 593 | "AMAT -0.037364 0.657139 -0.212029 0.013207 -0.125598 2\n", 594 | "ARNC -0.037364 1.360567 -0.212029 0.013207 1.172481 2\n", 595 | "ADSK -0.037364 1.470466 -0.212029 0.013207 -0.125598 2\n", 596 | "AN -0.037364 0.200197 -0.212029 0.013207 -0.125598 2\n", 597 | "AVY -0.037364 0.352203 -0.212029 0.013207 -0.125598 2\n", 598 | "BIIB -0.037364 0.025548 -0.212029 0.013207 -0.125598 2\n", 599 | "BLK -0.037364 1.133764 -0.212029 0.013207 -0.125598 2\n", 600 | "BA -0.037364 0.249591 -0.212029 0.013207 -0.125598 2\n", 601 | "BWA -0.037364 1.339120 -0.212029 0.013207 0.282327 2\n", 602 | "... ... ... ... ... ... ...\n", 603 | "SIG -0.037364 0.236889 -0.212029 0.013207 0.322643 2\n", 604 | "SWKS -0.037364 2.072223 -0.212029 0.013207 -0.125598 2\n", 605 | "SLG -0.037364 0.733702 -0.212029 0.013207 -0.125598 2\n", 606 | "LUV -0.037364 0.570605 -0.212029 0.013207 -1.993673 2\n", 607 | "SWK -0.037364 0.669428 -0.212029 0.013207 -0.125598 2\n", 608 | "TROW -0.037364 1.262785 -0.212029 -1.219284 -0.125598 2\n", 609 | "TEL -0.037364 0.428200 -0.212029 0.013207 -0.125598 2\n", 610 | "TSO -0.037364 1.159344 -0.212029 0.013207 -0.125598 2\n", 611 | "TXN -0.037364 0.246218 -0.212029 0.013207 -0.125598 2\n", 612 | "TXT -0.037364 1.733603 -0.212029 0.013207 -0.125598 2\n", 613 | "TMO -0.037364 0.138587 -0.212029 0.013207 -0.125598 2\n", 614 | "TIF -0.037364 0.485470 -0.212029 0.013207 -0.125598 2\n", 615 | "FOXA -0.037364 0.760309 -0.212029 0.013207 -0.125598 2\n", 616 | "FOX -0.037364 0.711779 -0.212029 0.013207 -0.125598 2\n", 617 | "UAA -0.037364 1.104841 -0.212029 0.013207 -0.125598 2\n", 618 | "UNP -0.037364 0.195325 -0.212029 0.013207 -0.125598 2\n", 619 | "UAL -0.037364 1.661166 -0.212029 0.013207 -2.916260 2\n", 620 | "URI -0.037364 2.998708 -0.212029 0.013207 1.318902 2\n", 621 | "UNM -0.037364 0.769045 -0.212029 0.013207 0.216445 2\n", 622 | "VLO -0.037364 1.283163 -0.212029 0.013207 -0.125598 2\n", 623 | "VRTX -0.037364 0.527766 -0.212029 0.013207 -0.125598 2\n", 624 | "VIAB -0.037364 0.288969 -0.212029 0.013207 0.497322 2\n", 625 | "VNO -0.037364 0.101117 -0.212029 0.013207 -0.384345 2\n", 626 | "VMC -0.037364 0.853668 -0.212029 0.013207 -0.125598 2\n", 627 | "WAT -0.037364 0.249949 -0.212029 0.013207 -0.125598 2\n", 628 | "WDC -0.037364 1.100756 -0.212029 0.013207 -0.125598 2\n", 629 | "WRK -0.037364 0.309683 -0.212029 0.013207 1.270766 2\n", 630 | "WY 2.055581 0.401595 -0.212029 0.013207 -0.125598 2\n", 631 | "WHR -0.037364 1.145382 -0.212029 0.013207 -0.125598 2\n", 632 | "WYN -0.037364 1.099951 -0.212029 0.013207 -0.125598 2\n", 633 | "\n", 634 | "[167 rows x 6 columns]\n", 635 | " Gold Index Intercept NaturalGas Oil Group\n", 636 | "ALK -0.037364 1.171283 4.895146 0.013207 -2.135875 3\n", 637 | "MO 1.222066 -1.667088 2.581727 0.013207 -0.572746 3\n", 638 | "AZO 1.570637 -1.670183 3.953464 0.013207 -0.729499 3\n", 639 | "CTAS -0.037364 -0.220810 2.835500 0.013207 -0.388660 3\n", 640 | "STZ -0.037364 -0.287369 5.549112 0.013207 -0.125598 3\n", 641 | "COO -0.037364 -0.695445 3.505214 0.013207 -0.475174 3\n", 642 | "DLTR -0.037364 -1.323702 3.752665 0.013207 -0.859026 3\n", 643 | "EXR 1.311295 -0.126970 3.544450 0.013207 -0.694154 3\n", 644 | "FISV -0.037364 -0.245105 2.552038 0.013207 -0.480629 3\n", 645 | "FL -0.037364 0.281993 4.224519 0.013207 -0.640637 3\n", 646 | "IT -0.037364 -0.306754 3.531436 0.013207 -0.484292 3\n", 647 | "HD -0.037364 -0.386481 2.809116 0.013207 -0.726768 3\n", 648 | "HRL -0.037364 -1.388930 2.811987 0.013207 -0.570441 3\n", 649 | "NYSELMT -0.037364 -1.127999 2.250636 0.013207 -0.684056 3\n", 650 | "NFLX -0.037364 0.166396 9.405591 0.013207 -0.125598 3\n", 651 | "NOC -0.037364 -0.384869 2.487579 0.013207 -0.641965 3\n", 652 | "ORLY 1.928130 -1.064320 4.607136 0.013207 -0.863481 3\n", 653 | "REGN -0.037364 0.929923 7.368448 0.013207 -0.971786 3\n", 654 | "RAI 1.121632 -1.530970 3.321576 0.013207 -0.583742 3\n", 655 | "ROST -0.037364 -0.811672 4.292840 0.013207 -0.952582 3\n", 656 | "SHW -0.037364 -0.683939 2.831420 0.013207 -0.592588 3\n", 657 | "TJX -0.037364 -0.722322 2.876262 0.013207 -0.885723 3\n", 658 | "TSN -0.037364 -0.982704 4.769605 0.013207 -0.125598 3\n", 659 | "ULTA -0.037364 0.091139 7.022478 0.013207 -0.125598 3\n", 660 | "UNH -1.472044 -0.439227 3.145739 0.013207 -0.546574 3\n", 661 | " Gold Index Intercept NaturalGas Oil Group\n", 662 | "COG -0.037364 0.144115 -0.212029 5.898647 2.450504 4\n", 663 | "CHK -0.037364 0.226368 -0.212029 5.037403 4.945772 4\n", 664 | "EQT -0.037364 -0.216545 -0.212029 5.901897 1.688908 4\n", 665 | "NRG -0.037364 -0.465991 -0.212029 3.828920 1.323705 4\n", 666 | "QRVO -0.037364 2.494090 -0.212029 8.080680 -0.125598 4\n", 667 | "RRC -0.037364 -0.476543 -0.212029 6.646003 3.595249 4\n", 668 | " Gold Index Intercept NaturalGas Oil Group\n", 669 | "ATVI -0.037364 -0.550813 -0.212029 -2.323145 -0.602309 5\n", 670 | "ALLE -0.037364 0.074981 -0.212029 -1.840692 -0.125598 5\n", 671 | "AME -0.037364 0.398172 -0.212029 -1.351200 0.152526 5\n", 672 | "CHRW -0.037364 -0.734111 -0.212029 -1.839282 -0.125598 5\n", 673 | "CAT 1.663246 0.557075 -0.212029 -2.328826 1.086965 5\n", 674 | "CMG -0.037364 -0.622758 -0.212029 -4.364565 -0.125598 5\n", 675 | "CTSH -0.037364 0.663110 -0.212029 -2.131022 -0.125598 5\n", 676 | "DOV 1.294731 0.492649 -0.212029 -5.102337 1.070828 5\n", 677 | "ETN -0.037364 0.683734 -0.212029 -2.262920 0.583124 5\n", 678 | "EMR 1.078884 0.263763 -0.212029 -1.759968 0.409874 5\n", 679 | "EXPE -0.037364 0.549718 -0.212029 -3.142582 -0.125598 5\n", 680 | "FMC -0.037364 0.279363 -0.212029 -1.758351 0.888024 5\n", 681 | "HOG -0.037364 0.975937 -0.212029 -2.231008 -0.125598 5\n", 682 | "JEC -0.037364 0.739179 -0.212029 -2.017641 0.766014 5\n", 683 | "MHK -0.037364 1.287530 -0.212029 -1.910888 -0.696267 5\n", 684 | "PH -0.037364 0.925265 -0.212029 -1.437470 0.416644 5\n", 685 | "PYPL -0.037364 0.432367 -0.212029 -5.984245 -0.125598 5\n", 686 | "PKI -0.037364 0.228454 -0.212029 -2.734316 -0.125598 5\n", 687 | "PSX -0.037364 0.328316 -0.212029 -3.118581 0.943080 5\n", 688 | "RL -0.037364 0.061883 -0.212029 -2.328263 0.310095 5\n", 689 | "ROP -0.037364 0.091239 -0.212029 -1.445355 -0.125598 5\n", 690 | "SNA 0.935208 0.470112 -0.212029 -1.427104 -0.125598 5\n", 691 | "TRIP -0.037364 0.828456 -0.212029 -4.444568 -0.125598 5\n", 692 | " Gold Index Intercept NaturalGas Oil Group\n", 693 | "APC -0.037364 0.662560 -0.212029 0.013207 3.758347 6\n", 694 | "APA -0.037364 0.021442 -0.212029 0.013207 3.844829 6\n", 695 | "CVX -0.037364 -0.637414 -0.212029 0.013207 1.839171 6\n", 696 | "XEC -0.037364 0.249937 -0.212029 0.013207 3.787234 6\n", 697 | "CXO -0.037364 0.046259 -0.212029 0.013207 4.514565 6\n", 698 | "COP -0.037364 -0.697965 -0.212029 0.013207 3.022128 6\n", 699 | "DVN -0.037364 -0.130709 -0.212029 0.013207 4.212070 6\n", 700 | "EOG -0.037364 0.217549 -0.212029 0.013207 3.230052 6\n", 701 | "FLS 1.643877 0.690943 -0.212029 0.013207 1.701192 6\n", 702 | "FCX 2.666719 1.373591 -0.212029 -4.538934 4.581797 6\n", 703 | "HAL -0.037364 0.696967 -0.212029 0.013207 3.108741 6\n", 704 | "HP -0.037364 0.562014 -0.212029 0.013207 4.082044 6\n", 705 | "HES -0.037364 0.345869 -0.212029 0.013207 3.940547 6\n", 706 | "KMI -0.037364 -1.177047 -0.212029 0.013207 2.358735 6\n", 707 | "LYB 1.933452 1.249650 -0.212029 0.013207 2.038759 6\n", 708 | "MRO -0.037364 -0.080173 -0.212029 0.013207 5.084962 6\n", 709 | "MUR -0.037364 -0.110421 -0.212029 0.013207 4.632634 6\n", 710 | "NOV -0.037364 0.282712 -0.212029 0.013207 2.955504 6\n", 711 | "NFX -0.037364 0.599563 -0.212029 0.013207 4.377602 6\n", 712 | "NYSENBL -0.037364 -0.061520 -0.212029 0.013207 3.259042 6\n", 713 | "OXY -0.037364 -0.132415 -0.212029 0.013207 2.374759 6\n", 714 | "OKE -0.037364 -0.604437 -0.212029 0.013207 3.112968 6\n", 715 | "PXD -0.037364 0.606309 -0.212029 0.013207 3.844561 6\n", 716 | "SLB 1.235321 0.239626 -0.212029 0.013207 2.257183 6\n", 717 | "RIG -0.037364 -0.580909 -0.212029 0.013207 4.877028 6\n", 718 | "WMB -0.037364 0.748471 -0.212029 0.013207 2.445858 6\n", 719 | " Gold Index Intercept NaturalGas Oil Group\n", 720 | "BAC -3.741856 2.135568 -0.212029 0.013207 -0.125598 7\n", 721 | "BK -1.232297 0.881894 -0.212029 0.013207 -0.125598 7\n", 722 | "BBT -2.293568 0.625016 -0.212029 0.013207 -0.431078 7\n", 723 | "COF -1.407998 0.886260 -0.212029 0.013207 -0.125598 7\n", 724 | "SCHW -2.659574 1.341409 -0.212029 0.013207 -0.125598 7\n", 725 | "CFG -5.150936 0.050350 -0.212029 0.013207 -0.125598 7\n", 726 | "CMA -2.781929 0.917943 -0.212029 0.013207 0.311819 7\n", 727 | "DVA -3.769486 -1.302045 -0.212029 0.013207 -0.125598 7\n", 728 | "FITB -2.105578 1.444667 -0.212029 0.013207 -0.125598 7\n", 729 | "HBAN -1.808479 1.406817 -0.212029 0.013207 -0.125598 7\n", 730 | "JPM -2.255710 1.137596 -0.212029 0.013207 -0.125598 7\n", 731 | "KEY -2.151970 1.389809 -0.212029 0.013207 -0.125598 7\n", 732 | "LNC -1.961125 2.687208 -0.212029 0.013207 0.539791 7\n", 733 | "MTB -2.119380 0.215538 -0.212029 0.013207 -0.125598 7\n", 734 | "MET -2.160961 1.722846 -0.212029 0.013207 0.274669 7\n", 735 | "MU -2.955802 2.618412 -0.212029 0.013207 -0.125598 7\n", 736 | "MS -2.381532 2.352819 -0.212029 0.013207 -0.125598 7\n", 737 | "NTRS -2.583096 0.449667 -0.212029 0.013207 -0.125598 7\n", 738 | "PNC -1.933389 0.634961 -0.212029 0.013207 -0.125598 7\n", 739 | "PRU -1.384558 1.799133 -0.212029 0.013207 -0.125598 7\n", 740 | "RF -2.161320 2.170066 -0.212029 0.013207 0.276846 7\n", 741 | "STT -2.052816 1.065609 -0.212029 0.013207 -0.125598 7\n", 742 | "STI -2.387321 1.728233 -0.212029 0.013207 -0.125598 7\n", 743 | "USB -2.591863 0.402497 -0.212029 0.013207 -0.360830 7\n", 744 | "WFC -2.376377 0.934776 -0.212029 0.013207 -0.125598 7\n", 745 | "XRX -1.910188 0.849949 -0.212029 0.013207 -0.125598 7\n", 746 | "ZION -2.963010 1.439493 -0.212029 0.013207 0.658687 7\n" 747 | ] 748 | } 749 | ], 750 | "source": [ 751 | "for x in range(8):\n", 752 | " print(df[df[\"Group\"]==x])" 753 | ] 754 | }, 755 | { 756 | "cell_type": "code", 757 | "execution_count": 13, 758 | "metadata": { 759 | "collapsed": true 760 | }, 761 | "outputs": [], 762 | "source": [ 763 | "SP500 = pd.DataFrame.from_csv(\"SP500.csv\", encoding=\"UTF-8\")\n", 764 | "tickers = []\n", 765 | "for x in SP500[\"Ticker\"].values:\n", 766 | " if x == \"LMT\":\n", 767 | " x = \"NYSELMT\"\n", 768 | " if x ==\"NWL\":\n", 769 | " x = \"NYSENWL\"\n", 770 | " if x==\"NBL\":\n", 771 | " x = \"NYSENBL\"\n", 772 | " if \".\" in x:\n", 773 | " x = x.replace(\".\",\"\")\n", 774 | " tickers.append(x)\n", 775 | "SP500[\"Ticker\"]=tickers" 776 | ] 777 | }, 778 | { 779 | "cell_type": "code", 780 | "execution_count": 14, 781 | "metadata": {}, 782 | "outputs": [ 783 | { 784 | "name": "stdout", 785 | "output_type": "stream", 786 | "text": [ 787 | " Ticker Security SEC Filings \\\n", 788 | "A A Agilent Technologies Inc reports \n", 789 | "AAL AAL American Airlines Group reports \n", 790 | "AAP AAP Advance Auto Parts reports \n", 791 | "AAPL AAPL Apple Inc. reports \n", 792 | "ABBV ABBV AbbVie Inc. reports \n", 793 | "ABC ABC AmerisourceBergen Corp reports \n", 794 | "ABT ABT Abbott Laboratories reports \n", 795 | "ACN ACN Accenture plc reports \n", 796 | "ADBE ADBE Adobe Systems Inc reports \n", 797 | "ADI ADI Analog Devices, Inc. reports \n", 798 | "ADM ADM Archer-Daniels-Midland Co reports \n", 799 | "ADP ADP Automatic Data Processing reports \n", 800 | "ADS ADS Alliance Data Systems reports \n", 801 | "ADSK ADSK Autodesk Inc reports \n", 802 | "AEE AEE Ameren Corp reports \n", 803 | "AEP AEP American Electric Power reports \n", 804 | "AES AES AES Corp reports \n", 805 | "AET AET Aetna Inc reports \n", 806 | "AFL AFL AFLAC Inc reports \n", 807 | "AGN AGN Allergan, Plc reports \n", 808 | "AIG AIG American International Group, Inc. reports \n", 809 | "AIV AIV Apartment Investment & Management reports \n", 810 | "AIZ AIZ Assurant Inc reports \n", 811 | "AJG AJG Arthur J. Gallagher & Co. reports \n", 812 | "AKAM AKAM Akamai Technologies Inc reports \n", 813 | "ALB ALB Albemarle Corp reports \n", 814 | "ALGN ALGN Align Technology reports \n", 815 | "ALK ALK Alaska Air Group Inc reports \n", 816 | "ALL ALL Allstate Corp reports \n", 817 | "ALLE ALLE Allegion reports \n", 818 | "... ... ... ... \n", 819 | "VTR VTR Ventas Inc reports \n", 820 | "VZ VZ Verizon Communications reports \n", 821 | "WAT WAT Waters Corporation reports \n", 822 | "WBA WBA Walgreens Boots Alliance reports \n", 823 | "WDC WDC Western Digital reports \n", 824 | "WEC WEC Wec Energy Group Inc reports \n", 825 | "WFC WFC Wells Fargo reports \n", 826 | "WFM WFM Whole Foods Market reports \n", 827 | "WHR WHR Whirlpool Corp. reports \n", 828 | "WLTW WLTW Willis Towers Watson reports \n", 829 | "WM WM Waste Management Inc. reports \n", 830 | "WMB WMB Williams Cos. reports \n", 831 | "WMT WMT Wal-Mart Stores reports \n", 832 | "WRK WRK WestRock Company reports \n", 833 | "WU WU Western Union Co reports \n", 834 | "WY WY Weyerhaeuser Corp. reports \n", 835 | "WYN WYN Wyndham Worldwide reports \n", 836 | "WYNN WYNN Wynn Resorts Ltd reports \n", 837 | "XEC XEC Cimarex Energy reports \n", 838 | "XEL XEL Xcel Energy Inc reports \n", 839 | "XL XL XL Capital reports \n", 840 | "XLNX XLNX Xilinx Inc reports \n", 841 | "XOM XOM Exxon Mobil Corp. reports \n", 842 | "XRAY XRAY Dentsply Sirona reports \n", 843 | "XRX XRX Xerox Corp. reports \n", 844 | "XYL XYL Xylem Inc. reports \n", 845 | "YUM YUM Yum! Brands Inc reports \n", 846 | "ZBH ZBH Zimmer Biomet Holdings reports \n", 847 | "ZION ZION Zions Bancorp reports \n", 848 | "ZTS ZTS Zoetis reports \n", 849 | "\n", 850 | " GICS Sector \\\n", 851 | "A Health Care \n", 852 | "AAL Industrials \n", 853 | "AAP Consumer Discretionary \n", 854 | "AAPL Information Technology \n", 855 | "ABBV Health Care \n", 856 | "ABC Health Care \n", 857 | "ABT Health Care \n", 858 | "ACN Information Technology \n", 859 | "ADBE Information Technology \n", 860 | "ADI Information Technology \n", 861 | "ADM Consumer Staples \n", 862 | "ADP Information Technology \n", 863 | "ADS Information Technology \n", 864 | "ADSK Information Technology \n", 865 | "AEE Utilities \n", 866 | "AEP Utilities \n", 867 | "AES Utilities \n", 868 | "AET Health Care \n", 869 | "AFL Financials \n", 870 | "AGN Health Care \n", 871 | "AIG Financials \n", 872 | "AIV Real Estate \n", 873 | "AIZ Financials \n", 874 | "AJG Financials \n", 875 | "AKAM Information Technology \n", 876 | "ALB Materials \n", 877 | "ALGN Health Care \n", 878 | "ALK Industrials \n", 879 | "ALL Financials \n", 880 | "ALLE Industrials \n", 881 | "... ... \n", 882 | "VTR Real Estate \n", 883 | "VZ Telecommunication Services \n", 884 | "WAT Health Care \n", 885 | "WBA Consumer Staples \n", 886 | "WDC Information Technology \n", 887 | "WEC Utilities \n", 888 | "WFC Financials \n", 889 | "WFM Consumer Staples \n", 890 | "WHR Consumer Discretionary \n", 891 | "WLTW Financials \n", 892 | "WM Industrials \n", 893 | "WMB Energy \n", 894 | "WMT Consumer Staples \n", 895 | "WRK Materials \n", 896 | "WU Information Technology \n", 897 | "WY Real Estate \n", 898 | "WYN Consumer Discretionary \n", 899 | "WYNN Consumer Discretionary \n", 900 | "XEC Energy \n", 901 | "XEL Utilities \n", 902 | "XL Financials \n", 903 | "XLNX Information Technology \n", 904 | "XOM Energy \n", 905 | "XRAY Health Care \n", 906 | "XRX Information Technology \n", 907 | "XYL Industrials \n", 908 | "YUM Consumer Discretionary \n", 909 | "ZBH Health Care \n", 910 | "ZION Financials \n", 911 | "ZTS Health Care \n", 912 | "\n", 913 | " GICS Sub Industry \\\n", 914 | "A Health Care Equipment \n", 915 | "AAL Airlines \n", 916 | "AAP Automotive Retail \n", 917 | "AAPL Technology Hardware, Storage & Peripherals \n", 918 | "ABBV Pharmaceuticals \n", 919 | "ABC Health Care Distributors \n", 920 | "ABT Health Care Equipment \n", 921 | "ACN IT Consulting & Other Services \n", 922 | "ADBE Application Software \n", 923 | "ADI Semiconductors \n", 924 | "ADM Agricultural Products \n", 925 | "ADP Internet Software & Services \n", 926 | "ADS Data Processing & Outsourced Services \n", 927 | "ADSK Application Software \n", 928 | "AEE Multi-Utilities \n", 929 | "AEP Electric Utilities \n", 930 | "AES Independent Power Producers & Energy Traders \n", 931 | "AET Managed Health Care \n", 932 | "AFL Life & Health Insurance \n", 933 | "AGN Pharmaceuticals \n", 934 | "AIG Property & Casualty Insurance \n", 935 | "AIV Residential REITs \n", 936 | "AIZ Multi-line Insurance \n", 937 | "AJG Insurance Brokers \n", 938 | "AKAM Internet Software & Services \n", 939 | "ALB Specialty Chemicals \n", 940 | "ALGN Health Care Supplies \n", 941 | "ALK Airlines \n", 942 | "ALL Property & Casualty Insurance \n", 943 | "ALLE Building Products \n", 944 | "... ... \n", 945 | "VTR Health Care REITs \n", 946 | "VZ Integrated Telecommunication Services \n", 947 | "WAT Health Care Distributors \n", 948 | "WBA Drug Retail \n", 949 | "WDC Technology Hardware, Storage & Peripherals \n", 950 | "WEC Electric Utilities \n", 951 | "WFC Diversified Banks \n", 952 | "WFM Food Retail \n", 953 | "WHR Household Appliances \n", 954 | "WLTW Insurance Brokers \n", 955 | "WM Environmental & Facilities Services \n", 956 | "WMB Oil & Gas Storage & Transportation \n", 957 | "WMT Hypermarkets & Super Centers \n", 958 | "WRK Paper Packaging \n", 959 | "WU Internet Software & Services \n", 960 | "WY Specialized REITs \n", 961 | "WYN Hotels, Resorts & Cruise Lines \n", 962 | "WYNN Casinos & Gaming \n", 963 | "XEC Oil & Gas Exploration & Production \n", 964 | "XEL Multi-Utilities \n", 965 | "XL Property & Casualty Insurance \n", 966 | "XLNX Semiconductors \n", 967 | "XOM Integrated Oil & Gas \n", 968 | "XRAY Health Care Supplies \n", 969 | "XRX Technology Hardware, Storage & Peripherals \n", 970 | "XYL Industrial Machinery \n", 971 | "YUM Restaurants \n", 972 | "ZBH Health Care Equipment \n", 973 | "ZION Regional Banks \n", 974 | "ZTS Pharmaceuticals \n", 975 | "\n", 976 | " Address Date Added CIK Gold Index \\\n", 977 | "A Santa Clara, California 2000-06-05 1090872 -0.037364 1.360707 \n", 978 | "AAL Fort Worth, Texas 2015-03-23 6201 -0.037364 1.769704 \n", 979 | "AAP Roanoke, Virginia 2015-07-09 1158449 -0.037364 -0.889615 \n", 980 | "AAPL Cupertino, California 1982-11-30 320193 -0.037364 -0.272928 \n", 981 | "ABBV North Chicago, Illinois 2012-12-31 1551152 -0.037364 0.157857 \n", 982 | "ABC Chesterbrook, Pennsylvania 2001-08-30 1140859 -0.037364 -1.084857 \n", 983 | "ABT North Chicago, Illinois 1964-03-31 1800 -0.037364 -1.040657 \n", 984 | "ACN Dublin, Ireland 2011-07-06 1467373 -0.037364 -0.243264 \n", 985 | "ADBE San Jose, California 1997-05-05 796343 -0.037364 0.422960 \n", 986 | "ADI Norwood, Massachusetts NaN 6281 -0.037364 0.635097 \n", 987 | "ADM Decatur, Illinois 1981-07-29 7084 -0.037364 -0.265843 \n", 988 | "ADP Roseland, New Jersey 1981-03-31 8670 -0.037364 -0.441252 \n", 989 | "ADS Plano, Texas 2013-12-23 1101215 -0.037364 0.162804 \n", 990 | "ADSK San Rafael, California 1989-12-01 769397 -0.037364 1.470466 \n", 991 | "AEE St. Louis, Missouri 1991-09-19 1002910 -0.037364 -1.371433 \n", 992 | "AEP Columbus, Ohio NaN 4904 -0.037364 -1.630641 \n", 993 | "AES Arlington, Virginia NaN 874761 -0.037364 0.241443 \n", 994 | "AET Hartford, Connecticut 1976-06-30 1122304 -0.037364 -0.104926 \n", 995 | "AFL Columbus, Georgia NaN 4977 -0.037364 0.932364 \n", 996 | "AGN Dublin, Ireland NaN 884629 -0.037364 0.533200 \n", 997 | "AIG New York, New York 1980-03-31 5272 -0.037364 1.346869 \n", 998 | "AIV Denver, Colorado NaN 922864 -0.037364 0.110871 \n", 999 | "AIZ New York, New York 2007-04-10 1267238 -0.037364 -0.192013 \n", 1000 | "AJG Itasca, Illinois 2016-05-31 354190 -0.037364 -0.842017 \n", 1001 | "AKAM Cambridge, Massachusetts 2007-07-12 1086222 -0.037364 1.013205 \n", 1002 | "ALB Baton Rouge, Louisiana 2016-07-01 915913 1.608568 0.891388 \n", 1003 | "ALGN San Jose, California 2017-06-19 1097149 -0.037364 1.148354 \n", 1004 | "ALK Seattle, Washington 2016-05-13 766421 -0.037364 1.171283 \n", 1005 | "ALL Northfield Township, Illinois NaN 899051 -0.037364 -0.286923 \n", 1006 | "ALLE Dublin, Ireland 2013-12-02 1579241 -0.037364 0.074981 \n", 1007 | "... ... ... ... ... ... \n", 1008 | "VTR Chicago, Illinois 2009-03-04 740260 -0.037364 -0.613804 \n", 1009 | "VZ New York, New York 1983-11-30 732712 -0.037364 -1.604861 \n", 1010 | "WAT Milford, Massachusetts NaN 1000697 -0.037364 0.249949 \n", 1011 | "WBA Deerfield, Illinois 1979-12-31 1618921 -0.037364 0.022056 \n", 1012 | "WDC Irvine, California 2009-07-01 106040 -0.037364 1.100756 \n", 1013 | "WEC Milwaukee, Wisconsin 2008-10-31 783325 -0.037364 -1.685645 \n", 1014 | "WFC San Francisco, California 1976-06-30 72971 -2.376377 0.934776 \n", 1015 | "WFM Austin, Texas NaN 865436 -0.037364 -0.345890 \n", 1016 | "WHR Benton Harbor, Michigan NaN 106640 -0.037364 1.145382 \n", 1017 | "WLTW London, United Kingdom 2016-01-05 1140536 -0.037364 -1.156006 \n", 1018 | "WM Houston, Texas NaN 823768 -0.037364 -1.073051 \n", 1019 | "WMB Tulsa, Oklahoma 1975-03-31 107263 -0.037364 0.748471 \n", 1020 | "WMT Bentonville, Arkansas 1982-08-31 104169 -0.037364 -1.793953 \n", 1021 | "WRK Richmond, Virginia NaN 1636023 -0.037364 0.309683 \n", 1022 | "WU Englewood, Colorado NaN 1365135 -0.037364 -0.106561 \n", 1023 | "WY Federal Way, Washington NaN 106535 2.055581 0.401595 \n", 1024 | "WYN Parsippany, New Jersey NaN 1361658 -0.037364 1.099951 \n", 1025 | "WYNN Las Vegas, Nevada 2008-11-14 1174922 3.515203 1.424921 \n", 1026 | "XEC Denver, Colorado 2014-06-21 1168054 -0.037364 0.249937 \n", 1027 | "XEL Minneapolis, Minnesota NaN 72903 -0.037364 -1.759744 \n", 1028 | "XL Hamilton, Bermuda NaN 875159 -0.037364 -0.435564 \n", 1029 | "XLNX San Jose, California 1999-11-08 743988 -0.037364 -0.119870 \n", 1030 | "XOM Irving, Texas NaN 34088 -0.037364 -0.863888 \n", 1031 | "XRAY York, Pennsylvania 2008-11-14 818479 -0.037364 -0.032380 \n", 1032 | "XRX Norwalk, Connecticut NaN 108772 -1.910188 0.849949 \n", 1033 | "XYL White Plains, New York 2011-11-01 1524472 -0.037364 -0.358508 \n", 1034 | "YUM Louisville, Kentucky 1997-10-06 1041061 -0.037364 -0.346218 \n", 1035 | "ZBH Warsaw, Indiana 2001-08-07 1136869 -0.037364 -0.479457 \n", 1036 | "ZION Salt Lake City, Utah NaN 109380 -2.963010 1.439493 \n", 1037 | "ZTS Florham Park, New Jersey 2013-06-21 1555280 -0.037364 -0.428367 \n", 1038 | "\n", 1039 | " Intercept NaturalGas Oil Group \n", 1040 | "A -0.212029 0.013207 -0.125598 2.0 \n", 1041 | "AAL -0.212029 0.013207 -1.827080 2.0 \n", 1042 | "AAP -0.212029 0.013207 -0.743704 0.0 \n", 1043 | "AAPL -0.212029 2.679741 -0.535081 0.0 \n", 1044 | "ABBV -0.212029 0.013207 -0.125598 2.0 \n", 1045 | "ABC -0.212029 0.013207 -0.649327 0.0 \n", 1046 | "ABT -0.212029 0.013207 -0.482210 0.0 \n", 1047 | "ACN -0.212029 0.013207 -0.125598 0.0 \n", 1048 | "ADBE -0.212029 0.013207 -0.125598 2.0 \n", 1049 | "ADI -0.212029 0.013207 -0.125598 2.0 \n", 1050 | "ADM -0.212029 0.013207 0.474966 0.0 \n", 1051 | "ADP -0.212029 -1.128750 -0.469736 0.0 \n", 1052 | "ADS -0.212029 0.013207 -0.125598 2.0 \n", 1053 | "ADSK -0.212029 0.013207 -0.125598 2.0 \n", 1054 | "AEE -0.212029 0.013207 -0.536962 0.0 \n", 1055 | "AEP -0.212029 0.013207 -0.424683 0.0 \n", 1056 | "AES -0.212029 0.013207 0.715364 2.0 \n", 1057 | "AET -0.212029 0.013207 -0.542458 0.0 \n", 1058 | "AFL -0.212029 0.013207 -0.125598 2.0 \n", 1059 | "AGN -0.212029 0.013207 -0.782655 2.0 \n", 1060 | "AIG -0.212029 0.013207 -0.125598 2.0 \n", 1061 | "AIV -0.212029 2.010638 -0.489773 0.0 \n", 1062 | "AIZ -0.212029 0.013207 -0.125598 0.0 \n", 1063 | "AJG -0.212029 0.013207 -0.325530 0.0 \n", 1064 | "AKAM -0.212029 0.013207 -0.125598 2.0 \n", 1065 | "ALB -0.212029 0.013207 0.951058 2.0 \n", 1066 | "ALGN -0.212029 0.013207 -0.688967 2.0 \n", 1067 | "ALK 4.895146 0.013207 -2.135875 3.0 \n", 1068 | "ALL -0.212029 0.013207 -0.394047 0.0 \n", 1069 | "ALLE -0.212029 -1.840692 -0.125598 5.0 \n", 1070 | "... ... ... ... ... \n", 1071 | "VTR -0.212029 2.465387 -0.485161 0.0 \n", 1072 | "VZ -0.212029 0.013207 -0.125598 0.0 \n", 1073 | "WAT -0.212029 0.013207 -0.125598 2.0 \n", 1074 | "WBA -0.212029 0.013207 -0.867695 0.0 \n", 1075 | "WDC -0.212029 0.013207 -0.125598 2.0 \n", 1076 | "WEC -0.212029 0.013207 -0.480405 0.0 \n", 1077 | "WFC -0.212029 0.013207 -0.125598 7.0 \n", 1078 | "WFM -0.212029 0.013207 -0.125598 0.0 \n", 1079 | "WHR -0.212029 0.013207 -0.125598 2.0 \n", 1080 | "WLTW -0.212029 0.013207 -0.125598 0.0 \n", 1081 | "WM -0.212029 0.013207 -0.425257 0.0 \n", 1082 | "WMB -0.212029 0.013207 2.445858 6.0 \n", 1083 | "WMT -0.212029 0.013207 -0.903630 0.0 \n", 1084 | "WRK -0.212029 0.013207 1.270766 2.0 \n", 1085 | "WU -0.212029 0.013207 -0.125598 0.0 \n", 1086 | "WY -0.212029 0.013207 -0.125598 2.0 \n", 1087 | "WYN -0.212029 0.013207 -0.125598 2.0 \n", 1088 | "WYNN -0.212029 0.013207 0.419235 1.0 \n", 1089 | "XEC -0.212029 0.013207 3.787234 6.0 \n", 1090 | "XEL -0.212029 0.013207 -0.544619 0.0 \n", 1091 | "XL -0.212029 0.013207 -0.125598 0.0 \n", 1092 | "XLNX -0.212029 0.013207 -0.125598 0.0 \n", 1093 | "XOM -0.212029 0.013207 1.194556 0.0 \n", 1094 | "XRAY -0.212029 0.013207 -0.489745 0.0 \n", 1095 | "XRX -0.212029 0.013207 -0.125598 7.0 \n", 1096 | "XYL -0.212029 0.013207 0.542919 0.0 \n", 1097 | "YUM -0.212029 0.013207 -0.125598 0.0 \n", 1098 | "ZBH -0.212029 0.013207 -0.125598 0.0 \n", 1099 | "ZION -0.212029 0.013207 0.658687 7.0 \n", 1100 | "ZTS -0.212029 0.013207 -0.532184 0.0 \n", 1101 | "\n", 1102 | "[505 rows x 14 columns]\n" 1103 | ] 1104 | } 1105 | ], 1106 | "source": [ 1107 | "SP500.index = SP500[\"Ticker\"]\n", 1108 | "SP500 = pd.concat([SP500,df],axis=1)\n", 1109 | "print(SP500)" 1110 | ] 1111 | }, 1112 | { 1113 | "cell_type": "code", 1114 | "execution_count": 15, 1115 | "metadata": {}, 1116 | "outputs": [ 1117 | { 1118 | "name": "stdout", 1119 | "output_type": "stream", 1120 | "text": [ 1121 | "['Health Care' 'Industrials' 'Consumer Discretionary'\n", 1122 | " 'Information Technology' 'Consumer Staples' 'Utilities' 'Financials'\n", 1123 | " 'Real Estate' 'Materials' 'Energy' 'Telecommunication Services']\n" 1124 | ] 1125 | } 1126 | ], 1127 | "source": [ 1128 | "print(SP500[\"GICS Sector\"].unique())" 1129 | ] 1130 | }, 1131 | { 1132 | "cell_type": "code", 1133 | "execution_count": 16, 1134 | "metadata": {}, 1135 | "outputs": [ 1136 | { 1137 | "name": "stdout", 1138 | "output_type": "stream", 1139 | "text": [ 1140 | "Consumer Discretionary 85\n", 1141 | "Information Technology 69\n", 1142 | "Financials 67\n", 1143 | "Industrials 65\n", 1144 | "Health Care 61\n", 1145 | "Consumer Staples 36\n", 1146 | "Energy 34\n", 1147 | "Real Estate 31\n", 1148 | "Utilities 28\n", 1149 | "Materials 25\n", 1150 | "Telecommunication Services 4\n", 1151 | "Name: GICS Sector, dtype: int64\n" 1152 | ] 1153 | } 1154 | ], 1155 | "source": [ 1156 | "print(SP500[\"GICS Sector\"].value_counts())" 1157 | ] 1158 | }, 1159 | { 1160 | "cell_type": "code", 1161 | "execution_count": 17, 1162 | "metadata": {}, 1163 | "outputs": [ 1164 | { 1165 | "name": "stdout", 1166 | "output_type": "stream", 1167 | "text": [ 1168 | "7.0 24\n", 1169 | "0.0 22\n", 1170 | "2.0 21\n", 1171 | "Name: Group, dtype: int64\n" 1172 | ] 1173 | } 1174 | ], 1175 | "source": [ 1176 | "print(SP500[SP500[\"GICS Sector\"]==\"Financials\"][\"Group\"].value_counts())" 1177 | ] 1178 | }, 1179 | { 1180 | "cell_type": "code", 1181 | "execution_count": 18, 1182 | "metadata": {}, 1183 | "outputs": [ 1184 | { 1185 | "name": "stdout", 1186 | "output_type": "stream", 1187 | "text": [ 1188 | "0.0 37\n", 1189 | "2.0 19\n", 1190 | "3.0 3\n", 1191 | "5.0 1\n", 1192 | "7.0 1\n", 1193 | "Name: Group, dtype: int64\n", 1194 | "2.0 29\n", 1195 | "0.0 19\n", 1196 | "5.0 10\n", 1197 | "3.0 4\n", 1198 | "1.0 1\n", 1199 | "6.0 1\n", 1200 | "Name: Group, dtype: int64\n", 1201 | "2.0 38\n", 1202 | "0.0 30\n", 1203 | "3.0 8\n", 1204 | "5.0 7\n", 1205 | "1.0 2\n", 1206 | "Name: Group, dtype: int64\n", 1207 | "2.0 32\n", 1208 | "0.0 27\n", 1209 | "3.0 3\n", 1210 | "5.0 3\n", 1211 | "7.0 2\n", 1212 | "4.0 1\n", 1213 | "Name: Group, dtype: int64\n", 1214 | "0.0 30\n", 1215 | "3.0 5\n", 1216 | "1.0 1\n", 1217 | "Name: Group, dtype: int64\n", 1218 | "0.0 26\n", 1219 | "4.0 1\n", 1220 | "2.0 1\n", 1221 | "Name: Group, dtype: int64\n", 1222 | "7.0 24\n", 1223 | "0.0 22\n", 1224 | "2.0 21\n", 1225 | "Name: Group, dtype: int64\n", 1226 | "0.0 20\n", 1227 | "2.0 9\n", 1228 | "1.0 1\n", 1229 | "3.0 1\n", 1230 | "Name: Group, dtype: int64\n", 1231 | "2.0 14\n", 1232 | "0.0 6\n", 1233 | "6.0 2\n", 1234 | "3.0 1\n", 1235 | "1.0 1\n", 1236 | "5.0 1\n", 1237 | "Name: Group, dtype: int64\n", 1238 | "6.0 23\n", 1239 | "4.0 4\n", 1240 | "2.0 3\n", 1241 | "0.0 1\n", 1242 | "5.0 1\n", 1243 | "Name: Group, dtype: int64\n", 1244 | "0.0 3\n", 1245 | "2.0 1\n", 1246 | "Name: Group, dtype: int64\n" 1247 | ] 1248 | } 1249 | ], 1250 | "source": [ 1251 | "for x in SP500[\"GICS Sector\"].unique():\n", 1252 | " print(SP500[SP500[\"GICS Sector\"]==x][\"Group\"].value_counts())" 1253 | ] 1254 | }, 1255 | { 1256 | "cell_type": "code", 1257 | "execution_count": 19, 1258 | "metadata": {}, 1259 | "outputs": [ 1260 | { 1261 | "name": "stdout", 1262 | "output_type": "stream", 1263 | "text": [ 1264 | "Group GICS Sector \n", 1265 | "0.0 Health Care 37\n", 1266 | " Consumer Discretionary 30\n", 1267 | " Consumer Staples 30\n", 1268 | " Information Technology 27\n", 1269 | " Utilities 26\n", 1270 | " Financials 22\n", 1271 | " Real Estate 20\n", 1272 | " Industrials 19\n", 1273 | " Materials 6\n", 1274 | " Telecommunication Services 3\n", 1275 | " Energy 1\n", 1276 | "1.0 Consumer Discretionary 2\n", 1277 | " Consumer Staples 1\n", 1278 | " Industrials 1\n", 1279 | " Materials 1\n", 1280 | " Real Estate 1\n", 1281 | "2.0 Consumer Discretionary 38\n", 1282 | " Information Technology 32\n", 1283 | " Industrials 29\n", 1284 | " Financials 21\n", 1285 | " Health Care 19\n", 1286 | " Materials 14\n", 1287 | " Real Estate 9\n", 1288 | " Energy 3\n", 1289 | " Telecommunication Services 1\n", 1290 | " Utilities 1\n", 1291 | "3.0 Consumer Discretionary 8\n", 1292 | " Consumer Staples 5\n", 1293 | " Industrials 4\n", 1294 | " Health Care 3\n", 1295 | " Information Technology 3\n", 1296 | " Materials 1\n", 1297 | " Real Estate 1\n", 1298 | "4.0 Energy 4\n", 1299 | " Information Technology 1\n", 1300 | " Utilities 1\n", 1301 | "5.0 Industrials 10\n", 1302 | " Consumer Discretionary 7\n", 1303 | " Information Technology 3\n", 1304 | " Energy 1\n", 1305 | " Health Care 1\n", 1306 | " Materials 1\n", 1307 | "6.0 Energy 23\n", 1308 | " Materials 2\n", 1309 | " Industrials 1\n", 1310 | "7.0 Financials 24\n", 1311 | " Information Technology 2\n", 1312 | " Health Care 1\n", 1313 | "Name: GICS Sector, dtype: int64\n" 1314 | ] 1315 | } 1316 | ], 1317 | "source": [ 1318 | "print(SP500.groupby(\"Group\")[\"GICS Sector\"].value_counts())" 1319 | ] 1320 | } 1321 | ], 1322 | "metadata": { 1323 | "kernelspec": { 1324 | "display_name": "Python 3", 1325 | "language": "python", 1326 | "name": "python3" 1327 | }, 1328 | "language_info": { 1329 | "codemirror_mode": { 1330 | "name": "ipython", 1331 | "version": 3 1332 | }, 1333 | "file_extension": ".py", 1334 | "mimetype": "text/x-python", 1335 | "name": "python", 1336 | "nbconvert_exporter": "python", 1337 | "pygments_lexer": "ipython3", 1338 | "version": "3.6.1" 1339 | } 1340 | }, 1341 | "nbformat": 4, 1342 | "nbformat_minor": 2 1343 | } 1344 | -------------------------------------------------------------------------------- /7 Machine Learning Function.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from sklearn.cluster import KMeans\n", 12 | "import pandas as pd\n", 13 | "def group(n):\n", 14 | " df = pd.DataFrame.from_csv(\"RegressionMatrix1.csv\",encoding=\"UTF-8\")\n", 15 | " SP500 = pd.DataFrame.from_csv(\"SP500.csv\", encoding=\"UTF-8\")\n", 16 | " df = df.fillna(0)\n", 17 | " tickers = []\n", 18 | " for x in SP500[\"Ticker\"].values:\n", 19 | " if x == \"LMT\":\n", 20 | " x = \"NYSELMT\"\n", 21 | " if x ==\"NWL\":\n", 22 | " x = \"NYSENWL\"\n", 23 | " if x==\"NBL\":\n", 24 | " x = \"NYSENBL\"\n", 25 | " if \".\" in x:\n", 26 | " x = x.replace(\".\",\"\")\n", 27 | " tickers.append(x)\n", 28 | " SP500[\"Ticker\"]=tickers\n", 29 | " SP500.index = SP500[\"Ticker\"]\n", 30 | "\n", 31 | " SP500.drop([\"FTI\",\"COL\",'DXC',\"BHGE\"],inplace=True)\n", 32 | "\n", 33 | " df = df.transpose()\n", 34 | "\n", 35 | " zScores = pd.DataFrame()\n", 36 | " for x in df.columns:\n", 37 | " zScores[x] = (df[x] - df[x].mean())/df[x].std(ddof=0)\n", 38 | " \n", 39 | " SP500 = pd.concat([SP500,df],axis=1)\n", 40 | " \n", 41 | " \n", 42 | " model = KMeans(n_clusters=n)\n", 43 | " model = model.fit(zScores)\n", 44 | "\n", 45 | " SP500[\"Group\"] = model.labels_\n", 46 | " return SP500" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 2, 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "name": "stdout", 56 | "output_type": "stream", 57 | "text": [ 58 | "Group GICS Sector \n", 59 | "0 Health Care 37\n", 60 | " Consumer Discretionary 30\n", 61 | " Consumer Staples 30\n", 62 | " Information Technology 27\n", 63 | " Utilities 26\n", 64 | " Financials 22\n", 65 | " Real Estate 20\n", 66 | " Industrials 19\n", 67 | " Materials 6\n", 68 | " Telecommunication Services 3\n", 69 | " Energy 1\n", 70 | "1 Consumer Discretionary 38\n", 71 | " Information Technology 32\n", 72 | " Industrials 30\n", 73 | " Financials 21\n", 74 | " Health Care 19\n", 75 | " Materials 14\n", 76 | " Real Estate 9\n", 77 | " Energy 3\n", 78 | " Telecommunication Services 1\n", 79 | " Utilities 1\n", 80 | "2 Energy 23\n", 81 | " Materials 1\n", 82 | "3 Consumer Discretionary 8\n", 83 | " Consumer Staples 5\n", 84 | " Industrials 4\n", 85 | " Health Care 3\n", 86 | " Information Technology 3\n", 87 | " Materials 1\n", 88 | " Real Estate 1\n", 89 | "4 Energy 4\n", 90 | " Information Technology 1\n", 91 | " Utilities 1\n", 92 | "5 Consumer Discretionary 2\n", 93 | " Consumer Staples 1\n", 94 | " Industrials 1\n", 95 | " Materials 1\n", 96 | " Real Estate 1\n", 97 | "6 Industrials 10\n", 98 | " Consumer Discretionary 7\n", 99 | " Information Technology 3\n", 100 | " Materials 2\n", 101 | " Energy 1\n", 102 | " Health Care 1\n", 103 | "7 Financials 24\n", 104 | " Information Technology 2\n", 105 | " Health Care 1\n", 106 | "Name: GICS Sector, dtype: int64\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "SP500 = group(8)\n", 112 | "print(SP500.groupby(\"Group\")[\"GICS Sector\"].value_counts())" 113 | ] 114 | } 115 | ], 116 | "metadata": { 117 | "kernelspec": { 118 | "display_name": "Python 3", 119 | "language": "python", 120 | "name": "python3" 121 | }, 122 | "language_info": { 123 | "codemirror_mode": { 124 | "name": "ipython", 125 | "version": 3 126 | }, 127 | "file_extension": ".py", 128 | "mimetype": "text/x-python", 129 | "name": "python", 130 | "nbconvert_exporter": "python", 131 | "pygments_lexer": "ipython3", 132 | "version": "3.6.1" 133 | } 134 | }, 135 | "nbformat": 4, 136 | "nbformat_minor": 2 137 | } 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is the clustering industries project for FinanceAndPython.com 2 | --------------------------------------------------------------------------------