├── .gitignore ├── LICENSE ├── README.md ├── analyzing_and_manipulating_data_with_the_pydata_stack ├── example_00 │ └── sample_cycle.csv ├── example_01 │ └── SNL Data │ │ ├── .DS_Store │ │ ├── SNL_18650_NMC_15C_0-100_0.5-1C_a_timeseries.csv │ │ ├── SNL_18650_NMC_25C_0-100_0.5-1C_a_timeseries.csv │ │ └── SNL_18650_NMC_35C_0-100_0.5-1C_a_timeseries.csv ├── example_02 │ └── SNL_18650_NMC_15C_0-100_0.5-1C_a_timeseries.csv ├── example_03 │ └── SNL_18650_LFP_35C_0-100_0.5-1C_a_timeseries.csv ├── example_04 │ ├── ORNL_abuse │ │ └── file.xlsx │ ├── SNL_abuse │ │ └── file.xlsx │ └── cell_list.xlsx └── readme.md ├── battery_life_prediction_with_neural_networks ├── README.md ├── cnn_life_predict.ipynb ├── exercise-1_keras.ipynb └── neural-networks-tutorial.pdf ├── environment.yml ├── figures └── eis_to_q.png └── predict_capacity_from_eis ├── data ├── data_eis_zhang2020.csv ├── freq_eis_zhang2020.csv └── ref.txt ├── eisplot.py ├── ml_predicting_capacity_with_eis.ipynb ├── ml_predicting_capacity_with_eis_solutions.ipynb └── plotyy.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | *.DS_Store 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 battery-data-commons 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MRS SP22 Tutorial 2 | 3 | Materials for a tutorial on Battery Data Science and MRS SP22 4 | 5 | ## Using these tutorial materials 6 | 7 | These tutorials are designed to be run on cloud resources. 8 | Simply click the following box to open the resources on Binder. 9 | 10 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/battery-data-commons/mrs-sp22-tutorial/HEAD) 11 | 12 | A few of the examples are best run on systems with a GPU. You'll see links to running those examples on Google Colab where appropriate. 13 | 14 | ### Local Installation 15 | 16 | There are plenty of good reasons to run these examples on your own compute. 17 | For example, using the cloud means that the results and any notes will disappear once you are done. 18 | 19 | The first step is to install [Anaconda](https://docs.conda.io/en/latest/miniconda.html) on your computer and then build the environment by calling the following command from the command line. 20 | 21 | ```bash 22 | conda env create --file environment.yml --force 23 | ``` 24 | 25 | Once done installing, Anaconda will give you a command to activate the computation environment. Active then call `jupyter lab` to open Jupyter. 26 | 27 | ## Machine learning with battery data 28 | Tutorial instructor: Dr. Paul Gasper, Staff Scientist, National Renewable Energy Laboratory 29 | 30 | 31 | 32 | This tutorial walks through the development of a machine-learning model predicting battery capacity from electrochemical impedance spectroscopy data, illustrating basic machine-learning topics such as evaluating model fitness with train/test splits, feature engineering methods, model interrogation, and visualizing predictions using the sklearn library for model development and matplotlib for visualization. The tutorial will use open-source data from Zhang et al, *Nature Communications* (2020) 11:1706 ([paper](https://www.nature.com/articles/s41467-020-15235-7.pdf), [github](https://github.com/YunweiZhang/ML-identify-battery-degradation)). This tutorial uses the following files in the 'predict_capacity_from_eis' folder: 33 | - [ml_predicting_capacity_with_eis.ipynb](predict_capacity_from_eis/ml_predicting_capacity_with_eis.ipynb): Notebook for the tutorial 34 | - [ml_predicting_capacity_with_eis_solutions.ipynb](predict_capacity_from_eis/ml_predicting_capacity_with_eis_solutions.ipynb): Notebook with solutions to challenge problems 35 | - eisplot.py: visualization function for plotting impedance data 36 | - plotyy.py: visualization function for creating parity plots 37 | - data/data_eis_zhang2020.csv: capacity and EIS measurements from 8 cells throughout their lifetime, from Zhang et al 2020 (see ref. above) 38 | - data/freq_eis_zhang2020.csv: frequency vector for plotting impedance data 39 | -------------------------------------------------------------------------------- /analyzing_and_manipulating_data_with_the_pydata_stack/example_01/SNL Data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/battery-data-commons/mrs-sp22-tutorial/64b420d2365f2ff26b6ea50617923db3a80c819b/analyzing_and_manipulating_data_with_the_pydata_stack/example_01/SNL Data/.DS_Store -------------------------------------------------------------------------------- /analyzing_and_manipulating_data_with_the_pydata_stack/example_04/ORNL_abuse/file.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/battery-data-commons/mrs-sp22-tutorial/64b420d2365f2ff26b6ea50617923db3a80c819b/analyzing_and_manipulating_data_with_the_pydata_stack/example_04/ORNL_abuse/file.xlsx -------------------------------------------------------------------------------- /analyzing_and_manipulating_data_with_the_pydata_stack/example_04/SNL_abuse/file.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/battery-data-commons/mrs-sp22-tutorial/64b420d2365f2ff26b6ea50617923db3a80c819b/analyzing_and_manipulating_data_with_the_pydata_stack/example_04/SNL_abuse/file.xlsx -------------------------------------------------------------------------------- /analyzing_and_manipulating_data_with_the_pydata_stack/example_04/cell_list.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/battery-data-commons/mrs-sp22-tutorial/64b420d2365f2ff26b6ea50617923db3a80c819b/analyzing_and_manipulating_data_with_the_pydata_stack/example_04/cell_list.xlsx -------------------------------------------------------------------------------- /analyzing_and_manipulating_data_with_the_pydata_stack/readme.md: -------------------------------------------------------------------------------- 1 | ## Analyzing and Manipulating Data with The PyData Stack 2 | 3 | Hello, 4 | 5 | Below are the Colab examples presented at MRS 2022. Each of these examples are run using Google Colab. 6 | 7 | Colab, or "Colaboratory", allows you to write and execute Python in your browser, with 8 | - Zero configuration required 9 | - Access to GPUs free of charge 10 | - Easy sharing 11 | - 12 | [Example 0. Intro to APIs - Jupyter Notebook Demo](https://colab.research.google.com/drive/1I0_EJR6uNo0hpLHfz46oP8szLPNwMboI?usp=sharing) 13 | 14 | [Example 1. Simple Manipulations - Jupyter Notebook Demo](https://colab.research.google.com/drive/1AlV8EVjGv4gtwIXgWpz9GJWD-fbkR-VR?usp=sharing) 15 | 16 | [Example 2. Compare with Simulation Data - Jupyter Notebook Demo](https://colab.research.google.com/drive/1gSOjMaVLE24EBzHxe0wx6QP-9rSrWOPc?usp=sharing) 17 | 18 | [Example 3. Compare with Synthetic Data - Jupyter Notebook Demo](https://colab.research.google.com/drive/1Fi28cNffOjF1ZOsy8UZqWo21yg0wr4Gh?usp=sharing) 19 | 20 | [Example 4. Abuse Test Data - Jupyter Notebook Demo](https://colab.research.google.com/drive/12GoNag6UjhNSmYKwE97OUYhu-3SVXSz2?usp=sharing) 21 | -------------------------------------------------------------------------------- /battery_life_prediction_with_neural_networks/README.md: -------------------------------------------------------------------------------- 1 | # Life Prediction with Neural Networks 2 | 3 | Neural networks are an exceptionally-useful tool for creating complex machine learning models. 4 | In these examples, we demonstrate two examples: 5 | 6 | 1. [An general overview of how to train neural networks with Keras](./exercise-1_keras.ipynb) ([Colab](https://colab.research.google.com/github/battery-data-commons/mrs-sp22-tutorial/blob/main/battery_life_prediction_with_neural_networks/exercise-1_keras.ipynb)) 7 | 1. [A specific example of training a neural network for predicting battery life](./cnn_life_predict.ipynb) ([Colab](https://colab.research.google.com/github/battery-data-commons/mrs-sp22-tutorial/blob/main/battery_life_prediction_with_neural_networks/cnn_life_predict.ipynb)) 8 | 9 | Both can run on small computing resources, but are much faster to run on a GPU. 10 | Click the "Colab" link to access GPU resources hosted by Google Colaboratory. 11 | -------------------------------------------------------------------------------- /battery_life_prediction_with_neural_networks/cnn_life_predict.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "d8544be7", 6 | "metadata": {}, 7 | "source": [ 8 | "# Training a CNN to Predict Battery Cycle Life\n", 9 | "*Noah Paulson, Argonne National Laboratory*\n", 10 | "\n", 11 | "\n", 12 | "In this notebook we will collect data, visualize the machine learning features, define a convolutional neural network architecture, train it, and evaluate the results.\n", 13 | "\n", 14 | "### Import packages, define data loader" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "c8ca0704", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import h5py\n", 25 | "import time\n", 26 | "import numpy as np\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "import seaborn as sns\n", 29 | "import tensorflow as tf\n", 30 | "from tensorflow import keras\n", 31 | "from urllib import request" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "id": "92b53dba", 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "def get_data(logy=True, output=False):\n", 42 | "\n", 43 | " remote_url = 'https://drive.google.com/uc?export=download&id=1orXDHcq2TzRLhGkHyWVtsuGn7_10pSAn'\n", 44 | " fname = 'energy_capacity.h5'\n", 45 | " request.urlretrieve(remote_url, fname)\n", 46 | " \n", 47 | " # load the raw data from hdf5 file\n", 48 | " f = h5py.File(fname, 'r')\n", 49 | " \n", 50 | " # evaluated cycles and voltages\n", 51 | " cyceval = f['evaluated cycles']\n", 52 | " veval = f['evaluated voltages']\n", 53 | " \n", 54 | " Xin = [] # feature array\n", 55 | " cycfailV = np.array([]) # failure cycle array\n", 56 | " batchV = np.array([]) # batch array\n", 57 | "\n", 58 | " anameL = f['cell data'].keys()\n", 59 | " ncell = len(anameL)\n", 60 | " for aname in anameL:\n", 61 | "\n", 62 | " g = f['cell data'][aname]\n", 63 | "\n", 64 | " cycfail = g['cycles to 90pct capacity'][0]\n", 65 | " cycfailV = np.append(cycfailV, cycfail)\n", 66 | "\n", 67 | " VvC = g['discharging voltage vs capacity']\n", 68 | "\n", 69 | " # our features are based on the difference in capacity between cycles\n", 70 | " farr = VvC[1:, :] - VvC[0, :] \n", 71 | " Xin += [farr]\n", 72 | "\n", 73 | " if 'Batch1' in aname:\n", 74 | " batchV = np.append(batchV, 1)\n", 75 | " elif 'Batch2' in aname:\n", 76 | " batchV = np.append(batchV, 2)\n", 77 | " elif 'Batch3' in aname:\n", 78 | " batchV = np.append(batchV, 3)\n", 79 | " else:\n", 80 | " batchV = np.append(batchV, 0)\n", 81 | "\n", 82 | " nfeat = farr.size\n", 83 | " ncyc = len(cyceval)\n", 84 | "\n", 85 | " Xin = np.stack(Xin)\n", 86 | " if output:\n", 87 | " print('number of cells:', ncell)\n", 88 | " print('number of features:', nfeat)\n", 89 | " print(Xin.shape)\n", 90 | "\n", 91 | " # create the train, validation, and test sets\n", 92 | "\n", 93 | " dsetL = ['trn', 'val', 'tst']\n", 94 | "\n", 95 | " iD = {}\n", 96 | "\n", 97 | " # batch 3 is reserved for test\n", 98 | " iD['tst'] = batchV == 3\n", 99 | " ntst = np.sum(iD['tst'])\n", 100 | "\n", 101 | " irst = np.invert(iD['tst'])\n", 102 | " nrst = ncell - ntst\n", 103 | "\n", 104 | " # validation and training sets are pulled from batches 1 and 2\n", 105 | " nval = np.int16(0.2*nrst)\n", 106 | "\n", 107 | " irnd = np.random.rand(ncell)\n", 108 | " irnd[iD['tst']] = 0\n", 109 | " top_nval = np.argsort(irnd)[-nval:]\n", 110 | "\n", 111 | " iD['val'] = np.zeros((ncell,), dtype='bool')\n", 112 | " iD['val'][top_nval] = True\n", 113 | " iD['trn'] = np.invert(iD['val'])*np.invert(iD['tst'])\n", 114 | "\n", 115 | " X = {} # features dictionary\n", 116 | " y = {} # failure cycles dictionary\n", 117 | " n = {} # number of cells dictionary\n", 118 | " for dset in dsetL:\n", 119 | " X[dset] = Xin[iD[dset], ..., None]\n", 120 | " if logy:\n", 121 | " y[dset] = np.log10(cycfailV[iD[dset]])\n", 122 | " else:\n", 123 | " y[dset] = cycfailV[iD[dset]]\n", 124 | " n[dset] = np.sum(iD[dset])\n", 125 | "\n", 126 | " # define weights for each cell based on the relative number of\n", 127 | " # cells available for that cycle-life bin. For example, there\n", 128 | " # are many low cycle-life cells so those have low weights, and\n", 129 | " # there are relatively few high cycle-life cells so those have\n", 130 | " # higher weights.\n", 131 | " bins = 4\n", 132 | " hist, bin_edges = np.histogram(y['trn'], bins=bins, density=True)\n", 133 | "\n", 134 | " invhist = np.mean(hist)/hist\n", 135 | "\n", 136 | " weights = np.ones(y['trn'].shape)\n", 137 | " for ii in range(bins):\n", 138 | " lwr = bin_edges[ii]\n", 139 | " upr = bin_edges[ii+1]\n", 140 | " if ii == 0:\n", 141 | " sel = (lwr <= y['trn'])*(y['trn'] <= upr)\n", 142 | " else:\n", 143 | " sel = (lwr < y['trn'])*(y['trn'] <= upr)\n", 144 | " weights[sel] = invhist[ii]\n", 145 | "\n", 146 | "\n", 147 | " if output:\n", 148 | " print('bin_edges:', bin_edges)\n", 149 | " print('original training densities:', hist)\n", 150 | " print('category weights:', invhist)\n", 151 | " print('y_trn weights:', weights)\n", 152 | " histw, bew = np.histogram(\n", 153 | " y['trn'], bins=bins, density=True, weights=weights)\n", 154 | " print('new densities:', histw)\n", 155 | "\n", 156 | " return X, y, n, iD, weights, cyceval, veval" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "id": "b1e3f0e1", 162 | "metadata": {}, 163 | "source": [ 164 | "### Define key variables for the fitting and extract the data" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "id": "b6063130", 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "epochs = 200\n", 175 | "logy=True # apply a log transform to the cycle lives as a normalization scheme\n", 176 | "output=True\n", 177 | "batch_size=32\n", 178 | "dsetL = ['trn', 'val', 'tst']\n", 179 | "\n", 180 | "sns.set()\n", 181 | "np.random.seed(1)\n", 182 | "\n", 183 | "X, y, n, iD, weights, cyceval, veval = get_data(logy, output)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "id": "abd81b88", 189 | "metadata": {}, 190 | "source": [ 191 | "### Visualize features for machine learning\n", 192 | "\n", 193 | "In this work, we define the features as the difference in capacity at different voltages during discharge between cycle X and cycle 10. Cycle X can be in the set [20, 30, 40, 50, 60 ,70, 80, 90, 100]. There are a total of 180 features." 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "id": "64110f2c", 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "arr = X['trn'][0, ..., 0]\n", 204 | "f, ax = plt.subplots()\n", 205 | "img = ax.imshow(arr)\n", 206 | "plt.colorbar(img, ax=ax, label='Q(Voltage, Cycle_X) - \\nQ(Voltage, Cycle_10) (Ah)')\n", 207 | "ax.set_xticks(np.arange(arr.shape[1])[::2])\n", 208 | "ax.set_yticks(np.arange(arr.shape[0]))\n", 209 | "yticklabels = cyceval[()][1:]\n", 210 | "yticklabels = yticklabels.astype(str)\n", 211 | "ax.set_xticklabels(np.round(veval[::2], 1))\n", 212 | "ax.set_yticklabels(yticklabels)\n", 213 | "ax.grid(False)\n", 214 | "plt.xlabel('Voltage')\n", 215 | "plt.ylabel('Cycle')" 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "id": "2acd07f9", 221 | "metadata": {}, 222 | "source": [ 223 | "### Define the model\n", 224 | "\n", 225 | "We define the convolutional network with two convolutional layers and one dense layer. Feel free to explore the number of filters/neurons, the shape of the filters, and the number of different types of layers. It is also interesting to see the effect of learning rate and loss metric on fitting and performance. We print a summary of the model that shows the types of layers, the shape of their outputs, and the number of parameters associated with the layer." 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "id": "33169007", 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "inputs = keras.Input(shape=(X['trn'].shape[1], X['trn'].shape[2], 1), name='farr')\n", 236 | "x = keras.layers.Conv2D(16, (3, 3), activation='relu')(inputs)\n", 237 | "x = keras.layers.Conv2D(8, (3, 3), activation='relu')(inputs)\n", 238 | "x = keras.layers.Flatten()(x)\n", 239 | "# x = keras.layers.Dropout(0.2)(x)\n", 240 | "# x = keras.layers.BatchNormalization()(x)\n", 241 | "x = keras.layers.Dense(16, activation='relu')(x)\n", 242 | "outputs = keras.layers.Dense(1)(x)\n", 243 | "model = keras.Model(inputs=inputs, outputs=outputs, name='functional_1')\n", 244 | "\n", 245 | "model.summary()\n", 246 | "\n", 247 | "model.compile(optimizer=tf.optimizers.Adam(learning_rate=0.01, amsgrad=False),\n", 248 | " loss='MSE',\n", 249 | " metrics=[])" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "id": "04ed5136", 255 | "metadata": {}, 256 | "source": [ 257 | "### Train the model" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": null, 263 | "id": "321f8637", 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "cb2 = keras.callbacks.EarlyStopping(\n", 268 | " monitor='val_loss',\n", 269 | " patience=50,\n", 270 | " restore_best_weights=True)\n", 271 | "\n", 272 | "st = time.time()\n", 273 | "\n", 274 | "history = model.fit(X['trn'], y['trn'],\n", 275 | " epochs=epochs, batch_size=batch_size,\n", 276 | " sample_weight=weights, callbacks=[cb2],\n", 277 | " validation_data=(X['val'], y['val']))\n", 278 | "\n", 279 | "print('fit time:', np.round(time.time()-st))" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "id": "6fa7b826", 285 | "metadata": {}, 286 | "source": [ 287 | "### Make predictions" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "id": "b32e75e9", 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "ypred = {}\n", 298 | "for dset in dsetL:\n", 299 | " ypred[dset] = np.squeeze(model.predict(X[dset]))" 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "id": "142d14a4", 305 | "metadata": {}, 306 | "source": [ 307 | "### Plot loss versus epoch" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": null, 313 | "id": "ee1956d7", 314 | "metadata": {}, 315 | "outputs": [], 316 | "source": [ 317 | "plt.figure('metrics_vs_epoch')\n", 318 | "nhist = len(history.history['loss'])\n", 319 | "plt.semilogy(range(1, nhist+1), history.history['loss'],\n", 320 | " 'b-', alpha=0.5, label='training')\n", 321 | "plt.semilogy(range(1, nhist+1), history.history['val_loss'],\n", 322 | " 'r-', alpha=0.5, label='validation')\n", 323 | "plt.xlabel('epoch')\n", 324 | "plt.ylabel('loss')\n", 325 | "plt.legend()\n", 326 | "plt.tight_layout()" 327 | ] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "id": "76ca4dc1", 332 | "metadata": {}, 333 | "source": [ 334 | "### Parity Plot\n", 335 | "\n", 336 | "The parity plot shows the experimental cycle lives versus the predicted cycle lives. Accurate predictions are close to the 45 degree centerline." 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "id": "762645e1", 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "minL = []\n", 347 | "maxL = []\n", 348 | "cL = sns.color_palette('cubehelix', 5)[1:-1]\n", 349 | "mL = ['o', 's', 'D']\n", 350 | "lblL = ['train', 'validate', 'test']\n", 351 | "plt.figure(num='parity-plot', figsize=(5.5, 5))\n", 352 | "for dset, c, m, lbl in zip(dsetL, cL, mL, lblL):\n", 353 | " if logy:\n", 354 | " yP = 10**ypred[dset]\n", 355 | " yT = 10**y[dset]\n", 356 | " else:\n", 357 | " yP = ypred[dset]\n", 358 | " yT = y[dset]\n", 359 | "\n", 360 | " minL += [np.min(yP), np.min(yT)]\n", 361 | " maxL += [np.max(yP), np.max(yT)]\n", 362 | "\n", 363 | " plt.plot(\n", 364 | " yT, yP,\n", 365 | " color=c, marker=m,\n", 366 | " ls='', ms=5, alpha=0.6, label=lbl)\n", 367 | "\n", 368 | "ymin = np.min(minL)\n", 369 | "ymax = np.max(maxL)\n", 370 | "yrng = ymax - ymin\n", 371 | "lwr = ymin - .05*yrng\n", 372 | "upr = ymax + .05*yrng\n", 373 | "\n", 374 | "plt.plot([lwr, upr], [lwr, upr], 'k-')\n", 375 | "plt.xlim([lwr, upr])\n", 376 | "plt.ylim([lwr, upr])\n", 377 | "plt.xlabel('experimental cycles to failure')\n", 378 | "plt.ylabel('predicted cycles to failure')\n", 379 | "plt.legend()\n", 380 | "plt.gca().set_aspect('equal', adjustable='box')\n", 381 | "plt.tight_layout()" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "id": "2b2113db", 387 | "metadata": {}, 388 | "source": [ 389 | "### Error Metrics" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": null, 395 | "id": "a5f4ae66", 396 | "metadata": {}, 397 | "outputs": [], 398 | "source": [ 399 | "# calculate errors and load into dictionary\n", 400 | "metricD = {}\n", 401 | "for dset in dsetL:\n", 402 | "\n", 403 | " metricD[dset + '_loss'] = np.mean((ypred[dset]-y[dset])**2)\n", 404 | "\n", 405 | " if logy:\n", 406 | " mapre = 100*np.mean(np.abs(10**ypred[dset]-10**y[dset])/10**y[dset])\n", 407 | " mape = 100*np.mean(np.abs(10**ypred[dset]-10**y[dset])/10**y[dset].max())\n", 408 | " else:\n", 409 | " mapre = 100*np.mean(np.abs(ypred[dset]-y[dset])/y[dset])\n", 410 | " mape = 100*np.mean(np.abs(ypred[dset]-y[dset])/y[dset].max())\n", 411 | "\n", 412 | " metricD[dset + '_mapre'] = mapre \n", 413 | " metricD[dset + '_mape'] = mape\n", 414 | "\n", 415 | "# print results\n", 416 | "for dset in dsetL:\n", 417 | " print('\\n')\n", 418 | " print(dset, 'mean squared error:',\n", 419 | " np.round(metricD[dset + '_loss'], 4))\n", 420 | " print(dset, 'mean absolute percent relative error:',\n", 421 | " np.round(metricD[dset + '_mapre'], 1))\n", 422 | " print(dset, 'mean absolute percent error:',\n", 423 | " np.round(metricD[dset + '_mape'], 1))" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": null, 429 | "id": "917ae945", 430 | "metadata": {}, 431 | "outputs": [], 432 | "source": [] 433 | } 434 | ], 435 | "metadata": { 436 | "kernelspec": { 437 | "display_name": "Python 3 (ipykernel)", 438 | "language": "python", 439 | "name": "python3" 440 | }, 441 | "language_info": { 442 | "codemirror_mode": { 443 | "name": "ipython", 444 | "version": 3 445 | }, 446 | "file_extension": ".py", 447 | "mimetype": "text/x-python", 448 | "name": "python", 449 | "nbconvert_exporter": "python", 450 | "pygments_lexer": "ipython3", 451 | "version": "3.7.10" 452 | } 453 | }, 454 | "nbformat": 4, 455 | "nbformat_minor": 5 456 | } 457 | -------------------------------------------------------------------------------- /battery_life_prediction_with_neural_networks/exercise-1_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Exercise 1: Regression in Keras\n", 8 | "In this exercise, you will learn how to effectively use [Keras](https://keras.io/) to train neural network models. You will learn how to make your own Keras models, evaluate their performance, and see the different ways that models can overfit (and how to prevent them). Elements of this exercise have been adapted from the Keras tutorials for Keras.\n", 9 | "\n", 10 | "This notebook contains many sections that are filled out for you and many that you will need to fill out to complete the exercise (marked in RED). You are finished when \"Restarting and Run All Cells\" executes the entire notebook without producing any errors. Do not remove assert statements.\n", 11 | "\n", 12 | "**HINT**: An answer to each of the blank cells is hidden after each . Double click the cell to see the answer in its contents" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "%matplotlib inline\n", 22 | "from matplotlib import pyplot as plt\n", 23 | "from sklearn.model_selection import train_test_split\n", 24 | "from sklearn.metrics import mean_absolute_error\n", 25 | "from tensorflow.keras import optimizers as opt\n", 26 | "from tensorflow.keras import Sequential\n", 27 | "from tensorflow.keras import layers\n", 28 | "from tensorflow import keras\n", 29 | "from time import perf_counter\n", 30 | "from math import isclose\n", 31 | "import pandas as pd\n", 32 | "import numpy as np\n", 33 | "import warnings\n", 34 | "np.random.seed(1)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "## Load Data\n", 42 | "We are going to use the [Auto MPG Dataset](https://archive.ics.uci.edu/ml/datasets/auto+mpg)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "First step is to download and cache it locally" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "Now, we read in the data using Pandas. As the dataset lacks headers, we have to make them ourselves" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "with open(dataset_path) as fp:\n", 75 | " for i in range(5):\n", 76 | " print(fp.readline().strip())" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "column_names = ['MPG', 'Cylinders', 'Displacement', 'Horsepower', 'Weight',\n", 86 | " 'Acceleration', 'Model Year', 'Origin']\n", 87 | "data = pd.read_csv(dataset_path, names=column_names,\n", 88 | " na_values = \"?\", comment='\\t',\n", 89 | " sep=\" \", skipinitialspace=True)\n", 90 | "data.head()" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "The data has some missing values, and we'll just remove them" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": null, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "data.dropna(inplace=True)\n", 107 | "print(f'Total number of entries: {len(data)}')" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "This is far from \"big data\", but will help us learn Keras with minimal waiting around for computations to finish" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "### Preprocessing\n", 122 | "There are a few things we need to deal with about the data first to make it more accessible to machine learning.\n", 123 | "\n", 124 | "The first is the \"Origin\" column, which is a categorical variable expressed as a list of numbers. \n", 125 | "Categorial variables expressed this way cause problems with machine learning because the ordering of numbers suggest cardinality that does not exist\n", 126 | "(i.e., Origin \"1\" is not less than Origin \"2\").\n", 127 | "We will use [one-hot encoding](https://en.wikipedia.org/wiki/One-hot) to remove this false ordering." 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "new_origin = pd.get_dummies(data['Origin'], prefix='Origin')\n", 137 | "new_origin.head()" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "Replace the origin column with these new columns\n", 145 | "\n", 146 | "*Pro Tip*: [Pandas](https://pandas.pydata.org/) makes working with tabular data in Python very easy. If you are about to write a loop to operate on tabular data, check to see if Pandas already has a function for what you're trying to do. Pandas makes your code much faster and easier to read/maintain. Start with the excellent [10 Minutes to Pandas](https://pandas.pydata.org/pandas-docs/stable/getting_started/10min.html)." 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "data = pd.concat([data, new_origin], axis=1)" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "data.drop('Origin', axis=1, inplace=True)" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "data.head()" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "We now are going to mark which columns are inputs and outputs" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": null, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "y_col = 'MPG'\n", 190 | "X_cols = data.columns[1:]" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": {}, 196 | "source": [ 197 | "Output is \"MPG\" and input is anything except MPG" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "Another trick needed for training a Neural Network is that the input values should all be the same order of magnitude. \n", 205 | "As shown above, our data is not. So, we will normalize the datasets" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [ 214 | "data[X_cols] = data[X_cols].apply(lambda x: (x - x.mean()) / x.std())" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": {}, 221 | "outputs": [], 222 | "source": [ 223 | "data.head(2)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "Our data is now ready for training.\n", 231 | "A best practice for evaluating neural networks is to split data into a training and validation sets.\n", 232 | "When comparing different machine learning models, you should test them against data that was neither used to train the model nor select the hyperparameters of the model - the \"validation set\"." 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "train_data, valid_data = train_test_split(data, test_size=0.1)\n", 242 | "print(f'Training data has {len(train_data)} entries')\n", 243 | "print(f'Validation data has {len(valid_data)} entries')" 244 | ] 245 | }, 246 | { 247 | "cell_type": "markdown", 248 | "metadata": {}, 249 | "source": [ 250 | "## Quick Tutorial: Working with Keras\n", 251 | "Building Keras models is decomposed into three different phases: defining the architecture, setting the optimizer, and training the model. We will go through each step independentally." 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "### Defining an Architecture\n", 259 | "Keras provides a wide variety of layers that can be combined together to form complicated networks.\n", 260 | "We are only going to focus on the [Sequential](https://keras.io/getting-started/sequential-model-guide/) type of model for simplicity, \n", 261 | "but you will get to make complicated models with branches in the Generative models session later today.\n", 262 | "\n", 263 | "The Sequential model takes a list of layers to generate combine in to a single model.\n", 264 | "The first element must specify the shape of the input data, which is the number of features in this case.\n", 265 | "The last element defines the shape of the outputs and must allow for the outputs to output on the entire range of the data.\n", 266 | "`linear` is a good choice for regression problems" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": null, 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "model = Sequential([\n", 276 | " layers.Dense(64, activation='relu', input_shape=(len(X_cols),)),\n", 277 | " layers.Dense(64, activation='relu'),\n", 278 | " layers.Dense(1, activation='linear')\n", 279 | "])" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": {}, 285 | "source": [ 286 | "Taking a look at the network we created" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [ 295 | "print(f'Shape of the input: {model.input_shape}')\n", 296 | "print(f'Shape of the output: {model.output_shape}')" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "It has an input shape of $N \\times 9$, where $N$ can be any number of training entries, and $N \\times 1$ outputs (i.e., just MPG)" 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": null, 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [ 312 | "model.summary()" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "The model has 5 layers: an input layer (not shown), 2 hidden layers, and an output layer. " 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "#### Concept Review! How do we get 4865 parameters?\n", 327 | "Remember our simple neural network from the lecture:\n", 328 | "\n", 329 | "\n", 330 | "\n", 331 | "_First Hidden Layer_: Each input node is connected to each node in the first hidden layer (i.e., the network is \"fully connected\"): $([\\text{9 Inputs}] + [\\text{1 Bias per Hidden Node}]) \\times [\\text{64 Hidden Nodes}] = 640\\text{ parameters}$\n", 332 | "\n", 333 | "_Second Hidden Layer_: Each node in the first hidden layer is connected to each node in the second hidden layer: $([\\text{64 hidden nodes}] + [\\text{1 Bias per Hidden Node}]) \\times [\\text{64 Hidden Nodes}] = 4160\\text{ parameters}$\n", 334 | "\n", 335 | "_Output Layer_: Each second hidden layer node is connected to the output node and that output node has a bias term: $[\\text{64 hidden nodes}] + [\\text{1 Bias}] = 65\\text{ parameters}$\n" 336 | ] 337 | }, 338 | { 339 | "cell_type": "markdown", 340 | "metadata": {}, 341 | "source": [ 342 | "### Training a Model\n", 343 | "The first step to train a model is to \"compile\" it.\n", 344 | "Compiling defines the optimizer and the loss function for the network." 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": null, 350 | "metadata": {}, 351 | "outputs": [], 352 | "source": [ 353 | "model.compile('adam', loss='mean_squared_error')" 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "metadata": {}, 359 | "source": [ 360 | "We compile with the Adam optimizer and using a mean squared error loss.\n", 361 | "Note that you can define more settings for these optimizers by creating the optimizer as an object." 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": null, 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [ 370 | "adam = keras.optimizers.Adam(lr=1e-3)\n", 371 | "model.compile(adam, loss=keras.metrics.mean_squared_error)" 372 | ] 373 | }, 374 | { 375 | "cell_type": "markdown", 376 | "metadata": {}, 377 | "source": [ 378 | "The model is ready to be trained once you compile it.\n", 379 | "\n", 380 | "The `fit` operation, like its analogue in scikit-learn, takes the inputs and outputs for the training data as arguments.\n", 381 | "It also has many other options specific to neural networks, such as the \"batch size\" (see later module) and a \"validation split.\" \n", 382 | "\n", 383 | "Validation sets are a very important tool in Neural Networks as they help determine if the model is overfitting during the training process." 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": null, 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [ 392 | "history = model.fit(train_data[X_cols], train_data[y_col], batch_size=32, validation_split=0.1, epochs=8)" 393 | ] 394 | }, 395 | { 396 | "cell_type": "markdown", 397 | "metadata": {}, 398 | "source": [ 399 | "The `history` object returned by the fit function contains details about the training process." 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": null, 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [ 408 | "fig, ax = plt.subplots()\n", 409 | "\n", 410 | "ax.plot(history.epoch, history.history['loss'], label='Training Loss')\n", 411 | "ax.plot(history.epoch, history.history['val_loss'], label='Validation Loss', linestyle='--')\n", 412 | "\n", 413 | "fig.set_size_inches(3.5, 2.5)\n", 414 | "ax.set_xlabel('Epoch')\n", 415 | "ax.set_ylabel('Loss ($MPG^2$)')\n", 416 | "ax.legend()" 417 | ] 418 | }, 419 | { 420 | "cell_type": "markdown", 421 | "metadata": {}, 422 | "source": [ 423 | "The loss for both the training data and the validation are both decreasing with epoch, which means our model is training correctly!" 424 | ] 425 | }, 426 | { 427 | "cell_type": "markdown", 428 | "metadata": {}, 429 | "source": [ 430 | "### Running the Model\n", 431 | "Now that our model is trained, we can use it to predict the properties of the validation set we held out with the predict option." 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": null, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "pred_y = model.predict(valid_data[X_cols])" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": null, 446 | "metadata": {}, 447 | "outputs": [], 448 | "source": [ 449 | "fig, ax = plt.subplots()\n", 450 | "\n", 451 | "ax.scatter(valid_data[y_col], pred_y)\n", 452 | "\n", 453 | "ax.set_xlim(0, max(valid_data[y_col].max(), pred_y.max()) + 5)\n", 454 | "ax.set_ylim(ax.get_xlim())\n", 455 | "\n", 456 | "ax.plot(ax.get_xlim(), ax.get_xlim(), 'k--')\n", 457 | "\n", 458 | "fig.set_size_inches(3.5, 3.5)\n", 459 | "ax.set_xlabel('Actual MPG')\n", 460 | "ax.set_ylabel('Predicted MPG')" 461 | ] 462 | }, 463 | { 464 | "cell_type": "markdown", 465 | "metadata": {}, 466 | "source": [ 467 | "Admittedly, the fitness is not very good. But, we only trained for 8 epochs." 468 | ] 469 | }, 470 | { 471 | "cell_type": "markdown", 472 | "metadata": {}, 473 | "source": [ 474 | "## Part 1: Overfitting a Neural Network\n", 475 | "A key problem with neural networks is that the more you train them, the more likely they are to become overfit.\n", 476 | "In this part, we need you to create a neural network and overfit it.\n", 477 | "\n", 478 | "Step 1: Make a neural network with 3 hidden layers of 128 elements per hidden layer each and ReLU activations." 479 | ] 480 | }, 481 | { 482 | "cell_type": "code", 483 | "execution_count": null, 484 | "metadata": {}, 485 | "outputs": [], 486 | "source": [] 487 | }, 488 | { 489 | "cell_type": "markdown", 490 | "metadata": {}, 491 | "source": [ 492 | "\n", 493 | "model = Sequential([\n", 494 | " layers.Dense(128, activation='relu', input_shape=(len(X_cols),)),\n", 495 | " layers.Dense(128, activation='relu'),\n", 496 | " layers.Dense(128, activation='relu'),\n", 497 | " layers.Dense(1, activation='relu')\n", 498 | "])\n", 499 | "" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": null, 505 | "metadata": {}, 506 | "outputs": [], 507 | "source": [ 508 | "assert model.count_params() == 34433" 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "metadata": {}, 514 | "source": [ 515 | "Fit the model with a batch size of 32 for 1024 epochs with a hold out set of 10%. Use Adam with the default settings and mean squared error loss. Save the history as a variable named `history`\n", 516 | "\n", 517 | "*Pro Tip*: Keras produces a lot of status messages with its default `verbose` setting of the `fit` function. Consider using `verbose=0` to turn it off if that bothers you." 518 | ] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": null, 523 | "metadata": {}, 524 | "outputs": [], 525 | "source": [] 526 | }, 527 | { 528 | "cell_type": "markdown", 529 | "metadata": {}, 530 | "source": [ 531 | "\n", 532 | "model.compile('adam', 'mean_squared_error')\n", 533 | "history = model.fit(train_data[X_cols], train_data[y_col], batch_size=32, validation_split=0.1, verbose=0, epochs=1024)\n", 534 | "" 535 | ] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": {}, 540 | "source": [ 541 | "Plot the loss as a function of the number of epochs" 542 | ] 543 | }, 544 | { 545 | "cell_type": "code", 546 | "execution_count": null, 547 | "metadata": {}, 548 | "outputs": [], 549 | "source": [ 550 | "fig, ax = plt.subplots()\n", 551 | "\n", 552 | "ax.semilogy(history.epoch, history.history['loss'], label='Training Loss')\n", 553 | "ax.plot(history.epoch, history.history['val_loss'], label='Validation Loss', linestyle='--')\n", 554 | "\n", 555 | "fig.set_size_inches(3.5, 2.5)\n", 556 | "ax.set_xlabel('Epoch')\n", 557 | "ax.set_ylabel('Loss ($MPG^2$)')\n", 558 | "ax.legend()" 559 | ] 560 | }, 561 | { 562 | "cell_type": "markdown", 563 | "metadata": {}, 564 | "source": [ 565 | "Note how the validation loss stops decreasing around the 50th epoch and then increases. \n", 566 | "This is overfitting.\n", 567 | "The network is continuing to get better at predicting the training data at the expense of being generalizable to other data" 568 | ] 569 | }, 570 | { 571 | "cell_type": "code", 572 | "execution_count": null, 573 | "metadata": {}, 574 | "outputs": [], 575 | "source": [ 576 | "pred_y = model.predict(valid_data[X_cols])\n", 577 | "overfit_score = mean_absolute_error(valid_data[y_col], pred_y)\n", 578 | "print(f'Mean absolute error on held-out set: {overfit_score : 0.2f} MPG')" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "execution_count": null, 584 | "metadata": {}, 585 | "outputs": [], 586 | "source": [ 587 | "fig, ax = plt.subplots()\n", 588 | "\n", 589 | "ax.scatter(valid_data[y_col], pred_y)\n", 590 | "\n", 591 | "ax.set_xlim(0, max(valid_data[y_col].max(), pred_y.max()) + 5)\n", 592 | "ax.set_ylim(ax.get_xlim())\n", 593 | "\n", 594 | "ax.plot(ax.get_xlim(), ax.get_xlim(), 'k--')\n", 595 | "\n", 596 | "fig.set_size_inches(3.5, 3.5)\n", 597 | "ax.set_xlabel('Actual MPG')\n", 598 | "ax.set_ylabel('Predicted MPG')" 599 | ] 600 | }, 601 | { 602 | "cell_type": "markdown", 603 | "metadata": {}, 604 | "source": [ 605 | "The model does predict better than when we trained with only 8 epochs, but we can do better" 606 | ] 607 | }, 608 | { 609 | "cell_type": "markdown", 610 | "metadata": {}, 611 | "source": [ 612 | "## Part 2: Preventing Overfitting with Early Stopping\n", 613 | "Early stopping is the idea that you detect when the error on the validation set is increasing and roll back to the network state with the best valiation error." 614 | ] 615 | }, 616 | { 617 | "cell_type": "markdown", 618 | "metadata": {}, 619 | "source": [ 620 | "#### Quick Tutorial: Pre-trained Models\n", 621 | "If you were to call the `fit` method again, the Keras model will resume training from its previous state.\n", 622 | "This is a nice feature if you need to restart a training from a saved checkpoint or using the weights from another model for pretraining (which we will dicuss in a later tutorial).\n", 623 | "However, it interferes with the lesson on early stopping we want here.\n", 624 | "\n", 625 | "So, we are going to make a new model before training.\n", 626 | "To make this more convenient, we are going to make a function that generates a Keras model so to avoid having to write out the architecture each time.\n", 627 | "\n", 628 | "*Pro Tip*: Making these \"model building functions\" can save you a lot of time when testing new architectures." 629 | ] 630 | }, 631 | { 632 | "cell_type": "code", 633 | "execution_count": null, 634 | "metadata": {}, 635 | "outputs": [], 636 | "source": [ 637 | "def build_model(n_layers=3, hidden_size=128):\n", 638 | " model = Sequential([layers.Dense(hidden_size, activation='relu', input_shape=(len(X_cols),))])\n", 639 | " \n", 640 | " for i in range(n_layers-1):\n", 641 | " model.add(layers.Dense(hidden_size, activation='relu'))\n", 642 | " \n", 643 | " model.add(layers.Dense(1, activation='linear'))\n", 644 | " return model" 645 | ] 646 | }, 647 | { 648 | "cell_type": "code", 649 | "execution_count": null, 650 | "metadata": {}, 651 | "outputs": [], 652 | "source": [ 653 | "model = build_model()" 654 | ] 655 | }, 656 | { 657 | "cell_type": "code", 658 | "execution_count": null, 659 | "metadata": {}, 660 | "outputs": [], 661 | "source": [ 662 | "model.compile('adam', 'mean_squared_error')" 663 | ] 664 | }, 665 | { 666 | "cell_type": "markdown", 667 | "metadata": {}, 668 | "source": [ 669 | "### Creating the Callback\n", 670 | "Keras implement early stopping using a \"callback.\"\n", 671 | "Callback functions by performing an operation (e.g., assessing validation loss) at different points of the model training.\n", 672 | "In our case, we will use the [`EarlyStopping`](https://keras.io/callbacks/#earlystopping) callback, which is one of many [available in Keras](https://keras.io/callbacks/)" 673 | ] 674 | }, 675 | { 676 | "cell_type": "code", 677 | "execution_count": null, 678 | "metadata": {}, 679 | "outputs": [], 680 | "source": [ 681 | "callbacks = [keras.callbacks.EarlyStopping(restore_best_weights=True, patience=25)]" 682 | ] 683 | }, 684 | { 685 | "cell_type": "markdown", 686 | "metadata": {}, 687 | "source": [ 688 | "The callback we defined will stop only if the validation loss does not improve after 25 epochs and will restore the state of the model at the point with lowest loss. We use the callbacks when calling the fit operation." 689 | ] 690 | }, 691 | { 692 | "cell_type": "markdown", 693 | "metadata": {}, 694 | "source": [ 695 | "Train the model using the same batch size, but now add the early-stopping callback. Store the result in `history`" 696 | ] 697 | }, 698 | { 699 | "cell_type": "code", 700 | "execution_count": null, 701 | "metadata": {}, 702 | "outputs": [], 703 | "source": [] 704 | }, 705 | { 706 | "cell_type": "markdown", 707 | "metadata": {}, 708 | "source": [ 709 | "history = model.fit(train_data[X_cols], train_data[y_col], batch_size=32, validation_split=0.1, verbose=0, epochs=256, callbacks=callbacks)" 710 | ] 711 | }, 712 | { 713 | "cell_type": "markdown", 714 | "metadata": {}, 715 | "source": [ 716 | "The following cells will plot your results" 717 | ] 718 | }, 719 | { 720 | "cell_type": "code", 721 | "execution_count": null, 722 | "metadata": {}, 723 | "outputs": [], 724 | "source": [ 725 | "fig, ax = plt.subplots()\n", 726 | "\n", 727 | "ax.semilogy(history.epoch, history.history['loss'], label='Training Loss')\n", 728 | "ax.plot(history.epoch, history.history['val_loss'], label='Validation Loss', linestyle='--')\n", 729 | "\n", 730 | "fig.set_size_inches(3.5, 2.5)\n", 731 | "ax.set_xlabel('Epoch')\n", 732 | "ax.set_ylabel('Loss ($MPG^2$)')\n", 733 | "ax.legend()" 734 | ] 735 | }, 736 | { 737 | "cell_type": "markdown", 738 | "metadata": {}, 739 | "source": [ 740 | "The model now stops training at around 60 epochs." 741 | ] 742 | }, 743 | { 744 | "cell_type": "code", 745 | "execution_count": null, 746 | "metadata": {}, 747 | "outputs": [], 748 | "source": [ 749 | "assert len(history.epoch) < 256 # Model should stop training before 256 epochs" 750 | ] 751 | }, 752 | { 753 | "cell_type": "code", 754 | "execution_count": null, 755 | "metadata": {}, 756 | "outputs": [], 757 | "source": [ 758 | "pred_y = model.predict(valid_data[X_cols])\n", 759 | "earlystop_score = mean_absolute_error(valid_data[y_col], pred_y)\n", 760 | "print(f'Mean absolute error on held-out set: {earlystop_score : 0.2f} MPG')" 761 | ] 762 | }, 763 | { 764 | "cell_type": "code", 765 | "execution_count": null, 766 | "metadata": {}, 767 | "outputs": [], 768 | "source": [ 769 | "assert earlystop_score < overfit_score" 770 | ] 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": null, 775 | "metadata": {}, 776 | "outputs": [], 777 | "source": [ 778 | "fig, ax = plt.subplots()\n", 779 | "\n", 780 | "ax.scatter(valid_data[y_col], pred_y)\n", 781 | "\n", 782 | "ax.set_xlim(0, max(valid_data[y_col].max(), pred_y.max()) + 5)\n", 783 | "ax.set_ylim(ax.get_xlim())\n", 784 | "\n", 785 | "ax.plot(ax.get_xlim(), ax.get_xlim(), 'k--')\n", 786 | "\n", 787 | "fig.set_size_inches(3.5, 3.5)\n", 788 | "ax.set_xlabel('Actual MPG')\n", 789 | "ax.set_ylabel('Predicted MPG')" 790 | ] 791 | }, 792 | { 793 | "cell_type": "markdown", 794 | "metadata": {}, 795 | "source": [ 796 | "The performance should be better than when you ran the network to its full number of epochs.\n", 797 | "\n", 798 | "Early stopping is one of the many approaches to preventing overfitting. Some other techniques include:\n", 799 | "\n", 800 | "- Reduce the model complexity\n", 801 | "- Employ [Dropout](https://keras.io/layers/core/#dropout) layers, which randomly drop connections in the neural network during training. \n", 802 | "- Using [regularization](https://keras.io/regularizers/), which ensures the weights do not become too large" 803 | ] 804 | }, 805 | { 806 | "cell_type": "code", 807 | "execution_count": null, 808 | "metadata": {}, 809 | "outputs": [], 810 | "source": [] 811 | } 812 | ], 813 | "metadata": { 814 | "kernelspec": { 815 | "display_name": "Python 3 (ipykernel)", 816 | "language": "python", 817 | "name": "python3" 818 | }, 819 | "language_info": { 820 | "codemirror_mode": { 821 | "name": "ipython", 822 | "version": 3 823 | }, 824 | "file_extension": ".py", 825 | "mimetype": "text/x-python", 826 | "name": "python", 827 | "nbconvert_exporter": "python", 828 | "pygments_lexer": "ipython3", 829 | "version": "3.7.10" 830 | } 831 | }, 832 | "nbformat": 4, 833 | "nbformat_minor": 4 834 | } 835 | -------------------------------------------------------------------------------- /battery_life_prediction_with_neural_networks/neural-networks-tutorial.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/battery-data-commons/mrs-sp22-tutorial/64b420d2365f2ff26b6ea50617923db3a80c819b/battery_life_prediction_with_neural_networks/neural-networks-tutorial.pdf -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: mrsen05 2 | channels: 3 | - defaults 4 | - conda-forge 5 | dependencies: 6 | - python==3.7.* 7 | 8 | # General requirements for making plots and managing results 9 | - pandas==1.* 10 | - matplotlib==3.* 11 | - scikit-learn==1.0.* 12 | - seaborn==0.11.* 13 | 14 | # Putting Jupyter in this environment so we can run it locally 15 | - jupyterlab 16 | 17 | # Key tools for software development 18 | - pytest 19 | 20 | # ML modeling 21 | - xgboost 22 | 23 | # Put any requirements from pypi here 24 | - pip 25 | - pip: 26 | - jupyterlab-spellchecker 27 | - tensorflow==2.8.* # Needed for life prediction 28 | -------------------------------------------------------------------------------- /figures/eis_to_q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/battery-data-commons/mrs-sp22-tutorial/64b420d2365f2ff26b6ea50617923db3a80c819b/figures/eis_to_q.png -------------------------------------------------------------------------------- /predict_capacity_from_eis/data/freq_eis_zhang2020.csv: -------------------------------------------------------------------------------- 1 | 20004.453 2 | 15829.126 3 | 12516.703 4 | 9909.4424 5 | 7835.48 6 | 6217.2461 7 | 4905.291 8 | 3881.2737 9 | 3070.9827 10 | 2430.7778 11 | 1923.1537 12 | 1522.4358 13 | 1203.8446 14 | 952.86591 15 | 754.27557 16 | 596.71857 17 | 471.96338 18 | 373.20856 19 | 295.47278 20 | 233.87738 21 | 185.05922 22 | 146.35823 23 | 115.77804 24 | 91.6721 25 | 72.51701 26 | 57.36816 27 | 45.3629 28 | 35.93134 29 | 28.40909 30 | 22.48202 31 | 17.79613 32 | 14.06813 33 | 11.1448 34 | 8.81772 35 | 6.97545 36 | 5.5173 37 | 4.36941 38 | 3.45686 39 | 2.73547 40 | 2.16054 41 | 1.70952 42 | 1.35352 43 | 1.07079 44 | 0.84734 45 | 0.67072 46 | 0.53067 47 | 0.41976 48 | 0.33183 49 | 0.26261 50 | 0.20791 51 | 0.16452 52 | 0.13007 53 | 0.10309 54 | 0.08153 55 | 0.06443 56 | 0.05102 57 | 0.04042 58 | 0.03192 59 | 0.02528 60 | 0.01999 61 | -------------------------------------------------------------------------------- /predict_capacity_from_eis/data/ref.txt: -------------------------------------------------------------------------------- 1 | Zhang, Yunwei, Tang, Qiaochu, Zhang, Yao, Wang, Jiabin, Stimming, Ulrich, & Lee, Alpha A. (2020). Identifying degradation patterns of lithium ion batteries from impedance spectroscopy using machine learning [Data set]. Zenodo. http://doi.org/10.5281/zenodo.3633835 2 | https://github.com/YunweiZhang/ML-identify-battery-degradation 3 | 4 | https://www.nature.com/articles/s41467-020-15235-7.pdf 5 | -------------------------------------------------------------------------------- /predict_capacity_from_eis/eisplot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | def plot_eis(frequencies, impedance, title=None, cmap='tab10'): 8 | """ Creates a single figure w/ both Bode and Nyquist plots of a single EIS spectrum. 9 | Plots the results of a simulated circuit as well if provided 10 | 11 | Args: 12 | frequency (np.ndarray): numpy array of frequency values. Real, positive numbers 13 | impedance (np.ndarray): numpy array of impedance values. Imaginary numbers 14 | title (str): A figure title. Defaults to None. 15 | cmap (str): name of a matplotlib colormap for coloring multiple lines 16 | """ 17 | fig, ax = plt.subplots(1, 3, figsize=(12,4)) 18 | 19 | cmap = plt.get_cmap(cmap) 20 | if impedance.shape != frequencies.shape: 21 | colors = cmap(np.linspace(0,1,impedance.size)) 22 | colors = colors[:,0:3] 23 | ax[0].set_prop_cycle(color=colors) 24 | ax[1].set_prop_cycle(color=colors) 25 | ax[2].set_prop_cycle(color=colors) 26 | else: 27 | colors = cmap(0) 28 | 29 | if impedance.shape != frequencies.shape: 30 | # plot multiple lines 31 | frequencies = np.repeat(frequencies, impedance.size).reshape((frequencies.size, impedance.size)) 32 | impedance = np.vstack(impedance).transpose() 33 | 34 | # Bode Plot (1) 35 | ax[0].semilogx(frequencies, np.abs(impedance), "o") 36 | ax[0].set_title("Bode, |Z| vs. frequency") 37 | ax[0].set_xlabel("Freq [Hz]") 38 | ax[0].set_ylabel(r"|Z| [$\Omega$]", color="k") 39 | # Bode Plot (2) 40 | ax[1].semilogx(frequencies, np.angle(impedance, deg=True), "o") 41 | ax[1].set_title(r"Bode, $\angle$Z vs. frequency") 42 | ax[1].set_xlabel("Freq [Hz]") 43 | ax[1].set_ylabel(r"$\angle$Z [$^\circ$]", color="k") 44 | # Nyquist Plot 45 | ax[2].plot(np.real(impedance), -np.imag(impedance), "o") 46 | ax[2].set_aspect("equal") 47 | ax[2].invert_yaxis() 48 | ax[2].set_title("Nyquist") 49 | ax[2].set_xlabel(r"Re(Z) [$\Omega$]", color="k") 50 | ax[2].set_ylabel(r"Im(Z) [$\Omega$]", color="k") 51 | 52 | if title is not None: 53 | fig.suptitle(title) 54 | 55 | fig.tight_layout() 56 | plt.show() 57 | 58 | 59 | if __name__ == "__main__": 60 | ... 61 | -------------------------------------------------------------------------------- /predict_capacity_from_eis/plotyy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # this is also called a 'parity' plot 5 | def plotyy(y, y_pred, color=None, title=None): 6 | # Customize colors to match with capacity vs cycle plot earlier 7 | if color is not None: 8 | n_series = np.unique(color).size 9 | else: 10 | n_series = 1 11 | cmap = plt.get_cmap('tab10', n_series) 12 | # Get axes and plot 13 | fig, ax = plt.subplots(1,1) 14 | sc = ax.scatter(y.values, y_pred, c=color, cmap=cmap) 15 | ax.set_aspect('equal') 16 | plt.xlabel('Actual discharge capacity (mAh)') 17 | plt.ylabel('Predicted discharge capacity (mAh)') 18 | plt.axis('square') 19 | # Diagonal line for guiding the eye 20 | xlim = ax.get_xlim() 21 | ylim = ax.get_ylim() 22 | lims = np.concatenate((xlim, ylim)) 23 | lims = np.array([min(lims), max(lims)]) 24 | ax.set_xlim(lims) 25 | ax.set_ylim(lims) 26 | ax.set_autoscale_on(False) 27 | plt.plot([-100, 100], [-100,100], '--k') 28 | if n_series > 1: 29 | # colorbar 30 | cbar = plt.colorbar(sc) 31 | tick_locs = (np.arange(n_series) + 1.5)*(n_series-1)/n_series 32 | cbar.set_ticks(tick_locs) 33 | cbar.set_ticklabels(np.arange(n_series)+1) 34 | # title 35 | if title is not None: 36 | plt.title(title) 37 | plt.tight_layout() --------------------------------------------------------------------------------