├── img ├── dla.png ├── plot.png ├── ml_map.png ├── neuron.png ├── sigmoid.png ├── tikz12.png ├── foo_loop.png ├── math_graph.png ├── not_neurons.png ├── perceptron.png ├── screenshot.png ├── sklearn_demo.pdf ├── sklearn_demo.png ├── braided_river.jpg ├── pandas_cheatsheet.jpg ├── graph_vis_animation.gif ├── an_object_has_no_name.jpg └── data_exploration_cheatsheet.jpg ├── README.md ├── NeuralNetBasics.ipynb ├── ImageClassifier_CPU.ipynb ├── sklearn_demo_Server_Health.ipynb ├── hellotensorflow_demo_end.ipynb ├── data └── nginx_requests.bal-demoA.deltas.csv ├── plot.ipynb ├── AnomalyDetectionScikit-learn.ipynb └── ImageClassifier_GPU.ipynb /img/dla.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/dla.png -------------------------------------------------------------------------------- /img/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/plot.png -------------------------------------------------------------------------------- /img/ml_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/ml_map.png -------------------------------------------------------------------------------- /img/neuron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/neuron.png -------------------------------------------------------------------------------- /img/sigmoid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/sigmoid.png -------------------------------------------------------------------------------- /img/tikz12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/tikz12.png -------------------------------------------------------------------------------- /img/foo_loop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/foo_loop.png -------------------------------------------------------------------------------- /img/math_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/math_graph.png -------------------------------------------------------------------------------- /img/not_neurons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/not_neurons.png -------------------------------------------------------------------------------- /img/perceptron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/perceptron.png -------------------------------------------------------------------------------- /img/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/screenshot.png -------------------------------------------------------------------------------- /img/sklearn_demo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/sklearn_demo.pdf -------------------------------------------------------------------------------- /img/sklearn_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/sklearn_demo.png -------------------------------------------------------------------------------- /img/braided_river.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/braided_river.jpg -------------------------------------------------------------------------------- /img/pandas_cheatsheet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/pandas_cheatsheet.jpg -------------------------------------------------------------------------------- /img/graph_vis_animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/graph_vis_animation.gif -------------------------------------------------------------------------------- /img/an_object_has_no_name.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/an_object_has_no_name.jpg -------------------------------------------------------------------------------- /img/data_exploration_cheatsheet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ricardoamaro/MachineLearning4SRE/HEAD/img/data_exploration_cheatsheet.jpg -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MachineLearning4SRE 2 | 3 | Repo used to share the demos used in D.C. Vienna 2017 4 | https://events.drupal.org/vienna2017/sessions/intelligent-automation-and-machine-learning-site-reliability-engineering 5 | -------------------------------------------------------------------------------- /NeuralNetBasics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# A simple code to train a neural network from scratch:\n" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "collapsed": true 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | " import numpy as np" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": { 25 | "collapsed": true 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "# Create a \"Sigmoid\". This is the activation of a neuron.\n", 30 | "# A function that will map any value to a value bettween 0 and 1\n", 31 | "# Creates probabilities out of numbers\n", 32 | "def nonlin(x,deriv=False):\n", 33 | " if(deriv==True):\n", 34 | " return (x*(1-x))\n", 35 | " return (1/(1+np.exp(-x)))" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "Resulting in the Sigmoid:\n", 43 | "![sigmoid](img/sigmoid.png)\n", 44 | "A perceptron classifier is a simple model of a neuron. It has different inputs (x1...xn) with different weights (w1...wn).\n", 45 | "image\n", 46 | "The weighted sum s of these inputs is then passed through a step function f (usually a Heaviside step function).\n", 47 | "\n", 48 | "![perceptron](img/perceptron.png)" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 3, 54 | "metadata": { 55 | "collapsed": true 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "# Initialize the dataset as a matrix with input Data:\n", 60 | "# Each row is a diferent training example\n", 61 | "# Each column represents a diferent neuron \n", 62 | "X = np.array([[0,0,1],\n", 63 | " [0,1,1],\n", 64 | " [1,0,1],\n", 65 | " [1,1,1]])" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 4, 71 | "metadata": { 72 | "collapsed": true 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "# Output Data with one output neuron each\n", 77 | "Y = np.array([[1],\n", 78 | " [0.7],\n", 79 | " [3],\n", 80 | " [0]])" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 5, 86 | "metadata": { 87 | "collapsed": true 88 | }, 89 | "outputs": [], 90 | "source": [ 91 | "# seed them to make them deterministic\n", 92 | "# give random numbers with the same starting point (useful for debuging)\n", 93 | "# so we can get the same sequence of generated numbers everytime we run the program\n", 94 | "np.random.seed(1)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 6, 100 | "metadata": { 101 | "collapsed": true 102 | }, 103 | "outputs": [], 104 | "source": [ 105 | "# Create synapse matrices.\n", 106 | "# Initialize the weights of a neural network\n", 107 | "# (it is a neural network with two layers of weights):\n", 108 | "syn0 = 2 * np.random.random((3, 4)) - 1\n", 109 | "syn1 = 2 * np.random.random((4, 1)) - 1" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 7, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "name": "stdout", 119 | "output_type": "stream", 120 | "text": [ 121 | "Error:0.934545246367\n", 122 | "Error:0.502784580514\n", 123 | "Error:0.501866999994\n", 124 | "Error:0.501483292108\n", 125 | "Error:0.501261879281\n", 126 | "Error:0.501114122775\n", 127 | "Output after\n", 128 | "[[ 0.99999979]\n", 129 | " [ 0.69998478]\n", 130 | " [ 0.99999684]\n", 131 | " [ 0.00400894]]\n", 132 | "Objective\n", 133 | "[[ 1. ]\n", 134 | " [ 0.7]\n", 135 | " [ 3. ]\n", 136 | " [ 0. ]]\n" 137 | ] 138 | } 139 | ], 140 | "source": [ 141 | "# Training code (loop)\n", 142 | "for j in xrange(60000):\n", 143 | " # optimize the network for the given data set\n", 144 | " # First layer it's just our input data\n", 145 | " l0 = X\n", 146 | " # Prediction step\n", 147 | " # preform matrix multiplication bettween each layer and its synapse\n", 148 | " # Then run sigmoid function on the matrix to create the next layer\n", 149 | " l1 = nonlin(np.dot(l0, syn0))\n", 150 | " l2 = nonlin(np.dot(l1, syn1))\n", 151 | " \n", 152 | " # With the above prediction of the output in l2 we can compare it to the expected \n", 153 | " # output data using subtraction to get an error rate\n", 154 | " l2_error = Y - l2\n", 155 | " \n", 156 | " # Print the average error at a set interval to make sure it goes down every time\n", 157 | " if(j % 10000) == 0:\n", 158 | " print \"Error:\" + str(np.mean(np.abs(l2_error)))\n", 159 | " \n", 160 | " # Multiply the error rate by the slope of the sigmoid at the values in l2\n", 161 | " l2_delta = l2_error * nonlin(l2, deriv=True)\n", 162 | " \n", 163 | " # (Backpropagation) How much did l1 contributed to the error on l2 ?\n", 164 | " # Multiply layer 2 delta by synapse 1 transpose.\n", 165 | " l1_error = l2_delta.dot(syn1.T)\n", 166 | " \n", 167 | " # Get l1's delta by multlying it's error by the result of the sigmoid function\n", 168 | " l1_delta = l1_error * nonlin(l1, deriv=True)\n", 169 | " \n", 170 | " # (Gradient Descent) update weights \n", 171 | " # Now that we have deltas for each of our layers, we can use them to update\n", 172 | " # our synapses rates to reduce the error rate more and more every iteration.\n", 173 | " syn1 += l1.T.dot(l2_delta)\n", 174 | " syn0 += l0.T.dot(l1_delta)\n", 175 | " \n", 176 | "print \"Output after\"\n", 177 | "print l2\n", 178 | "print \"Objective\"\n", 179 | "print Y" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 8, 185 | "metadata": {}, 186 | "outputs": [ 187 | { 188 | "name": "stdout", 189 | "output_type": "stream", 190 | "text": [ 191 | "[[ 0.99999979]\n", 192 | " [ 0.69998478]\n", 193 | " [ 0.99999684]\n", 194 | " [ 0.00400894]]\n" 195 | ] 196 | } 197 | ], 198 | "source": [ 199 | "print l2" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "source": [ 208 | "## Next demo: \n", 209 | "http://localhost:8888/notebooks/final/sklearn_demo_Server_Health.ipynb" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": { 216 | "collapsed": true 217 | }, 218 | "outputs": [], 219 | "source": [] 220 | } 221 | ], 222 | "metadata": { 223 | "kernelspec": { 224 | "display_name": "Python 2", 225 | "language": "python", 226 | "name": "python2" 227 | }, 228 | "language_info": { 229 | "codemirror_mode": { 230 | "name": "ipython", 231 | "version": 2 232 | }, 233 | "file_extension": ".py", 234 | "mimetype": "text/x-python", 235 | "name": "python", 236 | "nbconvert_exporter": "python", 237 | "pygments_lexer": "ipython2", 238 | "version": "2.7.6" 239 | } 240 | }, 241 | "nbformat": 4, 242 | "nbformat_minor": 2 243 | } 244 | -------------------------------------------------------------------------------- /ImageClassifier_CPU.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Image Classifier (Using Keras and Tensorflow backend)\n", 8 | "\n", 9 | "### Convolutional Neural Network\n", 10 | "\n", 11 | "Keras runs on top of TensorFlow. It makes building models intuitive, since we can define each layer as it's own line of code.\n", 12 | "```\n", 13 | "cat ~/.keras/keras.json \n", 14 | "{\n", 15 | " \"epsilon\": 1e-07,\n", 16 | " \"floatx\": \"float32\",\n", 17 | " \"image_dim_ordering\": \"tf\", \n", 18 | " \"image_data_format\": \"channels_last\",\n", 19 | " \"backend\": \"tensorflow\"\n", 20 | "}\n", 21 | "```\n", 22 | "### This example uses the CPU which is much slow compared to GPU (other example)" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "##This notebook is built around using tensorflow as the backend for keras\n", 34 | "# !pip install pillow\n", 35 | "# !KERAS_BACKEND=tensorflow python -c \"from keras import backend\"" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 4, 41 | "metadata": { 42 | "collapsed": true 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "import os\n", 47 | "import numpy as np # math\n", 48 | "# from parser import load_data # data loading\n", 49 | "from keras.models import Sequential # machine learning loads ~/.keras/keras.json\n", 50 | "from keras.layers import Activation, Dropout, Flatten, Dense\n", 51 | "from keras.preprocessing.image import ImageDataGenerator\n", 52 | "from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D\n", 53 | "from keras import optimizers" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 5, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "# dimensions of our images.\n", 65 | "img_width, img_height = 150, 150\n", 66 | "\n", 67 | "train_data_dir = '/home/ricardo/Downloads/kaggle/train'\n", 68 | "validation_data_dir = '/home/ricardo/Downloads/kaggle/test1'" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "## Imports" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 4, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "Found 23778 images belonging to 2 classes.\n", 88 | "Found 1222 images belonging to 2 classes.\n" 89 | ] 90 | } 91 | ], 92 | "source": [ 93 | "# used to rescale the pixel values from [0, 255] to [0, 1] interval\n", 94 | "datagen = ImageDataGenerator(rescale=1./255)\n", 95 | "\n", 96 | "# automagically retrieve images and their classes for train and validation sets\n", 97 | "train_generator = datagen.flow_from_directory(\n", 98 | " train_data_dir,\n", 99 | " target_size=(img_width, img_height),\n", 100 | " batch_size=16,\n", 101 | " class_mode='binary')\n", 102 | "\n", 103 | "validation_generator = datagen.flow_from_directory(\n", 104 | " validation_data_dir,\n", 105 | " target_size=(img_width, img_height),\n", 106 | " batch_size=32,\n", 107 | " class_mode='binary')" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "# Small Convolutional Net\n", 115 | "\n", 116 | "## Model architecture definition" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 5, 122 | "metadata": { 123 | "collapsed": true 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "model = Sequential()\n", 128 | "model.add(Convolution2D(32, (3, 3), input_shape=(img_width, img_height,3)))\n", 129 | "model.add(Activation('relu'))\n", 130 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 131 | "\n", 132 | "model.add(Convolution2D(32, (3, 3)))\n", 133 | "model.add(Activation('relu'))\n", 134 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 135 | "\n", 136 | "model.add(Convolution2D(64, (3, 3)))\n", 137 | "model.add(Activation('relu'))\n", 138 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 139 | "\n", 140 | "model.add(Flatten())\n", 141 | "model.add(Dense(64))\n", 142 | "model.add(Activation('relu'))\n", 143 | "model.add(Dropout(0.5))\n", 144 | "model.add(Dense(1))\n", 145 | "model.add(Activation('sigmoid'))" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 6, 151 | "metadata": { 152 | "collapsed": true 153 | }, 154 | "outputs": [], 155 | "source": [ 156 | "model.compile(loss='binary_crossentropy',\n", 157 | " optimizer='rmsprop',\n", 158 | " metrics=['accuracy'])" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "# Training" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 7, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "nb_epoch = 30\n", 177 | "nb_train_samples = 128\n", 178 | "nb_validation_samples = 128" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 8, 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "name": "stdout", 188 | "output_type": "stream", 189 | "text": [ 190 | "Epoch 1/30\n", 191 | "128/128 [==============================] - 151s - loss: 0.7454 - acc: 0.5386 - val_loss: 0.6781 - val_acc: 0.5463\n", 192 | "Epoch 2/30\n", 193 | "128/128 [==============================] - 142s - loss: 0.6685 - acc: 0.6089 - val_loss: 0.6297 - val_acc: 0.6239\n", 194 | "Epoch 3/30\n", 195 | "128/128 [==============================] - 142s - loss: 0.6301 - acc: 0.6562 - val_loss: 0.5699 - val_acc: 0.7066\n", 196 | "Epoch 4/30\n", 197 | "128/128 [==============================] - 153s - loss: 0.6083 - acc: 0.6636 - val_loss: 0.5764 - val_acc: 0.6844\n", 198 | "Epoch 5/30\n", 199 | "128/128 [==============================] - 145s - loss: 0.6048 - acc: 0.6904 - val_loss: 0.5525 - val_acc: 0.7369\n", 200 | "Epoch 6/30\n", 201 | "128/128 [==============================] - 146s - loss: 0.5620 - acc: 0.7256 - val_loss: 0.5628 - val_acc: 0.7106\n", 202 | "Epoch 7/30\n", 203 | "128/128 [==============================] - 141s - loss: 0.5760 - acc: 0.7114 - val_loss: 0.5118 - val_acc: 0.7586\n", 204 | "Epoch 8/30\n", 205 | "128/128 [==============================] - 151s - loss: 0.5585 - acc: 0.7192 - val_loss: 0.5307 - val_acc: 0.7315\n", 206 | "Epoch 9/30\n", 207 | "128/128 [==============================] - 161s - loss: 0.5587 - acc: 0.7310 - val_loss: 0.4969 - val_acc: 0.7680\n", 208 | "Epoch 10/30\n", 209 | "128/128 [==============================] - 147s - loss: 0.5372 - acc: 0.7383 - val_loss: 0.4747 - val_acc: 0.7751\n", 210 | "Epoch 11/30\n", 211 | "128/128 [==============================] - 159s - loss: 0.5408 - acc: 0.7397 - val_loss: 0.4740 - val_acc: 0.7825\n", 212 | "Epoch 12/30\n", 213 | "128/128 [==============================] - 152s - loss: 0.5227 - acc: 0.7456 - val_loss: 0.4824 - val_acc: 0.7615\n", 214 | "Epoch 13/30\n", 215 | "128/128 [==============================] - 160s - loss: 0.5131 - acc: 0.7568 - val_loss: 0.5142 - val_acc: 0.7539\n", 216 | "Epoch 14/30\n", 217 | "128/128 [==============================] - 175s - loss: 0.4996 - acc: 0.7856 - val_loss: 0.4584 - val_acc: 0.7837\n", 218 | "Epoch 15/30\n", 219 | "128/128 [==============================] - 178s - loss: 0.5280 - acc: 0.7588 - val_loss: 0.4732 - val_acc: 0.7738\n", 220 | "Epoch 16/30\n", 221 | "128/128 [==============================] - 175s - loss: 0.5021 - acc: 0.7642 - val_loss: 0.4399 - val_acc: 0.8021\n", 222 | "Epoch 17/30\n", 223 | "128/128 [==============================] - 172s - loss: 0.4961 - acc: 0.7720 - val_loss: 0.4442 - val_acc: 0.8006\n", 224 | "Epoch 18/30\n", 225 | "128/128 [==============================] - 182s - loss: 0.5141 - acc: 0.7749 - val_loss: 0.4926 - val_acc: 0.7690\n", 226 | "Epoch 19/30\n", 227 | "128/128 [==============================] - 166s - loss: 0.4877 - acc: 0.7803 - val_loss: 0.4810 - val_acc: 0.7710\n", 228 | "Epoch 20/30\n", 229 | "128/128 [==============================] - 177s - loss: 0.4919 - acc: 0.7690 - val_loss: 0.5438 - val_acc: 0.7708\n", 230 | "Epoch 21/30\n", 231 | "128/128 [==============================] - 168s - loss: 0.5203 - acc: 0.7563 - val_loss: 0.5304 - val_acc: 0.7485\n", 232 | "Epoch 22/30\n", 233 | "128/128 [==============================] - 166s - loss: 0.4894 - acc: 0.7783 - val_loss: 0.4622 - val_acc: 0.7977\n", 234 | "Epoch 23/30\n", 235 | "128/128 [==============================] - 163s - loss: 0.4898 - acc: 0.7822 - val_loss: 0.4884 - val_acc: 0.7668\n", 236 | "Epoch 24/30\n", 237 | "128/128 [==============================] - 168s - loss: 0.4693 - acc: 0.7886 - val_loss: 0.4487 - val_acc: 0.7922\n", 238 | "Epoch 25/30\n", 239 | "128/128 [==============================] - 156s - loss: 0.4743 - acc: 0.7827 - val_loss: 0.4539 - val_acc: 0.7740\n", 240 | "Epoch 26/30\n", 241 | "128/128 [==============================] - 162s - loss: 0.4562 - acc: 0.7988 - val_loss: 0.4474 - val_acc: 0.7877\n", 242 | "Epoch 27/30\n", 243 | "128/128 [==============================] - 175s - loss: 0.4486 - acc: 0.8081 - val_loss: 0.4203 - val_acc: 0.8143\n", 244 | "Epoch 28/30\n", 245 | "128/128 [==============================] - 168s - loss: 0.4786 - acc: 0.7793 - val_loss: 0.4341 - val_acc: 0.8250\n", 246 | "Epoch 29/30\n", 247 | "128/128 [==============================] - 176s - loss: 0.4929 - acc: 0.7852 - val_loss: 0.4535 - val_acc: 0.7949\n", 248 | "Epoch 30/30\n", 249 | "128/128 [==============================] - 174s - loss: 0.4435 - acc: 0.8052 - val_loss: 0.4284 - val_acc: 0.8031\n" 250 | ] 251 | }, 252 | { 253 | "data": { 254 | "text/plain": [ 255 | "" 256 | ] 257 | }, 258 | "execution_count": 8, 259 | "metadata": {}, 260 | "output_type": "execute_result" 261 | } 262 | ], 263 | "source": [ 264 | "model.fit_generator(\n", 265 | " train_generator,\n", 266 | " validation_data=validation_generator,\n", 267 | " steps_per_epoch=nb_train_samples,\n", 268 | " epochs=nb_epoch,\n", 269 | " validation_steps=nb_validation_samples)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 10, 275 | "metadata": { 276 | "collapsed": true 277 | }, 278 | "outputs": [], 279 | "source": [ 280 | "model.save_weights('models/basic_cnn_20_epochs.h5')" 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "metadata": { 287 | "collapsed": true 288 | }, 289 | "outputs": [], 290 | "source": [ 291 | "#model.load_weights('models_trained/basic_cnn_20_epochs.h5')" 292 | ] 293 | }, 294 | { 295 | "cell_type": "markdown", 296 | "metadata": {}, 297 | "source": [ 298 | "## Evaluating on validation set\n", 299 | "\n", 300 | "Computing loss and accuracy :" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 11, 306 | "metadata": {}, 307 | "outputs": [ 308 | { 309 | "data": { 310 | "text/plain": [ 311 | "[0.42391207523462249, 0.80388252856186693]" 312 | ] 313 | }, 314 | "execution_count": 11, 315 | "metadata": {}, 316 | "output_type": "execute_result" 317 | } 318 | ], 319 | "source": [ 320 | "model.evaluate_generator(validation_generator, nb_validation_samples)" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": {}, 326 | "source": [ 327 | "**After ~30 epochs on the CPU the neural network reached ~80% accuracy. But again we can improve it.**\n", 328 | "\n" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "# Back to Slides" 336 | ] 337 | } 338 | ], 339 | "metadata": { 340 | "kernelspec": { 341 | "display_name": "Python 2", 342 | "language": "python", 343 | "name": "python2" 344 | }, 345 | "language_info": { 346 | "codemirror_mode": { 347 | "name": "ipython", 348 | "version": 2 349 | }, 350 | "file_extension": ".py", 351 | "mimetype": "text/x-python", 352 | "name": "python", 353 | "nbconvert_exporter": "python", 354 | "pygments_lexer": "ipython2", 355 | "version": "2.7.6" 356 | } 357 | }, 358 | "nbformat": 4, 359 | "nbformat_minor": 2 360 | } 361 | -------------------------------------------------------------------------------- /sklearn_demo_Server_Health.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from sklearn import tree\n", 12 | "import graphviz" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "## Decision Tree with scikit learn for Server Health check\n", 20 | "We will train a decision tree to check if a server is healthy or not, based on our sample data against the current CPU, RAM and STORAGE % usage over the last 6 hours: " 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [ 28 | { 29 | "name": "stdout", 30 | "output_type": "stream", 31 | "text": [ 32 | "['healthy', 'unhealthy', 'unhealthy', 'unhealthy', 'unhealthy', 'unhealthy', 'healthy', 'healthy', 'unhealthy', 'healthy', 'healthy', 'unhealthy', 'healthy', 'unhealthy', 'healthy', 'healthy']\n", 33 | "[[45, 32, 65], [87, 67, 100], [100, 1, 1], [76, 70, 90], [1, 1, 100], [31, 100, 50], [12, 65, 39], [20, 10, 46], [100, 50, 50], [34, 70, 37], [1, 50, 50], [50, 50, 100], [50, 1, 50], [1, 100, 1], [50, 50, 1], [53, 53, 80]]\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "# Average Usage in % for the last 6h: [CPU, RAM, STORAGE]\n", 39 | "\n", 40 | "data = [ \n", 41 | " ['healthy', 45, 32, 65], \n", 42 | " ['unhealthy', 87, 67, 100], \n", 43 | " ['unhealthy', 100, 1, 1], \n", 44 | " ['unhealthy', 76, 70, 90], \n", 45 | " ['unhealthy', 1, 1, 100], \n", 46 | " ['unhealthy', 31, 100, 50], \n", 47 | " ['healthy', 12, 65, 39], \n", 48 | " ['healthy', 20, 10, 46], \n", 49 | " ['unhealthy', 100, 50, 50], \n", 50 | " ['healthy', 34, 70, 37], \n", 51 | " ['healthy', 1, 50, 50],\n", 52 | " ['unhealthy', 50, 50, 100], \n", 53 | " ['healthy', 50, 1, 50],\n", 54 | " ['unhealthy', 1, 100, 1], \n", 55 | " ['healthy', 50, 50, 1],\n", 56 | " ['healthy', 53, 53, 80], \n", 57 | "]\n", 58 | "# state to Y\n", 59 | "states = [row[0] for row in data]\n", 60 | "# metrics to X\n", 61 | "metrics = [row[1:] for row in data]\n", 62 | "print states\n", 63 | "print metrics" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 3, 69 | "metadata": { 70 | "collapsed": true 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "# Use a Decision Tree classifier for my tree\n", 75 | "mytree = tree.DecisionTreeClassifier()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 4, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "low CPU, RAM OK, LOW Storage: \n", 88 | "['healthy']\n", 89 | "high CPU and Storage: \n", 90 | "['unhealthy']\n", 91 | "high RAM usage: \n", 92 | "['unhealthy']\n" 93 | ] 94 | } 95 | ], 96 | "source": [ 97 | "# train the Decision Tree with our data\n", 98 | "mytree = mytree.fit(metrics, states)\n", 99 | "\n", 100 | "# CHALLENGE compare their reusults and print the best one!\n", 101 | "print(\"low CPU, RAM OK, LOW Storage: \") \n", 102 | "print(mytree.predict([[10, 80, 10]]))\n", 103 | "print(\"high CPU and Storage: \") \n", 104 | "print(mytree.predict([[80, 10, 90]]))\n", 105 | "print(\"high RAM usage: \") \n", 106 | "print(mytree.predict([[60, 90, 10]]))" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 5, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "# Visulize the decision tree\n", 116 | "dot_data = tree.export_graphviz(mytree, \n", 117 | " feature_names=['CPU','RAM','Storage'],\n", 118 | " class_names=['healthy','unhealthy'],\n", 119 | " filled=True, rounded=True,\n", 120 | " out_file=None) " 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "#### Next demo: http://localhost:8888/notebooks/final/hellotensorflow_intro.ipynb" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 6, 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "data": { 137 | "image/svg+xml": [ 138 | "\n", 139 | "\n", 141 | "\n", 143 | "\n", 144 | "\n", 146 | "\n", 147 | "Tree\n", 148 | "\n", 149 | "\n", 150 | "0\n", 151 | "\n", 152 | "Storage <= 85.0\n", 153 | "gini = 0.5\n", 154 | "samples = 16\n", 155 | "value = [8, 8]\n", 156 | "class = healthy\n", 157 | "\n", 158 | "\n", 159 | "1\n", 160 | "\n", 161 | "RAM <= 85.0\n", 162 | "gini = 0.444\n", 163 | "samples = 12\n", 164 | "value = [8, 4]\n", 165 | "class = healthy\n", 166 | "\n", 167 | "\n", 168 | "0->1\n", 169 | "\n", 170 | "\n", 171 | "True\n", 172 | "\n", 173 | "\n", 174 | "6\n", 175 | "\n", 176 | "gini = 0.0\n", 177 | "samples = 4\n", 178 | "value = [0, 4]\n", 179 | "class = unhealthy\n", 180 | "\n", 181 | "\n", 182 | "0->6\n", 183 | "\n", 184 | "\n", 185 | "False\n", 186 | "\n", 187 | "\n", 188 | "2\n", 189 | "\n", 190 | "CPU <= 76.5\n", 191 | "gini = 0.32\n", 192 | "samples = 10\n", 193 | "value = [8, 2]\n", 194 | "class = healthy\n", 195 | "\n", 196 | "\n", 197 | "1->2\n", 198 | "\n", 199 | "\n", 200 | "\n", 201 | "\n", 202 | "5\n", 203 | "\n", 204 | "gini = 0.0\n", 205 | "samples = 2\n", 206 | "value = [0, 2]\n", 207 | "class = unhealthy\n", 208 | "\n", 209 | "\n", 210 | "1->5\n", 211 | "\n", 212 | "\n", 213 | "\n", 214 | "\n", 215 | "3\n", 216 | "\n", 217 | "gini = 0.0\n", 218 | "samples = 8\n", 219 | "value = [8, 0]\n", 220 | "class = healthy\n", 221 | "\n", 222 | "\n", 223 | "2->3\n", 224 | "\n", 225 | "\n", 226 | "\n", 227 | "\n", 228 | "4\n", 229 | "\n", 230 | "gini = 0.0\n", 231 | "samples = 2\n", 232 | "value = [0, 2]\n", 233 | "class = unhealthy\n", 234 | "\n", 235 | "\n", 236 | "2->4\n", 237 | "\n", 238 | "\n", 239 | "\n", 240 | "\n", 241 | "\n" 242 | ], 243 | "text/plain": [ 244 | "" 245 | ] 246 | }, 247 | "execution_count": 6, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "graphviz.Source(dot_data)" 254 | ] 255 | } 256 | ], 257 | "metadata": { 258 | "kernelspec": { 259 | "display_name": "Python 2", 260 | "language": "python", 261 | "name": "python2" 262 | }, 263 | "language_info": { 264 | "codemirror_mode": { 265 | "name": "ipython", 266 | "version": 2 267 | }, 268 | "file_extension": ".py", 269 | "mimetype": "text/x-python", 270 | "name": "python", 271 | "nbconvert_exporter": "python", 272 | "pygments_lexer": "ipython2", 273 | "version": "2.7.6" 274 | } 275 | }, 276 | "nbformat": 4, 277 | "nbformat_minor": 2 278 | } 279 | -------------------------------------------------------------------------------- /hellotensorflow_demo_end.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# TensorFlow Intro" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "* Names and execution in Python and TensorFlow\n", 15 | "* The simplest TensorFlow neuron\n", 16 | "* Making the neuron learn\n", 17 | "* Training diagnostics in TensorBoard\n", 18 | "* Flowing onward\n", 19 | "\n", 20 | "*courtesy of [Aaron Schumacher](https://www.oreilly.com/learning/hello-tensorflow)*" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": {}, 26 | "source": [ 27 | "### Names and execution in Python and TensorFlow\n", 28 | "#### A name in Python is separated from the object. A name points at the object and not the other way around." 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "\n", 36 | "\"An\n", 37 | "\n", 38 | "\n", 39 | "\n", 40 | "*Image courtesy of [Hadley Wickham](https://twitter.com/hadleywickham/status/732288980549390336).*\n", 41 | "\n", 42 | "Let's see a code example of foo pointing to an object:" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 1, 48 | "metadata": { 49 | "collapsed": true 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "foo = []" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 2, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "bar = foo" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 3, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "data": { 74 | "text/plain": [ 75 | "True" 76 | ] 77 | }, 78 | "execution_count": 3, 79 | "metadata": {}, 80 | "output_type": "execute_result" 81 | } 82 | ], 83 | "source": [ 84 | "foo == bar" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 4, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/plain": [ 95 | "True" 96 | ] 97 | }, 98 | "execution_count": 4, 99 | "metadata": {}, 100 | "output_type": "execute_result" 101 | } 102 | ], 103 | "source": [ 104 | "foo is bar" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 5, 110 | "metadata": {}, 111 | "outputs": [ 112 | { 113 | "data": { 114 | "text/plain": [ 115 | "36658424" 116 | ] 117 | }, 118 | "execution_count": 5, 119 | "metadata": {}, 120 | "output_type": "execute_result" 121 | } 122 | ], 123 | "source": [ 124 | "id(foo)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 6, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "text/plain": [ 135 | "36658424" 136 | ] 137 | }, 138 | "execution_count": 6, 139 | "metadata": {}, 140 | "output_type": "execute_result" 141 | } 142 | ], 143 | "source": [ 144 | "id(bar)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "You can now see that id(foo) and id(bar) are the same." 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 7, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "# Nesting lists is one way to represent a graph structure like a TensorFlow computation graph.\n", 161 | "foo.append(bar) \n" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 8, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "data": { 171 | "text/plain": [ 172 | "[[...]]" 173 | ] 174 | }, 175 | "execution_count": 8, 176 | "metadata": {}, 177 | "output_type": "execute_result" 178 | } 179 | ], 180 | "source": [ 181 | "# and now foo is inside itself...\n", 182 | "foo" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "Python is managing foo and bar and both names point at the same thing.\n", 190 | "\n", 191 | "\"foo\n", 192 | "\n", 193 | "*Image made with [draw.io](https://draw.io/).*\n", 194 | "\n", 195 | "\n", 196 | "Meaning that...\n", 197 | "### Tensorflow has also a system to keep track of things somewhere and giving access via names.\n" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "### Let's start by the simplest TensorFlow neuron\n", 205 | "*Will just have an input and a weight and we will be able to multiply that*" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": 9, 211 | "metadata": { 212 | "collapsed": true 213 | }, 214 | "outputs": [], 215 | "source": [ 216 | "# Inport tensorflow, create the graph and define the values\n", 217 | "import tensorflow as tf\n", 218 | "sess = tf.Session()\n", 219 | "graph = tf.get_default_graph()\n", 220 | "input_value = tf.constant(1.0)\n", 221 | "weight = tf.Variable(0.8)" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 10, 227 | "metadata": { 228 | "collapsed": true 229 | }, 230 | "outputs": [], 231 | "source": [ 232 | "# We define our operation in the graph\n", 233 | "output_value = weight * input_value\n", 234 | "op = graph.get_operations()[-1]" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "### What is the last operation in the graph? Multiplication." 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": 11, 247 | "metadata": {}, 248 | "outputs": [ 249 | { 250 | "data": { 251 | "text/plain": [ 252 | "u'mul'" 253 | ] 254 | }, 255 | "execution_count": 11, 256 | "metadata": {}, 257 | "output_type": "execute_result" 258 | } 259 | ], 260 | "source": [ 261 | "op.name" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 12, 267 | "metadata": {}, 268 | "outputs": [ 269 | { 270 | "data": { 271 | "text/plain": [ 272 | "0.80000001" 273 | ] 274 | }, 275 | "execution_count": 12, 276 | "metadata": {}, 277 | "output_type": "execute_result" 278 | } 279 | ], 280 | "source": [ 281 | "# Generates an operation which will initialize all our variables a finally run it\n", 282 | "\n", 283 | "sess.run(tf.global_variables_initializer())\n", 284 | "sess.run(output_value)" 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "*Recall that's 0.8 x 1.0 with 32-bit floats, and 32-bit floats have a hard time with 0.8; 0.80000001 is as close as they can get.*\n", 292 | "\n", 293 | "This is the neuron's \"inference\" or \"forward pass\"." 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "### Neat! let's look at our graph in TensorBoard" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 13, 306 | "metadata": {}, 307 | "outputs": [], 308 | "source": [ 309 | "# reset previous stuff\n", 310 | "tf.reset_default_graph()\n", 311 | "sess = tf.Session()\n", 312 | "!rm -rf log_simple_graph\n", 313 | "!rm -rf log_simple_stat\n", 314 | "\n", 315 | "x = tf.constant(1.0, name='input')\n", 316 | "w = tf.Variable(0.8, name='weight')\n", 317 | "y = tf.multiply(w, x, name='output')" 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": 14, 323 | "metadata": {}, 324 | "outputs": [], 325 | "source": [ 326 | "# FileWriter needs an output directory.\n", 327 | "summary_writer = tf.summary.FileWriter('log_simple_graph', sess.graph)" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 15, 333 | "metadata": { 334 | "scrolled": true 335 | }, 336 | "outputs": [ 337 | { 338 | "name": "stdout", 339 | "output_type": "stream", 340 | "text": [ 341 | "TensorBoard 0.1.5 at http://ricardo-T440s:6006 (Press CTRL+C to quit) ^C\n", 342 | "\n" 343 | ] 344 | } 345 | ], 346 | "source": [ 347 | "!tensorboard --logdir=log_simple_graph" 348 | ] 349 | }, 350 | { 351 | "cell_type": "markdown", 352 | "metadata": {}, 353 | "source": [ 354 | "To continue with the notebook, interrupt the kernel by using the square \"stop\" button or by typing `esc`, `i`, `i`." 355 | ] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": {}, 360 | "source": [ 361 | "### Making the neuron learn with gradient descent\n", 362 | "Now the system takes the input one and returns 0.8, which is worg. We need a way to measure how wrong the system is. We'll call that measure the “loss” and give our system the goal of minimizing the loss. \n" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": 16, 368 | "metadata": { 369 | "collapsed": true 370 | }, 371 | "outputs": [], 372 | "source": [ 373 | "y_ = tf.constant(0.0)" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": {}, 379 | "source": [ 380 | "We make the loss the **square of the difference** between the **current output** and the **desired output**, so it doesn't get to a negative values." 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 17, 386 | "metadata": { 387 | "collapsed": true 388 | }, 389 | "outputs": [], 390 | "source": [ 391 | "loss = (y - y_)**2\n", 392 | "\n", 393 | "# Optimizer with a learning rate of 0.025.\n", 394 | "optim = tf.train.GradientDescentOptimizer(learning_rate=0.025)" 395 | ] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "metadata": {}, 400 | "source": [ 401 | "\"plot\n", 402 | "
Plot of Loss and Derivative
" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 18, 408 | "metadata": {}, 409 | "outputs": [ 410 | { 411 | "data": { 412 | "text/plain": [ 413 | "1.6" 414 | ] 415 | }, 416 | "execution_count": 18, 417 | "metadata": {}, 418 | "output_type": "execute_result" 419 | } 420 | ], 421 | "source": [ 422 | "grads_and_vars = optim.compute_gradients(loss)\n", 423 | "sess.run(tf.global_variables_initializer())\n", 424 | "sess.run(grads_and_vars[0][0])" 425 | ] 426 | }, 427 | { 428 | "cell_type": "markdown", 429 | "metadata": {}, 430 | "source": [ 431 | "Why is the value of the gradient 1.6? Our loss is error squared, and the derivative of that is two times the error. Currently the system says 0.8 instead of 0, so the error is 0.8, and two times 0.8 is 1.6. It's working!\n", 432 | "\n", 433 | "We have `y = 0.8` and `y_ = 0`.\n", 434 | "\n", 435 | "So the loss is `(0.8 - 0)^2 = 0.64`.\n", 436 | "\n", 437 | "And the derivative of the loss is `2*(0.8 - 0)`.\n", 438 | "\n", 439 | "#### Let's apply the gradient, finishing the backpropagation." 440 | ] 441 | }, 442 | { 443 | "cell_type": "code", 444 | "execution_count": 19, 445 | "metadata": {}, 446 | "outputs": [ 447 | { 448 | "data": { 449 | "text/plain": [ 450 | "0.75999999" 451 | ] 452 | }, 453 | "execution_count": 19, 454 | "metadata": {}, 455 | "output_type": "execute_result" 456 | } 457 | ], 458 | "source": [ 459 | "sess.run(optim.apply_gradients(grads_and_vars))\n", 460 | "sess.run(w)" 461 | ] 462 | }, 463 | { 464 | "cell_type": "markdown", 465 | "metadata": {}, 466 | "source": [ 467 | "The weight decreased by 0.04 because the optimizer subtracted the gradient times the learning rate, 1.6 * 0.025, pushing the weight in the right direction.\n", 468 | "\n", 469 | "`0.8 - 0.025*1.6 = 0.8 - 0.04 = 0.76`\n", 470 | "\n", 471 | "Instead of hand-holding the optimizer like this, we can make one operation that calculates and applies the gradients: the `train_step`" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 20, 477 | "metadata": {}, 478 | "outputs": [ 479 | { 480 | "data": { 481 | "text/plain": [ 482 | "0.0044996012" 483 | ] 484 | }, 485 | "execution_count": 20, 486 | "metadata": {}, 487 | "output_type": "execute_result" 488 | } 489 | ], 490 | "source": [ 491 | "train_step = tf.train.GradientDescentOptimizer(0.025).minimize(loss)\n", 492 | "\n", 493 | "for i in range(100):\n", 494 | " sess.run(train_step)\n", 495 | "\n", 496 | "sess.run(y)" 497 | ] 498 | }, 499 | { 500 | "cell_type": "markdown", 501 | "metadata": {}, 502 | "source": [ 503 | "Running the training step many times, the weight and the output value are now very close to zero. \n", 504 | "\n", 505 | "The neuron has learned!\n" 506 | ] 507 | }, 508 | { 509 | "cell_type": "markdown", 510 | "metadata": {}, 511 | "source": [ 512 | "# Last Step: The all example on Tensorboard" 513 | ] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "execution_count": 25, 518 | "metadata": { 519 | "collapsed": true 520 | }, 521 | "outputs": [], 522 | "source": [ 523 | "# reset everything \n", 524 | "tf.reset_default_graph()\n", 525 | "!rm -rf log_simple_stats" 526 | ] 527 | }, 528 | { 529 | "cell_type": "markdown", 530 | "metadata": {}, 531 | "source": [ 532 | "The script below is designed to stand alone, and will also work here with the default graph reset." 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": 26, 538 | "metadata": { 539 | "collapsed": true 540 | }, 541 | "outputs": [], 542 | "source": [ 543 | "import tensorflow as tf\n", 544 | "\n", 545 | "x = tf.constant(1.0, name='input')\n", 546 | "w = tf.Variable(0.8, name='weight')\n", 547 | "y = tf.multiply(w, x, name='output')\n", 548 | "y_ = tf.constant(0.0, name='correct_value')\n", 549 | "loss = tf.pow(y - y_, 2, name='loss')\n", 550 | "train_step = tf.train.GradientDescentOptimizer(0.025).minimize(loss)\n", 551 | "\n", 552 | "for value in [x, w, y, y_, loss]:\n", 553 | " tf.summary.scalar(value.op.name, value)\n", 554 | "\n", 555 | "summaries = tf.summary.merge_all()\n", 556 | "\n", 557 | "session = tf.Session()\n", 558 | "summary_writer = tf.summary.FileWriter('log_simple_stats', session.graph)\n", 559 | "\n", 560 | "session.run(tf.global_variables_initializer())\n", 561 | "for i in range(100):\n", 562 | " summary_writer.add_summary(session.run(summaries), i)\n", 563 | " session.run(train_step)\n", 564 | "\n", 565 | "summary_writer.close()" 566 | ] 567 | }, 568 | { 569 | "cell_type": "code", 570 | "execution_count": 27, 571 | "metadata": { 572 | "scrolled": true 573 | }, 574 | "outputs": [ 575 | { 576 | "name": "stdout", 577 | "output_type": "stream", 578 | "text": [ 579 | "TensorBoard 0.1.5 at http://ricardo-T440s:6006 (Press CTRL+C to quit) ^C\n", 580 | "\n" 581 | ] 582 | } 583 | ], 584 | "source": [ 585 | "!tensorboard --logdir=log_simple_stats" 586 | ] 587 | }, 588 | { 589 | "cell_type": "markdown", 590 | "metadata": {}, 591 | "source": [ 592 | "After startup, go to http://localhost:6006/#events to see the interface.\n", 593 | "\n", 594 | "To continue with the notebook, interrupt the kernel by using the square \"stop\" button or by typing `esc`, `i`, `i`." 595 | ] 596 | }, 597 | { 598 | "cell_type": "markdown", 599 | "metadata": { 600 | "collapsed": true 601 | }, 602 | "source": [ 603 | "### Next Demo: http://localhost:8888/notebooks/final/ImageClassifier_GPU.ipynb \n" 604 | ] 605 | }, 606 | { 607 | "cell_type": "code", 608 | "execution_count": null, 609 | "metadata": { 610 | "collapsed": true 611 | }, 612 | "outputs": [], 613 | "source": [] 614 | } 615 | ], 616 | "metadata": { 617 | "kernelspec": { 618 | "display_name": "Python 2", 619 | "language": "python", 620 | "name": "python2" 621 | }, 622 | "language_info": { 623 | "codemirror_mode": { 624 | "name": "ipython", 625 | "version": 2 626 | }, 627 | "file_extension": ".py", 628 | "mimetype": "text/x-python", 629 | "name": "python", 630 | "nbconvert_exporter": "python", 631 | "pygments_lexer": "ipython2", 632 | "version": "2.7.6" 633 | } 634 | }, 635 | "nbformat": 4, 636 | "nbformat_minor": 1 637 | } 638 | -------------------------------------------------------------------------------- /data/nginx_requests.bal-demoA.deltas.csv: -------------------------------------------------------------------------------- 1 | 455001 2 | 442973 3 | 437442 4 | 445951 5 | 439219 6 | 462725 7 | 498109 8 | 613896 9 | 778066 10 | 884924 11 | 905967 12 | 886269 13 | 873787 14 | 868160 15 | 787165 16 | 735387 17 | 677927 18 | 626825 19 | 614388 20 | 594544 21 | 577593 22 | 493081 23 | 454072 24 | 426678 25 | 420059 26 | 417025 27 | 426057 28 | 431692 29 | 469363 30 | 564114 31 | 690581 32 | 791161 33 | 846474 34 | 903309 35 | 895457 36 | 806941 37 | 817156 38 | 767361 39 | 716363 40 | 698105 41 | 533199 42 | 512643 43 | 496724 44 | 480436 45 | 520345 46 | 504876 47 | 463866 48 | 394116 49 | 383860 50 | 407791 51 | 388684 52 | 415923 53 | 478803 54 | 600958 55 | 627162 56 | 642389 57 | 587983 58 | 531306 59 | 559893 60 | 549130 61 | 783845 62 | 644688 63 | 664794 64 | 699052 65 | 564625 66 | 499852 67 | 512757 68 | 457385 69 | 427234 70 | 416618 71 | 409250 72 | 467281 73 | 483753 74 | 426853 75 | 468020 76 | 555479 77 | 606357 78 | 632808 79 | 562323 80 | 585635 81 | 581527 82 | 582509 83 | 588778 84 | 596011 85 | 566716 86 | 658179 87 | 647793 88 | 556708 89 | 552320 90 | 520337 91 | 478471 92 | 456733 93 | 441299 94 | 417682 95 | 407654 96 | 417018 97 | 453925 98 | 490929 99 | 603980 100 | 767749 101 | 908347 102 | 895436 103 | 726432 104 | 655927 105 | 637997 106 | 751289 107 | 620687 108 | 502214 109 | 481831 110 | 518703 111 | 457984 112 | 468997 113 | 464629 114 | 543303 115 | 573866 116 | 638207 117 | 909604 118 | 854130 119 | 727643 120 | 680629 121 | 777145 122 | 757096 123 | 646076 124 | 640477 125 | 638977 126 | 528548 127 | 451215 128 | 440351 129 | 436129 130 | 521346 131 | 524852 132 | 547441 133 | 643999 134 | 778558 135 | 821337 136 | 707959 137 | 647995 138 | 623575 139 | 600637 140 | 583978 141 | 561419 142 | 495816 143 | 473739 144 | 449465 145 | 447073 146 | 464764 147 | 462290 148 | 460372 149 | 516574 150 | 612130 151 | 793743 152 | 898323 153 | 864518 154 | 783842 155 | 730968 156 | 703025 157 | 679793 158 | 685474 159 | 574127 160 | 505333 161 | 488782 162 | 692413 163 | 576107 164 | 557092 165 | 557419 166 | 662333 167 | 686769 168 | 785178 169 | 903366 170 | 855117 171 | 765028 172 | 682386 173 | 689840 174 | 731323 175 | 690939 176 | 682707 177 | 654434 178 | 605568 179 | 593931 180 | 538863 181 | 505771 182 | 516301 183 | 526605 184 | 560297 185 | 610677 186 | 802189 187 | 755248 188 | 797307 189 | 799609 190 | 803513 191 | 785770 192 | 887171 193 | 836785 194 | 789287 195 | 802764 196 | 754993 197 | 663908 198 | 724794 199 | 691501 200 | 589659 201 | 548407 202 | 526829 203 | 545518 204 | 551106 205 | 616816 206 | 641955 207 | 653322 208 | 671458 209 | 732130 210 | 702858 211 | 715510 212 | 733449 213 | 728855 214 | 743386 215 | 727514 216 | 709707 217 | 691619 218 | 691177 219 | 709110 220 | 700742 221 | 661117 222 | 615026 223 | 545039 224 | 501996 225 | 471655 226 | 450115 227 | 463004 228 | 465712 229 | 493725 230 | 538358 231 | 654036 232 | 824614 233 | 835106 234 | 774609 235 | 759958 236 | 736942 237 | 687490 238 | 639036 239 | 551684 240 | 498683 241 | 483434 242 | 476713 243 | 471165 244 | 474925 245 | 495145 246 | 551208 247 | 665127 248 | 831589 249 | 833423 250 | 752170 251 | 741373 252 | 728318 253 | 713897 254 | 676545 255 | 589117 256 | 512004 257 | 475657 258 | 470979 259 | 456017 260 | 458440 261 | 486249 262 | 534154 263 | 645519 264 | 817354 265 | 887240 266 | 806901 267 | 775348 268 | 738329 269 | 715183 270 | 681346 271 | 630720 272 | 560014 273 | 503223 274 | 478685 275 | 449920 276 | 452706 277 | 476296 278 | 486875 279 | 519300 280 | 636880 281 | 825643 282 | 907401 283 | 862452 284 | 760259 285 | 709001 286 | 673764 287 | 678832 288 | 661628 289 | 640381 290 | 572731 291 | 520166 292 | 493188 293 | 499991 294 | 500387 295 | 512932 296 | 533220 297 | 560073 298 | 664938 299 | 791736 300 | 890279 301 | 895137 302 | 876531 303 | 761884 304 | 684649 305 | 660633 306 | 652625 307 | 596094 308 | 596410 309 | 546508 310 | 510514 311 | 502345 312 | 497707 313 | 504325 314 | 485141 315 | 452844 316 | 467307 317 | 521567 318 | 618652 319 | 655181 320 | 669940 321 | 692980 322 | 702634 323 | 695068 324 | 710561 325 | 688136 326 | 653066 327 | 630367 328 | 610856 329 | 602000 330 | 595301 331 | 577791 332 | 543409 333 | 496257 334 | 463386 335 | 442098 336 | 427660 337 | 422554 338 | 433784 339 | 448733 340 | 468738 341 | 508982 342 | 547773 343 | 595947 344 | 624808 345 | 661219 346 | 666008 347 | 670039 348 | 661687 349 | 683039 350 | 662022 351 | 647086 352 | 685042 353 | 710884 354 | 695050 355 | 678318 356 | 633107 357 | 567164 358 | 550586 359 | 527898 360 | 457548 361 | 449190 362 | 461697 363 | 490957 364 | 535265 365 | 652607 366 | 860110 367 | 819072 368 | 767453 369 | 766082 370 | 732443 371 | 676042 372 | 587074 373 | 506391 374 | 491563 375 | 499196 376 | 476951 377 | 451416 378 | 462422 379 | 524366 380 | 657710 381 | 838153 382 | 852173 383 | 778333 384 | 748712 385 | 734847 386 | 709383 387 | 662449 388 | 587727 389 | 524915 390 | 494891 391 | 456672 392 | 407335 393 | 418690 394 | 425548 395 | 414630 396 | 410168 397 | 405710 398 | 410962 399 | 389932 400 | 403256 401 | 399394 402 | 393875 403 | 391504 404 | 402610 405 | 383215 406 | 379897 407 | 401211 408 | 411811 409 | 397962 410 | 404809 411 | 414566 412 | 395351 413 | 380295 414 | 385980 415 | 389068 416 | 404489 417 | 412094 418 | 417190 419 | 410851 420 | 426300 421 | 413696 422 | 396951 423 | 384345 424 | 401641 425 | 422255 426 | 410186 427 | 399183 428 | 417912 429 | 392276 430 | 413053 431 | 485986 432 | 511407 433 | 506876 434 | 515363 435 | 489853 436 | 420773 437 | 409488 438 | 423216 439 | 430510 440 | 414053 441 | 399116 442 | 414533 443 | 399437 444 | 393275 445 | 389202 446 | 402153 447 | 382650 448 | 380887 449 | 456617 450 | 448384 451 | 434020 452 | 424750 453 | 423387 454 | 442748 455 | 385083 456 | 453312 457 | 508556 458 | 518336 459 | 571820 460 | 681590 461 | 851737 462 | 903444 463 | 835880 464 | 735781 465 | 717465 466 | 700596 467 | 673412 468 | 625550 469 | 567255 470 | 514003 471 | 522841 472 | 483273 473 | 494324 474 | 557757 475 | 540291 476 | 572860 477 | 681542 478 | 829614 479 | 902831 480 | 857518 481 | 749475 482 | 671139 483 | 627910 484 | 622781 485 | 609599 486 | 570936 487 | 530629 488 | 475306 489 | 461489 490 | 457559 491 | 452644 492 | 445567 493 | 464431 494 | 520590 495 | 616969 496 | 775773 497 | 876433 498 | 897702 499 | 895503 500 | 860667 501 | 815460 502 | 731375 503 | 664625 504 | 568646 505 | 629356 506 | 637221 507 | 646523 508 | 504739 509 | 471529 510 | 437079 511 | 404918 512 | 417775 513 | 408913 514 | 395852 515 | 396549 516 | 430686 517 | 461770 518 | 502253 519 | 547352 520 | 556861 521 | 575106 522 | 569515 523 | 568382 524 | 583473 525 | 564876 526 | 550322 527 | 548855 528 | 527414 529 | 514684 530 | 509322 531 | 511990 532 | 490656 533 | 482979 534 | 446742 535 | 419768 536 | 432695 537 | 427517 538 | 427972 539 | 420591 540 | 441517 541 | 486806 542 | 522046 543 | 546285 544 | 573774 545 | 599620 546 | 596639 547 | 663589 548 | 633384 549 | 634310 550 | 627261 551 | 626640 552 | 611014 553 | 621440 554 | 623016 555 | 606304 556 | 558993 557 | 515278 558 | 470582 559 | 440408 560 | 440813 561 | 435563 562 | 456259 563 | 461260 564 | 506840 565 | 623317 566 | 806683 567 | 877810 568 | 776749 569 | 692795 570 | 655444 571 | 641922 572 | 622866 573 | 589944 574 | 539014 575 | 482142 576 | 479752 577 | 463612 578 | 470848 579 | 465452 580 | 473364 581 | 516863 582 | 641609 583 | 819138 584 | 830890 585 | 733171 586 | 657254 587 | 637237 588 | 632638 589 | 609656 590 | 569502 591 | 511489 592 | 461032 593 | 441920 594 | 440945 595 | 436351 596 | 444424 597 | 447276 598 | 508772 599 | 616198 600 | 774781 601 | 898157 602 | 889852 603 | 814564 604 | 728408 605 | 722360 606 | 643528 607 | 616802 608 | 604309 609 | 563464 610 | 547165 611 | 484730 612 | 474260 613 | 477625 614 | 471706 615 | 491094 616 | 543494 617 | 655411 618 | 826373 619 | 805498 620 | 727939 621 | 709434 622 | 717345 623 | 667795 624 | 622939 625 | 524563 626 | 480417 627 | 451523 628 | 454278 629 | 457917 630 | 459028 631 | 384861 632 | 408668 633 | 435878 634 | 408353 635 | 403129 636 | 402666 637 | 401818 638 | 387641 639 | 387624 640 | 399249 641 | 395790 642 | 397051 643 | 391404 644 | 394075 645 | 386846 646 | 392558 647 | 404828 648 | 406293 649 | 380876 650 | 397163 651 | 382404 652 | 412076 653 | 433551 654 | 423651 655 | 414100 656 | 420332 657 | 413325 658 | 402954 659 | 382536 660 | 401291 661 | 417351 662 | 412726 663 | 409473 664 | 406622 665 | 398357 666 | 394154 667 | 398119 668 | 407715 669 | 400879 670 | 388330 671 | 388857 672 | 397954 673 | 383121 674 | 405045 675 | 411976 676 | 405696 677 | 397609 678 | 394536 679 | 385195 680 | 388263 681 | 381270 682 | 404794 683 | 420104 684 | 414781 685 | 404432 686 | 420690 687 | 403096 688 | 393041 689 | 385488 690 | 423470 691 | 426553 692 | 426945 693 | 413644 694 | 416598 695 | 410419 696 | 398439 697 | 906050 698 | 839330 699 | 782767 700 | 405442 701 | 424924 702 | 418922 703 | 394203 704 | 396947 705 | 394141 706 | 384448 707 | 397285 708 | 398438 709 | 384719 710 | 385299 711 | 386031 712 | 416682 713 | 407850 714 | 409259 715 | 407155 716 | 413171 717 | 407659 718 | 389961 719 | 396335 720 | 397218 721 | 394297 722 | 386095 723 | 395011 724 | 397734 725 | 379377 726 | 381302 727 | 390611 728 | 400607 729 | 391840 730 | 384493 731 | 383644 732 | 381482 733 | 406832 734 | 425510 735 | 416597 736 | 394488 737 | 405945 738 | 406679 739 | 405885 740 | 390293 741 | 386651 742 | 392443 743 | 381135 744 | 389339 745 | 390846 746 | 398183 747 | 385345 748 | 391388 749 | 407546 750 | 402236 751 | 409038 752 | 384387 753 | 389885 754 | 405768 755 | 398866 756 | 382985 757 | 384675 758 | 392222 759 | 382656 760 | 390528 761 | 383935 762 | 406665 763 | 384223 764 | 382342 765 | 385444 766 | 382038 767 | 385308 768 | 381085 769 | 381859 770 | 380780 771 | 788240 772 | 751295 773 | 749537 774 | 748045 775 | 753484 776 | 772474 777 | 814553 778 | 877945 779 | 900345 780 | 869652 781 | 824364 782 | 798740 783 | 769584 784 | 744502 785 | 759122 786 | 783337 787 | 833284 788 | 870420 789 | 793913 790 | 768199 791 | 777921 792 | 810731 793 | 811872 794 | 813263 795 | 855320 796 | 904056 797 | 824313 798 | 786591 799 | 760842 800 | 770434 801 | 769976 802 | 785054 803 | 857516 804 | 888338 805 | 797893 806 | 769894 807 | 751147 808 | 762406 809 | 767875 810 | 790899 811 | 850403 812 | 904447 813 | 819120 814 | 786730 815 | 763244 816 | 759618 817 | 767087 818 | 807721 819 | 877005 820 | 891128 821 | 840653 822 | 843610 823 | 805096 824 | 714113 825 | 654607 826 | 581504 827 | 561525 828 | 548024 829 | 517283 830 | 527144 831 | 554303 832 | 642541 833 | 629776 834 | 697733 835 | 781255 836 | 803596 837 | 756865 838 | 752323 839 | 781057 840 | 791695 841 | 797019 842 | 760526 843 | 721474 844 | 714813 845 | 711771 846 | 716980 847 | 745838 848 | 730700 849 | 684872 850 | 615704 851 | 621363 852 | 605688 853 | 614860 854 | 587567 855 | 612734 856 | 649637 857 | 690322 858 | 734218 859 | 771030 860 | 793224 861 | 816030 862 | 821251 863 | 836308 864 | 870263 865 | 844707 866 | 816996 867 | 776077 868 | 778758 869 | 790404 870 | 812663 871 | 823270 872 | 808766 873 | 758167 874 | 710383 875 | 657238 876 | 637499 877 | 624127 878 | 622948 879 | 652352 880 | 703702 881 | 787317 882 | 885080 883 | 890375 884 | 842256 885 | 841319 886 | 837333 887 | 832895 888 | 775962 889 | 726816 890 | 770477 891 | 646868 892 | 643530 893 | 647026 894 | 648453 895 | 681336 896 | 715640 897 | 776728 898 | 803699 899 | 840198 900 | 853676 901 | 865718 902 | 861491 903 | 905475 904 | 895301 905 | 903101 906 | 864184 907 | 832093 908 | 804708 909 | 802809 910 | 817207 911 | 772334 912 | 755463 913 | 731337 914 | 704796 915 | 670944 916 | 636302 917 | 648334 918 | 668467 919 | 681134 920 | 742836 921 | 857650 922 | 909597 923 | 882390 924 | 822173 925 | 741143 926 | 707842 927 | 677407 928 | 705688 929 | 705650 930 | 716579 931 | 780151 932 | 907504 933 | 901951 934 | 883710 935 | 831137 936 | 734502 937 | 722346 938 | 715934 939 | 700123 940 | 702558 941 | 725220 942 | 774319 943 | 882141 944 | 879139 945 | 864424 946 | 831545 947 | 817614 948 | 760488 949 | 722419 950 | 701036 951 | 725737 952 | 716006 953 | 693664 954 | 719979 955 | 736245 956 | 778365 957 | 844231 958 | 876986 959 | 899397 960 | 905249 961 | 897326 962 | 907518 963 | 894147 964 | 886251 965 | 859127 966 | 867479 967 | 862338 968 | 822592 969 | 776281 970 | 787709 971 | 691153 972 | 685103 973 | 681920 974 | 685993 975 | 695121 976 | 747730 977 | 778733 978 | 821749 979 | 888282 980 | 887634 981 | 907464 982 | 890351 983 | 898835 984 | 898392 985 | 861510 986 | 797383 987 | 757911 988 | 751300 989 | 721668 990 | 705892 991 | 738248 992 | 736590 993 | 806185 994 | 892085 995 | 837750 996 | 746412 997 | 711102 998 | 692775 999 | 702504 1000 | 708119 1001 | 733551 1002 | 807682 1003 | 908602 1004 | 798898 1005 | 793918 1006 | 775779 1007 | 743440 1008 | 738732 1009 | 753271 1010 | 749986 1011 | 795194 1012 | 865561 1013 | 808896 1014 | 793900 1015 | 727065 1016 | 701243 1017 | 712170 1018 | 725422 1019 | 752545 1020 | 807381 1021 | 848915 1022 | 783146 1023 | 775484 1024 | 803457 1025 | 827183 1026 | 824496 1027 | 818556 1028 | 823739 1029 | 891943 1030 | 881225 1031 | 841058 1032 | 808710 1033 | 790860 1034 | 797091 1035 | 773308 1036 | 725196 1037 | 738564 1038 | 845810 1039 | 831059 1040 | 839762 1041 | 869048 1042 | 889961 1043 | 894137 1044 | 898617 1045 | 851714 1046 | 808778 1047 | 755293 1048 | 725357 1049 | 732718 1050 | 778270 1051 | 794494 1052 | 758477 1053 | 786562 1054 | 828345 1055 | 830146 1056 | 781751 1057 | 736247 1058 | 738532 1059 | 747324 1060 | 768788 1061 | 810707 1062 | 856016 1063 | 872854 1064 | 813175 1065 | 765038 1066 | 740548 1067 | 762097 1068 | 792901 1069 | 804612 1070 | 902413 1071 | 868592 1072 | 801960 1073 | 772424 1074 | 796912 1075 | 751114 1076 | 762477 1077 | 766452 1078 | 848224 1079 | 880373 1080 | 810229 1081 | 786205 1082 | 741731 1083 | 751821 1084 | 772033 1085 | 799978 1086 | 854858 1087 | 875625 1088 | 813483 1089 | 771599 1090 | 739128 1091 | 761373 1092 | 773734 1093 | 826140 1094 | 855252 1095 | 904768 1096 | 895968 1097 | 887001 1098 | 804627 1099 | 756826 1100 | 727341 1101 | 734252 1102 | 728483 1103 | 736224 1104 | 778619 1105 | 860040 1106 | 860980 1107 | 906862 1108 | 874792 1109 | 868494 1110 | 864427 1111 | 831698 1112 | 780125 1113 | 737264 1114 | 738453 1115 | 719718 1116 | 711206 1117 | 723051 1118 | 746453 1119 | 813279 1120 | 836505 1121 | 890256 1122 | 855234 1123 | 800296 1124 | 787340 1125 | 798232 1126 | 779782 1127 | 798098 1128 | 845478 1129 | 890382 1130 | 830963 1131 | 756082 1132 | 762837 1133 | 746596 1134 | 752487 1135 | 743204 1136 | 777918 1137 | 855229 1138 | 902221 1139 | 846240 1140 | 788710 1141 | 752565 1142 | 732189 1143 | 714044 1144 | 727605 1145 | 771378 1146 | 821943 1147 | 881860 1148 | 799676 1149 | 751839 1150 | 749821 1151 | 736501 1152 | 760466 1153 | 745415 1154 | 785420 1155 | 816338 1156 | 902505 1157 | 822229 1158 | 764248 1159 | 750859 1160 | 745785 1161 | 731706 1162 | 742489 1163 | 782318 1164 | 832004 1165 | 902643 1166 | 868069 1167 | 806144 1168 | 757307 1169 | 723223 1170 | 707036 1171 | 709311 1172 | 725518 1173 | 786379 1174 | 806540 1175 | 819615 1176 | 901108 1177 | 882490 1178 | 814396 1179 | 783552 1180 | 753354 1181 | 709675 1182 | 706644 1183 | 720278 1184 | 749493 1185 | 782927 1186 | 836565 1187 | 887528 1188 | 838927 1189 | 796317 1190 | 746545 1191 | 725050 1192 | 738187 1193 | 740220 1194 | 787422 1195 | 865200 1196 | 902201 1197 | 827827 1198 | 757347 1199 | 646980 1200 | 580932 1201 | 552084 1202 | 509427 1203 | 503587 1204 | 514882 1205 | 547463 1206 | 628421 1207 | 730636 1208 | 825475 1209 | 797426 1210 | 806065 1211 | 796093 1212 | 735187 1213 | 658360 1214 | 590732 1215 | 563947 1216 | 586039 1217 | 579716 1218 | 565190 1219 | 592552 1220 | 644313 1221 | 776959 1222 | 795236 1223 | 811109 1224 | 847252 1225 | 839940 1226 | 786322 1227 | 755682 1228 | 672038 1229 | 597490 1230 | 587776 1231 | 599672 1232 | 616528 1233 | 629566 1234 | 678905 1235 | 818700 1236 | 904056 1237 | 837238 1238 | 842675 1239 | 883154 1240 | 832699 1241 | 714687 1242 | 647961 1243 | 616476 1244 | 587363 1245 | 586515 1246 | 619843 1247 | 652681 1248 | 698663 1249 | 791251 1250 | 904394 1251 | 897126 1252 | 802725 1253 | 797258 1254 | 796958 1255 | 782086 1256 | 741466 1257 | 683240 1258 | 623912 1259 | 598199 1260 | 610393 1261 | 617163 1262 | 604952 1263 | 620533 1264 | 666937 1265 | 721690 1266 | 770181 1267 | 787308 1268 | 813160 1269 | 871811 1270 | 826037 1271 | 807418 1272 | 791957 1273 | 797811 1274 | 825816 1275 | 839172 1276 | 775496 1277 | 757788 1278 | 743839 1279 | 754966 1280 | 743338 1281 | 684059 1282 | 627161 1283 | 595882 1284 | 589806 1285 | 632312 1286 | 610762 1287 | 638059 1288 | 704013 1289 | 695572 1290 | 738182 1291 | 754708 1292 | 794284 1293 | 793761 1294 | 812113 1295 | 829940 1296 | 845318 1297 | 900462 1298 | 821349 1299 | 812459 1300 | 802917 1301 | 824175 1302 | 805707 1303 | 776419 1304 | 729383 1305 | 679750 1306 | 637945 1307 | 628651 1308 | 615290 1309 | 629551 1310 | 651581 1311 | 741060 1312 | 814755 1313 | 906068 1314 | 875537 1315 | 890618 1316 | 862239 1317 | 826470 1318 | 752094 1319 | 670253 1320 | 640548 1321 | 662760 1322 | 666173 1323 | 657608 1324 | 733495 1325 | 769910 1326 | 873468 1327 | 873382 1328 | 787708 1329 | 712840 1330 | 681254 1331 | 651635 1332 | 657167 1333 | 666287 1334 | 676124 1335 | 739680 1336 | 842605 1337 | 899279 1338 | 870697 1339 | 808185 1340 | 757097 1341 | 730923 1342 | 700919 1343 | 682964 1344 | 719324 1345 | 706420 1346 | 763265 1347 | 859442 1348 | 894474 1349 | 885938 1350 | 833786 1351 | 765530 1352 | 691382 1353 | 664317 1354 | 644039 1355 | 637996 1356 | 662947 1357 | 702058 1358 | 751191 1359 | 832484 1360 | 905913 1361 | 869708 1362 | 849105 1363 | 842046 1364 | 768272 1365 | 729806 1366 | 693352 1367 | 652442 1368 | 653518 1369 | 633397 1370 | 636038 1371 | 686740 1372 | 729984 1373 | 761610 1374 | 810719 1375 | 848792 1376 | 858743 1377 | 868835 1378 | 875968 1379 | 898893 1380 | 896987 1381 | 859136 1382 | 850662 1383 | 835281 1384 | 826766 1385 | 821156 1386 | 818795 1387 | 809278 1388 | 801447 1389 | 738283 1390 | 686612 1391 | 647129 1392 | 682147 1393 | 669443 1394 | 634327 1395 | 681360 1396 | 707303 1397 | 740493 1398 | 783614 1399 | 824842 1400 | 849515 1401 | 844127 1402 | 860633 1403 | 896906 1404 | 897694 1405 | 901041 1406 | 896407 1407 | 789989 1408 | 702869 1409 | 667964 1410 | 653403 1411 | 673417 1412 | 724309 1413 | 707582 1414 | 758930 1415 | 849941 1416 | 843917 1417 | 743994 1418 | 696496 1419 | 670598 1420 | 666370 1421 | 655446 1422 | 729603 1423 | 713078 1424 | 742632 1425 | 879457 1426 | 877297 1427 | 774352 1428 | 699658 1429 | 679852 1430 | 673784 1431 | 689571 1432 | 689322 1433 | 707705 1434 | 763846 1435 | 893401 1436 | 870140 1437 | 785159 1438 | 736255 1439 | 712163 1440 | 702218 1441 | 710543 1442 | 730519 1443 | 731581 1444 | 798437 1445 | 905835 1446 | 743304 1447 | 674830 1448 | 665585 1449 | 685376 1450 | 682799 1451 | 730628 1452 | 776488 1453 | 884664 1454 | 892850 1455 | 853169 1456 | 881010 1457 | 872343 1458 | 792444 1459 | 700533 1460 | 677284 1461 | 699970 1462 | 692116 1463 | 689521 1464 | 722099 1465 | 765571 1466 | 828120 1467 | 881373 1468 | 893640 1469 | 885634 1470 | 868421 1471 | 837139 1472 | 762195 1473 | 754515 1474 | 751437 1475 | 754514 1476 | 865334 1477 | 898508 1478 | 863646 1479 | 846668 1480 | 886506 1481 | 764061 1482 | 729017 1483 | 742207 1484 | 758023 1485 | 698692 1486 | 744404 1487 | 764318 1488 | 798480 1489 | 905359 1490 | 893866 1491 | 843454 1492 | 765265 1493 | 704168 1494 | 691533 1495 | 705247 1496 | 685833 1497 | 719477 1498 | 736165 1499 | 774304 1500 | 897211 1501 | 862902 1502 | 784723 1503 | 725267 1504 | 702068 1505 | 714028 1506 | 726997 1507 | 745389 1508 | 721924 1509 | 781621 1510 | 895814 1511 | 909586 1512 | 875737 1513 | 799540 1514 | 731029 1515 | 720120 1516 | 684046 1517 | 701350 1518 | 767602 1519 | 786500 1520 | 801928 1521 | 890841 1522 | 898875 1523 | 842034 1524 | 752826 1525 | 712809 1526 | 682146 1527 | 666645 1528 | 661202 1529 | 693626 1530 | 735309 1531 | 782900 1532 | 871283 1533 | 848173 1534 | 865150 1535 | 882039 1536 | 829715 1537 | 739094 1538 | 714669 1539 | 707051 1540 | 722561 1541 | 748094 1542 | 812697 1543 | 811461 1544 | 764000 1545 | 745554 1546 | 840286 1547 | 876956 1548 | 872100 1549 | 879890 1550 | 906241 1551 | 904452 1552 | 893149 1553 | 862551 1554 | 881626 1555 | 896428 1556 | 797294 1557 | 720302 1558 | 702189 1559 | 726906 1560 | 684976 1561 | 685555 1562 | 693189 1563 | 709107 1564 | 777502 1565 | 842897 1566 | 909266 1567 | 900533 1568 | 879032 1569 | 903733 1570 | 895354 1571 | 869770 1572 | 822525 1573 | 759986 1574 | 699751 1575 | 702352 1576 | 685057 1577 | 697998 1578 | 762741 1579 | 733372 1580 | 785137 1581 | 907421 1582 | 829101 1583 | 757576 1584 | 694198 1585 | 753227 1586 | 802424 1587 | 688202 1588 | 680207 1589 | 715488 1590 | 768548 1591 | 904511 1592 | 909389 1593 | 873973 1594 | 714707 1595 | 696291 1596 | 660797 1597 | 736521 1598 | 693248 1599 | 725936 1600 | 807343 1601 | 898406 1602 | 894273 1603 | 901469 1604 | 876101 1605 | 768224 1606 | 496820 1607 | 469104 1608 | 511919 1609 | 523889 1610 | 520047 1611 | 613979 1612 | 809081 1613 | 832330 1614 | 854843 1615 | 818158 1616 | 768414 1617 | 760867 1618 | 736955 1619 | 701697 1620 | 742353 1621 | 680495 1622 | 621834 1623 | 552560 1624 | 565783 1625 | 581668 1626 | 611208 1627 | 590654 1628 | 650168 1629 | 687354 1630 | 690669 1631 | 726118 1632 | 758342 1633 | 755428 1634 | 745804 1635 | 749080 1636 | 741105 1637 | 753552 1638 | 701613 1639 | 696732 1640 | 680911 1641 | 709740 1642 | 733492 1643 | 710044 1644 | 677370 1645 | 588619 1646 | 557076 1647 | 535218 1648 | 528244 1649 | 531576 1650 | 539011 1651 | 590030 1652 | 646655 1653 | 642341 1654 | 683417 1655 | 717388 1656 | 783504 1657 | 799929 1658 | 763358 1659 | 793779 1660 | 833037 1661 | 855981 1662 | 746632 1663 | 739075 1664 | 717275 1665 | 715919 1666 | 752219 1667 | 813960 1668 | 788123 1669 | 680282 1670 | 670955 1671 | 630248 1672 | 568047 1673 | 545664 1674 | 559872 1675 | 633846 1676 | 629498 1677 | 661796 1678 | 716065 1679 | 773203 1680 | 809874 1681 | 848884 1682 | 836247 1683 | 896153 1684 | 898372 1685 | 903563 1686 | 870030 1687 | 820710 1688 | 820550 1689 | 772586 1690 | 747744 -------------------------------------------------------------------------------- /plot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 48, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "%matplotlib inline" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 49, 19 | "metadata": { 20 | "collapsed": true 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "x = np.linspace(-0.4, 1.2, 200)\n", 25 | "y = x**2" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 50, 31 | "metadata": { 32 | "collapsed": true 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "x_ = 0.8 + np.linspace(-1, 1, 100)\n", 37 | "y_ = 0.64 + 1.6 * np.linspace(-1, 1, 100)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 53, 43 | "metadata": {}, 44 | "outputs": [ 45 | { 46 | "data": { 47 | "text/plain": [ 48 | "" 49 | ] 50 | }, 51 | "execution_count": 53, 52 | "metadata": {}, 53 | "output_type": "execute_result" 54 | }, 55 | { 56 | "data": { 57 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAIXCAYAAAC/ygWHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAIABJREFUeJzs3Xd8VfX9x/HXYQiIIE5UrKgoooLi3gu1IO5KQYurtQpu\nBaX2RwtYUdS6sBaVobUqUreiYuOqRRS1DqpREJkOiKMgK8yc3x/fRC6YQMZNzh2v5+ORx72545xv\nJDGffL7f7/tEcRwjSZKUyeolPQBJkqT1sWCRJEkZz4JFkiRlPAsWSZKU8SxYJElSxrNgkSRJGc+C\nRZIkZTwLFkmSlPEsWCRJUsazYJEkSRnPgkWSJGU8CxZJkpTxGiQ9gHwWRZFXnpQk5Z04jqOqvscO\niyRJynh2WDJAHNtoyVZRFPnvJyXAn73M9r//QatWsHQpTJkCbduGx6Ooyo2VH9lhkSRJaXXffaFY\n6dx5dbFSUxYskiQpbVatgmHDwv2LL07fcS1YJElS2owbBzNmwPbbQ9eu6TuuBYskSUqbO+8Mtxdd\nBPXrp++4kYuWklO2rdl/g+zlwj8pGf7sZaZPPoHdd4cmTeDLL2HTTdd8vmzRrduaJUlSYv7yl3B7\n9tk/LVZqyoJFqoGBAwcmPQQpL/mzl3nmzYO//z3cv/TS9B/fKaEEOSUkScoVt94KV10FRx8NL79c\n/mucEpIkSYlZtQruuivcv/zy2jmHSbdZpCYJgcptdukkJWnsWJg5E3bcMb1bmVPZYZEkSTVStpX5\n0kvTu5U5lWtYElTVNSwpc3+1NyhlFb8nJCXtv/+FPfeEpk3hq69g440rfq1rWCRJUiLKtjKfe+66\ni5WassOSIDssqim/JyQl6bvv4Gc/Cxc6nDwZdtll3a+3wyJJkurcyJGhWOnSZf3FSk1ZsEiSpCpb\nuRL++tdw/7LLav98FiySJKnKnnoqXC+obVvo3Ln2z2fBIkmSqix1K3O9OqgmXHSbIBfdqqb8npCU\nhPffh332gebNQ5elWbPKvc9Ft5Ikqc6UdVd+85vKFys1ZcGivHHkkUdSr149/l52OVFJUpV98w08\n8ghEEVxySd2d14JFeSOKIq/HJEk1NHw4LF8OJ5wAbdrU3XktWCRJUqUsXw7DhoX7dbGVOZUFiyRJ\nqpTHH4c5c2C33eDoo+v23BYskiRpveIYbr893L/ssrCGpS5ZsEjAwoULGTRoEB07dqRZs2Y0a9aM\nPffck0GDBrFgwYIK3/f666/TrVs3fvazn9GoUSNatGhB27ZtOfXUUxk+fPhPXr9o0SKuu+469t13\nX5o3b06jRo1o1aoV++23H/369aOwsLA2v0xJqrYJE+A//4HNNoOzz6778zeo+1NKmeXzzz/nmGOO\nYfbs2URRxIYbbgjAxx9/zEcffcTf/vY3XnnlFdqstbps+PDh9O7d+8eFvBtuuCElJSVMmzaNadOm\n8eyzz3LuueeywQYbALBgwQIOOuggPv30U6Iool69emy88cZ88803zJ07l/fff58GDRpwww031O1/\nAEmqhNtuC7cXXghNmtT9+e2wKK+tWLGC0047jdmzZ7Pddtvx0ksvsXDhQhYuXMjLL79M69atmT17\nNqeeeiorVqz48X3FxcVcddVVRFHEeeedx+zZs1m4cCELFizg+++/Z9y4cZxxxhnUS4l/vOOOO/j0\n00/Zcsstef7551m2bBnfffcdS5cu5bPPPuPGG2/8SVEkSZlg2jR4+mlo2BAuuiihQcRx7EdCH0Ac\n/gkqp6qv15qOPPLIuF69evEDDzzw42MPPvhgHEVR3KhRo/iTTz75yXsKCwvjDTbYIK5Xr158//33\n//j4O++8E0dRFDdr1iwuKSmp1Pm7du0a16tXL7755ptr/LWU8XtCUl247LI4hjg+55yaHSfl/1lV\n/p1phyXPRFHmfGSCxx9/nCiKOPnkk9l1111/8vxuu+1Gt27diOOYRx999MfHmzdvDoQOzffff1+p\nc5W9Z86cOWkYuSTVjfnz4b77wv0rr0xuHBYsymvvv/8+AEcddVSFr+nUqdMarwXYeeed2XnnnVm2\nbBkHHnggd9xxB1OmTFnnubp27UocxwwdOpSzzz6bF198kUWLFqXhq5Ck2jNyJCxaBJ06wZ57JjcO\nC5Y8E5p6mfGRCb799lsAWrVqVeFrtt12W4A1Oin16tVj9OjRbLvttsyYMYM+ffqw6667svnmm9O9\ne3fGjh37k+OcddZZ9OrVC4CHH36Yrl270qJFC/bee28GDhzI3Llz0/mlSVKNrVy5+rpBNe6ulJTU\n6O0WLBKwdOnSKr9nn332YerUqTz00EOcc845tGnThnnz5vHEE09w8sknc8IJJ5StVfrR3Xffzccf\nf8yAAQM46qijaNy4MZMmTeK6665j55135pVXXknXlyRJNfbEE/DFF9C2LXTtWsODjRpVo7dbsCiv\nbbHFFgDMnj27wtd8+eWXAGy22WY/ea5Ro0acccYZ3H///UydOpXp06fz+9//niiKGDduHPfcc89P\n3rPrrrsycOBAXnnlFebPn8/YsWPZY489WLx4Meeccw6rVq1K01cnSdUXx6u3Ml95JdSrScVQVAT9\n+tVoPBYsymt77703cRzz2muvVfiaV1999cfXrk/r1q0ZPHgwPXr0AEKw3Lo0aNCArl27/rigd86c\nOUydOrWyw5ekWvPWW/DOO7DppmkIiuvTJ6zerQELFuW1bt26ATBu3DgmTZr0k+cLCwt/3ElUVoQA\na2SylKdJkybEccyyZcsq9Z7GjRv/eD/1PZKUlLLuSu/eUJqnWT0FBTB6dI3T5ixYlNd69OjBHnvs\nQRzHnHzyyWusIXnllVc4/vjjWbFiBe3bt+dXv/rVj8+98MILHHzwwYwcOXKN6aTi4mJGjBjBww8/\nTBRFdOnS5cfnjjnmGC6//HLGjx+/xpqZwsJCzj33XAC22WYbOnToUItfsSSt34wZ8NRTISju4otr\ncKDi4hCNCzBgQI3GZDS/8lrDhg154oknOPbYY5k1axbHHnvsj9H8S5YsIYoitt9+e5588kkaNmy4\nxnsnTpzIxIkTgdBRady4MfPnzyeOY6Io4vjjj+f888//8fULFizgrrvu4i9/+cuPsfzFxcUsXbqU\nKIpo2rQpDz744BrpuJKUhDvvDJt6evaEbbapwYEGD4bp06F9e+jbF37/+2ofyv8zKu+1adOGSZMm\nMWDAADp06EAURURRRIcOHRgwYACTJk36SWT+0UcfzUMPPcS5557LHnvsQdOmTVm0aBGbb745P//5\nz3nwwQd59tln1yg+Ro0axbXXXkunTp1o3br1j4XKrrvuyqWXXsrHH3/MkUceWcdfvSSt6YcfVm/o\nqdFW5sJCuPnmcH/48NCuqYFo7W2XqjtRFIV8/kr+G5RdZM9/M5Xxe0JSut12W2iGHHkkrGM/wrqV\nlMDhh4dLPPfuDXffDazx/6wq551bsCTIgkU15feEpHRauRJ22glmzYJnnoGTTqrmgUaMgAsugJYt\nYfJkaNECqFnB4pSQJEkCwkLbWbNC0XLCCdU8SGrmytChPxYrNWXBIkmSALj99nBbo6C4ssyVLl2g\ne/e0jc0poQQ5JaSa8ntCUrpMnAgHHQSbbBLi+Js2rcZBCgqgc+eQuVJYCDvssMbTTglJkqQaueWW\ncHvBBdUsVtbOXFmrWKkpOywJssOimvJ7QlI6fP55uMBhgwYwc2Y1s1f694cbbgiZK++/X+42Zjss\nkiSp2m67LVzs8Mwzq1mspDlzpTx2WBJkh0U15feEpJr69lvYbjtYujTUHbvtVsUDVJC5Uh47LJIk\nqVqGDQvFyvHHV6NYgRCLO2FCyFwZMiTt4ytjhyVBdlhUU35PSKqJJUugdWv47jv417/giCOqeICi\nImjXLmxjHjMGUq5qXx47LJIkqcoeeCAUK/vuG2Z1qqyWMlfKY4clQXZYVFN+T0iqrlWrYJddYNo0\n+Mc/qlFvrCdzpTw16bA0qOoblLyyf3BJkqrr6adDsbLDDvCLX1TxzbWcuVIep4QkScozcQx//nO4\n36dPyF+pksGDYfr0kLnSt2/ax1cep4QSVNUpIUmS0uGNN+Cww2DTTWH27Com2xYWQseO4dLOb74Z\n8vwryUW3kiSp0sq6KxdfXMVipaQEevUKxUrv3lUqVmrKDkuC7LBIkura5Mmw667QqFHormy5ZRXe\nPGJEuNhQy5bhQC1aVOncdlgkSVKl3HpruD3nnCoWK0VF0K9fuD90aJWLlZqyw5IgOyySpLo0d24I\niluxIjRI2ratwpt79oTRo0PmygsvQDV2rNphkSRJ63XXXbB8OZx8chWLlYKCUKw0aRKy/BOI17Bg\nkSQpDyxaFGoNgKuvrsIbUzNXBg6sk8yV8liwSJKUB+67D+bNg4MPDh+Vlpq50qdPrY1vfVzDkiDX\nsEiS6sLKlbDzzjBzJjz1FJxySiXfWIPMlfK4hkWSJFXo8cdDsbLzznDiiZV8U4KZK+WxYJEkKYfF\nMdx4Y7h/1VVQv34l3zhqFEyYEDJXhgyptfFVllNCCXJKSJJU28aNg65dYeutYcaMEBi3XkVF0K4d\nzJ8PY8ZAjx5pGYtTQpIkqVxl3ZUrr6xksQJhce38+SFzpXv3WhtbVdhhSZAdFklSbXrzTTjkkBBK\nO2sWNG9eiTcVFEDnziFzpbAwrduY7bBIkqSfKOuuXHxxJYuVDMlcKY8dlgTZYZEk1ZaPP4YOHaBx\n49BdqdR1g/r3hxtuCJkr778PDRumdUx2WCRJ0hpuvjnc/va3lSxWCgtXv2n48LQXKzVlhyVBdlgk\nSbVh5kzYaadw//PPYfvt1/OGkhI4/PCwjblXL7jnnloZlx0WSZL0o1tvhVWr4Fe/qkSxAmtmrpQt\nfMkwdlgSZIdFkpRu33wDrVvD0qXw0UdhOco61VLmSnnssEiSJADuvDMUKyeeWIliBTIyc6U8dlgS\nZIdFkpROCxaE7sr8+ZW8VmEtZq6Uxw6LJEni3ntDsXL44ZUoVjI4c6U8FiySJOWApUvhttvC/Wuu\nqcQbBg+G6dPDvFGfPrU6tnSwYJEkKQc8+CDMnQt77hmWo6xThmeulMeCRZKkLLdq1er645prIFrX\nCpGSkpC1snIl9O5dibmjzGDBIklSlnviiRAQt+OO0K3bel6cmrkyZEidjC8dLFgkScpicbw6661f\nP2jQYB0vLioKLwK4445wGecsYcEiSVIWKyiADz4IDZNzzlnPi1MzV2oxIK42WLBIkpSl4hiuuy7c\n79MnXJm5QgUFMHp0yFwZNmw9C10yjwWLJElZ6t//DstRNt10daRKubIsc6U8FiySJGWpsu7KFVdA\ns2breGGWZa6Ux2j+BBnNL0mqrrfegoMPhubNYdasdayfLSyEjh3DNuZK5fXXHqP5JUnKM9dfH24v\nuWQdxUqWZq6Uxw5LguywSJKq44MPYO+9YcMNYeZM2GKLCl44YgRccEHYQjR5cuLbmO2wSJKUR8q6\nK717r6NYyeLMlfLYYUmQHRZJUlUVFoa1s40awYwZsPXWFbywZ8+wjblLF3jhhYzYxmyHRZKkPHHD\nDeH2t79dR7GS5Zkr5bHDkiA7LJKkqpg6Fdq1g3r1YNo02G67cl5UXBxaMNOnh8z+3/2uzsdZETss\nkiTlgRtvDBt/zjmngmIFciJzpTx2WBJkh0WSVFmzZsFOO4WCZcqUcP8nMihzpTx2WCRJynE33xzq\nkDPOqKBYyaHMlfLYYUmQHRZJUmV8/TXsuCMsXw4ffwy77VbOizIsc6U8dlgkScpht9wCy5bBaadV\nUKykZq4MHZqRxUpN2WFJkB0WSdL6fPsttG4dNv988EFYovITZZkrnTvDuHEZu43ZDoskSTnq9ttD\nsXLCCRUUK6mZK3ffnbHFSk3ZYUmQHRZJ0rrMmxe6KwsXhqszH3jgWi/I4MyV8thhkSQpB91xRyhW\njjmmnGIFcjZzpTx2WBJkh0WSVJF582D77WHBAhg/Hg49dK0XZHjmSnnssEiSlGPuuCMUK8ccU06x\nkuOZK+Wxw5IgOyySpPKst7uSBZkr5bHDIklSDllndyUPMlfKY4clQXZYJElrW2935cwz4eGHoUsX\neOGFrNrGbIdFkqQcsc7uyksvhWKlSRP461+zqlipKTssCbLDIklKtc7uSnExdOgA06ZlReZKeeyw\nSJKUA26/fR3dleuvD8VKHmSulMcOS4LssEiSyvzvf7DDDhV0VwoLYa+9YMWKrMlcKY8dFkmSslyF\na1dKSkLWyooVeZO5Uh47LAmywyJJgvV0V0aOhPPPz7rMlfLYYZEkKYtV2F0pKoKrrw738yhzpTx2\nWBJkh0WSlNpdeeMNOOSQlCezOHOlPHZYJEnKUmXdlWOPXatYSc1cGTYs64uVmrLDkiA7LJKU3yrs\nrqRmrgwZAtdck+g408UOiyRJWajC7kpq5krfvomNL5PYYUmQHRZJyl8VdldyJHOlPHZYJEnKMrfc\nsnpn0I/FipkrFbLDkiA7LJKUn775JnRXliyBiRPhgANKn8ihzJXy2GGRJCmLDBkSipUTT0wpVsxc\nWSc7LAmywyJJ+efLL2GnnWDZMvjgA+jYsfSJHMtcKY8dFkmSssT114dipXv3lGLFzJX1ssOSIDss\nkpRfpk+HXXYJa2s//hh23ZU1M1duvBF+97ukh1lr7LBIkpQF/vQnWLkyzP7sumvpg6mZK336JDq+\nTGaHJUF2WCQpf3z6aahJ6tWDKVNgxx3J6cyV8thhkSQpww0aFKaCzjuvtFgxc6VK7LAkyA6LJOWH\nDz8MjZRGjeDzz2HbbYERI+CCC3I2c6U8dlgkScpgAwaE2wsvLC1WioqgX7/woJkrlWKHJUF2WCQp\n9739Nhx4IGy4IcyYAVtuCfTsCaNH53TmSnnssEiSlKH+8Idwe/nlpcVKQUEoVsxcqRI7LAmywyJJ\nue1f/4KjjoKNNw7dlU0aF4etQtOn53zmSnlq0mFpkPbRSJIk4nh1d6VvX9hkE6D/4FCsmLlSZXZY\nEmSHRZJy14svwnHHwWabhRql+ReFIYt/5UqYMAEOPjjpIdY517BIkpRBUrsr11wDzTcqgV69QrHS\nu3deFis1ZYclQXZYJCk3PfEEdOsGW20VUvc3fDj/MlfKY4dFkqQMsXIl9O8f7v/xj7DhQjNX0sFF\nt5IkpdH994drBbVpA+efD5zbB+bPD5kr3bsnPbys5ZRQgpwSkqTcUlwMO+0EX38NjzwCp29aAJ07\nh8yVwkLYYYekh5gotzVLkpQB/vKXUKzstRd0P7EY9rgwPDFwYN4XKzVlhyVBdlgkKXfMmxeuwjx/\nPvzzn/Dz1/vDDTeEzJX334eGDZMeYuJcdCtJUsJuuikUK506wbHbFMLNN4cnhg+3WEkDOywJssMi\nSbnhq6/C2pWlS+GdiSXs1/fwEA7Xqxfcc0/Sw8sYdlgkSUrQtdeGYqVbN9jvv6NCsdKyZbhekNLC\nDkuC7LBIUvabMgV23730/r+LaHN8uzA3NGYM9OiR7OAyjB0WSZIS0r8/rFoF550Hbf5q5kptscOS\nIDsskpTd3nkHDjggxKx8MaqAzX5l5sq62GGRJKmOxXG4sCFA34uK2ewPZq7UJgsWSZKqoaAAXnsN\nNtkE+seDYfr0kLnSp0/SQ8tJFiySJFVRScnq7sotvy6k8Z1mrtQ2CxZJkqro0Ufhww9h221KOOet\nXuESzb17w0EHJT20nOWi2wS56FaSss/y5bDbbjBtGvz7rBEc9uAFIXNl8mRo0SLp4WU0F91KklRH\n7r03FCsHtyni0LH9woNDh1qs1DI7LAmywyJJ2WX+/BDB//338MXhPdn236ND5soLL0BU5aZB3rHD\nIklSHRgyJBQrfTsUhGKlSRMYNsxipQ7YYUmQHRZJyh6zZsEuu0C0rJh5rdrT+Kvp4VpBv/td0kPL\nGnZYJEmqZf37w7Jl8Mjug0OxYuZKnbLDkiA7LJKUHf7zH9hvP+jYsJD3445EK1fCm2+6jbmK7LBI\nklRL4hiuugoiSnhyy16hWDFzpc5ZsEiStA5jx8Lrr8PlTUexw1cTQubKkCFJDyvvOCWUIKeEJCmz\nrVgBHTrAvClFzGrSjsbF8+GRR+D005MeWlaqyZRQg7SPRpKkHDFyJEyZAs8060PjhfND5kqPHkkP\nKy/ZYUmQHRZJylwLFoSQuI7fFlBA55C5UlgIO+yQ9NCylotuJUlKs5tugoXfFnN/4wvDAwMHWqwk\nyA5LguywSFJm+uILaNsW/rC0P/25IWSuvP8+NGyY9NCyWk06LBYsCbJgkaTMdO658O4DhUyKOtIg\nNnMlXVx0K0lSmnzwATz4QAn/jnqFYsXMlYxghyVBdlgkKbPEMRxzDOz46ghGcEHIXJk8GVq0SHpo\nOcEOiyRJafD88/Dxq0U8EfWDGBg61GIlQ7hLSJIkYPnycC3D2+hDi7g0c6V796SHpVIWLJIkAX/5\nC2w/tYCejCZu0gSGDYOoyjMXqiVOCUmS8t4338DN1xbzJiFzJTJzJePYYZEk5b0//hEuWziYNkwP\nmSt9+iQ9JK3FXUIJcpeQJCVv0iQ4c69C3o870hAzV2qT0fySJFVDHMOVl5dwd9wrFCtmrmQsCxZJ\nUt566ilo8/ooDmUCJVu2hCFDkh6SKuCUUIKcEpKk5CxdCofvUsQ/Z7djE+bDmDHQo0fSw8ppTglJ\nklRFt98Ol8/uwybMp6SzmSuZzg5LguywSFIy5syBXjsU8Oyyzqxq1IT6n3wMO+6Y9LBynh0WSZKq\nYGC/Ym5fFjJX6l870GIlC1iwSJLyyn/+A60fCpkry9qauZItnBJKkFNCklS34hjO3qeQ+z4wcyUJ\nTglJklQJ/3ikhF4fhMyVZb8xcyWbWLBIkvLCkiXwwSUhc2VJ85Y0utXMlWxiwSJJygt/+UMR18zr\nB0Dje4ZCixYJj0hVYcEiScp506ZB66Ehc2XeAV2od7qZK9nGgkWSlPMe6FnA6SWjWVa/CZs8Mgyi\nKq/5VMIsWCRJOW3ck8Wc83bIXFnWbwDssEPCI1J1WLBIknLW0qUw47yQufLdVu1pfm3fpIekarJg\nkSTlrL//rpDz598MQItHh0PDhgmPSNVlwSJJykmzZ5bQ/q6QufLlCb1pcJiZK9nMgkWSlJPGdRvF\nwSUTmN+oJds+aOZKtrNgkSTlnPGPF9H9vZC5svJWM1dygQWLJCmnrFgBP5wXMlem7dyFzS8ycyUX\nWLBIknLKs5cUcMKC0RRHTdj2WTNXcoUFiyQpZ3w9rZi9R4TMlVnnDKRROzNXcoUFiyQpZ0w8cTA7\nxNOZ2aw97Yb3SXo4SiMLFklSTpgwvJATPw2ZK43+ZuZKrrFgkSRlvaVLSmh0echc+eCA3mz9CzNX\nco0FiyQp6710xn3su3QC39Vvye7PmrmSiyxYJElZbebbRRz67NUAfNN/KBtsaeZKLsrqgiWKolZR\nFN0XRdFXURQtjaJoRhRFt0dRVOXv1iiKjo6i6KkoiuaUHuurKIpejKKoSyXeOzKKopLSjx2r99VI\nkqoqjmH6qX3ZhPlM2qYLuw0ycyVXNUh6ANVVWhi8BWwOPA1MAfYHLgc6R1F0SBzH8yp5rJuBq4Av\ngGeA74AtgH2AI4EX1/HeE4HfAAuBjar55UiSquGNgS/Rac7DLKEJ2zxl5kouy9qCBbibUKxcGsfx\nsLIHoyi6FbgSuB64aH0HiaLofEKxcj/QK47jlWs9X38d790cGA6MAbYGDq/6lyFJWp84jnnnnXe4\n5ZZhvPDCWIqLf6BFo+a8u2wVAJNOHsBB+5u5ksuyckqotLtyLDAztVgpNRBYDJwVRVGT9RxnA2Aw\nMItyihWAOI5XreMQI4AYuLgKw5ckVcGKFSv41a/Oo1On03nyyfYsWfIxcbyMvkvPok28kI9oyF0b\nTGbFihVJD1W1qE4KliiK2kVRdGUURb2iKNo4DYc8qvS2YO0n4jheBEwANgQOXM9xjiVM/TwBxFEU\nHR9FUb8oii6Lomid742i6FzgJOCCyk49SZKqJo5jzj67F88++zVLlnxMScnVwDbsxhT6cTcAF/Ai\nTz03h7PP7kUcx8kOWLUmrQVLFEUDShetbpry2DHAB8AtwDDg/SiKNqvhqXYhdDY+q+D5qaW3bddz\nnP1Kj7O8dIxjgSHA7cCbURT9q3TaZw1RFLUG7gAejOP4uaoPX5JUGe+88w5jx77GkiVPAE0BiCjh\nXkLmyt30ZiKdKC5+krFjX+Pdd99NdsCqNenusBwHTI7j+H8pjw0hFAUDCetOdiAsjK2Jsi7NDxU8\nX/b4+nYLbQlEwNVACXAI0AzYA/gnYU3Ko6lviKIoAh4gLLKt6dchSVqHW2+9m+LiiygrVgDOYxSH\nMoG5tOT3lGWuNKW4+EJuvfXuRMap2pfugmV74NOyT6IoakXYaTMsjuPBcRxfArwKnJLm81ZX2de/\nAjgxjuO34jheEsdxIfAL4EvgiCiKDkh5Tx/gMOC3cRxXVDBJktLg+eefpaSk54+fb0kRN9MPgCu4\ngx9S/i4tKenJ888/W+djVN1Id8GyCZDaXTmE0F1JnTZ5D9iuhucpKxQqWg9T9vj89Ryn7PkP4jj+\nIvWJOI6LCV0WCNuliaJoZ8Ii3fvjOP4naRJFUYUfgwYNStdpJCnrFBf/QGiGB7fRh02Yzzi68A96\nrPXqLUtfryQNGjSowt9pNZHuguVboFXK50cRuhdvpzy2QRrOO4UwlVPRGpWdS28rWuOSehyouLAp\nW0xbtttoN6AR8JuUoLiSKIpKgCNKX/N56WMnrefcP4rjuMIPCxZJ+axJk42BbwA4lgJ6MpolNOEi\nhhF+DaT6pvT1StKgQYMq/J1WE+nOYfkQOCmKovbAUqAH8EZpt6LM9sCcGp7ntdLbn6/9RBRFGxE6\nO0uAies5ziuEDtBuFTzfvvR2RuntTGBkBa89AWhJWPOyoPS1kqQaOP74k3jiiYfZoOQS7uZCAK5l\nIDP5aeZKvXoPc/zxlf5bUVkmSucWsCiKDiMUE6ll7wlxHI8rfb4+oVh5KY7jnuUcoirnepGwLfny\nOI7vSnnV8cwXAAAgAElEQVT8NuAK4O44ji8ufawB0AZYEcfx9LWO8zRwItA3juM7Uh7/OTCO0GXZ\nIY7jhesZz2uERbo7r32OdbwnBtyGJ0kVePvttzn66NP5/eJu9OcWPqI9e/M+K2m41isXseGG7Xnt\ntUfZf//9Exmr1q9sWiiO4yrPD6W1wxLH8fgoik4Azid0Lh4uK1ZKHQx8BTyVhtNdRMhbGRpF0dGE\nxb4HEqL0JwN/SHltq9LnZwJrX+vnYqAjcGsURccTtjfvCJwMrCQsrl1nsSJJqh37778/v9qzI/3e\nvBWACxhebrHSpMlpnHRSJ/bbb7+6H6TqRNqj+eM4fpEKrr0Tx/F4YK80nWd6FEX7An8CuhC2VM8h\nZKj8qZwdPHHpx9rH+SqKon2AAYQguMMIUzrPADfGcfyfqgyryl+IJKlCq1bEXPjhNzQk5t6oGe9E\n46FkO8JC3G+oV+9hGjcexkkndeLvf7+3xgs7lbnSOiW0zhNF0SbA8jiOF9fJCbOAU0KStG4FvxzB\nzx+/gG/qteSzZx7mzr8/+OO1hJo02Zjjjz+Jq666yM5KlqjJlFC617AcDXQGhpTF1UdRtCXwGHAo\nYYrlr3Ec90nbSbOYBYskVWz2u0U0278dmzCfD/o9wl43nZ70kFRDmVSwPA20j+N4p5TH/g6cCXwO\nbETYSXNGHMePln+U/GHBIknli2N4bZuedJo7mg+26sJeX78ATvdkvZoULOnOYdkTeKPsk9KrJXcj\n7ApqS7gG0BdA7zSfV5KUQ17uV0CnuSFzZdtnhlmsKO0Fy5bA1ymfHwA0Bv4GULrb5jlC4SJJ0k/M\nnVHMTreFzJVPuw9ki/1/mrmi/JPugmUZq1NhIey4iYF/pzy2ANgUSZLKMeG4wexQMp0ZG7Vn7wdd\n8qgg3QXLDKBTyuenAVPjOP4q5bGfAd+l+bySpBxQcEchJ075MwCNHxhOtMHamSvKV+kuWB4AOkRR\n9HYUReOBDsDotV6zB6uv4SNJEgDzvi+heb/ebMAKPj60N1v/4qCkh6QMku6C5W5gDLAv4Xo+zwE3\nlT1Zeo2hDsC/0nxeSVKWe/rk+zhwxRt837Aluz0zJOnhKMPUSnBcFEXNgXjtSPsoijYnxOTPLCeJ\nNu+4rVmSgtcfLWKPHiFz5evbxrDNlT2SHpJqQcbksKhqLFgkCRYtgpe36skpi0czrW0X2kw2cyVX\nZczFD8tEUbQh8AvCdYNaAD8A7wNPGc0vSUr19zMLuGjxaIqjJmw31swVlS/tBUsURV0Ji283BVK/\n62Lg9iiKfh3H8XPpPq8kKfu8+UoxnZ8JmSvzLh3INm3NXFH50h3NvzfwJlAfeAR4lXAF5a0J253P\nAFYBh8Rx/F7aTpylnBKSlM8WL4YHftafi+bdwNwt2rPVV+9DQ7cx57KMWcMSRdETQFfgqDiOJ5bz\n/AGEHUIvxHF8WtpOnKUsWCTlsyFnFnLVwx1pyEpWvP4mDQ93G3Ouy6RrCR0GPFZesQIQx/HbwOOl\nr5Mk5anXXyvhsId70ZCVfHtab4sVrVe6C5aNCRc3XJfZQPM0n1eSlCUWLYJ/dh/FoUxgUdOWbDHS\nzBWtX7oLlq+B/dfzmn0J61okSXno+suKuPq7fgA0vucOaNEi4REpG6S7YHkB6BRF0TVRFNVPfSKK\nonpRFPUFjil9nSQpz7z6KrS/vw+bMJ8Fh3ShQU8D4lQ56V50uxXwHrAVYepnPKGbshVwKLA9MBfY\nN47jvO+yuOhWUj5ZuBAu3rmAvxd1ZkXDJjScUgg7uI05n2RMcFwcx3OjKDoEuBc4Fmi91kteAnpb\nrEhS/unfp5iBRSFzpd6ggRYrqpJai+aPoqgVIel2Y0LS7QdxHH9VKyfLUnZYJOWLl1+Gt4/tT39u\nYOlO7Wn8iZkr+ShjclhUNRYskvLB/PnQbddCxs0NmSu8+SYc5DbmfJTYlFAURfdV861xHMfn1eTc\nkqTscNklJQyaGzJXVl3Qm/oWK6qGGnVYoigqqeZb4ziO66//ZbnNDoukXPf44/DPX45gBBewcvOW\nNJg62W3MeSyxKaEoitZeVFtpcRzPqvaJc4QFi6RcNmcOHLVbEW/Nb8cmzIcxY6CH25jzWWJTQhYd\nkqTyxDGcdx78cX7IXIm7dCHq3j3pYSmLpTs4TpIk7r0XVo4roCejKWnchGjYMIiq/Ee19KO05rBI\nkjR1ashceQczV5Q+dlgkSWmzciWcdRb0KR5MG6ZD+/bQp0/Sw1IOsGCRJKXNkCGw8O1C+nFzeGD4\ncAPilBZOCUmS0uK99+C6a0t4lZC5Qu/eBsQpbUy6TZDbmiXlisWLYe+94fDPQuYKLVvCZDNXtKaa\nbGt2SkiSVGNXXAHzPyvi1nr9wgNDh1qsKK2cEpIk1cgTT8DIkfBIvT40L5kPXbqAmStKM6eEEuSU\nkKRs98UXsOeesO+8AgroDE2awMcfw447Jj00ZSCnhCRJdW7VqrCFuXheMQ9sGDJXGDjQYkW1wikh\nSVK13HwzvP463N50MFsvNnNFtcspoQQ5JSQpW73zDhxyCLRdWchH9TtSb9VKePNNtzFrnZwSkiTV\nmYUL4Ve/glUrSxi7Ta9QrJi5olpmwSJJqpJLL4Vp02DQtqPY8esJIXNlyJCkh6Uc55RQgpwSkpRt\nRo+Gnj1hu0ZFTG/UjvoL5sOYMdCjR9JDUxZwSkiSVOumToVevcL9l/foE4oVM1dURyxYJEnrtXRp\nqEsWLYLBhxew87ujQ+bKsGEQVfmPZanKLFgkSet19dXw4Yew6/bFXDO7NHNlwADYYYdkB6a8YcEi\nSVqnJ5+Eu+6Chg3hlaMGU39maeZK375JD015xEW3CXLRraRMN2MG7LUX/PADPHhNIWfe0hFWmrmi\n6nHRrSQp7ZYvh9NPD8XKKSeV0HN8r1CsmLmiBFiwSJLK9X//FxJtt9sOHjpqFNEEM1eUHKeEEuSU\nkKRM9dxzcOKJUL8+THymiH3PbAfzzVxRzTglJElKm9mz4Zxzwv0bboB9R/cJxYqZK0qQHZYE2WGR\nlGmWLYPDDw9TQccdB89dVkC94zqHzJXCQrcxq0Zq0mFpkPbRSJKyVp8+oVhp3RoeGlFMvcNLM1cG\nDbJYUaKcEpIkAeE6QcOGwQYbwOOPw6bDBsP06dChA1x5ZdLDU55zSihBTglJyhSFhbD//rBkCdx9\nN/Q+rBA6doRVq2DCBLcxKy2cEpIkVdvChXDaaaFYOfNM6HV+CRxRmrly4YUWK8oIdlgSZIdFUtLi\nOITDPfoo7L47vP02NB09Ai64ALbaCj79FFq0SHqYyhFua5YkVcudd4ZipVkzeOIJaLqoCPr1C08O\nHWqxoozhlJAk5am33oKrrgr377sPdtkF6FmauXLccfDLXyY6PimVU0IJckpIUlLmzIF99gm3V14J\nt90GFBRAZzNXVHucEpIkVdry5aF5MmcOHHYY3HQTUFwcFtiCmSvKSBYskpRnLr887FTedlt47DFo\n2BAYbOaKMptTQglySkhSXRs5Es4/Hxo1gvHjYb/9CNM/HTuGbcxvvuk2ZtUap4QkSes1cSJcfHG4\nf889pcVKSQn0Ks1c6d3bYkUZyw5LguywSKorqYtsL7kE/vKX0idGlGautGwJkye7jVm1qiYdFguW\nBFmwSKoLy5dDp05h3cphh8Err5SuWykqgnbtwjbmMWOgR4+kh6oc55SQJKlC5S6yhXBp5vnzoUsX\n6N490TFK62OHJUF2WCTVtuHDwxKVNRbZgpkrSoQdFknST/zrX+UssoU1M1cGDrRYUVawYJGkHDRt\nWrgC88qV0LcvnHtuypNlmSvt24dpISkLOCWUIKeEJNWGH34Iu5M//RSOPx6eeQbq1y99MjVzZcIE\nOPjgRMeq/OKUkCQJgFWr4IwzQrGy++4wenRKsZKauXLBBRYryioWLJKUQ66+GsaNg802g2efhebN\nU54cNSp0VVq2hBtvTGyMUnU4JZQgp4QkpdOoUfDb34Ztyy+/DIcfnvJkaubKI4/A6acnNk7lL6eE\nJCnP/fvfqzf+3H33WsUKrJm5YkCcspAdlgTZYZGUDtOmwQEHwPffhwst33bbWi8wc0UZwg6LJOWp\n77+Hrl3D7XHHwZ//vNYLzFxRjrBgkaQstXQpnHIKfPYZ7Lkn/OMfKTuCypi5ohzhlFCCnBKSVF0l\nJdCzZ7hmYatWMHFiuFbQGlIzV958M4SzSAlySkiS8swf/xiKlY02guefL6dYSc1c6d3bYkVZzw5L\nguywSKqOkSPh/PPD9M9zz4WNPz8xYkQIh2vZEiZPhhYt6nyc0tpq0mGxYEmQBYukqiooCItsV62C\ne+8NNclPpGaujBnjNmZlDKeEJCkP/Pe/0K1bKFauuaaCYgXWzFzp3r1OxyjVFjssCbLDIqmyZs8O\nl/756qvQMBk9GuqV9yenmSvKYHZYJCmHff99aJZ89RUceij87W8VFCtmriiHWbBIUgZbsgROOCFc\nfbl9+3BBw8aNK3ixmSvKYU4JJcgpIUnrsmIFnHpq2La83XYhSqVVqwpeXFgIe+0V3mTmijKUU0KS\nlGPiOCyqff552HRT+Oc/11GslJSErJUVK8xcUc6yYJGkDPR//xfWqmy4IbzwQtilXKH77oM33giZ\nK0OG1NUQpTrllFCCnBKSVJ6hQ+GKK0Iw3Nix4aKGFUrNXHnkETj99Dobp1RVTglJUo545JFQrEBo\nnKyzWAHo23d15ooBccphdlgSZIdFUqqxY+EXvwiX/7npJujXbz1veOkl+PnPzVxR1rDDIklZ7tVX\n4Ze/DMXK734HV1+9njeYuaI8Y8EiSQmbOBFOOgmWLQs1yJAhEK3v78/rr4dp08xcUd5wSihBTglJ\nmjQJjjwyLEM566x1pNimMnNFWcopIUnKQlOmwLHHhmLl1FPDItv1FislJdCrl5kryjt2WBJkh0XK\nXzNnwmGHwZdfhnWzzz4LjRpV4o0jRoREuZYtYfJkaNGitocqpU1NOiwWLAmyYJHy09dfw+GHhyUo\nhx4KL74ITZtW4o2pmStjxriNWVnHKSFJyhJz5kCnTqFY2XtveO65ShYrEBbXzp8PnTtD9+61Ok4p\n09hhSZAdFim/FBWFBbaTJ8Mee8Arr8Dmm1fyzQUFoVAxc0VZzA6LJGW4oqLQWZk8GTp0qGKxYuaK\nZMEiSbXtm2/g6KPhk09CbEqVihWAwYNh+nQzV5TXnBJKkFNCUu779ttQrHz0Eey2G7z2Gmy5ZRUO\nUFgIHTuGCFwzV5TlnBKSpAz03XdwzDGhWNl11xC/X6VipSxzZeVKM1eU9+ywJMgOi5S7vv02hMJN\nmhR2Ir/2Gmy1VRUPYuaKcow5LFnKgkXKTXPmhM7KJ59A27bwr3/B1ltX8SBmrigHOSUkSRli9uwQ\nCvfJJ7D77vD669UoVmB15kqXLmauSNhhSZQdFim3TJsWFtjOmhWuTVhQUMXdQGVeeink9TdpAh9/\nDDvumPaxSkmwwyJJCZs8OXRWZs2CAw8MC2yrVaysnblisSIBFiySVGOTJoVi5euv4YgjQmel2utj\nr78+tGrMXJHW4JRQgpwSkrLfu++GxPx588IszlNPwYYbVvNghYVhLmnFCjNXlJOcEpKkBLzySojb\nnzcPTj4Znn22BsVKSUnIWlmxwswVqRwWLJJUDY89Bl27wqJFcMYZ4fNGjWpwwPvugzfeCJkrQ4ak\nbZxSrrBgkaQquvvuEIuyfDlcdhk89BA0bFiDAxYVwdVXh/tDhxoQJ5XDgkWSKimOYdAguOiicP/6\n6+GOO6BeTf9P2revmSvSerjoNkEuupWyx6pVoZsybFgoUO69F3772zQcODVzpbAQdtghDQeVMlNN\nFt02SPtoJCnHLFsGZ521ep3KI4/Aqaem4cCpmSsDBlisSOtgwSJJ6zBvXihOXn8dmjcPO4GOOCJN\nBx88eHXmSt++aTqolJucEkqQU0JSZps+PewEmjIlXA/ohRegY8c0HbywMBxs5UozV5Q3zGGRpDR7\n++0QsT9lCnToED5PW7FSUgK9eoVixcwVqVIsWCRpLU88AUceCd9+G9bDvvEG/OxnaTzBqFEwYYKZ\nK1IVWLBIUqk4hltvhV/+EpYuDbuAnnsurF1Jm6Ii6Ncv3DdzRao0CxZJIszOXHIJXHVVKFyGDIHh\nw2sYCFeePn3MXJGqwUW3CXLRrZQZ/ve/kFz78sth2/IDD4TP066gIFwp0cwV5SlzWCSpmgoLw4UL\np02DLbeEJ5+EQw6phROlZq4MHGixIlWRU0KS8tazz4adQNOmwd57w7vv1lKxAiFzZfr0kLnSp08t\nnUTKXRYskvJO2XWATjklXG359NNh/HjYbrtaOmFhIdx8c7hfKwtjpNxnwZIBFi9OegRS/li8OKxP\n+cMfwudDhsDo0bDhhrV0QjNXpLSwYMkABx4In32W9Cik3DdtWpjyeeyxsFV57Fi45hqIqrz8rwrM\nXJHSwoIlA3z8Mey3Hzz1VNIjkXLX00/DPvvApEmw884wcSIcf3wtn9TMFSltLFgyQLdusGAB/OIX\n8Lvfhc6xpPRYuRKuvjpcwPCHH8LP2bvvwq671sHJzVyR0sYclgSV5bCUlMTcfnv4Q2zVqhAJPmZM\n6CBLqr45c8J6lfHjoX79sO71yitreQqojJkr0k/UJIfFgiVBawfHjR8f/gibOzdcGfYf/4DDDkt0\niFLW+te/wu6foiLYZpvw83TooXV08uLisH15+nS48cbQOpXk1ZpzxWGHwfvvh9s5c0Kn5dprQ9dF\nUuWsWhW2LB99dChWOnUKP1d1VqyAmStSLbDDkqCKovlXrAhBmDfeGPIiDj8cHnoozVeLlXLQl1/C\nWWeF7gpA//6h6K9fvw4HUVgIHTuGxTMTJsDBB9fhyaXMZoclxzRsCDfcAC+9FKaG/v1v2HNPdxFJ\n6/Lkk7DHHqFYadkSXnwxNDrqtFhZO3PFYkVKGwuWDHb00WELZteuMG9e2N1w0UVhelxSsHhxqBFO\nOy38nHTtCv/9b1jvWufMXJFqjVNCCars1ZrjGO68M+wiWr48TIs/9FDoukj57MMP4YwzYPLkcJXl\nP/8ZLrmkjnYBra2oCNq1C9uYx4yppcs9S9nNKaEcF0Vw+eXw1lvQtu3qoLkbbjCzRflp1apQnBxw\nQChWdtsN3nkHLr00oWIFzFyRapkdlgRVtsOSatGisENy2LDw+f77wwMPhD/spHwwZQqce25IqoWw\nVOTWW2vxWkCVYeaKVCl2WPLIRhvBX/8a/v+47bbhr8q99oLbbw/r/aRctWoV3HZb2IAzcSK0agXj\nxsHddydcrBQXw4UXhvsDB1qsSLXEgiVLHXssfPRR+Etz6dLQjT7qqBD9IOWaqVPhiCOgb9/w/X7u\nuWFqtEuXpEeGmStSHXFKKEHVmRIqz7PPwgUXhDV/G24I110Hl10GDRqkZZhSYlatgrvugt//PjQy\ntt4aRoyog4sWVlZq5sqbb8JBByU9IimjOSWU5046Kfy12aMHLFkS/go94ICQ7illqw8+CL//r7gi\nFCtnnRXqg4wpVlIzV3r1sliRapkFS47YfPOwk3Ls2JCI+/77YSdR375hoa6ULRYtCjMr++4brqrc\nqhU8/TT8/e+wySZJjy5FaubKjTcmPRop5zkllKB0TQmtbdEiGDAAhg4NfwS2bh12FXXtmtbTSGn3\nzDNha/IXX0C9euH+dddBs2ZJj2wtZq5I1eKUkNaw0UZhN0XZDqJZs0Ib/Ze/DPelTPPFF3DqqXDK\nKeH+PvuE79877sjAYgXMXJESYIclQbXVYUm1cmXotAwYENa3NG4M11wDV1+d8FZQibA25ZZbwozK\nkiWh2L7+erj44jq+BlBVmLkiVVtNOiwWLAmqi4KlzBdfhGj/MWPC59ttF35RdOuWYDKo8lYcw6OP\nhu/J2bPDY6edFjoq226b7NjWqbg4bF+ePj1UWb/7XdIjkrKKU0Jar5/9DB55BF5/PVyDaPbs0Mnu\n1ClcKE6qK++9B4cfDqefHr4P99gDXnsNHn88w4sVMHNFSpAdlgTVZYcl1apVMHIk9O8P338fFjf+\n5jchpDPjf2Eoa82ZA3/4A9x/f+iwbLFF+P1/3nkZPP2TyswVqcbssKhK6tcPsRFTp66+WNzIkbDz\nzqFF/7//JT1C5ZJ580LwW5s2cN994fuvb9/w/XfBBVlSrKRmrvTubbEiJcAOS4KS6rCsbcoU+OMf\n4bHHwuctWoSp+csuc2Guqm/x4rDg++ab4YcfwmOnnAI33RSuOp5VRowI1VXLluHy0C1aJD0iKSu5\n6DZLZUrBUubdd8Nfwq+8Ej7feuswTfTrX8MGGyQ7NmWP5cth+PAw3VNUFB7r1AluuCEkMGcdM1ek\ntLFgyVKZVrCUefnlsPX5vffC59ttFzouv/lN2BYtlWfZMnjgARgyBGbODI/tt1/4/OijEx1azfTs\nCaNHh8yVF15wW51UAxYsWSpTCxYIU/aPPw7XXguffBIe23rrkN9ywQXQtGmy41PmWLw4dFRuuQW+\n/jo8tttuocNyyilZ/vvdzBUprSxYslQmFyxlSkrgqafCL58PPwyPbbFF2NF50UXQvHmy41Ny5s8P\nV1K+446w2wygQwf4v/8LqcpZsZh2XcxckdLOgiVLZUPBUiaO4fnnw3Vd3nknPLbxxvDb38Ill8D2\n2yc6PNWhL76Av/41XJ9q4cLw2AEHhG3yJ5yQ5R2VVP37h4U37duHq4k2bJj0iKSsZ8GSpbKpYCkT\nx2GNy+DB8O9/h8fq1QvXgbnySjj44Bz6haUfxXG4MPGdd8KTT4YsHwhrU/r3hyOPzLF/dzNXpFph\nwZKlsrFgSfWf/4Rtq2PGhP+vA+y7L1xxRZgScGdR9lu6NPz73nknfPBBeKxBg3BJhyuuyNJdP+tT\nUhKieCdMCJkrd9+d9IiknGHBkqWyvWAp8/XXYXrgnntWr2XYYgs455yQYtquXbLjU9V99llIpB01\nCr79Njy2xRYhO613b2jVKtnx1aqyzJWttoJPPzVzRUojC5YslSsFS5niYnj44fDX+EcfrX78kEPC\nWpdf/tLdRZls4cIQHnj//fDGG6sf32svuPzyED+S89vazVyRapUFS5bKtYKlTByHELqRI8MFFxct\nCo83awZnnAFnnhmKmHpeGCJxZWtT7rsvXD158eLweNOm4Xf1eeeF5Rs5tT5lXcxckWqVBUuWytWC\nJdWiReEX4ahRYe1imVatwtWie/SA/ff390JdiuOw0+uxx8LH7NmrnzvssBAQ2K0bbLRRcmNMhJkr\nUq3L24IliqJWwHVAZ2AzYA7wNHBtHMfza/M4URQdDPwBOABoAkwF7gP+EsdxSSXPm/MFS6pPPglJ\nqP/4B8yatfrx7bcPhUv37mH6weIl/dZVpLRqBWefDeeem4XX+EkXM1ekOpGXBUsURTsCbwGbE4qL\nKcD+QCdgMnBIHMfzauM4URSdDDwOFAP/AP4HnAi0Ax6L47hSE9/5VrCUiWN4++1QuDz66Op0VIBt\ntoGuXeH44+GYY/Lwr/w0WrQIXn0Vxo0LsxtrFynduoV1RQcd5PScmStS3cjXguWfwDHApXEcD0t5\n/FbgSuCeOI4vSvdxoihqBkwDmgEHx3H8QenjGwCvAQcCZ8Rx/Gglzp2XBUuqkpKwwHPMGHjmmTWL\nlw02gCOOCMVLly7hr3+7LxWL43Ah4RdeCEXK+PHhQoRlLFIqYOaKVGfyrmAp7Yp8DsyI47jNWs9t\nRJjSAdgyjuPidB4niqLfACOBv8Vx/Ju13nMU8ArwehzHR1Xi68j7giVVHIf4/+efDx9vvx0eK9Oy\nZYjHOOKI8LHbbvn9S7ekJPyufeONUJyMHw9ffrn6+SgKOSnHHRc+9tknv/97lcvMFalO5WPBch4w\nArg3juMLy3n+ReBY4Jg4jl9L53GiKHoQ+BXwqziO/7HW6+sDPwANgY3iOF6xnq/DgmUdvv0WXnwx\nFC+vvQbffLPm85ttFn7XHHAA7L13+Nhss2TGWhfmzYNJk8JalPHjw+/YeWtNem6xRehGHXcc/Pzn\nuf3fIy3KMldatgztKTNXpFpVk4KlQdpHUzd2AWLgswqen0ooNNoSpmnSeZxdSm9/8p44jldFUTQD\n2A3YkbAeRtW0xRZw1lnhI45hyhR4/fVwSYDXX4evvgoXZnzqqdXvad06dBL23jss4G3XLjyWTRfi\nKykJ1+uZNCmky374YfiYOfOnr/3Zz8LOnsMOg0MPtetUJUVF0K9fuD90qMWKlOGytWDZuPT2hwqe\nL3t8ff8Hqs5x0nVuVUEUheKjXbuQthrHYUPH+PHw3nvh48MPw+6jWbPC9W7KbLAB7LQT7LxzWAfT\nti20aRMW+G6zTciHqWtLlsDcuTBjBnz+efiYOjXcTpsWIvHX1rhxuBry3nuH4uSww0Ixpmrq0ycE\nxHXuHLaoScpo2VqwKM9FUSg62rQJ23EhXJBv8uSwyeO990KHYurU0In55JPwUZ6mTVcXL1tvHaZR\nmjcPhUyzZmver18/nHvtj5KSELpW9rFo0erbefNCcTJ3bvijfu7c1Vc5rsiWW8Iee4S1oHvtFW7b\ntg3X8VEaFBSEgLjGjcN1JVzNLWW8bP3fX1kXY+MKni97fH1ZLNU5TrrOrTSrXx923z18nHXW6scX\nLQqFy2efhY8pU0IXZs6csCtp8eLw/NSpdTfWRo3Csonttw/dn9SPNm1CkaRaUlwMF5YuWRs4EHbc\nMdnxSKqUbC1YpgARYW1JeXYuva1obUpNjjMF2Kf0PR+kvrh00e0OwEpg+nrOnfq+Cp8bOHAggwYN\nquyhVI6NNgpdir32+ulzcQw//LC6ePn669ARWbgwfCxYsPp20aLQxYnjn35EUThP06bhI/V+ixah\nc9OyZbie3lZbwcYb+0d9YgYPDvOJ7dtD375Jj0bKOYMGDeLaa69N+3GzdZdQktuafw2MAh6I4/jX\na4M6cIwAABGHSURBVL2nE/Ay8K84jjtV4utwl5BUl1IzVyZMgIMPTnpEUl6pyS6hrNxPEMfxdKAA\n2D6KokvWevpPQFPg7ylFRoMoinYpLVCqfZxSjwPfAadHUbRP2YNRFDUCBhN2HRnmIGWakpKwYnvl\nynBrsSJllazssMCP3ZEJwJbAs8CnhJTZI1krUj+KotbADGBmHMc7Vvc4Ke85GXgMWAaMIUTzn0SY\nJnosjuPTK/k12GGR6kpq5sqnn8ImmyQ9Iinv5F1wXJnSixb+CejC6osWPgn8KY7jH1Je15qwpmTm\n2lM/VTnOWu85COgPHAQ0JkwtjSJc/LBS/1EtWKQ6UlQU9sTPnw+PPAKnV+pvCklplrcFS7azYJHq\nSM+eYRtz587hQkuueJYSYcGSpSxYpDpQUBAKlcaNw6JbtzFLicm7RbeSVClmrkg5w4JFUu4yc0XK\nGU4JJcgpIakWpWauvPkmHHRQ0iOS8p5TQpKUKjVzpXdvixUpB9hhSZAdFqmWpGauTJ4cro8gKXHu\nEspSFixSLUjNXBkzBnr0SHpEkko5JSRJZfr2DcVKly7QvXvSo5GUJnZYEmSHRUqzl16Cn/8cmjQJ\ni2532CHpEUlKYYdFklIzVwYMsFiRcowFi6TccP31MG2amStSjnJKKEFOCUlpUlgIe+0FK1aYuSJl\nMKeEJOWvkpKQtbJihZkrUg6zw5IgOyxSGpi5ImUNc1iylAWLVENmrkhZxSkhSfmpTx8zV6Q8YYcl\nQXZYpBooKIDOnc1ckbKIHRZJ+aW4GC66KNwfONBiRcoDFiySss/gwaszV/r0SXo0kuqAU0IJckpI\nqobCQujYEVauNHNFyjJOCUnKDyUl0KtXKFbMXJHyih2WBNlhkarIzBUpq5nDkqUsWKQqMHNFynpO\nCUnKfWauSHnNDkuC7LBIlWTmipQT7LBIyl3FxXDhheG+mStS3rJgkZTZBg+G6dPNXJHynFNCCXJK\nSFqP1MyVCRPg4IOTHpGkGnBKSFLuWTtzxWJFymt2WBJkh0VaBzNXpJxjDkuWsmCRKmDmipSTnBKS\nlFvMXJG0FjssCbLDIpXDzBUpZ9lhkZQbzFyRVAELFkmZw8wVSRVwSihBTglJKVIzV958Ew46KOkR\nSUozp4QkZbfUzJVevSxWJP2EHZYE2WGRSpm5IuUFc1iylAWLhJkrUh5xSkhS9jJzRVIl2GFJkB0W\n5T0zV6S8YodFUvYxc0VSFViwSDUwaNCgpIeQvcxcUQ34s5d/nBJKkFNC2S+KIv/9qsPMFdWQP3vZ\nySkhSdkjNXOld2+LFUmVYoclQXZYsp9/5VWDmStKA3/2spMdFknZoagI+vUL94cOtViRVGkWLJLq\njpkrkqrJKaEEOSWU/WxLV4GZK0ojf/ayk9H8WaqsYJEkKZ+4hkWSJOUkOyySJCnj2WGRJEkZz4JF\nkiRlPAsWSZKU8SxYJElSxrNgkSRJGc+CRZIkZTwLFkmSlPEsWKRSURS1iqLoviiKvoqiaGkURTOi\nKLo9iqIqXaEvXceR8kUaf/aOj6KoIIqiL6IoWhJF0bQoih6NoujA2hq76o7BcRIQRdGOwFvA5sDT\nwBRgf6ATMBk4JI7jeXV1HClfpPFn7ybgauC70uN8B+wEnAQ0BM6K43h0bXwNqhsWLBIQRdE/gWOA\nS+M4Hpby+K3AlcA9cRxfVFfHkfJFOn5moihqCXwFfAN0iOP4+5TnjgBeA6bHcbxTLXwJqiMWLMp7\npX/hfQ7MiOO4zVrPbQTMKf10yziOi2v7OFK+SOPP3v7AROCZOI5PLef5HwDiON44XWNX3XMNiwRH\nld4WrP1EHMeLgAnAhsD65sHTdRwpX6TrZ2YqsBzYP4qizVKfiKLocKAZ8FKNR6tEWbBIsAsQA59V\n8PzU0tu2dXQcKV+k5WemdI1LP6Al8EkURfdGUXRDFEWPAv8s/eidniErKQ2SHoCUAcraxD9U8HzZ\n4+vbsZCu40j5Im0/M3Ec3xlF0SzgPuC3KU99DjwQx/F31R6lMoIdFklS1ouiqB/wOKFgaQM0BfYB\nZgCjoyi6McHhKQ0sWKTVf8VVtCDv/9u78yA5yjKO498fBPGOiiDEIxwGUfGAUvAgmIgEMGKJVpmg\nKEiUeIAoiMYSCisoAoJ4IiqXciuSFF4RpQJERIjE4H0UEBASkEPAABGz+/jH805q6MzsTnZmd5ua\n36eqq5N+33777Tc728/0e6Rx/P4xKsesX/TkM1NmAp0ALIyIoyJiRUSsiYjlwH7kDKIjJW3dfZVt\nvDhgMct1H0T7fvIpZd+un73X5Zj1i159Zt5CjoW5sppQZhddTz7vdhpRLa0WHLCY5RoNADOqCWVq\n5euBh8lpk2NRjlm/6NVnZtOy37xNeuP4oxtaQasPByzW9yLiZnJa5daSDq0kzyf7wr/XWAdC0gRJ\nLyprSIy4HLN+16vPHrCEfFNziKRJzQmS9iEDnzXAr0fhNmyMeOE4M9YtYHUNsAVwGfAXcu2HaVSW\nB5c0mRzItyIith1pOWbWm8+eJAGLyBVzVwMLgDuBlwAzS7bDI+LrY3BLNkocsJgVkp5LfqvbG9iM\nXGXzUmB+RDzQlG8ycDP5S3O7kZZjZqkXnz1JGwMfAWaTgcqTgfuA64CvRsQVY3ArNoocsJiZmVnt\neQyLmZmZ1Z4DFjMzM6s9ByxmZmZWew5YzMzMrPYcsJiZmVntOWAxMzOz2nPAYmZmZrXngMXMzMxq\nzwGLmZmZ1Z4DFjMzM6s9ByxmZmZWew5YzMzMrPYcsJiZmVntOWAxMzOz2nPAYmZmZrXngMXMOibp\no5L+JOlhSYOSPjredWqQNLnU6azRPMdGxm1t3XLAYmYdkTQb+DLwCHAq8FngN2N4/dF64EXZzKzG\nJox3BczscWMm+WCfGRF3jXdleuQO4MXAA+NdETMbmgMWM+vUJIBxDFbU6wIjYi3w916Xa2a95y4h\nMxuSpGMlDQLT868aLNtAJd87JV0t6f4yxuX3kuZJekIl37quHUlTJF0s6S5JA5J2b1cH4GbyDc9B\nTXUYlPTeFvknS7pI0t2SHpG0VNLMNvnW62aS9FZJV0haKWmNpDskXSnpQxvYdrtKukTSKkn/lXSb\npNMlbdUi74JSl0NbpB1X0r7TdOygUvZNpb0fkPQrSe9uU5fmdt+2nHuPpAcl/VzSS0u+Z0v6drn3\nRyRdL2naEGW9SNJCSfdKWi1piaQ9R6udrH/5DYuZDWcxGSi8D3gBOXZFNI37kHQ8MA+4GzgfWA3s\nAxwPzJA0o7zNaPZC4Drgb8B5wJOAB4eow0TgY8ByYGFT2vJK3q2B64GbgO8BzwJmAQslvSkirhrq\nZiUdApwOrAIuA+4BtgBeDhwEfHOo85vKORj4FrCmlPNPYAowB9hX0q4RcXvTKQcDvwNOkrQkIm4s\n5ewBfBr4I3BYU/7TyrGrSl03A94MnCtp+4g4tk3VtiHb/c/A2WR7vR1YLOl1wCKyi+wisu32B35a\nyry9Uta2wLXA78k224ps659J2j8ifjAK7WT9KiK8efPWw418IHwN+DGwfyXtCODCUb7+DsA3gB8B\nsyppC4EPlD9vDtwCfLLDchcDAy2OvwYYLGVt3nR8I/IBNADMazo+ueQfAI7bgPtqnHfWMOkDwNGV\ntBkl7cfDlQn8lhxYvFmLazyrw7pOAf5LBmNbVtKmA2uBH7Y477XAo8BfgScDzyGDkdXADtWfsxbn\nTwB+Wa691RDtM6+SdnRJuxf4RiXtgJJ2SpuyTqjk37ncw73AU4dp6xG1k7f+3NwlZNZ7RwEfBy4H\nPlVJW6/7AkDSmZJ+J2lZB1sj33rdJ5I2Ltc+DPgF+c28kfZMYF/yoQH5sHki8IYu73cO+bblcxFx\nd+NgRAwCR5a097c47y5gfpfXbuVW4PPNByLicuA2YJcOy1hLts9jRMR9HZ7/YTJ4+FhE3FkpYzEZ\nyO0r6SmVtGuBY8gH+bfJN0RbAIdFxF8reW9pUb+1ZLA6AdijTd1WACdWjn237J8AfLKSdgHZHq9s\nUdYDwHGVOiwj37I9A9ivTR0aRtRO1p/cJWTWQ5JeD1wdEWsl7U3TgE5JE4EdadGlEBFzelSFmcDC\niBiUtA/5zbVht7K/slzzPklHAHt3ec2dyn5xNSEi/iHpdmAbSU+LiP80Jd8YEf/r8tqtLI+IVtOU\n/0m+DRrO+cDJwJ8lXUR2uVwTEfdsQB0a15kmqVWQtAWwMbA92Q20TkScKGk68C4y2LsgIs6uFiDp\n+WQ33BvJrronNRcDPLdN3Vq1z8qy/3tEPFSpz6Cku4DntShrWTV/cSVwIPmzcW6bekAX7WT9xwGL\nWW/9A7he0iSyG+LtTWm7k2M/rh7F698ArJL0PGBP8o1Kw1RgZUTc1HRsBTneoxsTy35Vm/RVwPPJ\nb9zNAcudrbN37f42x9fSwUSDiDhV0t3kt//DgMMBJF0FHBURN3RQh83K/hNDXQp4apu0S8mfH4Cv\nVBMlbQMsJdt+CfBz8m3HADkm5UBg0zZlrzeFOyIGJLVMK9YCm7Q43m7GWOPfdmKb9IZu28n6iAMW\nsx6KiH8BSJpFPpx/1pS8G3BPRPxlFK9/R7n+AeSD+/Km5Knk24JmryYHWXaj8ZDbkhzHUrVVJV9D\nbRdri4jzgPMkPR14Hdm1MQdYJGmHiLh3mCIa9/r0Nm8g2pI0hXzDcx/5wD9D0i4R8WhTtiOBZwIH\nRcS5lfNnk4ODx8Jz2hzfsuyHW99mxO1k/cdjWMxGxwxgcaXLY3favF0p00iXlimkw22NfFOHuP6e\nwBURMVDK34gcDPnrSr4dI6LbdUgar+qntbiv7ciuhFsiot0MoE41xpRs3GU5HYuIByNiUUTMBc4h\nZ820nHpd0VgBuJO86yingF9MDridBXyBnJ305UrW7cr+0hbFTGPsgsGd24wvmV7qMFw3zojayfqT\nAxaz0TGZpvEj5Zf6zqz/hgOAiDgkIl4dEbt0sDXyLRni+luS65Y0/32T5mOStie7sLp1FtnVdbSk\nZzeVvxFwSkk7owfX+Tf5EHxBD8pqq7rmSJPG24SHOyjm62Q3yqnljUn1GptI2m390zgFeAVwYkRc\nQU4hvwaYK+kdTflWlP1j6ippL/JN0FiZCDxm+rSkV5Hjb+4HFgxz/kjbyfqQu4TMRset5LfxhpPI\nz9uQa4D00A3kOicNs8hpppNgXTBxBDmjqCsRca2kk8jZUX+UdAnwELkOy0vJMRYn9+A6D0m6Dpgq\n6TxyQPMAcFlE/KHb8psskLSa/Pa/ggy4ppLdZ0vJacPD1fVvZX2RM4E/SVpU6rsJGXBNBf4FvKRx\njqT9gI+Q65ocU8oZlLQ/udbMGZKWldlBp5Hr4lxS2nslOaB7L+D7wOwu26BTVwNzJO1KBlaTgHeS\nbTY3IlYPdfJI2sn6lwMWs9FxOPmA+Sq5zsTLgH/3+ME6lI8DX5F0JjlOYDE5AHe+pB3JbpUvRcQj\nG1huy66GiJgnaRlwKPAe8oFzE/CZcp3qonEj/Q8HDyD/48W9yIeyyNk/jXYdrtxWadVzPlXK34kM\nutaQAehRwOmNbrbhRMT5kpaT402mk910D5HBxQ/Irh9g3YyfM8i3SLPLlPBGObeXh/oC4EJJu0XE\nH8qboM+Ri8VNAG4kx9o8SAaondxrp2m0SbsF+CBwAjCXHOj7W2B+RLQK7Na7xoa0k/U3tZ79Z2a9\nopx+sRL4SUS0Wo/E7HFF0mQyWDknIg4e7/pYf/AYFrMek3Rh+cbY8Dayr//4caqSmdnjngMWs957\nE2WsSlmP5WTgwIi4ecizzMysLY9hMeu9ucAukr5Izs6ZHRFLx7lOZr020nFIZiPiMSxmZmZWe+4S\nMjMzs9pzwGJmZma154DFzMzMas8Bi5mZmdWeAxYzMzOrPQcsZmZmVnsOWMzMzKz2HLCYmZlZ7Tlg\nMTMzs9pzwGJmZma154DFzMzMas8Bi5mZmdWeAxYzMzOrPQcsZmZmVnsOWMzMzKz2/g/KRHTdTyW8\n3wAAAABJRU5ErkJggg==\n", 58 | "text/plain": [ 59 | "" 60 | ] 61 | }, 62 | "metadata": { 63 | "image/png": { 64 | "height": 267, 65 | "width": 278 66 | } 67 | }, 68 | "output_type": "display_data" 69 | } 70 | ], 71 | "source": [ 72 | "plt.plot(x, y, label='loss')\n", 73 | "plt.plot([0.8], [0.64], 'bo')\n", 74 | "plt.plot(x_, y_, color='r')\n", 75 | "plt.xlim([-0.4, 1.2])\n", 76 | "plt.ylim([-0.2, 1.4])\n", 77 | "plt.gca().set_aspect('equal', adjustable='box')\n", 78 | "plt.legend(loc='upper left')\n", 79 | "ax = plt.gca()\n", 80 | "ax.set_xticks([0.0, 0.8])\n", 81 | "ax.set_yticks([0.0, 0.64])\n", 82 | "plt.xlabel('$y=w$ for this example')\n", 83 | "plt.ylabel('loss')" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": { 90 | "collapsed": true 91 | }, 92 | "outputs": [], 93 | "source": [] 94 | } 95 | ], 96 | "metadata": { 97 | "kernelspec": { 98 | "display_name": "Python 2", 99 | "language": "python", 100 | "name": "python2" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 2 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython2", 112 | "version": "2.7.6" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 1 117 | } 118 | -------------------------------------------------------------------------------- /AnomalyDetectionScikit-learn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import matplotlib.pyplot as plt\n", 22 | "import matplotlib.font_manager\n", 23 | "from sklearn import svm" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "collapsed": true 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "# Generate train data\n", 35 | "X = 0.3 * np.random.randn(100, 2)\n", 36 | "X_train = np.r_[X + 2, X - 2]" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": { 43 | "collapsed": true 44 | }, 45 | "outputs": [], 46 | "source": [ 47 | "# Generate some regular novel observations\n", 48 | "X = 0.3 * np.random.randn(20, 2)\n", 49 | "X_test = np.r_[X + 2, X - 2]" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "collapsed": true 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "# Generate some abnormal novel observations\n", 61 | "X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": { 68 | "collapsed": true 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "# fit the model\n", 73 | "clf = svm.OneClassSVM(nu=0.1, kernel=\"rbf\", gamma=0.1)\n", 74 | "clf.fit(X_train)\n", 75 | "y_pred_train = clf.predict(X_train)\n", 76 | "y_pred_test = clf.predict(X_test)\n", 77 | "y_pred_outliers = clf.predict(X_outliers)\n", 78 | "n_error_train = y_pred_train[y_pred_train == -1].size\n", 79 | "n_error_test = y_pred_test[y_pred_test == -1].size\n", 80 | "n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": { 87 | "collapsed": true 88 | }, 89 | "outputs": [], 90 | "source": [ 91 | "# plot the line, the points, and the nearest vectors to the plane\n", 92 | "xx, yy = np.meshgrid(np.linspace(-5, 5, 500), np.linspace(-5, 5, 500))\n", 93 | "Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\n", 94 | "Z = Z.reshape(xx.shape)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": { 101 | "collapsed": true 102 | }, 103 | "outputs": [], 104 | "source": [ 105 | "plt.title(\"Novelty Detection\")\n", 106 | "plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.Blues_r)\n", 107 | "a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='red')\n", 108 | "plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors='orange')\n", 109 | " \n", 110 | "b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white')\n", 111 | "b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='green')\n", 112 | "c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='red')\n", 113 | "plt.axis('tight')\n", 114 | "plt.xlim((-5, 5))\n", 115 | "plt.ylim((-5, 5))\n", 116 | "plt.legend([a.collections[0], b1, b2, c],\n", 117 | " [\"learned frontier\", \"training observations\",\n", 118 | " \"new regular observations\", \"new abnormal observations\"],\n", 119 | " loc=\"upper left\",\n", 120 | " prop=matplotlib.font_manager.FontProperties(size=11))\n", 121 | "plt.xlabel(\n", 122 | " \"error train: %d/200 ; errors novel regular: %d/40 ; \"\n", 123 | " \"errors novel abnormal: %d/40\"\n", 124 | " % (n_error_train, n_error_test, n_error_outliers))\n", 125 | "plt.show()" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 1, 131 | "metadata": {}, 132 | "outputs": [ 133 | { 134 | "data": { 135 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEWCAYAAACT7WsrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXd8U9X7gJ+TpG262ZuCsmTvWaDsvZG9viqgwE9EQRQK\nqEwHCKIgguJgqchGhohsBAVlb4GWvbsoHUnu74/bhLRNSkfS3LT3+Xz6ob2549yS3ifvec85r5Ak\nCRUVFRUVFVtoXN0AFRUVFRXlokpCRUVFRcUuqiRUVFRUVOyiSkJFRUVFxS6qJFRUVFRU7KJKQkVF\nRUXFLqokVNwSIUQzIcR1V7cjOxBCBAkhYoQQWle3RSX3oUpCJUsIIa4KIe4KIXyttg0VQux2QTta\nZfLY0kIIKelBHCOEuCOE2CyEaJ2Bc7wvhFiemevbOFeye5EkKVySJD9JkoyOOL+KSkZQJaHiCLTA\nG65uhAPII0mSH1Ad2AGsE0L8z7VNUlFxLaokVBzBJ8A4IUQeWy8KIRoJIf4WQkQm/dsoaXsfIcSR\nFPu+KYTYmPS9lxBithAiPOnT/SIhhLeN8y8DgoBNSZHAeCHEr0KI11Psd0II0f1ZNyNJ0m1Jkj4D\n3gc+EkJoko4vJoRYI4S4J4S4IoQYnbS9HTAR6JN0/eNJ2wOFEN8IIW4JIW4IIaZbdxkJIYYJIc4K\nIaKFEGeEELXs3Is50tFZtWOjEOKhEOKSEGKY1TnfF0L8LIT4Iem8p4UQdZ51zyoqdpEkSf1SvzL9\nBVwFWgFrgelJ24YCu5O+zwc8AgYBOqBf0s/5AR8gGihndb6/gb5J388FNiadwx/YBMxKeq0ZcD1l\nO6x+7g0ctvq5OvAA8LRxD6UBCdCl2P580vaKyB+ojgJTAM+k1y4DbZP2fR9YnuL4dcBXgC9QCPgL\neDXptV7ADaAuIICyQCk795KsfcBeYCGgB2oA94AWVu2IAzogR3izgEOufp+oX+77pUYSKo5iCvC6\nEKJgiu0dgYuSJC2TJMkgSdIq4BzQWZKkWGADsjgQQpQDXgA2CiEEMBx4U5Kkh5IkRQMzgb7pbM9G\noHzSOUGW1E+SJCVk4J5uJv2bD/lhXlCSpKmSJCVIknQZWGKvPUKIwsgP6jGSJD2WJOkusvTM+w8F\nPpYk6W9J5pIkSWHPapAQoiQQDLwjSVKcJEnHgK+BwVa77ZckaYsk5zCWIQtSRSVT6FzdAJWcgSRJ\np4QQm4F3gbNWLxUDUj78woDiSd+vBOYAU4H+wHpJkmKFEIWQI42jsi8A+RN3ukb4SJIUJ4T4CRgo\nhPgAWUQvZvC2zG18CFQFigkhIqxe1wL77BxbCvAAblm1XwNcS/q+JPBfBtsD8u/TLE0zYYB1l9Jt\nq+9jAb0QQidJkiET11PJ5aiSUHEk7wH/ID/0zdxEfmBaEwRsS/p+B1BQCFED+UH+ZtL2+8AToLIk\nSTfScW1byxl/j/xJej8QK0nSn+m5CSu6A3eB80Ae4IokSeXs7Jvy+teAeKCAnYfzNaBMOs9lzU0g\nnxDC30oUQchdVyoqDkftblJxGJIkXQJ+AkZbbd6C3O3TXwihE0L0ASoBm5OOSQRWIye/8yFLA0mS\nTMjdOXOTogqEEMWFEG3tXP4Ocp7Auj1/AiZkaS1L730IIQoLIf4PWXoTktryFxAthHhHCOEthNAK\nIaoIIepaXb+0OcktSdIt4DdgjhAiQAihEUKUEUKEJO3/NXKyv7aQKSuEKGV1rmT3YnVP14CDwCwh\nhF4IUQ14BXDI8FsVlZSoklBxNFORE7UASJL0AOgEjEVOHI8HOkmSdN/qmJXIye/VKT51vwNcAg4J\nIaKA34EKdq47C5gkhIgQQoyz2v4DcldReh6iEUKIx8BJ5HxCL0mSlibdhzHpPmoAV5Ajna+BwKRj\nVyf9+0AI8U/S94ORk9xnkJP1vwBFk863GpiRdO/RwHpkSaZ1L2b6ISezbyInx9+TJOn3dNyfikqG\nEZKkFh1SybkIIQYDwyVJauzqtqiouCNqJKGSYxFC+AAjgcWubouKiruiSkIlR5KUu7iH3L+/0sXN\nUVFxW9TuJhUVFRUVu6iRhIqKioqKXVwyT0J4+krCO9+zd1TJsXjo9a5uQpbx9vZwdRNUnMCTJ4mu\nboJdEu5eui9JUspVDZyKayThnQ+vBjlh0VCVjFKkUkVXNyHLVK9c2NVNUMkGjp++4+ompOLqvE7P\nXLrF0agzrlWyDXcWhCqG3If1/7kShZFdqJJQcTqqHFTcHfP7IDfKQpWEilNxV0GoclCxRW6UhSoJ\nFafhjoLITXJoWt45g0f2XnjolPMqidwkC0VJIo+vB6G9KlOmiB+ap8srq7gZWg/3G/Xj4eHa0eAS\ncPeJxPorRmIdsKC3swTgqGvnFJHkBlkoShKhvSpTt3IpdF5+oErCLfHwdq+hrT56ZQhNkiTyx0QC\nd1l50Zju41wpg6xgr93uKo+cLAtFSaJMET9VEG6KKoesIYRA7xdIIe97dvdxVyFkBFv36E7iqF65\ncI4ThaIkoRFCFYQb4k6CUJocrBFCYP3uzw1SSA8pfw9Kl0ZOiyoUJQkV98NdBKFkOZjx0+vQe2hp\nWj7A1U1RNNbSULIwcoos1LWb0qBOhaLEPn7s6mbYZPigHuzbtcPmawvnfkjPdo0Z2r+rQ6+58rvF\nPHzwtFbQul9WsuzrhQ69hqPx0XsoWhB+ep3lSyXjNC2fL9mXEnH3EXPqOzMbMRqNaLVap19nxbdf\nsXn33+TNV8ChbVj1wxLqNWpK4RIlAOg96OUMn8NkMsndKtnQrahUOahCcB5mUSgtwnDnqELR79ba\nFYo65bxHz9/K8DFXL1/i05lTiHj0kMTERPoNGUaXnn0BmDR2JGFX/iMhMYGSQaWZMnMuAYF5OHL4\nILOnT6JilWqcP3OKEWPeYef2zXh6ehF+9TJ3bt+kao3afPDRfIQQxMREM3fW+1w8f4aE+Hjq1G/E\nmxM+QKvVcvnSeT6Y8CZPYh9TpnxFEuLjbbZzaP+uxMfHMWJIbxo2bkZws1ap2lCpag1mvTee6+Fh\nSEgMemUEnbr1BqBzi7p07NqLwwf3cv/eHQa+PII+A1/mmy/nce/uHd4dMxwvLy9mzV/Cb5vXExv7\nmLGTpgGwdOE8ft+6CaPRQKHCRXnvo88oUKgwX376If9dOEd0dBS3b15n2brfCMiTJ5P/e89GiXJQ\nxZC9KLVLyh0T2+o7Nx0YDAYmjRvJ9E8WULpMOR7HxDCoZzuq1ahN6TLlGBc6jTz58gNyV8/3Sxbw\n+rhQAC5fOs/EqR9TrWYdAHZu38x/F8+x8Luf0QgNA7q35vDBvTQIDmHurPepVbcBk2fMwWQyMWnc\nKDauWUX33gOZMv51+g4aSqfuvTl57Civ9Otis61fr9xAnQpFWfrjJnx8fTly+GCqNkwY8yplyr3A\n7AXfcv/uHQb2aMsLlapRtvwLAMTFPeHbnzZz8/o1+nRuRufufXhlxBg2/LKS2Yu+o1yFSqmuu3nt\nT1wPu8ryDTvQaDT8vOwb5kyfxKz5SwA4eewIP27ZQ96k35MzUOWgYgulCcPdogpFv4Mz84nfGYRf\n/Y8r/11k4lsjLNsSE+O5cvkipcuUY/OG1WzbtJbExETiYmMJKv28Zb+SpZ6zPJzNNGvVHi8vOeFb\noVJVrodfheAQ9v6xndMn/mXFt18B8sO6cOGixMRE89+F83To+iIAVWvUpmz59M9mTtmGv/7cx5h3\n3wOgQKHCBIe05MjhAxZJtOkg5zKKlShJQEAe7t6+SbkqVdO8xp4d2zh94l/6dggBZLH6+z9NwDZu\n3jrXCEIVg3JRUneUu0QV6rs5HUgS5Mmbj5Ubfk/12r9HDrFm1Q8s/XEjefMVYNumtaz9ebnldR8f\n31THeHp5Wb7XarUYjcak60jMXvgtJUqWSrZ/TEx0ltpvqw1p4en1dMSSVqcFj2fnMCRJYtjocXTv\nM9B2G3z9MtSG9KLKQSUzKEUW7iAKdXRTOij1XBn0em9+Xb/asu3qfxeJiYkmOioKPz9/AvPkIyEh\nno1rfsz0dZq2aMv3iz+3SCPi4QNuXAvHz8+fsuVfYNumtQCcOvEvly6czfR16jVswvqfVwBw/95d\nDuzZSd0GjVPtl3J4q6+fPzFRUTbPGdK6HT//8A1REREAJMTHc/7MyUy3MT0oRRDq6CT3RQmjoqpX\nLqzoEVDqOzsd6HQ65i76gTkzp7Dsmy8xmYzky1+QD+d9RaMmzdm6cQ092gaTJ28+atZpwOmT/2bq\nOmMnTmX+J9Po17UlQgg8PTwZO3EqxUsG8cHH8/lgwpt8t+QLypavSKWqNTJ9P+MmTWfmlLfp27kF\nEhL/N24iZcpVSLaPrfkP/V96lSnj/g9vb29LrsFM5559iXj0kJd7dwTkUUx9Br9ChUppd1NlFiUI\nQhVDzkEJkYVSowohSZJjTiSEFjgC3JAkqVNa+2oCS0q2KtOtn9CEAsVK2ThCJTtR8gS5nC6HsEsX\nOBCdse7BjNCoWPZ+aj540/V9/5nBlbJISxRX53U6KklSHbs7OAFHvtvfAM4C6nRRN0YVhH3cJXLI\nbhGkhb22KF0erowslBZROORdL4QoAXQEZgBvOeKcKtmLkuUArhWE0uWgJCmkF1ttVqI4mpbPl+tF\n4ah3/zxgPOBvbwchxHBgOAB6502kUsk4BUmkyL3beJgMJGp03PbLT4S33f/KbEWVQ2rcUQrpwfq+\nlCQMV0UVShFFlv8KhBCdgLuSJB0VQjSzt58kSYuBxSDnJLJ6XUeTLzGWYnGReEpGEoSWm/pAHnr4\nuLpZTqcgiRSPvos2KTflaTJQPPougMtFoQpCJqdKIS1S3rMSpOGKqEIJonDEX0Iw0EUI0QHQAwFC\niOWSJNkeMK9A8iXGEvTkEVrkB6WXZCToySOAHC0KD289Re7dtgjCjFaSKBLzwKWScJUglCKH3CiG\ntFBKlOGKqMLVosjyPAlJkiZIklRCkqTSQF/gD3cSBECxuEiLIMxokSgWF+miFjkfcw7Cw2S7Vqa9\n7c7Glau2KkEQXlqNKohn0KhYPpf/jrJ7boUr51Gok+kAT8l2uUh7290ZD299siR1osb2g9Hedmfi\nSjm4UhB+HlrLl0r6cbUscosoHCoJSZJ2P2uOhBJJELb/OCcvXkJiQkKmznnm5DEmjR35zP3u3bnN\nq4N6ZuoaGcXDW88rvTux5/dtlm23/fJjTLFst1EIbvs5b52llNy4Fs7Gn5cl2za0Xw/Crlx2+rWV\nIAeVrGGWhSuE4erZ2tmB20cSef31VHmuALXKFabKcwXI65/xoZw39YEYSfGgRDBjyWISExNtHmMw\npN0dU6lqDabPeXZBnoKFi/DVsjXpb2wmsTfENcLbnxv+hUjQ6JCABI2OG/6FMpWPeNbvxB4P797g\np2XfJtv29aq1lHrueTtHZB1XRg+qHJyHK2ShhKU9nIlbSyKvv55ShQPw8tAhhMDLQ0epwgEZFsVD\nDx/CvfMSL+TMRLzQ8r858wB4uW9n+ndtRXRUJO+/+wbTQscytH9XBvdsB8i1JAb1aEufzs0ZN+ol\noiLltYuOHD7IoB5tAbh5/Rot61diwdxZ9O/Wmh5tG3PsyOFkr5mpU6EoSxd9xuCe7ejasj47t2+2\nvLZz+2Z6tmtM/26tWbroM7uV82IfP+aDCWPo3akZvTs1Y/l3XyV7/fD+PfTv3JJOTWox/6OpRHj7\nc65gaUatXEOF3r1p070Dvds3JSpSzsmc+PcIr/TpTN8OzejboRl7d24H5AggpHoZ5kyfTJ8OIaxZ\n+R0h1cvw6OEDy7XmTJ/MorkfATBh9DD6dWxOz9aNGDNsIFEREfjoPXj/3be4dOEcnZs35P9eHgBA\ns9qVuHD2NABhl/9jcM+OdAqpT9eWwez942lFvnKF/Phy3if0aNOU5nWqsG3TegCexMby+isDade4\nNp2bNWD00EGWY1whB7VLKXtxlSxyIq7P1GWB4gX80GqSe06r0VC8gB+PouMydK6HHj7JRjKN+eAT\nlv+43FKXwcyFs6dYvHwd3j7yvmnVkrAmMuIR1WrUYdSbE9i6cQ3zZ89g6Y8bbbbF18+fH9Zs49jR\nv5gw5lVatu3Eg/v3mDllPN/+tJmg0s+zIsWD35qvF87FZDLx06ZdJBgTGdStLeVeqETj5q0B+O/i\neX5Yt52E+DgGdWtD9dr1qFGnPsu/WcjvR86h13vzOCYaL703UZGRTJ/4Fgu++5mChYtw785t+ndu\nyZodBwGIePSQytVqWgoPnT11gi3rf2HAy69iMBjYuv4Xvl8nS2X8+x9algv/4pPpLFvyOW9Pnsr7\nH37Kh++Hsm7HPpv389bIV+g76CV6DRjCxfNnGdC1LVv3HyV/gYIA+Pn7s/a3vRw9/CdvDBtMu87d\n2Lfrd2Kio9m2/6jl9w+uE4SKa2hULF+2joZy1eQ7Z+LWkYSnzvYfn73tjqBlu04WQQBs3rCagT3a\n0Kdzc7ZvXseFs6dsHufj40uTpId01Rq1uXHtqt1rtE2q51C1Rm3u3b1NfHwcp47/Q4VKVS21Krr2\n7Gf3+L/+3Ev3XgPw9PHGzz+A9l17cmj/HsvrXV7si06nw8fXj3ZdevDXwX34+QdQstTzTBozgjUr\nvyf28WN0Oh3Hjx7m5rUwRg3pRe92TRg1pBdCwLWrcr7Ay0tP287drc7dj42/rARg/64dlC5bjuIl\ngwDYtOZH+nZoRq+2wWzdsIazp07YvQczMTHRnD11gp795EigXIWKVKxcjWNH/7bs07GbXGejRp16\n3Ll9i/i4OF6oXJX/Lp7n/XfeZOvGteQN8M12QaiRgzLI7qgip0UUbh1JJBiMeHmkvoUEg/NGJXlb\n1WZ4Vi0Jazw8PS3fazTaNPvvzfUczLWojRm8HyE0aL08n72jFVqtlmUbdnDsyGH+OrCXfh2bs3DZ\naiRJotwLlfn2ly2pjrlxLRxvH59k9apr1WtIbEwMF8+dZuMvq+jaqz8A/xw+yOplS/l5y07yFyjI\nxjU/89OypRlqoz28Uvy+DAYDQaWfY8u+v/lz724O7vmduTM/4I9DR9Hrnb/8iCoGZZKdUUVOiijc\nOpK4cT8Go8mUbJvRZOLG/RiHnN/X14+YGNv1EwCH1pJ4FlWq1+L8mZNyFTtg87qfbe7n4a2nfuMQ\n1v20HEmSeBwTzbaNa2jYpJlln1/X/YzBYCA29jG/bV5PvUZNeBwTzaMH96nTIJiRYydQpkJFLp0/\nS43a9Qm/cpm/Dj7tCjp1/B/SWj2484v9+GHxAv45fJBWHeQyq1FRkQQEBpI3X37i4+NZs/IHy/5+\n/gFE26lT4efnT8Uq1Vj7k1z/4tKFc5w9c5Lgii9Q6sYlAIJuXcbvcfI5Lbdu3kCr0dK9R3c+mPUJ\nDx7cJ+KRc/9o3TFyKF3Q1yFf7kJ2RhU5JaJw60jCnHcoXsAPT52WBIORG/djMpyPsMeAl1/jtcG9\n0Ov1NkcgObKWxLPIX6AgE97/iNHDBqL39qZJs1boPDzQe3tb9jGPYHr1jbeZNXk8PVs3AqBTjz4E\nN2tl2e+5MuUZ3L0tURGPaNOpGyGt2nHn1g3eenUI8XFPMJlMVKxSnZbtOuOl1/PZ0pXMnTGFTz6Y\nQGJiIiWCSjF/qX0hdu7Zhw7BNejaewDe3nLXXOt27dm+8RdaN6hB3vz5qdsgmBP/HgGgQqUqPFe2\nHB2a1uX5suX5YumKZOf7dOE3TH57NN8t+gKtTscXcz6jEgloDLKoPAyJBD5IXur2wtnTfDpDLtFq\nNBp5/a23KVK0WKZ+9+lByXLIjod4Wte4ei/14ApXk11RRU6IKBxWTyIjqPUkMsfjmBh8/eQyoBvX\n/MiGX1byzSo5+a3kVVwdPUmu1I1LeBhSD01O1HkQVrxstuYeHC2HSxfOE+dTPEvnUPone6VJIztk\n4ShRbBhW163rSag4mR+Xfc3ObZsxGA0EBuZh0vTZipYDOGcWtc6GIMzb3VkQWUHpYrDGuq1KEEZ2\nj4ByN1RJuBGvjBjDKyPGWH7OjYIAMOg8bEYSJo/sWdZDKXJwJzHYQynCcLYo3Lnbya0T17mZ3CoI\ngAd5CmJKsZSISWiILlDEadc042pBuGOyOL24+r6cndB210S2Gkm4GUqXA6RPEH6PI8kfcQ+dIRGD\nzoMHeQoS4xuYrvOb9zMfb/LwILpAEeICnFvMypWCyIlSsIf5Xl0RWagRRWpUSbgROUkQhR7cQiM9\nHZ1UKGl0UkZEEeMbmG05CFcJIjfJISWZlUX+jasJmjMVz1s3SChanPCxU3jQpVe6j1dzFMlRJaEA\n0lMVL6cIAuQIQJNiVJ1GksgfcS/dkoDsWWJDlYPryYgs8m9czfOhb6CNewKA183rPB8qj6RUiijc\nLZpQcxIuxlwVz0uS16E1V8XLlxgLpK7/kFn+/nM//To2z/J5bGEuFDT/4xl8+N7EZ+6f1uik9JIR\nQcyeOY0EqyXfP57+ARvWrH72NVwgCJ1WqIKwQ3pyFkFzploEYUYb94SgOVMzfD1n5ijcKT/h9pLY\nenUtnTbUpe6qYnTaUJetV9e6ukkZImVVPIPBYKmKp4TowWhMe0mQzCSoDTrbx9jbDsmXIc9oBDHn\nwxnJ6oKMn/QeXXum/akyuwXhpdPgpXP7P8dsIS1ReN66kaHtz8LVFfCUgFu/K7deXcuMv8ZxO/Y6\nEhK3Y68z469xDhNFWst2nzr+D68O6snAHm0Y2KMN+3f/DsAXc2bww9dyHYkdWzZS94ViPHxwH4DR\nwwZwaP/uZNfwlIz87/33GTp9Ok2GDaPO4MEA/HvyuM3luQFWfbeYzk1r079TCxbOmUVI9TJA6mjB\nXvRgMBh4bWBP+nVsTveWDZk8dpTlIbph9UqG9+vGmGED6dGqIRfPnUl1/NKF8+jRqiEvtmnEO6Nf\n43HM02VQbt64xqAeHWjbqCajXupPdJS8XMbvWzfTMaQenZs3pGqfPvxxVF6d9db9+7z4zjvUGzKE\nan378uW8Tyznala7Ep9Mm0LPtiFMHjeaIT07sX/nVsvrO7ZuoWfHNgAs+nwe7UKCad24Pp1ahnDq\nxHEAJrwldzV0bt2MVsH1iIyI4I3XhrL0qy8BeYLimBHDaFa/Fs3q12LBvDkWQbRv05LQCe/QukUI\nVSuWZ8qkp1HSrBnTqFW9Co3q1ya4QR0iIiJS/Z7SiyqHjGNPFAlFbU9EtLfdlbhLNOHW784Fx2cR\nZ0weWsYZn7Dg+CyHXcO8bPcHH3/O7OmTAYiOimTme+8wY85Clq/9jXmLljFzyniioyKp27AJf/8p\nr3P016F9VK1Rm78P7ceQmMjpE/9So3a9ZOc3V8U7duEC2+bP59jKlURER/Pahx/y4fwl/LhlN59/\n+yPTJrxFVGQkF86e4psF8/h+3XZWbv7D8hDOCFqtlg8/X8KqX3ex9veDmIxG1v30dGHCE/8eYWzo\nNNb+/icvVK6a7Nj9u3awee1PrN76B7/u+QuT0ciCTz+yvH7k0EHmLvqW7Qf/xT8gkAVz5Nc++2g6\n02d/zqZdf7Jhz1+UbNSMRJ0Hg997j5H9B7Dx1x2s2XmQvTt/Y//uPyzni4mOYs32Pcyat5B+gwbz\n88qn7fxxxQ/0GSBL9cV+A9i25wA79h9m/KQpvDPmdQBmffoZAJt27Ob3A38RmCf5CKi5H8/CZDKx\n69BRNu3YzS+rVvDb9qeV+65fu8b233ex/9DffP/dUi5dusjDhw9Z8PlnHDj0NwcPH2Xbjl34Jc2E\nzwhq9JA1bIkifOwUjHrvZNuMem/Cx07J9HVyezTh1onrO7G2Q0h72zODrWW7j/97hJvXwxk9bIBl\nPyEE18KuUL1WXSa++RqJCQkc/+dvxoyfws7tv1KocBHKlKuA3jt5QvqmPhAJwYstWuCbtA7TvhMn\nuHLzFqOG9LI6v7w897Gjf9GyaQiNTDF43IngzTbN2WpnsT97mEwmvv/qcw7s/h2j0UhUZGSyNaBq\n1m1AydLP2Tz20P49dO7RC3//AAD6DHqJ6ZPGW15v3qY9BQrJtXh79R/M1InjAGjQJIQZU96lbceu\nhLRsTfmKlTn7+DG7//mH63PnwVy5yNPjmBj+u3iOxs1aANCtt7yKrJ9eR4cu3XhvwngePpCLGv25\nfx+ff/UNACf+/Zf5cz4i4tEjNBoNly9dTNfvYu/uP5j20WyEEBTNn5devfuw64+dtGkrF5Xq3qMn\nGo2GwMBAKlR4gSuXL/Pcc8/zfJkyDH/lJVq0akX79h3x989YJT9VDo4hZVLbnJzOyugmWzgrke0O\nSWy3lkRhn+Lcjr1uc7ujsLlstyRRrkJFlqxYb/OYsuUrsv3X9RQoWJg6DYKZ99EHFCpSlLoNmqTa\n96GHD7GeevS+fkhAokbHfe9Ayla0vTz3hUN78U2Mw9Mk99F7mIxoJBN5nkSj1WoxSU9XxU2It73Q\n4Zb1q/n370N8+8sWfP38+fqLOYRd/s/yuo+P/T5fD23mHm6h0z7i/JlT/Ll/D68PHcTLr71Ox+4v\nIoRg7W978bAzW9rX92kdCB8fH9p17MS61T8B0K5jJ3x8fUlISGDY4H6s2/o71WrU5Patm9SskLHS\np/ZyEF5WS4trtfIS71qtlj/2HODQnwfZs3sXTRrVZ93GzVSpWu2Z11GKHPQeAj+9Fq0AowQxcUbi\nEm2v45aRfV1F6YK+yUSRVSmoPEUZ79hMMqr6BPTa5KGlXuvNqOoTnHrdajXrEB52hSOHDli2nT5x\nzLJ8dt2Gjfnq80+o17Axnp5eFCpSlM3rfqJuw8bJzmMeuZSg9eCOXz5OFi7LuYKlKdewmd3luTtW\neYFtBw9yP6kP/PtffwWgSMwDSgSV5kb4VaIiIpAkia0bbNfOjo6KIk++/Pj6+RMdFcmW9b+k6759\n9B40CmnOlg1riImJRpIkfl7xPcEhLSz77N6xjQf37wGw5sflNGwcAsDlSxeoUKkK/xs+iq4v9uXk\nsaP4+flTp0Ejvpo/x3L8rRvXuXfnztNreiX/HNN7wCB+WrmMn1Yuo89AuaspPi4Oo8FAsRIlAPhu\nSfKqfX61V41rAAAgAElEQVT+/kTZ6ZZr2qwFq5d/jyRJREdH88vqn2nRspXNfc1ER0dz/949Gjdp\nSujk96hUuTJnTp9O8xhQliACvLXoNAIhBDqN/LPeQ2RpX1fj7FFhzup2UnpuQhnv2kzSvnQPQuvN\npohPCQSCIj4lCK03m/alezj1ugGBefh04fcsXjCHfl1a8mL7Jiz+YrZFEvUaNuHWjeuWyKFugyZE\nPnpElWo1LedIa+RSQJ48fLZ0JV/N+4hebRvTrUV9Fs39EEmSqF22DOMHDaLhyy9Te9AgdFotgX5+\neJgMFCpSlEHD/o++HZsxuHtbS7dPSjr37ENsTAxdm9dj9Mv9qFWv4TPv2TyKKaRlG7r07EvvDi3o\nGCLnV0a++bS7qU6DRrz56v9o26gmEY8eMmrsOwDMnjaFDk3r0rl5Qw7s+YNh//cWAHMWLuXShXN0\nDKlHx5B6vDFsCFFRsgCFjedQ/YbBxERFERMVRf2GwQD4BwTwdugU2jcLpk3ThsnKzQK89n9v0KtT\nO0vi2ppJoZOQJIn6dWrSslkT+vYbQOs2bdP8XURFRtKvd08a1K1Jvdo1KFy4MF26dU/zmPQIQjx6\niObmDQrn9aJA1bLoV6965jGZwU+vRZPil6sRcrSQlX1VcibqUuHZTFaHtb5w7yrx0ZH4Jz0I31+8\nmEvXrrF0xizOFSztgBYmx5lrMKVFTpool15BaK+Fc+7uXSq2bw+AyduHqPlfEtfLfqnazFA4QJes\nmqAZSZJSdSelte+dKPvVFV2Js5fzcNYku/TkJtSlwnMwjprzcNsvPx9+OJODx46RYDDwfPHifBka\nym2//A45vzWqILJOeruYtLduIlJUWdQ8icVv6mSHS8Iogc5GlCaEQCcgwFsLyKKwt69RWSmJZFjn\nJ1SyjiqJbMCRk+IivP15d/ocisQ8wMNkIFGj47ZffiK8Mza65lmogsg6GcpBWE32s0Z7/ZqDWvOU\nmDgjAd6pu5HMmLuT4hINNvc1SRIxcemru+6qpLczRZHb1nZSJeFEnDVjOsLb3+FSsEYVRNbJcJLa\n09OmKIwlSjqoRU+RH9JG/Lw0aC9eQPz7L5w5A//9BzdvwoMHaB8/pqDBhOThgcnfDwoVQpQsgalC\nBZ68UJX4GnUhMO11tsxJb7NgUkYpGUW/ehV+UyejvX4NY4mSxEyZ5vAoy5UodTisKgknoITlNDKL\nKoisk5lRTMaixdBeC0+2zeTtQ8yUaY5qlvypPi4a7eZN8Ouv8McfiPv3be4rAFu/LS3gB/gJSKhT\nn/gOnXnSsw+moFKsPbeKWQcmcyP6GsX9SzKz5UwGVR+Q7HjrKCVDbV+9ioDRI9A8kdc0010LJ2D0\nCAC7olCjCcegSsKBqHLIHNlZctTZZHaYq5Q3H0ZAevAASQiHflLW68D/8H40Xy1CrF+fPGIpUgTq\n1YOqVaFcOShRAvL4wqUZcGszGIEngD4EtC3h0m048if8cxzPvw/j+fdh/KdO4mqjGvxS6SzXi8cD\ncD06nNd+HY5GAwOqJheFNhOjZ/2mTrYIwoyzcjYqyck5f50uxJ3lALlDENkRRWR1HoSUNx+mYsW5\n8yjeMQ2SJPx2bsV31jRE0lpZCAHNm0P37tCuHZQtK28zGUBoWHFiBaG7JhEeeY0gvQ8zAmMZ0KAf\nNPgWtF5Pzx35ABZ2h0374AiUPnCMPw7Ahgowuj2E54HYxFhCd4amkgTIUU1Gupzs5WaelbNRk9hZ\nR5VEFnB3OYAqCEehlIlyZnRnTxMw7g08D+yVNxQqBCNGwNChcrSQEqFhxfHvGL7ldWKTlqkPi4tl\neIIW9E0ZYC0IgMD8rGg3mFDTPmIbw+uHYcwh6HoeWlyBVzvBqmoQHhme+lKZ6HIyliiJ7lrqczkj\nZ6OSHGW9s90ER9V4SA/tG1Xj4vnUK7FmFXMNCEdSrpBfshVh08IsCOsVWZ1BZgQRERHB3Dmzk20b\nNWI4B/bvd1SznIfRiO+nH1GgcU1ZEPnzw9y5cPUqvP++bUEASCZCd0+1CMJMrMlI6L7UC2auOLmC\n4VvfIMwA93xhSgso/zr8UhH8E2DlWnhvF5QMsH29jHY5xUyZhinFumeOztlkFGfMwFbi7Gu3l0Te\nTWup0rwutV4oRpXmdcm7yXn1JLJTDs7EWg7WdRqyi+yKIPQic0MtIyMjmDc3uSQWfLmY4MaNbe6v\nlChCRDwib68u+E+dLOcSWgDHdsKYMeDtbf9ASQKhtfmpHyA8MnWXTujO0FRCue0PvXrDyA5gFPD+\nHthwpYHNc2Z0nkVcr35Ezf8SQ8kgJCEwlAxK90RDtYhT1lDGuzuT5N20llKTxuF18zpCkvC6eZ1S\nk8Y5TBTJ6km0asDvWzZaXjvx7xGb9R4++/ADvls0H4Dtm9ZRo1Q+y1pGo4b04uDeP1JdZ8v61Qzo\n0ore7ZvSu31TDu/fk+z1X9f+TN8OzejUpBarvlts2d6+UTUWzJnJoG5taN+oWrLXTh3/h0Hd2vBi\nm2AGdWvDqeP/4KP34OHdm9R7IYgP35tI15bB/Lz8O+Z/PIMxw4cwtF8PWtarxhvDBnP65HEG9ehA\ni7pV+fD9UMt5v1k4nx5tmtKlRSN6tW/BmZMnnvl7NBqNfPjeRDo0rUunkHp8EPpusmJGp0+doHOr\nZgTXrMLY10dYqsgt+/ZrmtSpTqvgerRoWIeLF84DcOniBfr36EK7kGBaNqrLj8u/t5yraICe2TOn\n0bFZMLNmTKNGlYqcTKotAfDVlwt4bfgrAEx8dzwhwQ1oWK8Wndq3ITwsDIC3xowmMiKCRvVr07KZ\nvLRK+zYt2bpFXifr7p079Ov9Ig3q1qRBnZosW/aD5fzlyj7P+++/R5PGwZQr+zwLFywA5JV3X/+/\nUVSpUonatWoS0jT1Yo9ZQXPjOvnbhuD1xw4okB+2/Aobr4DmNkgmRv46Et1UHeIDgW6qjpG/jrQc\nu+LUSkp/9hwStp/cQYFBqbbZEwoCFtUTjB4QgCQENT5fDdu2JdslI/MsrInr1Y/7Jy9x51E8909e\nUhPW2YRb5ySKfzrLZqnC4p/O4lFnx6zfFJA3L6u27Obfvw/x9siXadWhC1GRkUyf+BYLvvuZgoWL\ncO/Obfp3bsmaHQepHxzC94u/4H+vjebwgT1Uq1WHvw7upVX7Lpz89yg166b+ZNUopCXtu8orol79\n7yLD+nVjx19PF4x7+OA+P27ZzYN7d+nTPoTa9RtRvmIVAOKePGHZ+t+4cS2cnq0b0bVXfzw8PBn7\n6mCmzl5A/cYhHNq3m3GvDeH3w/ID/dHDh1StWYt3P5gJwPyPZ3Dq+DHW7diLj68f3VoGM3vaFL5e\ntQ6jwUDzupXpO/glSj9flm69+/HKyNEAHNiziynj3+CXrbvS/B3++MNSzp46we/7DwMwoGcXln/7\nDUOGDgfg3yN/s2nHbrz0egb07Mryb7/h5VdHMG3yRPYdOU7hIkWJj4/HaDRiMBgY+coQFnz9HeXK\nVyAmOpp2IY2oXa8B5cpXAEDv7c2eA4cA8PLyYsXyZXz4cXUAli/7nlkfywsKvjVuPDM//BiA7779\nhimTJvLdshV8Om8+TYMbcPDwUZv38/bYN6lUuTJr167l1q1bNKhfl5o1a1Glivx/8iQ2ln37D3D1\n6lVq1qjG4CFDuHjxIrv37ObEiVNoNBoePXqU5u8sI2hu3SRfx1borl6GypVgy1YISnqw+5Ri5JZR\nfHnkaZeeUTJafg4OCmb4puGpogIzPh4+zGg5I9X2oMAgwiLDUm0vFViKq2OuytFJxVkQGgrDh8P5\n8+DtjSRJRD1R3iqyKvbJciQhhCgphNglhDgjhDgthEi9KJOTcHSpQmvM3UrtuvQEoFqtuty7c4v4\nuDiOHz3MzWthjBrSi97tmjBqSC9LvYcadetz+vg/JCYkcOzIYV59YzyH9+/h5D9/U7ZCRbxT9KsC\nXAu7wmsDe9K9ZUPGj3qZB/fucP/u05VQu/cZCED+goVo0qINf//5dPXZdkkyLF4yiIDAQO7cusnV\nyxfx8PCkfuMQfPQetGjdGg8PT65cugDIy1936NozWRuaNG+Jf0AgWq2WCpWqEBzSAi8vL3x8fXm+\nTDnCr14B5NVu+3VpQ4emdZk15V3Onnp2JHFw7y76DRqMp6cnnp6e9BkwmL1WhYW69HgRXz8/dDod\nvfsPZP/e3QAEN23G6FeH8s2ihdy+eQMfHx/+u3SRi+fPMeKlQbQKrke3di2JT4jn4vlzlvO9NHiI\n5ft+/Qfyy+qfMBgMnD51ksiISIKD5W6jHb9to3nTYOrVrsH8eXM5YRVxpMWuXTt5+ZVhABQtWpR2\n7dqzZ/duy+u927dHd+YUZSMektfPjxtnTvP8889jSExk+LChLF++LF3XSQ/i8WPy9u4qC6JmVdi3\n/6kgAIRg8dHFNo9dfHSxzW4jM6UCS7G482Kbo5NmtJyBj0fy93IyoQgB77wD1avDtWuwdKllP1UQ\n7oUjIgkDMFaSpH+EEP7AUSHEDkmSHJ9tTUFC0eJ43UxdTyIrpQpT5hy8vORRHZZ6EkYDkiRR7gXb\n9R4AylWszNYNayhYqAh1GzZh9rTJFC5SjHrBITb3f/f/hjJ28nRatO2IyWSifoVixMenbxikp/7p\nqBOtVovB+DTHYC8x7ePjk2rRNi+vp/et0WrxsjqvJqmOQkJCAq+/PJCVG7dTuVoN7ty+ReNq5Z7Z\nRl0ma1AsXfETx44eYf/e3fTs1JaP5n5BiZIlyZc/P78f+Mvucb5WVeJKBgVRsWIlftu+jX179zBg\n0GCEEISHhfHu+HHs2f8npUs/x6E/D/LK/wanu22e9vIQJhO+jx4i8soV8LQaDdL1a+QpV45jx0+y\nZ89u/ti5k9CJEzj81xGKFCmS7mvawn/8GDxOHofCwPbfIW/eVPsYJdtdO0bJaDMaABAIOSKwg1kc\noTtDCY8MJygwiBktZyQXilYLEydCnz7w7bcwapSi13xSsU2WIwlJkm5JkvRP0vfRwFkgWwrK3nhr\ngs1ShTfeyng9iYwkpWvUrm+33gNA/eAQvpw7i3rBTfH08qJw0WJs/GUl9YOb2jxfdFQkxUvKn/7W\n/7SchBSC2Ji0ZPTDB/fZt2sH9RrZTqCaqVipEgZDIoeScht/7tuNwZDIc2XLW/bxexxJqRuXKBN2\nlsDoh3gkPltK8fFxGIwGihST/3tXfGv7E6o1fnqdXLNh1XISExNJTExk9arlNG3e0rLP5vVriX38\nGIPBwC8/rqRx02YYDAbCrlymZp26vP7W24S0aMWpE8coU6483t4+rF61wnL8xQvniY6KsjuSacCg\nwXz/3VJ++fkn+g8YBEB0dBSeHp4ULlwEk8nE0q+f3ou/fwBPYmPtJvVbtmzJN998DcDt27fZtm0r\nzZon1RI3GlMt1CdMJh6ePkVsbCxt2rRlxsxZBAQGcuXy5Wf+/qxZe24Vdb8pS7F5XtT9pixHlr6L\nz4rvwQP4pD3ktz0yRisyPsLLOg+x4uQKSs8rjeYDDaXnlWbkryMpPa80g9bKv8tlPZZxdczV1BGH\nJEHnzuDlBUePYnr4MFO5CBXX4tDEtRCiNFATOOzI89rjUecehE2fTXyxEkhCEF+sBGHTZ6c7H2EW\nQ0ZHLKVV7wGgfnBTbl6/Rv2kgjv1g5sS8egRVWrUtnm+t9+byZhhA+nTIYTr4VfJkzf5H3uefPno\n26EZg7u14ZVRYyj3QmWb5xFC4O2pw9PTky+WrmDOjA/oFFKfT2dO5fNvluPp6SnvKEkUenALD0Oi\nvPyCyYRP3GP8HqddL9vfP4A3xk+iZ9sQurVqnGYFO3g6imngS69QsXJVWjeuT+vG9alYuSoD//ey\nZb/qtWrTt1tHmtapTrESJRj40isYjUbeGDGM5g1q07JRXe7evs2gl4ai0+n44ae1bFizmhYN6xBS\nryYT3hqNh51PywBdunZn/949vFCxIiWTumIqV6lK9x49qVuzGs2bNKKUVbnWfPny0btvP0uNiZTM\nnfsZJ04cp1bNGrRv15bpM2ZSuXJly+/WFteuX6dd2zbUrlWTWrVq0K5tO+o3sD3yxxZrz61i3O8j\nuB4djoTEzchwys6aK784rAb0Ww0a2x0Dw2sPT/d1IHm30YqTKxi+aThhkWFISIRFhvHlkS+T/fzS\n+pco8HEBi0RWnFwh/x5u/QamO1CxIgCPT51Tu5rcEIfVkxBC+AF7gBmSJKUaXiSEGA7I71Z9ntr6\npqEpd8m2ehI5YRhrSjIy56HUjUt4GBJTbU/UeRBWvGyW2+KKZTaUsi6T7swphI2F+iRPTwyVqjzz\n/OfPn6NAidRdeHW/Kcv16KcjinqcgTU/Q3heCPrvPOQtn+oYa0b+OpLFRxfb7XoCuYspZbdR6Xml\n7XZJ2cPHw4fF7T9jwFk5b8N04Cw83PgbCU2bZehcjsJZs66dsX5TWov8uaKehEMiCSGEB7AGWGFL\nEACSJC2WJKmOJEl1hKefrV2cTk6Z52BNZibF6WwIIq3tGSEnCyI9GIsWQ9Ik/7OSNBqMRYtl6bw3\nopPPVXj5X/nfOfWBPM/OCy3suBDDFAOlbAxnBcjvnR/Te6ZU3UZ2h7qmQWxiLKF7pkE/I3S9Agny\narGmgIAMn0vF9ThidJMAvgHOSpL0adab5HhUOSTHoLN9nL3t6SWnCyI9k+akvPkwlgxC8vREQo4g\njCWDkPJmbSZtcf+ny0/oE6HVZTAB+xuVsF3n1QrrnEJMwmObOYqo+ChLrsG62yifd+baHR55DYQG\nKAjhsaARGMs8W2YqysMRf9XBwCDgpBDiWNK2iZIk2R76kwYmSUqa/emYIus5TQzgmLWWHuQpSKEH\nt9BYdTWahOBBnoKZPmdOF0RGkPLmw5AJKaTs+rUu2DOz5Uxe3TycJ4ZYqt4BLyOcKSR4q9uHaZ7T\nnFMwD3N98OSBzf0STYksOrLIMqEuLDKM4ZuGI8jc36Il8b1xIyQmIjVogOTvvBooKs4jy3/ZkiTt\nh0y+k1Lw3+0Y8uWPQefll2VR5DRBOHKdpRhfOfzPH3EPnSERg86DB3kKWrZnBFct853dgnD20huS\nJPHw4UN0HvLQ45QFewZVH4AQEPrHRILOyF1A/pVrJR9RZOMDVlrzIFK1IcWM62cdJxDk885HdEI0\nCcaneRhL4ttkgjnyxEUGDUpXG1SUh6JmXM9YfZpQoEwRP7ulFZ+F1sN1q5o6Aw8P+eHkuPm5KdFB\nggR37wJ3M3Sk3kOL7c+lzsUrk/MusoIuM0UQMnoNDy/888u5Cz996vKiA6sNYEDV/ohCq+DnAZQs\nnKL7xsbfTGZyCukhv3d+7o+XCxatOLki2XyJDuU6ELozlN2hA1lyFGIL5sFz8BB5PSkVt0NRkoh4\nnMjb3x179o42KFKpooNb41qqVy6c9J0pzf1cgWWlyjjXXN8Zq2+mhSsWiEvLSVLevHLofvu21Ubb\n3bT2ls8QiGSRQ8qfM8KAqgMsEY25e6v0jVjmJi3ZNKp5LI3OraVzub6ZOn9WySkjm1yFWy/wB7Ic\ncpIgqlcubCUI5eHqpYyzWxCuwt7MZKMEUWWT3u///CNXmYu7B5LtDxM2l8/QefNandcoFVgKgaBU\nYBCv1Xo51X5p5SMePrH9MAvdGUr+e7FsWQF+ibCiKnxXMYGpeyfZPZeKsnFbSahyyF6als+XKwXh\nqmWmY+KM8kAOK8yrpz4pVJzEipUgKgre8YK1heDilzYn8g2oOoDFnRc/FUJAcRZXqMHCZuO5+sZl\nTOMuc7VuMAujv2Fxp68s++X3zp9mZGFrZVgAn0th7PsWSkXCwRIwrDMgUg/hVXEfFNXdlB5ykhgA\nRYvBjKvlALkngjAjz0w2WkY3GSVZHOYZy0/+NwyPd96EX5DXODj6unxgudfAPMQ1qfvJujsIyQSr\ntLDhOVIyoExjBlS7aukysodA2FwZljVrOPy1wD9e4mAJ6DAAniRN8rcewpudqKVLs47bRBJq5JD9\nKCF6gNwnCDNxiRL3ow3ciTJwP9qQbEmL2CGvYHiuDFwHVgASsih+9IBVGoi1M0s6No1E9rGJYHic\n5ogogeC1Oq8lH1V17x4MHgwvvoh/vMSaKlpaDYbIpGXVvHU+TAh2XQU5Z5Bb8hHgBpFEThODO6AE\nMZhxpSAUXdFMryfyq2/J374p/Ia8yF9fnn7sOzYR6i8BndU9GB7L2+0RJi8kmdaIqGWdFjKgck85\nInl0DRYthY8/g8hI0Ovhk0+Ia5qHQn9MIjwynJKBJZnefCady/XJ9nWb1CjCMThs7aaMoAksKXk1\neHbZiZwiCFUOmcPVEYSiJZGE14Y15H25nzy8tAowFDDPiSzVD2rMBJ8gOYIwCyLltiQ5mCl9BcJs\nLIBbSgdXnwMigF3IcoqSX5Nat4EFXyDK2Z5VbXJBsSF3GtUE6YskXLF2kyIjCVUO2Y8qiOS4gyAA\n4rv25OGareQb1B5OAW8DLYG2AKuSC6BUv+TRhW9p+WdItt+MCg0Zfv4YsYanVR+LGPQsjygJay/C\ncSxzHgy1aqOZPg1Nu3aWHIgkSanqlWiEPHs8LjF7aqq7WxSh1K4mUJgkVDlkP6oc3J+EZi25eyQc\n/0nj8V79I2wDtgMvALWBSkAJ5AhCl0J+Ol95u5UkBrRfiVeBLWxa9QFlzt+l7XUv6oYlojFclHfQ\nQFzHLsQOH0VAx1ZoUkxuTCkIM9kwHxFwP0EoHcVIIicIwl3koDQxmFEFYRv96lX4TZ2M9vo1jCVK\nEjNlGnG9+iXbx1S4CJFLfuDx62/hu+AzvNeukMt/nU3awQP4tLtc2rR4cQgMlHMIAE+ewFEgCu7f\nBc3wMrz42MSLlrPHg0aD1LQp0e27EtejF6bCckU9rSb9T/7sqErnbEE4q6tJybg8J6HKIftQqhxA\nWYJQUleTfvUqAkaPQPPk6Wgjk7cPUfO/TCUKa0RkJF7bf8Vr5w48Dh1EF3YlQ9eN8oTjReBYSR3V\nXhxFyJD3MATm4X508u6iAv46dDZEkbLLKbtyEs6UhCtzEWZckZNwmSRKvbwo26/rSNxFDKBsOYCy\nBAHKkkSBqmXRXUs92shQMoj7Jy+l+zwiIgKfqxfwvXsTzZ078mik+HgkIE7jQaJvABNOTOOExwMu\n54Wb/liW7SwVWIrLb1yx+ZBPuRAhyEJ4kmDCy0Njc46Hs3DXKELpknBJd5OH3n1XaFXl4DiUJgcl\nor1ue6ayve32kPLk4XGNehitlh9P+fD+ct6bNudYh0eG240C0pr0Fx2XfeuO5QZBuArF5CSUjioH\nx6JUQSgpigAwlihpM5IwlsjcDOa4RMnuCKPi/iWTlUi13p5WFJDWObMDNVHtXNxmxrWrcIeZ0fB0\ndrTSBdGoWD7FCkKJxEyZhsk7+cJ7Jm8fYqY4fgbzhOBpeOuSX0vps6WzQxC5OYoANZKwizuIAdwj\nagDlRg5Kx5ycftboJkfQ4wX5nLMOTOZG9DWK+5dkQvA0y3al4c6CcCdckrj2KlxOKtZ/XrZfNz24\ngxzcRQxm3EUQSutqUrFNdnUvOVMQmY0ick3iWomocnA87iIHFfchNwvCVeR6SahycDyqHFScQU4Q\nhDuSayWhdDm4mxjAveWgdjUpl5w0esndogjIhZJQ5eBY3FkMKsonuwWhdjOlJldJQqmCcDcxQM6S\ngxpFKA9XRA+qIGyTKyShysEx5CQxqCiTnCaHnECOloQqB8egykHF2bgq75AdgnDnKAJyqCSUKAdV\nDMpE7WpyLa5MSquCSB85ThJKE4Q7ySG3iEHF9bh6xJIqiPSTYyShyiFz5GYxZCWKSE8hIJXUuFoO\noAoio+QISShJEO4gh9wsBkeQshCQ7lo4AaNHAKiisIESxADZl6DOSYIAN5eEKof0o4ohOVmJIvym\nTk5WKQ5A8yQWv6mTVUlYoRQ5gCqIrOC2klCKIJQsB1UMtslqstpRhYByIkoSA2Tv8NacKAhwQ0mo\nckgbVQxp44jRTI4uBOTOKE0K1qiCcAxuJQklCEIJcqiwYwNNlszG/+4togsVZd+wceQf8pKrm6V4\nHDXcNWbKtGQ5CXBeISAlomQxQPZPjsvJggA3koSrBaEEOYAsiDafhOIR/wSAgDs3aTc7lMt5/XjQ\npZeLW6dMHD0XIjsLASkBpUvBjCtmTmenII6fvpNt17JG8ZJQ5ZCcVks/tQjCjDbuCUFzpqqSSIEz\nJ8rF9eqnSkFB5PTowVWCAAdJQgjRDvgM0AJfS5L0oSPO60pBKEkO1nkGz1s3bO5jb3tuRJ1FnX7c\nUQjW5PToAVwrCHCAJIQQWmAB0Bq4DvwthNgoSdKZrJxXFYTtJHRC0eJ43bxuc3tuRZXCs3F3GaTE\nWg62cnTnW3d1ynVzmyDAMZFEPeCSJEmXAYQQPwJdgUxLwlWCUIIcnjU6KXzsFJ4PfQNt3NMuJ6Pe\nm/CxU5zdNEWgCsE++tWr8H4vFM9bN0goWpzwsVNyXBdkysjBVo6uzSehAA4VhSuS00oQBDhGEsUB\n6wHi14H6KXcSQgwHhgNo/QvaPJEqh2dj/qMPmjM1Rz8MVBnYx1ZUkH/jagpafXjwunmd50PfAMgR\n7w173UpNlsxOlaPziH9CkyWzHSaJ3CwIyMbEtSRJi4HFAF6Fy0kpX8+tgsjMvIYHXXrliD98VQT2\nyWj3UNCcqcmiS3D/AQ3pyTf4372Voe0ZwVVDW5UkCHCMJG4A1rOISiRtSzeuEIQ7ysEdUUVgH0fm\nCXLSgIaMJKOjCxUl4M5Nm9uzglkQTQ5tZdC6hRR4eIf7+QqzrPtI9jVon6Vzp4XSBAGOkcTfQDkh\nxHPIcugL9E/vwblJEDlVDKoI0iY7ksY5YUBDZkYq7Rs2LllOAiDRy5t9w8Zlqg3W0UOTQ1sZtWwm\n+oQ4AAo9vM2oZTPl6zpBFEoUBDhAEpIkGYQQ/wdsRx4Cu1SSpNPpOTa3CCInyUEVQtq4ahSRuw5o\nyFq9EeQAACAASURBVOoQVnPewRGjm1J2Lw1at9AiCDP6hDgGrVvocEkoVRDgoJyEJElbgC0ZOSa7\nBaHKIWOoMkg/Shhe6k4DGhw9t+F8665ZSlLbyz0UeGj7wW1ve2ZQshzMuGTGtbe3R7ZeTxXEs1Gl\nkHGUIAdrlD6gwRUT39LiWYnp+/kKU+jhbZvbHYE7CALcYFmOrJLdgnAXOahSyDxKk4OSUZoYIP2j\nlpZ1H5ksJwEQ56lnWfeRWW6DuwgCcrgkslMQSpeDKoWso8ohfShRDGYyMqzVnHdw9OgmdxIE5FBJ\nqNGDKgVHowoibZQsBsj8nId9Ddo7LEntbnIwk+MkkZujB1UMzkEVhG2ULgZQTq0HdxUE5DBJ5EZB\nqGJQyU7cQQygysGR5BhJZJcgVDnkLtQoQhVDZskJgoAcIoncIghVDNmLKwWRf+Nql855cBcxgCoH\nZ+P2ksgOQahyUMlO8m9cnWz2dHat6OpOYgDlyQFyniDAzSWR0wWhysF1uDKKyM4VXd1NDKDKIbtx\na0k4G1cJQpVD7sbZK7qqYnAcOVkOZtxWEs6OIlwhCFUOysDVyWpnreiqysGx5AZBgJtKIqcJQpWD\nijWOXNHVHcUAqhyUhNtJIicJQpWDii0csaKrO8pByWKA3CcHM24lCVUQKs7G1V1NZjKzoqs7igFU\nOSgdt5KEM8kuQahyUHE07igHpYsBVDmYcRtJODOKUAWh4o6ocnAOqhyS4xaSUAWhkh0opavpWbib\nHNxBDKDKwR5uIQlnoQpCxd1wJ0GocsgZKF4SzooiskMQSpCD3kPgp9eiFWCUICbOSFyilOV9VbIX\nd5GDu4gBVDmkF0VLwhW1qR2FUgQR4K1FIwQAOgEB3log9cM/I/vmRJTa1aTKwfGocsgYipaEs3B2\nFKEEQQD46Z8+9M1ohBwtxCUaMr2vSvagdEG4kxjA/eVw+8xZl1xXsZJw124mpQgCQCvsby/gr7N0\nJ4mHD9CeC4MbN+DuXYiMhLg4kCS0np74ePphLFgQU8lSGMqUQ8qTJ3tvJBeiZEGocsheXCUHM4qV\nhErWMUpyt1Ey4uIQBw6g27uXwEOHCPj3GJp7d+2eQwABKbYZSj1HYoNGxLduS3ybDkgBKfdwL5TU\n1aTKwTG4uxjMuFoQkMskkZuiCJATzwHeWjSxsbB+PaxeDb/9Bk/kNYFE0hdeQMk8UDEYihSDPHlA\nrwcpEa5sgNtnIRK4C9wCXdgVdGFX8P5pBXjAk54DSXzzLXzq1FCT3llAqYJQ5ZD9KEEOZhQpCXdM\nWCtNEACJp86QuGQBXj+ugBirT8vVq0OLFhAcDLVqwsEyICKgVADUmAg+QRAbDscmQliKN6sRuAac\nAY4C58D7x+WyMAYPhtmz0RUokKuS3o5AiYJQ5ZD9KEkOZhQpCWfgzChCaYLwuXQWvxkfoFm39unG\nBvVh4CDo1g2KWy05/fhqUjgBhK2Sv9JCCyvyQWhFCC8HjSNhyfXGVNjwF3z/PWzfDqtXo2nc2C2S\n3kroalKaIFQ5uAYlCgIUKAl3jCKUgj42ioDpU9AsWgQmE3h5wf/+B6NHQ6VKIElgPYLJ8FiOFqwp\n1Q9qzEwRTTwVx4ooGH4XYpMChH2BUKvAP6wcMpOuH22AffugVStYtw5tu3bOv2k3R0mCUOXgGpQq\nBzOKk4QzyA1RhN/+Xfi9+hLcuAlaLYwYAZMnQ9GiT3cSAkwGEBpWnFhB6K5JhEdeI0jvw4zAWAZU\n7QcNvgWtl7y/b2n5Z7CIIvTBU0GYiU2M5Y2Ln9P1j0vw+uuwaBG8+CIcPIi+XGW1y8kOShGEu8gh\nJ4kBlC8HM7lCEjkaScJ33if4TZ0sRwr168OSJVC1qu39hYYVx79j+JbXiU2MBSAsLpbhCVrQN2WA\nWRBmtF6s8GlB6JVVhBvA3uM+PDIcdDpYuBAeP4ZlyxBDhuD319/EJTrudh2JK7ualCAIVQ6uwV3k\nYEbj6gZY425dTS6PIkwmAsa+jv8Hk+Sfp0yB/fvtCwJAMhG6e6pFEGZiTUZC981KtfuKkysYvvUN\nwtIQBEBQYEn5GyHgyy+hVCk4fhztj8/IceRCVEGkj+On76iCUAA5PpJwRa3qbEGS8H93LD5LF4MH\n8P2X0O/VZx6D0Mqf+m0QHnkt1bbQnaGphJISH42WGc2nP93g6yt3dQ0dKkc1Xfs9626ynf9v78zj\nrCjOvf99ZgZkhyBqZB1fMMF9QyVBIorXfXn1GhPvKKJ5w1W81+TG5VWJJOKu9ybGa1DR+LoENWrE\n3SugqAmiERRBBIEYNlcEHEZgYJbn/aOqZ/qc6T7nzMyZ093n1Pfzmc909+murq6uql8/9dQSlRUR\ntUAkRRyKjSSKg0esLIkkEbUV0euhe+k+7ffQuTM89TAcWQnayMQXJlIxpQK5VqiYUsHEFyY2XyTC\n9A8eoUyCX/vg3oNbHAsTFABBGNJjV6adMo2qA85N/fGss6BTJ2TuX5Hq6rY8YtERpUC8sXxj7AWi\nWC2HJAsElIAlUYx0W7WCbldcanbuvx9OrgJVJr54MXfNv6vpvAZtaNqfetJU03T03AQatKFlmJ26\nccPYG1ocH9x7MKurV7c4PqT3EFb9fFXLHlMe3bvCHj1g+SYqli+j7tDD2/awHUAUVkTUAhFXik0U\nPJIuDH7aZUmIyG0iskxEFonIDBGJ1aQ+xdrU1OPqK2D7djjvPKiqMgdFmLZgWuD53vGwpqNyKTfW\nwH5VLX67YewNdOvULeVYiqAECQSAlEHXTQCUbdyQy2MVLVEJRJyth2K0GqA4LId02tvcNAvYV1X3\nB5YDV7U1oCQ5raNsaqpY/D5lL75g2v1vvTXltyALwTteeXtloEUA0KiNgQIBULVfFdNOmcaQ3kNM\n81LvIaGCksLWNWDH0WnnzpnPLWKiFIg44sQhebSruUlVZ/p23wLObF90HABPLXuUm+Zewyc1axnQ\ncxBXjbqOM4Yb52+Px6ebk84/H3bdNeW6cikPFYrV1asRBA3oo+T5IqYvns6kVyaxpnoNg3sP5sQ9\nT+TFFS827T98xsPB4hA2SM/WBQ39B7S8JiIK2dTkBKKZYhQGKK5mpTDy6bi+AHgpj+GVJE8te5TL\nZl/Eupo1KMq6mjVcNvsinlr2KF06CTu9/qo58Yc/bHHthEMmZAxbUYTU5iGv6cjzV6yuXo2irK5e\nzV3z70rZP//p8+l3az/Kri2j8vZKpi+ebgTis5lmeg9tNP/f/im8+yh8BXSBhqF75idxHBmJY/OS\nsxyST1ZLQkRmA98O+GmSqj5jz5mEaVyYniGcCcAEgK59g4JzANw09xq21af6DbbVb+WWeddw0Ygq\nZKnNmIcc0uLaqSdNBYwPIsyiUJQhvYc0WQc3jL2Bqv2qqLy9MmtX17rGOjZsM/6F1dWrmfDcBKjf\nRtXSn7Y8+U3zr/bY080guxhQzFZE3MQBitN6KBVh8JO19KrqMZl+F5HxwMnAWFUNHW+lqtOAaQB9\nKvd28zSE8ElNy7EKAGur1yK1tVBXZ6bx7h7sF5l60lSmnjSVytuHsDqg++rOXXc2vZLSCPNXZGJr\n3VYmvX4dVT9rSJ3nqQ7jrQK2/cu4VoebdEpdIJw4FBft+sQTkeOBK4AjVTXzZ6gjJwb0HMS6mpaV\n++Deg404lJWZVeO2bzcT+Pnw+xT6du0b6KPYvH0zE1+YmOJruGHsDRn9GZlYU73W9GTqXgmH32sO\n3vkobIC6vfdh+7EntDrMjqBQVkQpC4QTh+KkvT6JO4GewCwRWSgid+chTiVHl05Cv54V7NarghvH\n3kjXipAup+XlMHSoObhoUco56T6FDds2BFb6dY113D3/7hRfQ9jYiVxIGYBX0R06j4c/m92a6281\nohYxTiA6lmL0O5SSzyEb7e3dNCxfESlVunQSs3qc7R107gFViMCkV69mbfXaFL8BAGPHwooV8OST\ncOihTeHkMn2GR3oPp611W0MtiXIpp1Eb6du1LzU7atjRsKPptxYD8DZsgKp/gzrYWnUeO47+p1yT\nwdFK4iQQxYQThpZIBjdCh9Gncm8d88uHUo51xDiJjhpMl89xEv16VlBR1nJAmqoiAQPVdN485Pvf\nh169YOVK2GUXAMquLQvs3tpeLhpxUZNDPFMX2X06D2TOI53p98Hfqdt3fza+/Doa4jcpJMVoRcRB\nIIpJHJIkDLUzL1+gqiMKec94dDspYcpDBiwDNKo2WRhghIORI+G448wKcBMnwuOPg0jo9BnpYyPC\nxkqE8eKKF5u2q/ararJovOatrXVb2X0zPPjIWvp9DjW792Pbn56OhUAUikIJRBzEAYpHIJIkDlES\nfYNxidMQUl83KGze1kB9o6Kq1Dcq1dtsc9DUqdCjh2lymjwZCJk+o6IrF4640DdaejAXHnxBi/PS\nx074CZvgz2ve+sEqmD8NDv4cVvSFYy/oTOOAgbk9fAdTCCuilASiWHwPzt/QOpwlETHf1Dak+CTA\nWBDf1DZQW6ct1ojuVN5Itz32QKZPh9NPh+uvh82bqfrP/wRobg7q1Z8bBg2maswVcOKdvi6qf2DU\nyQ8z6dVfNvWC8sY+BBE0MyzAhi9X819z4OdvmS+N14fAmWfBhk6ftT9R8oATiPxSLOLgaD1OJNrA\nqvVb8uaXMEt7NtCjSznlYiwITyCCqKltBKDbKafA9OnIuHFwxx3w9ttU/eEPVHljILQRHi2HZ/Zo\nEUbV0COo2n9VU5NRGIK0nBm2rg4eeICVvy9nt80NNAhcNxquPRIaymFgz0FtSYa8EuWKc/kmaoFw\n4uCITXNTRxSGqBd4yZXaOuWrmnq+2FzPVzX1WdeErqlt5IvN9Xxxwj9T/8qrMGAAvP02HHAA/Ou/\nwscfG8shjIVXQ/2WjD2iBOHCERc296pav95MKDh0KEyYwG6bG3hnYBmH/RQmH20EomtFN64adV1b\nkyFRFCJvOYFoH65ZKT84SyLhfHPISHotXkzZ1VfDtGnm79574bBvw0HAgZiRLH5WmyVFMy0o9PDp\nD1HV5wi471546gmYNQfqbdPXXnvB5Mks36uODXOuQarXMKj3IK4/6kZO2fNHWUWuIymWZqYoBaIY\nxMGRP2LTBRaS1Q0Wol+dzqNLJzHNVR8tQ2+6mbLH/gh1plkKEdh3OFTugD5/h12B3kA3OOIz+KQe\nuu+AnbfBgM0wdBMcvh5O/gJY77uJQN1xJ1J20YWUnXQiUl4eGJdGVTZvC28u60iKRSAgOpFwAhFv\nXBfYDuDNTzcW7eJDHk0O7v7D4L/vo+v1t9DzuT9TNmMG/OUvsHgpLG553V+zBdwFao88ke3HnwSn\nnUbPyv4tuuSmj+UoEyNY6Q73jsYJRPtJskAUuzhESaxE4o3lGxO1+FA+Hdj5pPvA3Si7+GK4+GLY\ntg3mz4cFC2hctoy6v6+ibMNXSHU15XXb2dKwjc8aqvmycx1f9+3GHgcdxXePOINNww+gfq99zFQg\nmEF/ZWmCEDTYDzKP/egInKO6/SRVIJw4dDyxEokkEkehSKmku3aF0aNh9GhElU2bzRf+U8se5ZZ5\n17C2ekOLqT9UlfrN9eFhZiFs7EdHUEwjqqMQiCSIw6nL5nD53IfoX/MVn/bsx22jxjGt0S03UChi\n07upI0lKL6d8kWmAHjQvbLSmek3KJH/TF08PvT4szHSfljfGoxA4gWgfSRGIm2bfycCa9ZShDKxZ\nz40z7+CsT9+NOmolQ0mIREcTt+aOb2obaMxQeQctbLS1biuTXpkUWsmHhbl1R2PKqPBCOa2dQLSP\nJAgEwOVzH6Jb/faUY90b65iy0i2CWShiJxIdVWA6urDHSShq67TFlB7+yjtsYaM11WtCK/mwMGtq\nG1s1xiMfOIFoH0kRCID+NV8FHh9U+3WBY1K6OJ9EHomTfyJoSg+PsIWNBvQclLGSzxRmoXAC0T6S\nJBCff7iUtV16MyRAENZ26RNBjEqT2FkSkFxrAuJlUYRx1ajrWixslITR0k4g2kfSBAJg8rAT2FLW\nKeW3LWWdmDwsHiselgLOkugA4mRRBHHG8LMB45v4pGYtA3oO4qpR1zUdjxuFFF4nENGS3qX18f4H\nAzBl5UsMqv2atV36MHnYCU3HHR1PrEZcp9NRYyYKNbguzkKRFJxAtJ+kCoSjJW7EdYEo1CjsuFsU\ncccJRPtw4uDIB7H0SXh0ZKEq1NiJJPgo4saq9VucQLQTJxCOfFGSlkSh8So8Z1VkJgpBLRaBGP3W\nS5w7Yyr9Nn7RNCr52eFHdfh924oTh+QQa0sCisOa8HBWRTCFthw8ikkgLn74Rnbd+HnTqOSbZt/J\nqcvmdPi924ITiGQRe5GA4hMKJxbNRJUWxSIQAOfOmEqXHbUpx7rVb+fyudk7hxQaJxDJwzU3Ec10\n4qXcBBWlSBbjdN/9Ngb7H8JGK0eBE4fkkghLAjq+0EU1CWApWRZRP2sxCgTApz37tep4oXECkWwS\nIxJQvEIB0VegHUkcnq1YBeL9JV9w26hxbK3YKeX41oqduG3UuILGJQgnEMkncc1NHb0wUdQr2fkr\n0yQ3RUUtCh6FFP6ourl6vZjS11yIsneTE4fiIXEiUQi8iiXqZU+TJBhxEQU/pSAQHs8OPyo2XV6d\nQBQXiRSJQi1zGrVV4Se9Eo5aNOIoCh7FLA4Q74FyTiCKj0SKBJSmUPgJq6TzLR5xFoMgnEBEhxOI\n4iSxIgGFFQqIvvkpF5JWqeeLQnc6cAKRihOI4iVRvZuCKGRhLbW1spOCE4jo+PzDpU4gipxEWxIe\nhbIoIFlWRbEThWhH7aCOE04cSoOiEAkorFCAE4soKQVxACcQjniQ+OYmP1EUZNcEVTje/HSjE4gY\n4ASitMibSIjIpSKiIhLpXABvLN9Y8EIdVeVVKkSZvk4gUnECUXrkpblJRAYBxwJr8hFePih08xOk\nWhWuGap9RC26UYgDxFcgnDiULvmyJH4LXAEUfsHsDERV0MFZF20lDunmBCIVJxClTbstCRE5DfhE\nVd8XkUznTQAmAHTt++323jZnvAJfaKvCw1kX2YlaFDyi/KhwAuGIKzmJhIjMBoJq9knA1Zimpoyo\n6jRgGsBOu+1ZcIsjiuandJxgNBMXYfBw1kNLnEA4IEeRUNVjgo6LyH7AHoBnRQwE3hWRw1T180xh\nvr/kCw7YZ7dWRrd9xEEoPEpRMOImDOCshzCcQDg82tXcpKqLgV29fRFZBYxQ1ZyWxIpKKCC65qcg\n0ivPYhGNOIqCR5TiAE4gHMkh8sF0UQgFxFMsPJIoGnEWBD9OHDLjBMKRTl5FQlUr23JdVEIB8WqC\nCiNTBVxoAUmKGAThBCIcJw6OMCK3JDyiFgqIp1WRjSRX2oUianEAJxCO5BIbkYBohQKSLRaOljhx\nyI4TCEc2YiUS0FyonFg42kIchMHDCYSjGIidSHhEbVWAE4sk4cShdTiBcORKbEUC4iEUkFoBOcGI\nF3ESB4i/QDhxcLSWWIsExKP5yY+zLqInbsIA8RcHcALhaBuxFwmPuFgVHs66KCxxFAYPJxCOYiYx\nIgHxsyo8nGB0DHEWBkiGOIATCEf7SJRIeMTNqvDjBKN9xF0YIDniAE4gHO0nkSIB8bUq/DjByI0k\nCINHUgTCiYMjXyRWJDySIBbQsiIsZdFIkih4JEUcwAmEI78kXiQ8kiIWHkEVZTEKRxIFwU+SxAGc\nQDjyT9GIhEec/RXZCKtQkyAeSReDdJw4OByGohMJSJ5VkY1cKuCOFJJiE4BMJE0cwAmEo2MpSpHw\nKDaxyEQpVeQdQRLFAZxAODqeohYJD38FUAqC4cgdJw4OR2ZEVQt/U5H1wOqC37gl/YCcllotAVxa\nNOPSwuDSoZm4pMUQVd2lkDeMRCTigojMV9URUccjDri0aMalhcGlQzOlnBZlUUfA4XA4HPHFiYTD\n4XA4Qil1kZgWdQRihEuLZlxaGFw6NFOyaVHSPgmHw+FwZKbULQmHw+FwZMCJhMPhcDhCcSIBiMil\nIqIi0i/quESFiNwmIstEZJGIzBCRPlHHqdCIyPEi8pGIrBSRK6OOT1SIyCARmSMiH4rIEhH5WdRx\nihoRKReR90Tk+ajjUmhKXiREZBBwLLAm6rhEzCxgX1XdH1gOXBVxfAqKiJQDvwdOAPYGzhaRvaON\nVWTUA5eq6t7ASODiEk4Lj58BJTnMveRFAvgtcAVQ0h58VZ2pqvV29y1gYJTxiYDDgJWq+rGq7gAe\nA06LOE6RoKqfqeq7drsGUzkOiDZW0SEiA4GTgPuijksUlLRIiMhpwCeq+n7UcYkZFwAvRR2JAjMA\nWOvbX0cJV4weIlIJHAS8HW1MIuV2zIdkY9QRiYKin+BPRGYD3w74aRJwNaapqSTIlBaq+ow9ZxKm\nuWF6IePmiB8i0gP4M/BzVd0cdXyiQEROBr5U1QUiMibq+ERB0YuEqh4TdFxE9gP2AN4XETDNK++K\nyGGq+nkBo1gwwtLCQ0TGAycDY7X0BtB8Agzy7Q+0x0oSEemEEYjpqvpU1PGJkFHAqSJyItAF6CUi\nf1TVcyKOV8Fwg+ksIrIKGKGqcZjpseCIyPHAb4AjVXV91PEpNCJSgXHYj8WIwzvAv6jqkkgjFgFi\nvpoeBDaq6s+jjk9csJbEZap6ctRxKSQl7ZNwpHAn0BOYJSILReTuqCNUSKzT/t+AlzGO2sdLUSAs\no4BzgaNtXlhov6QdJYizJBwOh8MRirMkHA6HwxGKEwmHw+FwhOJEwuFwOByhOJFwOBwORyhOJBwO\nh8MRSqxEQkTGiMj323DdCBG5ox33DZwBVUR2trNhfiMidwZcd6WIVInIL+yMmYtE5BURGeI75zwR\nWWH/zvMdP0REFtsZR++wfdMdPkRkVaFn5s33PW2eXmhnU3097be7RWSUbz9lNmIx3GHzyCIROThf\n8SoWRGR8UNnM1/mFREReE5ERWc7ZSUT+ZPPE23baFP/vC0RkJ9/+syLygW+/r4jMsvXRLBH5VrZ4\n5UUk7AyaofsZrksf8T0GCBSJgHObUNX5qnpJLvcMIWwG1FrgGuCykOuOA2YC72EG4u0PPAncauPc\nF/gVcDhmArlf+V7KXcBPgT3t3/HtiH8LbAVTlnYs1/eS03lxIOq4Zrq//diYCpyqqvsAP0w7ZSRm\nMsWw2YhPoDl/TMDkmbySXq4ylbO2nFcqFDA9fgJsUtVhmMlJb/HFYQ/MXHTb7f4ZwDdp118JvKKq\newKv2P3MqGrGP+Ac4G/AQuAeoNwe/wb4L+B94AhglY3wu8CPgQMxBWARMAP4lr3uNcyEWfMx0xF7\n96kEPseMdl0IjAYeAO7GTC72G0xFOw9TKb8JfNdeOwZ43m7/Grjf3udj4JJsz5j2vKdjpiLwHxsP\n3Jl2rBcwN+D6g7zjwNnAPb7f7rHHdgeW+Y6nnOc7fiowJSSel2NGBS8CrvWl4UfAQ8ASYEjAexpr\n02+xTaed7LXp7+8S4EMb/mMB9x8PPAX8D7ACuDXteRYDHwC32GMXArcFpSnheWwV0C/g3unPdAjw\nOrAAMxhud3veoTb+C4HbgA+C3ifwPDAm/Z7A0zbMJcCEsPtnyEsTgetDftsLM2DP238SOCDt/vcA\nZ/vO+ch7trSwXgT6Bxwfat/PAuAvwHB7/AFSy9WvgYeBucCjmOkn/p99h+8BR/nS7VngVZveuwNv\n2PT9ABgdEIdVwLWYfLXYF4e+Nn0XYeqJ/TEfrauAPr7rVwC7Abtgpgl5x/6NCiub9nhYXTEeeAZT\nP6wAfuUrO0uBe+37ngl0tb/lVJfZdL3Lnvsxpl6634b7gC9ud9lrlmDLri+8EVnqp5eB79ntCuAr\nmse7XQRMtNs9gL9ipr3/ICgP2ff3UdY6MUuE9gKeAzrZ/anAOLutwFlpmeEK3/4izBQPAFOA230J\nMTXkfr/GDHv39h/AFGCv0ugFVNjtY4A/2+0xpIrEm8BOQD9ggy/+gYUpLQ7PAecEVIjpInEGARU4\nZuTyL+32Zd623feskhHAbN/x0V78c/nDfHFOAwRTsJ4HfoDJ6I3ASN+5Te8JU/jXAt+x+w9hJm8L\nen+f0iwgfQLiMB5TEHrbcFdj5j7qj/ka3gWTiV8F/rfdX+m7/iVMBZ8pj60iWCT8z9TJvu9d7P6P\ngPvt9gc0F6ibab1I9LX/u9qwdg7J+1Mw1kJ6PG/HrFHxGqaiHuf77RfABXb7NOB3Afd/Hp8IYb78\nMlYiafd/BdjTbh8OvBpSrn5t4+dVipf60nC4fZ9dbLqt86XLpZjJIQHKgZ4BcVgF/LvdngjcZ7f/\nm+YK+mhgod3+HXC+L86z7fYjXloAg4GlYWUzS10xHvgM2Nn3Xkdgyk49cKA973FsPUCOdZlN18cw\n5fI0YDOwH6aMLvCF3deXZq8B+/vCG2G37wt61za+A337f6c5vzwD/C+7/VvMB28lqSLxtW9b/Pth\nf9lMpLGYr7R3bJN5V+BL+1sDRtn9/AlARHpjKhavDfZB4In083LkCVVtsNu9gQdFZE9MQe0Ucs0L\nakyu7SLyJeZLZJ2qZpxaoJUzoB6P+dryX38OJsMdmcP17eFY+/ee3e+BaZJYA6xW1bd85/rf03eB\nf6jqcrv/IHAxpjKD1PeyCJguIk9jvviCeEVVqwFE5EOM5bIz8Jra+Z9EZDrwA1V9WkQ+FpGRmC+4\n4Zgv14sJz2NhpD/TvpjpRMAUvM9sU09PVZ1nz3sEM3lha7hERE6324Mwabwh7f6o6uSQ6yswzzYW\n81zzROQtm/7HAeeLSDc6YDZiMTO4fh94wufu2sl3ir9cATyrqtvs9hGYShxVXSYiq4Hv2N9mqepG\nu/0OcL+YyQCfVtWFIdHxJghcgPm48u7xz/Yer1r/Xy9MHpyMKVs/pjlPHgPs7XuWXvYZw8hUV8xS\n1Q0AIvKUjcvTmLLhPcMCoLINddlzqqoishj4QlUX2/sswVTYC4GzRGQCJn/sjvnaX+QPRFX/RLjp\nCwAABRhJREFUT4Zna4GIdMaIx8ciciAwVFX/I91nkXYPFRHNFnY2kRDgQVUNWqWsNi2TAWzJdsNW\nnpd+7nXAHFU93T78ayHXbPdtN5DDbLfS+hlQD8OYd971x2CmHz/SChSYprMxvmsG2jh/QuqiPq2d\ncVSAm1T1nrRnqKRl2ga9pzD8156EsU5OASaJyH7avCiRR2vT+THgLGAZMMNm0kx5LAz/MwmwRFW/\n5z9BMi+/Wk+qP65L+gliJnM7BmOJbBWR13zn5Zqm64ANqroF2CIibwAHiMg6TMXzqWSYjZj2zUxb\nhvlKPDDk9/R80uqyq6pviMgPMHnlARH5jao+FHCNl09yySPzgGEisgvGAr3eHi/DWMi1/pMz9PfI\nVFekl29vPz0/d80SV2iZbl4YjWnhNQIV1m9wGXCoqm4SkQcIyH8Z8PLEOusH6Y35cDka07wE8D1g\nhJhJSyuAXUXkNVUdA3whIrur6mcisjvZP8iyOq5fAc4UkV2hyTM+JFug9utyk4iMtofOxbRhZqMG\nM8lcGL1pLiTjcwgvJ8TMgHoFpslgaw7n74PxKTTY/YMw7cenqqo/0V8GjhWRb1mH9bHAy6r6GbBZ\nREbaSnIcxlRMv8/pInJTQBReBi7wvqREZID3jrLwEebraJjdD3wv1uE9SFXnAP8Xk+6Zvtr8/A04\nUkT6Wafu2b57zMCY4WdjBAPamMfSnmkXEfmevb6TiOyjql8DNSJyuD3vx75rVgEHikiZdRgfFhBu\nb4yDcKuIDMc4mVvLM8ARIlJhLYbDMe3TRwFzAFR1saruqqqVqlqJEZaD1UxX/ywwznZCGAlU27yT\ngpgedSkLJKlZ/+EfIvJDe46IyAE5xvsvQJW97juY5p2PAu47BPO1fC+meaQ1va/89xgDfKWqm+0H\n2gyMr2Sp98WP8RH8u+/eYeLnkamu+Cebz7pihGhuWCDtqMvC6IURlmoR2Q3TOaE1PAucZ7fPxDQh\nKqZl4yUb57tUtb/NT0cAy61ApF9/HgH1TjoZRUJVPwR+CcwUkUWYXkC75/gw5wG32esOxLTlZeM5\n4HQxXQZHB/x+K3CTiLxHG9bCEJEXRaR/wE+hM6BaNf4NMF5E1olZ6/cEjEPQ4zZMJfqEvf5ZAGuW\nX0ezs22Kz1SfiClYKzHtikErwQ3FtGumoKozMc0n86xZ+ySZxdW7rhY438ZzMebrJmi213Lgj/ac\n94A7bKWbFVuJXYmpBN8HFqhd0EhVN2EqySGq+jd7rD15DDVLjZ4J3CIi72PMea+H3E+Ae0VkIdAd\nqLbH5wL/wDjm78A4VdP5H8yX31KMP+OtgHMAEJEpInJqQNyW2nAWYcTzPlX9gJb5J4wXMX6flRiH\n6sSAe5cBw4CN6b9hKuGf2HRZQu7LsU4Fyuz7/xMw3mcd+xmDsYDew/iCfpdj+GD8IIfYd34zzRUX\n9p7nkNqUcwnm63iRmKbNC7OEn6mu+BumuXARxlcxP0tYbanLAlGzCuZ7GGv6EUIESkTuk+DusH8A\ndhaRlRi/ltc7aQy5idfNGJFcgbGUb852gZsFtg2IyCyME7LFV12e7/NH4D+0BNd3yAci0kNVv7Hb\nV2J6dfws4mghIu8Ch6tqXR7C2hfjAP9F+2PmSCJi1uC+V1Vba5XkFr4TCUexIiI/wox5qcD0vhrv\nBNfhaB1OJBwOh8MRSqym5XA4HA5HvHAi4XA4HI5QnEg4HA6HIxQnEg6Hw+EIxYmEw+FwOEL5/3fv\nymR3kI6TAAAAAElFTkSuQmCC\n", 136 | "text/plain": [ 137 | "" 138 | ] 139 | }, 140 | "metadata": {}, 141 | "output_type": "display_data" 142 | } 143 | ], 144 | "source": [ 145 | "\n", 146 | " \n" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "metadata": {}, 152 | "source": [ 153 | "Introduction\n", 154 | "\n", 155 | "The goal of anomaly detection is to identify cases that are unusual within data that is seemingly homogeneous. Anomaly detection is an important tool for detecting fraud, network intrusion, and other rare events that may have great significance but are hard to find.\n", 156 | "\n", 157 | "Outliers are cases that are unusual because they fall outside the distribution that is considered normal for the data. The distance from the center of a normal distribution indicates how typical a given point is with respect to the distribution of the data. Each case can be ranked according to the probability that it is either typical or atypical.\n", 158 | "\n", 159 | "Anomaly detection is a form of classification and is implemented as one-class classification, because only one class is represented in the training data. An anomaly detection model predicts whether a data point is typical for a given distribution or not. An atypical data point can be either an outlier or an example of a previously unseen class. Normally, a classification model must be trained on data that includes both examples and counter-examples for each class so that the model can learn to distinguish between them.\n", 160 | "Implementation\n", 161 | "\n", 162 | "Scikit-learn implements One-class SVM algorithm, which detects the soft boundary of that set so as to classify new points as belonging to that set or not. The class that implements this is called OneClassSVM. \n", 163 | "\n", 164 | "Anomaly detection is applicable in a variety of domains, such as intrusion detection, fraud detection, fault detection, system health monitoring, event detection in sensor networks, and detecting Eco-system disturbances. It is often used in preprocessing to remove anomalous data from the dataset. In supervised learning, removing the anomalous data from the dataset often results in a statistically significant increase in accuracy. \n", 165 | "\n", 166 | "So" 167 | ] 168 | } 169 | ], 170 | "metadata": { 171 | "kernelspec": { 172 | "display_name": "Python 2", 173 | "language": "python", 174 | "name": "python2" 175 | }, 176 | "language_info": { 177 | "codemirror_mode": { 178 | "name": "ipython", 179 | "version": 2 180 | }, 181 | "file_extension": ".py", 182 | "mimetype": "text/x-python", 183 | "name": "python", 184 | "nbconvert_exporter": "python", 185 | "pygments_lexer": "ipython2", 186 | "version": "2.7.6" 187 | } 188 | }, 189 | "nbformat": 4, 190 | "nbformat_minor": 2 191 | } 192 | -------------------------------------------------------------------------------- /ImageClassifier_GPU.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Building an image classification model using very little data \n", 8 | "\n", 9 | "Based on the tutorial by Francois Chollet @fchollet https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html and the workbook by Guillaume Dominici https://github.com/gggdominici/keras-workshop\n", 10 | "\n", 11 | "This tutorial presents several ways to build an image classifier using keras from just a few hundred or thousand pictures from each class you want to be able to recognize.\n", 12 | "\n", 13 | "We will go over training a small network from scratch.\n", 14 | "This will lead us to cover the following Keras features: \n", 15 | " \n", 16 | "- fit_generator for training Keras a model using Python data generators \n", 17 | "- ImageDataGenerator for real-time data augmentation \n", 18 | "- layer freezing and model fine-tuning \n", 19 | "- ...and more. \n" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "## Data" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "Data can be downloaded at:\n", 34 | "https://www.kaggle.com/c/dogs-vs-cats/data " 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "### Folder structure" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "```python\n", 49 | "data/\n", 50 | " train/\n", 51 | " dogs/ ### 1024 pictures\n", 52 | " dog001.jpg\n", 53 | " dog002.jpg\n", 54 | " ...\n", 55 | " cats/ ### 1024 pictures\n", 56 | " cat001.jpg\n", 57 | " cat002.jpg\n", 58 | " ...\n", 59 | " validation/\n", 60 | " dogs/ ### 416 pictures\n", 61 | " dog001.jpg\n", 62 | " dog002.jpg\n", 63 | " ...\n", 64 | " cats/ ### 416 pictures\n", 65 | " cat001.jpg\n", 66 | " cat002.jpg\n", 67 | " ...\n", 68 | "```" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "### Data loading" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 1, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "Collecting pillow\n", 88 | " Downloading Pillow-3.4.2-cp27-cp27mu-manylinux1_x86_64.whl (5.6MB)\n", 89 | "\u001b[K 100% |################################| 5.6MB 321kB/s \n", 90 | "\u001b[?25hInstalling collected packages: pillow\n", 91 | "Successfully installed pillow-3.4.2\n", 92 | "\u001b[33mYou are using pip version 8.1.1, however version 9.0.1 is available.\n", 93 | "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n", 94 | "Using TensorFlow backend.\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "##This notebook is built around using tensorflow as the backend for keras\n", 100 | "!pip install pillow\n", 101 | "!KERAS_BACKEND=tensorflow python -c \"from keras import backend\"" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 2, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stderr", 111 | "output_type": "stream", 112 | "text": [ 113 | "Using TensorFlow backend.\n" 114 | ] 115 | } 116 | ], 117 | "source": [ 118 | "import os\n", 119 | "import numpy as np\n", 120 | "from keras.models import Sequential\n", 121 | "from keras.layers import Activation, Dropout, Flatten, Dense\n", 122 | "from keras.preprocessing.image import ImageDataGenerator\n", 123 | "from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D\n", 124 | "from keras import optimizers" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 3, 130 | "metadata": { 131 | "collapsed": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "# dimensions of our images.\n", 136 | "img_width, img_height = 150, 150\n", 137 | "\n", 138 | "train_data_dir = 'data/train'\n", 139 | "validation_data_dir = 'data/validation'" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### Imports" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 4, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stdout", 156 | "output_type": "stream", 157 | "text": [ 158 | "Found 2048 images belonging to 2 classes.\n", 159 | "Found 832 images belonging to 2 classes.\n" 160 | ] 161 | } 162 | ], 163 | "source": [ 164 | "# used to rescale the pixel values from [0, 255] to [0, 1] interval\n", 165 | "datagen = ImageDataGenerator(rescale=1./255)\n", 166 | "\n", 167 | "# automagically retrieve images and their classes for train and validation sets\n", 168 | "train_generator = datagen.flow_from_directory(\n", 169 | " train_data_dir,\n", 170 | " target_size=(img_width, img_height),\n", 171 | " batch_size=16,\n", 172 | " class_mode='binary')\n", 173 | "\n", 174 | "validation_generator = datagen.flow_from_directory(\n", 175 | " validation_data_dir,\n", 176 | " target_size=(img_width, img_height),\n", 177 | " batch_size=32,\n", 178 | " class_mode='binary')" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "## Small Convolutional Neural Network" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "### Model architecture definition" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 5, 198 | "metadata": { 199 | "collapsed": true 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "model = Sequential()\n", 204 | "model.add(Convolution2D(32, 3, 3, input_shape=(img_width, img_height,3)))\n", 205 | "model.add(Activation('relu'))\n", 206 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 207 | "\n", 208 | "model.add(Convolution2D(32, 3, 3))\n", 209 | "model.add(Activation('relu'))\n", 210 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 211 | "\n", 212 | "model.add(Convolution2D(64, 3, 3))\n", 213 | "model.add(Activation('relu'))\n", 214 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 215 | "\n", 216 | "model.add(Flatten())\n", 217 | "model.add(Dense(64))\n", 218 | "model.add(Activation('relu'))\n", 219 | "model.add(Dropout(0.5))\n", 220 | "model.add(Dense(1))\n", 221 | "model.add(Activation('sigmoid'))" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 6, 227 | "metadata": { 228 | "collapsed": true 229 | }, 230 | "outputs": [], 231 | "source": [ 232 | "model.compile(loss='binary_crossentropy',\n", 233 | " optimizer='rmsprop',\n", 234 | " metrics=['accuracy'])" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "### Training" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": 7, 247 | "metadata": { 248 | "collapsed": true 249 | }, 250 | "outputs": [], 251 | "source": [ 252 | "nb_epoch = 30\n", 253 | "nb_train_samples = 2048\n", 254 | "nb_validation_samples = 832" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 8, 260 | "metadata": {}, 261 | "outputs": [ 262 | { 263 | "name": "stdout", 264 | "output_type": "stream", 265 | "text": [ 266 | "Epoch 1/30\n", 267 | "2048/2048 [==============================] - 38s - loss: 0.6943 - acc: 0.5269 - val_loss: 0.6779 - val_acc: 0.5769\n", 268 | "Epoch 2/30\n", 269 | "2048/2048 [==============================] - 39s - loss: 0.6795 - acc: 0.5674 - val_loss: 0.6578 - val_acc: 0.6154\n", 270 | "Epoch 3/30\n", 271 | "2048/2048 [==============================] - 39s - loss: 0.6624 - acc: 0.5981 - val_loss: 0.6664 - val_acc: 0.5216\n", 272 | "Epoch 4/30\n", 273 | "2048/2048 [==============================] - 42s - loss: 0.6346 - acc: 0.6543 - val_loss: 0.6188 - val_acc: 0.6430\n", 274 | "Epoch 5/30\n", 275 | "2048/2048 [==============================] - 40s - loss: 0.6050 - acc: 0.6724 - val_loss: 0.6060 - val_acc: 0.6671\n", 276 | "Epoch 6/30\n", 277 | "2048/2048 [==============================] - 42s - loss: 0.5843 - acc: 0.6875 - val_loss: 0.5939 - val_acc: 0.6779\n", 278 | "Epoch 7/30\n", 279 | "2048/2048 [==============================] - 41s - loss: 0.5624 - acc: 0.7178 - val_loss: 0.5784 - val_acc: 0.6851\n", 280 | "Epoch 8/30\n", 281 | "2048/2048 [==============================] - 42s - loss: 0.5269 - acc: 0.7456 - val_loss: 0.5734 - val_acc: 0.6911\n", 282 | "Epoch 9/30\n", 283 | "2048/2048 [==============================] - 41s - loss: 0.4919 - acc: 0.7646 - val_loss: 0.5773 - val_acc: 0.7067\n", 284 | "Epoch 10/30\n", 285 | "2048/2048 [==============================] - 43s - loss: 0.4695 - acc: 0.7773 - val_loss: 0.5626 - val_acc: 0.7200\n", 286 | "Epoch 11/30\n", 287 | "2048/2048 [==============================] - 43s - loss: 0.4294 - acc: 0.8047 - val_loss: 0.6142 - val_acc: 0.6983\n", 288 | "Epoch 12/30\n", 289 | "2048/2048 [==============================] - 42s - loss: 0.4107 - acc: 0.8198 - val_loss: 0.5890 - val_acc: 0.7163\n", 290 | "Epoch 13/30\n", 291 | "2048/2048 [==============================] - 42s - loss: 0.3741 - acc: 0.8384 - val_loss: 0.5833 - val_acc: 0.7007\n", 292 | "Epoch 14/30\n", 293 | "2048/2048 [==============================] - 42s - loss: 0.3381 - acc: 0.8618 - val_loss: 0.7326 - val_acc: 0.7055\n", 294 | "Epoch 15/30\n", 295 | "2048/2048 [==============================] - 43s - loss: 0.3178 - acc: 0.8721 - val_loss: 0.7492 - val_acc: 0.7151\n", 296 | "Epoch 16/30\n", 297 | "2048/2048 [==============================] - 42s - loss: 0.2731 - acc: 0.8862 - val_loss: 0.6983 - val_acc: 0.6947\n", 298 | "Epoch 17/30\n", 299 | "2048/2048 [==============================] - 42s - loss: 0.2751 - acc: 0.8945 - val_loss: 0.8021 - val_acc: 0.7067\n", 300 | "Epoch 18/30\n", 301 | "2048/2048 [==============================] - 41s - loss: 0.2449 - acc: 0.9038 - val_loss: 0.6808 - val_acc: 0.7103\n", 302 | "Epoch 19/30\n", 303 | "2048/2048 [==============================] - 43s - loss: 0.2164 - acc: 0.9170 - val_loss: 0.6530 - val_acc: 0.6959\n", 304 | "Epoch 20/30\n", 305 | "2048/2048 [==============================] - 43s - loss: 0.2181 - acc: 0.9219 - val_loss: 0.8731 - val_acc: 0.7115\n", 306 | "Epoch 21/30\n", 307 | "2048/2048 [==============================] - 41s - loss: 0.1855 - acc: 0.9248 - val_loss: 0.9053 - val_acc: 0.7296\n", 308 | "Epoch 22/30\n", 309 | "2048/2048 [==============================] - 43s - loss: 0.1854 - acc: 0.9351 - val_loss: 1.0213 - val_acc: 0.7127\n", 310 | "Epoch 23/30\n", 311 | "2048/2048 [==============================] - 43s - loss: 0.1866 - acc: 0.9419 - val_loss: 1.2990 - val_acc: 0.7067\n", 312 | "Epoch 24/30\n", 313 | "2048/2048 [==============================] - 42s - loss: 0.1605 - acc: 0.9414 - val_loss: 1.0048 - val_acc: 0.7019\n", 314 | "Epoch 25/30\n", 315 | "2048/2048 [==============================] - 43s - loss: 0.1768 - acc: 0.9448 - val_loss: 1.0695 - val_acc: 0.7103\n", 316 | "Epoch 26/30\n", 317 | "2048/2048 [==============================] - 43s - loss: 0.1310 - acc: 0.9526 - val_loss: 1.2172 - val_acc: 0.7091\n", 318 | "Epoch 27/30\n", 319 | "2048/2048 [==============================] - 43s - loss: 0.1636 - acc: 0.9419 - val_loss: 1.1059 - val_acc: 0.7019\n", 320 | "Epoch 28/30\n", 321 | "2048/2048 [==============================] - 41s - loss: 0.1512 - acc: 0.9517 - val_loss: 1.2849 - val_acc: 0.7103\n", 322 | "Epoch 29/30\n", 323 | "2048/2048 [==============================] - 43s - loss: 0.1461 - acc: 0.9517 - val_loss: 1.1104 - val_acc: 0.7007\n", 324 | "Epoch 30/30\n", 325 | "2048/2048 [==============================] - 43s - loss: 0.1437 - acc: 0.9463 - val_loss: 1.7152 - val_acc: 0.7188\n" 326 | ] 327 | }, 328 | { 329 | "data": { 330 | "text/plain": [ 331 | "" 332 | ] 333 | }, 334 | "execution_count": 8, 335 | "metadata": {}, 336 | "output_type": "execute_result" 337 | } 338 | ], 339 | "source": [ 340 | "model.fit_generator(\n", 341 | " train_generator,\n", 342 | " samples_per_epoch=nb_train_samples,\n", 343 | " nb_epoch=nb_epoch,\n", 344 | " validation_data=validation_generator,\n", 345 | " nb_val_samples=nb_validation_samples)" 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "execution_count": 9, 351 | "metadata": { 352 | "collapsed": true 353 | }, 354 | "outputs": [], 355 | "source": [ 356 | "model.save_weights('models/basic_cnn_20_epochs.h5')" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": 10, 362 | "metadata": { 363 | "collapsed": true 364 | }, 365 | "outputs": [], 366 | "source": [ 367 | "#model.load_weights('models_trained/basic_cnn_20_epochs.h5')" 368 | ] 369 | }, 370 | { 371 | "cell_type": "markdown", 372 | "metadata": {}, 373 | "source": [ 374 | "### Evaluating on validation set" 375 | ] 376 | }, 377 | { 378 | "cell_type": "markdown", 379 | "metadata": {}, 380 | "source": [ 381 | "Computing loss and accuracy :" 382 | ] 383 | }, 384 | { 385 | "cell_type": "code", 386 | "execution_count": 11, 387 | "metadata": {}, 388 | "outputs": [ 389 | { 390 | "data": { 391 | "text/plain": [ 392 | "[0.90210915643435263, 0.72956730769230771]" 393 | ] 394 | }, 395 | "execution_count": 11, 396 | "metadata": {}, 397 | "output_type": "execute_result" 398 | } 399 | ], 400 | "source": [ 401 | "model.evaluate_generator(validation_generator, nb_validation_samples)" 402 | ] 403 | }, 404 | { 405 | "cell_type": "markdown", 406 | "metadata": {}, 407 | "source": [ 408 | "Evolution of accuracy on training (blue) and validation (green) sets for 1 to 32 epochs :" 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": {}, 414 | "source": [ 415 | "![Accuracy evolution](pictures/scores_no_dataaugmentation.png)" 416 | ] 417 | }, 418 | { 419 | "cell_type": "markdown", 420 | "metadata": {}, 421 | "source": [ 422 | "**After ~30 epochs the neural network reach ~72% accuracy. But of course we could improve our model, by preventing overfiting, data augmentation, applying random transformations to the training set, etc. This example was seen to reach ~90% accuracy**\n", 423 | "\n", 424 | "# CPU Version: http://localhost:8888/notebooks/final/ImageClassifier_CPU.ipynb" 425 | ] 426 | }, 427 | { 428 | "cell_type": "markdown", 429 | "metadata": {}, 430 | "source": [ 431 | "## Data augmentation for improving the model" 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": {}, 437 | "source": [ 438 | "By applying random transformation to our train set, we artificially enhance our dataset with new unseen images. \n", 439 | "This will hopefully reduce overfitting and allows better generalization capability for our network." 440 | ] 441 | }, 442 | { 443 | "cell_type": "markdown", 444 | "metadata": {}, 445 | "source": [ 446 | "Example of data augmentation applied to a picture:\n", 447 | "![Example of data augmentation applied to a picture](pictures/cat_data_augmentation.png)" 448 | ] 449 | }, 450 | { 451 | "cell_type": "code", 452 | "execution_count": 12, 453 | "metadata": {}, 454 | "outputs": [ 455 | { 456 | "name": "stdout", 457 | "output_type": "stream", 458 | "text": [ 459 | "Found 2048 images belonging to 2 classes.\n" 460 | ] 461 | } 462 | ], 463 | "source": [ 464 | "train_datagen_augmented = ImageDataGenerator(\n", 465 | " rescale=1./255, # normalize pixel values to [0,1]\n", 466 | " shear_range=0.2, # randomly applies shearing transformation\n", 467 | " zoom_range=0.2, # randomly applies shearing transformation\n", 468 | " horizontal_flip=True) # randomly flip the images\n", 469 | "\n", 470 | "# same code as before\n", 471 | "train_generator_augmented = train_datagen_augmented.flow_from_directory(\n", 472 | " train_data_dir,\n", 473 | " target_size=(img_width, img_height),\n", 474 | " batch_size=32,\n", 475 | " class_mode='binary')" 476 | ] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "execution_count": 13, 481 | "metadata": {}, 482 | "outputs": [ 483 | { 484 | "name": "stdout", 485 | "output_type": "stream", 486 | "text": [ 487 | "Epoch 1/30\n", 488 | "2048/2048 [==============================] - 10s - loss: 0.5532 - acc: 0.7285 - val_loss: 0.6922 - val_acc: 0.7127\n", 489 | "Epoch 2/30\n", 490 | "2048/2048 [==============================] - 9s - loss: 0.5568 - acc: 0.7275 - val_loss: 0.5770 - val_acc: 0.7115\n", 491 | "Epoch 3/30\n", 492 | "2048/2048 [==============================] - 9s - loss: 0.5271 - acc: 0.7451 - val_loss: 0.6073 - val_acc: 0.7440\n", 493 | "Epoch 4/30\n", 494 | "2048/2048 [==============================] - 9s - loss: 0.5038 - acc: 0.7642 - val_loss: 0.5640 - val_acc: 0.7512\n", 495 | "Epoch 5/30\n", 496 | "2048/2048 [==============================] - 9s - loss: 0.5324 - acc: 0.7510 - val_loss: 0.5619 - val_acc: 0.6887\n", 497 | "Epoch 6/30\n", 498 | "2048/2048 [==============================] - 9s - loss: 0.5079 - acc: 0.7671 - val_loss: 0.5678 - val_acc: 0.7127\n", 499 | "Epoch 7/30\n", 500 | "2048/2048 [==============================] - 9s - loss: 0.5016 - acc: 0.7695 - val_loss: 0.5821 - val_acc: 0.7452\n", 501 | "Epoch 8/30\n", 502 | "2048/2048 [==============================] - 9s - loss: 0.4631 - acc: 0.7847 - val_loss: 0.6128 - val_acc: 0.7476\n", 503 | "Epoch 9/30\n", 504 | "2048/2048 [==============================] - 9s - loss: 0.4762 - acc: 0.7778 - val_loss: 0.5489 - val_acc: 0.7416\n", 505 | "Epoch 10/30\n", 506 | "2048/2048 [==============================] - 9s - loss: 0.4841 - acc: 0.7769 - val_loss: 0.5276 - val_acc: 0.7584\n", 507 | "Epoch 11/30\n", 508 | "2048/2048 [==============================] - 10s - loss: 0.4541 - acc: 0.7974 - val_loss: 0.5752 - val_acc: 0.7512\n", 509 | "Epoch 12/30\n", 510 | "2048/2048 [==============================] - 9s - loss: 0.4770 - acc: 0.7886 - val_loss: 0.5560 - val_acc: 0.7716\n", 511 | "Epoch 13/30\n", 512 | "2048/2048 [==============================] - 9s - loss: 0.4706 - acc: 0.7935 - val_loss: 0.5922 - val_acc: 0.7188\n", 513 | "Epoch 14/30\n", 514 | "2048/2048 [==============================] - 10s - loss: 0.4693 - acc: 0.7847 - val_loss: 0.5398 - val_acc: 0.7500\n", 515 | "Epoch 15/30\n", 516 | "2048/2048 [==============================] - 9s - loss: 0.4419 - acc: 0.7988 - val_loss: 0.5946 - val_acc: 0.7692\n", 517 | "Epoch 16/30\n", 518 | "2048/2048 [==============================] - 9s - loss: 0.4532 - acc: 0.8003 - val_loss: 0.5396 - val_acc: 0.7656\n", 519 | "Epoch 17/30\n", 520 | "2048/2048 [==============================] - 9s - loss: 0.4529 - acc: 0.7983 - val_loss: 0.5153 - val_acc: 0.7656\n", 521 | "Epoch 18/30\n", 522 | "2048/2048 [==============================] - 9s - loss: 0.4253 - acc: 0.8135 - val_loss: 0.8682 - val_acc: 0.7404\n", 523 | "Epoch 19/30\n", 524 | "2048/2048 [==============================] - 9s - loss: 0.4090 - acc: 0.8169 - val_loss: 0.6031 - val_acc: 0.7127\n", 525 | "Epoch 20/30\n", 526 | "2048/2048 [==============================] - 10s - loss: 0.4442 - acc: 0.8110 - val_loss: 0.5669 - val_acc: 0.7740\n", 527 | "Epoch 21/30\n", 528 | "2048/2048 [==============================] - 9s - loss: 0.4253 - acc: 0.8169 - val_loss: 0.5418 - val_acc: 0.7764\n", 529 | "Epoch 22/30\n", 530 | "2048/2048 [==============================] - 9s - loss: 0.4352 - acc: 0.8076 - val_loss: 0.5494 - val_acc: 0.7692\n", 531 | "Epoch 23/30\n", 532 | "2048/2048 [==============================] - 9s - loss: 0.4277 - acc: 0.8022 - val_loss: 0.5858 - val_acc: 0.7776\n", 533 | "Epoch 24/30\n", 534 | "2048/2048 [==============================] - 9s - loss: 0.4232 - acc: 0.8115 - val_loss: 0.6827 - val_acc: 0.7224\n", 535 | "Epoch 25/30\n", 536 | "2048/2048 [==============================] - 9s - loss: 0.4312 - acc: 0.8081 - val_loss: 0.5408 - val_acc: 0.7764\n", 537 | "Epoch 26/30\n", 538 | "2048/2048 [==============================] - 9s - loss: 0.4178 - acc: 0.8130 - val_loss: 0.5425 - val_acc: 0.7632\n", 539 | "Epoch 27/30\n", 540 | "2048/2048 [==============================] - 9s - loss: 0.3949 - acc: 0.8262 - val_loss: 0.6941 - val_acc: 0.7668\n", 541 | "Epoch 28/30\n", 542 | "2048/2048 [==============================] - 9s - loss: 0.4196 - acc: 0.8149 - val_loss: 0.8530 - val_acc: 0.7284\n", 543 | "Epoch 29/30\n", 544 | "2048/2048 [==============================] - 9s - loss: 0.4228 - acc: 0.8232 - val_loss: 0.5460 - val_acc: 0.7752\n", 545 | "Epoch 30/30\n", 546 | "2048/2048 [==============================] - 9s - loss: 0.3989 - acc: 0.8242 - val_loss: 0.6910 - val_acc: 0.7680\n" 547 | ] 548 | }, 549 | { 550 | "data": { 551 | "text/plain": [ 552 | "" 553 | ] 554 | }, 555 | "execution_count": 13, 556 | "metadata": {}, 557 | "output_type": "execute_result" 558 | } 559 | ], 560 | "source": [ 561 | "model.fit_generator(\n", 562 | " train_generator_augmented,\n", 563 | " samples_per_epoch=nb_train_samples,\n", 564 | " nb_epoch=nb_epoch,\n", 565 | " validation_data=validation_generator,\n", 566 | " nb_val_samples=nb_validation_samples)" 567 | ] 568 | }, 569 | { 570 | "cell_type": "code", 571 | "execution_count": 14, 572 | "metadata": { 573 | "collapsed": true 574 | }, 575 | "outputs": [], 576 | "source": [ 577 | "model.save_weights('models/augmented_30_epochs.h5')" 578 | ] 579 | }, 580 | { 581 | "cell_type": "code", 582 | "execution_count": 15, 583 | "metadata": { 584 | "collapsed": true 585 | }, 586 | "outputs": [], 587 | "source": [ 588 | "#model.load_weights('models_trained/augmented_30_epochs.h5')" 589 | ] 590 | }, 591 | { 592 | "cell_type": "markdown", 593 | "metadata": {}, 594 | "source": [ 595 | "### Evaluating on validation set" 596 | ] 597 | }, 598 | { 599 | "cell_type": "markdown", 600 | "metadata": {}, 601 | "source": [ 602 | "Computing loss and accuracy :" 603 | ] 604 | }, 605 | { 606 | "cell_type": "code", 607 | "execution_count": 16, 608 | "metadata": {}, 609 | "outputs": [ 610 | { 611 | "data": { 612 | "text/plain": [ 613 | "[0.69096329235113585, 0.76802884615384615]" 614 | ] 615 | }, 616 | "execution_count": 16, 617 | "metadata": {}, 618 | "output_type": "execute_result" 619 | } 620 | ], 621 | "source": [ 622 | "model.evaluate_generator(validation_generator, nb_validation_samples)" 623 | ] 624 | }, 625 | { 626 | "cell_type": "markdown", 627 | "metadata": {}, 628 | "source": [ 629 | "Evolution of accuracy on training (blue) and validation (green) sets for 1 to 100 epochs :" 630 | ] 631 | }, 632 | { 633 | "cell_type": "markdown", 634 | "metadata": {}, 635 | "source": [ 636 | "![Accuracy evolution](pictures/scores_with_dataaugmentation.png)" 637 | ] 638 | }, 639 | { 640 | "cell_type": "markdown", 641 | "metadata": {}, 642 | "source": [ 643 | "**Thanks to data-augmentation, the accuracy on the validation set improved to ~80%**" 644 | ] 645 | }, 646 | { 647 | "cell_type": "markdown", 648 | "metadata": {}, 649 | "source": [ 650 | "## Using a pre-trained model" 651 | ] 652 | }, 653 | { 654 | "cell_type": "markdown", 655 | "metadata": {}, 656 | "source": [ 657 | "The process of training a convolutionnal neural network can be very time-consuming and require a lot of datas. \n", 658 | "\n", 659 | "We can go beyond the previous models in terms of performance and efficiency by using a general-purpose, pre-trained image classifier. This example uses VGG16, a model trained on the ImageNet dataset - which contains millions of images classified in 1000 categories. \n", 660 | "\n", 661 | "On top of it, we add a small multi-layer perceptron and we train it on our dataset." 662 | ] 663 | }, 664 | { 665 | "cell_type": "markdown", 666 | "metadata": {}, 667 | "source": [ 668 | "### VGG16 + small MLP\n", 669 | "![VGG16 + Dense layers Schema](pictures/vgg16_original.png)" 670 | ] 671 | }, 672 | { 673 | "cell_type": "markdown", 674 | "metadata": {}, 675 | "source": [ 676 | "#### VGG16 model architecture definition" 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "execution_count": 17, 682 | "metadata": { 683 | "collapsed": true 684 | }, 685 | "outputs": [], 686 | "source": [ 687 | "model_vgg = Sequential()\n", 688 | "model_vgg.add(ZeroPadding2D((1, 1), input_shape=(img_width, img_height,3)))\n", 689 | "model_vgg.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))\n", 690 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 691 | "model_vgg.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))\n", 692 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 693 | "\n", 694 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 695 | "model_vgg.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))\n", 696 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 697 | "model_vgg.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))\n", 698 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 699 | "\n", 700 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 701 | "model_vgg.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))\n", 702 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 703 | "model_vgg.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))\n", 704 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 705 | "model_vgg.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))\n", 706 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 707 | "\n", 708 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 709 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))\n", 710 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 711 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))\n", 712 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 713 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))\n", 714 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 715 | "\n", 716 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 717 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))\n", 718 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 719 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))\n", 720 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 721 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))\n", 722 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))" 723 | ] 724 | }, 725 | { 726 | "cell_type": "markdown", 727 | "metadata": {}, 728 | "source": [ 729 | "#### Loading VGG16 weights\n", 730 | "This part is a bit complicated because the structure of our model is not exactly the same as the one used when training weights. \n", 731 | "Otherwise, we would use the `model.load_weights()` method." 732 | ] 733 | }, 734 | { 735 | "cell_type": "markdown", 736 | "metadata": {}, 737 | "source": [ 738 | "*Note : the VGG16 weights file (~500MB) is not included in this repository. You can download from here : \n", 739 | "https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3*" 740 | ] 741 | }, 742 | { 743 | "cell_type": "code", 744 | "execution_count": 18, 745 | "metadata": { 746 | "collapsed": true 747 | }, 748 | "outputs": [], 749 | "source": [ 750 | "import h5py\n", 751 | "f = h5py.File('models/vgg/vgg16_weights.h5')\n", 752 | "for k in range(f.attrs['nb_layers']):\n", 753 | " if k >= len(model_vgg.layers) - 1:\n", 754 | " # we don't look at the last two layers in the savefile (fully-connected and activation)\n", 755 | " break\n", 756 | " g = f['layer_{}'.format(k)]\n", 757 | " weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]\n", 758 | " layer = model_vgg.layers[k]\n", 759 | "\n", 760 | " if layer.__class__.__name__ in ['Convolution1D', 'Convolution2D', 'Convolution3D', 'AtrousConvolution2D']:\n", 761 | " weights[0] = np.transpose(weights[0], (2, 3, 1, 0))\n", 762 | "\n", 763 | " layer.set_weights(weights)\n", 764 | "\n", 765 | "f.close()" 766 | ] 767 | }, 768 | { 769 | "cell_type": "markdown", 770 | "metadata": {}, 771 | "source": [ 772 | "### Using the VGG16 model to process samples" 773 | ] 774 | }, 775 | { 776 | "cell_type": "code", 777 | "execution_count": 19, 778 | "metadata": {}, 779 | "outputs": [ 780 | { 781 | "name": "stdout", 782 | "output_type": "stream", 783 | "text": [ 784 | "Found 2048 images belonging to 2 classes.\n", 785 | "Found 832 images belonging to 2 classes.\n" 786 | ] 787 | } 788 | ], 789 | "source": [ 790 | "train_generator_bottleneck = datagen.flow_from_directory(\n", 791 | " train_data_dir,\n", 792 | " target_size=(img_width, img_height),\n", 793 | " batch_size=32,\n", 794 | " class_mode=None,\n", 795 | " shuffle=False)\n", 796 | "\n", 797 | "validation_generator_bottleneck = datagen.flow_from_directory(\n", 798 | " validation_data_dir,\n", 799 | " target_size=(img_width, img_height),\n", 800 | " batch_size=32,\n", 801 | " class_mode=None,\n", 802 | " shuffle=False)" 803 | ] 804 | }, 805 | { 806 | "cell_type": "markdown", 807 | "metadata": {}, 808 | "source": [ 809 | "This is a long process, so we save the output of the VGG16 once and for all. " 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 20, 815 | "metadata": { 816 | "collapsed": true 817 | }, 818 | "outputs": [], 819 | "source": [ 820 | "bottleneck_features_train = model_vgg.predict_generator(train_generator_bottleneck, nb_train_samples)\n", 821 | "np.save(open('models/bottleneck_features_train.npy', 'wb'), bottleneck_features_train)" 822 | ] 823 | }, 824 | { 825 | "cell_type": "code", 826 | "execution_count": 21, 827 | "metadata": { 828 | "collapsed": true 829 | }, 830 | "outputs": [], 831 | "source": [ 832 | "bottleneck_features_validation = model_vgg.predict_generator(validation_generator_bottleneck, nb_validation_samples)\n", 833 | "np.save(open('models/bottleneck_features_validation.npy', 'wb'), bottleneck_features_validation)" 834 | ] 835 | }, 836 | { 837 | "cell_type": "markdown", 838 | "metadata": {}, 839 | "source": [ 840 | "Now we can load it..." 841 | ] 842 | }, 843 | { 844 | "cell_type": "code", 845 | "execution_count": 22, 846 | "metadata": { 847 | "collapsed": true 848 | }, 849 | "outputs": [], 850 | "source": [ 851 | "train_data = np.load(open('models/bottleneck_features_train.npy', 'rb'))\n", 852 | "train_labels = np.array([0] * (nb_train_samples // 2) + [1] * (nb_train_samples // 2))\n", 853 | "\n", 854 | "validation_data = np.load(open('models/bottleneck_features_validation.npy', 'rb'))\n", 855 | "validation_labels = np.array([0] * (nb_validation_samples // 2) + [1] * (nb_validation_samples // 2))" 856 | ] 857 | }, 858 | { 859 | "cell_type": "markdown", 860 | "metadata": {}, 861 | "source": [ 862 | "And define and train the custom fully connected neural network :" 863 | ] 864 | }, 865 | { 866 | "cell_type": "code", 867 | "execution_count": 23, 868 | "metadata": { 869 | "collapsed": true 870 | }, 871 | "outputs": [], 872 | "source": [ 873 | "model_top = Sequential()\n", 874 | "model_top.add(Flatten(input_shape=train_data.shape[1:]))\n", 875 | "model_top.add(Dense(256, activation='relu'))\n", 876 | "model_top.add(Dropout(0.5))\n", 877 | "model_top.add(Dense(1, activation='sigmoid'))\n", 878 | "\n", 879 | "model_top.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])" 880 | ] 881 | }, 882 | { 883 | "cell_type": "code", 884 | "execution_count": 24, 885 | "metadata": {}, 886 | "outputs": [ 887 | { 888 | "name": "stdout", 889 | "output_type": "stream", 890 | "text": [ 891 | "Train on 2048 samples, validate on 832 samples\n", 892 | "Epoch 1/40\n", 893 | "2048/2048 [==============================] - 0s - loss: 0.9784 - acc: 0.6855 - val_loss: 0.4140 - val_acc: 0.8077\n", 894 | "Epoch 2/40\n", 895 | "2048/2048 [==============================] - 0s - loss: 0.4831 - acc: 0.7891 - val_loss: 0.3646 - val_acc: 0.8365\n", 896 | "Epoch 3/40\n", 897 | "2048/2048 [==============================] - 0s - loss: 0.3974 - acc: 0.8271 - val_loss: 0.3369 - val_acc: 0.8558\n", 898 | "Epoch 4/40\n", 899 | "2048/2048 [==============================] - 0s - loss: 0.3438 - acc: 0.8525 - val_loss: 0.3136 - val_acc: 0.8594\n", 900 | "Epoch 5/40\n", 901 | "2048/2048 [==============================] - 0s - loss: 0.3131 - acc: 0.8647 - val_loss: 0.5777 - val_acc: 0.7873\n", 902 | "Epoch 6/40\n", 903 | "2048/2048 [==============================] - 0s - loss: 0.2883 - acc: 0.8823 - val_loss: 0.2913 - val_acc: 0.8786\n", 904 | "Epoch 7/40\n", 905 | "2048/2048 [==============================] - 0s - loss: 0.2321 - acc: 0.9048 - val_loss: 0.3724 - val_acc: 0.8462\n", 906 | "Epoch 8/40\n", 907 | "2048/2048 [==============================] - 0s - loss: 0.2057 - acc: 0.9092 - val_loss: 0.4276 - val_acc: 0.8438\n", 908 | "Epoch 9/40\n", 909 | "2048/2048 [==============================] - 0s - loss: 0.2000 - acc: 0.9141 - val_loss: 0.3741 - val_acc: 0.8618\n", 910 | "Epoch 10/40\n", 911 | "2048/2048 [==============================] - 0s - loss: 0.1811 - acc: 0.9248 - val_loss: 0.3447 - val_acc: 0.8726\n", 912 | "Epoch 11/40\n", 913 | "2048/2048 [==============================] - 0s - loss: 0.1540 - acc: 0.9395 - val_loss: 0.3352 - val_acc: 0.8714\n", 914 | "Epoch 12/40\n", 915 | "2048/2048 [==============================] - 0s - loss: 0.1511 - acc: 0.9414 - val_loss: 0.3706 - val_acc: 0.8738\n", 916 | "Epoch 13/40\n", 917 | "2048/2048 [==============================] - 0s - loss: 0.1362 - acc: 0.9399 - val_loss: 0.3913 - val_acc: 0.8798\n", 918 | "Epoch 14/40\n", 919 | "2048/2048 [==============================] - 0s - loss: 0.1134 - acc: 0.9497 - val_loss: 0.3854 - val_acc: 0.8726\n", 920 | "Epoch 15/40\n", 921 | "2048/2048 [==============================] - 0s - loss: 0.1117 - acc: 0.9609 - val_loss: 0.5750 - val_acc: 0.8534\n", 922 | "Epoch 16/40\n", 923 | "2048/2048 [==============================] - 0s - loss: 0.0998 - acc: 0.9565 - val_loss: 0.4339 - val_acc: 0.8870\n", 924 | "Epoch 17/40\n", 925 | "2048/2048 [==============================] - 0s - loss: 0.0678 - acc: 0.9736 - val_loss: 0.4881 - val_acc: 0.8762\n", 926 | "Epoch 18/40\n", 927 | "2048/2048 [==============================] - 0s - loss: 0.0852 - acc: 0.9658 - val_loss: 0.4335 - val_acc: 0.8870\n", 928 | "Epoch 19/40\n", 929 | "2048/2048 [==============================] - 0s - loss: 0.0733 - acc: 0.9692 - val_loss: 0.4505 - val_acc: 0.8750\n", 930 | "Epoch 20/40\n", 931 | "2048/2048 [==============================] - 0s - loss: 0.0616 - acc: 0.9756 - val_loss: 0.5825 - val_acc: 0.8606\n", 932 | "Epoch 21/40\n", 933 | "2048/2048 [==============================] - 0s - loss: 0.0585 - acc: 0.9771 - val_loss: 0.6856 - val_acc: 0.8510\n", 934 | "Epoch 22/40\n", 935 | "2048/2048 [==============================] - 0s - loss: 0.0494 - acc: 0.9800 - val_loss: 0.5624 - val_acc: 0.8750\n", 936 | "Epoch 23/40\n", 937 | "2048/2048 [==============================] - 0s - loss: 0.0464 - acc: 0.9839 - val_loss: 0.8560 - val_acc: 0.8401\n", 938 | "Epoch 24/40\n", 939 | "2048/2048 [==============================] - 0s - loss: 0.0626 - acc: 0.9727 - val_loss: 0.5666 - val_acc: 0.8738\n", 940 | "Epoch 25/40\n", 941 | "2048/2048 [==============================] - 0s - loss: 0.0285 - acc: 0.9912 - val_loss: 0.5543 - val_acc: 0.8786\n", 942 | "Epoch 26/40\n", 943 | "2048/2048 [==============================] - 0s - loss: 0.0463 - acc: 0.9805 - val_loss: 0.6137 - val_acc: 0.8738\n", 944 | "Epoch 27/40\n", 945 | "2048/2048 [==============================] - 0s - loss: 0.0260 - acc: 0.9917 - val_loss: 0.6608 - val_acc: 0.8882\n", 946 | "Epoch 28/40\n", 947 | "2048/2048 [==============================] - 0s - loss: 0.0414 - acc: 0.9824 - val_loss: 0.6423 - val_acc: 0.8702\n", 948 | "Epoch 29/40\n", 949 | "2048/2048 [==============================] - 0s - loss: 0.0308 - acc: 0.9878 - val_loss: 0.6941 - val_acc: 0.8726\n", 950 | "Epoch 30/40\n", 951 | "2048/2048 [==============================] - 0s - loss: 0.0273 - acc: 0.9897 - val_loss: 0.6677 - val_acc: 0.8678\n", 952 | "Epoch 31/40\n", 953 | "2048/2048 [==============================] - 0s - loss: 0.0315 - acc: 0.9927 - val_loss: 0.6830 - val_acc: 0.8738\n", 954 | "Epoch 32/40\n", 955 | "2048/2048 [==============================] - 0s - loss: 0.0258 - acc: 0.9922 - val_loss: 0.7593 - val_acc: 0.8798\n", 956 | "Epoch 33/40\n", 957 | "2048/2048 [==============================] - 0s - loss: 0.0266 - acc: 0.9922 - val_loss: 0.8203 - val_acc: 0.8546\n", 958 | "Epoch 34/40\n", 959 | "2048/2048 [==============================] - 0s - loss: 0.0198 - acc: 0.9941 - val_loss: 0.9712 - val_acc: 0.8450\n", 960 | "Epoch 35/40\n", 961 | "2048/2048 [==============================] - 0s - loss: 0.0291 - acc: 0.9902 - val_loss: 0.8274 - val_acc: 0.8594\n", 962 | "Epoch 36/40\n", 963 | "2048/2048 [==============================] - 0s - loss: 0.0168 - acc: 0.9941 - val_loss: 0.8353 - val_acc: 0.8738\n", 964 | "Epoch 37/40\n", 965 | "2048/2048 [==============================] - 0s - loss: 0.0324 - acc: 0.9912 - val_loss: 0.7481 - val_acc: 0.8702\n", 966 | "Epoch 38/40\n", 967 | "2048/2048 [==============================] - 0s - loss: 0.0191 - acc: 0.9932 - val_loss: 0.8011 - val_acc: 0.8750\n", 968 | "Epoch 39/40\n", 969 | "2048/2048 [==============================] - 0s - loss: 0.0254 - acc: 0.9897 - val_loss: 0.7558 - val_acc: 0.8702\n", 970 | "Epoch 40/40\n", 971 | "2048/2048 [==============================] - 0s - loss: 0.0267 - acc: 0.9897 - val_loss: 0.7828 - val_acc: 0.8798\n" 972 | ] 973 | }, 974 | { 975 | "data": { 976 | "text/plain": [ 977 | "" 978 | ] 979 | }, 980 | "execution_count": 24, 981 | "metadata": {}, 982 | "output_type": "execute_result" 983 | } 984 | ], 985 | "source": [ 986 | "nb_epoch=40\n", 987 | "model_top.fit(train_data, train_labels,\n", 988 | " nb_epoch=nb_epoch, batch_size=32,\n", 989 | " validation_data=(validation_data, validation_labels))" 990 | ] 991 | }, 992 | { 993 | "cell_type": "markdown", 994 | "metadata": {}, 995 | "source": [ 996 | "The training process of this small neural network is very fast : ~2s per epoch" 997 | ] 998 | }, 999 | { 1000 | "cell_type": "code", 1001 | "execution_count": 25, 1002 | "metadata": { 1003 | "collapsed": true 1004 | }, 1005 | "outputs": [], 1006 | "source": [ 1007 | "model_top.save_weights('models/bottleneck_40_epochs.h5')" 1008 | ] 1009 | }, 1010 | { 1011 | "cell_type": "markdown", 1012 | "metadata": {}, 1013 | "source": [ 1014 | "### Bottleneck model evaluation" 1015 | ] 1016 | }, 1017 | { 1018 | "cell_type": "code", 1019 | "execution_count": 26, 1020 | "metadata": { 1021 | "collapsed": true 1022 | }, 1023 | "outputs": [], 1024 | "source": [ 1025 | "#model_top.load_weights('models/with-bottleneck/1000-samples--100-epochs.h5')\n", 1026 | "#model_top.load_weights('/notebook/Data1/Code/keras-workshop/models/with-bottleneck/1000-samples--100-epochs.h5')" 1027 | ] 1028 | }, 1029 | { 1030 | "cell_type": "markdown", 1031 | "metadata": {}, 1032 | "source": [ 1033 | "Loss and accuracy :" 1034 | ] 1035 | }, 1036 | { 1037 | "cell_type": "code", 1038 | "execution_count": 27, 1039 | "metadata": {}, 1040 | "outputs": [ 1041 | { 1042 | "name": "stdout", 1043 | "output_type": "stream", 1044 | "text": [ 1045 | "704/832 [========================>.....] - ETA: 0s" 1046 | ] 1047 | }, 1048 | { 1049 | "data": { 1050 | "text/plain": [ 1051 | "[0.782802758165277, 0.87980769230769229]" 1052 | ] 1053 | }, 1054 | "execution_count": 27, 1055 | "metadata": {}, 1056 | "output_type": "execute_result" 1057 | } 1058 | ], 1059 | "source": [ 1060 | "model_top.evaluate(validation_data, validation_labels)" 1061 | ] 1062 | }, 1063 | { 1064 | "cell_type": "markdown", 1065 | "metadata": {}, 1066 | "source": [ 1067 | "Evolution of accuracy on training (blue) and validation (green) sets for 1 to 32 epochs :" 1068 | ] 1069 | }, 1070 | { 1071 | "cell_type": "markdown", 1072 | "metadata": {}, 1073 | "source": [ 1074 | "![Accuracy evolution](pictures/scores_with_bottleneck.png)" 1075 | ] 1076 | }, 1077 | { 1078 | "cell_type": "markdown", 1079 | "metadata": {}, 1080 | "source": [ 1081 | "**We reached a 90% accuracy on the validation after ~1m of training (~20 epochs) and 8% of the samples originally available on the Kaggle competition !**" 1082 | ] 1083 | }, 1084 | { 1085 | "cell_type": "code", 1086 | "execution_count": 28, 1087 | "metadata": { 1088 | "collapsed": true 1089 | }, 1090 | "outputs": [], 1091 | "source": [ 1092 | "##Fine-tuning the top layers of a a pre-trained network" 1093 | ] 1094 | }, 1095 | { 1096 | "cell_type": "markdown", 1097 | "metadata": { 1098 | "collapsed": true 1099 | }, 1100 | "source": [ 1101 | "Start by instantiating the VGG base and loading its weights." 1102 | ] 1103 | }, 1104 | { 1105 | "cell_type": "code", 1106 | "execution_count": 30, 1107 | "metadata": { 1108 | "collapsed": true 1109 | }, 1110 | "outputs": [], 1111 | "source": [ 1112 | "model_vgg = Sequential()\n", 1113 | "model_vgg.add(ZeroPadding2D((1, 1), input_shape=(img_width, img_height,3)))\n", 1114 | "model_vgg.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))\n", 1115 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1116 | "model_vgg.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))\n", 1117 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 1118 | "\n", 1119 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1120 | "model_vgg.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))\n", 1121 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1122 | "model_vgg.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))\n", 1123 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 1124 | "\n", 1125 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1126 | "model_vgg.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))\n", 1127 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1128 | "model_vgg.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))\n", 1129 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1130 | "model_vgg.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))\n", 1131 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 1132 | "\n", 1133 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1134 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))\n", 1135 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1136 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))\n", 1137 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1138 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))\n", 1139 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))\n", 1140 | "\n", 1141 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1142 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))\n", 1143 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1144 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))\n", 1145 | "model_vgg.add(ZeroPadding2D((1, 1)))\n", 1146 | "model_vgg.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))\n", 1147 | "model_vgg.add(MaxPooling2D((2, 2), strides=(2, 2)))" 1148 | ] 1149 | }, 1150 | { 1151 | "cell_type": "code", 1152 | "execution_count": 31, 1153 | "metadata": { 1154 | "collapsed": true 1155 | }, 1156 | "outputs": [], 1157 | "source": [ 1158 | "import h5py\n", 1159 | "f = h5py.File('models/vgg/vgg16_weights.h5')\n", 1160 | "for k in range(f.attrs['nb_layers']):\n", 1161 | " if k >= len(model_vgg.layers) - 1:\n", 1162 | " # we don't look at the last two layers in the savefile (fully-connected and activation)\n", 1163 | " break\n", 1164 | " g = f['layer_{}'.format(k)]\n", 1165 | " weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]\n", 1166 | " layer = model_vgg.layers[k]\n", 1167 | "\n", 1168 | " if layer.__class__.__name__ in ['Convolution1D', 'Convolution2D', 'Convolution3D', 'AtrousConvolution2D']:\n", 1169 | " weights[0] = np.transpose(weights[0], (2, 3, 1, 0))\n", 1170 | "\n", 1171 | " layer.set_weights(weights)\n", 1172 | "\n", 1173 | "f.close()" 1174 | ] 1175 | }, 1176 | { 1177 | "cell_type": "markdown", 1178 | "metadata": {}, 1179 | "source": [ 1180 | "Build a classifier model to put on top of the convolutional model. For the fine tuning, we start with a fully trained-classifer. We will use the weights from the earlier model. And then we will add this model on top of the convolutional base." 1181 | ] 1182 | }, 1183 | { 1184 | "cell_type": "code", 1185 | "execution_count": 32, 1186 | "metadata": { 1187 | "collapsed": true 1188 | }, 1189 | "outputs": [], 1190 | "source": [ 1191 | "top_model = Sequential()\n", 1192 | "top_model.add(Flatten(input_shape=model_vgg.output_shape[1:]))\n", 1193 | "top_model.add(Dense(256, activation='relu'))\n", 1194 | "top_model.add(Dropout(0.5))\n", 1195 | "top_model.add(Dense(1, activation='sigmoid'))\n", 1196 | "\n", 1197 | "top_model.load_weights('models/bottleneck_40_epochs.h5')\n", 1198 | "\n", 1199 | "model_vgg.add(top_model)" 1200 | ] 1201 | }, 1202 | { 1203 | "cell_type": "markdown", 1204 | "metadata": {}, 1205 | "source": [ 1206 | "For fine turning, we only want to train a few layers. This line will set the first 25 layers (up to the conv block) to non-trainable." 1207 | ] 1208 | }, 1209 | { 1210 | "cell_type": "code", 1211 | "execution_count": 33, 1212 | "metadata": { 1213 | "collapsed": true 1214 | }, 1215 | "outputs": [], 1216 | "source": [ 1217 | "for layer in model_vgg.layers[:25]:\n", 1218 | " layer.trainable = False" 1219 | ] 1220 | }, 1221 | { 1222 | "cell_type": "code", 1223 | "execution_count": 34, 1224 | "metadata": { 1225 | "collapsed": true 1226 | }, 1227 | "outputs": [], 1228 | "source": [ 1229 | "# compile the model with a SGD/momentum optimizer\n", 1230 | "# and a very slow learning rate.\n", 1231 | "model_vgg.compile(loss='binary_crossentropy',\n", 1232 | " optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),\n", 1233 | " metrics=['accuracy'])" 1234 | ] 1235 | }, 1236 | { 1237 | "cell_type": "code", 1238 | "execution_count": 35, 1239 | "metadata": {}, 1240 | "outputs": [ 1241 | { 1242 | "name": "stdout", 1243 | "output_type": "stream", 1244 | "text": [ 1245 | "Found 2048 images belonging to 2 classes.\n", 1246 | "Found 832 images belonging to 2 classes.\n" 1247 | ] 1248 | } 1249 | ], 1250 | "source": [ 1251 | "# prepare data augmentation configuration . . . do we need this?\n", 1252 | "train_datagen = ImageDataGenerator(\n", 1253 | " rescale=1./255,\n", 1254 | " shear_range=0.2,\n", 1255 | " zoom_range=0.2,\n", 1256 | " horizontal_flip=True)\n", 1257 | "\n", 1258 | "test_datagen = ImageDataGenerator(rescale=1./255)\n", 1259 | "\n", 1260 | "train_generator = train_datagen.flow_from_directory(\n", 1261 | " train_data_dir,\n", 1262 | " target_size=(img_height, img_width),\n", 1263 | " batch_size=32,\n", 1264 | " class_mode='binary')\n", 1265 | "\n", 1266 | "validation_generator = test_datagen.flow_from_directory(\n", 1267 | " validation_data_dir,\n", 1268 | " target_size=(img_height, img_width),\n", 1269 | " batch_size=32,\n", 1270 | " class_mode='binary')" 1271 | ] 1272 | }, 1273 | { 1274 | "cell_type": "code", 1275 | "execution_count": 36, 1276 | "metadata": {}, 1277 | "outputs": [ 1278 | { 1279 | "name": "stdout", 1280 | "output_type": "stream", 1281 | "text": [ 1282 | "Epoch 1/40\n", 1283 | "2048/2048 [==============================] - 39s - loss: 0.3593 - acc: 0.8862 - val_loss: 0.3669 - val_acc: 0.8738\n", 1284 | "Epoch 2/40\n", 1285 | "2048/2048 [==============================] - 35s - loss: 0.2559 - acc: 0.9077 - val_loss: 0.3483 - val_acc: 0.8594\n", 1286 | "Epoch 3/40\n", 1287 | "2048/2048 [==============================] - 35s - loss: 0.1942 - acc: 0.9292 - val_loss: 0.3380 - val_acc: 0.8618\n", 1288 | "Epoch 4/40\n", 1289 | "2048/2048 [==============================] - 35s - loss: 0.1900 - acc: 0.9258 - val_loss: 0.3268 - val_acc: 0.8858\n", 1290 | "Epoch 5/40\n", 1291 | "2048/2048 [==============================] - 35s - loss: 0.1638 - acc: 0.9395 - val_loss: 0.3094 - val_acc: 0.8894\n", 1292 | "Epoch 6/40\n", 1293 | "2048/2048 [==============================] - 35s - loss: 0.1307 - acc: 0.9502 - val_loss: 0.3038 - val_acc: 0.8930\n", 1294 | "Epoch 7/40\n", 1295 | "2048/2048 [==============================] - 35s - loss: 0.1223 - acc: 0.9556 - val_loss: 0.3267 - val_acc: 0.8990\n", 1296 | "Epoch 8/40\n", 1297 | "2048/2048 [==============================] - 35s - loss: 0.1233 - acc: 0.9570 - val_loss: 0.2864 - val_acc: 0.8930\n", 1298 | "Epoch 9/40\n", 1299 | "2048/2048 [==============================] - 35s - loss: 0.1147 - acc: 0.9648 - val_loss: 0.3657 - val_acc: 0.8990\n", 1300 | "Epoch 10/40\n", 1301 | "2048/2048 [==============================] - 35s - loss: 0.1070 - acc: 0.9590 - val_loss: 0.3192 - val_acc: 0.8990\n", 1302 | "Epoch 11/40\n", 1303 | "2048/2048 [==============================] - 35s - loss: 0.0825 - acc: 0.9683 - val_loss: 0.4118 - val_acc: 0.8990\n", 1304 | "Epoch 12/40\n", 1305 | "2048/2048 [==============================] - 35s - loss: 0.0953 - acc: 0.9663 - val_loss: 0.3561 - val_acc: 0.9002\n", 1306 | "Epoch 13/40\n", 1307 | "2048/2048 [==============================] - 35s - loss: 0.0605 - acc: 0.9756 - val_loss: 0.3643 - val_acc: 0.9026\n", 1308 | "Epoch 14/40\n", 1309 | "2048/2048 [==============================] - 35s - loss: 0.0826 - acc: 0.9736 - val_loss: 0.3549 - val_acc: 0.8990\n", 1310 | "Epoch 15/40\n", 1311 | "2048/2048 [==============================] - 35s - loss: 0.0758 - acc: 0.9707 - val_loss: 0.3911 - val_acc: 0.9014\n", 1312 | "Epoch 16/40\n", 1313 | "2048/2048 [==============================] - 35s - loss: 0.0547 - acc: 0.9756 - val_loss: 0.4181 - val_acc: 0.9002\n", 1314 | "Epoch 17/40\n", 1315 | "2048/2048 [==============================] - 35s - loss: 0.0573 - acc: 0.9795 - val_loss: 0.3912 - val_acc: 0.9038\n", 1316 | "Epoch 18/40\n", 1317 | "2048/2048 [==============================] - 35s - loss: 0.0499 - acc: 0.9810 - val_loss: 0.4097 - val_acc: 0.9075\n", 1318 | "Epoch 19/40\n", 1319 | "2048/2048 [==============================] - 35s - loss: 0.0504 - acc: 0.9790 - val_loss: 0.4063 - val_acc: 0.9159\n", 1320 | "Epoch 20/40\n", 1321 | "2048/2048 [==============================] - 35s - loss: 0.0398 - acc: 0.9858 - val_loss: 0.4282 - val_acc: 0.9099\n", 1322 | "Epoch 21/40\n", 1323 | "2048/2048 [==============================] - 35s - loss: 0.0477 - acc: 0.9824 - val_loss: 0.4364 - val_acc: 0.8930\n", 1324 | "Epoch 22/40\n", 1325 | "2048/2048 [==============================] - 35s - loss: 0.0401 - acc: 0.9873 - val_loss: 0.4338 - val_acc: 0.9038\n", 1326 | "Epoch 23/40\n", 1327 | "2048/2048 [==============================] - 35s - loss: 0.0357 - acc: 0.9897 - val_loss: 0.4830 - val_acc: 0.9026\n", 1328 | "Epoch 24/40\n", 1329 | "2048/2048 [==============================] - 35s - loss: 0.0397 - acc: 0.9878 - val_loss: 0.4229 - val_acc: 0.9111\n", 1330 | "Epoch 25/40\n", 1331 | "2048/2048 [==============================] - 35s - loss: 0.0353 - acc: 0.9863 - val_loss: 0.4715 - val_acc: 0.9171\n", 1332 | "Epoch 26/40\n", 1333 | "2048/2048 [==============================] - 35s - loss: 0.0308 - acc: 0.9888 - val_loss: 0.4751 - val_acc: 0.9159\n", 1334 | "Epoch 27/40\n", 1335 | "2048/2048 [==============================] - 35s - loss: 0.0432 - acc: 0.9834 - val_loss: 0.4557 - val_acc: 0.9099\n", 1336 | "Epoch 28/40\n", 1337 | "2048/2048 [==============================] - 35s - loss: 0.0385 - acc: 0.9868 - val_loss: 0.4859 - val_acc: 0.9038\n", 1338 | "Epoch 29/40\n", 1339 | "2048/2048 [==============================] - 35s - loss: 0.0270 - acc: 0.9893 - val_loss: 0.4334 - val_acc: 0.9062\n", 1340 | "Epoch 30/40\n", 1341 | "2048/2048 [==============================] - 35s - loss: 0.0356 - acc: 0.9897 - val_loss: 0.4468 - val_acc: 0.9038\n", 1342 | "Epoch 31/40\n", 1343 | "2048/2048 [==============================] - 35s - loss: 0.0344 - acc: 0.9888 - val_loss: 0.4405 - val_acc: 0.9087\n", 1344 | "Epoch 32/40\n", 1345 | "2048/2048 [==============================] - 35s - loss: 0.0273 - acc: 0.9897 - val_loss: 0.5071 - val_acc: 0.9050\n", 1346 | "Epoch 33/40\n", 1347 | "2048/2048 [==============================] - 35s - loss: 0.0179 - acc: 0.9937 - val_loss: 0.5163 - val_acc: 0.9111\n", 1348 | "Epoch 34/40\n", 1349 | "2048/2048 [==============================] - 35s - loss: 0.0397 - acc: 0.9868 - val_loss: 0.4168 - val_acc: 0.9123\n", 1350 | "Epoch 35/40\n", 1351 | "2048/2048 [==============================] - 35s - loss: 0.0240 - acc: 0.9941 - val_loss: 0.4653 - val_acc: 0.9111\n", 1352 | "Epoch 36/40\n", 1353 | "2048/2048 [==============================] - 35s - loss: 0.0206 - acc: 0.9932 - val_loss: 0.5174 - val_acc: 0.9062\n", 1354 | "Epoch 37/40\n", 1355 | "2048/2048 [==============================] - 35s - loss: 0.0308 - acc: 0.9902 - val_loss: 0.5016 - val_acc: 0.9087\n", 1356 | "Epoch 38/40\n", 1357 | "2048/2048 [==============================] - 35s - loss: 0.0159 - acc: 0.9966 - val_loss: 0.4876 - val_acc: 0.9111\n", 1358 | "Epoch 39/40\n", 1359 | "2048/2048 [==============================] - 35s - loss: 0.0335 - acc: 0.9883 - val_loss: 0.4280 - val_acc: 0.9050\n", 1360 | "Epoch 40/40\n", 1361 | "2048/2048 [==============================] - 35s - loss: 0.0198 - acc: 0.9922 - val_loss: 0.4693 - val_acc: 0.9038\n" 1362 | ] 1363 | }, 1364 | { 1365 | "data": { 1366 | "text/plain": [ 1367 | "" 1368 | ] 1369 | }, 1370 | "execution_count": 36, 1371 | "metadata": {}, 1372 | "output_type": "execute_result" 1373 | } 1374 | ], 1375 | "source": [ 1376 | "# fine-tune the model\n", 1377 | "model_vgg.fit_generator(\n", 1378 | " train_generator,\n", 1379 | " samples_per_epoch=nb_train_samples,\n", 1380 | " nb_epoch=nb_epoch,\n", 1381 | " validation_data=validation_generator,\n", 1382 | " nb_val_samples=nb_validation_samples)" 1383 | ] 1384 | }, 1385 | { 1386 | "cell_type": "code", 1387 | "execution_count": 37, 1388 | "metadata": { 1389 | "collapsed": true 1390 | }, 1391 | "outputs": [], 1392 | "source": [ 1393 | "model_vgg.save_weights('models/finetuning_20epochs_vgg.h5')" 1394 | ] 1395 | }, 1396 | { 1397 | "cell_type": "code", 1398 | "execution_count": 38, 1399 | "metadata": { 1400 | "collapsed": true 1401 | }, 1402 | "outputs": [], 1403 | "source": [ 1404 | "model_vgg.load_weights('models/finetuning_20epochs_vgg.h5')" 1405 | ] 1406 | }, 1407 | { 1408 | "cell_type": "markdown", 1409 | "metadata": {}, 1410 | "source": [ 1411 | "### Evaluating on validation set" 1412 | ] 1413 | }, 1414 | { 1415 | "cell_type": "markdown", 1416 | "metadata": {}, 1417 | "source": [ 1418 | "Computing loss and accuracy :" 1419 | ] 1420 | }, 1421 | { 1422 | "cell_type": "code", 1423 | "execution_count": 39, 1424 | "metadata": {}, 1425 | "outputs": [ 1426 | { 1427 | "data": { 1428 | "text/plain": [ 1429 | "[0.46926221093879295, 0.90384615384615385]" 1430 | ] 1431 | }, 1432 | "execution_count": 39, 1433 | "metadata": {}, 1434 | "output_type": "execute_result" 1435 | } 1436 | ], 1437 | "source": [ 1438 | "model_vgg.evaluate_generator(validation_generator, nb_validation_samples)" 1439 | ] 1440 | }, 1441 | { 1442 | "cell_type": "code", 1443 | "execution_count": 40, 1444 | "metadata": {}, 1445 | "outputs": [ 1446 | { 1447 | "data": { 1448 | "text/plain": [ 1449 | "[0.69096328031558252, 0.76802884615384615]" 1450 | ] 1451 | }, 1452 | "execution_count": 40, 1453 | "metadata": {}, 1454 | "output_type": "execute_result" 1455 | } 1456 | ], 1457 | "source": [ 1458 | "model.evaluate_generator(validation_generator, nb_validation_samples)" 1459 | ] 1460 | }, 1461 | { 1462 | "cell_type": "code", 1463 | "execution_count": 41, 1464 | "metadata": {}, 1465 | "outputs": [ 1466 | { 1467 | "name": "stdout", 1468 | "output_type": "stream", 1469 | "text": [ 1470 | "704/832 [========================>.....] - ETA: 0s" 1471 | ] 1472 | }, 1473 | { 1474 | "data": { 1475 | "text/plain": [ 1476 | "[0.782802758165277, 0.87980769230769229]" 1477 | ] 1478 | }, 1479 | "execution_count": 41, 1480 | "metadata": {}, 1481 | "output_type": "execute_result" 1482 | } 1483 | ], 1484 | "source": [ 1485 | "model_top.evaluate(validation_data, validation_labels)" 1486 | ] 1487 | } 1488 | ], 1489 | "metadata": { 1490 | "kernelspec": { 1491 | "display_name": "Python 2", 1492 | "language": "python", 1493 | "name": "python2" 1494 | }, 1495 | "language_info": { 1496 | "codemirror_mode": { 1497 | "name": "ipython", 1498 | "version": 2 1499 | }, 1500 | "file_extension": ".py", 1501 | "mimetype": "text/x-python", 1502 | "name": "python", 1503 | "nbconvert_exporter": "python", 1504 | "pygments_lexer": "ipython2", 1505 | "version": "2.7.6" 1506 | } 1507 | }, 1508 | "nbformat": 4, 1509 | "nbformat_minor": 1 1510 | } 1511 | --------------------------------------------------------------------------------