├── image ├── notmnist.png ├── learn_rate_tune.png ├── mean_variance.png └── weight_biases.png ├── docker_tensor_setup.sh ├── run_jupyter.sh ├── jupyter_notebook_config.py ├── environment.yml ├── .gitignore ├── Dockerfile ├── README.MD ├── solutions.ipynb └── lab.ipynb /image/notmnist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimoralea/CarND-TensorFlow-Lab/master/image/notmnist.png -------------------------------------------------------------------------------- /image/learn_rate_tune.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimoralea/CarND-TensorFlow-Lab/master/image/learn_rate_tune.png -------------------------------------------------------------------------------- /image/mean_variance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimoralea/CarND-TensorFlow-Lab/master/image/mean_variance.png -------------------------------------------------------------------------------- /image/weight_biases.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimoralea/CarND-TensorFlow-Lab/master/image/weight_biases.png -------------------------------------------------------------------------------- /docker_tensor_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "WARNING: This installation method will no longer be supported in the future" 3 | apt-get update 4 | apt-get install -y bzip2 git wget python3-pip python3-yaml 5 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh 6 | bash Miniconda3-latest-Linux-x86_64.sh -b 7 | export PATH=$PATH:/root/miniconda3/bin/ 8 | conda install -y psutil 9 | python3.5 -m pip install pyyaml 10 | -------------------------------------------------------------------------------- /run_jupyter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | . activate CarND-TensorFlow-Lab 18 | jupyter notebook "$@" 19 | -------------------------------------------------------------------------------- /jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | import os 17 | from IPython.lib import passwd 18 | 19 | c.NotebookApp.ip = '*' 20 | c.NotebookApp.port = int(os.getenv('PORT', 8888)) 21 | c.NotebookApp.open_browser = False 22 | c.MultiKernelManager.default_kernel_name = 'python' 23 | 24 | # sets a password if PASSWORD is set in the environment 25 | if 'PASSWORD' in os.environ: 26 | c.NotebookApp.password = passwd(os.environ['PASSWORD']) 27 | del os.environ['PASSWORD'] 28 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: CarND-TensorFlow-Lab 2 | dependencies: 3 | - openssl=1.0.2j 4 | - pip=8.1.2 5 | - psutil=4.4.1 6 | - python>=3.4.0 7 | - readline=6.2 8 | - setuptools=27.2.0 9 | - sqlite=3.13.0 10 | - tk=8.5.18 11 | - wheel=0.29.0 12 | - xz=5.2.2 13 | - zlib=1.2.8 14 | - pip: 15 | - appnope==0.1.0 16 | - cycler==0.10.0 17 | - decorator==4.0.10 18 | - entrypoints==0.2.2 19 | - ipykernel==4.5.0 20 | - ipython==5.1.0 21 | - ipython-genutils==0.1.0 22 | - ipywidgets==5.2.2 23 | - jinja2==2.8 24 | - jsonschema==2.5.1 25 | - jupyter==1.0.0 26 | - jupyter-client==4.4.0 27 | - jupyter-console==5.0.0 28 | - jupyter-core==4.2.0 29 | - markupsafe==0.23 30 | - matplotlib==1.5.3 31 | - mistune==0.7.3 32 | - nbconvert==4.2.0 33 | - nbformat==4.1.0 34 | - notebook==4.2.3 35 | - numpy==1.11.2 36 | - pexpect==4.2.1 37 | - pickleshare==0.7.4 38 | - pillow==3.4.2 39 | - prompt-toolkit==1.0.8 40 | - protobuf==3.1.0.post1 41 | - ptyprocess==0.5.1 42 | - pygments==2.1.3 43 | - pyparsing==2.1.10 44 | - python-dateutil==2.5.3 45 | - pytz==2016.7 46 | - pyzmq==16.0.0 47 | - qtconsole==4.2.1 48 | - scikit-learn==0.18 49 | - scipy==0.18.1 50 | - simplegeneric==0.8.1 51 | - six==1.10.0 52 | - sklearn==0.0 53 | - terminado==0.6 54 | - tornado==4.4.2 55 | - tqdm==4.8.4 56 | - traitlets==4.3.1 57 | - wcwidth==0.1.7 58 | - widgetsnbextension==1.2.6 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # Data 92 | notMNIST.pickle 93 | *.zip 94 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python 2 | 3 | MAINTAINER @joshuacook 4 | 5 | # Pick up some TF dependencies 6 | RUN apt-get update && apt-get install -y --no-install-recommends \ 7 | build-essential \ 8 | curl \ 9 | libfreetype6-dev \ 10 | libpng12-dev \ 11 | libzmq3-dev \ 12 | pkg-config \ 13 | python3-dev \ 14 | rsync \ 15 | software-properties-common \ 16 | unzip \ 17 | && \ 18 | apt-get clean && \ 19 | rm -rf /var/lib/apt/lists/* 20 | 21 | RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ 22 | python get-pip.py && \ 23 | rm get-pip.py 24 | 25 | ADD https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh tmp/Miniconda3-latest-Linux-x86_64.sh 26 | RUN bash tmp/Miniconda3-latest-Linux-x86_64.sh -b 27 | ENV PATH $PATH:/root/miniconda3/bin/ 28 | COPY environment.yml . 29 | RUN conda install --yes pyyaml 30 | RUN conda env create -f environment.yml 31 | 32 | RUN conda install --name CarND-TensorFlow-Lab -c conda-forge tensorflow 33 | 34 | # Set up our notebook config. 35 | COPY jupyter_notebook_config.py /root/.jupyter/ 36 | 37 | # Copy sample notebooks. 38 | COPY . /notebooks 39 | 40 | # Jupyter has issues with being run directly: 41 | # https://github.com/ipython/ipython/issues/7062 42 | # We just add a little wrapper script. 43 | COPY run_jupyter.sh / 44 | RUN chmod +x run_jupyter.sh 45 | 46 | # TensorBoard 47 | EXPOSE 6006 48 | # IPython 49 | EXPOSE 8888 50 | 51 | WORKDIR "/notebooks" 52 | 53 | CMD ["/run_jupyter.sh"] 54 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # TensorFlow Neural Network Lab 2 | **Preprocess notMNIST data and train a neural net on it** 3 | 4 | [notMNIST dataset samples](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) 5 | 6 | We've prepared a Jupyter notebook that will guide you through the process of creating a single layer neural network in TensorFlow. 7 | 8 | ## Setup 9 | ### Windows 10 | #### Install Docker 11 | If you don't have Docker already, download and install Docker from [here](https://docs.docker.com/engine/installation/windows/). 12 | #### Clone the Repository 13 | Run the command below to clone the Lab Repository: 14 | ``` 15 | $ git clone https://github.com/udacity/CarND-TensorFlow-Lab.git 16 | ``` 17 | 18 | After running the command above, follow the “Run the Notebook” instructions to start the lab. 19 | ### OS X and Linux 20 | #### Install Anaconda 21 | This lab requires [Anaconda](https://www.continuum.io/downloads) and [Python 3.4](https://www.python.org/downloads/) or higher. If you don't meet all of these requirements, install the appropriate package(s). 22 | #### Run the Anaconda Environment 23 | Run these commands in your terminal to install all requirements: 24 | ``` 25 | $ git clone https://github.com/udacity/CarND-TensorFlow-Lab.git 26 | $ conda env create -f environment.yml 27 | $ conda install --name CarND-TensorFlow-Lab -c conda-forge tensorflow 28 | ``` 29 | ## Run the Notebook 30 | ### Start a Jupyter Server 31 | Make sure to run the server from the same directory that you ran in the *Setup* steps above. 32 | #### Windows 33 | ``` 34 | $ docker run -it -p 8888:8888 -v `pwd`:/notebooks udacity/carnd-tensorflow-lab 35 | ``` 36 | #### OS X and Linux 37 | ``` 38 | $ source activate CarND-TensorFlow-Lab 39 | $ jupyter notebook 40 | ``` 41 | ### View The Notebook 42 | You setup your environment in the *Setup* steps and started a Jupyter server in the *Start a Jupyter Server* instructions above. Open a browser window and go [here](http://localhost:8888/notebooks/CarND-TensorFlow-Lab/lab.ipynb). This is the notebook you'll be working on. The notebook has 3 problems for you to solve: 43 | - Problem 1: Normalize the features 44 | - Problem 2: Use TensorFlow operations to create features, labels, weight, and biases tensors 45 | - Problem 3: Tune the learning rate, number of steps, and batch size for the best accuracy 46 | 47 | This is a self-assessed lab. Compare your answers to the solutions [here](https://github.com/udacity/CarND-TensorFlow-Lab/blob/master/solutions.ipynb). If you have any difficulty completing the lab, Udacity provides a few services to answer any questions you might have. 48 | ## Help 49 | Remember that you can get assistance from your mentor, the [Forums](https://carnd-udacity.atlassian.net/wiki/questions), or the [Slack channel](https://carnd-inviter.herokuapp.com/). You can also review the concepts from the previous lessons. 50 | -------------------------------------------------------------------------------- /solutions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "collapsed": true 7 | }, 8 | "source": [ 9 | "# Solutions\n", 10 | "## Problem 1\n", 11 | "Implement the Min-Max scaling function ($X'=a+{\\frac {\\left(X-X_{\\min }\\right)\\left(b-a\\right)}{X_{\\max }-X_{\\min }}}$) with the parameters:\n", 12 | "\n", 13 | "$X_{\\min }=0$\n", 14 | "\n", 15 | "$X_{\\max }=255$\n", 16 | "\n", 17 | "$a=0.1$\n", 18 | "\n", 19 | "$b=0.9$" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": { 26 | "collapsed": true 27 | }, 28 | "outputs": [], 29 | "source": [ 30 | "# Problem 1 - Implement Min-Max scaling for greyscale image data\n", 31 | "def normalize_greyscale(image_data):\n", 32 | " \"\"\"\n", 33 | " Normalize the image data with Min-Max scaling to a range of [0.1, 0.9]\n", 34 | " :param image_data: The image data to be normalized\n", 35 | " :return: Normalized image data\n", 36 | " \"\"\"\n", 37 | " a = 0.1\n", 38 | " b = 0.9\n", 39 | " greyscale_min = 0\n", 40 | " greyscale_max = 255\n", 41 | " return a + ( ( (image_data - greyscale_min)*(b - a) )/( greyscale_max - greyscale_min ) )" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "## Problem 2\n", 49 | "- Use [tf.placeholder()](https://www.tensorflow.org/api_docs/python/io_ops.html#placeholder) for `features` and `labels` since they are the inputs to the model.\n", 50 | "- Any math operations must have the same type on both sides of the operator. The weights are float32, so the `features` and `labels` must also be float32.\n", 51 | "- Use [tf.Variable()](https://www.tensorflow.org/api_docs/python/state_ops.html#Variable) to allow `weights` and `biases` to be modified.\n", 52 | "- The `weights` must be the dimensions of features by labels. The number of features is the size of the image, 28*28=784. The size of labels is 10.\n", 53 | "- The `biases` must be the dimensions of the labels, which is 10." 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "features_count = 784\n", 65 | "labels_count = 10\n", 66 | "\n", 67 | "# Problem 2 - Set the features and labels tensors\n", 68 | "features = tf.placeholder(tf.float32)\n", 69 | "labels = tf.placeholder(tf.float32)\n", 70 | "\n", 71 | "# Problem 2 - Set the weights and biases tensors\n", 72 | "weights = tf.Variable(tf.truncated_normal((features_count, labels_count)))\n", 73 | "biases = tf.Variable(tf.zeros(labels_count))" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "# Problem 3\n", 81 | "Configuration 1\n", 82 | "* **Epochs:** 1\n", 83 | "* **Batch Size:** 50\n", 84 | "* **Learning Rate:** 0.01\n", 85 | "\n", 86 | "Configuration 2\n", 87 | "* **Epochs:** 1\n", 88 | "* **Batch Size:** 100\n", 89 | "* **Learning Rate:** 0.1\n", 90 | "\n", 91 | "Configuration 3\n", 92 | "* **Epochs:** 4 or 5\n", 93 | "* **Batch Size:** 100\n", 94 | "* **Learning Rate:** 0.2" 95 | ] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.5.2" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 0 119 | } 120 | -------------------------------------------------------------------------------- /lab.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

TensorFlow Neural Network Lab

" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "\n", 15 | "In this lab, you'll use all the tools you learned from *Introduction to TensorFlow* to label images of English letters! The data you are using, notMNIST, consists of images of a letter from A to J in differents font.\n", 16 | "\n", 17 | "The above images are a few examples of the data you'll be training on. After training the network, you will compare your prediction model against test data. Your goal, by the end of this lab, is to make predictions against that test set with at least an 80% accuracy. Let's jump in!" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "To start this lab, you first need to import all the necessary modules. Run the code below. If it runs successfully, it will print \"`All modules imported`\"." 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 1, 30 | "metadata": { 31 | "collapsed": false 32 | }, 33 | "outputs": [ 34 | { 35 | "name": "stdout", 36 | "output_type": "stream", 37 | "text": [ 38 | "All modules imported.\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "import hashlib\n", 44 | "import os\n", 45 | "import pickle\n", 46 | "from urllib.request import urlretrieve\n", 47 | "\n", 48 | "import numpy as np\n", 49 | "from PIL import Image\n", 50 | "from sklearn.model_selection import train_test_split\n", 51 | "from sklearn.preprocessing import LabelBinarizer\n", 52 | "from sklearn.utils import resample\n", 53 | "from tqdm import tqdm\n", 54 | "from zipfile import ZipFile\n", 55 | "\n", 56 | "print('All modules imported.')" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "The notMNIST data is a large dataset to handle for most computers. It contains 500 thousands images for just training. You'll be using a subset of this data, 15,000 images for each label (A-J)." 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 3, 69 | "metadata": { 70 | "collapsed": false, 71 | "scrolled": false 72 | }, 73 | "outputs": [ 74 | { 75 | "name": "stdout", 76 | "output_type": "stream", 77 | "text": [ 78 | "Downloading notMNIST_train.zip...\n", 79 | "Download Finished\n", 80 | "Downloading notMNIST_test.zip...\n", 81 | "Download Finished\n", 82 | "All files downloaded.\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "def download(url, file):\n", 88 | " \"\"\"\n", 89 | " Download file from \n", 90 | " :param url: URL to file\n", 91 | " :param file: Local file path\n", 92 | " \"\"\"\n", 93 | " if not os.path.isfile(file):\n", 94 | " print('Downloading ' + file + '...')\n", 95 | " urlretrieve(url, file)\n", 96 | " print('Download Finished')\n", 97 | "\n", 98 | "# Download the training and test dataset.\n", 99 | "download('https://s3.amazonaws.com/udacity-sdc/notMNIST_train.zip', 'notMNIST_train.zip')\n", 100 | "download('https://s3.amazonaws.com/udacity-sdc/notMNIST_test.zip', 'notMNIST_test.zip')\n", 101 | "\n", 102 | "# Make sure the files aren't corrupted\n", 103 | "assert hashlib.md5(open('notMNIST_train.zip', 'rb').read()).hexdigest() == 'c8673b3f28f489e9cdf3a3d74e2ac8fa',\\\n", 104 | " 'notMNIST_train.zip file is corrupted. Remove the file and try again.'\n", 105 | "assert hashlib.md5(open('notMNIST_test.zip', 'rb').read()).hexdigest() == '5d3c7e653e63471c88df796156a9dfa9',\\\n", 106 | " 'notMNIST_test.zip file is corrupted. Remove the file and try again.'\n", 107 | "\n", 108 | "# Wait until you see that all files have been downloaded.\n", 109 | "print('All files downloaded.')" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 4, 115 | "metadata": { 116 | "collapsed": false 117 | }, 118 | "outputs": [ 119 | { 120 | "name": "stderr", 121 | "output_type": "stream", 122 | "text": [ 123 | "100%|██████████| 210001/210001 [00:25<00:00, 8243.63files/s]\n", 124 | "100%|██████████| 10001/10001 [00:01<00:00, 8357.10files/s]\n" 125 | ] 126 | }, 127 | { 128 | "name": "stdout", 129 | "output_type": "stream", 130 | "text": [ 131 | "All features and labels uncompressed.\n" 132 | ] 133 | } 134 | ], 135 | "source": [ 136 | "def uncompress_features_labels(file):\n", 137 | " \"\"\"\n", 138 | " Uncompress features and labels from a zip file\n", 139 | " :param file: The zip file to extract the data from\n", 140 | " \"\"\"\n", 141 | " features = []\n", 142 | " labels = []\n", 143 | "\n", 144 | " with ZipFile(file) as zipf:\n", 145 | " # Progress Bar\n", 146 | " filenames_pbar = tqdm(zipf.namelist(), unit='files')\n", 147 | " \n", 148 | " # Get features and labels from all files\n", 149 | " for filename in filenames_pbar:\n", 150 | " # Check if the file is a directory\n", 151 | " if not filename.endswith('/'):\n", 152 | " with zipf.open(filename) as image_file:\n", 153 | " image = Image.open(image_file)\n", 154 | " image.load()\n", 155 | " # Load image data as 1 dimensional array\n", 156 | " # We're using float32 to save on memory space\n", 157 | " feature = np.array(image, dtype=np.float32).flatten()\n", 158 | "\n", 159 | " # Get the the letter from the filename. This is the letter of the image.\n", 160 | " label = os.path.split(filename)[1][0]\n", 161 | "\n", 162 | " features.append(feature)\n", 163 | " labels.append(label)\n", 164 | " return np.array(features), np.array(labels)\n", 165 | "\n", 166 | "# Get the features and labels from the zip files\n", 167 | "train_features, train_labels = uncompress_features_labels('notMNIST_train.zip')\n", 168 | "test_features, test_labels = uncompress_features_labels('notMNIST_test.zip')\n", 169 | "\n", 170 | "# Limit the amount of data to work with a docker container\n", 171 | "docker_size_limit = 150000\n", 172 | "train_features, train_labels = resample(train_features, train_labels, n_samples=docker_size_limit)\n", 173 | "\n", 174 | "# Set flags for feature engineering. This will prevent you from skipping an important step.\n", 175 | "is_features_normal = False\n", 176 | "is_labels_encod = False\n", 177 | "\n", 178 | "# Wait until you see that all features and labels have been uncompressed.\n", 179 | "print('All features and labels uncompressed.')" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "\n", 187 | "## Problem 1\n", 188 | "The first problem involves normalizing the features for your training and test data. I want you to implement Min-Max scaling in the `normalize()` function to a range of `a=0.1` and `b=0.9`. Since the notMNIST image data is in [greyscale](https://en.wikipedia.org/wiki/Grayscale), you'll have to use a max of 255 and min of 0.\n", 189 | "\n", 190 | "Min-Max Scaling:\n", 191 | "$\n", 192 | "X'=a+{\\frac {\\left(X-X_{\\min }\\right)\\left(b-a\\right)}{X_{\\max }-X_{\\min }}}\n", 193 | "$\n", 194 | "\n", 195 | "*If you're having trouble solving problem 1, you can view the solution [here](https://github.com/udacity/CarND-TensorFlow-Lab/blob/master/solutions.ipynb).*" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 9, 201 | "metadata": { 202 | "collapsed": false 203 | }, 204 | "outputs": [ 205 | { 206 | "name": "stdout", 207 | "output_type": "stream", 208 | "text": [ 209 | "Tests Passed!\n" 210 | ] 211 | } 212 | ], 213 | "source": [ 214 | "# Problem 1 - Implement Min-Max scaling for greyscale image data\n", 215 | "def normalize_greyscale(image_data, a=0.1, b=0.9):\n", 216 | " \"\"\"\n", 217 | " Normalize the image data with Min-Max scaling to a range of [0.1, 0.9]\n", 218 | " :param image_data: The image data to be normalized\n", 219 | " :return: Normalized image data\n", 220 | " \"\"\"\n", 221 | " return a + ((image_data - np.min(image_data))*(b-a))/(np.max(image_data)-np.min(image_data))\n", 222 | "\n", 223 | "\n", 224 | "### DON'T MODIFY ANYTHING BELOW ###\n", 225 | "# Test Cases\n", 226 | "np.testing.assert_array_almost_equal(\n", 227 | " normalize_greyscale(np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 255])),\n", 228 | " [0.1, 0.103137254902, 0.106274509804, 0.109411764706, 0.112549019608, 0.11568627451, 0.118823529412, 0.121960784314,\n", 229 | " 0.125098039216, 0.128235294118, 0.13137254902, 0.9],\n", 230 | " decimal=3)\n", 231 | "np.testing.assert_array_almost_equal(\n", 232 | " normalize_greyscale(np.array([0, 1, 10, 20, 30, 40, 233, 244, 254,255])),\n", 233 | " [0.1, 0.103137254902, 0.13137254902, 0.162745098039, 0.194117647059, 0.225490196078, 0.830980392157, 0.865490196078,\n", 234 | " 0.896862745098, 0.9])\n", 235 | "\n", 236 | "if not is_features_normal:\n", 237 | " train_features = normalize_greyscale(train_features)\n", 238 | " test_features = normalize_greyscale(test_features)\n", 239 | " is_features_normal = True\n", 240 | "\n", 241 | "print('Tests Passed!')" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": 10, 247 | "metadata": { 248 | "collapsed": false 249 | }, 250 | "outputs": [ 251 | { 252 | "name": "stdout", 253 | "output_type": "stream", 254 | "text": [ 255 | "Labels One-Hot Encoded\n" 256 | ] 257 | } 258 | ], 259 | "source": [ 260 | "if not is_labels_encod:\n", 261 | " # Turn labels into numbers and apply One-Hot Encoding\n", 262 | " encoder = LabelBinarizer()\n", 263 | " encoder.fit(train_labels)\n", 264 | " train_labels = encoder.transform(train_labels)\n", 265 | " test_labels = encoder.transform(test_labels)\n", 266 | "\n", 267 | " # Change to float32, so it can be multiplied against the features in TensorFlow, which are float32\n", 268 | " train_labels = train_labels.astype(np.float32)\n", 269 | " test_labels = test_labels.astype(np.float32)\n", 270 | " is_labels_encod = True\n", 271 | "\n", 272 | "print('Labels One-Hot Encoded')" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 11, 278 | "metadata": { 279 | "collapsed": false 280 | }, 281 | "outputs": [ 282 | { 283 | "name": "stdout", 284 | "output_type": "stream", 285 | "text": [ 286 | "Training features and labels randomized and split.\n" 287 | ] 288 | } 289 | ], 290 | "source": [ 291 | "assert is_features_normal, 'You skipped the step to normalize the features'\n", 292 | "assert is_labels_encod, 'You skipped the step to One-Hot Encode the labels'\n", 293 | "\n", 294 | "# Get randomized datasets for training and validation\n", 295 | "train_features, valid_features, train_labels, valid_labels = train_test_split(\n", 296 | " train_features,\n", 297 | " train_labels,\n", 298 | " test_size=0.05,\n", 299 | " random_state=832289)\n", 300 | "\n", 301 | "print('Training features and labels randomized and split.')" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 12, 307 | "metadata": { 308 | "collapsed": false 309 | }, 310 | "outputs": [ 311 | { 312 | "name": "stdout", 313 | "output_type": "stream", 314 | "text": [ 315 | "Saving data to pickle file...\n", 316 | "Data cached in pickle file.\n" 317 | ] 318 | } 319 | ], 320 | "source": [ 321 | "# Save the data for easy access\n", 322 | "pickle_file = 'notMNIST.pickle'\n", 323 | "if not os.path.isfile(pickle_file):\n", 324 | " print('Saving data to pickle file...')\n", 325 | " try:\n", 326 | " with open('notMNIST.pickle', 'wb') as pfile:\n", 327 | " pickle.dump(\n", 328 | " {\n", 329 | " 'train_dataset': train_features,\n", 330 | " 'train_labels': train_labels,\n", 331 | " 'valid_dataset': valid_features,\n", 332 | " 'valid_labels': valid_labels,\n", 333 | " 'test_dataset': test_features,\n", 334 | " 'test_labels': test_labels,\n", 335 | " },\n", 336 | " pfile, pickle.HIGHEST_PROTOCOL)\n", 337 | " except Exception as e:\n", 338 | " print('Unable to save data to', pickle_file, ':', e)\n", 339 | " raise\n", 340 | "\n", 341 | "print('Data cached in pickle file.')" 342 | ] 343 | }, 344 | { 345 | "cell_type": "markdown", 346 | "metadata": {}, 347 | "source": [ 348 | "# Checkpoint\n", 349 | "All your progress is now saved to the pickle file. If you need to leave and comeback to this lab, you no longer have to start from the beginning. Just run the code block below and it will load all the data and modules required to proceed." 350 | ] 351 | }, 352 | { 353 | "cell_type": "code", 354 | "execution_count": 13, 355 | "metadata": { 356 | "collapsed": false 357 | }, 358 | "outputs": [ 359 | { 360 | "name": "stderr", 361 | "output_type": "stream", 362 | "text": [ 363 | "/root/miniconda3/envs/CarND-TensorFlow-Lab/lib/python3.5/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n", 364 | " warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n", 365 | "/root/miniconda3/envs/CarND-TensorFlow-Lab/lib/python3.5/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n", 366 | " warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n" 367 | ] 368 | }, 369 | { 370 | "name": "stdout", 371 | "output_type": "stream", 372 | "text": [ 373 | "Data and modules loaded.\n" 374 | ] 375 | } 376 | ], 377 | "source": [ 378 | "%matplotlib inline\n", 379 | "\n", 380 | "# Load the modules\n", 381 | "import pickle\n", 382 | "import math\n", 383 | "\n", 384 | "import numpy as np\n", 385 | "import tensorflow as tf\n", 386 | "from tqdm import tqdm\n", 387 | "import matplotlib.pyplot as plt\n", 388 | "\n", 389 | "# Reload the data\n", 390 | "pickle_file = 'notMNIST.pickle'\n", 391 | "with open(pickle_file, 'rb') as f:\n", 392 | " pickle_data = pickle.load(f)\n", 393 | " train_features = pickle_data['train_dataset']\n", 394 | " train_labels = pickle_data['train_labels']\n", 395 | " valid_features = pickle_data['valid_dataset']\n", 396 | " valid_labels = pickle_data['valid_labels']\n", 397 | " test_features = pickle_data['test_dataset']\n", 398 | " test_labels = pickle_data['test_labels']\n", 399 | " del pickle_data # Free up memory\n", 400 | "\n", 401 | "\n", 402 | "print('Data and modules loaded.')" 403 | ] 404 | }, 405 | { 406 | "cell_type": "markdown", 407 | "metadata": {}, 408 | "source": [ 409 | "\n", 410 | "## Problem 2\n", 411 | "For the neural network to train on your data, you need the following float32 tensors:\n", 412 | " - `features`\n", 413 | " - Placeholder tensor for feature data(`train_features`/`valid_features`/`test_features`)\n", 414 | " - `labels`\n", 415 | " - Placeholder tensor for label data(`train_labels`/`valid_labels`/`test_labels`)\n", 416 | " - `weights`\n", 417 | " - Variable Tensor with random numbers from a truncated normal distribution.\n", 418 | " - See `tf.truncated_normal()` documentation for help.\n", 419 | " - `biases`\n", 420 | " - Variable Tensor with all zeros.\n", 421 | " - See `tf.zeros()` documentation for help.\n", 422 | "\n", 423 | "*If you're having trouble solving problem 2, review \"TensorFlow Linear Function\" section of the class. If that doesn't help, the solution for this problem is available [here](https://github.com/udacity/CarND-TensorFlow-Lab/blob/master/solutions.ipynb).*" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": 29, 429 | "metadata": { 430 | "collapsed": false 431 | }, 432 | "outputs": [ 433 | { 434 | "name": "stdout", 435 | "output_type": "stream", 436 | "text": [ 437 | "Tests Passed!\n" 438 | ] 439 | } 440 | ], 441 | "source": [ 442 | "features_count = 784\n", 443 | "labels_count = 10\n", 444 | "# tf.placeholder(\"float\", [None, self.noutputs])\n", 445 | "# ToDo: Set the features and labels tensors\n", 446 | "features = tf.placeholder(tf.float32, [None, features_count])\n", 447 | "labels = tf.placeholder(tf.float32)\n", 448 | "\n", 449 | "# ToDo: Set the weights and biases tensors\n", 450 | "weights = tf.Variable(tf.truncated_normal([features_count, labels_count]))\n", 451 | "biases = tf.Variable(tf.zeros([labels_count]))\n", 452 | "\n", 453 | "### DON'T MODIFY ANYTHING BELOW ###\n", 454 | "\n", 455 | "#Test Cases\n", 456 | "from tensorflow.python.ops.variables import Variable\n", 457 | "\n", 458 | "assert features._op.name.startswith('Placeholder'), 'features must be a placeholder'\n", 459 | "assert labels._op.name.startswith('Placeholder'), 'labels must be a placeholder'\n", 460 | "assert isinstance(weights, Variable), 'weights must be a TensorFlow variable'\n", 461 | "assert isinstance(biases, Variable), 'biases must be a TensorFlow variable'\n", 462 | "\n", 463 | "assert features._shape == None or (\\\n", 464 | " features._shape.dims[0].value is None and\\\n", 465 | " features._shape.dims[1].value in [None, 784]), 'The shape of features is incorrect'\n", 466 | "assert labels._shape in [None, 10], 'The shape of labels is incorrect'\n", 467 | "assert weights._variable._shape == (784, 10), 'The shape of weights is incorrect'\n", 468 | "assert biases._variable._shape == (10), 'The shape of biases is incorrect'\n", 469 | "assert features._dtype == tf.float32, 'features must be type float32'\n", 470 | "assert labels._dtype == tf.float32, 'labels must be type float32'\n", 471 | "\n", 472 | "# Feed dicts for training, validation, and test session\n", 473 | "train_feed_dict = {features: train_features, labels: train_labels}\n", 474 | "valid_feed_dict = {features: valid_features, labels: valid_labels}\n", 475 | "test_feed_dict = {features: test_features, labels: test_labels}\n", 476 | "\n", 477 | "# Linear Function WX + b\n", 478 | "logits = tf.matmul(features, weights) + biases\n", 479 | "\n", 480 | "prediction = tf.nn.softmax(logits)\n", 481 | "\n", 482 | "# Cross entropy\n", 483 | "cross_entropy = -tf.reduce_sum(labels * tf.log(prediction), reduction_indices=1)\n", 484 | "\n", 485 | "# Training loss\n", 486 | "loss = tf.reduce_mean(cross_entropy)\n", 487 | "\n", 488 | "# Create an operation that initializes all variables\n", 489 | "init = tf.initialize_all_variables()\n", 490 | "\n", 491 | "# Test Cases\n", 492 | "with tf.Session() as session:\n", 493 | " session.run(init)\n", 494 | " session.run(loss, feed_dict=train_feed_dict)\n", 495 | " session.run(loss, feed_dict=valid_feed_dict)\n", 496 | " session.run(loss, feed_dict=test_feed_dict)\n", 497 | " biases_data = session.run(biases)\n", 498 | "\n", 499 | "assert not np.count_nonzero(biases_data), 'biases must be zeros'\n", 500 | "\n", 501 | "print('Tests Passed!')" 502 | ] 503 | }, 504 | { 505 | "cell_type": "code", 506 | "execution_count": 30, 507 | "metadata": { 508 | "collapsed": false 509 | }, 510 | "outputs": [ 511 | { 512 | "name": "stdout", 513 | "output_type": "stream", 514 | "text": [ 515 | "Accuracy function created.\n" 516 | ] 517 | } 518 | ], 519 | "source": [ 520 | "# Determine if the predictions are correct\n", 521 | "is_correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))\n", 522 | "# Calculate the accuracy of the predictions\n", 523 | "accuracy = tf.reduce_mean(tf.cast(is_correct_prediction, tf.float32))\n", 524 | "\n", 525 | "print('Accuracy function created.')" 526 | ] 527 | }, 528 | { 529 | "cell_type": "markdown", 530 | "metadata": {}, 531 | "source": [ 532 | "\n", 533 | "## Problem 3\n", 534 | "You're given 3 parameter configurations for training the neural network. One of the parameters in each configuration has multiple options. Choose the option for each configuration that gives the best acccuracy.\n", 535 | "\n", 536 | "Parameter configurations:\n", 537 | "\n", 538 | "Configuration 1\n", 539 | "* **Epochs:** 1\n", 540 | "* **Batch Size:**\n", 541 | " * 2000\n", 542 | " * 1000\n", 543 | " * 500\n", 544 | " * 300\n", 545 | " * 50\n", 546 | "* **Learning Rate:** 0.01\n", 547 | "\n", 548 | "Configuration 2\n", 549 | "* **Epochs:** 1\n", 550 | "* **Batch Size:** 100\n", 551 | "* **Learning Rate:**\n", 552 | " * 0.8\n", 553 | " * 0.5\n", 554 | " * 0.1\n", 555 | " * 0.05\n", 556 | " * 0.01\n", 557 | "\n", 558 | "Configuration 3\n", 559 | "* **Epochs:**\n", 560 | " * 1\n", 561 | " * 2\n", 562 | " * 3\n", 563 | " * 4\n", 564 | " * 5\n", 565 | "* **Batch Size:** 100\n", 566 | "* **Learning Rate:** 0.2\n", 567 | "\n", 568 | "The code will print out a Loss and Accuracy graph, so you can see how well the neural network performed.\n", 569 | "\n", 570 | "*If you're having trouble solving problem 3, you can view the solution [here](https://github.com/udacity/CarND-TensorFlow-Lab/blob/master/solutions.ipynb).*" 571 | ] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "execution_count": 40, 576 | "metadata": { 577 | "collapsed": false 578 | }, 579 | "outputs": [ 580 | { 581 | "name": "stderr", 582 | "output_type": "stream", 583 | "text": [ 584 | "Epoch 1/5: 100%|██████████| 1425/1425 [00:05<00:00, 283.35batches/s]\n", 585 | "Epoch 2/5: 100%|██████████| 1425/1425 [00:05<00:00, 281.62batches/s]\n", 586 | "Epoch 3/5: 100%|██████████| 1425/1425 [00:05<00:00, 283.69batches/s]\n", 587 | "Epoch 4/5: 100%|██████████| 1425/1425 [00:05<00:00, 256.39batches/s]\n", 588 | "Epoch 5/5: 100%|██████████| 1425/1425 [00:05<00:00, 284.96batches/s]\n" 589 | ] 590 | }, 591 | { 592 | "data": { 593 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkwAAAGGCAYAAACJ/96MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xl4FEX+P/D3J5AQiNyH6IonKEG8CLoe6HrjBeKKrhF1\nd0EU12tZfl9dddVwyg0it4hAkCiKaEQuFRQEBEm4QUCQQ+5whJCQc96/P3oyTpJJZgJJZqLv1/PM\nA1Nd1V1dmXQ+U1VdbSQhIiIiIsULC3YFREREREKdAiYRERERPxQwiYiIiPihgElERETEDwVMIiIi\nIn4oYBIRERHxQwGTiIiIiB8KmERERET8UMAkIiIi4ocCJhERERE/FDCJSEgws7+bmcvMWgW7LiIi\nhSlgEpFQoodbikhIUsAkIiIi4ocCJhGpNMysoZm9Z2b7zeykma02syd85HvEzFaa2XEzSzWztWb2\ngtf2qmb2ppltce8nxcwWm9ltFXtGIlJZVA12BUREAmFmkQC+BXARgHcA7ADwEIBJZlab5DvufHcA\nmAbgKwAvuYtHA7gOwAj3+54A/gtgPIAfAdQC0BpAKwDflP/ZiEhlo4BJRCqLpwE0B9CJ5IcAYGZj\nASwC0MfMJpJMB3APgGMk25awr3sAfEnymfKutIj8PmhITkQqi7sB7M8PlgCAZB6cXqMzAPzFnXwM\nwBlmVlLAdAzApWbWtLwqKyK/LwqYRKSyOA/AVh/pmwCYezsAjAawBcBsM9vtnvNUOHh6A0AdAFvc\n85sGmNll5VVxEan8FDCJSGVhgWQieQjAlQDaA/gcwM0A5pjZ+155FsOZC/VPAOsAPAkg2cw6l3Gd\nReR3QgGTiFQWOwA085Ee7f53Z34CyVySX5J8juRFAMYBeMLMLvTKc4zkZJKdADQBsBZAXHlVXkQq\nNwVMIlJZzAbQ2Mz+lp9gZlUAPA8gDcB37rR6Psquc/9bzVcekhkAfs7fLiJSmO6SE5FQYgC6mNnd\nPra9DedOuUlm1hq/LStwHYAX3XfIAcAEd0C0AMCvAM4H8ByA1SQ3ufNsNLNvASQBOALgagAd8duy\nAyIiBRipJxGISPCZ2d8BTCwhSxMA2QD6A2gHZ+2kzQCGkIz32s8DAJ6CM4+pDoD9cHqnepI86M7z\nCpw5ThfD6VXaCWAKgMHuO+9ERApQwCQiIiLix2nPYTKzG80s0cz2uJ803t5Hnmgz+9zMjpnZCTNb\nbmbnnO6xRURERCpCWUz6jgKwGsCz8PGkcTO7CMBiABsB3ATgMgC9AWSWwbFFREREyl2ZDsmZmQtA\nB5KJXmkJALJJ/r3MDiQiIiJSgcp1WQEzMwD3AthqZnPN7ICZ/WBm95fncUVERETKUnmvw9QIzjOe\nXoZzl8odAGYC+NTMbiznY4uIiIiUifJehyk/IPuMZP76JmvN7HoA3eDMbSrCzOoDaAtnnRXNdRIR\nEZHyEglnvbZ5JA8Xl6m8A6YUALlwHo7pbROAG0oo1xbAB+VVKREREZFCOgGYVtzGcg2YSOaY2Y8A\nLim06WJ4PffJhx0AMHXqVERHR5eQ7Y+pe/fuGDZsWLCrUSmorQKntgqc2qp01F6BU1sFrqzaatOm\nTXjssccAd+xRnNMOmMwsCkBT/PYk8QvN7AoAR0juBjAIwIdmthjAQgB3A7gPwF9K2G0mAERHR6NV\nq1anW8Xfndq1a6tdAqS2CpzaKnBqq9JRewVObRW4cmirEqcAlUUPU2s4gRDdryHu9MkAOpP8zMy6\nAXgVzrOgNgP4K8llZXBsERERkXJ32gETye/g5247kpMATDrdY4mIiIgEQ3kvKyAiIiJS6SlgqoRi\nY2ODXYVKQ20VOLVV4NRWpaP2CpzaKnAV3VZl+miUsmJmrQAkJSUlafKbiIiIlJvk5GTExMQAQAzJ\n5OLyqYdJRERExA8FTCIiIiJ+KGASERER8eO0AyYzu9HMEs1sj5m5zKx9CXnHufO8cLrHFREREako\nZdHDFAVgNYBn4Sxc6ZOZdQBwDYA9ZXBMERERkQpTFgtXzgUwFwDMzHzlMbM/ARgB56G6s0/3mCIi\nIiIVqdznMLmDqCkABpLcVN7HExERESlrFTHp+78AskmOrIBjiYiIiJS5snj4brHMLAbACwCuKs/j\niIiIiJSncg2YALQB0BDAbq/pTVUADDWzf5O8sKTC3bt3R+3atQukxcbGaul4ERERKbWEhAQkJCQU\nSEtNTQ2obJk+GsXMXAA6kEx0v68L4KxC2ebDmdP0PsmtxexHj0YRERGRchfoo1FOu4fJzKIANAWQ\n34V0oZldAeAIyd0AjhbKnwNgf3HBkoiIiEioKYshudYAFsJZg4kAhrjTJwPo7CN/6D3tV0RERKQE\nZbEO03coxd12/uYtiYiIiISakH6WXG5ebrCrICIiIhLaAdPJ3JPBroKIiIhIiAdMOQqYREREJPhC\nO2BSD5OIiIiEgJAOmDJzM4NdBREREZHQDpg0JCciIiKh4LQDJjO70cwSzWyPmbnMrL3XtqpmNsDM\n1prZCXeeyWZWePVvnzQkJyIiIqGgLHqYogCsBvAsii5KWQPAlQB6wnkA7wMALgHweSA71pCciIiI\nhIKyWLhyLoC5AGBeT9h1bzsOoK13mpk9B2C5mZ1D8teS9q0eJhEREQkFwZjDVAdOT9QxfxnVwyQi\nIiKhoEIDJjOrBqA/gGkkT/jLr0nfIiIiEgoqLGAys6oAPobTu/SvQMqoh0lERERCwWnPYQqEV7DU\nBMCtgfQuAcAXI77A7sTdBdJiY2MRGxtb9pUUERGR37WEhAQkJCQUSEtNTQ2orJGFb2w7dWbmAtCB\nZKJXWn6wdCGAW0geCWA/rQAkPTLiESQ8n+Avu4iIiMgpSU5ORkxMDADEkEwuLt9p9zCZWRSApgDy\n75C70MyuAHAEwF4AM+AsLXAfgHAzO9Od7wjJnJL2rSE5ERERCQVlMSTXGsBCOHOTCGCIO30ynPWX\n2rnTV7vTzf3+FgCLStqxlhUQERGRUFAW6zB9h5Inj5/yxHIFTCIiIhIK9Cw5ERERET9COmDSHCYR\nEREJBSEdMGlITkREREJBSAdMmTnqYRIREZHgC+2ASUNyIiIiEgJCOmDSkJyIiIiEgtMOmMzsRjNL\nNLM9ZuYys/Y+8vQys71mlmFmX5lZ00D2fTLnJMpyJXIRERGRU1EWPUxRcBalfBbOgpQFmNnLAJ4D\n8DSAawCkA5hnZhH+duyiC9l52WVQRREREZFTVxYLV84FMBcAzMx8ZHkRQG+SX7jzPAHgAIAOAKb7\n239GTgaqVa12utUUEREROWXlOofJzC4A0BjAN/lpJI8DWA7gukD2kZ6TXj6VExEREQlQeU/6bgxn\nmO5AofQD7m1+pWcrYBIREZHgCtZdcvkP4PUrIyejnKsiIiIiUrLTnsPkx344wdGZKNjL1AjAKr+l\n5wLPb38e9arX8yTFxsYiNja2jKspIiIiv3cJCQlISEgokJaamhpQWSvL2/bNzAWgA8lEr7S9AAaR\nHOZ+XwtO8PQEyY+L2U8rAEl4Cpj7f3PRtmnbMqujiIiISL7k5GTExMQAQAzJ5OLynXYPk5lFAWgK\npycJAC40sysAHCG5G8BwAP8zs58B7ADQG8CvAD4PZP+a9C0iIiLBVhZDcq0BLIQzJ4kAhrjTJwPo\nTHKgmdUAMA5AHQCLAdxNMqAFljSHSURERIKtLNZh+g5+Jo+TjAMQV9p9m5nukhMREZGgC+lnyVWv\nWl1DciIiIhJ0IR0wRVaN1JCciIiIBF1IB0zVw6trSE5ERESCLrQDJg3JiYiISAgI7YBJPUwiIiIS\nAkI6YIqsGomMXM1hEhERkeAK+YBJPUwiIiISbOUeMJlZmJn1NrPtZpZhZj+b2f8CKVs9XHOYRERE\nJPjK++G7APBfAE8DeALARjgrg08ys2MkR5ZUMLJKJA7mHKyAKoqIiIgUryICpusAfE5yrvv9LjN7\nFMA1/gpq0reIiIiEgoqYw7QUwG1m1gwA3A/mvQHAbH8FtayAiIiIhIKK6GHqD6AWgJ/MLA9OkPYa\nyQ/9FaweXh3p6QqYREREJLgqImD6G4BHATwCZw7TlQDeNrO9JONLKjhv1DykZKag/aL2nrTY2FjE\nxsaWZ31FRETkdyghIQEJCQkF0lJTUwMqayTLo06/HcBsF4B+JMd6pb0GoBPJFsWUaQUg6dX4V9F/\ne3/kvp4LMyvXeoqIiMgfT3JyMmJiYgAghmRycfkqYg5TDQCFozJXIMeuHl4dLrqQlZdVLhUTERER\nCURFDMl9AeA1M9sNYAOAVgC6A5jgr2Bk1UgAQEZOhuf/IiIiIhWtIgKm5wD0BjAKQCMAewGMcaeV\nKD9ISs9OR73q9cqxiiIiIiLFK/eAiWQ6gP+4X6VSvWp1ANDSAiIiIhJUIf0sOU/ApMUrRUREJIhC\nOmDynsMkIiIiEiwhHTBVD9eQnIiIiARfaAdMGpITERGREBDSAZOG5ERERCQUhHTAFF4lHFWsiobk\nREREJKgqJGAys7PNLN7MUswsw8zWuB9/4q8coiKiNCQnIiIiQVXu6zCZWR0ASwB8A6AtgBQAzQAc\nDaR8VHiUephEREQkqCpipe//AthF8kmvtJ2BFq4RXkNzmERERCSoKmJIrh2AlWY23cwOmFmymT3p\nt5SbhuREREQk2CoiYLoQwDMANgO4E8BYACPM7LFACmtITkRERILNSJbvAcyyAKwgeaNX2tsAWpO8\noZgyrQAk3XTTTdiYthERYRGIOTsGABAbG4vY2NhyrbOIiIj8/iQkJCAhIaFAWmpqKhYtWgQAMSST\niytbEQHTDgDzST7lldYNwGskmxRTphWApKSkJPTc0hMuuvBF7BflWk8RERH540lOTkZMTAzgJ2Cq\niCG5JQAuKZR2CQKc+B0VrjlMIiIiElwVETANA3Ctmb1iZheZ2aMAngQwMpDCmsMkIiIiwVbuARPJ\nlQAeABALYB2A1wC8SPLDQMprWQEREREJtopYhwkkZwOYfSpltayAiIiIBFtIP0sO0JCciIiIBF/o\nB0wRURqSExERkaAK+YCpRngNpGenw0VXsKsiIiIif1AhHzA1qdUEBLE7dXewqyIiIiJ/UCEfMEU3\njAYAbErZFOSaiIiIyB9VyAdM59Y+F9WrVsemQwqYREREJDgqPGByL2DpMrOhgeQPszBc0uAS/JTy\nU3lXTURERMSnCg2YzOxqAF0BrClNuegG0RqSExERkaCpsIDJzM4AMBXOY1GOlaasAiYREREJpors\nYRoF4AuSC0pbsHmD5kjJSEFKRko5VEtERESkZBUSMJnZIwCuBPDKqZTPv1NO85hEREQkGMo9YDKz\ncwAMB/AYyZxT2Uezes0QZmG6U05ERESCoiIevhsDoCGAJDMzd1oVADeZ2XMAqpGkr4Ldu3dH7dq1\nAQDVf6mO/rP744wXz0BsbGwFVFtERER+TxISEpCQkFAgLTU1NaCyVkysUmbMLArAeYWSJwHYBKA/\nySLdRmbWCkBSUlISWrVqBQBon9AeOa4czOk0p1zrKyIiIn8cycnJiImJAYAYksnF5Sv3HiaS6QA2\neqeZWTqAw76CpeJEN4jG9I3Ty7p6IiIiIn4Fa6XvUndrRTeMxs5jO5GRk1Ee9REREREpVlACJpK3\nkvxPaco0b9AcBLE5ZXN5VUtERETEp5B/lly+6AZ6CK+IiIgER6UJmGpH1sZZZ5yltZhERESkwlWa\ngAlw5jGph0lEREQqWqUKmJrXb67FK0VERKTCVaqAKbphNLYc3oJcV26wqyIiIiJ/IJUrYGoQjRxX\nDn45+kuwqyIiIiJ/IBXxLLlXzGyFmR03swNmNtPMLj6VfeU/hLc085gOZxzGkZNHkJGTARddp3JY\nj9MtLyIiIpVTRfQw3QjgHQB/BnA7gHAA882seml3dNYZZ6FmRM2A5zEN/2E4GgxqgPoD6yOqXxSq\n9KqCu6behTxXXqmOu2b/Glz/3vW4YuwVSMtKK221RUREpJIr94CJ5D0k40luIrkOwD8AnAvnobyl\nYmaIbhiNpH1JyMzNLDFv0t4kvPTVS3jyqicx4+EZ+OCvH2DInUMwf9t8DP9heEDHS8tKQ495PRAz\nPgapWanYeWwnuiR2QXk/fy8U5OTlYM/xPcGuhoiISEgo92fJ+VAHzqNRjpxK4VaNW2Fs0ljM2DQD\nF9a9EC0atkDXVl1x38X3efKcyD6B2BmxuPzMyzHq3lGIqBLh2bY7dTf+t/B/aH9JezSr36zAvrNy\ns7Dh0Aas2rcKq/avwmc/fYYjJ4+g76190f267kjcnIiHPn4IbVa0wQt/fuGUTr6y6Lu4L4YuG4rd\n3XejdmTtYFdHREQkqCp00reZGYDhAL4nudFffl+G3zUcSzsvxfj7xqP9xe1xKP0Q2iW0w3Ozn8PJ\nnJMAgOfnPI+9aXuR8GBCgWAJAPre1hd/qvkndEnsUmBOUvyaeDQa3Agx42Pw1KynsOCXBWh7UVts\nfHYjXm7zMiKqRKBji47495//jR7ze2DZ7mWn3A6hLis3C2NWjkFadhqmrJkS7OqIiIgEXUXfJTca\nQAsAj5zqDqpVrYbrmlyHLq26YEjbIVjSeQlG3TMK7616D9dMuAb9FvfDpNWTMPKekUV6kACgRngN\nTGg/AYt3LcboH0cjLSsNj898HE989gQ6NO+AZV2W4fh/j2Pjsxvx3v3v4fw65xcoP+COAbj67Kvx\n8CcP41D6IZ91XLJrCS4dfWmlXTPqk42f4GD6QVx99tUYvXJ0qYcgdxzbgVErRpVT7URERCqeVdR8\nHDMbCaAdgBtJ7vKTtxWApJtuugm1axccDoqNjUVsbGyRMusPrscjnzyCDYc24JGWj2DaX6fB6dDy\n7dkvn8XkNZPR+IzGOJB+AGPvHYtOl3cK6Fx+Pf4rrhp3FW694FZ81PGjAttIos37bbB091K0aNgC\nK55cgaiIqBL3t+HgBlQJq4Km9ZqialgwRkkLuv6961EjvAZeu/E13DrlVix4YgFuueCWgMt3+LAD\nPt/8OdY9sw4tG7Usx5qKt0mrJ2Hiqon4+omvi/SsiogIkJCQgISEhAJpqampWLRoEQDEkEwurmyF\nBEzuYOl+AH8huT2A/K0AJCUlJaFVq1YBH+dkzkkkrE/AQy0eQs1qNUvMm5aVhqvGXYW61esi4cEE\nNK3XNODjAMDk1ZPxj8//gcX/XIw257bxpM/fNh9tp7bFkDuH4PWFr+OB5g8g/oH4YoO38Unj8fSs\npwEA1apUQ4uGLdCsfjNk52UjLSsNadlpuKDOBRjfbjxqVatVqjqeiuR9yYgZH4NPH/4UHZp3QIvR\nLXBZo8sw/aHpAZVP2puE1u+2hsHQ47oeGHTnoHKucfGycrOQmZv5h5iDlZ6djgvevgCHMg5hzL1j\n0K11t2BXSQJAEpm5mageXuqbhkWkjCQnJyMmJgbwEzBVxDpMowF0AvAogHQzO9P9iizrY1UPr47O\nV3X2GywBQM1qNbH+X+ux/MnlpQ6WAODxKx5HzFkx+Pfcf3vmQpHEGwvfwLXnXIvu13bHu+3exQfr\nPsC4pHE+9/H+qvfx9Kyn8a/W/8KCJxZgwO0D0OqsVjiYfhBZuVmoX6M+WjZsiXnb5uGWybcUOwTo\n7WTOSaw/uB6f/fQZBi0ZhImrJpbqvEatGIUmtZqg3SXtYGZ4pvUzmPnTTOxN2xtQ+bjv4nBJ/UvQ\nrXU3xK+NP61V2VMyUtA1sSueTHyy1PvZdmQbYsbHoMmwJhi6bChy8nJKzH8o/RDe/uFtv/lC1egf\nR+No5lHcdsFt6L2ot2c+X6BW71+NPov6/CHuAA0l/5n3H1w88mKkZqYGuyoi4g/Jcn0BcAHI8/F6\nooQyrQAwKSmJoWzxzsVEHDh59WSS5JdbviTiwPk/z/fk+desfzGidwR/3PNjgbLxa+Jpccanv3ia\nLperxOOs3reajQY1YvORzbk7dXex+RLWJTCidwQRByIOjOwTScSBP+z+IaDzOZxxmJF9Itl3UV9P\n2tGTR1mjbw32/LZngby+6rz81+VEHPjB2g+YtDeJiANnbZ4V0LEL73vSqkmsP6A+6/Svwyo9q7DL\n5118HnPbkW08mXOyQNpX275i3f512WxEMz6V+BTDeoYxemQ0v9r2VbHHfPLzJ4k4sMe8HqWub7Cl\nZaWxwcAG7JrYlVsPb2WVnlU4ZOmQgMtn52bz0lGXEnHgkl1LyrGm4m31vtUM6xlGxIHd53YvdXmX\ny8W0rLRyqJmUZPiy4bz3g3uZk5cT7KpIgFwuF7Nys4rdnpSURDh377diSfFMSRuD9aosARNJPjT9\nIZ495GymZaWx9fjWvOG9Gwr8Yc/MyeTV469mWM8wXvT2RWw3rR2f/uJphvUMY5fPuzDPlRfQcbak\nbOG5w87lecPO46ZDm4psn75+Oqv0rMJHZzzK73d+z/1p+5mbl8vLRl/Gm96/yWewMW3tNM7dOpfZ\nudkkyUFLBjGidwQPnDhQIN+Tnz/JPw35E3PycpiWlcZ+i/qx3oB6jP0ktsAF++6pdzN6ZDRz83Lp\ncrl42ejL2HF6x4DOL99Ph37izZNuJuLATjM6cX/afsaviSfiwJe/etmT71D6IT4641EiDqzRtwbv\nT7if41eO54DvBzCsZxjbxrflkYwjJJ0/TG0mtiHiwF7f9ipyzPwg49oJ1xJx4CcbPvFbzx1Hd/CH\n3T9w8c7F/Gb7N5z/83wu3bWU6w+s565ju5iZk1mq8z4d/Rf3Z3ivcO44uoOk8/NqMLBBwH9Mhy8b\nToszNh7cmA9//PBp1eXAiQPsMa8He37b0+8XAV++2/Edp6yewqMnj/rN+8vRX/j6gteLBMyVgcvl\n4o0Tb2T0yGj2/q43q/SswvUH1pdqH89++SzPHHQmj2ceL6daSmGr9q1i1V5ViTiw/+L+pS7vcrkq\n9NogTpt3mtGJF79zcbG/KwqYKsj2I9tZrXc13vT+TUQc+M32b4rkOZJxhBOSJrDHvB68e+rdvGD4\nBXz2y2cDDpby7U7dzeYjmzOidwRfmv8SUzNTSZIzN81k1V5V+eiMR5mbl1ugzJytc4g48POfPi+Q\n/v6q9z09UXX61+ETM5/gucPO5WOfPlbkuPm9RY9/+jgbDGzAiN4R7DSjE8/odwabj2zODQc3cOmu\npUQc+OG6Dz3lhiwdwojeEUxJT/F7bidzTvLNhW8yoncEL3r7ogK9dCQ5bNkwIg4cvGQwP1r/ERsO\nbMi6/etyzI9j2H9xf7aZ2Mbzbf3/5v9fkXZwuVx85etXWKVnlSK9fY9/+jjPGnwW07PT2XF6R9bs\nV5ObUzYXW9e3f3ibFmee9vP1qjegHkcuH+n3W+i6A+vYcXpH/n3m3/nK16/wneXvMGlv4J/745nH\nWX9AfT79xdOetJ3HdjKidwT7fNfHb/kDJw6w9lu12e2Lbnxn+Tus0rMKdx3bFfDx8x09eZSvffMa\no/pGMapvFBEHn8c/lH6Ivb7txTlb5xT4GR08cZCPffqYp/2q9a7GBz96kDM3zfQE9N5cLhdvn3I7\nEQfGfhJb6t+lYEtYl+Dpjc7MyeTF71zMWybdEnCQOf/n+Z62Gvj9wNOqS54rjwt/WcjV+1afUvnc\nvNwSv717S81M5dytc08pmA62rNwsXjHmCl42+jK+OOdFVutdrcTrhC9dE7vyguEXMD07vZxqKYWN\nXzmeiAMjekfwqcSnfOZRwFSBXvn6FSIO/Mv7fyn3C8GJrBN8c+GbrN6nOhsNasSX5r/E8F7hfGj6\nQz7/OLtcLt42+TY2H9ncsz1pbxKr9a7GLp934ap9q/i/b/7H6JHRDOsZxhW/rvB53OsmXMeqvary\nqcSnuPPYTpLkpkObeOmoSxnVN4otRrXgpaMuLfCH68CJA6zaqypHLh9Z7PmkZ6dz7ta5vPidixne\nK5yvffMaM7IzfObNb2fEgX/96K/cl7avwPbDGYe58eDGYo+VnZvNK8deyZajW3q+5W08uJEWZ546\npmam8pJ3LmHL0S15IutEgfK5ebl8cc6LRBz4n7n/4dr9a7np0CZuO7KNvxz9hesOrOOSXUs4Z+sc\ndv6sMy3O2GJUC87dOtdnfVbtW8X6A+rzwrcv5HUTruN5w85jRO8IRvSOCLi3od+ifgzvFe75meR7\nfvbzrP1WbR7JOMJNhzZxxA8j+Mgnj3DSqkkFPqOdP+vMuv3rMiU9hcczj7PWW7UK9OT5k5OXw+HL\nhrNO/zqs3qc6X/7qZR7OOMxe3/Yi4sDxK8d78ibtTeJ5w87zfENvMrQJX1/wOkevGM16A+qx3oB6\nnJg8kb+m/srBSwbzqrFXEXFgx+kdi/xefbrxUyIO/NesfxFx4GvfvFZiPXPzcjlr8ywOXjKYL8x+\ngR0+7MDr37ueN71/E++Mv5PtprVj97ndufXwVr/nfPDEQQ5eMpjvr3qfC39ZyB1HdxQJ0EuSlpXG\nPw35Ex/48AFP2tytc4k48KP1H/ktn5qZyiZDm/C2ybex82edeeagM0+pl+3AiQMc8P0ANh3RlIgD\no/pGcemupUXyrdq3irdPuZ0Dvx9Y4MtPZk4mR68YzSZDmzCyTyTbxrfl0KVDueHghmKvgx2ndyTi\nwNcXvF7pgqa4hXGs0rMKk/YmMT07nRe9fRFvnHhjwMH6xOSJRBxoccbBSwafVl1SM1M5Y+MMrt2/\n1uf23Lxczvt5XpFrWL49x/dwc8rmgH4GGw9uZM9ve1bKntwNBzewep/qfCrxKY75cQwRB87eMrtI\nPgVMFeh45nF2+LADV+5ZWWHH3J262/ON/IEPH/D5LTxf8t5kIg4ct3IcU9JTeN6w89h6fOsivwD5\nPVa+HDxx0Of8qRNZJ9hpRiciDpyxcUaR7e0T2rP1+NYF6t3ti25sPb41Gw5s6AmA2kxs4zdIcLlc\nHLRkUEBDZsVZvW81q/aqytcXvE6SfPjjh3nusHMLdJOvP7CeNfrW4A3v3cBBSwbx21++5b60fbw/\n4X6G9QzjqBWjAjpW0t4kT8/jHVPuKPDHaOWelazbvy5jxsXwcMZhT/rJnJOMHhnNVuNa+fyZLv91\nOePXxHN5KluhAAAgAElEQVTUilF8a/FbrDegHp+Z9UyRfPvS9rF6n+o8o98ZRBwY3iucV4y5wlOX\n7Ue2e+aceZ9P97ndWbd/3YC+AS/dtZRXjLnCMxdv7/G9nm0ul4vPfvksw3qGceammZyyegoj+0Sy\n9fjW3HVsF5f/upxPJT7lqd/jnz7OgycOFjnGR+s/IuLAYcuGedIysjN4/vDzec8H95AkB34/kIgD\n30t+z2c95/08j5ePudwTFLQY1YJ3Tb2Lf5/5d3aa0YkPfvQg7/3gXjYY2IAWZ+zwYQcu2rHI5x+T\nnw//zKYjmjK8V3iBHsWWo1uW+Pvj7dWvX2Vkn0huP7K9QHqHDzvwnKHnMC0rjVtStjB+TTz/b/7/\nFZl71zWxK8/odwZ3HN3BLSlbGNYzjKNXjA7o2CR57OQxPj/7eYb3Cme13tX4+KeP85vt3/DGiTey\n9lu1mbw32ZP3q21fsWa/mjx/+PmM6B3ByD6R/Odn/+SQpUN4ztBzGNYzjI/OeJSDlgziHVPuYLXe\n1Yg48JlZzxRpv483fOy5XgUSNLlcLm5O2czp66fzzYVvsuP0jowZF8PmI5vzvGHnsdGgRrxq7FV8\nN+ndYr9k5dtwcAOf+/I5vrHgDb6b9C7nbp3LQ+mHAm6z/KG4/OsGSS7YvoCIA8f8OMZv+TX71zCy\nTySf/PxJPpX4FBsMbFDqodSU9BSO+GEE75hyh+fzF9knssgXsqzcLD40/SEiDqzbvy5f/upl7k7d\nzTxXHr/a9hUf+PABT2/8mYPO5EPTH+I7y9/xOQyelpXGi9+52NMhEMhQeWHHTh7j6n2rOXPTTA5b\nNozxa+IrZFgyIzuDLUe35KWjLmV6djpdLhfbxrflWYPPKnDNJRUw/WHsOLojoMmHj336GM8cdCZv\nn3I76w+o75nvUhZcLhd/Pvyzz235PQHLf13OuIVxrN6nOhsObMjOn3Vmr297cfLqyfx+5/cVOqSS\n/00xf1hyQtKEInm+3PIl20xswxp9a3j+KEb1jSr1JHaXy8UZG2d4JlW3jW/LSasmsfZbtfnnd//s\n8wL0454fWaVnFb658M0C6UOWDvHUpWqvqqw/oD5jxsXw19RffR57yuop/M/c/3D2ltmeb5qzt8zm\nucPOZY2+NXjB8At4xZgrCvSObD+ynWE9wzhu5bgi+0tJT+Gy3csYvyae//jsH0QcGDMuptheydy8\nXD40/SFW6VmFiAP/+dk/iwTpJ7JO+O3V6TGvB6v2qsrvd35Pkuz5bU+G9wr3DIe4XC4+/cXTrNqr\nKsetHMcvt3zJOVvnMPGnRN4ZfycRB97w3g1cumtpiX+gM7IzOH7leDYf2ZyIA6+dcC1nb5ntKbNy\nz0o2GtSIzUY047Yj25iRncFNhzZx5qaZrNmvJv/28d+K3X9Kego/2/QZe8zrwYjeEXxjwRtF8mw/\nsp2RfSI9QQfiwEaDGhFx4J3xd3LVvlWenijvn88jnzzC84efX+KXpvx2+nDdh2w8uDGj+kbxrcVv\nFfjDkZqZyqvHX80GAxtww8ENjF8Tz6q9qvLuqXczLSuNB04cYL9F/dhkaBOG9QxjpxmdisynTM9O\n5/Blw4vM8TmUfogNBzbkAx8+QJfL5QlyfQVNWw9vZa9vezF6ZLSnHRoObMibJ93Mrold2X1ud776\n9avs/V1vtpvWjhZnrDegHv/71X99/i4k/pTImv1q8qzBZ/HsIWd7htMbDWpUpGfWl6Mnj3qG4goP\nPXZN7Mqa/Woy8adEvv3D2+z8WWfe8N4NfOXrVzx1Sc1MZbMRzXjFmCuYkZ3BXcd2BTxkTjoB0LBl\nw1infx2G9wrnnfF38p3l73BzymbeN+0+RvSO4BebvyDpfIbv+eAeRvSO4LtJ77LHvB6s9VYtVu1V\nlecNO4+IAy8ddSlHrxjNOVvn8JWvX+EN793A8F7hvHLslZ55n/n++dk/GdU3ipNWTWLd/nXZcnTL\nYq83+QHu5NWT2WNeD94ZfyfPGnxWgS8W+TciNR7cmP0W9StyvHx5rjz2W9SPZw0+i81HNudN79/E\njtM78t2kdwNqM5J8ZtYzjOwTyXUH1nnSfk39lXX612HsJ7GetCMZRzhh1gQFTPKbHUd3sFrvagzr\nGVbi3WJlLSs3i/UH1GeVnlU8c6+OnTxWYccvrk75vS0XvX1RiX9ocvJyuO7AOk5ZPcXnZPtA5bny\nOH39dE/gdP1715fYI/HGgjdYpWcVrtyz0jP/CnHgf7/6LzOyM05rOON45nE+P/t5RvaJ5OKdi4ts\n7/BhB7YY1cJzZ8m4leN4/vDzC1z4zhl6DkcuH+l3KCozJ5NdPu/CsT+OPeU6Z+dms83ENjx7yNlc\n8esKRvaJLDJsmJOXw/um3VdkLlmzEc04c9PMUh07z5XHWZtneW4CaD2+NQctGcSovlG85t1rSuwJ\nG/vj2ALpa/av4dXjry7Qbl0+71JsD94nGz5hr297cd7P83g44zBdLhc/3fgpL3nnElqcsWa/mrxj\nyh0FzmfN/jVEHDhl9ZQC+1p3YB0/3vAxRy4fyf998z/eMukWz3B2cfPUDmcc5uVjLmftt2oTcWDn\nzzoX+f3IycvxOy/x9QWvF5jTGPtJLOsNqFdgGH3A9wOIOPDWybfytsm38erxV/Oity/yfDnpNKMT\nE39K9Nne3n4+/DP/PeffrPVWLYb3CmeXz7t4hpveWvyWp9cw/yaIrNwsbjq0iecPP5+txrXy+bM4\nlH6IE5Im8O6pd3t64nzNLTx68ijPHnK2Z95dq3GtPPMg8+eV3jftPtbsV5NbUrZ4yr0w+wXPkLkv\n6dnp3HVsF2dsnMFmI5oxrGcYn/7i6SI35GTlZvGvH/2V4b3COWX1FN486WZW71Od836e58lzPPM4\nhy8bzqcSnyq253Tt/rWsP6A+rx5/tee6NG3tNCIOfH/V+ySdXromQ5uwydAm/Gj9R5yyegrf/uFt\nvrnwTbZPaM8GAxt4PucXvn0h70+4n6998xo/WPsBl+1exv1p++lyubjp0CZ2TezKar2rMapvFF+Y\n/UKB3tZD6Yd419S7aHHGpxKfYve53dlpRifPjTu+vuDmy8jO4PT109luWrtie/+mrpnqGdXI/9nh\nKShgkoImr57M+DXxFX7cd5a/w8c+fYzbjmyr8GMXZ9W+VTyj3xkBzRkpS3muPC7YvqDYuQX5snOz\nedXYq9hiVAt2TezqmfBelorrmVz4y0LPre7nDz+fFmd85JNHOH39dK7atyood2XtOb6HjQY1Yniv\ncJ41+CyfdXC5XNyXto97ju/h7tTd3Hls52nd+u1yufjVtq88F+p7P7i3xJ/bM7OeYbXe1bhq3yq6\nXC5OSJrAyD6RvHzM5YxfE39avbo5eTkc++NY3jb5Np+9Iu2mtWP0yGjmufK4at+qAsFjeK9wnjP0\nHF7/3vUB9ZAeOHGAbSa2OeU7HUmn7R779DFG9I7gq1+/SsTB57VnzI9j2Da+LR+a/hCf/PxJ9pjX\ngx+t/+iUJkWnZqZywPcD2HhwY1qceYZhX1/wus8e7NX7VrNG3xqM/STWc565ebkcsnQII/tE0uKM\nN71/E0f8MKLYXhXSmWaw7sC6Ap+11MxUDls2jBcMv4CIAz/e8HGBMvvS9rFG3xqeuXcul4sfrf+I\nLUe39PTC5L/umHJHsXOVSOez8cgnjxBxYK23avn8EhSIpL1JrNO/Dq9/73qu3b+Wtd6qVaBtSKeH\n5rLRlxXoMWo8uDFvnXwrX1/wOudunRvwF+L9afv5xoI3WH9AfYb1DOPDHz/M+DXxPGfoOWwwsEGB\noC+/jbp90Y1VelYpsm3P8T3smtiVtd6qRcSBf373zxy3cpzPz6/L5WKPeT3YPqE9X/36VU5bO40f\nzv8wtAImAM8C+AXASQA/ALi6hLwKmEowbdq0YFeh0iiprUL99t51B9YxoncEw3qGcWLyxHI/Xn5b\nuVwuTw/cgx89WKBLO5gWbF/AyD6RTFiXUKHHdblc3HhwY4E/iL4+VydzTvLKsVey2YhmfPzTx4k4\nsGtiV7/za8rCst3LPEOP+T1rH6z9wNNLFQxZuVmeJUJaPdeqwupxMuckx/44ltdOuNbvF6Lp66cT\nceCA7wfwp0M/8boJ19HijC/OebHITSWnIjcvt9gvii9/9TKj+kZx5qaZvObdazxB+eM9H2f8mnjO\n2TqHa/avCajdcvNyOXjJ4FO+0zHfD7t/8PSOXTD8Ap+94Dl5Odx7fG+ZTQJPz07n6BWjPTcftJnY\nptgANScvh/d8cA9r9qvJNfvXcOrUqRzxwwjW7FeTjQY14psL3yzQkxeokJrDBOBvADIBPAGgOYBx\nAI4AaFBMfgVMJWjXrl2wq1BpVPa2mrV5Fr/e9nWFHMu7rXYe28kNBzdUyHFLoyKCj0AU97nakrKF\nNfvVZI2+NSq8N/fuqXezydAmnJA0IWQWVTyScYQvzX+Jd959Z7CrUqz/ffM/Wpwxsk8km41odso9\nNKWVkp7Cmv1qeoZ9F/6ykGTwr1mLdy7m5WMu5/Jfl1focXPzcrlm/xq/c/HSstJ41direM7Qc1j7\n8tq0OGO3L7oVO7wZiEADpqqnvkZ4qXQHMI7kFAAws24A7gXQGcDACqqDSKVz78X3BuW459Y+NyjH\n9SfUn7nWrH4zLO2yFDXCa+DCuhdW6LETYxMRZmEIs3J/4lXA6laviwF3DED7d9oHuyrF6nlLT6Rk\npKBmtZqIuzkONcJrVMhx69eoj+kPTceJ7BP4a/RfQ+bn1ubcNljTbU2FH7dKWBVcfublfvOdEXEG\nZj06C9e/dz1cdGFpl6W49pxrK6CGKP+AyczCAcQA6JefRpJm9jWA68r7+CIiFallo5ZBOW7VsIr6\n/vv7EmZhGHPfmKAc+66mdwXluJXd2TXPxubnNqPjgo4VFiwBFfDwXQANAFQBcKBQ+gEAjSvg+CIi\nIvI7Uq1qNRisQo8ZzK8kBmfM0JdIANi0aVPF1aYSSU1NRXJycrCrUSmorQKntgqc2qp01F6BU1sF\nrqzayivWiCwpn5HFxSxlwz0klwHgQZKJXumTANQm+YCPMo8C+KBcKyYiIiLym04kpxW3sdx7mEjm\nmFkSgNsAJAKAmZn7/Yhiis0D0AnADjh314mIiIiUh0gA58OJPYpV7j1MAGBmDwOYDOBpACvg3DXX\nEUBzkofKvQIiIiIip6FC5jCRnG5mDQD0AnAmgNUA2ipYEhERkcqgQnqYRERERCqz0FgpS0RERCSE\nKWASERER8UMBUxCY2Y1mlmhme8zMZWZFnhtgZr3MbK+ZZZjZV2bWtND2umb2gZmlmtlRM5tgZlGF\n8lxuZovM7KSZ7TSz/yvvcytrZvaKma0ws+NmdsDMZprZxYXyVDOzUWaWYmZpZvaJmTUqlKeJmX1p\nZulmtt/MBpoVfBaBmd1sZklmlmlmW8zs7xVxjmXFzLqZ2Rr3ZyLVzJaa2V1e29VOxXB/zlxmNtQr\nTe0FwMzedLeN92uj13a1kxczO9vM4t3tkeH+nWxVKM8f/vpuZr/4+Fy5zOwd9/bQ+1yV9KA5vcrt\nYcR3wZkA3wFAHoD2hba/DOfhxO0AtATwGYBtACK88swBkAygNYDrAWwBMNVre00A++DcnRgN4GEA\n6QCeDPb5l7KtZgN43H0OlwGYBWe5iepeeca40/4C4CoASwEs9toeBmAdnFtGLwPQFsBBAH288pwP\n4AScZxteAuBZADkA7gh2G5Sire51f7aaul99AGQBiFY7ldhuVwPYDmAVgKH6XBVpnzcBrAXQEEAj\n96ue2slnW9UB8AuACXAeCXYegNsBXOCVR9d35xzqe32eGsFZaigPwI2h+rkKeqP90V8AXCgaMO0F\n0N3rfS0AJwE87H4f7S53lVeetgByATR2v38GQAqAql553gKwMdjnfJrt1cB97m282iYLwANeeS5x\n57nG/f5u9y9JA688TwM4mt8+AAYAWFvoWAkAZgf7nE+zvQ4D+Kfaqdj2OQPAZgC3AlgId8Ck9ipQ\n3zcBJBezTe1UsM79AXznJ4+u777bZTiALaH8udKQXIgxswvgPGPvm/w0kscBLMdvDyu+FsBRkqu8\nin4N51Ezf/bKs4hkrleeeQAuMbPa5VT9ilAHznkecb+PgbM8hnd7bQawCwXbax3JFK/9zANQG8Cl\nXnm+LnSseaikD4g2szAzewRADQDLoHYqzigAX5BcUCi9NdRe3pqZM4Vgm5lNNbMm7nR9rgpqB2Cl\nmU03ZwpBspk9mb9R13ffzHkiSCcA77mTQvL3TwFT6GkM5xejpIcVN4bT9ehBMg9OEOGdx9c+gEr6\n0GMzMzjfQr4nmT+HojGAbPdFx1vh9vLXFsXlqWVm1U637hXFzFqaWRqcb2ej4XxD+wlqpyLcAeWV\nAF7xsflMqL3y/QDgH3B6OboBuADAIvecGn2uCroQTu/PZgB3AhgLYISZPeberuu7bw/ACXQmu9+H\n5O9fMB++K6VT0sOKA82T/2jnyrr41mgALQC0CSBvIO0FP3kqY3v9BOAKOD1xDwKYYmY3lZD/D9lO\nZnYOnOD7DpI5pSmKP1h7kfR+XMR6M1sBYCeceTPFPbrqD9dObmEAVpB83f1+jZldCieImlpCuT/6\n9b0zgDkk9/vJF9TPlXqYQs9+OD/QMwulN8JvkfJ+93sPM6sCoK57W34eX/sAikbcIc/MRgK4B8DN\nJPd6bdoPIMLMahUqUri9CrfFmV7bisvTCMBxktmnU/eKRDKX5HaSySRfA7AGwItQOxUWA2cSc5KZ\n5ZhZDpzJpS+aWTacNqmm9iqKZCqcSchNoc9VYfsAbCqUtgnAue7/6/peiJmdC2di/LteySH5uVLA\nFGJI/gLnh3xbfpr7Q/NnOHcJAM6clDpmdpVX0dvg/CKu8Mpzk/sXLd+dADa7L3iVhjtYuh/ALSR3\nFdqcBGcypHd7XQznAuXdXpeZ83iefHcCSMVvF7dl3vvwyrOsLM4hiMIAVIPaqbCv4dxZcyWcHrkr\nAKyE0wuQ//8cqL2KMLMzAFwEZ/KyPlcFLYEzOdnbJXB65HR9960znCBotldaaH6ugj0z/o/4AhAF\n56J8JZxZ//92v2/i3v4SnLub2sG5qH8GYCsK3nY6G85F/WoAN8AZM4/32l4LzgVtMpxhrL/Bub2y\nS7DPv5RtNRrOXQ83wvmmkP+KLJTnFwA3w+k5WIKit5+ugXOr7uVw5mIcANDbK8/57vYZAOcC9y8A\n2QBuD3YblKKt+sIZrjwPzu3Kb8G56Nyqdgqo/Tx3yam9CrTLIAA3uT9X1wP4yn2e9dVORdqqNZz5\ng6/ACSofBZAG4BGvPLq+/3YeBmfpgL4+toXc5yroDfZHfMHp+nfBWXPC+zXRK0+c+xciA86s/qaF\n9lEHzrfhVDgBxbsAahTKcxmA79z72AXg/wX73E+hrXy1Ux6AJ7zyVAPwDpzbbNMAfAygUaH9NIGz\nhtMJ9y/VAABhPn4uSXBu8d0K4PFgn38p22oCnPWETsL5Fjsf7mBJ7RRQ+y1AwYBJ7UXPbdi/uuu/\nC8A0FFxXSO1U8DzugbNuVQaADQA6+8gTB13fAeAO9/W8qY9tIfe50sN3RURERPzQHCYRERERPxQw\niYiIiPihgElERETEDwVMIiIiIn4oYBIRERHxQwGTiIiIiB8KmERERET8UMAkIiIi4ocCJhERERE/\nFDCJiIiI+KGASURERMQPBUwiIiIifihgEhEREfFDAZOIiIiIHwqYRERERPxQwCQiIiLihwImERER\nET8UMImIiIj4oYBJRE6bmf3LzFxmtizYdRERKQ9GMth1EJFKzsy+B3AWgPMBNCO5Pbg1EhEpW+ph\nEpHTYmYXALgewH8ApADoFNwa+WZmNYJdBxGpvBQwicjp6gTgKIAvAXwCHwGTOV40s7VmdtLMDprZ\nHDNrVSjfY2a23MzSzeyImX1nZnd4bXeZ2Rs+9r/DzCZ6vf+7O+9NZjbazA4A2O3edq477SczyzCz\nFDObbmbn+dhvbTMbZma/mFmmme02s8lmVs/MoszshJkN81HubDPLNbOXS9WSIhKyqga7AiJS6T0K\n4BOSuWaWAKCbmcWQTPLKMxHA3+EEVe/CufbcCOBaAMkAYGZvAngTwBIArwPIBvBnALcA+MpPHYqb\nWzAawEEAPQFEudOudh83AcCvcIYR/wVgoZm1IJnprk8UgO8BXALgPQCrADQA0B7AOSTXmtlMAH8z\ns/+w4PyG/KBxqp96i0gloYBJRE6ZmcUAaA7gWQAg+b2Z7YETMCS589wCJ1gaTvI/XsWHee3nIjhB\n0gySD3nlGXmaVUwBcFuhYGYWyRmFzuMLAD8AeBDAB+7klwC0APAAyUSv7P28/j8FTsB4B4D5Xumd\nACwiuec06y8iIUJDciJyOjoB2A/gW6+0jwA8Ymbmfv8gABeAXiXs5wEA5idPaRHAu4WCJZDMyv+/\nmVU1s3oAtsMZVvQeIvwrgDWFgqXCvgawD17DkGZ2KYDLAcSf9hmISMhQwCQip8TMwgD8DcBCABea\n2UXunqIVABoDuM2d9UIAe0keK2F3F8IJqjaVcTV3FE4ws0gz62VmuwBkwemFOgigDoDaXlkvArC+\npJ27g7EPAHQws0h38mMAMuHM5xKR3wkFTCJyqm6Fs5TAIwC2er0+gtO7k9/rYj5LFxRInpJUKSb9\npI+0kQBeAfAhgIfgDKfdDuAITu2aOAVATQAd3O9jASSSTDuFfYlIiNIcJhE5VY8BOABnwnThgOdB\nAA+YWTcAPwO4w8zqlNDL9DOcYKUFgLUlHPMonJ4gDzMLhxO4BepBAJNIvuS1j2qF9wtgG4CW/nZG\ncoOZrQLQyT1/61y453SJyO+HephEpNTcw08PAPiC5EySn3q/4PTi1IJzR9kMONeaN0vY5WdweqXe\n8Jr75Ms2ADcVSuuG4nuYfMlD0WvfCz72MQPAFWZ2fwD7jAfQFsC/4QzxzS1FfUSkElAPk4icivvh\nDEMVNyH6BwCHAHQi2cHM4gG8YGYXwwkmwuAsK7CA5GiS28ysL4D/AVhsZp/CmV90NYA9JF9z73cC\ngLFm9gmcpQauAHCn+1iFFRd4zQLwuJkdB7ARwHVw5lulFMo3CEBHAB+b2ftw7vqrD6AdgKdJrvPK\n+wGAgXCG5UaTzCvm2CJSSSlgEpFT8SiADDh3iRVBkmb2JYBHzawugH8AWAOgC5zAIhXASgBLvcq8\naWbbATwPoI97/2vhzBHK9y6cdZO6wOnRWQRnDtI3KLoWU3FrM70AINd9DpFw1lq6HcA87zIk082s\nDZw1nB4A8AScyeFfw1m/yft8D5nZfAB3Q2svifwulfpZcmZ2I4D/AxADZ95ABz+33cLMbgYwBMCl\nAHYB6Ety8qlUWEQkFLl7xVqSvDjYdRGRsncqc5iiAKyGM6nRb7RlZufD6QL/Bk73+dsAJng/7kBE\npDIzs7MA3IuCvWEi8jtS6h6mAoXNXPDTw2RmAwDcTfJyr7QEALVJ3nPKBxcRCTL3F8I2AJ6E0+t+\nEcmDwayTiJSPirhL7loUnecwD85ESxGRyuwvcHqVzgXwhIIlkd+vipj03RjOWi3eDgCoZWbVvB9T\nICJSmbjnYmo+psgfQLDuksu/3dfneKCZ1YdzB8wOOI8YEBERESkPkXDuvp1H8nBxmSoiYNoP4MxC\naY0AHCeZXUyZtvjtieEiIiIi5a0TgGnFbayIgGkZnLVJvN3pTi/ODgCYOnUqoqOjy6lalVf37t0x\nbNiwYFejUlBbBU5tFTi1VemovQKntgpcWbXVpk2b8NhjjwE+HtbtrdQBk5lFAWiK34bVLjSzKwAc\nIbnbzN4CcDbJv7u3jwXwnPtuuYlwVtTtCKCkO+QyASA6OhqtWrUqbRV/92rXrq12CZDaKnBqq8Cp\nrUpH7RU4tVXgyqGtSpwCdCp3ybUGsArOYwIIZ0HKZDir4QLOJO8m+ZlJ7oCzPsntcNZv6g6gC0mf\nKwSLiIiIhJpS9zCR/A4lBFok/1lMmZjSHktEREQkFFTEOkwiIiIilZoCpkooNjY22FWoNNRWgVNb\nBU5tVTpqr8CprQJX0W11Wo9GKS9m1gpAUlJSkia/iYiISLlJTk5GTEwMAMSQTC4un3qYRERERPxQ\nwCQiIiLihwImERERET8UMImIiIj4oYBJRERExA8FTCIiIiJ+KGASERER8UMBk4iIiIgfCphERERE\n/FDAJCIiIuKHAiYRERERPxQwiYiIiPihgElERETEDwVMIiIiIn6cUsBkZs+a2S9mdtLMfjCzq/3k\n/7eZ/WRmGWa2y8yGmlm1U6uyiIiISMUqdcBkZn8DMATAmwCuArAGwDwza1BM/kcBvOXO3xxAZwB/\nA9D3FOssIiIiUqFOpYepO4BxJKeQ/AlANwAZcAIhX64D8D3Jj0juIvk1gAQA15xSjUVEREQqWKkC\nJjMLBxAD4Jv8NJIE8DWcwMiXpQBi8oftzOxCAPcA+PJUKiwiIiJS0aqWMn8DAFUAHCiUfgDAJb4K\nkExwD9d9b2bmLj+W5IDSVlZEREQkGEobMBXHANDnBrObAbwKZ+huBYCmAEaY2T6Sfcro+CIiIqHF\n5QJSUoAGDYCwcr4pffFiYMsWIC3NeWVmAnXrOsdu2BCoVw+oUcN5Va/upFU7jXuvsrOB/fuBffuA\nI0eA1FTndfIk0LIlcM01QK1apdunywVs3w6sWgWsXg3s2AHUrg3Ur+/Uv/Dr3HOBqKii+9m713m1\nalWm7V7agCkFQB6AMwulN0LRXqd8vQBMIfm++/0GMzsDwDgAJQZM3bt3R+3atQukxcbGIjY2tpTV\nFhGp5FJSgI0bgUaNnD8UNWqUrjwJmPnPl50NjB8PrFsHHDwIHDoEHDsGNG4MnHcecP75wJlnAjk5\nQFaW82rUCLjtNmebt7w8YOdOp3xGBpCe7vwhB5y6mDnHO3rUyXPs2G//P3rUyX/55cBf/gLceKNz\n3CSTLRwAACAASURBVMJcLuDXX51g4cSJ39Jzc50/vhs3Aps2/f/27ju8iip94Pj33PTeE0pIQgkl\ngEqTIijCUgUUEOli2QVEWdvPFRQL9oJ9RXBZVzGA4LKgogLCuqI0JRGkhRIk9JKQ3m9yfn+ctJtC\nbkJJkPfzPPMkd+bMzJlz5868c86ZGTh61AQIxUGDl5c5EQcEmKCieXPo0AGiosDZ2cy/ezds3QpH\njsDo0dC+fcX1nztn1p+TY8oiI8Oc8Ddtgs2byTuXjrOHs1luu3bQsKHZvnPnzODkBI0amfEhISbo\nOHLE5DclBXr3huHDzcm/su8vMxMefBD++U8KsJDs1phzbo3JcPanccY+gjPiqfRbd3aG666Dbt1M\ncFNYaNZ59CgcP27WXTYIcnAoHbKy4OxZsnHlN67BiiO+pOBLKn7OmbjnpZi8tmtnytTd3ZS9i4tZ\n1rFjZh3Hj5tlFRaaITcX8vIoRBEf3IPDwdfjnXsWv4w9+KcdxiEzlWT8OIc/KfgSQQItWlrg2muh\nZUs4cAC2bOHMkWziac71rdJweGg6TJxoAquUFJY8+yxLVqww6y0ooNBawIlc+0IhZbog2U8ptQXY\nqrV+sOizAo4A72qtX68k/TbgO631zDLjxgILAE9dSQaUUh2BmJiYGDp27Fij/Akh7JCeDnFx5gq0\nUaOan3ztlZwMb70F+/ZBWpoZMjPNeoODzRAQYE4aDg7g6GhOHD17mpNzeamp5iB78qQZkpLMSbn4\ngFtYaPs5JQVOnzZDcrI5QfTrZ07uwcEVl3/2LPz2mwkWMjJKD/KOjuZqd+9eU24JCWacq6uZ7uVV\netJr1AiaNTMnyDZtzEkwI8OcuHftMvnu18+cpMqeAK1W2LLFnCzz882QnV1y8k3fd5x9tMKPZBpy\nEvcgT7M+d3eTDzc3MxT/7+JiAp6EBDMkJ5uT78iR5gRc2fZv3UrBvZOJ3tORhAZdKXD3otDdE+3i\nimduIj5px/BJPoxz2lnSHPxJcQwgxRJA05y9jNPRuDQLhT59TBC0axeZuw/zbe7NOJNHEGcJJJEg\nzuJDqs1J/DTB/OLem21O3Ul39sfJ1REnN0dcXQqJPLOJdqfX0ZL9OIaHmloLV1czpKfD/v0UZOVw\nhDBS8SEfJ/JxIhcXTrk25VhQB457tiLJMQQPx1y8LRl4q3Sc8zOxZuaQn5FHflYekWmx3MIqAp3T\noUULSEjgaKYf0epOfnbszpj8Txl5UxKODz1ggoAvv4QVK+CHH6CwkDS8+IUubKUrcU7tOeB+Lfvz\nIjiX7U7zgGQ6ee2ns3ULYdn7SXQN5axTIxItwajCAgLzjhOYmUBA2mGs7t5k+jUm07sRhY7O9P99\nPu3TN6LCwmDYMOjf33yPXl6wYweMGUPsYX8m+K0i7pQvWtuGR56emmZNrAT75JKeAalpFlIzLDT1\nSWZao5XcfurvuMTvASDfP4TvfYfzbeEATuoGJBb4kpjnQ06hEw09Mwj1SqWxZwrpBR5sPR3BjmP+\nWAsq1uAE+Vtp7nuO5uoQTXP2EqLOEMwZggtPkeXozQ7nLuwoaMfO9Ai0xUKQZzZBnjl4u+WzP6Mh\nOw77kJFpX81QM+9EBnhupFfmanZ6dWdNfh9iT4cCEOZ+lsnZ73Cv979pEOVP7tbtxBVGsrPRQLa5\n38jW1Fb8ei6c3IKdmO7ZdNJax1a1rtoETHcAnwBTME1sDwO3A6211meVUguBY1rrJ4rSP1OUZgqw\nFYgE5gK/aK3HVbEOCZhE5QoKzFVZcrIZUlPNibb4xFZ2cHU1J+aaVDsXFpqq3N9/N1eMERHmat7F\nxaz74EFTVbx3rzkh9u1rTviV0dqcZLdvN/9HRJggwN/fXMVt3WqGXbvMlZXVagY3N2jduvSEGxVl\nru7LX10WFpoAxNe38vWnppoT5blzJrA4e9acfLdsMessLEQDObiS4dUIx9AG+LVpYK7UWrUy25WX\nV1qTkJBggoW4OBO09O4NY8bAoEGmrMuyWuHDD8l+6iX+lTGK3UG9OUMQZ6z+nMv3ItgpmXDLMcIL\nDtEo7zCOhXlYCvJxKMijRc5OurLVnCB69jTbffAgJ/enszS5H34kE8kBIjlAoGsmSY4hHCOU4zQm\nGT+sygmrcqJAOWJ19cTq5YfVwwdcXGh2ahPXHPmKpvyOpWWk7b5x9iz61CniaM2Pjn3IcA3AuyAZ\nH+s5vApSOOffghMB7TnhEckZSwj5VgtWq8ZqBceCXPwLkwjIO4l/1jFapWzlhoIfCCTJnNjS0zlF\nCKsZxE7njtyU9x39muzD7Y6hEBkJ330H69aRn5rJSRqWDMctYcT63sxW3ZU9KQ1tToY+rjkEu6Ti\n45SFj0MmPg7p+Kh0fFQqPjoVL51GqnMQJ5zDOaEbkpbvziinldwT/yRepEPXrqbW4tprTU3A559z\n8K2vuMvjczZlXUvDhgqLxfy8wMQmqanmZ1DMy8u0mBw/rmnsn83jUau4N/FVklwa8ffCacw/cDMp\n2eX2DcDRURPoX0igfyFpGRaOHDMrCQw0u11xvJiVZXZdAGfHApr7JOLnnIm3QxbeDplka1cO5IQS\nf86PfGvlJ1gvLwgNNcvNyjLbkZZmdmknp9JY/ehRsFg0NzQ9QV+/X/kxpR3/jQ/H1RWi2mhiYi00\ncT7F9Lw3GMhq4h1aEdd8MHEBNxBzpgm7D7mhtcLHu5CoKEXLVorISBOX7t4NMTEQG2vy4OhoWsSC\ngkweExPNzzM/33xWylSIFBSYmLllaCajQn7kjhNv0/7kGpSjI3TpAjExfBLyN6aemU3bdhamTCmt\nNPPwMNsUH2+GxEQTa/r4mL8bN8K6dSaev2dcDqeTHFm5ypFz58xhqnlz830EBpqfycmTpRVDrq4m\n3u/a1fx1dy+tFExKMoe94vUePmzi9ry80u/E29vsdtdcY8r/7FkzJCeXVvRdd52JWzMySg/5Vmtp\na5yPjzmMrVljhvh4k9f+/WHAALMNn3wCny0pJD9P09z9FPFZDbAWmn2tWTOT/65dwdc3lrvuugQB\nk/ky1TTgb5imue3AdK31tqJp/wUOa63vKfpsAZ4EJgKNgbPAl8AsrXVaFcuXgKm+S042e3JBgRms\nVtvqdAcHuPlmswdXRWvzSzp+3AQF5U+6xQoKYMMGWLSI1H9/x/FUD9LwJh0vMvEghNOEk0ADTmEp\n6kqXjyPJ+OFuycWzXQR06mQGR0ezvhMnzBEgM7O0WSE93Vzdl/1lgzl6NWgAqamkZjnyDYNZ73IL\nbXNjuJUvadYlwJzYCwvN8jIzzfK3byc5VfEjvXAjmxvYiDvZ5uiTm0sBFmJDBrMx6DZOFgRzJs+X\n07k+6PwCoqy/0T5lA+0LttOOXbj4uJkyatnSlO/BgxAfT3qOI54NvVFdi45egYEmCNu8GfbsIU17\nsoVubKIHW+jGGdcwMpwDyFCeZFpdyci2UFhYehKOdD9Gd7bQPWsdTThKJh5k4kEW7mR6hJAZ0IQs\n30YUuHpwS9Kn3Bz/DyxenqWBo5cXeHqS+++vWBDXkxfdXuBsvg9t2yqCg83B2dfXVPgUV3ycrqQx\n/7qmKUyPXMPY5LkcKWjMnPQpLDx0AyhFntWhJJ2Dg+0JvCyLxXzdjo4mndalLTaervm09jmJv0sm\nPs45+Lhkk2T15ccTzUlMd8HBQePurkhPt12mt7epQAoONi0axcvPzy+NSxMTzQkZoHXjdDoHH2FX\nUkO2H/FHKU2jRnD8uMLDMYeBDt/RPi+GOP8b2O14DXFJQTYnfkdHaNu29MDevr0JWoq7jpw5U9pq\nUn5ISzMnlUaNoHFjs3t+/TV4uBfyl+67udvyCcHxm/E+GIuTzmWu00M8zqs0aOLIxx8revWqWKZa\nmxN+bq4pC8eilow9e+Dll2HxYnMyS0kxJ9G//AWmTTP/FwcFxX+L/3dzM+f+66831yblrwuSkkyF\n365dptUtLa006HF0ND+JyEgzFFdWOjmZ7yckxOyS9jh50pTPF1/A999D584waZKpkPP2NpU5b78N\nixcVkpdvviMfH3Ntce210L27ad1q1arqbjMFBSbvPj4Vt7N4/yyuuCxurVy3Dj7/HFauNOXasmke\no1rtZET+Uj5KHs77sd25+26YO7fqQ2hV9u6F9983gUWDBjBqlBmuu86+1lt7aW32yTNnzHcTEXFx\nlw9m2ZV1FUtOhoULzSGzbVvzG2rXznwHxWJjY+nU6RIFTJeaBEy1lJRkjlzFTRYnTpijZOfO5mjb\ntGnN9tLCQttaheLmiLg4c7QDClFk40Y6XpygEUdpwlGaYMWRIXxNi+5BMGSIWffx4+YS5dgx07fg\n4EFKzkgeHqaZYuhQc/Q8dMisb+9eWLeOhBOOzPF+ngVZY8mxOlWaXWenQgJ9raRmOJCZbU6qLo5W\nbguLZRIL6XfknxQUKtb73c4KNYJ1Wd1p6J7KtQHHuTbkFBGBGSQ4NWd/bjgHUoJIy3Yi0DmdAMs5\n/PPPsD2pCevjw8m3WmjTBg4d0uTmKtr7HGGAWouPcxZOzhacXCycdAzl+4wuxJ5oUFIr4OxUSI+W\nifRs9Dv7s0JZt6ch55ItuLqak1pIiDkRa21ODocOme1ycSqgW+gxbvT8le55P3DEoSk/Wruz4Uxr\njqZ44e+aRZTLQaIytxFkPclx37YccW/NkYJGxJ/xQmtFgH8h3bsrQpsoPD2xGTw8zN/MTFP5tHkz\nbN+uKSgo3VecnU0A4eFh0mdnm6vXiMZ53B25kcFZ/+ZsihNH0nxJyAxkUf4ojuUFM2GC4umnzVXj\n+Xazsq1pP/wAf/87fPNNSeUMISHw0EMwdao54B48aLornD5tDvSNG5taBH//0hqDyk5Ip06ZFrff\nfjOthGW7abi7m24yN91kTnyenqUnuPR0U1np6Vn9z0ZrE3dv3Ag//QTbtpmT6MCB5uo3KMise8UK\nWPEfzaFD0CZK0a6dOZA3a2Za2ho2vPh9hY8eNWX74Ydm24s5OmisBYpp0+DVV+3bzsocPGhO3OHh\ncM899gcrV5JTp0zg1rKl2S8v9om/KpUFT05O8N57MHnyheWjsLC0S9nVSAKm+kRrs0c6OFQcv3Ur\nrFplLrOaNjVDeLi5pHF3N0fL1FTzS/nmG/j2W3Nmi4gwQ2ioOQpu307O0TP8lz4cpQknncI56dYM\nq7ZwTfpGOvAr1wYcxyfCz6y3eP05OaW1ItnZ5hfj4ECG8iIp252sPAeycCfL2Y+Eht34za0rO/Kj\n2HWuIecyXcjJcyi/tTg5aSwWyM1VXOdziFHZn9Iybye/O7fmkHs7fndoTrazD7iaPheObo508dxL\n37NL6bnnQ9zIRgPHPNuwt/GfWKTGs/hAF7x9FH/9q6Jv39JqZTc3cwArrq04e9bUYPj5mWH/fnP1\ntHs3hITooip5U1U+aJCJMXfsMHFZQYH5iiIizMHQ19fUGiQmmqFpU9P147bbzJVwRoapCl650tyg\nkp1d2pTg62tarPr2NRVtmZnmK1y/3pxIIyPNybN/f3Nl7VRJDJiRYQKnLVtMBduGDSa/Dg6mJaVX\nL3Nle/So2b49ezSJZzVNwiyEhZk8tmkDN9xgtqcmB8OsLLPbeXiY3dCxXJ9IrU2f1n/+E5YtM9sH\nJm+NG5t1PvWUWX9txceb7y48HCZMuLAbeoStjAz4+efSbmVpaaZWoWfPus6ZsEdeHvzvf+a31rZt\nXefmyicB0+WUn29qdIKCzBm82Nmz8OmnsGCBOfq3b2+ahTp2NJ+XLSMlIYWFHtNw0FaaZMURxhEa\ncQIn8lFolLs7qTku7C5sza7gvuwJugkvHwv9fX+mt3UdnicPcCKgPR/k3cv8XT04m+aKxaIJCVE0\nbGiysXu3qQkBaON7kr6N4/hTaBw3NT5Iog5g67lItp5pyq+nG3Ii1YNTaR5k5VdeixMebk7S7dub\nq6viG048PU0tSWioKYacHBPbff45rFqlycxUeHmZq+emTW2vYLOyTBBx+jS4uGhaNMri8Gk3MrPM\npXVoKDz6qKner+wO0upobfoOfPaZCbSGDzetW2UDiJwcE3g1blx58FIfFBaa3aZhw9rXAFwK6emm\nYrO4z3P54EoIIeozCZguVE6OqV8+fNgMCQnm8r5s28Hp06bd5MgR81kpE1G0amUuh7/9ljS8eb/1\ne8S5XMsAtw0MSvwUv31bSPdtwjvN3+GNvYPJzHUEVEmHv6p4epor9uIbX5ycTPCyfbtpu77rLrjv\nPrP6spVZ+fmmCeDXX01NyPr1pc09xSIjTctdkyamiaNBA9MfwNOzNCgKCbFt961JUWZmmuaSqmo5\ntDa1JOvXm7w2b276PbdubWp8ylfOCSGEEBeDBEw1kZJi2hc2bTJtILt3m4iisJBUvDno2JqDAd3I\n9QqgjcdR2ngexdMxhyzfRnzv2I9vEq/nx4QmtAxIon9ALP0K1xCQkcC7fk/z5ubuZGZZaNnSLNrB\nAW7oXsjuvYr0dMWUKTBzpglGTp82sdfJk6V3RmttgpaoKNPEYrGYcQcPwtq1JgDq2tX0F6hJMPP7\n72be4GDTJOTvf+mKVwghhKivJGCqTnq66d341VewcyendRD/876VuKCeHHSK4mBeEw4m+ZOYWnn7\nTJMmpqYnN9c0Md18s6kZ2bLFBDvOzqY25S9/gccfN81Kx46Z7krffmvmnzHDjBdCCCFE3bA3YLr6\nehtoDcuXw0MP8XNiM/7d8hXWhnZlx1F/SINgV/Psh1aRcEsL83+LFqaJyMnJ3CBW/ODY4GAYPNi2\nQ21qqumMt38/jB9v+nUUCw01d/lMnVonWy6EEEKIWrq6AqaEBJg6lXOrt/JY2FI+yu1Hg7PmTqXH\n+sOf/lT5k/fL6tzZDFXx8YFbb7242RZCCCFE3bp6AqYzZ9C9b2ZR2lAe8VlJXqoL8+aZJrNL/U5E\nIYQQQlzZro6AKTsbPexWppyezT+yJzJ6tHlia4MGdZ0xIYQQQlwJ/ngBU2GhbZVRYSFMmsQHsV35\nR/5E/vEP+POf6y57QgghhLjy/LECpn/9C6ZPN7etDRpkhtWr+fHzUzzo8Bl//asES0IIIYSouT9G\nwGS1wv/9H+nv/JP3rvuIKL9TDPpkDi6vv84xGnO75z5u6GRhzpy6zqgQQgghrkRXfsB07hyMHs2Z\n73czOCyeHbuCsFoVPj7TGXlrEr/tdsAl151ly+rvKy+EEEIIUb9d2QGT1Qp9+nA4QdG/4UHSct3Z\nts08NHLJEsXixYGcOGFeWhocXNeZFUIIIcSV6soOmD79lF07rAwIisXV2ZlNP5iXuwI89xzMnm3e\nYVafXlQqhBBCiCvPlfsEotxcMp9+lQGuGwhq5MzGjaXBUjGlJFgSQgghxIWrVcCklLpfKfW7Uipb\nKbVFKdWlmvQ+Sqn3lVIniuaJU0oNrF2Wi8ybx1vH7yCxwJeVK+WZSkIIIYS4dGrcJKeUGg28AUwG\nfgYeBtYopVpqrRMrSe8ErANOASOAE0A4kFLrXKenc/b5ebzmGMv9D1iIiKj1koQQQgghqlWbPkwP\nA/O11gsBlFJTgVuAe4DXKkl/L+ALdNNaFxSNO1KL9ZZ6+22eT3kAi4cLTz55QUsSQgghhKhWjZrk\nimqLOgHri8dprTWmBql7FbMNBTYDc5VSp5RSO5VSM5VStes/lZRE/GvLmaenMGOmhYCAWi1FCCGE\nEMJuNa1hCgQcgNPlxp8GWlUxTzOgDxANDAIigblFy3mhhuuHN97gyZxZhDRQPPhgjecWQgghhKix\ni/VYAQXoKqZZMAHV5KLaqF+VUo2B/6OmAZPW/PLJHpZaX+KjF8DN7UKyLIQQQghhn5oGTIlAARBS\nbnwwFWudip0E8oqCpWJ7gQZKKUettbWqlT388MP4+PiUjkhL4/SJa2nTJIM775TnBQghhBDCfkuW\nLGHJkiU241JTU+2at0YBk9Y6XykVA/QFvgRQSqmiz+9WMdtGYGy5ca2Ak+cLlgDeeustOnbsWPI5\n+5lXCPxhOk9NdsHBoSY5F0IIIcTVbuzYsYwdaxuSxMbG0qlTp2rnrU3H6zeByUqpO5VSrYF5gDvw\nMYBSaqFS6qUy6T8AApRS7yilIpVStwAzgb/XdMXfLT5LFh4MH3VlP6BcCCGEEFeWGkceWutlSqlA\n4DlM09x2YIDW+mxRklDAWib9MaVUf+AtYAdwvOj/yh5BULUTJ1hxsB1tGqXQqpVvTbMthBBCCFFr\ntaqq0VrPxdzpVtm0PpWM2wr0qM26illXfMVXjGTKaJcLWYwQQgghRI1dMW1bP35yiCQCGT6urnMi\nhBBCiKvNlfHy3fR0VsSE0cQ3HTv6ZQkhhBBCXFRXRMCkv13NysKh3HarRqm6zo0QQgghrjZXRMAU\n8/FOjhLG8EnedZ0VIYQQQlyF6n/AlJ/Piu99CHDLpFevus6MEEIIIa5G9T9g2riRFTmDGNo3G8cr\npou6EEIIIf5I6n3AtO+/x9lLFMP/HFDXWRFCCCHEVareB0w/7vRFUUi//tLbWwghhBB1o94HTElJ\n4GtJw82trnMihBBCiKtVvQ+YklMUfk4ZdZ0NIYQQQlzF6n036uR0R/xcsuo6G0IIUWtHjhwhMTGx\nrrMhxFUpMDCQsLCwC15O/Q+YMp3xc8ut62wIIUStHDlyhDZt2pCVJRd+QtQFd3d39u7de8FBU/0P\nmLJd8QvJr+tsCCFErSQmJpKVlUV0dDRt2rSp6+wIcVXZu3cvEyZMIDEx8SoImPLcaeaVU9fZEEKI\nC9KmTRs6duxY19kQQtRS/e/0ne+Fn5+u62wIIYQQ4ipWvwOmggKStQ9+/vU7m0IIIYT4Y6vXkUhh\nWgYp+OIX6FDXWRFCCCHEVaxWAZNS6n6l1O9KqWyl1BalVBc75xujlCpUSv3HnvSZpzPQWPALcapN\nNoUQQgghLooaB0xKqdHAG8AzQAdgB7BGKRVYzXzhwOvABnvXlXbK3Ibr11Ae8y2EEFezffv2YbFY\nWLZsWY3nzc3NxWKx8Nprr12CnImrRW1qmB4G5mutF2qt44CpQBZwT1UzKKUsQDTwNPC7vStKO2Pu\njvNr7F6LbAohhLhULBZLtYODgwMbNth9jVwtpWr/TlGl1AXNfzH8+uuvWCwWvLy85LlcV6AaPVZA\nKeUEdAJeKh6ntdZKqXVA9/PM+gxwRmv9L6XUjfauLz3RPLDSL8yzJtkUQghxiUVHR9t8/uSTT1i3\nbh3R0dFoXXpn88V69lSrVq3Izs7G2dm5xvO6uLiQnZ2Nk1Pddu9YtGgRoaGhnD59mpUrVzJu3Lg6\nzY+omZo+hykQcABOlxt/GmhV2QxKqRuAu4Fra5q5tGQrAH6hEjAJIUR9Uv5kv3nzZtatW8fYsWPt\nmj8nJwdXV9carbM2wdLFmPdi0Frz2Wefcffdd/Prr7+yaNGiehswWa3m3OvoWO8f1XhZXay75BRQ\n4WFJSilP4FPgL1rr5JouNC25EEUhPn71+mY+IYQQ57FmzRosFgsrVqzg8ccfp3Hjxnh6epKXl0di\nYiIPP/ww7dq1w9PTE19fX4YOHcqePXtsllFZH6YxY8YQFBTE0aNHGTJkCF5eXoSEhPDkk0/azFtZ\nH6YZM2ZgsVg4evQoEyZMwNfXF39/f6ZMmUJeXp7N/FlZWUybNo2AgAC8vb25/fbbSUhIqFG/qPXr\n13Py5EnGjBnD6NGjWbduXZXvF/zqq6+48cYb8fLywtfXl27duvHvf//bJs3GjRsZMGAAfn5+eHp6\n0qFDB+bNm1cyvVu3bgwePLjCsseMGWNT61dcru+//z5z5syhWbNmuLm5cejQIXJycpg1axadOnXC\nx8cHLy8vbr75ZjZu3FhhuYWFhcyZM4f27dvj5uZGSEgIt9xyC7/99hsAXbt2pVu3bpVub0REBMOH\nD6++EOtYTcPHRKAACCk3PpiKtU4AzYFw4CtV2nhsAVBK5QGttNZV9mlaHDsfB77mtttKq1HHjh1r\n9xWMEEKI+uOpp57Cw8ODxx9/nMzMTBwcHNi3bx+rV6/m9ttvJzw8nJMnTzJv3jx69+7Nnj17CAys\n+n4ipRT5+fn069eP3r17M2fOHFavXs0rr7xCy5YtmTRp0nnnVUpx22230bJlS1599VV+/vlnFixY\nQKNGjXjmmWdK0o4dO5ZVq1Zxzz330KlTJ9atW8dtt91Woz5RixYtom3btrRt25bw8HCmTJnC0qVL\nuf/++23SzZs3j2nTptGhQwdmzZqFt7c3sbGxrF27lttvvx2AVatWMWLECMLDw3nkkUcICQlh9+7d\nfP3110ydOrVk+8633eV98MEHFBQUMG3aNBwdHfHx8SEpKYmFCxcyZswYpk6dSkpKCgsWLKBfv37E\nxsbSunXrkvnHjx/P0qVLufXWW0uCzh9++IFffvmFa665hjvvvJO//vWvHDp0iGbNmpXM9+OPP3Lk\nyBHefPNNu8vyQixZsoQlS5bYjEtNTbVvZq11jQZgC/BOmc8KOAo8VklaZyCq3LAC+A5oAzhWsY6O\ngJ4U+Z5u6nxUCyHElSomJkYDOiYmpq6zckk98MAD2mKxVDpt9erVWimlo6KidH5+vs203NzcCukP\nHDignZ2d9Zw5c0rGxcXFaaWUXrp0acm4MWPGaIvFot944w2b+du2bat79epV8jknJ0crpfSrr75a\nMm7GjBlaKaWnT59uM+/gwYN1kyZNSj5v2rRJK6X0k08+aZNu7Nix2mKx2CyzKjk5OdrHx0e/9NJL\nJeNGjhypu3fvbpMuKSlJu7u76969e1cop2L5+fm6cePGunXr1jojI6PKdXbr1k0PGjSowvgxY8bo\nNm3alHwuLtfAwECdmppqk7agoEBbrVabcefOndMBAQH6gQceKBn3zTffaKWUnjlzZpX5SUpK0s7O\nznr27Nk24ydPnqz9/Pwq3Q8uBnt+f8VpgI76PPFPbRoo3wQ+UUrFAD9j7ppzBz4GUEotBI5pSI4O\n0AAAIABJREFUrZ/QWucBNvWqSqkUE6fpvdWtKD3Lgr+L3EkghLhKZGVBXNylX0/r1uB++e8+vuee\neyr0iynbt6igoIDU1FR8fX1p2rQpsbGxdi138uTJNp979uzJqlWrqp1PKcWUKVNsxvXq1Ys1a9aQ\nn5+Pk5MTq1evRinFfffdZ5Nu+vTpfPbZZ3bl74svviA9PZ0xY8aUjBs7dix33HGHTY3Lt99+S05O\nDk888USV/Ye2bt3KiRMnmD9/Ph4eHnat3x5jxozB29vbZpzFUtodRmtNSkoKBQUFdOzY0ea7Wb58\nOc7OzhWaQsvy9/dn8ODBLFq0iKeffhqA/Px8li9fzqhRo+q8j5k9ahwwaa2XFT1z6TlM09x2YIDW\n+mxRklDAejEyl5bthJ+bvHhXCHGViIuDTp0u/XpiYqAOXgQcERFRYVxx35f58+eTkJBAYWEhYIKZ\nFi1aVLtMX19fPD1tbwzy8/MjOdm+brPl32Dv5+dXEhwEBQWRkJCAi4sLjRs3tklnT96KLVq0iFat\nWlFYWEh8fDwALVu2xNnZmcWLFzNr1iyAkmlt27atclnx8fEopc6bpjYq+24AFixYwNtvv83+/ftL\nOoMDREVFlfx/6NAhwsLCqg3g7rzzTm6//Xa2bdtG586d+eabb0hOTmbixIkXZRsutVp1gddazwXm\nVjGtTzXz3m3vetJynAkPyas+oRBC/BG0bm2Cmcuxnjrg5lbxIcRPP/00L730ElOnTuXmm2/Gz88P\ni8XCfffdVxI8nY+DQ+WvztLavpe2X+j81UlOTmb16tVYrVYiIyNtpimlWLRoUUnAZM867c1XVX2Y\nCgoKKh1f2XezYMECJk+ezB133MGTTz5JYGAgDg4OzJ49m7Nnz5akszdPQ4YMwc/Pj+joaDp37kx0\ndDRhYWH07NnTrvnrWr2+ZzA93xU/78q/XCGE+MNxd6+Tmp+6tHz5cgYPHszcubbX4OfOnaN58+Z1\nlKtS4eHh5Obmcvz4cZtapgMHDtg1/9KlS7FarXz00Ud4eXnZTNu1axezZ88mNjaWjh07ltRa7dq1\ni0aNGlW6vBYtWqC1ZteuXfTo0aPK9VZVy5aQkGBXvsF8N23btq3Q9Pi3v/2tQp42b95MRkZGhdq+\nspycnBg9ejRLly7lmWee4euvv+bRRx+1Oz91rV7fr59mdcfPt/orDCGEEPVbVTUeDg4OFWooPv30\nU5KSki5Htqo1YMAAtNYVArr33nvPrrvkFi1aRFRUFJMmTWLEiBE2w2OPPYaLiwuLFi0CYNCgQbi6\nuvLSSy+Rn59f6fK6du1K48aNeeONN0hPT69yvc2bN2fnzp02d4D9/PPPbNu2zZ7NBir/bjZs2FCh\nb9nIkSPJy8vjxRdfrHaZEydO5PTp00ydOpXc3FzGjx9vd37qWr2uYUrTnvj51+2j7IUQQly4qppt\nhgwZwuuvv87kyZPp0qULO3bsYOnSpVX2qbncevTowS233MIrr7zCqVOn6Ny5M+vXr+f3380Tcc4X\nNB0+fJhNmzYxc+bMSqe7ubnRt29fPvvsM+bMmYO/vz+vv/4606dPp2vXrowePRofHx+2b9+O1pr5\n8+fj6OjI3LlzGTlyJB06dGDSpEmEhISwd+9eDh06xBdffAHAvffey9///nf69+/PXXfdxfHjx1mw\nYAFt27a16Yt0PkOGDGHatGncfvvtDBgwgIMHD/Lhhx8SFRVl01w6cOBARo0axWuvvcaePXvo168f\nVquVH374gSFDhnDvvfeWpO3WrRuRkZF8/vnndOzY0ebRBPVdva5hSscLv4DK25eFEELUL+cLHqqa\n9uyzz/LXv/6Vr7/+mkceeYQ9e/awdu1aGjRoUGGeypZxvucNlf9sz/Iqs3TpUqZMmcLKlSuZOXMm\njo6OJa+AOd/Tyouf9zNkyJAq0wwdOpRTp06xfv16AKZNm8by5ctxc3Pj+eefZ+bMmezcuZOBAwfa\nzLN+/XqaNm3KnDlzeOyxx9iwYQNDhw4tSXPttdfy8ccfk5iYyCOPPMKaNWtYunQpbdu2tbscpkyZ\nwnPPPce2bdt46KGH+P777/n8889p3759hXmWLFnCyy+/zP79+3nsscd45ZVXKCwspGvXrhWWO3Hi\nRJRS3HnnnVWWS32kLlbHtotJKdURiIEYlj3pwKgXavxWFSGEqBdiY2Pp1KkTMTExdLzK+if9kW3Z\nsoUePXqwfPnyK+Ip1fXJq6++ylNPPcWxY8cIDg6+pOuy5/dXnAbopLWu8lkW9bqGCcCvYc3eNSSE\nEEJcTLm5uRXGvfPOOzg6Ol4xd3jVF1pr/vWvf9G/f/9LHixdbPW6DxOAX+PL/3A1IYQQothzzz1H\nXFwcN954I0opVq1axfr163nwwQcJCgqq6+xdETIyMvjqq69Yu3YtBw4c4P3336/rLNVY/Q+YmlR9\ni6IQQghxqfXs2ZP//e9/PPfcc2RmZhIeHs6LL77I448/XtdZu2IcP36c8ePHExAQwOzZs+nbt29d\nZ6nG6n/AFOZVfSIhhBDiEhk0aBCDBg2q62xc0YqfdH4lq+d9mArxCaj3MZ0QQggh/uDqdcDkSSaW\nep1DIYQQQlwN6nU44u2YVddZEEIIIYSo5wGTU3ZdZ0EIIYQQon4HTF4ueXWdBSGEEEKI+h0webtV\n/vJBIYQQQojLqX4HTB4FdZ0FIYQQQojaBUxKqfuVUr8rpbKVUluUUl3Ok/bPSqkNSqlzRcN350tf\nlpdn/XvPnRBCCCGuPjUOmJRSo4E3gGeADsAOYI1SKrCKWW4CFgO9gW7AUWCtUqphdevylmdWCiHE\nVSc0NJTJkyeXfF6/fj0Wi4VNmzZVO2/Pnj3p37//Rc3PrFmzcHJyuqjLFFee2tQwPQzM11ov1FrH\nAVOBLOCeyhJrrSdqredprX/TWu8H/ly03mqfi+7jV69bDIUQ4qo1bNgwPDw8yMzMrDLN+PHjcXFx\nITk5uUbLVkrZNc7eee2RmZnJ7Nmz+emnnypdpqWOHwp47tw5nJ2dcXBwID4+vk7zcrWq0R6glHIC\nOgHri8dprTWwDuhu52I8ACfgXHUJvfzlKd9CCFEfTZgwgZycHFasWFHp9OzsbL788ksGDx6Mn5/f\nBa2rb9++ZGdn06NHjwtazvlkZGQwe/ZsNmzYUGHa7NmzycjIuGTrtseyZctwcnIiODiYRYsW1Wle\nrlY1DZkDAQfgdLnxp4EGdi7jVeA4Jsg6L+9AlxplTgghxOUxbNgwPD09Wbx4caXTV65cSVZWFuPH\nj78o63N2dr4oy6mKufavnMViqfMmuejoaIYNG8bo0aPrdcCktSY3N7eus3FJXKw6RgVU20NbKTUD\nuAO4TWtd7UOWvIMlYBJCiPrI1dWVESNGsG7dOhITEytMX7x4MZ6engwdOrRk3KuvvsoNN9xAQEAA\n7u7udOnShZUrV1a7rqr6MH3wwQc0b94cd3d3unfvXmkfp9zcXJ566ik6deqEr68vnp6e9O7dmx9/\n/LEkTXx8PI0aNUIpxaxZs7BYLFgsFl566SWg8j5MVquV2bNn07x5c1xdXWnWrBlPP/00+fm2j8MJ\nDQ1lxIgRbNiwgeuvvx43NzdatGhRZaBZmcOHD7Np0ybGjh3L6NGjOXDgANu2bas07ebNmxk0aBB+\nfn54enpy3XXX8f7779uk2bt3L6NGjSIoKAh3d3fatGnDM888UzJ9woQJREZGVlh2+XIoKCjAYrHw\nyCOP8Omnn9K2bVtcXV1Zv940QtXk+164cCHXX389Hh4eBAQE0Lt3b/773/8Cpmm3QYMGlb68t0+f\nPrRv376aErw4ahowJQIFQEi58cFUrHWyoZT6P+BvQD+t9W57VvbO4pcZNmyYzbBkyZIaZlkIIcSl\nMH78eKxWK8uWLbMZn5yczNq1axk5ciQuLqUXvu+++y6dOnXihRde4OWXX8ZisTBy5EjWrl1b7brK\n902aP38+999/P02aNOH111+ne/fuDB06lBMnTtikS0lJ4eOPP6Zv37689tprPPvss5w6dYr+/fuz\ne7c5FTVo0ID3338frTWjRo0iOjqa6OhobrvttpJ1l1//XXfdxezZs+natStvvfUWvXr14oUXXmDC\nhAkV8r1v3z7GjBnDwIEDefPNN/Hx8WHSpEkcOHCg2u0GWLRoEb6+vgwaNIju3bsTHh5eaS3T6tWr\n6d27N/v37+fRRx/lzTffpHfv3nz99dclabZv3063bt3YsGED9913H++++y633nqrTZrKtvd849eu\nXcvjjz/OuHHjePvttwkLCwPs/76feuop7rrrLtzc3Hj++ed59tlnCQ0N5fvvvwfgzjvv5OzZs6xb\nZ9swdeLECTZs2MDEiRPtKkeAJUuWVIgrHn74Yftm1lrXaAC2AO+U+awwd749dp55HgOSgS52rqMj\noL9fs1kLIcSVLCYmRgM6JiamrrNy0RUUFOhGjRrpG264wWb8vHnztMVi0evWrbMZn5OTY/M5Pz9f\nR0VF6YEDB9qMDw0N1X/5y19KPq9bt05bLBa9ceNGrbXWeXl5OjAwUF9//fXaarXarFcppfv162eT\nx/z8fJvlp6Sk6KCgID116tSScadOndJKKf3iiy9W2M5Zs2ZpJyenks8xMTFaKaWnTZtmk+7hhx/W\nFotF//TTTzbbYrFY9JYtW2zW5ezsrGfOnFlhXZWJiorSd999d8nnxx9/XDds2FAXFhaWjLNarTos\nLExHRkbq9PT0KpfVo0cP7efnp0+cOFFlmgkTJujIyMgK48uXg9Vq1Uop7eTkpA8cOFAhvT3f9759\n+7TFYtGjR4+uMj/F+9nEiRNtxr/22mvawcFBHz16tMp57fn9FacBOurzxCa16VX9JvCJUioG+Blz\n15w78DGAUmohcExr/UTR578BzwFjgSNKqeLaqQytddW3VwCe/pe2zVoIIeqTrCyIi7v062ndGtzd\nL3w5FouFMWPG8Pbbb5OQkEB4eDhgmuNCQkLo06ePTfqytU0pKSlYrVZ69uxpV7NcWVu3biUpKYnX\nX38dBweHkvH33HMPf/vb3yrksfgON601KSkpFBQU0LlzZ2JjY2u03mLffPMNSikeeeQRm/GPPvoo\nb7/9Nl9//TU33HBDyfhrrrmGrl27lnwOCQkhMjKSQ4cOVbuu2NhY9u7dyzvvvFMybuzYsbz++uus\nW7eOfv36AbBt2zaOHj3K+++/j6enZ6XLOn36NJs3b+axxx6jYcNqn+xjt759+9KiRYsK4+35vv/z\nn/8A2DQJlmexWBg3bhzz588nOzsbNzc3wOxnN954I6GhoRdrU86rxgGT1npZ0TOXnsM0zW0HBmit\nzxYlCQWsZWa5D3NX3L/LLWp20TKqVMd3cQohxGUVFwedOl369cTEQMeOF2dZ48eP56233mLJkiXM\nmDGD48eP89NPP/HQQw9VaL758ssveemll9ixY4dNx+CaduhOSEhAKVXhJO3k5ERERESF9P/61794\n66232Ldvn00fo5YtW9ZovWXX7+joSPPmzW3GN27cGC8vLxISEmzGFzdRleXn52fX4xaio6Px8vKi\nSZMmJY8T8PDwIDQ0lEWLFpUETPHx8SilaNu2bZXLKp7/fGlqo7IyB/u+70OHDuHg4ECrVq3Ou45J\nkybxxhtv8MUXXzBmzBh2797Njh07+Oijjy7KNtijVvfta63nAnOrmNan3OemtVmHEEJcbVq3NsHM\n5VjPxdKxY0dat27N4sWLmTFjRkln5nHjxtmk+/777xk+fDh9+vRh3rx5NGjQACcnJ/7xj3+wfPny\nGq1TF93RVll/muJpxT7++GPuvfdebr/9dmbMmEFQUBAODg48//zzHD9+vEbrrWod1U0rWwtm73KK\npy9dupSMjAzatGljM00pxYoVK5g3bx6urq7VLsue9ZVddmUKCip/XVlxjU9Z9n7fWmu7np3Vrl07\nrr32WqKjoxkzZgzR0dG4ubkxcuRIu7bpYpAHHQkhRD3h7n7xan4up/Hjx/P000+zc+dOlixZQmRk\nJJ3KVZX95z//wcPDg9WrV9sEEPPnz6/x+iIiItBas3//fpumr/z8fBISEmjQoPQpN8uXL6dVq1YV\nOqY/8cQTNp9r8sDLiIgIrFYr8fHxNrVMJ06cICMjo6Rp8kKtX7+ekydP8vLLL1e4ay0xMZH77ruP\nL7/8kjvuuIMWLVqgtWbXrl3ceOONlS6vuEZu165d512vn58fKSkpFcYfPnzY7rzb+323aNECq9VK\nXFwcUVFR513mnXfeyYwZMzhz5gyfffYZw4YNw8vr8r0SRBq9hBBCXJDx48ejtebpp59m+/btFe4U\nA1PLYrFYbGopDh06xFdffVXj9XXt2hV/f3/mzZtns7wFCxaQnp5eYb3lbdy4kV9++cVmnIeHB0Cl\ngUJ5gwcPRmvN22+/bTP+jTfeQCnFLbfcYve2nE90dDTe3t48+uijjBgxwmaYPHkyTZs2LblbrkuX\nLoSFhfHWW2+RlpZW6fJCQkLo0aMHCxYsOG/tWvPmzUlKSmLv3r0l444fP16j78re73v48OGAeTho\ndTVg48aNo7CwkOnTp3PkyJFK97NLSWqYhBBCXJCIiAh69OjBF198gVKqQnMcwJAhQ3j33XcZMGAA\nY8eO5eTJk8ydO5dWrVqV3N5/PmVPpk5OTjz//PM88MAD3HzzzYwePZqDBw+ycOFCmja17QUyZMgQ\nvvzyS0aMGMGgQYOIj4/nww8/JCoqyqZfjYeHBy1btmTJkiU0a9YMPz8/rrnmmgpNYWCaIcePH8/c\nuXNJSkqiV69ebN68mejoaO644w6bWq/aKn6K+qBBg3B0rPxUPXToUD744APOnTuHv78/c+fOZfjw\n4Vx33XXcfffdNGjQgLi4OPbt28eqVasAeO+997jpppvo0KEDkydPJiIigkOHDrF27dqSZzuNGzeO\nJ554gmHDhjF9+nQyMjKYN28erVu3ZseOHXbl397vu2XLlsyYMYNXXnmFm266idtuuw1nZ2d++eUX\nwsPDee650q7OISEh9OvXj88//5zAwEAGDhxY2+KtnfPdQldXA0WPFfgj3oYrhLi6/JEfK1DW3Llz\ntcVi0d27d68yzYIFC3TLli21m5ubbtu2rf70008r3KqutdZNmjTRkydPLvlc/rECZdfZrFkz7ebm\nprt37643bdqke/Xqpfv372+T7sUXX9QRERHa3d1dd+7cWa9evVpPmDBBt2zZ0ibdxo0bdefOnbWr\nq6u2WCwljxiYNWuWdnZ2tklrtVr17NmzdbNmzbSLi4uOiIjQTz/9dIVHGDRp0kSPGDGiQln07Nmz\nQj7LWrZsmbZYLDo6OrrKNOvXr9cWi0V/8MEHJeN++ukn3a9fP+3t7a29vLx0hw4d9Pz5823m27Vr\nlx4+fLj29/fXHh4eOioqSj/33HM2adasWaPbtWunXVxcdFRUlF66dGmljxWwWCz6kUceqTR/9n7f\nWmv90Ucf6Y4dO2o3NzcdEBCg+/Tpo7///vsK6ZYsWaKVUnr69OlVlktZF/OxAkrb2QnsclJKdQRi\nYmJi6HglNugLIUSR2NhYOnXqhBzPhLhw//nPfxg1ahSbN2/m+uuvrza9Pb+/4jRAJ611lc+akD5M\nQgghhLgifPjhh0RGRtoVLF1s0odJCCGEEPXaZ599xvbt2/nuu++YO7fSpxpdchIwCSGEEKLeKigo\nYNy4cXh5eTF58mQmT55cJ/mQgEkIIYQQ9ZaDgwOFhYV1nQ3pwySEEEIIUR0JmIQQQgghqiEBkxBC\nCCFENSRgEkIIIYSohnT6FkKIy6Dse7mEEJfHxfzdScAkhBCXUGBgIO7u7pf9RaFCCMPd3Z3AwMAL\nXo4ETEIIcQmFhYWxd+9eEhMT6zorQlyVAgMDCQsLu+DlSMB0BVqyZAljx46t62xcEaSs7CdlZb+a\nllVYWNhFOWBfqWTfsp+Ulf0ud1nVqtO3Uup+pdTvSqlspdQWpVSXatKPUkrtLUq/Qyk1qHbZFWB2\nEmEfKSv7SVnZT8qqZqS87CdlZb/LXVY1DpiUUqOBN4BngA7ADmCNUqrSBkKlVHdgMfAP4DpgJbBS\nKRVV20wLIYQQQlxOtalhehiYr7VeqLWOA6YCWcA9VaR/EPhWa/2m1nqf1voZIBZ4oFY5FkIIIYS4\nzGoUMCmlnIBOwPricVprDawDulcxW/ei6WWtOU96IYQQQoh6paadvgMBB+B0ufGngVZVzNOgivQN\nzrMeV5DnllQlNTWV2NjYus7GFUHKyn5SVvaTsqoZKS/7SVnZ72KVVZlYw/V86ZSpILKPUqohcBzo\nrrXeWmb8a0BPrXWPSubJBe7UWi8tM24aMEtr3aiK9YwDFtmdMSGEEEKICzNea724qok1rWFKBAqA\nkHLjg6lYi1TsVA3Tg2myGw8cBnJqmEchhBBCCHu5AhGY2KNKNaphAlBKbQG2aq0fLPqsgCPAu1rr\n1ytJ/xngprW+tcy4jcAOrfW0Gq1cCCGEEKIO1ObBlW8CnyilYoCfMXfNuQMfAyilFgLHtNZPFKV/\nB/hBKfUI8DUwFtNx/C8XlnUhhBBCiMujxgGT1npZ0TOXnsM0tW0HBmitzxYlCQWsZdJvVkqNBV4s\nGg4At2qt91xo5oUQQgghLocaN8kJIYQQQlxtavVqFCGEEEKIq4kETHVAKdVLKfWlUuq4UqpQKTWs\nkjTPKaVOKKWylFLfKaValJvup5RapJRKVUolK6UWKKU8yqW5Rim1oegdfglKqccu9bZdbEqpmUqp\nn5VSaUqp00qpFUqpluXSuCil3ldKJSql0pVS/1ZKBZdL00Qp9bVSKlMpdUop9ZpSylIuTW+lVIxS\nKkcptV8pNelybOPFopSaWvSuxtSiYZNSamCZ6VJOVSjazwqVUm+WGSflBSilnikqm7LDnjLTpZzK\nUEo1Ukp9WlQeWUW/yY7l0lz1x3dl3kdbfr8qVEq9VzS9/u1XWmsZLvMADMT0AbsN85iGYeWmPw6c\nA4YC7TDv34sHnMuk+RbzipnOQA9gPxBdZroXcBL4BGgD3AFkAn+u6+2vYVl9A0ws2ob2wCrM4ybc\nyqT5oGjcTZj3G24Cfiwz3QLsxNwy2h4YAJwBXiiTJgLIAF7DPIT1fiAf6FfXZVCDsrqlaN9qUTS8\nAOQCbaSczltuXYBDwK/Am7JfVSifZ4DfgCDMI2GCAX8pp0rLyhf4HViAubkpHPgT0LRMGjm+m20I\nKLM/BQN9MefDXvV1v6rzQrvaB6CQigHTCeDhMp+9gWzgjqLPbYrm61AmzQBMZ/sGRZ/vwzw3y7FM\nmpeBPXW9zRdYXoFF296zTNnkAsPLpGlVlOb6os+Din4kgWXSTAGSi8sHeBX4rdy6lgDf1PU2X2B5\nJQF3SzlVWT6ewD6gD/A9RQGTlJdNfp8BYquYJuVkm+dXgB+qSSPH98rL5W1gf33er6RJrp5RSjXF\nvDam7Pv60oCtlL5/rxuQrLX+tcys6wANdC2TZoPW2lomzRqglVLK5xJl/3LwxWznuaLPnTB3e5Yt\nr32YZ4OVLa+dWuvEMstZA/gAbcuk+cO881ApZVFKjcE88mMzUk5VeR/4Smv933LjOyPlVVakMl0I\n4pVS0UqpJkXjZb+yNRTYppRapkwXglil1J+LJ8rxvXLKvKd2PPDPolH18vcnAVP90wDzwzjf+/ca\nYKoeS2itCzBBRNk0lS0Dzv8ev3pLKaUwVyE/6dLHUjQA8ooOOmWVL6/qyqKqNN5KKZcLzfvlopRq\np5RKx1ydzcVcocUh5VRBUUB5HTCzkskhSHkV2wLchanlmAo0BTYU9amR/cpWM0ztzz6gPzAPeFcp\nNaFouhzfKzccE+h8UvS5Xv7+avPgSlE3FOaHdiFpVNHfK/VZEnOBKKCnHWntKS+qSXMlllcccC2m\nJm4ksFApdeN50l+V5aSUCsUE3/201vk1mZWrrLy01mVfF7FLKfUzkIDpN1PVq6uuunIqYgF+1lo/\nVfR5h1KqLSaIij7PfFf78f0e4Fut9alq0tXpfiU1TPXPKcwXer73750q+lxCKeUA+BVNK05T2TKg\nYsRd7yml/g4MBnprrU+UmXQKcFZKeZebpXx5lS+LkDLTqkoTDKRprfMuJO+Xk9baqrU+pLWO1Vo/\nCewAHkTKqbxOmE7MMUqpfKVUPqZz6YNKqTxMmbhIeVWktU7FdEJugexX5Z0E9pYbtxcIK/pfju/l\nKKXCMB3j/1FmdL3cryRgqme01r9jvuS+xeOKdpqumLsEwPRJ8VVKdSgza1/MD/HnMmluLPqhFesP\n7Cs64F0xioKlW4GbtdZHyk2OwXSGLFteLTEHqLLl1V6ZJ9QX6w+kUnpw21x2GWXSbL4Y21CHLIAL\nUk7lrcPcWXMdpkbuWmAbphag+P98pLwqUEp5As0xnZdlv7K1EdM5uaxWmBo5Ob5X7h5MEPRNmXH1\nc7+q657xV+MAeGAOytdhev0/VPS5SdH0v2HubhqKOaivxLxSpuxtp99gDupdgBswbeaflpnujTmg\nfYJpxhqNub3y3rre/hqW1VzMXQ+9MFcKxYNruTS/A70xNQcbqXj76Q7MrbrXYPpinAaeL5Mmoqh8\nXsUc4KYBecCf6roMalBWL2KaK8Mxtyu/jDno9JFysqv8Su6Sk/KyKZfXgRuL9qsewHdF2xkg5VSh\nrDpj+g/OxASV44B0YEyZNHJ8L90OhXl0wIuVTKt3+1WdF9jVOGCq/gsxz5woO3xUJs2zRT+ILEyv\n/hblluGLuRpOxQQU/wDcy6VpD/xQtIwjwP/V9bbXoqwqK6cC4M4yaVyA9zC32aYDnwPB5ZbTBPMM\np4yiH9WrgKWS7yUGc4vvAWBiXW9/DctqAeZ5QtmYq9i1FAVLUk52ld9/sQ2YpLx0yW3Yx4ryfwRY\njO1zhaScbLdjMOa5VVnAbuCeStI8ixzfAfoVHc9bVDKt3u1X8i45IYQQQohqSB8mIYT8UQ/vAAAA\naUlEQVQQQohqSMAkhBBCCFENCZiEEEIIIaohAZMQQgghRDUkYBJCCCGEqIYETEIIIYQQ1ZCASQgh\nhBCiGhIwCSGEEEJUQwImIYQQQohqSMAkhBBCCFENCZiEEEIIIaohAZMQQgghRDX+H40pAXCQ2otj\nAAAAAElFTkSuQmCC\n", 594 | "text/plain": [ 595 | "" 596 | ] 597 | }, 598 | "metadata": {}, 599 | "output_type": "display_data" 600 | }, 601 | { 602 | "name": "stdout", 603 | "output_type": "stream", 604 | "text": [ 605 | "Validation accuracy at 0.7862666845321655\n" 606 | ] 607 | } 608 | ], 609 | "source": [ 610 | "# ToDo: Find the best parameters for each configuration\n", 611 | "epochs = 1\n", 612 | "batch_size = 50\n", 613 | "learning_rate = 0.01 \n", 614 | "\n", 615 | "epochs = 1\n", 616 | "batch_size = 100\n", 617 | "learning_rate = 0.01\n", 618 | "\n", 619 | "epochs = 5\n", 620 | "batch_size = 100\n", 621 | "learning_rate = 0.2\n", 622 | "\n", 623 | "\n", 624 | "### DON'T MODIFY ANYTHING BELOW ###\n", 625 | "# Gradient Descent\n", 626 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss) \n", 627 | "\n", 628 | "# The accuracy measured against the validation set\n", 629 | "validation_accuracy = 0.0\n", 630 | "\n", 631 | "# Measurements use for graphing loss and accuracy\n", 632 | "log_batch_step = 50\n", 633 | "batches = []\n", 634 | "loss_batch = []\n", 635 | "train_acc_batch = []\n", 636 | "valid_acc_batch = []\n", 637 | "\n", 638 | "with tf.Session() as session:\n", 639 | " session.run(init)\n", 640 | " batch_count = int(math.ceil(len(train_features)/batch_size))\n", 641 | "\n", 642 | " for epoch_i in range(epochs):\n", 643 | " \n", 644 | " # Progress bar\n", 645 | " batches_pbar = tqdm(range(batch_count), desc='Epoch {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')\n", 646 | " \n", 647 | " # The training cycle\n", 648 | " for batch_i in batches_pbar:\n", 649 | " # Get a batch of training features and labels\n", 650 | " batch_start = batch_i*batch_size\n", 651 | " batch_features = train_features[batch_start:batch_start + batch_size]\n", 652 | " batch_labels = train_labels[batch_start:batch_start + batch_size]\n", 653 | "\n", 654 | " # Run optimizer and get loss\n", 655 | " _, l = session.run(\n", 656 | " [optimizer, loss],\n", 657 | " feed_dict={features: batch_features, labels: batch_labels})\n", 658 | "\n", 659 | " # Log every 50 batches\n", 660 | " if not batch_i % log_batch_step:\n", 661 | " # Calculate Training and Validation accuracy\n", 662 | " training_accuracy = session.run(accuracy, feed_dict=train_feed_dict)\n", 663 | " validation_accuracy = session.run(accuracy, feed_dict=valid_feed_dict)\n", 664 | "\n", 665 | " # Log batches\n", 666 | " previous_batch = batches[-1] if batches else 0\n", 667 | " batches.append(log_batch_step + previous_batch)\n", 668 | " loss_batch.append(l)\n", 669 | " train_acc_batch.append(training_accuracy)\n", 670 | " valid_acc_batch.append(validation_accuracy)\n", 671 | "\n", 672 | " # Check accuracy against Validation data\n", 673 | " validation_accuracy = session.run(accuracy, feed_dict=valid_feed_dict)\n", 674 | "\n", 675 | "loss_plot = plt.subplot(211)\n", 676 | "loss_plot.set_title('Loss')\n", 677 | "loss_plot.plot(batches, loss_batch, 'g')\n", 678 | "loss_plot.set_xlim([batches[0], batches[-1]])\n", 679 | "acc_plot = plt.subplot(212)\n", 680 | "acc_plot.set_title('Accuracy')\n", 681 | "acc_plot.plot(batches, train_acc_batch, 'r', label='Training Accuracy')\n", 682 | "acc_plot.plot(batches, valid_acc_batch, 'b', label='Validation Accuracy')\n", 683 | "acc_plot.set_ylim([0, 1.0])\n", 684 | "acc_plot.set_xlim([batches[0], batches[-1]])\n", 685 | "acc_plot.legend(loc=4)\n", 686 | "plt.tight_layout()\n", 687 | "plt.show()\n", 688 | "\n", 689 | "print('Validation accuracy at {}'.format(validation_accuracy))" 690 | ] 691 | }, 692 | { 693 | "cell_type": "markdown", 694 | "metadata": {}, 695 | "source": [ 696 | "## Test\n", 697 | "Set the epochs, batch_size, and learning_rate with the best learning parameters you discovered in problem 3. You're going to test your model against your hold out dataset/testing data. This will give you a good indicator of how well the model will do in the real world. You should have a test accuracy of atleast 80%." 698 | ] 699 | }, 700 | { 701 | "cell_type": "code", 702 | "execution_count": 42, 703 | "metadata": { 704 | "collapsed": false 705 | }, 706 | "outputs": [ 707 | { 708 | "name": "stderr", 709 | "output_type": "stream", 710 | "text": [ 711 | "Epoch 1/10: 100%|██████████| 2969/2969 [00:01<00:00, 2601.10batches/s]\n", 712 | "Epoch 2/10: 100%|██████████| 2969/2969 [00:01<00:00, 2602.94batches/s]\n", 713 | "Epoch 3/10: 100%|██████████| 2969/2969 [00:01<00:00, 2591.54batches/s]\n", 714 | "Epoch 4/10: 100%|██████████| 2969/2969 [00:01<00:00, 2591.01batches/s]\n", 715 | "Epoch 5/10: 100%|██████████| 2969/2969 [00:01<00:00, 2584.63batches/s]\n", 716 | "Epoch 6/10: 100%|██████████| 2969/2969 [00:01<00:00, 2593.57batches/s]\n", 717 | "Epoch 7/10: 100%|██████████| 2969/2969 [00:01<00:00, 2591.17batches/s]\n", 718 | "Epoch 8/10: 100%|██████████| 2969/2969 [00:01<00:00, 2602.96batches/s]\n", 719 | "Epoch 9/10: 100%|██████████| 2969/2969 [00:01<00:00, 2583.04batches/s]\n", 720 | "Epoch 10/10: 100%|██████████| 2969/2969 [00:01<00:00, 2583.16batches/s]" 721 | ] 722 | }, 723 | { 724 | "name": "stdout", 725 | "output_type": "stream", 726 | "text": [ 727 | "Nice Job! Test Accuracy is 0.8587999939918518\n" 728 | ] 729 | }, 730 | { 731 | "name": "stderr", 732 | "output_type": "stream", 733 | "text": [ 734 | "\n" 735 | ] 736 | } 737 | ], 738 | "source": [ 739 | "# ToDo: Set the epochs, batch_size, and learning_rate with the best parameters from problem 3\n", 740 | "epochs = 10\n", 741 | "batch_size = 48\n", 742 | "learning_rate = 0.001\n", 743 | "\n", 744 | "### DON'T MODIFY ANYTHING BELOW ###\n", 745 | "# The accuracy measured against the test set\n", 746 | "test_accuracy = 0.0\n", 747 | "\n", 748 | "with tf.Session() as session:\n", 749 | " \n", 750 | " session.run(init)\n", 751 | " batch_count = int(math.ceil(len(train_features)/batch_size))\n", 752 | "\n", 753 | " for epoch_i in range(epochs):\n", 754 | " \n", 755 | " # Progress bar\n", 756 | " batches_pbar = tqdm(range(batch_count), desc='Epoch {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')\n", 757 | " \n", 758 | " # The training cycle\n", 759 | " for batch_i in batches_pbar:\n", 760 | " # Get a batch of training features and labels\n", 761 | " batch_start = batch_i*batch_size\n", 762 | " batch_features = train_features[batch_start:batch_start + batch_size]\n", 763 | " batch_labels = train_labels[batch_start:batch_start + batch_size]\n", 764 | "\n", 765 | " # Run optimizer\n", 766 | " _ = session.run(optimizer, feed_dict={features: batch_features, labels: batch_labels})\n", 767 | "\n", 768 | " # Check accuracy against Test data\n", 769 | " test_accuracy = session.run(accuracy, feed_dict=test_feed_dict)\n", 770 | "\n", 771 | "\n", 772 | "assert test_accuracy >= 0.80, 'Test accuracy at {}, should be equal to or greater than 0.80'.format(test_accuracy)\n", 773 | "print('Nice Job! Test Accuracy is {}'.format(test_accuracy))" 774 | ] 775 | }, 776 | { 777 | "cell_type": "markdown", 778 | "metadata": {}, 779 | "source": [ 780 | "# Multiple layers\n", 781 | "Good job! You built a one layer TensorFlow network! However, you want to build more than one layer. This is deep learning after all! In the next section, you will start to satisfy your need for more layers." 782 | ] 783 | } 784 | ], 785 | "metadata": { 786 | "kernelspec": { 787 | "display_name": "Python 3", 788 | "language": "python", 789 | "name": "python3" 790 | }, 791 | "language_info": { 792 | "codemirror_mode": { 793 | "name": "ipython", 794 | "version": 3 795 | }, 796 | "file_extension": ".py", 797 | "mimetype": "text/x-python", 798 | "name": "python", 799 | "nbconvert_exporter": "python", 800 | "pygments_lexer": "ipython3", 801 | "version": "3.5.2" 802 | } 803 | }, 804 | "nbformat": 4, 805 | "nbformat_minor": 0 806 | } 807 | --------------------------------------------------------------------------------