├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── exp ├── __init__.py ├── loss_function_sample_size.ipynb ├── solve_sym_pos_matrix.ipynb └── tgp_np_gradient_check.ipynb └── tgp ├── .idea ├── test.py ├── test2.py └── zero.py ├── __init__.py ├── gradient_check.py ├── np ├── __init__.py └── gaussian_process.py └── tf ├── gaussian_process.py └── troubleshoot.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Untitled notebooks 2 | Untitled* 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | 58 | # Sphinx documentation 59 | docs/_build/ 60 | 61 | # PyBuilder 62 | target/ 63 | 64 | #Ipython Notebook 65 | .ipynb_checkpoints 66 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Rui Shu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tensorflow-gp 2 | Implementation of gaussian processes and bayesian optimization in tensorflow 3 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiShu/tensorflow-gp/45f48c93475de7e49658745b34536a74c9cb6380/__init__.py -------------------------------------------------------------------------------- /exp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiShu/tensorflow-gp/45f48c93475de7e49658745b34536a74c9cb6380/exp/__init__.py -------------------------------------------------------------------------------- /exp/loss_function_sample_size.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 7, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import sys\n", 12 | "sys.path.append('/Users/void/Documents/github/tensorflow-gp')import numpy as np\n", 13 | "import tensorflow as tf\n", 14 | "import scipy.linalg as la\n", 15 | "from src.np.gaussian_process import GaussianProcess as GP" 16 | ] 17 | } 18 | ], 19 | "metadata": { 20 | "kernelspec": { 21 | "display_name": "Python 2", 22 | "language": "python", 23 | "name": "python2" 24 | }, 25 | "language_info": { 26 | "codemirror_mode": { 27 | "name": "ipython", 28 | "version": 2 29 | }, 30 | "file_extension": ".py", 31 | "mimetype": "text/x-python", 32 | "name": "python", 33 | "nbconvert_exporter": "python", 34 | "pygments_lexer": "ipython2", 35 | "version": "2.7.10" 36 | } 37 | }, 38 | "nbformat": 4, 39 | "nbformat_minor": 0 40 | } 41 | -------------------------------------------------------------------------------- /exp/solve_sym_pos_matrix.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Comparing matrix inversion v. solving\n", 8 | "\n", 9 | "KX = Y\n", 10 | "\n", 11 | "where K is pretty big, X is pretty big, and Y is pretty big. We're solving for X" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "# Set up matrix K and Y\n", 23 | "import numpy as np\n", 24 | "from scipy.linalg import solve\n", 25 | "from numpy.linalg import inv\n", 26 | "import time\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "% matplotlib inline" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": { 35 | "collapsed": true 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "n = 1000\n", 40 | "K = np.random.randn(n, n)\n", 41 | "K = K.dot(K.T)\n", 42 | "Y = np.random.randn(n, n)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 3, 48 | "metadata": { 49 | "collapsed": true 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "def generate(n):\n", 54 | " K = np.random.randn(n, n)\n", 55 | " K = K.dot(K.T)\n", 56 | " Y = np.random.randn(n, n)\n", 57 | " return K, Y" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 4, 63 | "metadata": { 64 | "collapsed": true 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "def solve_by_inv(K, Y):\n", 69 | " # Solve by matrix inversion\n", 70 | " t = time.time()\n", 71 | " X = inv(K).dot(Y)\n", 72 | " t = time.time() - t\n", 73 | " err = abs(K.dot(X) - Y).mean()\n", 74 | " return err, t" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 5, 80 | "metadata": { 81 | "collapsed": true 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "def solve_by_sol(K, Y):\n", 86 | " # Solve by matrix inversion\n", 87 | " t = time.time()\n", 88 | " X = solve(K, Y, sym_pos=True)\n", 89 | " t = time.time() - t\n", 90 | " err = abs(K.dot(X) - Y).mean()\n", 91 | " return err, t " 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 6, 97 | "metadata": { 98 | "collapsed": false 99 | }, 100 | "outputs": [ 101 | { 102 | "name": "stdout", 103 | "output_type": "stream", 104 | "text": [ 105 | "Solution check: 2.55433963354e-11\n", 106 | "Computation time: 0.166968822479\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "# Solve by matrix inversion\n", 112 | "err, t = solve_by_inv(K, Y)\n", 113 | "print \"Solution check:\", err\n", 114 | "print \"Computation time:\", t" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 7, 120 | "metadata": { 121 | "collapsed": false 122 | }, 123 | "outputs": [ 124 | { 125 | "name": "stdout", 126 | "output_type": "stream", 127 | "text": [ 128 | "Solution check: 1.64575786881e-11\n", 129 | "Computation time: 0.114394187927\n" 130 | ] 131 | } 132 | ], 133 | "source": [ 134 | "# Solve by solving\n", 135 | "err, t = solve_by_sol(K, Y)\n", 136 | "print \"Solution check:\", err\n", 137 | "print \"Computation time:\", t" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 14, 143 | "metadata": { 144 | "collapsed": false, 145 | "scrolled": true 146 | }, 147 | "outputs": [ 148 | { 149 | "name": "stdout", 150 | "output_type": "stream", 151 | "text": [ 152 | "10\n", 153 | "********************************************************************************\n", 154 | "Error: 4.42049175042e-15 \t3.43683415061e-15\n", 155 | "Time: 9.20295715332e-05 \t0.0001060962677\n", 156 | "********************************************************************************\n", 157 | "Error: 3.29209819128e-13 \t1.7948229708e-13\n", 158 | "Time: 3.09944152832e-05 \t6.29425048828e-05\n", 159 | "********************************************************************************\n", 160 | "Error: 4.79287616539e-15 \t3.7774384315e-15\n", 161 | "Time: 2.88486480713e-05 \t5.79357147217e-05\n", 162 | "18\n", 163 | "********************************************************************************\n", 164 | "Error: 8.63665575662e-14 \t5.66552234493e-14\n", 165 | "Time: 4.10079956055e-05 \t7.20024108887e-05\n", 166 | "********************************************************************************\n", 167 | "Error: 1.8808092381e-13 \t1.27801132187e-13\n", 168 | "Time: 3.50475311279e-05 \t6.07967376709e-05\n", 169 | "********************************************************************************\n", 170 | "Error: 5.53955163847e-15 \t2.85114786932e-15\n", 171 | "Time: 3.40938568115e-05 \t6.00814819336e-05\n", 172 | "35\n", 173 | "********************************************************************************\n", 174 | "Error: 9.55214531162e-13 \t7.06246148581e-13\n", 175 | "Time: 0.00029993057251 \t0.000163078308105\n", 176 | "********************************************************************************\n", 177 | "Error: 9.10603868917e-14 \t5.75306378854e-14\n", 178 | "Time: 0.000164985656738 \t0.000113964080811\n", 179 | "********************************************************************************\n", 180 | "Error: 1.80277426722e-11 \t9.85386184063e-12\n", 181 | "Time: 0.000129222869873 \t0.000195026397705\n", 182 | "68\n", 183 | "********************************************************************************\n", 184 | "Error: 3.22948889123e-12 \t1.7591507325e-12\n", 185 | "Time: 0.000412940979004 \t0.000247001647949\n", 186 | "********************************************************************************\n", 187 | "Error: 3.11033180812e-13 \t1.89298640525e-13\n", 188 | "Time: 0.000715017318726 \t0.000301837921143\n", 189 | "********************************************************************************\n", 190 | "Error: 3.09719012001e-12 \t1.70594262638e-12\n", 191 | "Time: 0.000520944595337 \t0.00047492980957\n", 192 | "129\n", 193 | "********************************************************************************\n", 194 | "Error: 1.68938517692e-10 \t9.88073725983e-11\n", 195 | "Time: 0.00245594978333 \t0.00132703781128\n", 196 | "********************************************************************************\n", 197 | "Error: 2.85277239044e-12 \t1.74371486702e-12\n", 198 | "Time: 0.00215888023376 \t0.00173306465149\n", 199 | "********************************************************************************\n", 200 | "Error: 3.85807851936e-13 \t2.35591375268e-13\n", 201 | "Time: 0.00109601020813 \t0.00109505653381\n", 202 | "244\n", 203 | "********************************************************************************\n", 204 | "Error: 1.42796057749e-09 \t9.20738798286e-10\n", 205 | "Time: 0.00374412536621 \t0.00209999084473\n", 206 | "********************************************************************************\n", 207 | "Error: 2.38886458949e-10 \t1.39761291498e-10\n", 208 | "Time: 0.0028510093689 \t0.00250101089478\n", 209 | "********************************************************************************\n", 210 | "Error: 4.71068022907e-13 \t3.10499473008e-13\n", 211 | "Time: 0.00812816619873 \t0.00343108177185\n", 212 | "464\n", 213 | "********************************************************************************\n", 214 | "Error: 1.2198434094e-11 \t7.78292422394e-12\n", 215 | "Time: 0.0160710811615 \t0.0093309879303\n", 216 | "********************************************************************************\n", 217 | "Error: 2.07141102474e-12 \t1.36553636803e-12\n", 218 | "Time: 0.0166411399841 \t0.00949811935425\n", 219 | "********************************************************************************\n", 220 | "Error: 2.90398855521e-11 \t1.8192453841e-11\n", 221 | "Time: 0.0132720470428 \t0.00915384292603\n", 222 | "879\n", 223 | "********************************************************************************\n", 224 | "Error: 1.00970254969e-09 \t6.40454072489e-10\n", 225 | "Time: 0.0777299404144 \t0.0480480194092\n", 226 | "********************************************************************************\n", 227 | "Error: 2.19341119894e-10 \t1.3761695871e-10\n", 228 | "Time: 0.0762650966644 \t0.0459249019623\n", 229 | "********************************************************************************\n", 230 | "Error: 7.73593810326e-11 \t4.52258853839e-11\n", 231 | "Time: 0.0735318660736 \t0.0467600822449\n", 232 | "1668\n", 233 | "********************************************************************************\n", 234 | "Error: 1.7599371871e-10 \t1.10621402707e-10\n", 235 | "Time: 0.593911886215 \t0.29734992981\n", 236 | "********************************************************************************\n", 237 | "Error: 5.9789171951e-10 \t3.73527719942e-10\n", 238 | "Time: 0.759345054626 \t0.336894035339\n", 239 | "********************************************************************************\n", 240 | "Error: 8.17685037585e-11 \t5.42736781058e-11\n", 241 | "Time: 0.756484031677 \t0.284887075424\n", 242 | "3162\n", 243 | "********************************************************************************\n", 244 | "Error: 1.64172481528e-09 \t1.00572685299e-09\n", 245 | "Time: 3.15070295334 \t2.03040194511\n", 246 | "********************************************************************************\n", 247 | "Error: 1.06867099803e-08 \t6.74737991947e-09\n", 248 | "Time: 2.91052508354 \t2.09549403191\n", 249 | "********************************************************************************\n", 250 | "Error: 8.36690771438e-10 \t5.61995860124e-10\n", 251 | "Time: 3.53569293022 \t2.42399406433\n" 252 | ] 253 | } 254 | ], 255 | "source": [ 256 | "ns = np.power(10, np.linspace(1, 3.5, num=10)).astype(int)\n", 257 | "trials = 3\n", 258 | "inv_err = np.zeros(trials*len(ns))\n", 259 | "sol_err = np.zeros(trials*len(ns))\n", 260 | "inv_t = np.zeros(trials*len(ns))\n", 261 | "sol_t = np.zeros(trials*len(ns))\n", 262 | "x = np.zeros(trials*len(ns))\n", 263 | "\n", 264 | "i = 0\n", 265 | "for n in ns:\n", 266 | " print n\n", 267 | " for _ in xrange(trials):\n", 268 | " K, Y = generate(n)\n", 269 | " err1, t1 = solve_by_inv(K, Y)\n", 270 | " err2, t2 = solve_by_sol(K, Y)\n", 271 | " inv_err[i] = err1\n", 272 | " sol_err[i] = err2\n", 273 | " inv_t[i] = t1\n", 274 | " sol_t[i] = t2\n", 275 | " x[i] = n\n", 276 | " i += 1\n", 277 | " print \"*\"*80\n", 278 | " print \"Error:\", err1, \"\\t\", err2\n", 279 | " print \"Time:\", t1, \"\\t\", t2" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": 15, 285 | "metadata": { 286 | "collapsed": false 287 | }, 288 | "outputs": [ 289 | { 290 | "data": { 291 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEKCAYAAADkYmWmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4FOX2wPHvofegdFEQropdLlfpSuyASi7SFemIIvYu\nKqAi6k+xIyJFqlSFIIKoJCItdFE6KtIkgDQpgZTz+2MWbghJdpPs7uxuzud59mF2dnbmDJucvHvm\nnfcVVcUYY0x4KuB2AMYYY3LPkrgxxoQxS+LGGBPGLIkbY0wYsyRujDFhzJK4McaEMUvixhgTxiyJ\nm5AkIltF5JiIHBaRfzz/fuB2XN6ISJyIdHM7DpN/FHI7AGOyoMAdqhrnbUMRKaiqqd7W5XQfudnG\nmGCzlrgJZZLpSpHOIrJARAaLyD6gXxbrRERe9LTqd4vI5yJSxrOP6iKSJiLdRORP4IdMjtNERLaL\nyDMi8hcwUkTKishMEdkjIn97ls/zbP8acD3wUfpvDiJyqYjM9Wy/XkTaBOa/y+RHQU/iIjJCRBJF\nZI2f9vemiPwiImtEpK0/9mnCQj1gC1ARGJjFuq5AJ6AJUBMoDXyUYT83AJcCt2dxnMpAWaAacD/O\n78xI4ALPumPAxwCq+iLwE9BHVcuo6iMiUgKYC4wDygPtgY9F5NI8nLsxp7nREh9F1r8wOSIizYHa\nwNVAfeApESnlj32bkDBdRPaLyAHPv93TvbZTVYeoapqqnshi3T3AYFX9U1WPAc8D7UXk1M+9Av1U\n9Xi6fWSU6tkmWVVPqOp+Vf3Ks3wUGITzhyArdwJ/qOoYdfwMfAlYa9z4RdBr4qq6QESqp18nIjVx\nWjPlcVo2PVV1kw+7uxyYr84oXsc8rfumwFQ/h23cEZNNTXy7D+vOA/5M9/xPnJ/5SunW7fASw15V\nTT71RESKA+/hNETK4pR8SomIaOajyVUH6ovI/lO7AAoCY70c1xifhEpNfBjOV9DrgKeBT3x8389A\nUxEpLiLlgRtxvuaayJBpTdwjs4SZcd0unCR6SnUgGUj0sp/s9vkkcDFwnaqW5X+tcMli++1AvKqe\n63mc4ym1POTluMb4xPXeKSJSEmgITBGRU78IhT2vtQRe4cxfDAF2qGozVf1ORK4DFgF7PP9a7wFz\nyhfAMyIyB9iHUyefqKppnh+17P5IZKU0cBw4LCLnAv0zvJ6IU38/5WtgkIh0BCZ6jnkNcERVN+Ti\n+MacIRRa4gWAA6paR1X/7XlcCeCpPV6lqlene1ylqs1OvVlVX/e853bPvnwpw5jwMNPTy+PUY1oO\n3z8Sp2wxH/gNp1T3SLrXczOY/ntACZw/CouAbzK8/j7QxtMT5T1VPQLchnNBc5fn8QZQJBfHNuYs\n4sakECJyITBTVa/yPF8AvKeqUz3Pr1ZVr71XPBeoyqrqfhG5GqcHQG1VTQtY8MYYE0KCnsRFZAIQ\nDZTD+erZD5gHDAWq4JR4Jqrqaz7sqyiwEqdFdRjopaq/BCZyY4wJPa60xI0xxvhHKNTEjTHG5JIl\ncWOMCWNB7WIoIla7McaYXFDVTLvEBr0lrqoR8+jXr5/rMdg55c9zitTzsnNK99i8Ga1QAU1IyDan\nWjnFGGNCTVIStGkDL78Mdetmu6klcWOMCTWPPgqXXAIPeR+dwfXb7sNZdHS02yH4nZ1T+IjE87Jz\nAsaNg/h4WLYMxPvIEEHtJ571QG/GGGNYtw6aNIEffoCrrz69WkTQULmwaYwxJhNHjkDr1vDWW2ck\ncG+sJW6MMW5Thfvug8KFYdSos17OriVuNXFjjHHbZ5/BmjWwZEmO32otcWOMcdPKlXD77bBgAdSq\nlekmVhM3xphQdPCg0x/8o4+yTODeWEvcGGPcoAqtWsF55zlJPBtWEzfGmFDz3nuwfTt88UWedmMt\ncWOMCbZFi6BlS+dCZo0aXje3mrgxxoSKffugfXsYPtynBO6Nz0lcRAqIyEoRic3i9Q9EZLOIrBaR\n2nmOzBhjIk1aGnTsCB06wF13+WWXOWmJPwqsy+wFEWkG/EtVLwZ64cyXaYwxJr3XX4ejR+E1r1MI\n+8ynJC4i5wPNgeFZbBIDjAFQ1QQgSkQq+SVCY4yJBPPmwccfw8SJzp2ZfuJrS/xd4GmcWeUzUxXY\nnu75Ts86Y4wxf/3llFHGjoWq/k2NXrsYisgdQKKqrhaRaMD72IjZ6N+//+nl6OjoiBx60hhjTktJ\ncS5k9uoFt9zi01vi4+OJj4/3aVuvXQxF5HWgI5ACFAdKA1+qaqd02wwF4lR1kuf5BqCJqiZm2Jd1\nMTTG5C8vvADLl8Ps2VCwYK52kacuhqr6gqpWU9WaQHtgXvoE7hELdPIcrD5wMGMCN8aYfGfWLKeE\nMn58rhO4N7m+Y1NEegGqqsNU9RsRaS4iW4CjQFe/RWiMMeHozz+hWzeYNg0qVAjYYeyOTWOM8beT\nJ+H6653BrZ56Ks+7y66cYkncGGP87dFHYetWmD7dp3kyvbEBsIwxJlimTIGZM2HFCr8kcG+sJW6M\nMf6yeTM0bOj0RLn2Wr/t1gbAMsaYQDt+3JnoeMAAvyZwb6wlbowx/tCjhzMuyoQJfi+jWE3cGGMC\nafRoZ47MZcuCUgdPz1rixhiTF7/+CjfeCHFxcOWVATmE1cSNMSYQ/vnHqYO//XbAErg31hI3xpjc\nUIV77oGSJZ1ZegLIauLGGONvQ4fCunXOPJkuspa4Mcbk1PLl0KyZM+HxxRcH/HBWEzfGGH85cADa\ntoUhQ4KSwL2xlrgxxvhKFVq2hGrV4IMPgnZYq4kbY4w/vPOOM9Xa5MluR3KatcSNMcYXCxfC3XfD\n0qVQvXpQD201cWOMyYu9e515MkeODHoC98ZrEheRoiKSICKrRGStZ87NjNs0EZGDIrLS83gxMOEa\nY0yQpabCvfc6s9XfcYfb0ZzFa01cVU+IyI2qekxECgILRaSRqi7MsOl8VW0RmDCNMcYlAwfCiRPw\n6qtuR5Ipny5squoxz2JRnNb7gUw2C+6oL8YYE2jff+/c1LNiBRQKzX4gPtXERaSAiKwCdgPxqrou\nk80aiMhqEZklIpf7NUpjjAm2nTvhvvtg3DioUsXtaLLkUxJX1TRV/TdwPnCDiDTJsMkKoJqq1gY+\nAqb7N0xjjAmilBTnQuZDD8FNN7kdTbZy9P1AVQ+LyCzgWuDHdOuPpFueLSJDRORcVd2fcR/9+/c/\nvRwdHU10dHQuwjbGmADq29cZ2OqFF1w5fHx8PPHx8T5t67WfuIiUB5JV9ZCIFAe+BQao6g/ptqmk\nqome5brAZFW9MJN9WT9xY0xoi42FPn1g5UooX97taIC837FZBRgtIoJTfhmrqj+ISC9AVXUY0FpE\nHgSSgeNAOz/FbowxwbN1qzPN2vTpIZPAvbE7No0xBpxuhI0bQ4cO8MQTbkdzhuxa4pbEjTEGnBLK\nrl0wbVrQ58n0xgbAMsaY7EyaBLNnO/3BQyyBe2MtcWNM/rZxo1NG+fZbqFPH7WgyZQNgGWNMZo4d\ncyY6fu21kE3g3lhL3BiTf3XrBidPwtixIV1GsZq4McZkNGqUM8nx0qUhncC9sZa4MSb/WbMGbr4Z\nfvwRLg/9oZ6sJm6MMaccPuzUwd99NywSuDfWEjfG5B+qzsBWUVEwbJjb0fjMauLGGAPw8cewaRMs\nXux2JH5jLXFjTP6wbJkzvdqiRXDRRW5HkyNWEzfG5G/790Pbts4sPWGWwL2xlrgxJrKlpUFMjJO8\n333X7WhyxWrixpj86+23Yd8+Z2CrCGRJ3BgTuebPh8GDnRt6ihRxO5qAsJq4MSYyJSbCPfc4d2ZW\nq+Z2NAFjNXFjTORJTYXbb4f69Z3BrcJcnnqniEhREUkQkVUislZEXs9iuw9EZLOIrBaR2nkN2hhj\ncu2VV5xEnm5i9kjltSauqidE5EZVPSYiBYGFItJIVRee2kZEmgH/UtWLRaQeMBSoH7iwjTEmC3Pn\nwvDhzgQPhSL/sp9PNXFVPeZZLOp5z4EMm8QAYzzbJgBRIlLJX0EaY4xPduyATp1g/HioXNntaILC\npyQuIgVEZBWwG4hX1XUZNqkKbE/3fKdnnTHGBEdyMrRrB488AtHRbkcTND5911DVNODfIlIGmCsi\nTVT1x9wcsH+6GlV0dDTR+eg/2xgTQM8/D2XLwnPPuR1JnsXHxxMfH+/TtjnunSIiLwHHVPWddOuG\nAnGqOsnzfAPQRFUTM7zXeqcYY/xvxgynBb5yJZQr53Y0fpfX3inlRSTKs1wcuBVYnWGzWKCTZ5v6\nwMGMCdwYYwLi99+hZ09nxvoITODe+FJOqQKMFhHBSfpjVfUHEekFqKoOU9VvRKS5iGwBjgJdAxiz\nMcY4kpKgTRvo29fpE54P2c0+xpjw1bs37NkDU6aE9TyZ3tgAWMaYyDNhAnz3HSxfHtEJ3BtriRtj\nws/69XDDDU4Srx35N4jbpBDGmMhx9KhTBx80KF8kcG+sJW6MCR+q0KWL8+/o0fmmjGI1cWNMZBgx\nwhkTJSEh3yRwb6wlbowJD6tXw623OhM9XHaZ29EEldXEjTHh7dAhpw7+/vv5LoF/suyTbF+3coox\nJrSpQvfucMstzkw9+URyajKPzXmMuK1x2W5nLXFjTGj78EP444+wnak+N/Yf30+z8c344+AfLO6+\nONttLYkbY0JXQoIzvdqUKVCsmNvRBMWGfRuoP7w+11S6hpkdZhJVLCrb7a2cYowJTX//DW3bwrBh\nULOm29EExdzf5tLxy44MunkQ3et09+k91jvFGBN60tLgrrvg0kvhnXe8bx/mVJUPl37I6z+9zuQ2\nk7mh+g1nvG79xI0x4eXNN+HgQXjjDbcjCbjk1GT6fNOHhdsXsrj7YmqcUyNH77ckbowJLT/+6HQl\nXL4cChd2O5qA+vvY37Se0ppSRUqxqPsiyhQtk+N92IVNY0zo2L3b6UY4ejScf77b0QTUur3rqDe8\nHnXPq8v0dtNzlcDBWuLGmFCRmuok8O7d4fbb3Y4moGZvnk3n6Z35v1v/j861O+dpX75Mz3a+iMwT\nkbUi8ouIPJLJNk1E5KCIrPQ8XsxTVMaY/Kd/f2c8lH793I4kYFSVwYsH0z22O9PbT89zAgffWuIp\nwBOqulpESgErRGSuqm7IsN18VW2R54iMMfnPnDkwapQzuFXBgm5HExAnU0/y4NcPsvyv5Szuvpjq\nZav7Zb9ek7iq7gZ2e5aPiMh6oCqQMYnbkGLGmJzbvt0ZXnbyZKhUye1oAmLv0b20mtyKciXKsbDb\nQkoVKeW3fefowqaIXAjUBhIyebmBiKwWkVkicrkfYjPGRLqTJ50beh5/3JmpJwL9uudX6g2vR+Nq\njZnWdppfEzjk4GYfTyklHnhVVWdk8lqaqh4TkWbA+6p6SSb7sJt9jDH/88QTsHkzzJgBBSKvs9zX\nm76m64yuvHf7e9x79b253k+eb/YRkULAVGBsxgQOTpkl3fJsERkiIueq6v6M2/bv3//0cnR0NNHR\n0b6EYIyJNF9+CV995dTBIyyBqypvL3qb9xLeY2aHmdQ/v36O3h8fH098fLxP2/rUEheRMcA+VX0i\ni9crqWqiZ7kuMFlVL8xkO2uJG2Pgt9+gfn2YNQvq1nU7Gr86kXKCXl/3Yk3iGma0n8EFURfkeZ95\naomLSCPgXuAXEVkFKPACUB1QVR0GtBaRB4Fk4DjQLs9RG2MiU1KSM8HDyy9HXAJPPJJIy0ktOa/0\nefzU9SdKFikZ8GPaAFjGmODq1QsOHIBJkyJqnsyfd/9MzMQYOl3Tif7R/Skg/isR2QBYxpjQMG4c\nxMU546JEUAKfvmE6PWf25KNmH9HuyuAWIqwlbowJjnXroEkT+OEHuPpqt6PxC1XljQVv8PGyj/mq\n3VdcV/W6gBzHWuLGGHcdOQKtWztDzEZIAk9KSaJHbA827NtAQo8Eqpap6koc1hI3xgSWKtx3nzOs\n7KhRbkfjF7uP7Oa/E/9L9bLVGRUzihKFSwT0eNm1xCOrc6YxJvR89hn8/DN8/LHbkfjFqr9WUfez\nutxx8R1MbDUx4AncG2uJG2MCZ+VKZ1jZBQugVi23o8mzaeum8cCsB/jkjk9ofXnroB3XauLGmOA7\ndMjpD/7RR2GfwFWVV+e/yvCVw/m247fUqVLH7ZBOs5a4Mcb/VKFVK6hSJezLKMeTj9N1Rlf+OPgH\n09tNp0rpKkGPwWrixpjgeu89Z4jZwYPdjiRPdv2zixs+v4GCBQoS3znelQTujSVxY4x/LV4MgwY5\n44MXLep2NLm2fNdy6g2vR8tLWzKu5TiKFy7udkiZspq4McZ/9u2Ddu1g+HCoUcPtaHJt0q+T6DO7\nD8PuHEbLy1q6HU62rCZujPGPtDRo3ty5meett9yOJlfSNI0B8QMY/fNoprefTu3Ktd0OCbDeKcaY\nYBg0CI4ehYED3Y4kV44lH6Pz9M7sPLyThB4JVCoVHlPFWU3cGJN3cXFOV8KJE507M8PMjsM7uH7U\n9ZQoXIK4znFhk8DBkrgxJq/++gvuvRfGjoWq7owfkhcJOxKoN7we7a5ox+cxn1O0UHhdjLVyijEm\n91JSoEMHZ4zwW25xO5ocm/DLBB6d8ygjWoygRa0WboeTK5bEjTG59/LLTvnkxRfdjiRH0jSNl+a9\nxIRfJzCv0zyuqnSV2yHlmi/Ts50PjAEqAWnAZ6r6QSbbfQA0A44CXVR1tZ9jNcaEklmznBLKihVQ\nsKDb0fjsyMkjdPqqE3uP7WVpj6VUKFnB7ZDyxJeaeArwhKpeATQAHhKRS9NvICLNgH+p6sVAL2Co\n3yM1xoSOP/+Ebt3giy+gYkW3o/HZtkPbaDyyMWWLleX7+74P+wQOPiRxVd19qlWtqkeA9UDGqxcx\nOK11VDUBiBKR8Lm8a4zx3cmTzg09Tz0FjRu7HY3PFm9fTP3h9el0TSdGtBgRdhcws5KjmriIXAjU\nBhIyvFQV2J7u+U7PusQ8xGaMCUVPPw2VKjlJPEyM/XksT859ks//+znNL27udjh+5XMSF5FSwFTg\nUU+LPFf69+9/ejk6Opro6Ojc7soYE2xTp8LMmU4dPAwmOk5NS6XvvL5MWTeF+C7xXF7hcrdD8kl8\nfDzx8fE+bevTbfciUgj4Gpitqu9n8vpQIE5VJ3mebwCaqGpihu3stntjwtXmzdCwIcyeDdde63Y0\nXv1z4h/u/fJeDp84zNS2UylforzbIeWaP4aiHQmsyyyBe8QCnTwHqw8czJjAjTFh7PhxZ6LjAQPC\nIoFvPbiVhiMbUrlUZebeNzesE7g3XlviItIImA/8Aqjn8QJQHVBVHebZ7iOgKU4Xw66qujKTfVlL\n3Jhw1LOnM2P9hAkhX0ZZsG0Bbaa04fnGz/Nw3YeREI/XF9m1xG0UQ2NM9saMgddfh2XLoHRpt6PJ\n1qhVo3j2+2cZ23Ist190u9vh+I2NYmiMyZ1ff4Unn3QGuArhBJ6alsoz3z1D7KZY5nedz6XlL/X+\npghhSdwYk7kjR5w6+Ntvw5VXuh1Nlg6fOEyHaR1ISkkioUcC5xY/1+2QgspGMTTGnE0V7r8fGjWC\nzp3djiZLvx/4nQYjGlA9qjpz7p2T7xI4WEvcGJOZoUNh7VpYssTtSLIUvzWe9lPb83KTl+l9XW+3\nw3GNXdg0xpxpxQpo2hQWLoRLLnE7mkx9tuIzXox7kfF3j+eWmuE3BG5O2YVNY4xvDhyANm1gyJCQ\nTOApaSk8+e2TzPltDj91/YlLyoVejMFmLXFjjEMVWraEatXgg7NGm3bdwaSDtJ/anjRNY1LrSZxT\n/By3Qwoaf9yxaYyJdIMHO1Otvf2225GcZfPfm2kwogGXlLuEb+79Jl8lcG+sJW6Mcerfd98NS5dC\n9epuR3OGeX/Mo8O0DrwS/Qq9ru3ldjiusJq4MSZre/dC+/YwcmTIJfBPln3CgB8HMLHVRG6scaPb\n4YQkS+LG5GepqdCxo/O44w63ozktJS2Fx+Y8xrw/5rGg2wIuOvcit0MKWZbEjcnPBg6EpCR49VW3\nIzntwPEDtJ3alsIFCrO4+2KiikW5HVJIswubxuRX33/v3NQzcSIUCo323MZ9G6k3vB5XVbyKmR1m\nWgL3QWh8csaY4Nq1C+67D8aPhypV3I4GgLm/zaXjlx0ZdPMgutfp7nY4YcOSuDH5TUqKcyGzd2+4\n6Sa3o0FV+WjpRwz8aSBT207lhuo3uB1SWLEkbkx+07cvlCjh/Ouy5NRkHp79MAu2LWBx98XUOKeG\n2yGFHa9JXERGAHcCiap6dSavNwFmAL97Vn2pqq/5NUpjjH/MnAlffAErV0IBdy+J/X3sb1pPaU2p\nIqVY1H0RZYqWcTWecOXLpzgK8DZFxnxVreN5WAI3JhRt3Qo9ejgXMsu7O+fk+r3rqTe8Hteddx3T\n2023BJ4HXlviqrpARLzdARD+k9gZE8lOnHAGtnr2WWfGehfN2TKHTl914q1b36JL7S6uxhIJ/FUT\nbyAiq4GdwNOqus5P+zXG+MNTT8H558Pjj7sWgqryfsL7vLXwLb5q9xWNqjVyLZZI4o8kvgKopqrH\nRKQZMB2w8SGNCRWTJsE33zjjhLs08/vJ1JP0ntWbZbuWsbj7YqqXDa3b+8NZnpO4qh5JtzxbRIaI\nyLmquj+z7fv37396OTo6mujo6LyGYIzJysaN0KcPfPstlC3rSgh7j+6l1eRWnFv8XBZ2W0ipIqVc\niSOcxMfHEx8f79O2Po1iKCIXAjNV9apMXqukqome5brAZFW9MIv92CiGxgTLsWNQv77TH/yBB1wJ\n4dc9v9Liixa0u6IdA28eSAGxm8RzI7tRDL0mcRGZAEQD5YBEoB9QBFBVHSYiDwEPAsnAceBxVU3I\nYl+WxI0Jlm7dnAua48a5Ukb5etPXdJvRjcG3D6bj1R2DfvxIkqck7udALIkbEwyjRsFbb8GyZVAq\nuOULVeWdxe/w7pJ3mdZ2GvXPrx/U40ciG0/cmPxkzRp45hmIjw96Aj+RcoIHZj3A6t2rWdJ9CRdE\nXRDU4+dHVqAyJpIcPuz0Bx88GK64IqiH3nN0DzeNuYnDJw6zoOsCS+BBYkncmEihCj17QpMmzgiF\nQbQmcQ11P6vLzTVuZkqbKZQsUjKox8/PrJxiTKQYMgQ2bYJFi4J62BkbZtBjZg8+bPYh7a9sH9Rj\nG7uwaUxkWLbMmV5t0SK4KDhTmakqbyx4g4+XfcyX7b6kbtW6QTlufmQXNo2JZPv3Q9u28MknQUvg\nSSlJ9JzZk/V715PQI4GqZaoG5bjmbFYTNyacpaVB584QEwOtWgXlkLuP7ObG0TdyIuUE87vOtwTu\nMkvixoSzt9+GffucPuFBsOqvVdQbXo9mFzVjUutJlChcIijHNVmzcoox4Wr+fHjnHaceXqRIwA/3\n5fov6fV1L4Y0H0KbK9oE/HjGN5bEjQlHe/bAPffA559DtWoBPZSqMvCngXy64lPm3DuH/5z3n4Ae\nz+SMJXFjwk1qqpPAO3eGZs0CeqjjycfpFtuN3/b/xtIeS6lSukpAj2dyzmrixoSbV15xEvmAAQE9\nzK5/dtHk8yYIwo9dfrQEHqIsiRsTTubOhc8+gwkToFDgvkgv37WcesPrEVMrhvF3j6d44eIBO5bJ\nGyunGBMuduxwSihffAFVAtcqnrx2Mg998xCf3vkpd192d8COY/zDkrgx4SA5Gdq1g4cfhgDNhpWm\nabzy4yuMWj2K7+77jtqVawfkOMa/LIkbEw5eeAGiouC55wKy+2PJx+gyvQvbD28noUcClUtVDshx\njP9ZEjcm1M2YAZMnw8qVUMD/l7F2HN5BzMQYrqhwBXGd4yhWqJjfj2ECx+tPhIiMEJFEEVmTzTYf\niMhmEVktIvYdzBh/+f13Z3jZSZOgXDm/737pzqXUH16ftpe3ZfR/R1sCD0O+/FkfBdye1Ysi0gz4\nl6peDPQChvopNmPyt6QkZ4KHF15wJjz2swm/TOCOCXcw5I4hPNv4WcSFeThN3nktp6jqAhGpns0m\nMcAYz7YJIhIlIpVUNdFfQRqTLz3xBNSoAY8+6tfdpmkaL8e9zPhfxvNDpx+4utLVft2/CS5/1MSr\nAtvTPd/pWWdJ3JjcmjDB6RO+YoVfZ6o/cvIInb7qxJ6je0jokUDFkhX9tm/jjqBf2Ozfv//p5ejo\naKID1F3KmLC1YYPT+v7uO6dHip9sO7SNFl+04N9V/s0Xrb6gaKGiftu38a/4+Hji4+N92tanmX08\n5ZSZqnrW9y4RGQrEqeokz/MNQJPMyik2s48xXhw9CvXqOUm8Z0+/7Xbx9sW0mtyKJxs8yRMNnrD6\nd5jJbmYfX/srieeRmVigk+dA9YGDVg83JhdUoXdvqFMHevTw227H/jyWmIkxDG8xnCcbPmkJPMJ4\nLaeIyAQgGignItuAfkARQFV1mKp+IyLNRWQLcBToGsiAjYlYI0fC8uWwdKlf6uBpmsYLP7zAlHVT\niOscxxUVr/BDkCbU2ETJxoSCn3+GW25xJnq47LI87+6fE//Q8auOHEw6yLS20yhforwfgjRu8Uc5\nxRgTKIcPQ+vW8P77fkngWw9updHIRlQsUZHv7vvOEniEs5a4MW5SdWaqL1cOhub9PrkF2xbQZkob\nnmv0HI/Ue8Tq3xEiu5a4jZ1ijJs+/NC5tX7s2Dzv6vPVn/PMd88wpuUYml7U1A/BmXBgLXFj3JKQ\nAHfdBUuWQM2aud5Naloqz37/LNM3TGdmh5lcViHvJRkTWqwlbkyo+ftvZ3zwTz/NUwI/fOIw90y7\nh+Mpx0nokUC5Ev4fJMuENruwaUywpaVBp07QqhW0bJnr3fx+4HcajmjIBWUuYM69cyyB51OWxI0J\ntrfeggMH4I03cr2LH7f+SMMRDXnw2gf55M5PKFywsB8DNOHEyinGBNOPP8J778GyZVA4d4l3+Mrh\n9J3Xl3Etx3Hrv271c4Am3FgSNyZYdu+Ge+6B0aPhggty/PaUtBSemvsUs7fM5qeuP3FJuUsCEKQJ\nN5bEjQmG1FQngXfvDrdnOcdKlg4lHaLd1HakaipLui/hnOLnBCBIE46sJm5MMPTv74yH0q9fjt+6\nZf8W6o9XnVCyAAAR7UlEQVSoz8XnXszse2dbAjdnsJa4MYE2Zw6MGuVM8FCwYI7eOu+PeXSY1oEB\n0QN44NoHAhSgCWeWxI0JpO3boUsXZ7b6SpVy9Nahy4fSP74/X7T6gptq3BSY+EzYsyRuTKAkJzs3\n9Dz2GNxwg89vS0lL4fE5j/P9H9+zoNsCLjr3ogAGacKdJXFjAuXZZ52BrZ55xue3HDh+gLZT21Ko\nQCGWdF9CVDH/Tc9mIpNd2DQmEL76Cr780ulOWMC3X7NNf2+i/oj6XFnhSmZ2mGkJ3PjEp58uEWkq\nIhtEZJOIPJvJ601E5KCIrPQ8XvR/qMaEid9+g169nDr4uef69JbvfvuOxiMb83TDp3m36bsUKmBf\nko1vfJmerQDwEXAzsAtYJiIzVHVDhk3nq2qLAMTou3ffde6EGzfO59aPMX6VlARt2sBLL0Hdul43\nV1U+XvYxr81/jSltptDkwiZBCNJEEl8yXV1gs6r+qarJwEQgJpPt3B19/sgRGDQINmyA5593NRST\njz32GFx0EfTp43XT5NRkes/qzSfLP2FR90WWwE2u+JLEqwLb0z3f4VmXUQMRWS0is0Tkcr9ElxOf\nfgo33ghz58K0aTBiRNBDMPnc+PEwbx4MH+51ouO/j/3N7eNuZ9vhbSzuvpia5+R+OFqTv/mr8LYC\nqKaqx0SkGTAdyHRgh/79+59ejo6OJjo6Ou9HT0qCd96B2bOhfHmYNcvp0lWzppPYjQm0deucVvgP\nP0CZMtluun7velpMbMF/a/2XN255g4IFcnYDkIl88fHxxMfH+7St15l9RKQ+0F9Vm3qePweoqr6Z\nzXv+AP6jqvszrA/MzD5Dhjh3xcXG/m/dvHnQoYMze3itWv4/pjGnHD0K110HTz0F3bplu+mcLXPo\n9FUn3rr1LbrU7hKc+EzYy25mH1+SeEFgI86Fzb+ApUAHVV2fbptKqproWa4LTFbVCzPZl/+TeHIy\nXHwxTJoE9eqd+dqIEc6YzUuWOP11jfE3VWeCh4IFnVvrsyijqCofJHzAGwvfYGqbqTSq1ijIgZpw\nlqfp2VQ1VUT6AHNxaugjVHW9iPRyXtZhQGsReRBIBo4D7fwXvhfjxjkXkjImcHBGjNu4Ee6+26mV\nFy0atLBclZTk/AEbORKKF4fKlZ1bvitVyny5eHG3Iw5fn30Gq1c782VmkcBPpp7koVkPkbAzgcXd\nF3Nh2QuDG6OJaOE9UXJqKlx2GQwbBlnV1tPSnGmwoqKybSlFhKNHnQu8b7/tfL1/7DGnhZiY6Ixl\nnZiY+XLRov9L6N4SfrFibp9l6Fi1Cm67DRYsyLJkt+/YPlpNbkXZYmUZ13IcpYuWDnKQJhJE7kTJ\nU6ZAhQrQJJuuWQUKOK31G25wuiC+8ELw4guWQ4fg44/h/fed/4vZs+Gaa3x7r6rz/sySfELCmesT\nE50k7i3Rn3pEcsI/dMjpD/7hh1km8LV71tJiYgvaXt6WgTcPpIDYvQvG/8K3JZ6WBrVrOzXv5s29\nb79rl1NyGTzY+eWLBPv3O4l7yBBo1szpH3/ZZYE7niocPOi9ZX9quUSJrBN9xoQfTqUuVefbXZUq\nzh/PTMzaNIuuM7ryzm3vcN819wU5QBNpIrMl/vXXUKiQk7w8Zm2axYZ9G3i8weNnt3rOO8/pvXLb\nbVC9uk9304WsPXucLpXDhzv1/iVL4F//CvxxReCcc5zHpZdmv62qMxlwZkn+t9/OXL9nD5Qs6Vs5\np2JF9xP+++/Dtm3wxRdnvaSqvLP4HQYvHsyM9jNocEEDFwI0+Ul4tsRVnVb1s886LSJg89+baTiy\nITXPqUnZYmUZ23IsFUtWPPu9sbHw4IOweDFUq5b3WIJp5074v/+DMWOcqb6eeSb8ziEzaWlZJ/yM\ny3v3QqlS3ss5lSs7Cb9IEf/GungxxMQ4paYaNc546UTKCR6c9SCrdq9iRvsZVIuKgM/GhITIa4l/\n/71zm33LlgAkpSTRdmpbBkQP4P7/3M/LcS9T59M6TGg1gRuqZxjHuUUL2LIF7rzTuSDl5caMkLB1\nK7z5ptONsmtXWLvW+SofKQoUcLqAlisHl3u52TctzSkjZZboN206O+GXKeN7wvc2+/y+fc744MOH\nn5XA9xzdQ6vJrahQogILui6gZJGSefxPMcY34dkSb9IEevaEjh0B6D2rN/uO7WNS60mIp/fJnC1z\n6DK9Cw/XfZjnr3/+zPKKKjzwAOzYATNmOGWZULR5M7z+uvPt4YEHnN4mFSq4HVX4SEuDv//2Xrvf\nvdtJ0GXKZJ/o338frrrK+TaUzprENcRMjKHjVR0ZcOMAu4Bp/C5PN/v4OZC8J/EFC6BzZ6f/d6FC\nTPp1En3n9WXF/SvOGn95x+EddJjWgRKFS5xdXklOdi6IXn6588sZStauhYED4bvv4OGHncc5vk+O\nu/foXr7e9DU//vkjhQsUJqpYFFFFo4gqFkWZomVOL0cV9Tz3LBctFEYXF/0tNdV7wq9Uyel/n67F\nPmPDDHrM7MEHTT+gw1UdXDwBE8kiK4k3a+ZczOvZ83Qd/NuO31KnSp1MN09JS+HluJcZ8/OYs8sr\nBw9CgwbOiHMPPZS3uPxh5UoneS9cCI8/Dr17Q2nf+hVv3LeRGRtnELsxll/2/MKtNW/l1pq3AnD4\nxGEOnTjEoaRDHDpxKPPnSYcoIAXOSOpnJH0f/ghEFYuiZOGSp78NRTJV5c2Fb/LR0o/4st2X1K0a\nxhfKTciLnCS+fLlTB9+yhaSCSoMRDehZpye9r+vt9a1ztsyh64yu9Lmuz5nlld9/h0aNnBuBmjbN\nfWx5sWQJvPaac/PI00/D/fc73fOykZqWyuIdi4ndGMuMjTM4cvIILS5pQYtaLbixxo0UK5SzPtqq\nSlJK0hlJPbNEf2rd4ZMZnnv+MJxIOUHpoqWzT/rZ/BEoU7QMZYqWCelJEZJSkrh/5v2s3buWGe1n\ncH6Z890OyUS4yEnid9/t1MMffZQHv36Q/Un7mdhqos8tvyzLKwsXOn8c5s2DK6/MfXw5oeoMzvXq\nq07t+7nnnIuW2dwgc/TkUeb+NpfYTbF8velrzit9HjG1YmhRqwX/qfKfkGgBJ6cmc/jE4Sxb+xmT\n/lnPkw7xz8l/KF6oeK7/CJxazukfMl/sPrKblpNackGZC/j8v59TonD2f2yN8YfISOK//gq33AK/\n/87E32N5Ke4lVty/gjJFc9a7JMvyyvjx8OKLTqu4UqXcxegLVWccl9dec2qtL7zgXKDNomfEX//8\nxcxNM4ndGMv8P+dTt2pdYmrFcFetuyJ2DI40TePoyaO5/iNw6l8ROSOp5+YPQqkipU5/a1u9ezUx\nE2PoVrsbLzV5yS5gmqCJjCR+771w1VVs7tmKhiMbMrfjXP5d5d+5jiXT8kq/fvDttxAX5/9BoVRh\n5kwneR89Cn37Qtu2Z/WMUVXna/qGGcRuimXT35toelFTYmrF0PSippQtVta/cUWwpJQk738EvFwn\nOJ5ynNJFShNVLIojJ48w9I6htLkiQu74NWEj/JP4li3QoAFJG9dSf/Jt9PpPLx687sE8x7Pz8E46\nTOtA8cLFnfJKiQrOTTSqMGGCf+bpTE11Zj1/7TVnfy++6JRu0u07OTWZBdsWnK5vp2kaLWq1IKZW\nDNdXv54iBf18w4rxWUpayunyUPFCxalUKoDf0ozJQvgn8R49oGpVHrxuT47r4N6kpKXQL64fo38e\nzfi7x9Okcj1nNqBbb4VXXsnDjlNg4kSnt0lUlDNxbvPmp0dRPHziMHO2zCF2Yyyzt8ym5jk1T1+Y\nvLrS1SFR3zbGhIbwTuLbt8M11zBx1pu8tPytXNXBfXHGzUGXdKNAg4bORUfPDUU+O3nSuS1+0CA4\n/3wned98M4iw7dA2Zm6cSeymWBZtX0Tjao2JqRXDnZfcaT0cjDFZCu8k/sgjbCp+jEblZ+S5Du7N\nGeWVy/pSsVlrpxTSuLH3N5+aiOHNN52RBPv2Ra+/ntW7V5/uv73t0DaaX9ycmFox3Pav22xsaWOM\nT/KcxEWkKfAe/5vZ56z5NUXkA6AZcBTooqqrM9kmZ0k8MZHjV15Kg35VeaB+Hx649gHf35tLZ5RX\nLniUJg+/43RBzGqUwPQTMVx7LSeff4b4isdOX5gsWrAoMbViiLk0hoYXNAzp/s/GmNCUXRJHVbN9\n4CTuLUB1oDCwGrg0wzbNgFme5XrAkiz2pT7bvl21TRvt9ewV2m5KO01LS/P9vX4we/Nsrfx2ZX3t\nrbs09dJaqgcOnLnBoUMa1727asWKur9tCx0XO1DbTmmrUYOitMHwBjrop0G6ds/aoMedV3FxcW6H\n4HeReE6qkXledk6Z8+TOTHO0L90v6gKbVfVPVU0GJgIxGbaJAcZ4snQCECUiOb+Mr+oM9dm+Pbvr\nX8nbF+7ih8rHGHbXsKBf6Gt6UVOW91zOt+UP0qzVcfZ0aOGMt7J/P/Trx++1q/PGmm+4qd+FVK8d\nx+SjCdxW8zY29tnIou6LeK7xc1xe4fKwu0AZHx/vdgh+F4nnBJF5XnZOOefLd/uqwPZ0z3fgJPbs\nttnpWZeY6R7T0pxeGqcS3MmT7Js4kvipbxNXci9xVxTnr9pww4Xl+OqmoQG5kOmLqmWqMq/zPPrN\ne4k6+h5vtr+CdSe2E3tNcfb0KkjV1ZXpf3Nfbql5i925Z4xxRdALtE0eKc1uOUoqSrG0AhSjEMcL\npLKjDDS6sTY3XXc/3WreTO3KtSlYoGCwwztLoQKFGHjLIK6veB2vT3+Shld0YVjdTtStWpdXX3mV\nFrVauB2iMSYf83phU0TqA/1Vtann+XM49Zk3020zFIhT1Ume5xuAJqqamGFfwesKY4wxEUTzMLPP\nMuAiEakO/AW0BzIOnBwLPARM8iT9gxkTeHZBGGOMyR2vSVxVU0WkDzCX/3UxXC8ivZyXdZiqfiMi\nzUVkC04Xw66BDdsYYwwE+WYfY4wx/mVjaWZDRLaKyM8iskpElnrWnSMic0Vko4h8KyJR6bZ/XkQ2\ni8h6EbnNvcjPJCIjRCRRRNakW5fj8xCROiKyRkQ2ich7wT6P9LI4p34iskNEVnoeTdO9Fg7ndL6I\nzBORtSLyi4g84lkftp9VJuf0sGd92H5WIlJURBI8eWGtiLzuWe/O55RVB3J7KMDvwDkZ1r0JPONZ\nfhZ4w7N8ObAKp0R1Ic4NUuL2OXhiawzUBtbk5TyABOA6z/I3wO0hdk79gCcy2fayMDmnykBtz3Ip\nYCNwaTh/VtmcU7h/ViU8/xYElgCN3PqcrCWePeHsbysxwGjP8mjgv57lFsBEVU1R1a3AZs7uT+8K\nVV0AHMiwOkfnISKVgdKqusyz3Zh07wm6LM4JnM8soxjC45x2q2e4ClU9AqwHzieMP6sszqmq5+Vw\n/qyOeRaL4uSIA7j0OVkSz54C34nIMhHp4VlXST09b1R1N+CZ4y3LG55CVcUcnkdVnBu9TtlBaJ5f\nHxFZLSLD032dDbtzEpELcb5pLCHnP3MheV7pzinBsypsPysRKSAiq4DdQLyqrsOlz8mSePYaqWod\noDnwkIhcj5PY04uUK8ORcB5DgJqqWhvnl+sdl+PJFREpBUwFHvW0XsP+Zy6Tcwrrz0pV01T13zjf\nlK4XkWhc+pwsiWdDVf/y/LsXmI5THkkUz7gwnq9Dezyb7wQuSPf28z3rQlVOzyPkz09V96qnuAh8\nxv/KWWFzTiJSCCfZjVXVGZ7VYf1ZZXZOkfBZAajqYZxa9rW49DlZEs+CiJTwtB4QkZLAbcAvODc2\ndfFs1hk49YsWC7QXkSIiUgO4CFga1KCzJ5xZg8zReXi+Hh4SkboiIkCndO9xyxnn5PnFOeVu4FfP\ncjid00hgnaq+n25duH9WZ51TOH9WIlL+VPlHRIoDt+JcuHTnc3Lr6m6oP4AaOMPursJJ3s951p8L\nfI9zlX0uUDbde57HufK8HrjN7XNIF9cEYBdwAtiGczPWOTk9D+A/nv+LzcD7IXhOY4A1ns9tOk6N\nMpzOqRGQmu7nbiXQNDc/c6FyXtmcU9h+VsBVnvNYBfwMPOVZ78rnZDf7GGNMGLNyijHGhDFL4sYY\nE8YsiRtjTBizJG6MMWHMkrgxxoQxS+LGGBPGLIkbY0wYsyRujDFh7P8BX3JnRZdkMoMAAAAASUVO\nRK5CYII=\n", 292 | "text/plain": [ 293 | "" 294 | ] 295 | }, 296 | "metadata": {}, 297 | "output_type": "display_data" 298 | } 299 | ], 300 | "source": [ 301 | "y1 = np.array([inv_err[x == n].mean() for n in ns])\n", 302 | "y2 = np.array([sol_err[x == n].mean() for n in ns])\n", 303 | "plt.axis((x.min(), x.max(), 0, max(y1.max(),y2.max())))\n", 304 | "plt.plot(ns, y1, color='r')\n", 305 | "plt.plot(ns, y2, color='g')\n", 306 | "plt.title(\"Error rate\")\n", 307 | "plt.show()" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 16, 313 | "metadata": { 314 | "collapsed": false, 315 | "scrolled": true 316 | }, 317 | "outputs": [ 318 | { 319 | "data": { 320 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEKCAYAAADkYmWmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGXax/HvTQihBALSBUVWRcSGuCYoKrFSXEERBARp\nRkVlseyqWMG6qKwFlVchgIJKlV6kCIGAkCBFkbIgotJBgVDSk/v94wwxYHomOTOT+3Ndc2XKmTP3\nYeDHyXOeIqqKMcYY/1TO7QKMMcYUnYW4Mcb4MQtxY4zxYxbixhjjxyzEjTHGj1mIG2OMH7MQN6YE\nicizIjLS7TpM4LIQN6VCRO4VkTUiclxE9ojIXBFp5XZd2YlIaxHZ5c33q+p/VPXB4ldnTM4sxE2J\nE5EngXeA14A6wLnAR8AdbtaVAwGKM/qtuO83ptAsxE2JEpFqwMvAI6o6U1WTVDVDVeep6iDPNhVE\n5D3PGfpuEXlXRII9r7UWkV0i8pSIHPRsc6eItBORbSLyu4gMyvZ5g0VkiohMFJFjIvKdiFye7fVM\nEflbtsdjReQVEakMzAPO9vy2cExE6oljkIj8JCKHPPutnsNx5vb+wSIy3rNNI8/n9xGR3zy19xeR\nv4vI9yJyWEQ+OGO//URks4j8ISLzReRcb34/xv9ZiJuSdg0QAszIY5sXgHDgcuAKz/0Xsr1eD6gA\n1AcGA6OAnkBz4AbgJRFplG37DsAkoAYwAZghIkGe13I8U1bVRKAdsFdVq6pqNVXdDwz07O964Gzg\nCDCiEO/P6TPDgQuA7sB7wPPATcClwD0icj2AiHQEBgF3ArWBWM/xGJPFQtyUtJrA76qamcc29wIv\nq+ofqvoHzpn7fdleTwXeUNUMYKJnn++qaqKqbgY244T/KWtVdbpn+3eAikBLz2tSyPofAp5X1X2q\nmga8AnQWkaL+21HgFVVNVdVFwAngC8+x78UJ6iuzffZ/VHWb589vKNBcRM4p4mebAGQhbkraH0Ct\nfELvbOC3bI9/9TyXtQ/9c6a2JM/Pg9leTwJCsz3Ourjoed/uM/ZXGI2A6Z6mjsM4/2GkAXWLuD/4\na+25HUsj4P1sn/0Hzn8CDYrx2SbAWIibkrYKSMFpEsjNHpzAOqURsLcYn5l1pioiAjT0fAZAIlA5\n27b1st3PqanlN6Cdqp7ludVQ1Sqqui+Hbb19UXMX8NAZnx2qqqu9/DnGj1mImxKlqsdw2rE/EpGO\nIlJJRMp7LkwO9Ww2EXhBRGqJSC3gRWB8MT72Ks/FzyDgCSAZiPO8th64V0TKiUhboHW29x0Aanou\nxp7yCfDGqQuKIlJbRDrk8rk5vf9MhWnO+Rh4TkSaeT47TEQ6F+L9pgwo73YBJvCp6jsisg/nYuXn\nwHFgLfC6Z5PXgKrADzhns5OzvZbjLvN5PBPoCowDtgN3edrHAR4HPgMexbnYOj1bnf8TkQnAz57m\nn2bA+56XF4pIfZymj0nArByOM6f3F7b2rMeqOkNEqgATPf+JJACLgKk57NeUUWKLQphAIiKDgfNV\ntZfbtRhTGqw5xRhj/JiFuDHG+DFrTjHGGD9mZ+LGGOPHSrV3iojYab8xxhSBqubYPbXUz8RVNWBu\ngwcPdr0GO6ayeUyBelx2TNluI0agF1yA7t+fZ6ZaP3FjjPE1U6bAa69BbCzUzXuGBwtxY4zxJd98\nA48+CosWwd/+lu/mdmGzGCIjI90uwevsmPxHIB5XmT+mtWuhe3fnTPyKK/LfnlLuYigiWpqfZ4wx\nfmP7dmjdGkaMgDtPny9ORFBfubBpjDHmDHv3Qps28Morfwnw/FiIG2OMm44ehbZtISrKuRWSNacY\nY4xbkpKcM/Arr4T33gPJeabiYjWniEiIiMSJyHoR2SQib+Sy3XAR2S4iG0SkeaEOxBhjypr0dOci\nZsOG8O67uQZ4fvLtYqiqKSJyo6omeibZXykirVR15altRKQdzvSfF4pIBM5k9i1z26cxxpRpqtC/\nv3MmPnkylCt6y3aB+omrs5I3OKuWl8NZ8Tu7jjgT8KOqcZ4VSOqq6oEiV2aMMYHq+edh40anT3iF\nCsXaVYHi37OU1XpgPxCjzgrj2TUg2+K0OOsZ2mKuxhhzpvffh2nTYO5cCA3Nf/t8FPRMPBO40rN2\n4EIRaa2qy4rygUOGDMm6HxkZGZCd+40xJkdffAHDhsGKFVCrVq6bxcTEEBMTU6BdFrp3ioi8CCSq\n6n+zPfcxsFRVJ3kebwVan9mcYr1TjDFl1tdfQ+/esGQJXHJJod5a3N4ptUQkzHO/EnArsOGMzWYB\nvTzbtASOWnu4McZ4xMXBfffB9OmFDvD8FKQ5pT7wmYgITuiPV9VvROQhQFV1pKrOE5H2IvITcBLo\n69UqjTHGX23ZAh07wqefwrXXen33NtjHGGNKyq5dcN118Oqr0KtXkXdjc6cYY0xpO3zYGU7/z38W\nK8DzY2fixhjjbYmJcMst0KoVvP12sXeX15m4hbgxxnhTWpozE2Ht2jBmTLFGY55izSnGGFMaMjPh\n/vudeVBGjfJKgOfHlmczxhhvefpp2LHDWVotOLhUPtJC3BhjvOHtt50BPcuXQ+XKpfaxFuLGGFNc\nn34KH33kDKc/66xS/WgLcWOMKY7Zs2HQIIiJceYGL2UW4sYYU1QrVkC/fs6MhE2bulKC9U4xxpii\n2LgR7r7bmZkwPNy1MizEjTGmsH79Fdq3d9bFvO02V0uxEDfGmMI4dMgJ7qeectbIdJmFuDHGFNTx\n484ZeJcuMHCg29UANuzeGGMKJjUVbr8dGjeGTz4p8ur0RWFzpxhjTHFkZkKPHpCcDFOmQPnS7diX\nV4hbF0NjjMmLKjz2GOzdCwsWlHqA58e3qjHGGF/z+usQGwvLlkHFim5X8xcW4sYYk5uRI2HsWGdQ\nT1iY29XkyNrEjTEmJ9OmwYABzoRWF1zgainWJm6MMYUREwP9+zuzEroc4PmxfuLGGJPdhg1wzz0w\naRK0aOF2NfmyEDfGmFN27HD6go8YATfe6HY1BWIhbowxAPv3Q5s28OKL0Lmz29UUmIW4McYkJEC7\ndtCrl9MW7kesd4oxpmxLTnYC/JJL4IMPSnU4fUHZsHtjjMlJRoZzETMoCCZMcH76oLxCPN/mFBFp\nKCJLRGSTiGwUkb9M3SUirUXkqIis89xe8EbhxhhTYlThkUecppTx4302wPNTkH7i6cCTqrpBREKB\ntSKyUFW3nrHdclXt4P0SjTGmBAweDGvXwtKlEBLidjVFlm+Iq+p+YL/n/gkR2QI0AM4Mcd9rSDLG\nmJx8+CFMnOgMp69a1e1qiqVQvVNE5DygORCXw8vXiMgGEZkrIs28UJsxxnjfpEkwdKgzI2GdOm5X\nU2wFHnbvaUqZCjymqifOeHktcK6qJopIO2AG0CSn/QwZMiTrfmRkJJGRkYUs2RhjimjRImdFnkWL\nnMUdfFRMTAwxMTEF2rZAvVNEpDwwB5ivqu8XYPudwFWqeviM5613ijHGHd995yyt9tVXcP31bldT\nKMXqneIxBticW4CLSN1s98Nx/nM4nNO2xhhT6rZtgzvugOhovwvw/OTbnCIirYAewEYRWQ8o8BzQ\nCFBVHQl0FpGHgTQgCehaciUbY0wh7N3rDKd//XXoEHgd6GywjzEmcB05Ajfc4KyPOWiQ29UUmY3Y\nNMaUPYmJcNttcPXV8M47PjmcvqAsxI0xZUt6Otx1l7Ok2rhxUM6/5/rzxoVNY4zxD6rw4INOkI8d\n6/cBnh9bns0YE1iefRY2b4ZvvoHgYLerKXEW4saYwPHOOzBrFsTGQpUqbldTKizEjTGBYfx4eO89\nWLkSatZ0u5pSYyFujPF/8+fDU0/BkiVwzjluV1OqLMSNMf5t1SpnWbVZs6BZ2Zt7L7Av2xpjAtvm\nzU5XwnHj4Jpr3K7GFRbixhj/tGsXtG0Lw4Y5a2SWURbixhj/88cfzmjMxx+Hnj3drsZVNmLTGONf\nTp6Em2+GyEhncYcywIbdG2MCQ1qaMxNh/fowerRfz4dSGBbixhj/l5np9EI5dgymTYPyZadzXV4h\nXnb+FIwx/ksV/vUv+PVXZ23MMhTg+bE/CWOM73vzTVi8GJYvh8qV3a7Gp1iIG2N825gx8MknznD6\nGjXcrqbUpWak5vm6dTE0xviuWbPg+eedJpSzz3a7mlJ36OQhbh1/a57bWIgbY3xTbCxERcHs2dCk\nidvVlLqNBzYSHh1Oq3Na5bmd9U4xxvieH36AW2+FL76AW25xu5pSN3PrTKJmR/F+2/e597J7rXeK\nMcaP7NwJ7dvD8OFlLsBVlaErhvLRmo+Ye+9cwhuE5/seC3FjjO84eBDatHFWpu/a1e1qSlVSWhJR\ns6PY9sc24qLiaFCtQYHeZ23ixhjfcPy4cwberRsMGOB2NaVqz7E9tP60NZmayfI+ywsc4GAhbozx\nBSkpzpSyV10FL7/sdjWlKn5PPBHREdzZ9E6+7PQllYIrFer9dmHTGOOujAzo3t35OXkyBAW5XVGp\n+XLjlzz29WNE3xFNx6Ydc93OLmwaY3yTKgwcCIcOOUuslZEAz9RMXljyAhN+nMCSXku4rO5lRd5X\nviEuIg2BcUBdIBMYparDc9huONAOOAn0UdUNRa7KGFM2vPoqfPstLFsGFSu6XU2pOJ5ynJ7Te3Ik\n6QjxUfHUrlK7WPsrSJt4OvCkql4CXAM8KiJNs28gIu2A81X1QuAh4ONiVWWMCXwff+wsqzZ/PlSr\n5nY1pWLnkZ1cO+Za6lSuw+Jei4sd4FCAEFfV/afOqlX1BLAFOPPSaUecs3VUNQ4IE5G6xa7OGBOY\npk51zsIXLIB69dyuplQs+2UZ1465lgdbPMjIO0ZSIaiCV/ZbqDZxETkPaA7EnfFSA2BXtsd7PM8d\nKEZtxphAtGQJPPIILFwI55/vdjWlYuTakby49EU+v+tzbj0/77lQCqvAIS4iocBU4DHPGXmRDBky\nJOt+ZGQkkZGRRd2VMcbfrFvn9AOfPBmaN3e7mhKXnpnOkwueZOGOhcT2jaVJzYLNARMTE0NMTEyB\nti1QF0MRKQ/MAear6vs5vP4xsFRVJ3kebwVaq+qBM7azLobGlFU//QQ33AAffgidOrldTYk7nHSY\nrlO7EiRBTOw8keoVqxd5X3l1MSzoYJ8xwOacAtxjFtDL82EtgaNnBrgxpgzbt88ZTv/yy2UiwLf+\nvpWI6Agur3M5c+6dU6wAz0++Z+Ii0gpYDmwE1HN7DmgEqKqO9Gz3IdAWp4thX1Vdl8O+7EzcmLIm\nIQFat4bOneGFF9yupsTN3z6f3jN68+Ytb9L3yr5e2actlGyMcUdysnMGfvnlzqyEAbw6vary7up3\nGfbtMKZ0mUKrc/OeB7wwLMSNMaUvPR26dHEG8XzxBZQL3KmaUtJT6D+3P+v3rWdmt5k0qt7Iq/u3\nYffGmNKlCg8/DCdPwqRJAR3gB04coNPkTtQLrceKfisIrRBaqp8fuH+yxhj3vPgibNgAX30FFbwz\nqMUXrd+3nvDocG5pfAtTukwp9QAHOxM3xnjb8OEwZQqsWAFVq7pdTYn5avNX9J/bnxHtR9Dlki6u\n1WEhbozxngkT4O23nQCvXfx5QXyRqvLq8leJXhfNgp4LaFG/hav1WIgbY7xjwQJ4/HH45hto5N0L\ne74iMS2RPjP6sOvYLuIfiKdeqPvzvlibuDGm+OLjoWdPmDYNLr3U7WpKxK6EXVw35joqlq/I0t5L\nfSLAwULcGFNcW7dChw4wdiy08l7faF+yatcqIqIj6H5pdz678zMqlveduc+tOcUYU3S7d0PbtjB0\nKPzjH25XUyI+2/AZTy16irEdx3J7k9vdLucvLMSNMUVz+LAT4I88An36uF2N12VkZjBo8SCmb51O\nTJ8YmtVu5nZJObIQN8YUXmIi3HGHE+JPPeV2NV53LOUY3b/qTlJaEnFRcdSsXNPtknJlbeLGmMJJ\nS4N77nEWdHjrrYCbD+Wnwz/RMrol54Wdx4KeC3w6wMFC3BhTGJmZEBXlDKsfPTrghtMv2bmEVmNa\n8c/wf/LR7R8RHBTsdkn5suYUY0zBPfMMbN8OixZBsO8HXGGMWDOCV5a9wsS7J3Jj4xvdLqfALMSN\nMQUzbBjMmwexsVClitvVeE1aRhoD5w9k+W/LWdlvJeef5V/rflqIG2PyN24cfPABrFwJZ53ldjVe\n83vi73SZ0oXQCqGsun8V1UKquV1SoQVWg5YxxvvmzoWnn3aG1Tds6HY1XrPp4CYioiMIPzucGV1n\n+GWAg52JG2Py8u23Th/wOXOgaVO3q/Ga2f+bTb9Z/Xi3zbv0vLyn2+UUi4W4MSZnmzbBXXfB559D\nRITb1XiFqvLWyrcYHj+cOd3nENHQ/4/LQtwY81e//uoM5HnnHWeNzACQnJ7MA7MfYPOhzcRFxdGw\nWmA0DVmbuDHmdL//7gT3v/8NPXq4XY1X7Du+j9aftiY1I5XYvrEBE+BgIW6Mye7ECWjfHjp1gsce\nc7sar/hu73eER4dzR5M7mHj3RCoHV3a7JK+y1e6NMY7UVGc+lHPOgVGjAmI4/aQfJzFg/gBG/mMk\nd118l9vlFFleq91biBtjnOH0PXs6E1tNnQrl/ftyWaZmMnjpYMb/MJ6Z3WZyRb0r3C6pWPIKcf/+\npowxxacKTzzhzA2+YIHfB/iJ1BP0mt6LQ4mHiH8gnjpV6rhdUomyNnFjyrr//AdiYmDWLKhUye1q\niuWXo7/QakwralSsweL7Fgd8gEMBQlxERovIARH5IZfXW4vIURFZ57m94P0yjTElIjrauX39NVSv\n7nY1xRL7ayzXjL6Gvs37Et0hmpDyIW6XVCoK8nvTWOADYFwe2yxX1Q7eKckYUyqmT4eXXoJly6B+\nfberKZbR60bz7DfPMv6u8bS5IDD6tRdUviGuqitEpFE+m/n/ZWxjypJly+Chh2D+fLjwQrerKbL0\nzHT+vfDfzNs+j9i+sVxU6yK3Syp13rqCcY2IbAD2AE+p6mYv7dcY423ffw9dusCECXDVVW5XU2RH\nk4/SdWpXVJW4qDhqVKrhdkmu8EaIrwXOVdVEEWkHzACa5LbxkCFDsu5HRkYSGRnphRKMMQXy88/O\nYJ6PPoKbb3a7miL73+//o8PEDrS7oB3DbhtG+XL+3aPmTDExMcTExBRo2wL1E/c0p8xW1csLsO1O\n4CpVPZzDa9ZP3Bi3HDgArVrBv/4FDz/sdjVFtnDHQu6bfh+v3/Q6US2i3C6nVHijn7iQS7u3iNRV\n1QOe++E4/zH8JcCNMS46dgzatXMG9PhpgKsqw+OGM3TlUKZ2mcr1ja53uySfkG+Ii8iXQCRQU0R+\nAwYDFQBV1ZFAZxF5GEgDkoCuJVeuMabQkpPhzjuhZUsYPNjtaookNSOVR+Y+QvyeeFbdv4rzqp/n\ndkk+w4bdGxPIMjKga1dnHpSJEyEoyO2KCu3gyYPcPflualaqyeedPie0QqjbJZW6vJpTbMSmMYFK\nFQYMgCNHnIUd/DDAv9//PeGjwmndqDXTuk4rkwGen8C6pGuM+dPLL0N8PCxdCiH+N3px+pbpPDjn\nQT5o9wHdLu3mdjk+y0LcmEA0YgR88YWzOn01/1oAWFV5PfZ1Pln7CfN7zOfvZ//d7ZJ8moW4MYFm\n8mR44w2IjYU6/jUBVGJaIvfPup+fj/xMXFQcZ1c92+2SfJ61iRsTSBYvdtrB586Fxo3drqZQ9hzb\nww1jbyBIgojpHWMBXkAW4sYEiu++g3vvdRZ1uMK/FkGI2x1HRHQEXZp1Yfxd46kU7N9T4pYma04x\nJhBs3+4srTZqFNxwg9vVFMrnP3zOEwueYEyHMdxx0R1ul+N3LMSN8Xd798Jtt8Grr0LHjm5XU2AZ\nmRk8v+R5Jm+azNLeS7m0zqVul+SXLMSN8WdHj0LbtvDggxDlP/OIHEs5Ro9pPTiecpz4B+KpVbmW\n2yX5LWsTN8ZfJSU5TSg33QSDBrldTYH9fORnrh19LQ2qNmDhfQstwIvJht0b44/S0+HuuyE0FMaP\nh3L+cT62dOdSun/VnRdveJFHrn4EEVtPpiBstXtjAomqsypPSgpMmeI3Af7xdx8zOGYwX3b6kpv/\n5r9zmfsaC3Fj/M1zz8GPP8I330CFCm5Xk6+0jDQe//pxlvyyhJX9VnLBWRe4XVJAsRA3xp+89x7M\nmOGMxgz1/cmgDicdpsuULoQEhbD6/tWEVQxzu6SA4x+/hxljnLlQ3nkHFiyAWr5/MXDzoc2Ejwqn\nRb0WzO4+2wK8hNiZuDH+YP58ePJJWLIEzj3X7WryNW/7PPrM6MPbt75N7+a93S4noFmIG+Pr4uKg\nVy+YNQsuucTtavKkqvx31X95Z9U7zOw2k2vOucbtkgKehbgxvmzLFmcU5qefwjW+HYjJ6ck8NOch\nNh7YSFxUHOeEneN2SWWCtYkb46t27XJGY771Ftx+u9vV5Gn/if3c+NmNJKYlEts31gK8FFmIG+OL\n/vgD2rSBgQOdphQftm7fOsJHhdP2/LZM6jyJKhWquF1SmWIjNo3xNSdPwi23wPXXO2fhPmzKpik8\nMu8R/u/2/6Nzs85ulxOw8hqxaSFujC9JS3PawOvUgbFjnVXqfVCmZvJyzMt8+v2nzOw2k+b1mrtd\nUkCzYffG+IPMTOjXz1mVftQonw3wk6kn6T2jN3uP7yU+Kp66oXXdLqlMszZxY3yBKjz1FOzcCZMm\nQXCw2xXl6LeE37hu7HVUDanK0t5LLcB9gIW4Mb7g7bedkZizZ0Plym5Xk6OVv62kZXRL7rv8PsZ0\nGENI+RC3SzJYc4ox7hs7FkaMgJUroUYNt6vJ0dj1Y3lm8TN8dudntLuwndvlmGzyDXERGQ38Azig\nqpfnss1woB1wEuijqhu8WqUxgWr2bGdWwpgYaNDA7Wr+IiMzg6cXPc2sbbNY1mcZF9e+2O2SzBkK\nciY+FvgAGJfTiyLSDjhfVS8UkQjgY6Cl90o0JkCtWAH33w9z58JFF7ldzV8kJCfQ7atupGWkERcV\nx1mVznK7JJODfNvEVXUFcCSPTTriCXhVjQPCRMSudhiTl40bnZV5vvgCrr7a7Wr+Yvsf22k5uiUX\n1LiA+T3mW4D7MG9c2GwA7Mr2eI/nOWNMTn75Bdq3h/ffh1tvdbuav1j882KuG3sdT7R8gg/af0Bw\nkG/2lDGOUr+wOWTIkKz7kZGRREZGlnYJxrjn0CFnOP3TT0O3bm5XcxpV5cP4D3k99nUmd55M6/Na\nu11SmRUTE0NMTEyBti3QiE0RaQTMzunCpoh8DCxV1Umex1uB1qp6IIdtbcSmKbuOH3dWpm/TBl57\nze1qTpOakcqAeQNYtXsVs7rNonGNxm6XZLLJa8RmQZtTxHPLySygl+eDWgJHcwpwY8q0lBTo1Amu\nvBJefdXtak7ze+Lv3Dr+Vvaf2M+3/b61APcz+Z6Ji8iXQCRQEzgADAYqAKqqIz3bfAi0xeli2FdV\n1+WyLzsTN2VPZibcey+kpjqr0wcFuV1Rlo0HNtJxYke6XdqN1256jXJi4/98kU2AZYxbVJ3pZDdu\nhK+/hooV3a4oy8ytM4maHcV7bd6jx+U93C7H5MEmwDLGLa+95qxMv2yZzwS4qjJ0xVA+WvMRc++d\nS3iDcLdLMsVgIW5MSfnkE2dZtZUrIcw3VnpPSksianYU2/7YRlxUHA2qWW9gf2cNYMaUhGnT4OWX\nYeFCqFfP7WoA2Ht8L60/bU2mZrK8z3IL8ABhIW6Mty1dCv37O8Ppzz/f7WoAWLNnDRHREdzZ9E6+\n7PQllYIruV2S8RJrTjHGm9avh65dnTnBr7zS7WoA+HLjlzz29WNE3xFNx6Yd3S7HeJmFuDHesmOH\nsyr9//0f3Hij29WQqZm8sOQFJvw4gSW9lnBZ3cvcLsmUAAtxY7xh/3647TYYPNiZ2Mplx1OOc9/0\n+zicdJj4qHhqV6ntdkmmhFibuDHFlZAAbdtCnz7w0ENuV8POIzu5dsy11K5cm8W9FluABzgLcWOK\nIznZWZ3+uuvghRfcroZlvyzj2jHX8mCLBxl5x0gqBFVwuyRTwmzEpjFFlZEBXbo4ixp/+aXrw+lH\nrh3Ji0tf5PO7PufW831viltTdDZi0xhvU4WHH3ZmJpwzx9UAT89M58kFT7Jwx0Ji+8bSpGYT12ox\npc9C3JiieOklpzvhkiUQ4t6q70eSjnDP1HsIkiBWR62mesXqrtVi3GFt4sYU1gcfOP3A582DqlVd\nK2Pr71uJiI7gsjqXMefeORbgZZSdiRtTGBMnwltvOZNa1Xav18fXP31Nr+m9GHrLUPpd2c+1Ooz7\nLMSNKahFi+Cxx2DxYjjvPFdKUFXeXf0uw74dxvSu02l1bitX6jC+w0LcmPykpsLo0c5AnmnT4DJ3\nRj6mpKfQf25/1u9bz6r7V9GoeiNX6jC+xdrEjclNRgaMGwdNm8LMmbBggdMf3AUHThzgpnE3cSzl\nGCv6rbAAN1ksxI05k6pzxn355TBqlDMn+Ndfuzah1Yb9GwiPDueWxrcwpcsUQiuEulKH8U3WnGLM\nKarO/N/PP++sizlsmDOcXnJbI7zkfbX5K/rP7c9H7T/inkvuca0O47ssxI0BWLHCCe+DB53V6Dt1\ngnLu/aKqqry6/FWi10WzoOcCWtRv4VotxrdZiJuybf16Z86TTZtgyBDo2RPKu/vPIjEtkT4z+rDr\n2C7iH4inXqhvrAxkfJO1iZuyaetWuOceZ/7v9u3hf/9zZiF0OcB3JeziujHXUbF8RZb2XmoBbvJl\nIW7Kll9+gb594YYb4KqrYPt2ePRRV4fOn7J692pajm5J90u789mdn1GxfEW3SzJ+wJpTTNmwfz+8\n/roz2+Cjj8K2bVDdd4apj/t+HP9e+G/GdhzL7U1ud7sc40csxE1gO3zYGSY/ahT07g1btkCdOm5X\nlSUjM4NBiwcxfet0YvrE0Kx2M7dLMn7GQtwEpuPH4f334b33nOXSvv8eGjZ0u6rTHEs5RvevupOU\nlkRcVBw1K9d0uyTjhwrUJi4ibUVkq4hsE5Fncni9tYgcFZF1npv7S5yYsik5Gd59Fy680DnrXr0a\nPvnE5wKUHPx/AAAN50lEQVT8p8M/0TK6JeeFnceCngsswE2R5XsmLiLlgA+Bm4G9wBoRmamqW8/Y\ndLmqdiiBGo3JX1qaM7LylVecC5aLFrk2x0l+luxcwr1f3cvg1oN5+OqH3S7H+LmCNKeEA9tV9VcA\nEZkIdATODHH3hrWZsisz05ke9qWXoHFjmDoVIiLcripXI9aM4JVlrzDh7gnc2PhGt8sxAaAgId4A\n2JXt8W6cYD/TNSKyAdgDPKWqm71QnzE5U4VZs+DFF6FKFRg5Em66ye2qcpWWkcbA+QNZ/ttyVvZb\nyflnne92SSZAeOvC5lrgXFVNFJF2wAwgx4X+hgwZknU/MjKSyMhIL5VgyoxvvoHnnnPav994wxmw\n4+L8Jvn5I/EPOk/pTGiFUFbdv4pqIdXcLsn4uJiYGGJiYgq0bb6r3YtIS2CIqrb1PB4EqKq+mcd7\ndgJXqerhM5631e5N0a1a5cxvsnu30/Z9zz2uzm9SEJsObqLDxA50vrgzb9z8BkHl3FtQ2fivvFa7\nL8i/gDXABSLSSEQqAN2AWWd8QN1s98Nx/nM4jDHe8MMP0KEDdO0KPXrA5s3QrZvPB/icbXO48bMb\nGdJ6CG/e+qYFuCkR+TanqGqGiAwAFuKE/mhV3SIiDzkv60igs4g8DKQBSUDXkizalBHbtjmr6Sxd\nCs8+C5MnQ0XfH4quqry18i2Gxw9ndvfZRDT03Qutxv/l25zi1Q+z5hRTELt2Oc0lM2bAE0/AwIEQ\n6h8LISSnJ/PA7AfYfGgzM7vNpGE13+qfbvxTcZtTjCkdBw/C449D8+bOSvLbtjkXMP0kwPcd30fk\np5GkZqQS2zfWAtyUCgtx476jR505vS++2Ok6uGmT0+ukRg23KyuwtXvXEh4dzj+a/IOJd0+kcnBl\nt0syZYTNnWLcc/IkDB8O77zjXLhctw4a+d8CwJN+nMSA+QP45B+f0OniTm6XY8oYC3FT+lJSnPlM\n/vMfaN3aWRrtoovcrqrQMjWTwUsHM/6H8Sy+bzFX1LvC7ZJMGWQhbkpPejqMGwcvv+ysJD9/vtP+\n7YdOpJ6g1/ReHEo8RPwD8dSp4jvT25qyxdrETcnLzIRJk+CSS5wQnzABZs/22wD/9eivtBrTihoV\na7D4vsUW4MZVdiZuSo4qzJvnjLKsUAE+/BBuucWnh8jnZ8VvK+gypQvPtHqGxyIeQ/z4WExgsBA3\nJSMmxukeeOwYvPYadOzo1+GtqoxZP4bnljzHuDvH0eaCNm6XZAxgIW68bc0a58x7xw6n7bt7dwjy\nv+HmCckJxO+JJ25PnHPbHUfd0Los77Oci2r530VYE7hsxKbxjh9/dKaFXbPG+dmvHwQHu11VgaRn\npvPjwR+J2x3H6j2ridsdx65ju2hRvwURDSKcW8MIG7xjXJPXiE0LcVM8O3bAkCGwcCE8/TQ88ghU\nquR2VXnafWw3cbudM+zVu1ezfv96zql2Di0btswK7EvrXEr5cvaLqvENFuLG+/bsgVdfdVbSGTjQ\nGS5fzffmyT6ZepK1+9ayevfqrGaR1IxUIho6Z9gtG7bk6rOvJqximNulGpMrC3HjPYcOwdChMHYs\nREXBM89ATd9Y5DdTM9n6+1anWcQT2tsPb+fyupef1izSuHpj61Vi/EpeIW6/L5qCSUhwhsd/+KEz\nl/ePP8LZZ7ta0sGTB7OaReL2xLFmzxpqVa5FRMMIWjZoyf0t7ueKulcQUj7E1TqNKUkW4iZviYlO\ncA8bBu3bw3ffOQsSl7KU9BTW719/2sXHI8lHCG8QTkSDCB6PeJzwBuHUrlK71Gszxk3WnGJylpoK\n0dHw+utwzTXO/N7NmpXKR6sqO47sOO3i46ZDm7io5kWnXXxsUrMJ5cQGHZvAZ80ppuAyMuDzz50e\nJ02bOivKX3VViX7k0eSjxO+JP+3iY+XgylkXH++55B5a1G9h07sakwM7EzcOVZg2zenjXbOmcwZ+\nww1e/5i0jDQ2Htx42ln2nuN7uKr+VaedZZ9d1d32dmN8ifVOMblThQULnEUZMjOdxRjatPHKEHlV\nZfex3X+eYe+JY/2+9ZxX/byssG7ZsCXNajezPtnG5MFC3OQsNtYZIn/okNPnu1OnYq0gfyL1BN/t\n/e60i48ZmvHnGXaDCK5ucDXVQnyvP7kxvsxC3Jxu7VrnzHvrVqftu0cPKF+4M+GMzAy2/L7ltGaR\nHUd2cEXdK7IG0UQ0jKBRWCPrk21MMVmIG8eWLfDSS7BypXMGHhUFIQXrQ33gxIGssI7bE8d3e7+j\nbpW6WRcfIxpEcEW9K6gQVKGED8KYssdCvKzbudOZUXDuXHjqKRgwACrn3tMjOT2ZdfvWnXaWfSzl\nGOENwrOaRsIbhFOzsm+M1DQm0FmIl1X79jm9TCZMgEcfhX/9C8JOnyNEVfnp8E+nXXzcfGgzF9e6\n+LSLjxeedaE1ixjjEusnHmgSE52A3r8/75/Hj8NDDzlt37WdkYyHkw4782R7Lj7G74kntEJo1hl2\n90u706J+CyoF+/ZMhMYYR4HOxEWkLfAezpqco1X1zRy2GQ60A04CfVR1Qw7b2Jl4bjIz4Y8/8g/m\nffuc0ZT16kH9+nn+TK1Vgx8On37xcf+J/fz97L+fdvGxXmg9t4/eGJOHYjWniEg5YBtwM7AXWAN0\nU9Wt2bZpBwxQ1dtFJAJ4X1Vb5rCvgArxmJgYIiMj894oOdkJ4PzC+eBBqFr1tBBOqV+bhDphJNSu\nRkKNSiSEhZBQpTxHg9JISDlGQkoCCckJHE05SkJyQtbjhJQEjiYf5XjKcZrUbJLVLBLRIIJmtZsR\nVC73lXYKdEx+JhCPCQLzuOyYclbc5pRwYLuq/urZ2USgI7A12zYdgXEAqhonImEiUldVDxSrcl+l\nCkeOEDN5MpEZGTmGcvqBfSQc3ktCRiIJDWpytF51J4zPqkJC9YokNAnmaPNyJITUJ6F8HRJI4Wjq\nqWD+kYSUlaRnphOmYVQ/Vp2w1DDCjoYRVjGMsJAwqlesTlhIGOeEncOlIZcSVvHP57JvU9gZ/Owf\nkf8IxOOyYyq8goR4A2BXtse7cYI9r232eJ4rnRBXhfR055aa6pz9ZrtpUhLpSSdJSzpBWvJJ0pJO\nkpacSFpKYtbP1NRE0lKSSUtNIi01mbS0U7cU0tJTSEtPJSU1iYSkI85ZcGgQ3/wqbJWpHK0SREKI\nkBCWQUKNNI42TSI5M5WwkGqEVWqYFaxZIet5XC8kjKYVq2eF7mkBXTGMSuUr2cVEY0yeSv3C5u1P\n1CETBVUUUBRVJYNMMvjrz8xsN1UlU7I/h/NTcG7lIKOckBYEaeUgNQjSyinp5SAoE4K1HMGU+/Mn\nQQSLc6sg5QkOKU9wpWCCy5UnOCjYc6tAcFAIweWrEhJciWpVaxFWox41Q2tTf1wsHfv3yPEsOLRC\nqAWwMabEFaRNvCUwRFXbeh4PAjT7xU0R+RhYqqqTPI+3Aq3PbE4RkcBpEDfGmFJUnDbxNcAFItII\n2Ad0A7qfsc0s4FFgkif0j+bUHp5bEcYYY4om3xBX1QwRGQAs5M8uhltE5CHnZR2pqvNEpL2I/ITT\nxbBvyZZtjDEGSnnEpjHGGO+yta3yICK/iMj3IrJeROI9z9UQkYUi8j8RWSAiYdm2f1ZEtovIFhG5\nzb3KTycio0XkgIj8kO25Qh+HiLQQkR9EZJuIvFfax5FdLsc0WER2i8g6z61tttf84ZgaisgSEdkk\nIhtFZKDneb/9rnI4pn96nvfb70pEQkQkzpMLm0TkDc/z7nxPqmq3XG7Az0CNM557E3jac/8ZYKjn\nfjNgPU4T1XnAT3h+03H7BlwHNAd+KM5xAHHA1Z7784A2PnZMg4Enc9j2Yj85pnpAc8/9UOB/QFN/\n/q7yOCZ//64qe34GAauBVm59T3Ymnjfhr7+tdAQ+89z/DLjTc78DMFFV01X1F2A7f+1P7wpVXQEc\nOePpQh2HiNQDqqrqGs9247K9p9TlckzgfGdn6oh/HNN+9UxXoaongC1AQ/z4u8rlmBp4Xvbn7yrR\nczcEJyOO4NL3ZCGeNwUWicgaEYnyPJc1ElVV9wN1PM/nNuDJV9Up5HE0wBnodcpufPP4BojIBhGJ\nzvbrrN8dk4ich/ObxmoK/3fOJ48r2zHFeZ7y2+9KRMqJyHpgPxCjqptx6XuyEM9bK1VtAbQHHhWR\n63GCPbtAuTIcCMcxAvibqjbH+cf1X5frKRIRCQWmAo95zl79/u9cDsfk19+Vqmaq6pU4vyldLyKR\nuPQ9WYjnQVX3eX4eAmbgNI8cEJG6AJ5fhw56Nt8DnJPt7Q09z/mqwh6Hzx+fqh5ST+MiMIo/m7P8\n5phEpDxO2I1X1Zmep/36u8rpmALhuwJQ1WM4bdl/x6XvyUI8FyJS2XP2gIhUAW4DNuIMbOrj2aw3\ncOof2iygm4hUEJHGwAVAfKkWnTfh9DbIQh2H59fDBBEJFxEBemV7j1tOOybPP5xTOgE/eu770zGN\nATar6vvZnvP37+ovx+TP35WI1DrV/CMilYBbcS5cuvM9uXV119dvQGNgg+fL2QgM8jx/FrAY5yr7\nQqB6tvc8i3PleQtwm9vHkK2uL3GmEU4BfsMZjFWjsMcBXOX5s9iOM92wrx3TOOAHz/c2A6eN0p+O\nqRWQke3v3TqgbVH+zvnKceVxTH77XQGXeY5jPfA98G/P8658TzbYxxhj/Jg1pxhjjB+zEDfGGD9m\nIW6MMX7MQtwYY/yYhbgxxvgxC3FjjPFjFuLGGOPHLMSNMcaP/T8KQEbxzClv6QAAAABJRU5ErkJg\ngg==\n", 321 | "text/plain": [ 322 | "" 323 | ] 324 | }, 325 | "metadata": {}, 326 | "output_type": "display_data" 327 | } 328 | ], 329 | "source": [ 330 | "y1 = np.array([inv_t[x == n].mean() for n in ns])\n", 331 | "y2 = np.array([sol_t[x == n].mean() for n in ns])\n", 332 | "plt.axis((x.min(), x.max(), 0, max(y1.max(),y2.max())))\n", 333 | "plt.plot(ns, y1, color='r')\n", 334 | "plt.plot(ns, y2, color='g')\n", 335 | "plt.title(\"Compute time\")\n", 336 | "plt.show()" 337 | ] 338 | } 339 | ], 340 | "metadata": { 341 | "kernelspec": { 342 | "display_name": "Python 2", 343 | "language": "python", 344 | "name": "python2" 345 | }, 346 | "language_info": { 347 | "codemirror_mode": { 348 | "name": "ipython", 349 | "version": 2 350 | }, 351 | "file_extension": ".py", 352 | "mimetype": "text/x-python", 353 | "name": "python", 354 | "nbconvert_exporter": "python", 355 | "pygments_lexer": "ipython2", 356 | "version": "2.7.10" 357 | } 358 | }, 359 | "nbformat": 4, 360 | "nbformat_minor": 0 361 | } 362 | -------------------------------------------------------------------------------- /exp/tgp_np_gradient_check.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sanity Check for Gradients" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "collapsed": false 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import sys\n", 19 | "sys.path.append('/Users/void/Documents/github/tensorflow-gp')\n", 20 | "import numpy as np\n", 21 | "from tgp.np.gaussian_process import GaussianProcess as GP\n", 22 | "from tgp.gradient_check import check_grad\n", 23 | "import matplotlib.pyplot as plt\n", 24 | "import time\n", 25 | "%load_ext autoreload\n", 26 | "%autoreload 2\n", 27 | "%matplotlib inline" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 2, 33 | "metadata": { 34 | "collapsed": true 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "np.random.seed(42)\n", 39 | "# make some random data\n", 40 | "n = 2\n", 41 | "X = np.random.uniform(0, 1, (n, 1))\n", 42 | "y = np.sin(X)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 3, 48 | "metadata": { 49 | "collapsed": false 50 | }, 51 | "outputs": [ 52 | { 53 | "data": { 54 | "text/plain": [ 55 | "" 56 | ] 57 | }, 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "output_type": "execute_result" 61 | } 62 | ], 63 | "source": [ 64 | "gp = GP(ls=[.5],\n", 65 | " amp=1.,\n", 66 | " noise=0.,\n", 67 | " keps=1e-9)\n", 68 | "gp.fit(X, y, train=False)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "### Gradients w.r.t. K" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 4, 81 | "metadata": { 82 | "collapsed": false 83 | }, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "[[ 1. 0.51481205]\n", 90 | " [ 0.51481205 1. ]]\n", 91 | "ls grad:\n", 92 | "[[ 0. 1.36724483]\n", 93 | " [ 1.36724483 0. ]]\n", 94 | "amp grad:\n", 95 | "[[ 2. 1.02962411]\n", 96 | " [ 1.02962411 2. ]]\n", 97 | "noise grad:\n", 98 | "[[ 0. 0.]\n", 99 | " [ 0. 0.]]\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "K, grads = gp.K(X, grad=True)\n", 105 | "print K\n", 106 | "print \"ls grad:\"\n", 107 | "for ls_grad in grads['ls']:\n", 108 | " print ls_grad\n", 109 | "print \"amp grad:\"\n", 110 | "print grads['amp']\n", 111 | "print \"noise grad:\"\n", 112 | "print grads['noise']" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 5, 118 | "metadata": { 119 | "collapsed": false 120 | }, 121 | "outputs": [ 122 | { 123 | "name": "stdout", 124 | "output_type": "stream", 125 | "text": [ 126 | "Numerical v. analytical gradient difference:\n", 127 | "[[ 0.00000000e+00 -2.49078536e-11]\n", 128 | " [ -2.49078536e-11 0.00000000e+00]]\n", 129 | "Max deviation: 2.49078535575e-11\n", 130 | "(array([[[ 0. , 1.36724483],\n", 131 | " [ 1.36724483, 0. ]]]), array([[[ 0. , 1.36724483],\n", 132 | " [ 1.36724483, 0. ]]]))\n" 133 | ] 134 | } 135 | ], 136 | "source": [ 137 | "name = 'ls'\n", 138 | "f = lambda x: (gp.K(X), gp.K(X, grad=True)[1][name])\n", 139 | "print check_grad(f, gp.params[name], h=1e-5, verbose=True)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 6, 145 | "metadata": { 146 | "collapsed": false 147 | }, 148 | "outputs": [ 149 | { 150 | "name": "stdout", 151 | "output_type": "stream", 152 | "text": [ 153 | "Numerical v. analytical gradient difference:\n", 154 | "[[ -2.00017780e-12 9.48352508e-12]\n", 155 | " [ 9.48352508e-12 -2.00017780e-12]]\n", 156 | "Max deviation: 9.48352507635e-12\n", 157 | "(array([[ 2. , 1.02962411],\n", 158 | " [ 1.02962411, 2. ]]), array([[ 2. , 1.02962411],\n", 159 | " [ 1.02962411, 2. ]]))\n" 160 | ] 161 | } 162 | ], 163 | "source": [ 164 | "name = 'amp'\n", 165 | "f = lambda x: (gp.K(X), gp.K(X, grad=True)[1][name])\n", 166 | "print check_grad(f, gp.params[name], h=1e-5, verbose=True)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 7, 172 | "metadata": { 173 | "collapsed": false 174 | }, 175 | "outputs": [ 176 | { 177 | "name": "stdout", 178 | "output_type": "stream", 179 | "text": [ 180 | "Numerical v. analytical gradient difference:\n", 181 | "[[ 0. 0.]\n", 182 | " [ 0. 0.]]\n", 183 | "Max deviation: 0.0\n", 184 | "(array([[ 0., 0.],\n", 185 | " [ 0., 0.]]), array([[ 0., 0.],\n", 186 | " [ 0., 0.]]))\n" 187 | ] 188 | } 189 | ], 190 | "source": [ 191 | "name = 'noise'\n", 192 | "f = lambda x: (gp.K(X), gp.K(X, grad=True)[1][name])\n", 193 | "print check_grad(f, gp.params[name], h=1e-5, verbose=True)" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "### Gradients w.r.t. L" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 8, 206 | "metadata": { 207 | "collapsed": false 208 | }, 209 | "outputs": [ 210 | { 211 | "name": "stdout", 212 | "output_type": "stream", 213 | "text": [ 214 | "(0.0,) \t0.000395059585571\n", 215 | "(0.0,) \t0.00114607810974\n", 216 | "(0.0,) \t0.000312089920044\n" 217 | ] 218 | } 219 | ], 220 | "source": [ 221 | "# Before checking gradient, here's a little demo of the different ways of computing the log determinant\n", 222 | "big_matrix = np.eye(100) # np.random.randn(30, 30)\n", 223 | "big_matrix = big_matrix.T.dot(big_matrix)\n", 224 | "t1 = time.time()\n", 225 | "ans1 = np.log(np.diag(np.linalg.cholesky(big_matrix)**2)).sum()\n", 226 | "t1 = time.time() - t1\n", 227 | "t2 = time.time()\n", 228 | "ans2 = np.linalg.slogdet(big_matrix)[1]\n", 229 | "t2 = time.time() - t2\n", 230 | "t3 = time.time()\n", 231 | "ans3 = 2*np.log(np.diag(np.linalg.cholesky(big_matrix))).sum()\n", 232 | "t3 = time.time() - t3\n", 233 | "print (ans1,), \"\\t\", t1\n", 234 | "print (ans2,), \"\\t\", t2\n", 235 | "print (ans3,), \"\\t\", t3" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "### Check if L changes significantly as a function of sample size\n", 243 | "\n", 244 | "It does change. I have modified the loss function so that it is divided by the number of samples" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 9, 250 | "metadata": { 251 | "collapsed": false 252 | }, 253 | "outputs": [ 254 | { 255 | "name": "stdout", 256 | "output_type": "stream", 257 | "text": [ 258 | "9.20589046451\n" 259 | ] 260 | } 261 | ], 262 | "source": [ 263 | "# make some random data\n", 264 | "np.random.seed(42)\n", 265 | "n = 1000\n", 266 | "X = np.random.uniform(0, 1, (n, 1))\n", 267 | "y = np.sin(X)\n", 268 | "\n", 269 | "gp = GP(ls=[.1],\n", 270 | " amp=1.,\n", 271 | " noise=0.0,\n", 272 | " keps=1e-9)\n", 273 | "print gp.L(X, y, grad=True)[0]" 274 | ] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "execution_count": 10, 279 | "metadata": { 280 | "collapsed": false 281 | }, 282 | "outputs": [ 283 | { 284 | "name": "stdout", 285 | "output_type": "stream", 286 | "text": [ 287 | "Numerical v. analytical gradient difference:\n", 288 | "0.00415114502771\n", 289 | "Max deviation: 0.00415114502771\n", 290 | "Ana gradient: 1.87503868876\n", 291 | "Num gradient: 1.87088754373\n" 292 | ] 293 | } 294 | ], 295 | "source": [ 296 | "name = 'ls'\n", 297 | "f = lambda x: (gp.L(X, y), gp.L(X, y, grad=True)[1][name])\n", 298 | "ana_grad, num_grad = check_grad(f, gp.params[name], h=1e-5, verbose=1)\n", 299 | "print \"Ana gradient:\", ana_grad[0]\n", 300 | "print \"Num gradient:\", num_grad[0]" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 11, 306 | "metadata": { 307 | "collapsed": false 308 | }, 309 | "outputs": [ 310 | { 311 | "name": "stdout", 312 | "output_type": "stream", 313 | "text": [ 314 | "Numerical v. analytical gradient difference:\n", 315 | "0.0119308041244\n", 316 | "Max deviation: 0.0119308041244\n", 317 | "Ana gradient: -0.0264338891645\n", 318 | "Num gradient: -0.0383646932889\n" 319 | ] 320 | } 321 | ], 322 | "source": [ 323 | "name = 'amp'\n", 324 | "f = lambda x: (gp.L(X, y), gp.L(X, y, grad=True)[1][name])\n", 325 | "ana_grad, num_grad = check_grad(f, gp.params[name], h=1e-5, verbose=1)\n", 326 | "print \"Ana gradient:\", ana_grad\n", 327 | "print \"Num gradient:\", num_grad" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 12, 333 | "metadata": { 334 | "collapsed": false, 335 | "scrolled": true 336 | }, 337 | "outputs": [ 338 | { 339 | "name": "stdout", 340 | "output_type": "stream", 341 | "text": [ 342 | "Numerical v. analytical gradient difference:\n", 343 | "0.0\n", 344 | "Max deviation: 0.0\n", 345 | "Ana gradient: 0.0\n", 346 | "Num gradient: 0.0\n" 347 | ] 348 | } 349 | ], 350 | "source": [ 351 | "name = 'noise'\n", 352 | "f = lambda x: (gp.L(X, y), gp.L(X, y, grad=True)[1][name])\n", 353 | "ana_grad, num_grad = check_grad(f, gp.params[name], h=1e-5, verbose=1)\n", 354 | "print \"Ana gradient:\", ana_grad\n", 355 | "print \"Num gradient:\", num_grad" 356 | ] 357 | }, 358 | { 359 | "cell_type": "code", 360 | "execution_count": 18, 361 | "metadata": { 362 | "collapsed": false, 363 | "scrolled": true 364 | }, 365 | "outputs": [ 366 | { 367 | "name": "stdout", 368 | "output_type": "stream", 369 | "text": [ 370 | "Batch size larger than sample size. Reducing it to. 67\n", 371 | "Epoch: 1 Negative log likelihood: 10.4488791064\n", 372 | "Epoch: 2 Negative log likelihood: 8.77794575183\n", 373 | "Epoch: 3 Negative log likelihood: 7.02011137203\n", 374 | "Epoch: 4 Negative log likelihood: 4.67777952181\n", 375 | "Epoch: 5 Negative log likelihood: 1.94095366678\n", 376 | "Epoch: 6 Negative log likelihood: 0.133018449251\n", 377 | "Epoch: 7 Negative log likelihood: -0.589810677274\n", 378 | "Epoch: 8 Negative log likelihood: -0.886640464514\n", 379 | "Epoch: 9 Negative log likelihood: -1.03350597035\n", 380 | "Epoch: 10 Negative log likelihood: -1.11859571878\n", 381 | "********************************************************************************\n", 382 | "Final negative log likelihood -1.11859571878\n", 383 | "Final parameters:\n", 384 | "amp : 0.269161863125\n", 385 | "noise : 0.05\n", 386 | "ls : [ 0.25104884]\n", 387 | "keps : 1e-09\n", 388 | "********************************************************************************\n" 389 | ] 390 | }, 391 | { 392 | "data": { 393 | "text/plain": [ 394 | "" 395 | ] 396 | }, 397 | "execution_count": 18, 398 | "metadata": {}, 399 | "output_type": "execute_result" 400 | } 401 | ], 402 | "source": [ 403 | "# make some random data\n", 404 | "np.random.seed(45)\n", 405 | "n = 67\n", 406 | "X = np.random.uniform(0., 1, (n, 1))\n", 407 | "y = np.squeeze(np.sin(X*10)) + np.random.randn(len(X))*.1\n", 408 | "\n", 409 | "gp = GP(ls=[.5],\n", 410 | " amp=.1,\n", 411 | " noise=5e-2)\n", 412 | "gp.fit(X, y, nepochs=10, batch_size=1000, learning_rate=1e-3, learning_rate_decay=.999, verbose=1)" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 19, 418 | "metadata": { 419 | "collapsed": false 420 | }, 421 | "outputs": [ 422 | { 423 | "data": { 424 | "text/plain": [ 425 | "[]" 426 | ] 427 | }, 428 | "execution_count": 19, 429 | "metadata": {}, 430 | "output_type": "execute_result" 431 | }, 432 | { 433 | "data": { 434 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEACAYAAABVtcpZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGXax/HvM5OeQEiB0CFA6E2aFJWIJgEVkKJYUGRd\ncVfdtaBrZWFfd11RsVdcC7KCqHRQhlBCEUGE0EPvJATSe53n/WMGNmJCApnkzGTuz3XlcmZymPOb\nMTP3Oc95itJaI4QQwj2ZjA4ghBDCOFIEhBDCjUkREEIINyZFQAgh3JgUASGEcGNSBIQQwo05pAgo\npT5TSiUrpXZV8PvBSqkMpdR2+89LjtivEEKI6vFw0PN8AbwHfHWZbdZrrUc4aH9CCCEcwCFnAlrr\njUB6JZspR+xLCCGE49TmNYEBSqkdSqnlSqnOtbhfIYQQFXBUc1BltgEttdZ5SqlhwCKgfS3tWwgh\nRAVqpQhorXPK3P5RKfWhUipYa5126bZKKZnMSAghrpDW+qqa3B3ZHKSooN1fKRVW5nY/QJVXAC7Q\nWrvkz9SpUw3PIPmNzyH5XfPHlfNXh0POBJRSc4BIIEQpdRKYCngBWms9ExirlPozUAzkA+McsV8h\nhBDV45AioLW+p5LffwB84Ih9CSGEcBwZMexAkZGRRkeoFslvLMlvLFfPf7VUdduTHE0ppZ0tkxBC\nODOlFNoJLgwLIYRwMVIEhBDCjUkREEIINyZFQAgh3JgUASGEcGNSBIQQwo1JERBCCDcmRUAIIdyY\nFAEhhHBjUgSEEMKNSREQQgg3JkVACCHcmBQBIYRwY1IEhBDCjUkREEIINyZFQAgh3JgUASGEcGNS\nBIQQwo1JERBCCDcmRcCNWCwWxkRHMyY6GovFYnQcIYQTkIXm3YTFYmHCqFFMz88H4FlfX2YtXEhM\nTIzByYQQ1VWdhealCLiJMdHRjIiNZYL9/ixgSVQU81euNDKWEMIBqlMEpDlICCHcmBQBNzFp8mRb\nExC2s4BnfX2ZNHmy0bFEHSbXoFyDNAe5EYvFwswZMwBbUZDrAaKmyDWo2iXXBNxMdb/MpRiImibX\noGqX4dcElFKfKaWSlVK7LrPNu0qpQ0qpHUqpno7Yrzu6cIQ1IjbW9iEbNeqKTrWr+++FEHWM1rra\nP8B1QE9gVwW/HwYst9++Fth8mefSomKjo6L0l6C1/edL0KOjoircfsWKFXp0VJQeHRV18faV/Hsh\nrsaKFSt0mK+v/tL+Nxbm66tXrFhhdKw6y/69eVXf3x4OKiQblVKtLrPJSOAr+7ZblFKBSqkwrXWy\nI/Yvyndpu+yEjRvp2LGjwamEO4iJiWHWwoUXmx1nSbOj03JIEaiCZsCpMvfP2B+TInCFJk2ezISN\nG6HsBbcKevnMnDGD6fn5F9tlyc/nU/u/qcq/F6I6YmJi5IvfBUgXURdz4QhrSVQUS6Kiyu1xobXm\ndNZpUutlkNAaDgVDif3/dFhoaKX/XojKlFhLKCotutCEK1xYbZ0JnAFalLnf3P5YuaZNm3bxdmRk\nJJGRkTWVyyVVdIR1NP0oH/zyAfP2zqPYWkzIwBDeDDLxWYCVfG8oPWLmjXtGyhGaKFdlvca2JW7j\nyx1fsvLoSo6mH0VrTYBXAL2a9GJs57Hc2+1eAn0CjYjuduLi4oiLi3PIczmsi6hSqjWwVGvdrZzf\n3QI8qrW+VSnVH3hba92/gufRcnRxZbIKs3hx9YvM2TOHSb3+wF3tutJAJVJYeJpTp07y66Z9nDrv\nQ2mvvixLX82gFoN4e+jbNA5o/Jvnka6j7uty/fqPZxzniRVPsD1pO5N6T2J4++F0atgJL7MXKXkp\nbDq1if/u+i/rTqzjpetf4tF+j2JS0shQmwwfJ6CUmgNEAiHY2vmnAl7YrljPtG/zPjAUyAUmaq23\nV/BcUgSqyGKx8OonU/ilwx6iW/fn5cjuZJyfTUlJWrnbm80BBIfeyfdnfPh45wLmjJ7DjeE3Xnwu\nGdzjvirq13/nmw/y2I+P8VT/p3jkmtFkpS8nIyOOgoJjaG3F07Mh9etfS2jo7ZzI9+fPP/wZXw9f\nvh79NWEBYUa+JLdieBFwJCkCVWOxWBj33AisUUVMS4f2IyCgnu13fn6dCAq6GV/ftoCZwsKTZGSs\nJzt7CwAmkw/F9cczftUSpke9zv097pfBPW7u0v//XwIzHoggq0chC0Z/jF/uHJKT5wDWCp8jIKAX\nrdu8xts71jJ3z1xW3LuCiJCIWkgvqlMEauuagHCwZ79+Av+bi/g2GIpvsT128lgwTftP5cujCWyO\n38jJzDkANA5oTJ+mfRgbcT+tdRwpKd9hzvgP8wcP4IG45ygqLTLwlQhncGmvs8du9qBhp2J+HPkE\nqSfGkV2ajVIehIaOJTR0BH5+nVDKk8LCk6SnryE5eTY5OdvZs+tm7mv2EK3rP0XkrEjWP7CetsFt\nDX514nLkTMAFfR7/OVMWPcKnbQrxawXmHNgyF15u5k+r8DaM7z6ewa0GEx4UjkJxKusUP5/6mTl7\n5nA25yyvDRpD44JZFBefw8M7gj//msWY0D/x8YOvSXOQG7twTehI45OkdMnihxEjSUv+GIDQ0Ntp\nFf4am5KOsv7Eeo5nHkehaFavGQNaDCA6fDDnkj7gxIl/onUh587W4701YRxuk8P2R7fTpF6TcvcF\ncv3JEaQ5yI1MmTuFLw9NZ0ZnRaPQIgpPwjvrYWWA4u/9pzFl3BSUqvhvYd3xdTy+4nE6BwXzl1Zn\nKCw4iPJswYO/5PJEy3+w9pMlgHww3VXskVgeWHQ/i27uT276IpTyILztu3x3Ipc3Ns+gdYPWRLeJ\npl1wOwCOZxxn7fG17D63m8f6Pkav/Prkpk+mcVNN8Ul4aJEZ30ER7Hh8B94e3oBcf6oJUgTcxEcL\nPuL57Y8wpzP4NYWjx+GFg174pTXi7ZEfMGLYiCo9T1FpEVPWTOHHg/P4sLcvJQX7KfFozV/jrWx6\naCcNfBrU7AsRTulU5in6ftqHbyL7Qc4yzOYAfJrOYKLlXcKDwvnXkH/RPax7uf/2cNphpqydwuJf\nF/Hc0gKGPQG57aD4JDy2K5QhkWP56LaPAJlcriYYPoGcqHkpeSk8v+1JPg23FQDzcZhyAJqeasvR\nj09VuQAAeJm9mB41ncmD/sHEzSmYvFrhUXKclzqZ+OOSB2UAkBsqsZZw5/d38nq/3pCzDJPJh4z6\nf+OWBS/y3HXPseSuJRUWAIB2we2YO2YuPY92ZMYw2DEL/I+AZ0v4e89i1h2PZf6++bX4ikRVSRFw\nAVZt5d4F9/BMa2jYFkwpMOkw3L4E6u/1uarntFgsLPnb14RsacrDP6ejTPVp6nGctqZNfBb/mWNf\ngHB6MzbNoFu9AlpYbTPKnvF+kD+v/oQV965gfPfxl21iLGvqH17F82tvXuwN7y+A1POKZs0z+Xhg\nVx778THO5Z6TBY6cjBQBJ2exWOg5sROBqRsZFFFIQQE8uRsGL4Fv90HJVT7nhemk//zNLhK/yeeV\nvSZAMSw0mVlbniYxO9HRL0U4qYTzCXyxbTrjmxwHrOT4juHpnxaydsJaejftfUXPFRMTw9dfLKbP\noRtY0NWPX4vHoJQ31qzFPN+zLw8ve5jo6GiZusSJyDUBJ2axWLjn4ZE0e7CQt/uByRve/hkOrYCO\n6yEcOHYVbanltcm+8mBr7r3Nyg0NTpKvg/ji/CDm3bGkykeAwjVZtZXrPh/ECxEZBFj3o3wHcOf6\nQ6y6fw3dwn43+P+KHM84znWfX8fHg28lIHsmJnMAzyc04tnBbzGiQ9WbL0Xl5JpAHfXRm69Rb3gh\nMyJsBWDbbvhxGzyzHkYA/3XgaXSXk+3YmteZLN0IX5VOV4+NLNq/SNaJreO+3vU1feqdJcC6H7M5\niD9vOcqXt39V7QIA0LpBa5bcvYSJa+bjXT8aa2kOL3dvwBMrHie/ON8B6YUjyJmAE+v4xzbc0fkY\nN/WCkrMwcTO02tSdsAa24fhX242zoi5611x3Dbd92YXXumQCxbwQX48Dz+bQvlgzCFvRkVP3uiOn\nKIcBn7Tjve5ZoPP5JrkjzRrfw5TBUxy6n8+2f8anW1/j9S4plJaksSa7N0eTO5L/7TlAuiM7gnQR\nrYN2J+/mgbkDea1bDiYFf9sMRz/1Zs7Hix3ygalosM4Ph35g1bZ7GNE4kyOpsOptGLoRngXGc3XN\nT8I5vbD6BdqVzqGN1wnSVBfeP96IVfevcvjkb1prJiyaQBffJK71WYVV+zFmTR5T34GgbBkn4AjS\nHFTHlFpL+dPSB3mlVwPMZlh+1Ie8w+0cVgDAdgFv/sqVzF+58jfPeUvELeT5jeJ8jpm2IeB5F9zi\nB9OBnxyyZ+EMTmWe4qeD79PG6wQoH17ckcjnIz+vkdk/lVKM9BrJP9av5/DJ+phUHlMbQMJgmABM\nz8+/eEAiap8UASf00a8fcX1QCt7W05SYGhFX2o4dn+4jJiamVtroX4mawQeHbQcV49vCq9G2xw+Z\nTNKVr454ZcP/8Xgb2//jrxM0MUH307pB6xrZl8Vi4dE77uPh74v44FAWpaXQoyf8eq1twSNhLCkC\nTuZ87nne+Wkqwxra2ktf31/Ex8O/wNPs+ZuunSNiY5kwalSNFIJg32D6hj7O9rPg5QOFw+HxQMWT\n//d/cspeBxxLP0bq2f8S7J1Fdgr8tK2QuX/6pMYOKi4sc/rWIeiyD3ZuA2WGJ5vC+CEyTsBoUgSc\nzEtrXmJKt8Zoay5nSlrRscX99GnaB/jtmsE1fRr92r2vE5sRDsDIVnDLG0Po06eP9BSqA15ZP5W7\nmxQD8NlJ+HoZvJZfUCtNMq/FwutZkJdvonkbUP09mPb1m3JwYSApAk5kx9kd7D39PS099oHy5NV9\n2UyNnGpIFqUUU2+bw89pvniaoU29dYy/f2SNn4WImnUw9SAlmQsI8i/lTDLsXgP/PAu7a3CfZUcI\nx2ZDzk8erM9oCsBLvZuwQa+vwb2LykgRcBJaa+6bM54/BNm6ba4+G8JD104l2Pd/jaa1Pdy+f/P+\nHNGDsWJiSMMSro8urJWzEOF4F64ljfjHYO5sZhtn/v5JeGqNbczJp0CvwYNrZN8xMTG/GSE8528L\nWJrqiTY1IIBTpKQs42j60RrZt6icFAEn8dLclwjM3UubxvkUZsM7W87SNu23i3Fc+mGqjW51z934\nHmvPe2IyQYdoOBtQo7sTNeDCtaQbfo5lYNez+JoLOXDeTLOl8Gi+rai/C2xft67GMpTtjTZ82HBe\nGvwvliXXB+CxCD9e+2l6je1bXJ4UASew9MelvP7rv/lza9v95YfhgR/h87fe+d22FXXtrCntgttR\n6D8aq1bc1BSeulkm/XI1F64lnb8OxjW3Pfafo6VEbTEu07iu41ibWg+rKZh6KpkjZ+bIfFUGkSJg\nMIvFwj3/HsN9AZpmLeB8Gqw6AL32G53sfx4d+G82pnriYYaw0SbmD71RBve4mHwvODEcvP3gRDKk\nbG3KVLNxM3malIlpN77C4iQvAB5uV593t7xba/sX/yNFwGAfvD0dU/9ibutsu//dEbCugeec6Ei7\nVYNWZPrcCsCtzSFqymApAC5k0uTJPDfAg1H2s4DPT8K7D35s+Eyew9sPJz6nOVblT6g5kQ0HPyav\nOK9WMwgpAoaLD0rgJh8IagxZZ2HxIchM8ne6I+1HBr7OljQvPJSVoydnUFxabHQkUY7yBhPeFHUT\nN93rR3B9SMw207XRnxk+bHitNy1eSinF366fyqrz/gBMbOPDnN1zaj2Hu5MiYKDknGTOtznHw11t\n9789Bo+tBU9lYuaMGU7VBbNtcFvOe9wIQFTDfBYmfGNwInGpigYTLtm/mDta2noELUn14/V73jA4\n6f/cEnELcWnBaDxp55PMvPjXZGW7WiZFwEAvr3+ZcQEN8G4EJUlw5Ch0OwzNsrOdsi/+xP4zOJzr\ngZ+5hM37XzY6jrhERYMJf9j9Lxp755Fn9aVL6yfw8/QzOupFJmXi0f4v8WtWKAA3BJ1n3Yma66Uk\nfk+KgEGOph9l3p453NfPdmHs82PQdyM8A0zDOfvid2nUhX2FtosXvfyPsfPsToMTicpk++bS3msf\nAIsTNY/0e9zgRL83rus4Fiba5jEaFJzHx784z5mKO5AiYJB/rv8n0/reiIfpLAWlwSzP9mX5+SAm\nAM5zJeD3hnZ/hYxiD5r7lrBg+wtGxxFllDeYMGyEP30bFGHVJhqEjifEL8TomL/jYfJgfO8XOVkY\niokivPPXcjLzpNGx3IYUAQMcSz/G0gOL6e2XAMCG7Oa8NuYtPvx6rm3hFpy3L35U21tYZR8x5pf3\nI98v/97gROKCSwcTfvL9XIJ9N6CUZmOqB4/0f8noiBWa0GMCc08UAnBHCw++iP/M4ETuQ4qAAf69\n8d+81GcIBfkJmDwa8+WRJO7vcb8hI4Kv1MqVK1n0RQ5FJXBtqOb5mXc71XULd1e2x8+5RqcY2tgK\nwFlTf1o1aGVwuor5e/nTpfUfydP1CDBlsfXIh5RaS42O5RZkZbFadiLjBF3f78LMtiaaNMkm9mwn\nvBqPM2yiuCs1JjqaoWtjOfkF3NQc5u6HDW+0Yd/hI0ZHE2VorXn4m1bc0+QUJ/N9aN5hGUPa3GR0\nrMs6ln6Mfy3ryvgWeSTk1ie8wzyGthtqdCyXYPjKYkqpoUqp/Uqpg0qpZ8v5/WClVIZSarv9x3nP\nS2vYY988RqcTBTRpkk1JJry5O4FOOZ2MjnVFLCWwbLXt9i2t4Yz1KP/6178MzSR+K/5sPAMCkwGI\nSwvixvAhBieqXHhQONleN2DFk07+Wczb8bbRkdxCtYuAUsoEvI/temYX4G6lVMdyNl2vte5l//ln\ndffrik5nncZyegUvtrGd5ibGw7W7Yd67/zE4WdVNmjyZNcA9i+FkNgT6wN1jYOZ0mQDMmSzYMZ1w\nvyIKrJ70jfgbSl3VQWKtm9T3aX7NsHVh9S2IIzkn2eBEdZ8jzgT6AYe01ie01sXAN8DIcrZzjb/C\nGvTaT6/RL78R9a8FVQTTC2HIVqNTXZmYmBg8zGYapcP+PbbHOlwLeaUy3N9ZFJQUUJy1GIDV50zc\n1/OPBiequiHhQ9iU0QCAW5uambXjC4MT1X2OKALNgFNl7p+2P3apAUqpHUqp5Uqpzg7Yr0tJyk7i\nv7v+y/MxnVEmOLoVis7Bh5nO1wOoMg1bt+avQOH3UFQC14RC8xGBRscSdosSviWyoe1s0ztwFAFe\nrjP/t1KKmzo+RWZpAP6mPDYffk9GENcwj1razzagpdY6Tyk1DFgEtK9o42nTpl28HRkZSWRkZE3n\nq3Gvb3qdB3vcQT2f2Vit8F5hfYLyW/L+wjecrgdQZd784APG3HYb83aU4JUEMS0gZoTRqcQFmw++\nzu0hJRzL8+TO3s8bHeeK3dtjPM989zfuawnXBWWx+fRmBrQYYHQspxIXF0dcXJxDnqvavYOUUv2B\naVrrofb7zwFaa11hI7FS6hjQW2udVs7v6lzvoHO55+j4fkfWjXqI1MTX8Aq4jtFxhzjxxAm8PbyN\njndVLBYLM2fMoKDXXp4ZmkhmsYnwblvp3riX0dHc2snMkyyOa0u3wBIWnGvFu3ceNzrSVZm0cDR3\nBy1GA0tyx/P2rbOMjuTUjO4dtBVop5RqpZTyAu4CllwSMKzM7X7Yis/vCkBdNWPTDMZ3G0d2ylcA\nrE4J4Y+9/uiyBQD+1x/9w+c2cTrfTKCnleW7phkdy+3Ni3+LboElFGsz3cKfMDrOVbvnmr+yJ9sf\nE1ayUr+nsKTQ6Eh1VrWLgNa6FHgMWAnsBb7RWicopR5WSk2ybzZWKbVHKRUPvA2Mq+5+XUVKXgr/\nif8PD3eMoKjoLL5+XXlr5zoe7v2w0dEcolWDVhwsbG27k2ORKaYNZNVWzibbLqRuTDFxZ7eJBie6\negX7C1h40PbFH92olB8OLjc4Ud3lkHECWusVWusOWusIrfWr9sc+0VrPtN/+QGvdVWt9jdZ6oNba\nwIXtatdbP7/FHZ3HkpdqO509Zu3D4NaRtAhsYXAyx+nU+q9oDX2DirEckmkkjPLTyQ1cF5wLQIHP\nEAJ9XPNivcVi4YHRo2m4uIicXGjsU8jcn/5hdKw6S6aNqEFp+Wl8vO1jnug5hNzcXXh6hvHunj08\n1Osho6M51Mguf2RHpgceSrP9sMwAaZRV+14nyLOEc4We3NrNdSf3uzAl9jvxsPa87bEmPrtJzUs1\nNlgdJUWgBr2z+R1GdRxFaeZ8ADwCb+dkVhIxbV2rN1Bl/Dz9yPO6DoAwvZPMgkyDE7mfotIiyFkF\nwJbMelzf6nqDE1Vf4xzI3ma7PaQhfLtnrrGB6igpAjUkoyCDD7Z+wDPXTiQlZSFgZuHpEib2nIjZ\nZDY6nsMN7vwS+aUmOtQrZcbCx3+3xKGoWSsOLWZgiG31sBZN/uAyI4TLU3ZKbK/VcDIDAn00W458\nYHS0OkmKQA15b8t73Nr+VrzyVqJ1CcEhI/ls92ImXuO6F+suZ1CrIfyaaRuUdC5l1u+WOBQ1y/LL\n3/E2lZKQqQhN72F0nGopO5vu6RY3sS7TtvBSO69jHEmTiQodTYpADcgqzOLdX97l+YFPk5Q0E4DD\nxV3oEdaDNkFtDE5XM5RSNAi1dfq6OQIiGzjn6mh10YLlC2jhux+AXQmap++a5PKF90IX5EWWVQSG\njAGgf3Ap3++ZbXCyukeKQA344JcPiG4bTZDeS1HRWfz8uvDR3k38sZfrzOFyNUZ0f5GkPAj1gWW3\ngQX4GNi5bZvLfyk5s3cXvkDfECgthf7L6l7hHd39EQ7n+uGhSjh8+nOj49Q5UgQcLKcoh7c2v8WL\n179IYqKtDdMnaBzxSTu4vePtBqerWa0atOLX87ZuiWci4V7gT8CUtDRpFqpBrbqfwmyC7alw+69G\np3G8QS0GsTHNF4DuAcnsT9lvcKK6RYqAg3209SNuDL+Rlr7FZGZuxGyux8LTOdzb7V58PHyMjlfj\n9m0LBqB/C3i8ma1JSJqFak5SdhIDwwsAOBgPiwqdc1nS6lBK0ab5REq1ma71ili4x3WmXncFUgQc\nKK84jxk/z+Cl61/izBnbWUBY2AQ+2zGXB3s9aHC62mE+3JJj2VDfC5LLm1BcONTCXe/SoZ6V/FIT\nB092d9plSavr7u6T2J5hRik4lfRfmVnUgaQIONAnv37CoJaD6BjcnOTkrwE4Ye1JI/9GdA/rbnC6\n2vGXx59n7X5b98QGA+ALBbOoe0enzuL0WdtI9A3JmuCUhkyaPLnOFQCAiJAIDhfaRtn3rJfBnnN7\nDE5Ud0gRcJD84nxe3/Q6U26YwtmzX2K15hEUdDNfJcQxoccEo+PVmpiYGNq2sE0ZdV1j+Di6bZ09\nOjXakbQjdPa1rbx17ifN7StX1+lrL11bTaJEm2kfUMjCPTONjlNnSBFwkKLSIv4R+Q96hHUnMfFD\nAIIb/YGlB5ZyV9e7DE5Xu54ePYP92WZ8PGDMUxHMX7lSCkANWL73PZr7WckqhInL6/61lzFdx/NL\nuu0rKyl5rjQJOYgUAQcJ9Ankod4PkZa2kvz8w3h7t2TN2XxuaHUDDf0bGh2vVvl7+ZNmugYAc95a\nSq2lBieqm5LPzQNgSyIMPGFwmFrQtF5TTpdEANAnMIftSdsNTlQ3SBFwsAvdQps2/ROzd8/h/h73\nG5zIGD3bPoFVQ4/AIjYck2mAHclisRAzYhCdfM4CELfZxFe4x7WX7q0nUaw9aOtfyGOvjZSpSRxA\nioAD5ecfIzV1OUp5of2HEX82ntva32Z0LENERdzJ7ixPPJRm66E3jY5TZ1gsFiaMGkW41yaa+0NG\nIfRq8TBLoqLc4trL6C53synF1gzUOeIMw2VqkmqrrTWG3UJi4keAplGjcXyzfwVjO411i7EB5fE0\ne1LoPQiIw7doEyXWEjxM8udWXRemWT54o+3+r2fgzPpDLFgZa2ywWtLIvxFbj/oyuGEO1zeFLk1h\neqLtOkhdLn41Sc4EHKS0NJ+kpM8AaNr0UWbvmu22TUEX9I14ihKt6FyvmDWH5xsdp85IDoYurWy3\n038ChevOGHo1zu9rTkERtA6CFQONTuP6pAg4yLlz31BSkka9en04nOtBQUkBA1u491/oDeG3sDPT\nE5OC+CPvGB2nTpg0eTKzo800DYDMQvh2jVedvgZQnmfufJkNKbbbZyLhb74+bvceOJIUAQc5Z++p\n0azZY3y18yvu636fS8/p7ghmk5lSX9tiM37FW20Ln4hqiYmJYchQ2zw6GxLh8QenuF0zyNhbx3K6\nwNZLqF8LxT++fsvt3gNHkiLgIN26LaZTp68JChnN3D1zua/7fUZHcgr92j1JiVZ0qVdC7MF5Rsdx\neV8s/py+ITkA5G+E9195xS0vikb1fp5Cqwdt6msygvYZHcelSRFwEJPJm7Cwe4g9to6IkAjaBrc1\nOpJTuD582MUmoR1H3zc6jsv7Zt00mgdAViE89GPdHhx2OSM6juFn+5LDZ5O/NTaMi5Mi4GBf7fyK\n+7u79wXhsso2CdUr2UZ+cb7BiVxb2062aSLiz0CPJIPDGKi+d30yPWwrqHXxT+Ng6kGDE7kuKQIO\nlJ6fjuWIhTu73Gl0FKdyoUmoa/1SLAe+MTqOyzqYcpDrmtvWET73E3xF3R8cdjk9Wz9EsTYTEVDM\n0r2y2MzVkiLgQN/t+46oNlEE+QYZHcWp2JqEvDAp2H1cFgu/Wj/se4+mvlYyixUH0q51i8Fhl3Nb\nh7FsTbN1vjh5do7BaVyXjN5xoNm7ZvPMwGeMjuF0/tcktJrA0h3kFefh5+lndCyXc+7ctxAKO7Lq\nE7vgZ7fvfRbiF8I5OgB7ifA5y/GM47Ru0NroWC5HzgQcJDknmSNpRxjabqjRUZxSv3ZPXGwSWiFN\nQlfsQMoBugXYroQGBN3m9gXAYrEwJjqaLT8UUWpVdK5XwpJ9XxkdyyVJEXCQsIAwjvz1CF5mL6Oj\nOKXfNAkdkyahK/Xj3vdo4lNKZrGJmzo9aXQcQ12YP2lEbCx3fXuIX1M1JqU5ckaKwNWQIuBAvp6+\nRkdwWmUcR9iQAAAdE0lEQVR7CQVZd5JblGtwItdy7vx3AOzIqkePxr0MTmOsC/MnTQAez4EDh2yP\nt/Q8QWJ2oqHZXJFDioBSaqhSar9S6qBS6tkKtnlXKXVIKbVDKdXTEfsVruXaiP/1EvrhgFzIqypp\nCqrEGrBaoWeglcX7vjY6jcupdhFQSpmA94EYoAtwt1Kq4yXbDAPaaq0jgIeBj6u7X+F6rms9lPh0\nD0wKlsZOdsuRrlejbFPQzZ3dsztoWZMmT+ZZX19mYVtDYekeb/ZmmzErKwmnvjA6nstxxJlAP+CQ\n1vqE1roY+AYYeck2I7F1a0ZrvQUIVEqFOWDfwoWsil3Fui22fu69W2Vz3523SyGogvPnbSNi47Pq\n0T1MTqJjYmKYtXDhxTUU/jtrMYcLmwHQxHSI87nnDU7oWhxRBJoBp8rcP21/7HLbnClnG1HHzZwx\ng15LNSWl0C0I7ulc4JZTHlSVxWIheuRAOvvZvtTqSVPQRTExMcxfufLi+tUtm9wDQJ8gzdL93xmc\nzrU45TiBadOmXbwdGRlJZGSkYVmEY4Xvh50p0DsMSoYBG41O5Jwu9IAZOTyfZn6QXgh+uQOMjuW0\nhneeyLqfZxARUMyu4/+B3o8YHalGxcXFERcX55DnUlrr6j2BUv2BaVrrofb7zwFaaz29zDYfA2u1\n1vPs9/cDg7XWyeU8n65uJuGcLnyx3f1YPiNvgfg0aOU9n9G3jjY6mtMZEx3NiNhYDnwI0Z0g9ggk\nzLvZbVYQuxpPzGvM7WHJxJ334Inh52ng08DoSLVGKYXW+qpOEx3RHLQVaKeUaqWU8gLuApZcss0S\n4H64WDQyyisAom670Ja793QPiq3QIwgKm501OpbTSgyF7i1tt91xBbEr1ajhGAAGhMDyA4sNTuM6\nql0EtNalwGPASmAv8I3WOkEp9bBSapJ9mx+AY0qpw8AnQN0+VxMViomJ4cf/bmNXljcmBftOfGJ0\nJKdjsVhITUnh65ugsT+kF8B3briC2JWK7jiR0wVeeJtK+PXoTKPjuAyHXBPQWq8AOlzy2CeX3H/M\nEfsSrq/sXEIN2UtGQYZbnbpfzoUms+n5+ex6yPbYxrMmZn+82G0niquq3k1689U6b5o3KcK7aCu5\nRbn4e/kbHcvpyYhhYYiycwktS5htdByncWE0bL9Q6GtvCjp0qBFDh8qcVJVRShHQYBgA14cqfjy0\nzOBErkGKgDBE2bmEEk7IqfulfizTFJS/r5XRcVxGZMQfSC32xt9cxE+H/2N0HJcgRUAYwtYkNAiA\nMJVAen66wYmcw4XRsGcG2+6vPwOTH55maCZXEhl+IxtSbLdV3gYKSwqNDeQCpAgIw/Rr9yQlVluT\n0JJ9s4yO4xRiYmL455x36GM/+FfeQ6Qp6Ap4mb3A73oArg+F2CMrDU7k/KQICMNcHz6MnVm2Rej3\nn/rU6DhOI6vBdpr4QWaxiQk3v2F0HJfTv81Eckq9CfIsJO6gNAlVRoqAMIzZZKbEPr10E3WA1LxU\ngxM5h9SU+YDMFXS1hkXcyobzVgDys2IpLi02OJFzkyIgDHVtmSahpdIkRML5BHrUSwMgIGiEzBV0\nFQJ9AskwdQNsTULrTqwzOJFzkyIgDPXbJiE5dV+x9x0aeZeSXmzm5k5PGB3HZfVoPYFCqxeNvfNZ\nuf9zo+M4NSkCwlBlm4Samg66/TTAaan2pqDMevRofI3BaVzXiI6j2Ww7oSIjbSlWbTU2kBOTIiAM\n16/dE/YmIatb9xJKOJ9A9wBbV9nAkNulKagamtdvzrGi5gAMDCll06lNBidyXlIEhOFuCL/F3iSk\nOeDGTUKWvW/R0LuUtCIz0Z3dezF5R2jf7F5KtAetffP5IUEWoa+IFAFhOLPJTLGPrUmohfkw53LP\nGZzIGGmpCwGIzw6ka6NuBqdxfcM73sHOTE8Aks/PR6aoL58UAeEU+kU8TolV0aW+lUV73W+d2ITz\n+y72CgqSpiCH6NqoK9uzbBPI9WmQz7akbQYnck5SBIRTuCH8FnbYm4QOnfrM6Di1zrL3TUK8rKQU\nmYnuJE1BjqCUolmjsVgx0cG/gKUJ/zU6klOSIiCcgofJgxJ7k1BLz2OczXGfxWa01qSlLgBgZ3YD\nuoZ1NThR3XFrx7s4lOuPSWmOJ30jTULlkCIgnMbFJqF6pSx2oyahjxZ+QHd/W6+g3Px+BqepWwa1\nHMQ6++jhrn7pRI0axJjoaCwWi8HJnIcUAeE0ftMkdNo9BvhYLBa+XPEEoT5wLg++en61fEE5kIfJ\ng/pBw9AautcrIiTvZ0bExjJh1Ch5n+2kCAjDWCwWxkRHXzwyszUJ2aaXbuV5nMTsRIMT1rxPZrxB\nZP9SAHadhBmni5g5Y4bBqeqWqIi7OJjmgbcneA+GCcD0/Hx5n+2kCAhDXFhGcURs7G+OzPq2+1+T\n0KK9df9sIC0wnYFNbLdD4gyNUmfFtIshLs1WaNu3hsPBxuZxNlIEhCEuLKM4gd8emQ1ucyvxmbYm\noaNu0CTUebgnDXzhbC4UroJnfX1lQXkHC/AKIL3EdrG9TyhM6Srvc1lSBIRT8TB5UORrWxSkvc9J\nTmedNjhRzSkuLSbENx6A9YmebO0VxayFC2VB+Rowqu+jnM33x8cbzt3iJ+9zGVIEhCEuLKM4C5jF\nb4/M+rd7kgKrifYBpSzZ/aGhOWvSqiM/MDC4BICIDo8wf+VK+WKqIcM7DGddqq1J6NrwUjr372xw\nIuchRUAYIiYmhlkLF7IkKoolUb89Ah7cJoYtabbh/onJs42MWaM2HXwLX3MpR3PNjOou00bXpKb1\nmnKmtC0A14cqFuz73uBEzkOKgDBMTEwM81eu/N0RsIfJAxUQBUBnv0SOpR8zKmKNySvOw6/INrPl\ngYIWtG7Q2thAbqBf63HkWOvjaypg81H3G5VeESkCwind1PkZMovNNPWxsnjXdKPjONyyhG/oG1SC\n1tC2+UNGx3ELIzqOZGOK7XZT8xGOZxw3NI+zkCIgnNLAFtfzS4Zt8q+U8/Pq3HD/bUfewkNpdmeZ\nGd1tktFx3EK3Rt3YlukHwI2NPPl2zzyDEzkHKQLCKSmlaBA6FoC+gdlsOrXB4ESOk5SdRHPTfgAS\nrV0I9Qs1OJF7UErRufkdFBFAgCmbDYfdZ2qSy5EiIJzW8K7PkFRgJtCzlJV76k6T0Le7PqRr/RJK\ntIkebf5idBy3MrLj7WzL8AWgtddpDqQcMDiR8apVBJRSQUqplUqpA0opi1IqsILtjiuldiql4pVS\nv1Rnn8J9dGzYkd25jWx38lZRUFJgbCAH0Fpz9PSnKAW/pJkY0eluoyO5letbXs/Ks/kARDX2Zt5e\naRKq7pnAc8AqrXUHYA3wfAXbWYFIrfU1WmuZJlFUWYsmfwBgYHApS/Z/Z3Ca6tuWuI0+9e0zhnoN\nwN/L3+BE7sXT7EnzRrdSih/1TWmsOjirzl1vulLVLQIjsY31wf7f2yvYTjlgX8INje7+Fw7kmPE2\nlbL1kOtP+LVk9xu08C0it9SDQR1k2gIj3NZ+FPvzGgDQxT+N3ed2G5zIWNX9Ym6ktU4G0FqfBRpV\nsJ0GYpVSW5VS0h9OVFlYQBjHijsC0MK0x6UXmykqLaIoczEA61I8iWp7i8GJ3NPQdkNZcNzWV7Sv\nVx6vLHnF4ETG8qhsA6VULBBW9iFsX+ovlbN5RedVg7TWSUqphtiKQYLWemNF+5w2bdrF25GRkURG\nRlYWU9Rh3cL/SnHan+keWMp3O9/lL4Nc80O7/MBCbgi1TRPhFzQGT7OnwYnc0+Z1m9m6rZiSdtAm\ntIi12+axYsUDDB061OhoVRYXF0dcXJxDnktVpz1MKZWAra0/WSnVGFirte5Uyb+ZCmRrrd+s4Pfa\n3dvoxG/lFOXwxuJgIhsWs/xcCK/dcd4lF2J/asE1jAjewbE8T/r03ka3sG5GR3JLY6Kj8cuM5eaH\noFU7+DweTm3ox9oFW4yOdtWUUmitr+pDUd3moCXAA/bbE4DFl26glPJTSgXYb/sD0cCeau5XuJEA\nrwAKfW8CYECDTNafiDM20FU4nnGclua9AMRnN5YCYDDfAzAv13b7hkA43dB1mxmrq7pFYDoQpZQ6\nANwEvAqglGqilFpm3yYM2KiUigc2A0u11iuruV/hZm7t9gIpRZ4Ee5WwfNfLRse5YrO3v0mP+sWU\nahOdWz9mdBy3Vq9pU77LhG2HobQEwltBbstUSqwlRkczRLWKgNY6TWt9s9a6g9Y6WmudYX88SWt9\nm/32Ma11T3v30G5a61cdEVy4l0Etr2NTen0AAoo2kpqXanCiqisuLSYp6QuUgk2pZu7s/kejI7kt\ni8XCktmzeRsYsAd2ngKzGW4rrU9hSaHR8Qwh3TaFS1BK0arZJLSGgSGlfL3jI6MjVdnSA4u5uVEx\nAJmeAwj2lfUNjTJzxgzeslqZAPz9AMRn2B7v37jEbcdsSBEQLuOuno+zI9MDD2XlwIl3XWaQT+y+\nVwj2LCSt2IubO//N6DjC7pok2HQOrFYIj8ikpCTH6EiGkCIgXEZYQBhn6A3A4OAM1h2PMzZQFRxN\nP0pb+wXh2HM+RLV1nW6IdVHZFe2+As7shsScQJQqIi1thdHxDCFFQLiUGzu9QEaxF428i1m88+9G\nx6nU57+8Qu8GRZRqEy2aTcJsMhsdya1duqLdlPv+ya4i2+jhlJSFBqczRrXGCdQEGScgLqfEWsJf\n54ZwZ7MstqZ7cseNB512Va7swmxemN+IMU0LWJ/ixYPRR2lWv5nRsUQZhSWFdHm3IctGTSG86X14\nezc2OtJVMXKcgBC1ysPkQYtmD1OqTfRpUMJnvzjv6OFZ8Z8yrLHtgOaceYAUACfk7eFNrxZD+Sk9\n2GULQHVJERAuZ2Kfp9iYakYpTUbKbLILs42O9DtWbWXTwen4mQo5U+DD7d1fNDqSqMDIDiNZcnCJ\n0TEMI0VAuJzGAY3J9IoEYFhjzVc7/mNsoHIsP7CMmJBMABYdKaX4kHsORHIFwyKGsfbYWvKK84yO\nYggpAsIlje05jRP53viZCtl26N+UWkuNjvQbX6x/glb+heTmg/e8YiaOHoPFYjE6lihHsG8wq+9f\njZfZy+gohpAiIFzSgBYD2JDRBIBhjbL5+zdTGBMdzZjoaMO/bDec2MC1DU4AsOwMvLcVpufnM3OG\n66+HUFf1bdYXD1OlkyrXSVIEhEtSSjGow4tklvjQ0KuAjcf+zW2rYhkRG8uEUaMMLQQf/fw81zay\nUlwMASsgyPVXxRR1mBQB4bLu7j6eRUm2OfnHt4fg9rapbI086v418VfC1TYAfkyCRmtsS+496+vL\npMmykphwPlIEhMvy8fChU+vJ5BQpIhrC7DEVr2rkCBaLpdImpymL/kRkSAHaClt/gumZJr645hpm\nLVxITExMDaYT4upIERAubVLfx1mebDsbGNwdJkfUzFG3xWJhwqhRjIituMkpPimejgHxeHrA1kR4\n7QeYYbUSEhoqBUA4LSkCwqU18GlAcONJFJZ60C0MNk/05fMF31/1l25FR/szZ8xgen4+E6i4yenl\ntY9zazONtsLP22HA6Wq8MCFqiRQB4fIe7f88i5JsPTvuiSji77OfuaoLw1U52q9I7JFYunrtwsOs\nWXcG2i6RawHCNUgREC6vab2mJOfeQHY+dA0tpV7nfdw/9vYrLgSXO9ovO/vkpV/uVm3l1bgnuDE0\nF41iQ1Y4R9vczJKoKLkWIJyee3aMFXVO/tJ8FgyHCR3g4XawoGcBM2fMcNgX8IXZJy8UhVmTJ198\n7q92fsXwRmdRlLA1M5Cnoz9g2KPDHLJfIWqaFAHhkiwWy2+O0v2KfDi/DAqaQeMwyB8LJavyr+g5\nJ02ezISNGyHf9u+e9fVlVpmmnJiYmN8VldS8VP7z81P8s1M6Gk9Wpbfm6XayZoBwHdIcJFxOeW33\nvQYP5pefffjoqG2bB8IhL+bK1oy9dK75qjTlPL/6OZ5s7w3A8mQ/nhv8Fkpd1Yy+QhhCzgSEyynb\ndg9Afj5L1q1j9teLePK7P3Gk+RnaBhfTP+wICxIWMLrT6Co/d3lH+xVZd3wdmSnfERKeSTEBHNcD\nuDH8xit/QUIYSM4ERJ0RExPD1g/3MPu8baWoYQ2zeGX1JBKzEx2+r4yCDB5ddi9/amMF4JOj8PJN\nbzp8P0LUNCkCwuVcrqeOv5c/fxr0LpsyQoAS/t7Vm4mL7seqrQ7N8NgPj/FMRz+UzuZsaXMahY2n\nU8NODt2HELVBioBwOZW13Y/rMo61Ge0pVsHUJ5He/od4ZcPlVyCrypQQF3y49UPIXUcrj0OgvHl1\nXyH/GuK8K5wJcTmyxrCok/ad38dfFgxgSocswIN/HgzmT9d9wNjOY3+37YULzdPL9gqq4KLwmmNr\neHzpON7vWYi2ZvNtUmMiu7/DnV3urOmXJESFZI1hIS7RuWFnoru8wKaslkAJUzpZeWHlw6w+uvp3\n21ZlSgiAzac3c/+CcbzbJwRtzeac7sBJ3Ys7Ot9R469HiJoiRUDUWZMHTmbZucZkm9qhS1OYeW1j\n7p03mkF39rrixWd+OvkTI+cO58tB7VFFB8CjMS/uTOc/Iz6TLqHCpUlzkKjTjqUfI2ZWP2b2MUNJ\nMjv3Kf7vhCZqA6zd4cNXCxcBVNgcpLXms/jPeHH183wd2QeP3BWYzPX5224/pkZ9zrAIGRksjFed\n5qBqFQGl1FhgGtAJ6Ku13l7BdkOBt7GdeXymtZ5+meeUIiAcasXhFbxgmcD/tU4nwL8Ynz3wXAIk\n5UOzjH6sWbDldyOQY2Ji2J28m2din+Fc7hk+7NeGgswlmEy+zDwVTniT0bw85GWDX5kQNkYWgQ6A\nFfgEeLq8IqCUMgEHgZuARGArcJfWen8FzylFQDjc6z+9zldrpvJm53w8Q8D3FHy7ED7q6EP3Nj2I\naRtDx9COKKU4nHaY2KOxHEo9xPP9H2CAz2pysn/BZPJnYXpfEosbMXfMXExKWlOFczCsCJQJsBaY\nXEER6A9M1VoPs99/DtAVnQ1IERA1Zdyn49iQ8C2fdQXfNmAtBauOobTZWNYlHuVYxjFKraW0DGzJ\njS26EOG5j6Qz76F1Id7eLfk6uT2HckwsGrcIX09fo1+OEBc5exEYA8RorSfZ748H+mmt/1rBc0kR\nEDVCa82Ql4fwS9Y6HvHw5JaYYpSy/a2lp/qRnupHq7YdqF8/i9zcPVxYrLJByFim7jpPifJn/p3z\n8fHwMfBVCPF7NdpFVCkVq5TaVeZnt/2/w69mh0IYZeXKlSS8uoV7Vmk+1kU89LZi04Z65GRDUEge\nbdqnYDb/RG7ubpTypGHDO8kNmc7I1b/QvmFfFt+1WAqAqHMqnUBOax1VzX2cAVqWud/c/liFpk2b\ndvF2ZGQkkZGR1YwgBLz8/POE5+eTsBNanIJDt1qZlpbN6Nfh2SII84UNwMr23bnu4Ud4bvOXpOfv\n5PMRn3NTm5uMji/ERXFxccTFxTnkuRzZHPS01npbOb8zAwewXRhOAn4B7tZaJ1TwXNIcJBzOYrFw\n9y238AerlVnAG/bH/9oc+vSAk23gZCCUmMCr2Jvh14zgnm73MLz9cMwms5HRhaiUkb2DbgfeA0KB\nDGCH1nqYUqoJ8KnW+jb7dkOBd/hfF9FXL/OcUgSEw42JjmZEbCxLgBFwcRrqp4HPgbcAq4LnfGxj\nB2RJSOFKqlMEqrWegNZ6EbConMeTgNvK3F8BdKjOvoSoCd2AMOAppfAPCCCiXTujIwlRq6Sjs3AL\nF6afDsd29H9hGuonTSY827bF6unJy9nZPBQfz4RRo654kXohXJVMGyHcxoVRwckpKXgAIaGhTJo8\nmZkzZtiWqbRvNwtYEhXF/JUrDUwrRNUZ1hwkhCupaOnI8mYMFcJdSBEQbm/S5MlM2LgRyk4gZ1+p\nTIi6TpqDhIByJ5ATwlUYPm2EI0kREEKIKyMriwkhhLgqUgSEEMKNSREQQgg3JkVACCHcmBQBIYRw\nY1IEhBDCjUkREEIINyZFQAgh3JgUASGEcGNSBIQQwo1JERBCCDcmRUAIIdyYFAEhhHBjUgSEEMKN\nSREQQgg3JkVACCHcmBQBIYRwY1IEhBDCjUkREEIINyZFQAgh3JgUASGEcGNSBIQQwo1VqwgopcYq\npfYopUqVUr0us91xpdROpVS8UuqX6uxTCCGE41T3TGA3MApYV8l2ViBSa32N1rpfNffptOLi4oyO\nUC2S31iS31iunv9qVasIaK0PaK0PAaqSTVV19+UKXP2PSPIbS/Iby9XzX63a+mLWQKxSaqtS6qFa\n2qcQQohKeFS2gVIqFggr+xC2L/UXtdZLq7ifQVrrJKVUQ2zFIEFrvfHK4wohhHAkpbWu/pMotRaY\nrLXeXoVtpwLZWus3K/h99QMJIYSb0VpX1ixfrkrPBK5AuQGUUn6ASWudo5TyB6KBf1T0JFf7QoQQ\nQly56nYRvV0pdQroDyxTSv1of7yJUmqZfbMwYKNSKh7YDCzVWq+szn6FEEI4hkOag4QQQrgmQ7tt\nKqWClFIrlVIHlFIWpVRgOds0V0qtUUrtVUrtVkr91Yisl2QaqpTar5Q6qJR6toJt3lVKHVJK7VBK\n9aztjJdTWX6l1D32wX07lVIblVLdjMhZkaq8//bt+iqlipVSo2szX2Wq+PcTaR9cucd+zc0pVOFv\nJ0Qp9aP97363UuoBA2JWSCn1mVIqWSm16zLbOPNn97L5r+qzq7U27AeYDvzNfvtZ4NVytmkM9LTf\nDgAOAB0NzGwCDgOtAE9gx6V5gGHAcvvta4HNRr7PV5G/PxBovz3U1fKX2W41sAwYbXTuK3z/A4G9\nQDP7/VCjc19B9qnAvy/kBlIBD6Ozl8l3HdAT2FXB7532s1vF/Ff82TV6ANdIYJb99izg9ks30Fqf\n1VrvsN/OARKAZrWW8Pf6AYe01ie01sXAN9heR1kjga8AtNZbgEClVBjOodL8WuvNWutM+93NGPt+\nX6oq7z/AX4DvgXO1Ga4KqpL/HmC+1voMgNY6pZYzVqQq2c8C9ey36wGpWuuSWsx4WdrWNT39Mps4\n82e30vxX89k1ugg00long+3LHmh0uY2VUq2xVcEtNZ6sYs2AU2Xun+b3b/Sl25wpZxujVCV/WX8E\nfqzRRFem0vxKqabA7Vrrj6h8NHttq8r73x4IVkqttQ+wvK/W0l1eVbJ/CnRRSiUCO4HHaymbozjz\nZ/dKVemz68guouW6zGCzl8rZvMKr1EqpAGxHdo/bzwhEDVNK3QhMxHYK6krexta8eIGzFYLKeAC9\ngCGAP/CzUupnrfVhY2NVyfPATq31jUqpttgGh3aXz2ztupLPbo0XAa11VEW/s1/gCNNaJyulGlPB\nqbtSygNbAZittV5cQ1Gr6gzQssz95vbHLt2mRSXbGKUq+VFKdQdmAkO11pc7fa5tVcnfB/hGKaWw\ntUsPU0oVa62X1FLGy6lK/tNAita6AChQSq0HemBrjzdSVbIPAv4FoLU+opQ6BnQEfq2VhNXnzJ/d\nKrnSz67RzUFLgAfstycAFX3Bfw7s01q/UxuhKrEVaKeUaqWU8gLuwvY6yloC3A+glOoPZFxo9nIC\nleZXSrUE5gP3aa2PGJDxcirNr7VuY/8Jx3bw8IiTFACo2t/PYuA6pZTZPtjyWmzXwoxWlewJwM0A\n9rb09sDRWk1ZOUXFZ4fO/Nm9oML8V/XZNfhKdzCwCluPn5VAA/vjTYBl9tuDgFJsPRHige3YKpyR\nuYfaMx8CnrM/9jAwqcw272M7ctsJ9DIy75Xmx9aum2p/r+OBX4zOfKXvf5ltP8eJegddwd/P09h6\nCO0C/mJ05iv42wkFltr/7ncBdxud+ZL8c4BEoBA4ia3JxJU+u5fNfzWfXRksJoQQbszo5iAhhBAG\nkiIghBBuTIqAEEK4MSkCQgjhxqQICCGEG5MiIIQQbkyKgBBCuDEpAkII4cb+H9vTZFk1Hhr6AAAA\nAElFTkSuQmCC\n", 435 | "text/plain": [ 436 | "" 437 | ] 438 | }, 439 | "metadata": {}, 440 | "output_type": "display_data" 441 | } 442 | ], 443 | "source": [ 444 | "X_ = np.linspace(0, 1, 1000).reshape(-1, 1)\n", 445 | "y_, var = gp.predict(X_)\n", 446 | "plt.scatter(X, y, c='r')\n", 447 | "plt.plot(X_, y_ + var**.5, c='g')\n", 448 | "plt.plot(X_, y_ - var**.5, c='g')\n", 449 | "plt.plot(X_, y_, c='y', linewidth=2)" 450 | ] 451 | }, 452 | { 453 | "cell_type": "markdown", 454 | "metadata": {}, 455 | "source": [ 456 | "### Higher Dimensions" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "metadata": { 463 | "collapsed": false 464 | }, 465 | "outputs": [], 466 | "source": [ 467 | "np.random.seed(45)\n", 468 | "n = 1000\n", 469 | "dim = 7\n", 470 | "X = np.random.uniform(0, 1, (n, dim))\n", 471 | "ls = [.1, .2, .4, .8, 1.6, 3.2, 6.4]\n", 472 | "amp = 10.\n", 473 | "noise = 0.001\n", 474 | "sqdist = GP.compute_sqdist(X/ls)\n", 475 | "K = amp**2 * np.exp(-0.5 * sqdist)\n", 476 | "A = np.linalg.cholesky( K + np.eye(n)*1e-9 )\n", 477 | "y = A.dot(np.random.randn(n, 1))" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": { 484 | "collapsed": false 485 | }, 486 | "outputs": [], 487 | "source": [ 488 | "shuffle = np.random.permutation(len(X))\n", 489 | "X = X[shuffle]\n", 490 | "y = y[shuffle]\n", 491 | "mask = len(X)/3*2\n", 492 | "X_train, y_train = X[:mask], y[:mask]\n", 493 | "X_test, y_test = X[mask:], y[mask:]" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": null, 499 | "metadata": { 500 | "collapsed": false 501 | }, 502 | "outputs": [], 503 | "source": [ 504 | "gp = GP(lsr=(.1, .2, 7),\n", 505 | " amp=1.,\n", 506 | " noise=1e-2)\n", 507 | "gp.fit(X_train, y_train, \n", 508 | " nepochs=500, batch_size=500, learning_rate=1e-2, \n", 509 | " learning_rate_decay=.999, \n", 510 | " verbose=2)\n", 511 | "y_, var = gp.predict(X_train)\n", 512 | "print \"mse:\", ((y_train - y_)**2).mean()/np.var(y_train)\n", 513 | "y_, var = gp.predict(X_test)\n", 514 | "print \"mse:\", ((y_test - y_)**2).mean()/np.var(y_test)\n", 515 | "print \"mse:\", ((y_test - y_train.mean())**2).mean()/np.var(y_test)" 516 | ] 517 | }, 518 | { 519 | "cell_type": "code", 520 | "execution_count": null, 521 | "metadata": { 522 | "collapsed": true 523 | }, 524 | "outputs": [], 525 | "source": [ 526 | "from sklearn.ensemble import RandomForestRegressor as RF" 527 | ] 528 | }, 529 | { 530 | "cell_type": "code", 531 | "execution_count": null, 532 | "metadata": { 533 | "collapsed": false 534 | }, 535 | "outputs": [], 536 | "source": [ 537 | "rf = RF(n_estimators=1000).fit(X_train, np.squeeze(y_train))\n", 538 | "\n", 539 | "y_ = rf.predict(X_train)\n", 540 | "print \"mse:\", ((np.squeeze(y_train) - y_)**2).mean()/np.var(y_train)\n", 541 | "y_ = rf.predict(X_test)\n", 542 | "print \"mse:\", ((np.squeeze(y_test) - y_)**2).mean()/np.var(y_test)" 543 | ] 544 | } 545 | ], 546 | "metadata": { 547 | "kernelspec": { 548 | "display_name": "Python 2", 549 | "language": "python", 550 | "name": "python2" 551 | }, 552 | "language_info": { 553 | "codemirror_mode": { 554 | "name": "ipython", 555 | "version": 2 556 | }, 557 | "file_extension": ".py", 558 | "mimetype": "text/x-python", 559 | "name": "python", 560 | "nbconvert_exporter": "python", 561 | "pygments_lexer": "ipython2", 562 | "version": "2.7.10" 563 | } 564 | }, 565 | "nbformat": 4, 566 | "nbformat_minor": 0 567 | } 568 | -------------------------------------------------------------------------------- /tgp/.idea/test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | sess1 = tf.Session(config=tf.ConfigProto( 4 | inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 5 | )) 6 | sess2 = tf.Session(config=tf.ConfigProto( 7 | inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 8 | )) 9 | 10 | a = tf.Variable(1.) 11 | d = tf.Variable([1., 1.]) 12 | add = tf.assign_add(a, 1.) 13 | b = tf.assign(tf.Variable(a, trainable=True), a) 14 | c = tf.convert_to_tensor(a) 15 | sess1.run(tf.initialize_all_variables()) 16 | sess2.run(tf.initialize_all_variables()) 17 | print sess1.run(a) 18 | print sess1.run(b) 19 | sess1.run(add) 20 | sess1.run(add) 21 | sess1.run(add) 22 | print sess1.run(b) 23 | print sess1.run(c) 24 | print tf.trainable_variables() 25 | 26 | loss_a = tf.square(a) 27 | loss_b = (b - d[0])*(10 - d[0]) 28 | loss_c = tf.square(c) 29 | 30 | print sess1.run(loss_b) 31 | train = tf.train.GradientDescentOptimizer(0.01).minimize(loss_b) 32 | 33 | for _ in xrange(100): 34 | sess1.run(train) 35 | 36 | 37 | print sess1.run(a) 38 | print sess1.run(b) 39 | print sess1.run(c) 40 | print sess1.run(loss_b) 41 | print sess1.run(d) 42 | print sess2.run(b) 43 | -------------------------------------------------------------------------------- /tgp/.idea/test2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | sess1 = tf.Session(config=tf.ConfigProto( 3 | inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 4 | )) 5 | sess2 = tf.Session(config=tf.ConfigProto( 6 | inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 7 | )) 8 | 9 | a = tf.Variable(1.) 10 | b = tf.assign(tf.Variable(2.), a+10) 11 | sess1.run(tf.initialize_all_variables()) 12 | sess2.run(tf.initialize_all_variables()) 13 | sess1.run(tf.assign(a, 100)) 14 | print sess1.run(a) 15 | print sess1.run(b) 16 | print sess2.run(a) 17 | print sess2.run(b) 18 | -------------------------------------------------------------------------------- /tgp/.idea/zero.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | # sess = tf.Session() 3 | # sess.run(tf.div(0,0)) 4 | -------------------------------------------------------------------------------- /tgp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiShu/tensorflow-gp/45f48c93475de7e49658745b34536a74c9cb6380/tgp/__init__.py -------------------------------------------------------------------------------- /tgp/gradient_check.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def check_grad(f, x, h=1e-4, verbose=0): 4 | """ checks whether the computed gradient is correct 5 | 6 | Parameters 7 | ---------- 8 | f : lambda function 9 | Takes in a particular argument x and computes the loss and gradient 10 | w.r.t. x 11 | x : np.array 12 | A parameter (or array of parameters) to be passed into f 13 | h : float 14 | The discrete step used for central difference 15 | """ 16 | max_dev = -np.inf 17 | if type(x) == np.ndarray and x.shape is not (): 18 | it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite']) 19 | _, ana_grad = f(x) 20 | num_grad = np.zeros_like(ana_grad) 21 | while not it.finished: 22 | ix = it.multi_index 23 | x[ix] += h/2 24 | fxph, _ = f(x) 25 | x[ix] -= h 26 | fxmh, _ = f(x) 27 | x[ix] += h/2 28 | num_grad[ix] = (fxph - fxmh) / h 29 | if verbose > 1: 30 | print "Numeric:", num_grad[ix] 31 | print "Analytic:", ana_grad[ix] 32 | if verbose > 0: 33 | diff = ana_grad[ix] - num_grad[ix] 34 | if abs(diff).max() > max_dev: 35 | max_dev = abs(diff).max() 36 | print "Numerical v. analytical gradient difference:\n", diff 37 | it.iternext() 38 | elif type(x) == np.ndarray and x.shape is (): 39 | _, ana_grad = f(x) 40 | original = x 41 | x += h/2 42 | fxph, _ = f(x) 43 | x -= h 44 | fxmh, _ = f(x) 45 | x += h/2 46 | num_grad = (fxph - fxmh) / h 47 | if verbose > 1: 48 | print "Numeric:", num_grad 49 | print "Analytic:", ana_grad 50 | if verbose > 0: 51 | diff = ana_grad - num_grad 52 | if abs(diff).max() > max_dev: 53 | max_dev = abs(diff).max() 54 | print "Numerical v. analytical gradient difference:\n", diff 55 | if verbose: 56 | print "Max deviation:", max_dev 57 | return ana_grad, num_grad 58 | 59 | if __name__ == "__main__": 60 | f = lambda x: (x.sum(), np.ones_like(x)) 61 | check_grad(f, x=np.random.randn(3, 3), verbose=True) 62 | -------------------------------------------------------------------------------- /tgp/np/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiShu/tensorflow-gp/45f48c93475de7e49658745b34536a74c9cb6380/tgp/np/__init__.py -------------------------------------------------------------------------------- /tgp/np/gaussian_process.py: -------------------------------------------------------------------------------- 1 | """ 2 | gaussian_process.py 3 | ------------------- 4 | This module performs basic Gaussian process regression using the squared 5 | exponential kernel. 6 | 7 | Author: Rui Shu 8 | """ 9 | 10 | import numpy as np 11 | from numpy.linalg import cholesky as chol 12 | 13 | class GaussianProcess(object): 14 | def __init__(self, ls=None, amp=None, noise=None, 15 | lsr=(.1, .2, 1), ampr=(1., 2.), noiser=(.1, .2), 16 | keps=1e-9): 17 | """ Gaussian Process Class 18 | 19 | Initialization of Gaussian process hyperparameters 20 | 21 | Parameters 22 | ---------- 23 | ls : (n_dim,) array/list of floats 24 | Array of length scales to be used 25 | amp : float 26 | Amplitude parameter in kernel 27 | noise : float 28 | Noise parameter in kernel 29 | lsr : 3-tuple (float, float, int) 30 | The range for random initializtion, and ndim number of ls's 31 | ampr : 2-tuple (float, float) 32 | The range of random initialization for amp 33 | noiser : 2-tuple (float, float) 34 | The range of random initialization for noise 35 | keps : float 36 | Small constant added to diag of kernel matrix for stability 37 | """ 38 | # parameters for non-learning 39 | self.params = {} 40 | self.params['ls'] = (np.array(ls) if ls is not None 41 | else np.random.uniform(*tuple(lsr))) 42 | self.params['amp'] = (np.array(amp) if amp is not None 43 | else np.random.uniform(*tuple(ampr))) 44 | self.params['noise'] = (np.array(noise) if noise is not None 45 | else np.random.uniform(*tuple(noiser))) 46 | self.params['keps'] = keps 47 | # self.Ki stores the inverse of the kernel 48 | self.Ki = None 49 | 50 | def fit(self, X, y, nepochs=1000, batch_size=None, train=True, 51 | learning_rate=1e-1, learning_rate_decay=1., 52 | verbose=0): 53 | """ Feeds in input data and trains parameters of the GP if desired 54 | 55 | Parameters 56 | ---------- 57 | X : (n_samples, n_dim) float matrix 58 | independent variables 59 | y : (n_samples,) float matrix 60 | response variable 61 | train : boolean 62 | Whether to train parameters or not 63 | """ 64 | # Preprocess: 65 | X = np.atleast_2d(X).astype(float) 66 | y = y.reshape(-1, 1).astype(float) 67 | # Scale to unit hypercube 68 | y_m = y.mean() 69 | y_scale = y.max() - y.min() 70 | X_m = X.min(axis=0) 71 | X_scale = X.max(axis=0) - X.min() 72 | # Check for scale being 0 73 | y_scale = y_scale if y_scale > 0 else 1. 74 | X_scale[X_scale <= 0] = 1. 75 | self.transform = {'ym': y_m, 76 | 'ys': y_scale, 77 | 'Xm': X_m, 78 | 'Xs': X_scale} 79 | X = (X - X_m)/X_scale 80 | y = (y - y_m)/y_scale 81 | 82 | if train: 83 | self.train(X, y, nepochs, batch_size, 84 | learning_rate, learning_rate_decay, 85 | verbose) 86 | 87 | K = self.K(X) 88 | self.Ki = np.linalg.inv(K) 89 | self.X = X 90 | self.y = y 91 | return self 92 | 93 | def train(self, X, y, nepochs, batch_size, learning_rate, 94 | learning_rate_decay, verbose): 95 | """ Update parameters via gradient ascend to maximize marginal log 96 | likelihood. 97 | """ 98 | batch_size = len(X) if batch_size is None else batch_size 99 | if batch_size > len(X): 100 | string = "Batch size larger than sample size. Reducing it to. {0:d}" 101 | print string.format(len(X)) 102 | batch_size = len(X) 103 | batch_count = 0 104 | 105 | # Compute the frequency with which print statements should arrive 106 | # I expect at least 10 epoch print outs 107 | 108 | for epoch in xrange(1, nepochs+1): 109 | shuffle = np.random.permutation(len(X)) 110 | X = X[shuffle] 111 | y = y[shuffle] 112 | nbatch = len(X)/batch_size # good enough approximation 113 | for i in xrange(nbatch): 114 | batch_count += 1 115 | X_batch = X[i*batch_size : (i+1)*batch_size] 116 | y_batch = y[i*batch_size : (i+1)*batch_size] 117 | L, dL = self.L(X_batch, y_batch, grad=True) 118 | print dL['ls'] 119 | for key in ['ls', 'amp']: 120 | self.params[key] += learning_rate * dL[key] 121 | if verbose > 2: 122 | print key, ":", self.params[key] 123 | print key, "gradient:", dL[key] 124 | if batch_count % 100 == 0 and verbose > 1: 125 | print "...Batch:", batch_count, "Negative log likelihood:", -L 126 | if epoch % (nepochs/10) == 0 and verbose > 0: 127 | print "Epoch:", epoch, "Negative log likelihood:", -L 128 | 129 | if verbose > 0: 130 | print "*"*80 131 | print "Final negative log likelihood", -L 132 | print "Final parameters:" 133 | for key in self.params: 134 | print key, ":", self.params[key] 135 | print "*"*80 136 | 137 | def L(self, X, y, grad=False): 138 | """ Computes the log likelihood and returns the gradient w.r.t. ls, amp, 139 | and noise if desired. 140 | """ 141 | # Set up cache 142 | cache = {} 143 | K = self.K(X) 144 | Ki = np.linalg.inv(K) 145 | Kiy = Ki.dot(y) 146 | norm = len(X) 147 | cache['K'] = K 148 | cache['Ki'] = Ki 149 | cache['Kiy'] = Kiy 150 | cache['norm'] = norm 151 | # Compute L and dL 152 | L = -0.5*y.T.dot(Kiy).sum() - np.log(np.diag(chol(K))).sum() 153 | L -= len(X)/2 * np.log(2*np.pi) 154 | if grad: 155 | return L/norm, self.L_grad(X, y, cache) 156 | else: 157 | return L/norm 158 | 159 | def L_grad(self, X, y, cache=None): 160 | """ Computes the gradient of the log likelihood w.r.t. ls, amp, and 161 | noise. 162 | """ 163 | if cache is None: 164 | K = self.K(X) 165 | Ki = np.linalg.inv(K) 166 | Kiy = Ki.dot(y) 167 | norm = len(X) 168 | else: 169 | K = cache['K'] 170 | Ki = cache['Ki'] 171 | Kiy = cache['Kiy'] 172 | norm = cache['norm'] 173 | pre = Kiy.dot(Kiy.T) - Ki 174 | dK = self.K_grad(X) 175 | grads = {} 176 | grads['ls'] = np.zeros(len(dK['ls'])) 177 | for i in xrange(len(dK['ls'])): 178 | grads['ls'][i] = np.sum(np.diag(pre.dot(dK['ls'][i])))/norm/2 179 | grads['amp'] = np.sum(np.diag(pre.dot(dK['amp'])))/norm/2 180 | grads['noise'] = np.sum(np.diag(pre.dot(dK['noise'])))/norm/2 181 | return grads 182 | 183 | def K(self, X, grad=False): 184 | """ Computes the covariance matrix using X and returns the gradient 185 | w.r.t. ls, amp, and noise if desired. 186 | """ 187 | keps = self.params['keps'] 188 | noise = self.params['noise'] 189 | K = ( 190 | self.compute_kernel(X) + 191 | (keps + noise**2)*np.eye(len(X)) 192 | ) 193 | if grad: 194 | return K, self.K_grad(X) 195 | else: 196 | return K 197 | 198 | def K_grad(self, X): 199 | """ Computes the gradient of the covariance matrix element-wise 200 | w.r.t. ls, amp, and noise. 201 | """ 202 | ls = self.params['ls'] 203 | amp = self.params['amp'] 204 | noise = self.params['noise'] 205 | grads = {} 206 | # Compute dK w.r.t. noise 207 | grads['noise'] = 2*noise*np.eye(len(X)) 208 | # Compute dK w.r.t. amp 209 | sqdist = self.compute_sqdist(X/ls) 210 | grads['amp'] = 2*amp * np.exp(-0.5*sqdist) 211 | # Compute dK w.r.t. ls 212 | grads['ls'] = [] 213 | precompute = amp**2 * np.exp(-0.5*sqdist) 214 | for i in xrange(X.shape[1]): 215 | val = precompute/ls[i]**3 216 | val *= self.compute_sqdist(X[:, i, None]) 217 | grads['ls'].append(val) 218 | grads['ls'] = np.array(grads['ls']) 219 | return grads 220 | 221 | def compute_kernel(self, X, Y=None): 222 | """ Compue the kernel matrix for X x Y, keeping in mind the scaling 223 | w.r.t. ls 224 | """ 225 | ls = self.params['ls'] 226 | amp = self.params['amp'] 227 | noise = self.params['noise'] 228 | if Y is None: 229 | sqdist = self.compute_sqdist(X/ls) 230 | else: 231 | sqdist = self.compute_sqdist(X/ls, Y/ls) 232 | return amp**2 * np.exp(-0.5*sqdist) 233 | 234 | @staticmethod 235 | def compute_sqdist(X, Y=None): 236 | """ Compute the square distance between the rows of X v. Y 237 | """ 238 | if Y is None: 239 | a = (X**2).sum(axis=1).reshape(-1, 1) 240 | b = a.T 241 | c = X.dot(X.T) 242 | else: 243 | a = (X**2).sum(axis=1).reshape(-1, 1) 244 | b = (Y**2).sum(axis=1) 245 | c = X.dot(Y.T) 246 | return a + b - 2*c 247 | 248 | def predict(self, X_): 249 | """ Predict w.r.t. X_ 250 | """ 251 | # Push X_ into hypercube 252 | X_ = (X_ - self.transform['Xm'])/self.transform['Xs'] 253 | K_ = self.compute_kernel(self.X, X_) 254 | y_ = K_.T.dot(self.Ki).dot(self.y) 255 | # Compute variance 256 | var = self.params['amp']**2 - (K_.T * K_.T.dot(self.Ki)).sum(axis=1) 257 | var = (var * self.transform['ys']**2).reshape(-1, 1) 258 | # Convert y_ back to original input space 259 | y_ = y_ * self.transform['ys'] + self.transform['ym'] 260 | return y_, var 261 | 262 | def main(): 263 | import matplotlib.pyplot as plt 264 | # make some random data 265 | # np.random.seed(45) 266 | n = 10 267 | X = np.random.uniform(0., 1, (n, 1)) 268 | y = np.squeeze(np.sin(X*10)) + np.random.randn(len(X))*.1 269 | 270 | gp = GaussianProcess(ls=[.1], 271 | amp=.1, 272 | noise=2e-2) 273 | gp.fit(X, y, nepochs=100, batch_size=1000, 274 | learning_rate=1e-2, learning_rate_decay=.999, 275 | verbose=1) 276 | X_ = np.linspace(0, 1, 1000).reshape(-1, 1) 277 | y_, var = gp.predict(X_) 278 | plt.scatter(X, y, c='r') 279 | plt.plot(X_, y_ + var**.5, c='g') 280 | plt.plot(X_, y_ - var**.5, c='g') 281 | plt.plot(X_, y_, c='y', linewidth=2) 282 | plt.show() 283 | 284 | if __name__ == "__main__": 285 | main() 286 | 287 | 288 | -------------------------------------------------------------------------------- /tgp/tf/gaussian_process.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('/Users/void/Documents/github/tensorflow-gp') 3 | from tgp.np.gaussian_process import GaussianProcess as GP 4 | import numpy as np 5 | import tensorflow as tf 6 | 7 | class DoubleAdamOptimizer(tf.train.AdamOptimizer): 8 | def _valid_dtypes(self): 9 | return set([tf.float32, tf.float64]) 10 | 11 | class GaussianProcess(object): 12 | def __init__(self, ls=.1, amp=.1, noise=1e-2, sess=tf.Session(), dim=1): 13 | self.sess = sess 14 | self.ls = tf.Variable(np.array(ls)) 15 | self.amp = tf.Variable(np.array(amp)) 16 | self.noise = tf.Variable(np.array(noise)) 17 | # construct loss computation graph 18 | self.xp = tf.placeholder(tf.float64, [None, dim]) 19 | self.yp = tf.placeholder(tf.float64, [None, 1]) 20 | self.x_p = tf.placeholder(tf.float64, [None, dim]) 21 | self.L = self.construct_loss_graph() 22 | # construct prediction computation graph 23 | self.y_, self.var_ = self.construct_prediction_graph() 24 | # set up optimizer 25 | opt = DoubleAdamOptimizer( 26 | learning_rate=tf.constant(1e-3, tf.float64), 27 | beta1=tf.constant(0.9, tf.float64), 28 | beta2=tf.constant(0.999, tf.float64), 29 | epsilon=tf.constant(1e-8, tf.float64) 30 | ) 31 | self.opt_op = opt.minimize(self.L, var_list=[self.ls, self.amp]) 32 | self.grad_ls = tf.gradients(self.L, self.ls) 33 | self.grad_amp = tf.gradients(self.L, self.amp) 34 | self.init = tf.initialize_all_variables() 35 | 36 | def construct_covariance_graph(self, xs, ys=None): 37 | add_noise = True if ys is None else False 38 | ys = xs if ys is None else ys 39 | # Compute covariance matrix K 40 | xsq = tf.reduce_sum(tf.square(xs), 1) 41 | ysq = tf.reduce_sum(tf.square(ys), 1) 42 | xsq = tf.reshape(xsq, tf.pack([tf.shape(xsq)[0], 1])) 43 | ysq = tf.reshape(ysq, tf.pack([1, tf.shape(ysq)[0]])) 44 | sqdist = xsq + ysq - 2*tf.matmul(xs, tf.transpose(ys)) 45 | K = tf.square(self.amp) * tf.exp(-0.5*sqdist) 46 | if add_noise: 47 | ones = tf.ones(tf.pack([tf.shape(xs)[0]]), dtype=tf.float64) 48 | K = K + tf.diag(ones)*(1e-9 + tf.square(self.noise)) 49 | # compute loss 50 | Ki = tf.matrix_inverse(K) 51 | return K, Ki 52 | else: 53 | return K 54 | 55 | def construct_loss_graph(self): 56 | x = self.xp 57 | y = self.yp 58 | xs = x/self.ls 59 | K, Ki = self.construct_covariance_graph(xs) 60 | yT = tf.transpose(y) 61 | Kiy = tf.matmul(Ki, y) 62 | lK = tf.log(tf.matrix_determinant(K)) 63 | L = tf.matmul(yT, Kiy) + lK 64 | ones = tf.ones(tf.pack([tf.shape(xs)[0]]), dtype=tf.float64) 65 | L = L/tf.reduce_sum(ones) * 0.5 66 | return L 67 | 68 | def construct_prediction_graph(self): 69 | x = self.xp 70 | y = self.yp 71 | x_ = self.x_p 72 | xs = x/self.ls 73 | xs_ = x_/self.ls 74 | _, Ki = self.construct_covariance_graph(xs) 75 | K_ = self.construct_covariance_graph(xs, xs_) 76 | # compute variance 77 | K_T = tf.transpose(K_) 78 | K_Ki = tf.matmul(K_T, Ki) 79 | var_ = tf.square(self.amp) - tf.reduce_sum(K_T * K_Ki, 1) 80 | # compute prediction 81 | y_ = tf.matmul(K_T, tf.matmul(Ki, y)) 82 | return y_, var_ 83 | 84 | def solve(self, X, y, epochs=20, batch_size=50, train=True): 85 | self.X = X 86 | self.y = y 87 | self.sess.run(self.init) 88 | if train: 89 | iterations = epochs * len(X) / batch_size 90 | epochiter = iterations/epochs 91 | for i in xrange(iterations): 92 | idx = np.random.choice(np.arange(len(X)), batch_size, replace=False) 93 | X_mini = X[idx] 94 | y_mini = y[idx] 95 | fd = {self.xp: X_mini, self.yp: y_mini} 96 | if i % epochiter == 0: 97 | print self.sess.run(self.L, feed_dict=fd) 98 | self.sess.run(self.opt_op, fd) 99 | 100 | def predict(self, X_): 101 | fd = {self.xp: self.X, self.yp: self.y, self.x_p: X_} 102 | y_ = self.sess.run(self.y_, feed_dict=fd) 103 | var_ = self.sess.run(self.var_, feed_dict=fd) 104 | return y_, var_ 105 | 106 | def main(): 107 | import matplotlib.pyplot as plt 108 | dim = 1 109 | X = np.random.uniform(0, 1, (10, dim)) 110 | y = (np.sin((X[:, 0])*10) - X[:,0]*3).reshape(-1, 1) 111 | 112 | ls = [.1]*dim 113 | sess = tf.Session() 114 | gp = GaussianProcess(dim=dim, sess=sess, ls=ls, noise=0.1) 115 | gp.solve(X, y, epochs=100, batch_size=len(X), train=False) 116 | print sess.run(gp.ls) 117 | 118 | X_ = np.linspace(0, 1, 1000).reshape(-1, 1) 119 | y_, var_ = gp.predict(X_) 120 | std_ = np.sqrt(var_).reshape(-1, 1) 121 | 122 | plt.scatter(X, y) 123 | plt.plot(X_, y_, c='r') 124 | plt.plot(X_, y_+2*std_, c='g') 125 | plt.plot(X_, y_-2*std_, c='g') 126 | plt.show() 127 | 128 | 129 | if __name__ == "__main__": 130 | main() 131 | -------------------------------------------------------------------------------- /tgp/tf/troubleshoot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 8, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from gaussian_process import GaussianProcess as GP\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "import numpy as np\n", 14 | "import tensorflow as tf\n", 15 | "%matplotlib inline" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 24, 21 | "metadata": { 22 | "collapsed": false 23 | }, 24 | "outputs": [ 25 | { 26 | "data": { 27 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEACAYAAABVtcpZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE4pJREFUeJzt3X+s3fV93/Hny/WQnKRjTZM4FALNQjNW1IaxjTKlWk6z\ncm2qFOO6miBSY1I1oHZ0VeVtBpHJl2lTIJK7NkVdZkQrZ0pEqyEvhgYdXxbOJkuF0AZDSA046UKB\ngLewkJXWjVz7vT/Owbm5udf3x/fre+7x9/mQrvz9fs/nfj+fj+71ed3v5/P9fE+qCklSN60bdwMk\nSeNjCEhShxkCktRhhoAkdZghIEkdZghIUoe1EgJJ7klyNMmTpynziSRHkhxKclkb9UqSmmnrSuD3\ngE0LvZjkauBdVfUjwE3AJ1uqV5LUQCshUFUHgW+epsgW4FOjso8C5ybZ2EbdkqSVW605gfOB52ft\nvzg6JkkaIyeGJanD1q9SPS8C75i1f8Ho2PdI4sOMJGmZqior+b42rwQy+prPfuBDAEmuBF6tqqML\nnaiqzsqvXbt2jb0N9s/+2b+z76uJVq4EknwG6AE/mOTPgV3AOUBV1Z6q+lySn0nyFeAvgQ+3Ua8k\nqZlWQqCqPriEMje3UZckqT1ODK+iXq837iacUfZvstm/bkrT8aS2Jam11iZJWsuSUGtgYliSNGEM\nAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwBSeowQ0CSOswQkKQOMwQkqcMM\nAUnqMENAkjrMEJCkDjMEJKnDWgmBJJuTPJ3k2SQ753n9fUleTfLF0ddH26hXZ0a/32dqahtTU9vo\n9/vjbo6kM6jxx0smWQc8C/wz4OvAY8B1VfX0rDLvA3ZU1TVLOJ8fLzlG/X6frVu3c+zYnQCsW/fr\nvOc9P8rHPvZv2bRp05hbJ2k+4/54ySuAI1X1XFUdB+4FtsxTbkUN1OravXvPKAC2A9s5efI/8vjj\nJ9i6dbtXBdJZqI0QOB94ftb+C6Njc/2TJIeS/GGSH22hXq2aH+LYsTvZvXvPuBsiqWXrV6mePwEu\nrKq/SnI18N+Ad69S3VqGHTtu5ODB7Rw79vqRncBe4OXxNUrSGdNGCLwIXDhr/4LRsVOq6rVZ2w8m\n+Z0kb66q/zvfCaenp09t93o9er1eC83UUmzatIl9+/Zy660f44knnuLkyRuAl9mwYSc7duwdd/Mk\nAYPBgMFg0Mq52pgY/j7gGYYTwy8BXwCur6rDs8psrKqjo+0rgD+oqh9e4HxODK8R/X7/1BDQjh03\nOjEsrVFNJoYbh8CoAZuB32I4x3BPVd2R5CagqmpPkn8B/DJwHDgG/HpVPbrAuQwBSVqGsYdAmwwB\nSVqecd8iKkmaUIaAJHWYISBJHWYISFKHGQJr1KQ9xG3S2itpyLuD1qC5D3HbsGEn+/btXbP36U9a\ne6WzjbeInmWmprYxM3MNw4e4Aezlqqv2c+DAfeNs1oImrb3S2cZbRCVJK7JaD5DTMsx9iNtaf27P\npLVX0nc4HLRGTdpzeyatvdLZxDkBSeow5wQkSStiCEhShxkCktRhhoAkdZghIEkdZghIUocZApLU\nYYaAJHVYKyGQZHOSp5M8m2TnAmU+keRIkkNJLmujXklSM41DIMk64C5gE3ApcH2SS+aUuRp4V1X9\nCHAT8Mmm9UqSmmvjSuAK4EhVPVdVx4F7gS1zymwBPgVQVY8C5ybZ2ELdkqQG2giB84HnZ+2/MDp2\nujIvzlNGkrTK1uSjpKenp09t93o9er3e2NoiSWvNYDBgMBi0cq7GTxFNciUwXVWbR/u3AFVVd84q\n80ng4ar6/dH+08D7quroPOfzKaKStAzjforoY8DFSS5Kcg5wHbB/Tpn9wIfgVGi8Ol8ASJJWV+Ph\noKo6keRm4ADDULmnqg4nuWn4cu2pqs8l+ZkkXwH+Evhw03olSc35oTKSNOHGPRwkSZpQhoAkdZgh\nIEkdZghIUocZApLUYYaAJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdZgh\nIEkdZghIUocZApLUYYaAJHWYISBJHdbog+aT/ADw+8BFwNeAf15V35qn3NeAbwEngeNVdUWTeiVJ\n7Wh6JXAL8FBV/T3g88CtC5Q7CfSq6h8YAJK0djQNgS3A3tH2XuDaBcqlhbokSS1r+sb8tqo6ClBV\nLwNvW6BcATNJHkvykYZ1SpJasuicQJIZYOPsQwzf1D86T/Fa4DTvraqXkryVYRgcrqqDC9U5PT19\narvX69Hr9RZrpiR1xmAwYDAYtHKuVC30vr2Eb04OMxzrP5rk7cDDVfX3F/meXcBfVNVvLPB6NWmT\nJHVNEqoqK/nepsNB+4EbRtvbgc/OLZDkDUneNNp+IzAFPNWwXklSC5peCbwZ+APgHcBzDG8RfTXJ\necDdVfWBJO8E9jEcKloPfLqq7jjNOb0SkKRlaHIl0CgEzgRDQJKWZ5zDQZKkCWYISFKHGQKS1GGG\ngCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLUYYaAJHWYISBJHWYISFKHGQKS1GGG\ngCR1mCEgSR1mCEhShxkCktRhjUIgyc8neSrJiSSXn6bc5iRPJ3k2yc4mdUqS2tP0SuBLwFbgfyxU\nIMk64C5gE3ApcH2SSxrWK0lqwfom31xVzwAkOd2n3F8BHKmq50Zl7wW2AE83qVuS1NxqzAmcDzw/\na/+F0TFJ0pgteiWQZAbYOPsQUMBtVXX/mWjU9PT0qe1er0ev1zsT1UjSRBoMBgwGg1bOlapqfpLk\nYWBHVX1xnteuBKaravNo/xagqurOBc5VbbRJkroiCVV1umH5BbU5HLRQAx4DLk5yUZJzgOuA/S3W\nK0laoaa3iF6b5HngSuCBJA+Ojp+X5AGAqjoB3AwcAL4M3FtVh5s1W5LUhlaGg9rkcJAkLc9aGQ6S\nJE0YQ0CSOswQkKQOMwQkqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwB\nSWtGv99namobU1Pb6Pf7425OJ/gUUUlrQr/fZ+vW7Rw7Nvy8qQ0bdrJv3142bdo05patfU2eImoI\nSFoTpqa2MTNzDbB9dGQvV121nwMH7htnsyaCj5KWJK3Ioh80L0mrYceOGzl4cDvHjg33N2zYyY4d\ne8fbqA5wOEjSmtHv99m9ew8wDAXnA5bGOQFJ6rCxzQkk+fkkTyU5keTy05T7WpInkjye5AtN6pQk\ntafpnMCXgK3Af16k3EmgV1XfbFifJKlFja4EquqZqjoCLHYZkqZ1SW1yUZI0tFp3BxUwk+QEsKeq\n7l6leqXvMXdR0sGD212UpM5aNASSzAAbZx9i+KZ+W1Xdv8R63ltVLyV5K8MwOFxVBxcqPD09fWq7\n1+vR6/WWWI20uN2794wCYLgo6dix4TFDQJNiMBgwGAxaOdeiIVBVVzWtpKpeGv37f5LsA64AlhQC\nkqTvNveP49tvv33F52pzOGjeeYEkbwDWVdVrSd4ITAErb7HUkIuSpO9otE4gybXAbwNvAV4FDlXV\n1UnOA+6uqg8keSewj+EQ0nrg01V1x2nO6ToBnXEuStLZxMViktRhPkBOkrQihoAk0d21Iw4HSeq8\nSf9AG+cEJKmBSf9AG+cEJEkr4ofKSOq8Lq8dcThIkvjetSPAxKwlcU5Aklo0aRPFhoAktWjSJoqd\nGJYkrYgTw5I0R5cmih0OkqR5vD5R/I1vvAL8DW95y8Y1O0HsnIAknQGTMkFsCEjSGTApE8RODEuS\nVsSJYUlaQBcmiB0OkqTTmIRPoXNOQJI6zDkBSdKKNAqBJB9PcjjJoST3JfnbC5TbnOTpJM8m2dmk\nTklSe5peCRwALq2qy4AjwK1zCyRZB9wFbAIuBa5PcknDeiVJLWgUAlX1UFWdHO0+AlwwT7ErgCNV\n9VxVHQfuBbY0qVeS1I425wR+EXhwnuPnA8/P2n9hdEySNGaLrhNIMgNsnH0IKOC2qrp/VOY24HhV\nfaaNRk1PT5/a7vV69Hq9Nk4rSWeFwWDAYDBo5VyNbxFNcgPwEeD9VfXteV6/Epiuqs2j/VuAqqo7\nFzift4hK0jKM7RbRJJuBfw1cM18AjDwGXJzkoiTnANcB+5vUK0lqR9M5gd8G3gTMJPlikt8BSHJe\nkgcAquoEcDPDO4m+DNxbVYcb1itJaoErhiVpwrliWJK0IoaAJHWYISBJHWYISFKHGQKS1GGGgCR1\nmCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLUYYaAJHWYISBJHWYISFKHGQKS1GGGgDRm\n/X6fqaltTE1to9/vj7s56phGHy+Z5OPAzwLfBr4KfLiq/t885b4GfAs4CRyvqitOc04/XlKd0e/3\n2bp1O8eO3QnAhg072bdvL5s2bRpzyzRJmny8ZNMQ+Gng81V1MskdQFXVrfOU+zPgH1bVN5dwTkNA\nnTE1tY2ZmWuA7aMje7nqqv0cOHDfOJulCTO2zxiuqoeq6uRo9xHgggWKpmldkqT2rW/xXL8I3LvA\nawXMJDkB7Kmqu1usV5pYO3bcyMGD2zl2bLi/YcNOduzYO95GqVMWHQ5KMgNsnH2I4Zv6bVV1/6jM\nbcDlVbVtgXOcV1UvJXkrMAPcXFUHFyjrcJA6pd/vs3v3HmAYCs4HaLnGNicwqvwG4CPA+6vq20so\nvwv4i6r6jQVer127dp3a7/V69Hq9Rm2UpLPJYDBgMBic2r/99tvHNjG8GdgN/NOqemWBMm8A1lXV\na0neCBwAbq+qAwuU90pAkpZhnHcHHQHOAV4PgEeq6leSnAfcXVUfSPJOYB/DIaT1wKer6o7TnNMQ\nkKRlGOtwUNsMAUlanrHdIipJmmyGgCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLU\nYYaAJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkd1igEkvy7JE8k\nOZTkoSQXLFBuc5KnkzybZGeTOiVJ7Wl6JfDxqnpPVV0GfBaYnlsgyTrgLmATcClwfZJLGtY7kQaD\nwbibcEbZv8lm/7qpUQhU1Wuzdt8IfGOeYlcAR6rquao6DtwLbGlS76Q6238J7d9ks3/dtL7pCZL8\ne+BDwF8BPzFPkfOB52ftv8AwGCRJY7bolUCSmSRPzvr60ujfnwWoqo9W1YXA7wG/eaYbLElqT6qq\nnRMl7wA+V1U/Nuf4lcB0VW0e7d8CVFXducB52mmQJHVIVWUl39doOCjJxVX1ldHutcCheYo9Blyc\n5CLgJeA64PqFzrnSjkiSlq/pnMAdSd4NnAD+DPhlgCTnAXdX1Qeq6kSSm4EDDIef7qmqww3rlSS1\noLXhIEnS5BnriuEkP5DkQJJnkvSTnDtPmQuSfD7Jl0eT0v9yHG1djqUsjkvyiSRHRgvtLlvtNjax\nWP+SfHC0iPCJJAeT/Nh851mrlrq4Mck/TnI8yc+tZvuaWOLvZi/J40meSvLwarexiSX8bv5gkgdH\n/+++lOSGMTRzxZLck+RokidPU2Z57y1VNbYv4E7g34y2dwJ3zFPm7cBlo+03Ac8Al4yz3Yv0aR3w\nFeAi4G8xnCe5ZE6Zq4E/HG3/BPDIuNvdcv+uBM4dbW8+2/o3q9x/Bx4Afm7c7W7xZ3cu8GXg/NH+\nW8bd7pb7twv42Ot9A14B1o+77cvo408ClwFPLvD6st9bxv3soC3A3tH2XoaTy9+lql6uqkOj7deA\nwwzXHqxVS1kctwX4FEBVPQqcm2Tj6jZzxRbtX1U9UlXfGu0+wtr+ec211MWNvwr8V+B/r2bjGlpK\n3z4I3FdVLwJU1XwLQNeqpfTvZeD7R9vfD7xSVX+zim1spKoOAt88TZFlv7eMOwTeVlVHYfhmD7zt\ndIWT/DDDFHz0jLds5eZbHDf3TXBumRfnKbNWLaV/s/0S8OAZbVG7Fu1fkh8Crq2q/wRM0t1sS/nZ\nvRt4c5KHkzyW5BdWrXXNLaV/dwOXJvk68ATwa6vUttWy7PeWxiuGF5NkBpidRAEK+Og8xRecpU7y\nJoZ/ef1afffjKrRGJfkp4MMML2HPJr/JcPjydZMUBItZD1wOvJ/ho2D+KMkf1XduBZ90twJPVNVP\nJXkXMJPkx7v8nnLGQ6CqrlrotdEEx8aqOprk7SxwaZ1kPcMA+C9V9dkz1NS2vAhcOGv/gtGxuWXe\nsUiZtWop/SPJjwN7gM1VdbrL17VmKf37R8C9ScJwXPnqJMerav8qtXGlltK3F4BvVNVfA3+d5H8C\n72E41r7WLaV/7wX+A0BVfTXJ/wIuAf54VVp45i37vWXcw0H7gRtG29sZPol0Pr8L/GlV/dZqNKqh\nU4vjkpzDcHHc3DeH/Qyft/T6iupXXx8WmwCL9i/JhcB9wC9U1VfH0MYmFu1fVf3d0dc7Gf5x8isT\nEACwtN/NzwI/meT7kryB4eTipKzrWUr/DgM/DTAaK383wzVOkyQsfPW5/PeWMc90vxl4iOEdPweA\nvzM6fh7wwGj7vQwXox0CHge+yPCvy7HP1J+mX5tHfToC3DI6dhNw46wydzH86+oJ4PJxt7nN/jEc\nd31l9LN6HPjCuNvc9s9vVtnfZULuDlpq34B/xfAOoSeBXx13m9vsH8Mrt/tH/++eBK4fd5uX2b/P\nAF8Hvg38OcPh1kbvLS4Wk6QOG/dwkCRpjAwBSeowQ0CSOswQkKQOMwQkqcMMAUnqMENAkjrMEJCk\nDvv/00C6zpicXT4AAAAASUVORK5CYII=\n", 28 | "text/plain": [ 29 | "" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | } 35 | ], 36 | "source": [ 37 | "# X = np.linspace(0., 1., 300).reshape(-1, 1).astype(np.float64)\n", 38 | "dim = 1\n", 39 | "X = np.random.uniform(0, 1, (10, dim))\n", 40 | "y = (np.sin((X[:, 0])*10) - X[:,0]*3).reshape(-1, 1)\n", 41 | "plt.scatter(X, y)\n", 42 | "plt.show()" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 37, 48 | "metadata": { 49 | "collapsed": false, 50 | "scrolled": true 51 | }, 52 | "outputs": [ 53 | { 54 | "name": "stdout", 55 | "output_type": "stream", 56 | "text": [ 57 | "[[-0.15558379]]\n", 58 | "[[-0.16170398]]\n", 59 | "[[-0.16779481]]\n", 60 | "[[-0.17385392]]\n", 61 | "[[-0.17987888]]\n", 62 | "[[-0.18586716]]\n", 63 | "[[-0.19181617]]\n", 64 | "[[-0.19772328]]\n", 65 | "[[-0.20358581]]\n", 66 | "[[-0.20940105]]\n", 67 | "[[-0.21516629]]\n", 68 | "[[-0.22087881]]\n", 69 | "[[-0.22653591]]\n", 70 | "[[-0.2321349]]\n", 71 | "[[-0.23767316]]\n", 72 | "[[-0.2431481]]\n", 73 | "[[-0.24855719]]\n", 74 | "[[-0.25389799]]\n", 75 | "[[-0.25916812]]\n", 76 | "[[-0.26436531]]\n", 77 | "[[-0.2694874]]\n", 78 | "[[-0.27453232]]\n", 79 | "[[-0.27949814]]\n", 80 | "[[-0.28438304]]\n", 81 | "[[-0.28918533]]\n", 82 | "[[-0.29390346]]\n", 83 | "[[-0.29853604]]\n", 84 | "[[-0.30308177]]\n", 85 | "[[-0.30753955]]\n", 86 | "[[-0.31190839]]\n", 87 | "[[-0.31618746]]\n", 88 | "[[-0.32037607]]\n", 89 | "[[-0.32447368]]\n", 90 | "[[-0.32847988]]\n", 91 | "[[-0.33239443]]\n", 92 | "[[-0.3362172]]\n", 93 | "[[-0.33994819]]\n", 94 | "[[-0.34358756]]\n", 95 | "[[-0.34713557]]\n", 96 | "[[-0.3505926]]\n", 97 | "[[-0.35395917]]\n", 98 | "[[-0.35723589]]\n", 99 | "[[-0.36042348]]\n", 100 | "[[-0.36352276]]\n", 101 | "[[-0.36653465]]\n", 102 | "[[-0.36946014]]\n", 103 | "[[-0.37230032]]\n", 104 | "[[-0.37505636]]\n", 105 | "[[-0.37772949]]\n", 106 | "[[-0.380321]]\n", 107 | "[[-0.38283225]]\n", 108 | "[[-0.38526466]]\n", 109 | "[[-0.38761969]]\n", 110 | "[[-0.38989885]]\n", 111 | "[[-0.39210368]]\n", 112 | "[[-0.39423576]]\n", 113 | "[[-0.39629672]]\n", 114 | "[[-0.39828818]]\n", 115 | "[[-0.40021181]]\n", 116 | "[[-0.40206928]]\n", 117 | "[[-0.4038623]]\n", 118 | "[[-0.40559257]]\n", 119 | "[[-0.40726179]]\n", 120 | "[[-0.40887168]]\n", 121 | "[[-0.41042396]]\n", 122 | "[[-0.41192034]]\n", 123 | "[[-0.41336254]]\n", 124 | "[[-0.41475226]]\n", 125 | "[[-0.41609118]]\n", 126 | "[[-0.41738098]]\n", 127 | "[[-0.41862333]]\n", 128 | "[[-0.41981988]]\n", 129 | "[[-0.42097225]]\n", 130 | "[[-0.42208205]]\n", 131 | "[[-0.42315085]]\n", 132 | "[[-0.42418023]]\n", 133 | "[[-0.42517169]]\n", 134 | "[[-0.42612676]]\n", 135 | "[[-0.42704689]]\n", 136 | "[[-0.42793353]]\n", 137 | "[[-0.42878809]]\n", 138 | "[[-0.42961194]]\n", 139 | "[[-0.43040641]]\n", 140 | "[[-0.43117282]]\n", 141 | "[[-0.43191243]]\n", 142 | "[[-0.43262647]]\n", 143 | "[[-0.43331614]]\n", 144 | "[[-0.43398258]]\n", 145 | "[[-0.43462691]]\n", 146 | "[[-0.43525021]]\n", 147 | "[[-0.43585352]]\n", 148 | "[[-0.43643784]]\n", 149 | "[[-0.43700413]]\n", 150 | "[[-0.43755331]]\n", 151 | "[[-0.43808626]]\n", 152 | "[[-0.43860383]]\n", 153 | "[[-0.43910683]]\n", 154 | "[[-0.43959602]]\n", 155 | "[[-0.44007215]]\n", 156 | "[[-0.44053592]]\n" 157 | ] 158 | } 159 | ], 160 | "source": [ 161 | "sess = tf.Session()\n", 162 | "gp = GP(sess=sess, ls=.1, noise=0.1, amp=1.)\n", 163 | "gp.solve(X, y, epochs=100, batch_size=len(X), train=True)\n", 164 | "X_ = np.linspace(0, 1, 1000).reshape(-1, 1)\n", 165 | "y_, var_ = gp.predict(X_)\n", 166 | "std_ = np.sqrt(var_).reshape(-1, 1)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 38, 172 | "metadata": { 173 | "collapsed": false 174 | }, 175 | "outputs": [ 176 | { 177 | "data": { 178 | "text/plain": [ 179 | "" 180 | ] 181 | }, 182 | "execution_count": 38, 183 | "metadata": {}, 184 | "output_type": "execute_result" 185 | }, 186 | { 187 | "data": { 188 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEACAYAAABVtcpZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FcX6wPHvJLTQe+9dAenVKAEJRZSmVynSuw0Br8IV\nBfXykyKigIhIl67Se40ISL2EAFJCkd5LaCHtvL8/NoSApJ6TbMr7eZ7zcHZ3dudNgPOendmZMSKC\nUkqp1MnN7gCUUkrZR5OAUkqlYpoElFIqFdMkoJRSqZgmAaWUSsU0CSilVCrmkiRgjJlqjLlsjPGL\npsw4Y4y/McbXGFPFFfUqpZRyjqvuBKYDTaI6aIxpBpQSkTJAb2CSi+pVSinlBJckARHZCtyMpkhL\nYFZ42Z1ANmNMPlfUrZRSKv4Sq0+gEHA20vb58H1KKaVspB3DSimViqVJpHrOA0UibRcO3/cPxhid\nzEgppeJIREx8znPlnYAJfz3NMqATgDGmDnBLRC5HdSERSZavoUOH2h6Dxm9/HBp/8nwl5/id4ZI7\nAWPMXMALyGWMOQMMBdIBIiKTRWSVMeZlY8xx4B7Q1RX1KqWUco5LkoCItI9FmXddUZdSSinX0Y5h\nF/Ly8rI7BKdo/PbS+O2V3OOPL+Nse5KrGWMkqcWklFJJmTEGSQIdw0oppZIZTQJKKZWKaRJQSqlU\nTJOAUkqlYpoElFIqFdMkoJRSqZgmAaWUSsU0CSilVCqWWLOIKhe4HXSb1f6r2XJ6C0evH+Xyvcu4\nG3dyeOSgYp6K1CtSj5fLvEy2DNnsDlUplUzoiOFk4PDVw4zYNoKlR5biWdSTBsUbUClfJfJnzk+Y\nI4xr969x4MoBNp3axJbTW2hZviX96/SnWoFqdoeulEoEzowY1iSQhN16cIv+a/uz8thKPqjzAT2r\n9SRPpjzRnnMz8CY//e8nxu0cR90idRnVaBQlcpRIpIiVUnbQaSNSoD9O/0GVSVXImCYjx98/zn9e\n+E+MCQAgh0cOPnr+I469d4zn8j5HjZ9qMHzLcEIdoYkQtVIqudE7gSQmJCyEz3//nKn7pvLTqz/x\nStlXnLremYAzdF/WnYAHAcxqPYvyucu7KFKlVFKhdwIpxPEbx/Gc7snei3vZ13uf0wkAoGi2oqx7\nax1dq3TlhekvMO/APBdEqpRKKVySBIwxTY0xR4wxx4wxHz/leH1jzC1jzP/CX0NcUW9KISJM3zed\nulPr0qFSB1a1X0X+zPmdvu7atWtp3Pg1mjR5nZI3SrK+43o+3fwp7616j+CwYBdErpRK7pxuDjLG\nuAHHgJeAC8BuoK2IHIlUpj4wUERaxOJ6qao56GbgTXqv6M3ha4eZ22YulfJVitP5InD7Nty7B+nS\nQc6c4OZmJYDWrTsTGDgSADe3/nzxxUDeGfgOXZZ04XrgdZa8uYRcGXMlxI+llEpEzjQHuWKcQC3A\nX0ROhwczH2gJHHmiXLwCTMlW+6+m14petC7fmpmtZuKR1iPGc0Rg+3ZYvRo2b4bDhyEkBDJlguBg\nuHsXnnkGbtyAwMBZQGMAHA747LOB1KhRg0VvLmLQhkHUm1aP1R1WUzJHyQT+SZVSSZUrmoMKAWcj\nbZ8L3/ekusYYX2PMSmPMsy6oN9m6eOci3ZZ2o+/KvsxoOYNxzcbFmABu34bRo6FcOejVy/pQ//JL\nOHrUOnbpkvXBf+cOTJkCHh6XgcjjBNxwOMowZsxk3Iwbo7xH0a92PzynebLnwp4E/XmVUklXYo0Y\n3gsUFZH7xphmwBKgbFSFhw0bFvHey8srWaz96RAHPn/74FnUk3Tu6Z5a5viN4/y450em+U6ja5Wu\n+PX1I2v6rNFe9/59GD8exowBb2+YORPq1AETxX1V+vRQsyaMH5+Pl18ui8PxM1AZqAPcA9ZHlH27\n5tsUylKI5nObs6LdCmoWqhmvn10plbh8fHzw8fFxybVc0SdQBxgmIk3DtwcBIiIjoznnFFBdRG48\n5Viy7BO4cu8KLee35Mi1I3gW9eTZ3M+SwyMHDnFw+tZp/jz3JxfvXqRL5S68U+sdimcvHuM116yB\nt9+GatWsb/3PPBO3mIYPH85nn43F4SgD9MSYJrzxRhCzZ5ckTaT0v+LYCrot7cbydsupXbh23CpR\nStnO1hHDxhh34ChWx/BFYBfQTkQORyqTT0Quh7+vBSwUkeJRXC9ZJoGHLt29xPaz2zl89TB3gu8A\nUCRrEWoUrEH1gtVJ4xbzzVdAALzzDvz5J0ycCE2axD+etWvXMmbMZAC6d3+XKVMaEBYGixdDtkhT\nDK08tpJuy7qxufNmns2TqlvrlEp2bJ82whjTFPgOq49hqoiMMMb0xrojmGyMeQfoC4QAgUB/EdkZ\nxbWSdRJw1s6d0K4dNG0KX38NGTO69vphYdCvH2zbZt1p5Mv36NjP+3/m082fsr37dgpmKejaipVS\nCcb2JOBKqTUJiMDYsTByJPzwA7RpE9/rCMFhwaRzT4eJouNABD7/HBYsgD/+gNy5Hx0bvmU4vx3+\nja3dtpIxrYszkFIqQWgSSOaCgqB3b9i/H5YsgWLFYn/u1XtXWXp0KVtOb2HX+V2cCThDqCOUMAmj\nYJaC1CxYkzbPtKHNM23+8aH+n//A+vWwaRNkyWLtExE6Lu6Iu5s7M1rOiDKRKKWSDk0CydiVK9C6\nNRQoYD35kylTzOeEOcJY6b+SCbsmsOv8LpqWbkqjko2oVagWJXOUJHO6zIQ5wjgdcJptZ7Yx9+Bc\n/C77Maz+MHpU6xHxwS4CffrA6dOwciW4u1vXvxd8j9pTavN+7ffpVb1XAv70SilX0CSQTB09arX9\nd+wIw4ZZI32jExIWwtR9Uxm9fTS5PHLRr3Y/2jzTJlaDzPZc2MM7q94hl0cuZrWeRe6MVhtQaKjV\n8Zw790lu3vw3AAMH9qJEjRJ4TvNkQ6cNPJfvOWd/VKVUAtIkkIw8fFonIKAU/v5fMmZMerp2jf4c\nhzhYeGghQzYNoWSOkgzzGkbdwnXj3FQTEhbC4I2DWXFsBes6rqNotqIALFy4ibZtSyNyEjiNh8fH\nLF48kwt5LzBu1zh29tgZ5dgHpZT9nEkCiEiSelkhpUxr1qwRD498AmsF7ku6dB1lzZo10Z7je9FX\n6k6pKzUn15SNJze6JI5vtn8jxb8tLudvnxcREW/vNgLLxGogEoEZ4u3dRhwOh7wy9xX5dNOnLqlX\nKZUwwj834/WZq1NJJ6IxYyYTGDgfaz4fD4KDX4p4hv9J94LvMWDtALx/9qZb1W7s6LGDhiUauiSO\n/nX707NaT5rNacbtoNvheyOP27O+UBhjmPzKZH7c+yN7L+x1Sd1KqaRFk0AiOnfOG2u+vejtPr+b\nqj9W5er9qxx6+xA9qvXAzbj2r2qw52BqFaxFj2U9GDCgJx4eHwOzgEukSXOGgQOtDuECWQow4qUR\nvL3qbRzicGkMSin7aRJIJGPGwM2bnUifvgEwE5iJh8fHER+2YLX9j9g6guZzm/Pfhv/l59Y/x2pJ\nyfgwxjD+5fEcv3Ec/xz+LF48E2/vpXh6DiNjxo8oUeLRMOXOVTrjbtyZtm9agsSilLKPdgwnMBFr\n3p85c2DjRjh06NE0DgMH9qJJ+JwQ94Lv0WVpF87fPs+C1xdQJFuRRInvxI0T1Jlahy1dtvBMHmty\norFjYflyK96Hfc++l3xpMrsJf739l65BoFQSo08HJVEiMGgQrFoFGzY8PkVDZJfuXqLZnGZUyV+F\nSc0nkT5N+kSN84fdPzDLbxZbu27F3c2d0FCoXRvefx86d35U7t1V7+Jm3BjXbFyixqeUip6uMZwE\nORzWh+jGjeDjE3UCOBtwlhenv0ib8m2Y1mJaoicAgN41epPePT3f7fwOgDRpYPJk+PhjuHXrUbmh\n9Ycy98BcTtw4kegxKqUSht4JJICwMOjZE44ds0biRp6tM7ILdy7gOc2T92q9R/+6/RM3yCccv3Gc\nOlPq4NfXL2LyuB49rOUqR416VO7L37+0lsJ8ba5NkSqlnqTNQUlISAh06mRNB7FsWdTTQAQ8CODF\nGS/StkJbBr8wOHGDjMKgDYO4dPcSM1rNAODiRahUCXbtgpLhK1DeDb5L2fFlWdF+BdUKVIv6Ykqp\nRKNJIIkICoI337SmYvjlF/CIYjaHUEcozeY0o1yucoxvNj7JTNJ2J+gO5SaUY0nbJdQqZD3KOnw4\n+PpaP89D3+/6nlXHV7Gy/UqbIlVKRaZ9AknA/fvQogWkTQuLFkWdAACGbh4KwHdNv0syCQAgS/os\nDG84nH5r+j0cvc2AAdYaB9u3PyrXvVp3fC/58r+L/7MpUqWUq2gScJGQEHj+eZg3D9JFM83Oav/V\nzPKbxZw2c3B3c0+8AGOpc5XO3A+5z/JjywErmQ0dCp9++qhMhjQZ+He9f/PfLf+1KUqllKu4JAkY\nY5oaY44YY44ZYz6Oosw4Y4y/McbXGFPFFfUmJdmywWef8djavU+6dv8a3ZZ1Y26bueTNlDfxgosD\nN+PGF15f8OnmTyNGCHfqZE03HXld617Ve7H97HYOXjloT6BKKZdwOgkYY9yACUAToALQzhhT/oky\nzYBSIlIG6A1Mcrbe5Kjfmn60r9ieF4q9YHco0WpRrgXp3NPx61+/AlYT12efWa+H3TUZ02ZkQN0B\nDP9juI2RKqWc5Yo7gVqAv4icFpEQYD7Q8okyLbEmpkGstYWzGWOieHI+ZVp2dBk7z+3ky4Zf2h1K\njIwx/LfBf/ls82eEOkIBaN8eLl+2xj081LdGX9afWM+pm6dsilQp5SxXJIFCwNlI2+fC90VX5vxT\nyqRYgSGBvLf6PX569adks25v41KNyZMpD/MPzgesZq5hwx6/G8iSPgvdq3Zn3E4dQaxUchVNC7Z9\nhg0bFvHey8sLLy8v22Jxha+3f02tQrVoUKKB3aHEmjGGIS8MYeC6gbSv1B4348Ybb1iJYMsWqF/f\nKvdurXep8mMVhnkNI1uGKEbFKaVcysfHB5/InXROcHqcgDGmDjBMRJqGbw/CWuBgZKQyk4DNIrIg\nfPsIUF9ELj/lesl2nMDTnLt9jsqTKrO3116KZy9udzhxIiJUn1ydYV7DaFGuBQBTpliPwK5a9ahc\nu9/aUbNgTQbUHWBTpEqlbnaPE9gNlDbGFDPGpAPaAsueKLMM6AQRSePW0xJASjR442D6VO+T7BIA\nWP+wBnkO4qutX0WMG+jYEfbvt14P9a/Tn3E7x0X0Hyilkg+nk4CIhAHvAuuAQ8B8ETlsjOltjOkV\nXmYVcMoYcxz4EXjb2XqTgwOXD7DuxDoGeQ6yO5R4e+2Z17h+/zpbTm8BIH16+OCDx+cTqlWoFoWy\nFmLJkSU2RamUii+dNiIBvbbwNeoVrsfAegPtDsUpU/43hV//+pU1b60B4PZtay6h3buhRAmrzMJD\nC5m4eyI+XXzsC1SpVMru5iD1FPsu7mPHuR30rdnX7lCc1vG5jvhd9osYGJY1qzVL6tixj8q0Kt+K\nI9eOcPjqYZuiVErFhyaBBPKZz2cMen5QsnkkNDrp06Snb42+jz0K+s47MHu2dVcAkM49Hd2qdmPy\n3sk2RamUig9NAglgz4U9+F7ypWf1nnaH4jK9a/Tml79+4dr9awAULgze3jBjxqMyPav15Ge/nwkM\nCbQnSKVUnGkSSAAjt43kw7ofkiFNBrtDcZm8mfLSunzrx77pv/8+jB9vraIGUCJHCWoWqskvf/0S\nxVWUUkmNJgEX87/uj8/fPnSv1t3uUFyuX+1+fL/7e0LCQgCoV8/qH1iz5lGZPtX7MGlPqpwaSqlk\nSZOAi329/Wv61uhL5nSZ7Q7F5Srnr0zZXGUjJpYzBvr1g3GRZo1oXrY5ZwLO4HfZz6YolVJxoUnA\nhS7dvcTCvxbybq137Q4lwXxQ+wPG7hgbMXjszTetlceOHLGOp3FLQ/eq3flp7082RqmUii1NAi40\nbuc42ldsn2TXCnCFV8q+wrX719h9YTdgDR7r2RMmTHhUpnOVzsw/NJ+g0CCbolRKxZYmARe5E3SH\nyXsnJ/uBYTFxd3Ond/Xej7X79+plrah27561XTJHSSrkqcBKf12DWKmkTpOAi7gZN6a1nEbJHCXt\nDiXBda3alUWHF3Ez8CYARYpYncQLFz4q06VKF2b4zrAnQKVUrGkScJFM6TJFzLSZ0uXNlJeXy7zM\nrP2zIvb17g2TI40Te/3Z19lyeguX76aKeQKVSrY0Cah46VujL5P2ToroIG7aFM6dA7/wh4Iyp8tM\nq/KtmHNgjo1RKqVioklAxYtnUU/cjTu/n/4dsFYe69Hj8buBh01CKWVCQKVSIk0CKl6MMfSp0Ycf\n9vwQsa9798c7iF8s9iK3g27je8nXpiiVUjHRJKDireNzHVl3Yh2X7l4CrPmEnn/+UQexm3Gjc+XO\n2kGsVBKmSUDFW7YM2Xj9mdeZtm9axL5eveDHHx+V6VS5E/MOzouYakIplbQ4lQSMMTmMMeuMMUeN\nMWuNMU9dadwY87cxZr8xZp8xZpczdaqkpU+NPkzeO5kwRxgAzZpZHcSHDlnHS+UsRamcpdhwcoON\nUSqlouLsncAgYIOIlAM2AYOjKOcAvESkqojUcrJOlYRUL1idnB452XhqIwDu7tY6xJGnmG5fsT1z\nD861J0ClYhDmCOOlWS9xO+i23aHYwtkk0BKYGf5+JtAqinLGBXWpJKpHtR5M3Tc1YrtLF2vBmdDw\ndeffqPAGy48u537IfXsCVCoaG09t5HbQbbKmz2p3KLZw9oM5r4hcBhCRS0BUk+YIsN4Ys9sYk3JW\nWlEAtK/UnrXH10YsOFOunLX28MMppvNlzkedwnVYdnSZjVEq9XQ/+/1Mx+c62h2GbdLEVMAYsx7I\nF3kX1of6kKcUj+qB8OdF5KIxJg9WMjgsIlujqnPYsGER7728vPDy8oopTGWj7Bmy80rZV5jtN5sP\n6nwAQNeuVpPQK69YZdpXas/cA3NpW7GtfYEq9YS7wXdZfnQ5YxqPsTuUOPHx8cHHx8cl1zLODOQx\nxhzGauu/bIzJD2wWkWdiOGcocEdEvoniuOjgouRn86nNvL/mffz6+GGMISAAihWD48chd264HXSb\nImOLcPL9k+TKmMvucJUCYNb+WSw8tJAV7VfYHYpTjDGIiInPuc42By0DuoS/7wwsfbKAMSajMSZz\n+PtMQGPgoJP1qiSmfvH6BIYEsuu89fBXtmzWXcC8edbxrOmz0qRUE347/JuNUSr1uJ/9fqZT5U52\nh2ErZ5PASMDbGHMUeAkYAWCMKWCMeZha8wFbjTH7gB3AchFZ52S9KolxM250q9rtHx3E06c/KtOh\nUgfmHtCnhFTScP72efZe2MurZV+1OxRbOdUclBC0OSj5On/7PBV/qMjZ/mfJnC4zDofVQbxsGVSu\nDEGhQRT8piC+vX0pkq2I3eGqVG70ttEcvX6UKS2m2B2K0+xsDlIqQqGshfAs6skvh34BwM0NOnV6\nNGYgfZr0tCnfhvkH59sXpFLhUvtTQQ9pElAu1b1q9380Cc2ZA8HB1naH5zrowDFlu/2X9hMQFMAL\nxV6wOxTbaRJQLtW8THOO3zjOkWvWyvOlSkHZsrAuvBfoxWIvcvXeVQ5fPWxjlCq1m7V/Fm9Vegs3\nox+B+htQLpXWPS2dK3dm6v8e3Q289Rb8/LP13s248WaFN5l3cJ5NEarULtQRytyDc+lYWZuCQJOA\nSgDdqnZj9oHZhDqseSPeeMMaPRwQYB1vV6kd8w7O08VmlC02nNxA4ayFKZ+7vN2hJAmaBJTLlctd\njuLZi7P2+FoAcuaEhg1h0SLrePUC1QHYc2GPXSGqVGy673S6VelmdxhJhiYBlSC6VO7CjP0zIrbf\nesuaVA6sx9naVWynTUIq0V2/f521x9fSrlI7u0NJMjQJqATxZsU3WXdiHdfvXwegeXPw9bXWGgBo\nV7EdCw4tiFiHQKnEMPfAXJqXbU72DNntDiXJ0CSgEkT2DNlpVrpZxJiADBngtddgbvjToc/keYa8\nmfLyx5k/bIxSpTbTfKdpU9ATNAmoBNOlShdm7p8ZsR25SQisu4F5B7RJSCWOfRf3cTPwJg1KNLA7\nlCRFk4BKMN4lvTl/5zyHrlhrTXp6Wk8I+flZx9tWbMtvh38jOCzYxihVajFt3zS6VumqYwOeoL8N\nlWDc3dzp+FzHiLsBN7fH7waKZitK+dzlWXdC5xNUCetB6APmHZxHlypd7A4lydEkoBJU58qd+dnv\n54gxAx06WNNIhIX3B7ev1F6fElIJbumRpVQtUJVi2YvZHUqSo0lAJahn8jxDsWzFIr7tP/ss5M8P\nDxdF+tez/2LlsZW6/rBKUFP3TdUO4ShoElAJrkuVLszwnRGxHblJKE+mPNQpXIflR5fbE5xK8U7c\nOMG+S/to/Uxru0NJkpxKAsaY140xB40xYcaYatGUa2qMOWKMOWaM+diZOlXy82aFN1l7Yi03A28C\n0K4dLFkC98O//OvAMZWQftz7I50rdyZDmgx2h5IkOXsncABoDfweVQFjjBswAWgCVADaGWN00o5U\nJIdHDpqWbhoxZiB/fqhdG5aHf/lv/UxrNv+9OSJJKOUqQaFBzPCdQe/qve0OJclyKgmIyFER8Qei\nW9GmFuAvIqdFJASYD7R0pl6V/DxtGomHM4tmTZ8V75LeLDq8yJ7gVIr161+/UiV/FcrkKmN3KElW\nYvQJFALORto+F75PpSLepbw5G3CWv67+BUCrVrB1K1y5Yh3XJiGVEH7Y8wN9avSxO4wkLcYkYIxZ\nb4zxi/Q6EP5n6l6dWcVJGrc01pgBX2vMQObM8OqrsGCBdfzlMi+z9+JeLt29ZGOUKiU5cPkAp26d\nokW5FnaHkqSliamAiHg7Wcd5oGik7cLh+6I0bNiwiPdeXl54eXk5GYJKCjpX6UyjWY0Y/tJw0ril\n4a234LPP4L33wCOtBy3KtWDhoYW8X/t9u0NVKcCkPZPoWa0nadxi/JhLdnx8fPB5+Jy1k4wrFvYw\nxmwGPhSRvU855g4cBV4CLgK7gHYi8tT1BY0xoouNpFy1p9RmWP1hNCvTjNBQKFLEGjNQrhysOb6G\nYT7D2NFjh91hqmTudtBtin9bHL++fhTOWtjucBKcMQYRia5vNkrOPiLayhhzFqgDrDDGrA7fX8AY\nswJARMKAd4F1wCFgflQJQKV8nSt3jphGIk0a63HROXOsYy+VeImTN09y8uZJGyNUKcG0fdPwLuWd\nKhKAs1xyJ+BKeieQst0IvEGJ70rwd7+/yeGRg337rCmmT5wAY+Cdle9QKGsh/vPCf+wOVSVTYY4w\nSo8vzbzX5lGncB27w0kUtt0JKBVXOT1y0qRUk4gxA1WqgIcHbNtmHX+4/rBS8bXkyBLyZ86fahKA\nszQJqEQXeZ0BY6Bjx0fTSNQrUo+ABwEcvHLQxghVcjZ2x1gG1BlgdxjJhiYBlegal2rMmYAzHL5q\ndQ21bw+//AJBQeBm3Ghbsa0uNqPiZff53Zy7fU7nCYoD7RNQtvh4vTWF1EjvkQA0aADvvw+tW1sr\nQL228DVOvH8CY+LVzJlqnQ04y0r/lew4t4ODVw5y/s55bgbeJDgsmPRp0pMpbSbyZ85PoayFKJyl\nMOVzl6dC3gpUyFOBItmKJPsFV9r/1p7qBaozsN5Au0NJVM70CWgSULb46+pfNJrViDP9z5DGLQ1T\np8LKlbBoEYgIz058luktp2u7biyEhIWw6PAixu0ax9FrR2lWphkvFn2RSvkqUTRbUXJkyEE693QE\nhQVxL/geF+9e5MKdCxF3Y4euHuLQ1UM8CH3A80We54WiL9CkdBMq5a2UrJLwiRsnqD2lNifeP0G2\nDNnsDidRaRJQyVLkMQMBAVCsGJw8CTlzwhe/f8H1+9f5rtl3doeZZIkIq/xXMXDdQPJmyssHdT6g\nRbkW8R4cdeHOBbae2crvf//OCv8VeKTxoG3FtvSu3psCWQq4OHrX67W8F/ky5ePLhl/aHUqi0ySg\nkqUfdv+Az2kfFrxuzR3x5pvQsCH07g3+1/15ccaLnOt/Dnc3d5sjTXpuBt6kz8o++F7y5ZvG3/By\nmZdd+q1dRNh1fhcz989k3sF5tC7fmmFewyiarWjMJ9vgbMBZKk+qzLH3jpE7Y267w0l0+oioSpba\nVmzL2uOP1hmIPLNomVxlKJSlED5/+9gXYBK158IeqvxYhXyZ8uHb25fmZZu7vNnGGEPtwrWZ2Hwi\nx987TuGshan6Y1WGbh5KUGiQS+tyhdHbR9O9avdUmQCcpUlA2ebhOgMPxwU0bQpHj1pNQmCtPzzn\nwBwbI0x6lh5ZSrM5zRjbZCzjmo3DI61HgteZK2MuvmjwBb69fdl/eT+1p9ROUo/wXrp7idl+s1Nd\nZ7CraBJQtoq89GTatFaT0MNpJNpXas/iI4u5F3zPvgCTkFn7Z9F3ZV9WtV9Fm2faJHr9RbIVYfGb\ni+lXux8NZjZg4aGFiR7D04zeNpoOlTqQP3N+u0NJljQJKFt5l/Tm/J3zHLpyCHg0cEwE8mfOj2dR\nT37961ebo7TfbL/ZDN44mE2dN1GzUE2XXHPt2rU0bvwajRu/xtq1a2N1jjGGrlW7sqHjBj5a/xFf\n/P4Fdvbhnb51mhn7Z/DJi5/YFkNyp0lA2crdzd1aZyB8BHGtWlYC2L3bOt61Slem+063MUL7LTmy\nhI/Wf8T6juspn9s1K7OuXbuW1q07s359C9avb0Hr1p1jnQgAKuevzM4eO1l8ZDEfrvvQtkQw1Gco\nfWv01bsAZ4hIknpZIanU5PDVw5L/6/wSEhYiIiKffy7y3nvWsaDQIMkzKo8cv37cxgjts+f8Hsk9\nKrfsOb/Hpdf19m4jMEOslCsCM8Tbu02cr3Pj/g2pObmmvLPyHXE4HC6NMSYHLh+QPKPyyK3AW4la\nb1IU/rkZr89cvRNQtiufuzzFsxdn3Yl1AHToAPPnQ0gIpHNPR4dKHSL6DVKTc7fP0XJ+Sya/Mpnq\nBau75JpXr8Jvv4G//1vAi5GOvMjx4+1ZsAAuX4799XJ45GBDpw3sOr+LTzd/6pIYY2vwxsEM9hyc\n6gaGuZrzpzlhAAAgAElEQVQmAZUkdKncJaLZp1QpKFMG1lk5ga5VuzJz/0zCHGE2Rpi4gkKDaL2g\nNe/Ves/peXBCQ2HePGjUyPq9TpsGL75YkbRpRwPrgQ2kTTsKT8/KzJ1rLfBTuzZMnAgBATFfP2v6\nrKxsv5KFhxYyYdcEp2KNrdX+qzl89TB9a/ZNlPpStPjeQiTUC20OSpVuBt6UrF9llWv3romIyMSJ\nIm+++eh4tR+rydrja22KLvG9v+p9aT2/tVNNLA6HyJw5IqVLi9SvLzJ/vkhg4KPja9asEW/vNuLt\n3UbWrFkTsT8kRGTVKpE33hDJlUvkk09Erl2Lub6TN05KwTEFZfHhxfGOOTYCQwKl9LjSsurYqgSt\nJznBieYgZz+wXwcOAmFAtWjK/Q3sB/YBu2K4ZgL9mlRS1/bXtjJh5wQRsT50smUTuXnTOjZh5wRp\n+2tbG6NLPIv+WiTFvy0uN+7fiPc1/P1FGjYUqV5dZNOm+Mdy4oRIz54iefKITJhgJYjo7D6/W3KP\nyi0HLx+Mf6Ux+PL3L6XV/FYJdv3kyM4kUA4oA2yKIQmcBHLE8poJ8ktSSd8a/zVSY3KNiO1//cu6\nIxARuX7/umT7KptTH4zJwambpyTPqDyy4+yOeF9j7lyR3LlFxoyJ+UM7tg4cEPHyEqlcWcTXN/qy\ns3xnSanvSsn1+9ddU3kkx64dk1wjc8mpm6dcfu3kzJkk4FSfgIgcFRF/IKYx6wbtf1AxaFSyERfv\nXIwYjdq9O0ydah3L6ZGTZmWa8bPfzzZGmLAc4qDr0q4MrDuQ2oVrx/n80FBrOu5PP4UNG2DAAGsd\nZ1eoWBE2bYL+/a2+hVGjICyKLpqOlTvSslxL2v7allBHqGsCwFo2svOSznxW/zOKZy/usuumdon1\nwSzAemPMbmNMz0SqUyUz7m7udKrciZm+1piBsLC1HDp0lbp1B7B27Vr61ujLD3t+eHjHmOL8sPsH\nAkMC+bDeh3E+NzDQWqv52DHYswcqV3Z9fMZA587WGI5Vq8DbG65ceXrZkd4jEYTBGwa7rP6vt39N\nhjQZeLfWuy67piLm5iCsxwf8Ir0OhP/5aqQym4m+OahA+J95AF/AM5qyCXbLpJK+I1ePSP6v88uK\nVSvEwyOfgK/AIfHwyCerV6+WCt9XkE0nnWjkTqJO3DghuUbmksNXD8f53Js3RTw9Rdq3FwkKSoDg\nniI0VOQ//xEpUkRk586nl7l275qU/K6kzPGb43R9f579U/KMyqPNQFHAieagGG8WRcTbBYnmYvif\nV40xi4FawNaoyg8bNizivZeXF15eXs6GoJKJcrnLUSJ7CYbMGE5g4EjA+kobGDiab775ib7/te4G\nGpRoYG+gLuQQB92WdmOQ56A4jwi+c8eaeK9mTfjuO3BLpHt7d3cYPtyq95VX4KuvrOa7yHJlzMWS\nN5fQcFZDyucuT7UC1eJV1+W7l/nXL/9iSosp2gwUzsfHBx8fH9dcLL7ZQx7/9r4ZqB7FsYxA5vD3\nmYBtQONorpUAeVIlJz/u+VHyvlvwiRGtm8Xbu40EPAiQ7COyy4XbF+wO02W+3/W91J1SV0LDQuN0\n3r17Ii++KNK7t/U4aHzcD74vR68dlb0X9srx68fjHIOIyJEjImXKiHz8sUhY2D+P/3LoFyk2tphc\nuXslztd+EPJA6k+vL59s/CTO56YmOHEn4NSiMsaYVsB4IDdwC/AVkWbGmALATyLyijGmBLAYq18g\nDTBHREZEc01xJiaV/AU8CKDQ14UI+yYjD26OBkrg5hbKqlUhNGnShN7Le1M4a2E+rZ+4I1QTwqW7\nl6j0QyV8OvtQIW+FWJ8XGgotW1qrsM2cGbc7gOCwYBYcXMCM/TPYeW4n+TPnJ0v6LNwMvMm1+9fw\nLOrJGxXe4I0Kb5A5XeZYXfPaNWjVCgoWtOLxeGKG6082fsL2c9tZ99Y60rqnjd3P6AjlzV/fxM24\nMf+1+bq4UDR0ZTGV4nRY1IHsd7LjP+cSYWHp2L17JgcPpqNoUfC95Mur817lVL9T8V5KManosKgD\nhbMUZqT3yFifIwLvvgvHj1vrMsflCaBV/qt4b/V7FM9enHdrvot3Ke/HPuhvPbjFuhPrmHtgLtvO\nbuPtGm8zoO6AWE3N8OABdOsGp07B0qWQN++jY2GOMFrMb0HRrEWZ2HxijIvgBIcF02VJF67dv8by\ndstJnyZ97H/IVMiZJOCS5iBXvtDmICUi646vk+o/Vo/Y7ttX5IsvHh2vO6Wu/PbXbzZE5jobT26U\nomOLyt2gu3E677vvRJ59VuRWHOZNCwoNkvdWvSclvi0ha/zXxHyCiBy9dlQ6L+4s+b/OL1P2TolV\nU1FYmDXCuGRJq5nooTVr1ohX01cl68Ds0mpSq2hHQl++e1lemvmStJzXUu4H349VrKkddg0WS4iX\nJgElIhIaFiqFvyksfpf8RERkzx6R4sUftTnPPzBfPKd52hihcx6EPJBy48vJksNL4nTeypUi+fOL\nnDwZ+3NuBd6SF6e/KK/OfVVuBt6MY6TWKOB6U+tJ7Z9qy6Erh2J1ztSpInnzimzZYiUA60mvGUKG\n78WtZ1qp810duXz38mPnhDnCZK7fXCnwdQEZtH5QvPonUitnkoA2B6kka8imIdwNvsu3Tb9FBKpX\nh//7P+tpmFBHKKXHlWbhvxZSq1Atu0ONs+FbhrPz/E6WtVsW63NOnIC6dWHJEqhXL3bn3Ai8QZPZ\nTahdqDbjmo3DzcTv8SGHOPhxz498uvlTBtQdwL/r/TvGtv1166x1o/PlG8vBgzmBztYB96kU6zqS\nO2Wu07xMc8rmKsvlu5dZ6b+SPJny8LX317xQ7IV4xZlaaZ+ASpFO3TxFrSm1ONv/LBnSZGDKFFi2\nzHoBjP1zLDvO72DB6wvsDTSOTt48Sc2farK3195YP/IYGGh98Hftao0Kjo07QXfwmulFw+INGeU9\nyiWL0Z++dZpeK3px7f41prWYRuX80Y9KO3AAatW6yoMHZ4CH02HPxNt7GdN/HcfyY8s5E3CGXB65\neKnkS1TOV9klcaY22iegUizvWd4Rg43u3hXJmVPk1CnrWMCDAMk5MmeyGkDkcDik+Zzm8n9b/i9O\n5/Xsac3qGdtHQYNDg6Xp7KbSc1lPly/24nA4ZNr/pkmeUXnkPxv+I4EhgdGWnz17sxhzQOCowEzx\n8Mj32KylynnoojIqpepVvReT904GIFMmaw3iydYmWdNnpVuVbozbOc7GCONm+bHlHL9xnIH1Bsb6\nnJkzYcsWmDLFmrohJiLC2yvfxs24xepJnLh6uM7w/j77OXr9KJUnVWbL6S1Rlu/QwYvffrtErlx3\nyZWrIh9+OJAxYybHaW3jhBQcDJ99Bnfv2h2JTeKbPRLqhd4JqEiCQoMk7+i8cvTaURGxnjjJm1fk\nwQPr+JlbZyTHiBzx6vBMbPeD70uJb0vI+hPrY33OkSPWjKAHDsS+nh/3/CgVJ1aUO0F34hFl3C36\na5EUGlNIei/vHe3MoSEhIs2anQm/K1goMMP2u4IHD0RatrRewcG2heE09E5ApVTp3NPRpXIXftr7\nE2CtelWpkrVEIkCRbEV4pewrfL/rexujjJ2R20ZSvWB1GpVsFKvyISFWx+qXX1qzeMbGngt7+GTT\nJ/z2xm+xHujlrNbPtObg2wdxN+6Um1COb/78hqDQoH+US5MGQkI+QCQI+BfQmcDAkYwZMzlR4nzS\nvXvw6quQNi0sXGj9mRppElBJXo9qPZjlNyvig+Xtt62lDx/65IVP+G7nd9wJumNThDE7efMk43eN\nZ0zjMbE+54svrAFXvXvHrvyNwBu8vvB1JjWfRNlcZeMZafxkz5Cd75t/z+9dfmfTqU08O/FZ5h+c\nj0Mcj5WzWqYORtpTHjueA7l+HRo3hkKFrKU306VL/BiSjPjeQiTUC20OUk/RYEYDWXBwgYhYzQqF\nCj2+uEn739rLV398ZVN0MWsxr4UM3zI81uW3bxfJl0/k4sXY19FpcSd5Z+U78YjO9Tac2CA1JteQ\nqpOqyhr/NRGd04+NGeA3gf2SPft6WbQo9k1kzjp40BrM9tFHT5/rKDlCB4uplG6u31xpNKtRxPaX\nX4p06fLo+KErhyTv6LyJ1g4eFyuPrZTS40rLg5AHsSp/545IqVIiixbFvo5Vx1ZJ8W+LJ6mf3+Fw\nyC+HfpGy48uK1wyviNXS1qxZI1Wr1hc3t1wCHwscFWNOy4gRuxI8pkWLrKUyZ85M8KoSlSYBleIF\nhgRK7lG55fj14yJirUGcPbvIhUiTib7xyxsyausomyJ8usCQQCn1Xak4LYreo4dI166xryPgQYAU\nHVtU1h1fF48IE15IWIhM3jNZCo0pJK3nt5YTN06It3ebJ2aJXS/p01+Tvn1FbiTACqK3b4t0727d\nAeyI/8qdSZYzSUD7BFSykCFNBjo+15Ep/5sCQK5c0L49TJjwqMynL37KmD/HcDvotk1R/tOY7WOo\nmLcizco0i1X5Zctg40ZrbYDYGrxhMI1KNMK7lNNLfySING5p6Fm9J/7v+VOzYE3qTq3L1RyXnih1\nnrp1++NwQPny8P33Vse4s0SsNv8KFaz3vr5QO+4rd6Zs8c0eCfVC7wRUFA5fPSx5R+eNaFbx97ce\nn7wbaf61Tos7yZCNQ2yK8HF/3/xbco7MKSdvxG6in0uXrHmBtm6NfR2///27FBxTMFk8IvvQtjPb\nJMfwHJK2arbwu4HHHxXdv1/E21ukaFFrsrw78WjhCgsTWbJEpHZtkapVrTmMUjK0OUilFt6zvGWW\n76yI7datRSZMeHT89K3TknNkTjl/+7wN0T3icDik5byWMmzzsFiWF3nlFZHBg2Nfx/3g+1JmXBlZ\nfHhxPKO0z/5L+yXH8BxS7XVP8fZu89SxAjt3irz2mki2bCLt2on88ovI5ctPuVi44GDrnP/8x2r2\nqVFDZP58aynMlM6ZJKBzB6lkZdnRZQz/Yzg7e+wEYNs2a/Hzo0etJQ8BPl7/MTcCb/BTi59si3PR\n4UV8sukTfHv7xmou/MmT4ccf4c8/Y/+44kfrP+J0wOlkN3fSQ5tPbabdb+3Y0WNHtHMoXb1qPce/\nahVs3w7Zs0OJEpAnj7WYzr17cO4cHDsGpUpBkybQti1UrRq7EdYpgW0TyBljRgGvAkHACaCriPyj\nQdYY0xT4FmtcwlQRiXIFDU0CKjphjjBKjy/NgtcXUKtQLUSsidX694c33rDK3Ay8SbkJ5djUeRMV\n88ZylJUL3Xpwi4oTKzL/9fl4FvWMsby/vzU76JYt8Oyzsatjz4U9NJ/bnAN9D5A3U96YT0iixmwf\nw2+Hf+OPrn/EauWwsDA4eRJOn7ZWMxOxVjErXBjKlIFsMa99kyLZNoEc0AhwC38/AvjqKWXcgONA\nMSAt4AuUj+aaLr5RUinN6G2j5a1Fb0Vsr1wpUrHi4898f7/re3lh2gsunzwtNvos7yO9l/eOVdmQ\nEKvd+rvvYn/9oNAgqTSxkszePzueESYdYY4wqT+9vozeNtruUJI17Ho6SEQ2iEQMCdwBFH5KsVqA\nv4icFpEQYD7Q0pl6VerWrWo3VhxbweW7lwFo1gwyZIDFix+V6V29N4GhgczaPytRY9t2ZhvLji1j\nRKMol9F+zP/9H2TNai0XGVsjto6gSLYitK/UPp5RJh1uxo1pLacxYusIjl0/Znc4qZIrHxHtBqx+\nyv5CwNlI2+fC9ykVLzk9cvL6M6/z0/+sNn9jrFkgv/gCHOFfSdzd3Pmh+Q98vMHqH0gM94Lv0XVp\nV8Y3G0/2DNljLL9zp/Uo5PTpsV8o/tCVQ4zfNZ5JzSelmHn3S+YoyWDPwfRb0+9ha4BKRDH2CRhj\n1gP5Iu8CBPhERJaHl/kEqCYirz3l/NeAJiLSK3z7LaCWiDx1aQxjjAwdOjRi28vLCy8vr7j8TCoV\n2H9pP83nNudUv1OkdU+LhK889tln0KrVo3LvrnqXwJBApracmuAx9V3Rl/uh95nZamaMZe/ehWrV\nrDuB11+P3fXDHGE8P+15ulbpSu8asZxQKJkIDgum8qTKjGo0ilfLvWp3OEmej48PPj4+Eduff/65\nPX0C4QmkC7ANSB/F8TrAmkjbg4CPo7meC1vKVErWYEYD+Xn/zxHbixeLVK78eN9AwIMAKfFtCVl6\nZGmCxrLy2EopNraY3AqM3ervvXqJdO4ctzrGbB8jXjO8JMyRQia8ecLa42ul5HclY1ykRv0Tdo0T\nAJoCh4Bc0ZRx51HHcDqsjuFnoimfUL8nlcKs9l8tlSZWiuj8dTisTtaff3683B+n/5D8X+eXS3cu\nJUgcF25fkAJfFxCfUz6xKr90qUjx4iIBAbGv4/j145JrZC7xv+4fzyiTh5bzWsqIP0bYHUayY2cS\n8AdOA/8Lf00M318AWBGpXFPgaHj5QTFcM+F+UypFcTgcUmliJVntvzpi3x9/WCNNA5/4Mjl4w2Bp\nNruZy79FB4cGi+c0T/nc5/NYlX84KjguI1gdDoc0mNFAvt72dTyjTD6OXD0iuUfljnZxGvVPziQB\nHSymkrXZfrOZtm8amzpvitjXurX13P1HHz0qFxwWTKNZjahfrD5fNvzSZfX3W92Pk7dOsrTtUtxM\n9L27Dge88gpUqWL1BcTW5L2TmbpvKtu7bY/Vs/TJXe/lvcmWIRujvEfZHUqyoQvNq1QrODRYinxT\nRHadezQN8ZEjIrlyiVy9+njZy3cvS9GxRWXhwYUuqXvirolSZlyZWM/bM2KESN26cVvG8GzAWck9\nKrccuByH9SWTuQu3L0jOkTnlzK0zdoeSbKCziKrUKq17WgbUHcCo7Y++NZYrBx06wKBBj5fNmykv\ni99czDur3mHjyY1O1bvw0EKG/zGcNW+tidXjoFu3wtixsGBB7JcxFBH6rOjDuzXftWXks10KZClA\n3xp9GeozNObCynnxzR4J9ULvBFQc3Qm6I3lH55VDVw5F7Lt1S6RgQauP4Em///275BmVRzaf2hyv\n+hYcXCB5R+cV34u+MRcWkStXRIoUsUY2x8UcvzlSaWIlCQoNikeUydutwFuSd3TeVHUH5Ax0FlGV\n2o3cOlLe+OWNx/YtWCBSocLTm182ntwoeUbleWxG0pg4HA4Z++dYKTSmkOy/tD9W54SGijRpIvLx\nx7GuRkRELt25JPlG53usmSu1+Wb7N9JiXgu7w0gWnEkC2jGsUoR7wfcoNa4U6zuup1K+SoA1udjL\nL1sTzH366T/POXTlEC3mt6B+sfp80+SbaJt1Lt65yDur3uHvW3/z2xu/USJHiVjF9eGHsG8frFkT\nt2ag1xa+Rvnc5fm/l+LQg5zCPAh9QLkJ5Zj32jzqFalndzhJmjMdw9onoFKETOky8dHzHz3WjmwM\n/PQTjB8Pu3f/85wKeStYUz27p6fM+DIM8xnGkWtHHt6R4hAHfpf9+Pe6f1Pph0qUy1WOP7v/GesE\nMHMmLFkCv/wS+wQAsODQAo5eP8rQ+qm7TTxDmgwMqz+MwRsHo18ME47eCagU437IfUqPK83ydsup\nXrB6xP6FC2HIEOsbeaZMTz/36LWjjNs5jqVHl3I3+C45PHJw/f51cmfMzZsV3qRPjT4Uy14s1rFs\n22Y9qurjE/vpoQEu371M5UmVWd5uOTUL1Yz9iSlUqCOU5354jm+afEPT0k3tDifJsm09gYSgSUA5\nY9KeSSw4tIBNnTY9NsFa587WncH06dEvNCIi3Ai8QUBQANkzZCenR844x3DgADRqZN0JNI3D55Y2\nAz3d4sOL+WLLF+zttTfGsRiplTYHKRWuR7UeXL13laVHlz62//vvrTuBceOiP98YQ66MuSiZo2S8\nEsDJk9bU1t9+G7cEADDnwBxtBnqKVuVbkc49HQsPLbQ7lBRJk4BKUdK4peGbJt/w4boPCQoNitif\nOTMsXQojRsDatQlT96lT4O1tjU9o1y5u5564cYL+a/szt83cWC1HmZoYY/jqpa8YsmkIIWEhdoeT\n4mgSUClO41KNKZ+7PON3jX9sf/Hi8Ouv0LGjtZSjKx05AvXrW8tcxmWBGICQsBDaL2rPkBeGUDl/\nZdcGlkI0LNGQkjlKMm3fNLtDSXG0T0ClSMeuH6Pe1Hrs7bX3Hx26mzZZC5H/+iu8+KLzdf3+u3W9\nESOsvoe4GrxhMH5X/FjRbkWKWSgmIey5sIeW81vi/54/GdNmtDucJEX7BJR6QtlcZelfpz99Vvb5\nx+OFDRvCvHnWYi6zZ8e/DhH47jtrgftZs+KXANYcX8Msv1lMbzldE0AMahSswfNFnmf8zvExF1ax\npncCKsUKCQuhxk81+LDuh3Ss3PEfxw8dghYtrKTwzTeQJUvsr33yJPTsCbdvW/MBlSwZ9/hO3DhB\nvWn1+PVfv/JCsRfifoFU6Oi1o3hO9+TYu8fI4ZHD7nCSDL0TUOop0rqnZVqLaXy4/kPOBJz5x/EK\nFcDX1/pG/8wzMGkSBAU95UKRnDkD/fpBjRrW0z9//hm/BHAv+B6tF7Tmsxc/0wQQB+Vyl6NVuVaM\n2qbTTLuK3gmoFG/E1hEsP7Ycn84+pHV/+tDd3but9Yl374bq1c9x7txSPDyu0KFDM0qXrsP+/bBx\nI/j5Wc0+H38M+fI99VIxcoiDtr+2JVO6TExrMU2bgeLo3O1zVJ5UmYN9D1IgSwG7w0kSbBssZowZ\nBbwKBAEngK4icvsp5f4GAgAHECIitaK5piYB5VIOcdBsTjOq5a/GV42+irbsrFm/06PHOkJCugEZ\nMeYC1asXpUGDPDz/PDRpAhkyxD8WEWHA2gHsvbiXdR3XkSGNExdLxf697t/cC7nHxOYT7Q4lSbCz\nOWgdUEFEqmAtHTk4inIOwEtEqkaXAJRKCG7GjZ9b/8y8g/OY7Rd9T/Ds2eMICSkLlAIKIHKQHDn6\nMGoUtGzpXAIAGPPnGNafXM/Stks1AThhkOcgFh5ayPEbx+0OJdlzKgmIyAYRcYRv7gAKR1HUOFuX\nUs7ImykvK9qvYMDaAWw57eJBArE0ac8kxu8az5q31minppNyZcxF/zr9+WzzZ3aHkuy58oO5G7A6\nimMCrDfG7DbG9HRhnUrFWsW8FZnTZg6vL3ydP8/++dQyAwf2wsPjY2AmMBMPj48ZOLCX03WP2zmO\nEVtHsKnTJgpnjeq7koqLfnX6sfnvzfhe8rU7lGQtxj4BY8x6IHIXmMH6UP9ERJaHl/kEqCYir0Vx\njQIictEYkwdYD7wrIlujKCtDhz6aO8XLywsvL6/Y/0RKxWCV/yq6LOnCojcX4VnU8x/H165dy5gx\nkwErKTRp0iTedTnEwec+nzP7wGw2ddoUp5lIVcwm7JrAimMrWPPWGrtDSVQ+Pj74+PhEbH/++ef2\nzSJqjOkC9AQaikgMD9iBMWYocEdEvoniuHYMqwS37sQ63lr0Fl83/ppOlTslSB33Q+7TdWlXzgSc\nYfGbi8mfOX+C1JOaBYcF89wPzzGm8Rial21udzi2sa1j2BjTFPg30CKqBGCMyWiMyRz+PhPQGDjo\nTL1KOatxqcZs7ryZL37/gndWvsPd4Lsuvf6+i/uo9VMt0runZ3PnzZoAEkg693R82/RbPlj7wWMT\nBqrYc7ZPYDyQGau9/3/GmIlgNf8YY1aEl8kHbDXG7MPqPF4uIuucrFcpp1XIW4HdPXdzL+Qez/3w\nHCuOrXB6Bau7wXcZsmkITWY3YbDnYGa2mqlPASWwpqWb8myeZxm7Y6zdoSRLOlhMKWC1/2r+vf7f\nZMuQjY/qfUTzss1J45Ym1uffenCLafumMebPMTQs0ZCvXvpKO4AT0YkbJ6g9pTb7++ynUNZCdoeT\n6HRlMaVcIMwRxoJDC5iwawKnA07TslxLmpRqQo2CNSiYpeBjI3sd4uDvW3+z49wOlh1dxtoTa2lW\nuhkD6w58bGlLlXiGbBrCqVunmNNmjt2hJDpNAkq52OGrh1lxbAXrT67H77If90PukytjLjKlzcTd\n4Ltcu3+NnB45qVagGs3LNOfVcq9qu7/N7gXf45nvn2Fmq5k0KNHA7nASlSYBpRLYzcCb3Hxwk3vB\n98iSPgs5PXKSNX1Wu8NST1h+dDkD1g3Ar48fHmk97A4n0WgSUEqpcG1/bUuxbMUY6T3S7lASjSYB\npZQKd+XeFSr9UIlV7Velmv4ZXU9AKaXC5c2Ul9Heo+m+rDvBYcF2h5PkaRJQSqU4HZ/rSJFsRfh0\n06d2h5LkaRJQSqU4xhimtZjG7AOz2XByg93hJGmaBJRSKVKeTHmY0XIGXZZ04eq9q3aHk2RpElBK\npVjepbxpX6k9nZZ0IswRZnc4SZImAaVUija84XAehD5gyKYhdoeSJGkSUEqlaGnd0/LLv35h/qH5\nzDswz+5wkhxNAkqpFC93xtwsbbuUfmv6se3MNrvDSVI0CSilUoXn8j3H7DazabOwjS5JGYkmAaVU\nqtG4VGO+f/l7Xp7zMn9d/cvucJKE2E+YrpRSKcDrz75OUGgQDWc2ZFm7ZdQqVMvukGzl7PKSXxhj\n9htjfI0xG4wxT11FwxjT1BhzxBhzzBjzsTN1KqWUszo814EpLabwytxXWHlspd3h2MqpCeSMMZlF\n5G74+/eAyiLS44kybsAx4CXgArAbaCsiR6K4pk4gp5RKFH+e/ZN//fIvelbryaf1P8XNJM8Wctsm\nkHuYAMJlAq49pVgtwF9ETotICDAfaOlMvUop5Qp1i9Rld8/d+N/w507QHbvDsYXTfQLGmP8CnYD7\nQO2nFCkEnI20fQ4rMSillO0KZCnA7Daz7Q7DNjEmAWPMeiBf5F2AAJ+IyHIRGQIMCW/r/xbo6mxQ\nw4YNi3jv5eWFl5eXs5dUSqkUw8fHBx8fH5dcy2WLyhhjigCrRKTSE/vrAMNEpGn49iBAROSpy/5o\nn4BSSsWNbX0CxpjSkTZbAU8bgbEbKG2MKWaMSQe0BZY5U69SSinXcLZPYIQxpiwQBpwE+gIYYwoA\nP+fdrbIAAAYaSURBVInIKyISZox5F1iHlXSmishhJ+tVSinlArrGsFJKJXO6xrBSSql40SSglFKp\nmCYBpZRKxTQJKKVUKqZJQCmlUjFNAkoplYppElBKqVRMk4BSSqVimgSUUioV0ySglFKpmCYBpZRK\nxTQJKKVUKqZJQCmlUjFNAkoplYppElBKqVTMqUVljDFfAC2x1hy+BnQRkXNPKfc3EAA4gBAR0YXm\nlVIqCXD2TmCUiFQWkSrAUmBYFOUcgJeIVE3JCcBVCz/bReO3l8Zvr+Qef3w5lQRE5G6kzUxYdwNP\nY5ytKzlI7v+INH57afz2Su7xx5ezawxjjPkv0Am4D9SOopgA640xYcBkEfnJ2XqVUko5L8Zv58aY\n9cYYv0ivA+F/vgogIkNEpCgwHfg2iss8LyLVgJeBd4wxni77CZRSSsWbyxaaN8YUAVaJSKUYyg0F\n7ojIN1Ec11XmlVIqjuK70LyzTweVFpHj4ZutAN+nlMkIuInIXWNMJqAx8HlU14zvD6KUUirunO0T\nGGGMKQuEASeBvgDGmALATyLyCpAPWBz+DT8NMEdE1jlZr1JKKRdwWXOQUkqp5MfWxzaNMTmMMeuM\nMUeNMWuNMdmeUqawMWaTMeZQeKf0+3bE+kRMTY0xR4wxx4wxH0dRZpwxxt8Y42uMqZLYMUYnpviN\nMe2NMfvDX1uNMdH28yS22Pz+w8vVNMaEGGPaJGZ8MYnlvx8vY8w+Y8xBY8zmxI4xKrH4t5PLGLM6\n/N/9AWNMFxvCjJIxZqox5rIxxi+aMkn5/2608cfr/66I2PYCRgIfhb//GBjxlDL5gSrh7zMDR4Hy\nNsbsBhwHigFpsfpByj9RphmwMvx9bWCHnb/neMRfB8gW/r5pcos/UrmNwAqgjd1xx/H3nw04BBQK\n385td9xxiH0o8NXDuIHrQBq7Y48UnydQBfCL4niS/b8by/jj/H/X7gFcLYGZ4e9nYnUuP0ZELomI\nb/j7u8BhoFCiRfhPtQB/ETktIiHAfKyfI7KWwCwAEdkJZDPG5EvcMKMUY/wiskNEAsI3d2Dv7/tJ\nsfn9A7wH/ApcSczgYiE28bcHfhOR8wAiEtUgzMQWm9gvAVnC32cBrotIaCLGGC0R2QrcjKZIUv6/\nG2P88fm/a3cSyCsil/n/du4eNIogDOP4/8FgZSFB/DZ+EgQhQhAUYhNRiJ2lFkEFURAkjYWCrb2C\nIERII4iNRaIgqCDYBI0YL4UWEgRNAoKCZcTitZg9PWPudjfxdjbs+4PA3WaKJ8tO3t3ZmSH8swfW\nt2osaQehCr5se7LmtgCfG77P8O+JXthmdpE2sWTJ3+gc8LitifJJzS9pM3DCzG4TVquXSZbz3w10\nSnouaULSYGHpWsuS/Q6wT9IcUAOGCsr2v5S57+aVqe8ue8VwGklPCTOEfh8irCC+tkjzpm+pJa0h\n3NkN2d/bVbg2kdQPnCU8gq4kNwjDi3VlKwRpOoBe4AhhO5ZxSeP2Zzp2mV0FambWL2k3YaeAHu+z\nxcrTd9teBMzsWLPfJS84NpjZF0kbafLoLqmDUADumtlom6JmNQt0NXzfmhxb2GZbSptYsuRHUg8w\nDAyYWavH56JlyX8AuC9JhHHp45J+mtlYQRlbyZJ/BvhqZvPAvKQXwH7CeHxMWbL3AdcBzGxa0kdg\nL/C6kITLV+a+m0nevht7OGgMOJN8Pk3YiXQxI8A7M7tZRKgUE8AeSdslrQZOEv6ORmOE/ZSQdAj4\nXh/2KoHU/JK6gAfAoJlNR8jYSmp+M9uV/Owk3DxcLEkBgGzXzyhwWNKqZLHlQcK7sNiyZH8PHAVI\nxtK7CWuIykQ0fzosc9+ta5p/SX038pvuTuAZYcbPE2BtcnwT8Cj53EdYjPYWmATeECpczNwDSeYP\nwJXk2AXgfEObW4Q7txrQGzNv3vyEcd1vybmeBF7Fzpz3/De0HaFEs4NyXD+XCTOEpoBLsTPnuHbW\nAQ+T634KOBU784L894A54AfwiTBkspL6bsv8S+m7vljMOecqLPZwkHPOuYi8CDjnXIV5EXDOuQrz\nIuCccxXmRcA55yrMi4BzzlWYFwHnnKswLwLOOVdhvwBXwQL5a+rYPQAAAABJRU5ErkJggg==\n", 189 | "text/plain": [ 190 | "" 191 | ] 192 | }, 193 | "metadata": {}, 194 | "output_type": "display_data" 195 | } 196 | ], 197 | "source": [ 198 | "plt.plot(X_, y_)\n", 199 | "plt.plot(X_, y_ + 2*std_, 'g')\n", 200 | "plt.plot(X_, y_ - 2*std_, 'g')\n", 201 | "plt.scatter(X, y)" 202 | ] 203 | } 204 | ], 205 | "metadata": { 206 | "kernelspec": { 207 | "display_name": "Python 2", 208 | "language": "python", 209 | "name": "python2" 210 | }, 211 | "language_info": { 212 | "codemirror_mode": { 213 | "name": "ipython", 214 | "version": 2 215 | }, 216 | "file_extension": ".py", 217 | "mimetype": "text/x-python", 218 | "name": "python", 219 | "nbconvert_exporter": "python", 220 | "pygments_lexer": "ipython2", 221 | "version": "2.7.10" 222 | } 223 | }, 224 | "nbformat": 4, 225 | "nbformat_minor": 0 226 | } 227 | --------------------------------------------------------------------------------