├── .coveragerc ├── .gitignore ├── .travis.yml ├── .travis ├── travis_before_install.sh └── travis_install.sh ├── LICENSE ├── README.md ├── examples └── mnist.ipynb ├── gelato ├── __init__.py ├── _compile.py ├── layers │ ├── __init__.py │ ├── base.py │ ├── conv.py │ ├── corrmm.py │ ├── cuda_convnet.py │ ├── dense.py │ ├── dnn.py │ ├── embedding.py │ ├── helper.py │ ├── input.py │ ├── local.py │ ├── merge.py │ ├── pool.py │ ├── recurrent.py │ ├── shape.py │ ├── special.py │ └── stats.py ├── random.py ├── specs │ ├── __init__.py │ ├── base.py │ └── dist.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── datasets.py │ ├── test_layers.py │ ├── test_magic.py │ └── test_spec.py └── version.py ├── img └── gelato.jpg ├── pytest.ini ├── requirements-dev.txt ├── requirements.txt └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | omit = gelato/tests/* 5 | include = gelato/* 6 | 7 | [report] 8 | # Regexes for lines to exclude from consideration 9 | exclude_lines = 10 | # Have to re-enable the standard pragma 11 | pragma: no cover 12 | 13 | # Don't complain about missing debug-only code: 14 | def __repr__ 15 | if self\.debug 16 | 17 | # Don't complain if tests don't hit defensive assertion code: 18 | raise AssertionError 19 | raise NotImplementedError 20 | 21 | # Don't complain if non-runnable code isn't run: 22 | if 0: 23 | if __name__ == .__main__.: 24 | 25 | ignore_errors = True -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # pycharm 92 | .idea 93 | /examples/ 94 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | cache: 4 | directories: 5 | - $HOME/.theano 6 | - $HOME/download # Sufficient to add miniconda.sh to TRAVIS cache. 7 | - $HOME/miniconda2 # Add the installation to TRAVIS cache. 8 | 9 | 10 | python: 11 | - "2.7" 12 | - "3.6" 13 | 14 | before_install: 15 | - "export DISPLAY=:99.0" 16 | - "sh -e /etc/init.d/xvfb start" 17 | - ./.travis/travis_before_install.sh 18 | - export PATH=/home/travis/miniconda2/bin:$PATH 19 | 20 | install: 21 | - ./.travis/travis_install.sh 22 | - "source activate pyenv" 23 | - "pip install --upgrade pip" 24 | - "pip install -r requirements.txt --no-cache-dir" 25 | - "pip install -r requirements-dev.txt" 26 | - "pip install . --no-deps" 27 | 28 | script: 29 | - THEANO_FLAGS='gcc.cxxflags="-march=core2"' py.test -v --cov=gelato --cov-report term-missing 30 | 31 | after_success: 32 | - coveralls -------------------------------------------------------------------------------- /.travis/travis_before_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Install miniconda to avoid compiling scipy 3 | if test -e $HOME/miniconda2/bin ; then 4 | echo "miniconda already installed." 5 | else 6 | echo "Installing miniconda." 7 | rm -rf $HOME/miniconda2 8 | mkdir -p $HOME/download 9 | if [[ -d $HOME/download/miniconda.sh ]] ; then rm -rf $HOME/download/miniconda.sh ; fi 10 | wget -c https://repo.continuum.io/miniconda/Miniconda2-4.1.11-Linux-x86_64.sh -O $HOME/download/miniconda.sh 11 | chmod +x $HOME/download/miniconda.sh 12 | $HOME/download/miniconda.sh -b 13 | fi 14 | -------------------------------------------------------------------------------- /.travis/travis_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # In Python 3.3, we test the min version of NumPy and SciPy. In Python 2.7, we test more recent version. 3 | if test -e $HOME/miniconda2/envs/pyenv; then 4 | echo "pyenv already exists." 5 | else 6 | echo "Creating pyenv." 7 | if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then conda create --yes -q -n pyenv python=2.7 ; fi 8 | if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then conda create --yes -q -n pyenv python=3.6 ; fi 9 | fi 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Maxim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gelato 2 | [![](https://travis-ci.org/ferrine/gelato.svg?branch=master)](https://travis-ci.org/ferrine/gelato) 3 | [![Coverage Status](https://coveralls.io/repos/github/ferrine/gelato/badge.svg?branch=master)](https://coveralls.io/github/ferrine/gelato?branch=master) 4 | ## Bayesian dessert for Lasagne 5 | 6 | ![](img/gelato.jpg) 7 | 8 | Recent results in Bayesian statistics for constructing robust neural networks have proved that it is one of the best ways to deal with uncertainty, overfitting but still having good performance. Gelato will help to use bayes for neural networks. 9 | Library heavily relies on [Theano](https://github.com/Theano/Theano), [Lasagne](https://github.com/Lasagne/Lasagne) and [PyMC3](https://github.com/pymc-devs/pymc3). 10 | 11 | Installation 12 | ------------ 13 | 14 | * from github (assumes bleeding edge pymc3 installed) 15 | ```bash 16 | # pip install git+git://github.com/pymc-devs/pymc3.git 17 | pip install git+https://github.com/ferrine/gelato.git 18 | ``` 19 | * from source 20 | ```bash 21 | git clone https://github.com/ferrine/gelato 22 | pip install -r gelato/requirements.txt 23 | pip install -e gelato 24 | ``` 25 | 26 | Usage 27 | ----- 28 | I use generic approach for decorating all Lasagne at once. Thus, for using Gelato you need to replace import statements for layers only. For constructing a network you need to be the in pm.Model context environment. 29 | 30 | **Warning** 31 | - `lasagne.layers.noise` is not supported 32 | - `lasagne.layers.normalization` is not supported (theano problems with default updates) 33 | - functions from `lasagne.layers` are hidden in `gelato` as they use Lasagne classes. Some exceptions are done for `lasagne.layers.helpers`. I'll try to solve the problem generically in future. 34 | 35 | Examples 36 | -------- 37 | For comprehensive example of using `Gelato` you can reference [this](https://github.com/ferrine/gelato/blob/master/examples/mnist.ipynb) notebook 38 | 39 | Life Hack 40 | --------- 41 | Any `spec` class can be used standalone so feel free to use it everywhere 42 | 43 | References 44 | ---------- 45 | Charles Blundell et al: "Weight Uncertainty in Neural Networks" ([arXiv preprint arXiv:1505.05424](https://arxiv.org/abs/1505.05424)) 46 | -------------------------------------------------------------------------------- /examples/mnist.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "I was inspired by [@twiecki](https://github.com/twiecki) and his great [post](http://twiecki.github.io/blog/2016/07/05/bayesian-deep-learning/) about Bayesian neural networks. But I thought that that way of creating BNNs is not obvious and easy for people. That's why I decided to make [Gelato](https://github.com/ferrine/gelato) that is a bridge for [PyMC3](https://github.com/pymc-devs/pymc3) and [Lasagne](https://github.com/Lasagne/Lasagne).\n", 8 | "I will use his convolution bnn from the post as an example of how to use gelato API." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": {}, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "env: THEANO_FLAGS=device=cuda0\n" 21 | ] 22 | }, 23 | { 24 | "name": "stderr", 25 | "output_type": "stream", 26 | "text": [ 27 | "Using cuDNN version 5105 on context None\n", 28 | "Mapped name None to device cuda0: Tesla K40m (0000:02:00.0)\n" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "%env THEANO_FLAGS=device=cuda0\n", 34 | "import matplotlib.pyplot as plt\n", 35 | "%matplotlib inline\n", 36 | "import gelato\n", 37 | "import theano\n", 38 | "import theano.tensor as tt\n", 39 | "theano.config.warn_float64 = 'warn'\n", 40 | "import numpy as np\n", 41 | "import lasagne\n", 42 | "import pymc3 as pm" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "## Load Data" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 2, 55 | "metadata": {}, 56 | "outputs": [ 57 | { 58 | "name": "stdout", 59 | "output_type": "stream", 60 | "text": [ 61 | "Loading data...\n" 62 | ] 63 | } 64 | ], 65 | "source": [ 66 | "from sklearn.datasets import fetch_mldata\n", 67 | "from sklearn.model_selection import train_test_split\n", 68 | "\n", 69 | "def load_dataset():\n", 70 | " # We first define a download function, supporting both Python 2 and 3.\n", 71 | " mnist = fetch_mldata('MNIST original')\n", 72 | " data = mnist['data'].reshape((70000, 1, 28, 28))\n", 73 | " target = mnist['target']\n", 74 | " # We can now download and read the training and test set images and labels.\n", 75 | " X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=10000)\n", 76 | " # We reserve the last 10000 training examples for validation.\n", 77 | " X_train, X_val = X_train[:-10000], X_train[-10000:]\n", 78 | " y_train, y_val = y_train[:-10000], y_train[-10000:]\n", 79 | "\n", 80 | " # We just return all the arrays in order, as expected in main().\n", 81 | " # (It doesn't matter how we do this as long as we can read them again.)\n", 82 | " return X_train, y_train, X_val, y_val, X_test, y_test\n", 83 | "\n", 84 | "print(\"Loading data...\")\n", 85 | "X_train, y_train, X_val, y_val, X_test, y_test = load_dataset()\n", 86 | "total_size = X_train.shape[0]" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "## Create priors for weights (Spec classes)\n", 94 | "Gelato has a flexible way to define a prior on weight instead of just shared variable. It supports a lot of features, listed below.\n", 95 | "\n", 96 | "Basicly, Specification (`Spec`) is a delayed expression that depends on shape. I made possible to combine them with tensor operations, manipulate shape and apply custom delayed functions. " 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 3, 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "data": { 106 | "text/plain": [ 107 | "SpecOp.__mul__" 108 | ] 109 | }, 110 | "execution_count": 3, 111 | "metadata": {}, 112 | "output_type": "execute_result" 113 | } 114 | ], 115 | "source": [ 116 | "from gelato.specs import NormalSpec\n", 117 | "expr = NormalSpec() * NormalSpec().with_shape(())\n", 118 | "expr" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "`Spec` behaves like a tensor and has the same methods " 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 4, 131 | "metadata": {}, 132 | "outputs": [ 133 | { 134 | "data": { 135 | "text/plain": [ 136 | "['arccosh',\n", 137 | " 'arcsin',\n", 138 | " 'arcsinh',\n", 139 | " 'arctan',\n", 140 | " 'arctanh',\n", 141 | " 'argmax',\n", 142 | " 'argmin',\n", 143 | " 'args',\n", 144 | " 'argsort',\n", 145 | " 'astype']" 146 | ] 147 | }, 148 | "execution_count": 4, 149 | "metadata": {}, 150 | "output_type": "execute_result" 151 | } 152 | ], 153 | "source": [ 154 | "dir(expr)[70:80]" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "Methods are used as usually, but instead of a tensor we get another `Spec` instance." 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 5, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "data": { 171 | "text/plain": [ 172 | "SpecOp.argmin" 173 | ] 174 | }, 175 | "execution_count": 5, 176 | "metadata": {}, 177 | "output_type": "execute_result" 178 | } 179 | ], 180 | "source": [ 181 | "expr.argmin()" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "These operations are dalayed until user calls `expr(shape)`. When that happens this shape is passed to all defined specs in expression tree. This tree is evaluated and pymc3 variables are created, one per spec instance and shape for it (there exsist some corner cases when one spec instance can get 2 different shapes)." 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 6, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "with pm.Model() as model:\n", 198 | " expr((100, 2))" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 7, 204 | "metadata": {}, 205 | "outputs": [ 206 | { 207 | "data": { 208 | "text/plain": [ 209 | "[auto_1, auto_4]" 210 | ] 211 | }, 212 | "execution_count": 7, 213 | "metadata": {}, 214 | "output_type": "execute_result" 215 | } 216 | ], 217 | "source": [ 218 | "model.vars" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": {}, 224 | "source": [ 225 | "Note that in example above we specified one variable without shape and other `.with_shape(())`. So when expression is evaluated this custom shape replaces the shape that was provided in `__call__`." 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 8, 231 | "metadata": {}, 232 | "outputs": [ 233 | { 234 | "data": { 235 | "text/plain": [ 236 | "[(100, 2), ()]" 237 | ] 238 | }, 239 | "execution_count": 8, 240 | "metadata": {}, 241 | "output_type": "execute_result" 242 | } 243 | ], 244 | "source": [ 245 | "list(map(lambda v: v.dshape, model.vars))" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "More complex cases can require more accurate shape handling. This can be achived with `tags`." 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 9, 258 | "metadata": { 259 | "collapsed": true 260 | }, 261 | "outputs": [], 262 | "source": [ 263 | "expr = NormalSpec().with_tag('one') * NormalSpec().with_tag('two') + NormalSpec()" 264 | ] 265 | }, 266 | { 267 | "cell_type": "markdown", 268 | "metadata": {}, 269 | "source": [ 270 | "Now if we try to call the expression without tags or we will get __KeyError__. Unspecified shape is *default* tag." 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": 10, 276 | "metadata": {}, 277 | "outputs": [ 278 | { 279 | "data": { 280 | "text/plain": [ 281 | "[(100, 2), (), (100, 1)]" 282 | ] 283 | }, 284 | "execution_count": 10, 285 | "metadata": {}, 286 | "output_type": "execute_result" 287 | } 288 | ], 289 | "source": [ 290 | "with pm.Model() as model:\n", 291 | " expr(dict(one=(100, 2), two=(), default=(100, 1)))\n", 292 | "list(map(lambda v: v.dshape, model.vars))" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "Sometimes it is usefull to change shape with a function, it maps shape to shape" 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": 11, 305 | "metadata": { 306 | "collapsed": true 307 | }, 308 | "outputs": [], 309 | "source": [ 310 | "expr = NormalSpec().with_shape(lambda s: (1,) * len(s)) * NormalSpec()" 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": 12, 316 | "metadata": {}, 317 | "outputs": [ 318 | { 319 | "data": { 320 | "text/plain": [ 321 | "[(1, 1), (10, 10)]" 322 | ] 323 | }, 324 | "execution_count": 12, 325 | "metadata": {}, 326 | "output_type": "execute_result" 327 | } 328 | ], 329 | "source": [ 330 | "with pm.Model() as model:\n", 331 | " expr((10, 10))\n", 332 | "list(map(lambda v: v.dshape, model.vars))" 333 | ] 334 | }, 335 | { 336 | "cell_type": "markdown", 337 | "metadata": {}, 338 | "source": [ 339 | "If you need some more complex transformation other than builtin tensor operations you can use a simple wrapper over a function" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": 13, 345 | "metadata": {}, 346 | "outputs": [ 347 | { 348 | "data": { 349 | "text/plain": [ 350 | "functools.partial(, )" 351 | ] 352 | }, 353 | "execution_count": 13, 354 | "metadata": {}, 355 | "output_type": "execute_result" 356 | } 357 | ], 358 | "source": [ 359 | "from gelato.specs import as_spec_op\n", 360 | "spec_det = as_spec_op(theano.tensor.nlinalg.det)\n", 361 | "spec_det" 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": {}, 367 | "source": [ 368 | "Determinant will be taken only after `__call__`" 369 | ] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "execution_count": 14, 374 | "metadata": {}, 375 | "outputs": [ 376 | { 377 | "data": { 378 | "text/plain": [ 379 | "SpecOp.Det" 380 | ] 381 | }, 382 | "execution_count": 14, 383 | "metadata": {}, 384 | "output_type": "execute_result" 385 | } 386 | ], 387 | "source": [ 388 | "spec_det(expr)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "### Disclamer\n", 396 | "I have to note that Gelato has some overhead magic with wrappring lasagne layers. Espessially there is no need to use gelato layer if you pass gelato `Spec` class to Lasagne layer to define a weight. But using them happens to be very convenient. There is a handy function `set_default_spec` that declares what prior to use if none specified. Let's see how it works." 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "## Priors\n", 404 | "In this setup I decided to use hyperprior to show it can be done very easily. Moreover it worked better for me, no need to choose regularization constant." 405 | ] 406 | }, 407 | { 408 | "cell_type": "code", 409 | "execution_count": 34, 410 | "metadata": { 411 | "collapsed": true 412 | }, 413 | "outputs": [], 414 | "source": [ 415 | "from gelato.specs import NormalSpec, LognormalSpec, set_default_spec\n", 416 | "# Now every layer without passed specs will use `NormalSpec(sd=hyper)` as a prior\n", 417 | "hyper = LognormalSpec(sd=10).with_shape(())\n", 418 | "set_default_spec(NormalSpec(sd=hyper))" 419 | ] 420 | }, 421 | { 422 | "cell_type": "markdown", 423 | "metadata": {}, 424 | "source": [ 425 | "## Network" 426 | ] 427 | }, 428 | { 429 | "cell_type": "markdown", 430 | "metadata": {}, 431 | "source": [ 432 | "The following print output should make you think *\"What's the hell is going on? Is it real?\"*" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 35, 438 | "metadata": {}, 439 | "outputs": [ 440 | { 441 | "name": "stdout", 442 | "output_type": "stream", 443 | "text": [ 444 | "True\n", 445 | "True\n" 446 | ] 447 | } 448 | ], 449 | "source": [ 450 | "print(issubclass(gelato.layers.InputLayer, pm.Model) and \n", 451 | " issubclass(gelato.layers.Conv2DLayer, pm.Model))\n", 452 | "print(issubclass(gelato.layers.InputLayer, lasagne.layers.Layer) and \n", 453 | " issubclass(gelato.layers.Conv2DLayer, lasagne.layers.Layer))" 454 | ] 455 | }, 456 | { 457 | "cell_type": "markdown", 458 | "metadata": {}, 459 | "source": [ 460 | "Yes, Gelato layers are Models and Layers at the same time. Moreover context is taken from the previous layers so that you no more need `with model: ...` construction for most usecases." 461 | ] 462 | }, 463 | { 464 | "cell_type": "markdown", 465 | "metadata": {}, 466 | "source": [ 467 | "## Minibatches\n", 468 | "I prefer storing all my dataset in GPU memory if it is possible to avoid unnessesary data transfers. It is possible with out of the bos pymc3 minibatches" 469 | ] 470 | }, 471 | { 472 | "cell_type": "code", 473 | "execution_count": 36, 474 | "metadata": { 475 | "collapsed": true 476 | }, 477 | "outputs": [], 478 | "source": [ 479 | "input_var = pm.Minibatch(X_train, 100, dtype='float32')\n", 480 | "target_var = pm.Minibatch(y_train, 100, dtype='int32')" 481 | ] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": 37, 486 | "metadata": {}, 487 | "outputs": [], 488 | "source": [ 489 | "network = gelato.layers.InputLayer(shape=(None, 1, 28, 28),\n", 490 | " input_var=input_var)\n", 491 | "network = gelato.layers.Conv2DLayer(\n", 492 | " network, num_filters=32, filter_size=(5, 5),\n", 493 | " nonlinearity=lasagne.nonlinearities.tanh)\n", 494 | "\n", 495 | "# Max-pooling layer of factor 2 in both dimensions:\n", 496 | "network = gelato.layers.MaxPool2DLayer(network, pool_size=(2, 2))\n", 497 | "\n", 498 | "# Another convolution with 32 5x5 kernels, and another 2x2 pooling:\n", 499 | "network = gelato.layers.Conv2DLayer(\n", 500 | " network, num_filters=32, filter_size=(5, 5),\n", 501 | " nonlinearity=lasagne.nonlinearities.tanh)\n", 502 | "\n", 503 | "network = gelato.layers.MaxPool2DLayer(network, \n", 504 | " pool_size=(2, 2))\n", 505 | "\n", 506 | "n_hid2 = 176\n", 507 | "network = gelato.layers.DenseLayer(\n", 508 | " network, num_units=n_hid2,\n", 509 | " nonlinearity=lasagne.nonlinearities.tanh,\n", 510 | ")\n", 511 | "\n", 512 | "n_hid3 = 64\n", 513 | "network = gelato.layers.DenseLayer(\n", 514 | " network, num_units=n_hid3,\n", 515 | " nonlinearity=lasagne.nonlinearities.tanh,\n", 516 | ")\n", 517 | "\n", 518 | "# Finally, we'll add the fully-connected output layer, of 10 softmax units:\n", 519 | "network = gelato.layers.DenseLayer(\n", 520 | " network, num_units=10,\n", 521 | " nonlinearity=lasagne.nonlinearities.softmax,\n", 522 | ")\n", 523 | "\n", 524 | "prediction = gelato.layers.get_output(network)\n", 525 | "\n", 526 | "# Gelato layers are designed to simlify the whole process\n", 527 | "# and are pm.Models as well as lasagne.layer.Layer\n", 528 | "# You can work in the contest of the last defined layer to define likelihood\n", 529 | "with network:\n", 530 | " likelihood = pm.Categorical('out', \n", 531 | " prediction,\n", 532 | " observed=target_var,\n", 533 | " total_size=total_size)" 534 | ] 535 | }, 536 | { 537 | "cell_type": "markdown", 538 | "metadata": {}, 539 | "source": [ 540 | "We now minimize the following objective\n", 541 | "$$-1 * ELBO=KL[q(w|mu,rho)||p(w)] - E_q[log p(D|w)]$$" 542 | ] 543 | }, 544 | { 545 | "cell_type": "markdown", 546 | "metadata": {}, 547 | "source": [ 548 | "Getting ELBO for model and optimizing it is quite easy and can be cone just in few lines. As we will perform batch training we have to say what population size we have for observed var or else we'll get invalid approximation.\n", 549 | "\n", 550 | "Let's see' what we've created" 551 | ] 552 | }, 553 | { 554 | "cell_type": "code", 555 | "execution_count": 38, 556 | "metadata": {}, 557 | "outputs": [ 558 | { 559 | "data": { 560 | "text/plain": [ 561 | "[DenseLayer_8770668666937_W:62_log__,\n", 562 | " DenseLayer_8770668666937_W,\n", 563 | " DenseLayer_8770668666937_b]" 564 | ] 565 | }, 566 | "execution_count": 38, 567 | "metadata": {}, 568 | "output_type": "execute_result" 569 | } 570 | ], 571 | "source": [ 572 | "network.vars" 573 | ] 574 | }, 575 | { 576 | "cell_type": "markdown", 577 | "metadata": {}, 578 | "source": [ 579 | "Seems something went wrong:(\n", 580 | "\n", 581 | "That's not the thing you expect, but that's just a nested model.\n", 582 | "You can find your root model accessing `network.root`. It stores all parameters, potentials, likelihood, etc" 583 | ] 584 | }, 585 | { 586 | "cell_type": "code", 587 | "execution_count": 39, 588 | "metadata": {}, 589 | "outputs": [ 590 | { 591 | "data": { 592 | "text/plain": [ 593 | "[Conv2DLayer_8770666142861_W:46_log__,\n", 594 | " Conv2DLayer_8770666142861_W,\n", 595 | " Conv2DLayer_8770666142861_b,\n", 596 | " Conv2DLayer_-9223363266188632965_W:50_log__,\n", 597 | " Conv2DLayer_-9223363266188632965_W,\n", 598 | " Conv2DLayer_-9223363266188632965_b,\n", 599 | " DenseLayer_8770666122735_W:54_log__,\n", 600 | " DenseLayer_8770666122735_W,\n", 601 | " DenseLayer_8770666122735_b,\n", 602 | " DenseLayer_8770666122553_W:58_log__,\n", 603 | " DenseLayer_8770666122553_W,\n", 604 | " DenseLayer_8770666122553_b,\n", 605 | " DenseLayer_8770668666937_W:62_log__,\n", 606 | " DenseLayer_8770668666937_W,\n", 607 | " DenseLayer_8770668666937_b]" 608 | ] 609 | }, 610 | "execution_count": 39, 611 | "metadata": {}, 612 | "output_type": "execute_result" 613 | } 614 | ], 615 | "source": [ 616 | "network.root.vars" 617 | ] 618 | }, 619 | { 620 | "cell_type": "markdown", 621 | "metadata": {}, 622 | "source": [ 623 | "## Inference" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": 40, 629 | "metadata": { 630 | "collapsed": true 631 | }, 632 | "outputs": [], 633 | "source": [ 634 | "# I prefer object oriented style for doing inference in PyMC3 it allows more flexibility\n", 635 | "# Remember that we need root model here\n", 636 | "with network.root:\n", 637 | " advi = pm.ADVI(scale_cost_to_minibatch=False)" 638 | ] 639 | }, 640 | { 641 | "cell_type": "code", 642 | "execution_count": null, 643 | "metadata": {}, 644 | "outputs": [ 645 | { 646 | "name": "stderr", 647 | "output_type": "stream", 648 | "text": [ 649 | "Average Loss = 3.2991e+05: 4%|▎ | 2945/80000 [00:22<09:51, 130.23it/s]" 650 | ] 651 | } 652 | ], 653 | "source": [ 654 | "advi.fit(80000, obj_optimizer=pm.adam(learning_rate=1e-3))" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": 35, 660 | "metadata": { 661 | "collapsed": true 662 | }, 663 | "outputs": [], 664 | "source": [ 665 | "#import pickle\n", 666 | "#params = advi.approx.shared_params\n", 667 | "#pickle.dump(params, open('params.pymc3','wb'))" 668 | ] 669 | }, 670 | { 671 | "cell_type": "code", 672 | "execution_count": 36, 673 | "metadata": { 674 | "collapsed": true 675 | }, 676 | "outputs": [], 677 | "source": [ 678 | "#advi.approx.shared_params = pickle.load(open('params.pymc3','rb'))" 679 | ] 680 | }, 681 | { 682 | "cell_type": "code", 683 | "execution_count": 42, 684 | "metadata": { 685 | "scrolled": false 686 | }, 687 | "outputs": [ 688 | { 689 | "data": { 690 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8EAAAIMCAYAAADVSa7eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3WuMn9d9J/bvmfvwTooXWaRkyhYlR/YmG4eRvUk28Q22\nnC5WfpEGDha1mhox2iTbbbPoxtm+MJDsAtl2UbcGkgDG2o29WMQx3G0tFMm68iXx5iJblB1fJEsW\nJVESafE+vMz9dvri/3A0JGeGV3Euz+cDDPj8z3Oe5zkjjmV955znd0qtNQAAANAGXcs9AAAAALhV\nhGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAA\nWqNnuQdwq2zfvr3u3bt3uYcBAADAa+CJJ544WWvdcaV+rQnBe/fuzYEDB5Z7GAAAALwGSikvXk0/\ny6EBAABoDSEYAACA1hCCAQAAaA0hGAAAgNYQggEAAGgNIRgAAIDWEIIBAABoDSEYAACA1hCCAQAA\naA0hGAAAgNYQggEAAGgNIRgAAIDWEIIBAABoDSEYAACA1hCCAQAAaA0hGAAAgNYQggEAAGgNIRgA\nAIDWEIIBAABoDSF4lTh4fDiPPX9quYcBAACwqvUs9wC4OodOjiz3EAAAAFa9K84El1I+XUo5Xkr5\n/iXt/7SU8nQp5clSyv8yr/13SikHSynPlFLeN6/9wabtYCnlo/Pa7y6lfKNp/9NSSl/T3t98Ptic\n33ulZwAAAMBSrmY59B8neXB+QynlnUkeSvITtdY3J/m3Tfv9ST6Y5M3NNX9YSukupXQn+YMk709y\nf5Jfafomyb9J8vFa6z1JhpJ8uGn/cJKhpv3jTb9Fn3Ht3zoAAABtc8UQXGv9epLTlzT/d0l+v9Y6\n0fQ53rQ/lORztdaJWusLSQ4meaD5Olhrfb7WOpnkc0keKqWUJO9K8oXm+s8k+cC8e32mOf5Cknc3\n/Rd7BgAAACzpegtj3ZvkHzbLlP+ylPLTTfvuJC/P63e4aVus/bYkZ2qt05e0X3Sv5vzZpv9i97pM\nKeUjpZQDpZQDJ06cuK5vFAAAgLXjekNwT5JtSd6e5H9K8vlmlnZFqbV+sta6v9a6f8eOHcs9HAAA\nAJbZ9Ybgw0n+Y+34ZpLZJNuTHEly57x+e5q2xdpPJdlSSum5pD3zr2nOb276L3YvAAAAWNL1huD/\nJ8k7k6SUcm+SviQnkzyS5INNZee7k+xL8s0kjyfZ11SC7kunsNUjtdaa5GtJfqm578NJvtgcP9J8\nTnP+q03/xZ4BAAAAS7riPsGllD9J8o4k20sph5N8LMmnk3y62TZpMsnDTUB9spTy+SRPJZlO8hu1\n1pnmPr+Z5EtJupN8utb6ZPOI307yuVLKv0ry7SSfato/leTfl1IOplOY64NJUmtd9BkAAACwlNLJ\nrmvf/v3764EDB5Z7GNfty08dS5K85/5dyzwSAACAlaeU8kStdf+V+l3vcmgAAABYdYRgAAAAWkMI\nBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1\nhGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAA\nWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYA\nAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRg\nAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFrj\niiG4lPLpUsrxUsr3Fzj3z0sptZSyvflcSimfKKUcLKV8t5Ty1nl9Hy6lPNt8PTyv/adKKd9rrvlE\nKaU07dtKKY82/R8tpWy90jMAAABgKVczE/zHSR68tLGUcmeS9yZ5aV7z+5Psa74+kuSPmr7bknws\nyduSPJDkYxdCbdPn1+Zdd+FZH03ylVrrviRfaT4v+gwAAAC4kiuG4Frr15OcXuDUx5P8iyR1XttD\nST5bOx5LsqWU8rok70vyaK31dK11KMmjSR5szm2qtT5Wa61JPpvkA/Pu9Znm+DOXtC/0DAAAAFjS\ndb0TXEp5KMmRWut3Ljm1O8nL8z4fbtqWaj+8QHuS7Kq1vtIcH02y6wrPAAAAgCX1XOsFpZR1Sf5l\nOkuhb4laay2l1Cv3vFgp5SPpLJnOXXfdddPHBQAAwOpyPTPBb0xyd5LvlFIOJdmT5FullNuTHEly\n57y+e5q2pdr3LNCeJMcuLHNu/jzetC92r8vUWj9Za91fa92/Y8eOa/w2AQAAWGuuOQTXWr9Xa91Z\na91ba92bznLkt9ZajyZ5JMmHmgrOb09ytlnS/KUk7y2lbG0KYr03yZeac+dKKW9vqkJ/KMkXm0c9\nkuRCFemHL2lf6BkAAACwpCsuhy6l/EmSdyTZXko5nORjtdZPLdL9z5L8YpKDSUaT/GqS1FpPl1J+\nL8njTb/frbVeKLb16+lUoB5M8ufNV5L8fpLPl1I+nOTFJL+81DMAAADgSkqnKPPat3///nrgwIHl\nHsZ1+/JTx5Ik77l/1xV6AgAAtE8p5Yla6/4r9buu6tAAAACwGgnBAAAAtIYQDAAAQGsIwQAAALSG\nEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABA\nawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAA\nALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAM\nAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsI\nwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrXDEEl1I+\nXUo5Xkr5/ry2/7WU8nQp5bullP+7lLJl3rnfKaUcLKU8U0p537z2B5u2g6WUj85rv7uU8o2m/U9L\nKX1Ne3/z+WBzfu+VngEAAABLuZqZ4D9O8uAlbY8meUut9ceT/DDJ7yRJKeX+JB9M8ubmmj8spXSX\nUrqT/EGS9ye5P8mvNH2T5N8k+Xit9Z4kQ0k+3LR/OMlQ0/7xpt+iz7jG7xsAAIAWumIIrrV+Pcnp\nS9r+v1rrdPPxsSR7muOHknyu1jpRa30hycEkDzRfB2utz9daJ5N8LslDpZSS5F1JvtBc/5kkH5h3\nr880x19I8u6m/2LPAAAAgCXdjHeC/5skf94c707y8rxzh5u2xdpvS3JmXqC+0H7RvZrzZ5v+i90L\nAAAAlnRDIbiU8j8nmU7yH27OcG6uUspHSikHSikHTpw4sdzDAQAAYJlddwgupfzXSf5Rkn9Sa61N\n85Ekd87rtqdpW6z9VJItpZSeS9ovuldzfnPTf7F7XabW+sla6/5a6/4dO3Zcx3cJAADAWnJdIbiU\n8mCSf5HkH9daR+edeiTJB5vKzncn2Zfkm0keT7KvqQTdl05hq0ea8Py1JL/UXP9wki/Ou9fDzfEv\nJflq03+xZwAAAMCSeq7UoZTyJ0nekWR7KeVwko+lUw26P8mjnVpVeazW+t/WWp8spXw+yVPpLJP+\njVrrTHOf30zypSTdST5da32yecRvJ/lcKeVfJfl2kk817Z9K8u9LKQfTKcz1wSRZ6hkAAACwlPLq\nSua1bf/+/fXAgQPLPYzr9uWnjiVJ3nP/rmUeCQAAwMpTSnmi1rr/Sv1uRnVoAAAAWBWEYAAAAFpD\nCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACg\nNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAA\nAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgG\nAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWE\nYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYRgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hGAAAABa\nQwgGAACgNa4Ygkspny6lHC+lfH9e27ZSyqOllGebP7c27aWU8olSysFSyndLKW+dd83DTf9nSykP\nz2v/qVLK95prPlFKKdf7DAAAAFjK1cwE/3GSBy9p+2iSr9Ra9yX5SvM5Sd6fZF/z9ZEkf5R0Am2S\njyV5W5IHknzsQqht+vzavOsevJ5nAAAAwJVcMQTXWr+e5PQlzQ8l+Uxz/JkkH5jX/tna8ViSLaWU\n1yV5X5JHa62na61DSR5N8mBzblOt9bFaa03y2UvudS3PAAAAgCVd7zvBu2qtrzTHR5Psao53J3l5\nXr/DTdtS7YcXaL+eZwAAAMCSbrgwVjODW2/CWG76M0opHymlHCilHDhx4sRrMDIAAABWk+sNwccu\nLEFu/jzetB9Jcue8fnuatqXa9yzQfj3PuEyt9ZO11v211v07duy4pm8QAACAted6Q/AjSS5UeH44\nyRfntX+oqeD89iRnmyXNX0ry3lLK1qYg1nuTfKk5d66U8vamKvSHLrnXtTwDAAAAltRzpQ6llD9J\n8o4k20sph9Op8vz7ST5fSvlwkheT/HLT/c+S/GKSg0lGk/xqktRaT5dSfi/J402/3621Xii29evp\nVKAeTPLnzVeu9RkAAABwJaXzuu3at3///nrgwIHlHsZ1+/JTx5Ik77l/1xV6AgAAtE8p5Yla6/4r\n9bvhwlgAAACwWgjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQ\nDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBr\nCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAA\ntIYQDAAAQGsIwQAAALSGEAwAAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGEAwA\nAEBrCMEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGsIwQAAALSGELzCHB4azXMnhpd7GAAAAGuSELzC\nPP3K+bxwYmS5hwEAALAmCcEAAAC0hhAMAABAawjBAAAAtIYQDAAAQGvcUAgupfyPpZQnSynfL6X8\nSSlloJRydynlG6WUg6WUPy2l9DV9+5vPB5vze+fd53ea9mdKKe+b1/5g03awlPLRee0LPgMAAACW\nct0huJSyO8l/n2R/rfUtSbqTfDDJv0ny8VrrPUmGkny4ueTDSYaa9o83/VJKub+57s1JHkzyh6WU\n7lJKd5I/SPL+JPcn+ZWmb5Z4BgAAACzqRpdD9yQZLKX0JFmX5JUk70ryheb8Z5J8oDl+qPmc5vy7\nSymlaf9crXWi1vpCkoNJHmi+DtZan6+1Tib5XJKHmmsWewYAAAAs6rpDcK31SJJ/m+SldMLv2SRP\nJDlTa51uuh1Osrs53p3k5eba6ab/bfPbL7lmsfbblngGAAAALOpGlkNvTWcW9+4kdyRZn85y5hWj\nlPKRUsqBUsqBEydOLPdwAAAAWGY3shz6PUleqLWeqLVOJfmPSX42yZZmeXSS7ElypDk+kuTOJGnO\nb05yan77Jdcs1n5qiWdcpNb6yVrr/lrr/h07dtzAtwoAAMBacCMh+KUkby+lrGve0313kqeSfC3J\nLzV9Hk7yxeb4keZzmvNfrbXWpv2DTfXou5PsS/LNJI8n2ddUgu5Lp3jWI801iz0DAAAAFnUj7wR/\nI53iVN9K8r3mXp9M8ttJfquUcjCd93c/1VzyqSS3Ne2/leSjzX2eTPL5dAL0f0ryG7XWmead399M\n8qUkP0jy+aZvlngGAAAALKrnyl0WV2v9WJKPXdL8fDqVnS/tO57kv1zkPv86yb9eoP3PkvzZAu0L\nPgMAAACWcqNbJAEAAMCqIQQDAADQGkIwAAAArXFD7wRz80xOz+brP7SXMQAAwGvJTPAKMTwxvdxD\nAAAAWPOEYAAAAFpDCAYAAKA1hGAAAABaQwgGAACgNYTgFWJsama5hwAAALDmCcErxPMnhhdsn5qZ\nvcUjAQAAWLuE4BViYurysHtmdDJ/+cyJHD8/vgwjAgAAWHuE4BXs3Fhn7+ChkallHgkAAMDaIAQD\nAADQGkIwAAAArSEErwIzs3XBYwAAAK6NELwK/OjM2NzxkaGxJXoCAACwFCF4lakxEwwAAHC9hGAA\nAABaQwgGAACgNYRgAAAAWkMIBgAAoDWE4FWmpCz3EAAAAFYtIRgAAIDWEIJXGVskAQAAXD8heAUr\nVj4DAADcVEIwAAAArSEErzIKYwEAAFw/IRgAAIDWEIIBAABoDSF4BasKQQMAANxUQjAAAACtIQQD\nAADQGkIwAAAArSEEAwAA0BpCMAAAAK0hBK9QwxPT+eGx88s9DAAAgDVFCF6hnhWAAQAAbjoheIU6\nNTy53EMAAABYc4RgAAAAWkMIBgAAoDWEYAAAAFpDCAYAAKA1hOBVZmp2drmHAAAAsGoJwavMCydG\nlnsIAAAAq5YQDAAAQGsIwQAAALTGDYXgUsqWUsoXSilPl1J+UEr5B6WUbaWUR0spzzZ/bm36llLK\nJ0opB0sp3y2lvHXefR5u+j9bSnl4XvtPlVK+11zziVJKadoXfAYAAAAs5UZngv+PJP+p1vqmJD+R\n5AdJPprkK7XWfUm+0nxOkvcn2dd8fSTJHyWdQJvkY0neluSBJB+bF2r/KMmvzbvuwaZ9sWcAAADA\noq47BJdSNif5+SSfSpJa62St9UySh5J8pun2mSQfaI4fSvLZ2vFYki2llNcleV+SR2utp2utQ0ke\nTfJgc25TrfWxWmtN8tlL7rXQM1rj7OhUpmZUigYAALgWNzITfHeSE0n+z1LKt0sp/66Usj7Jrlrr\nK02fo0l2Nce7k7w87/rDTdtS7YcXaM8Sz2iNxw+dzrdeHFruYQAAAKwqNxKCe5K8Nckf1Vp/MslI\nLlmW3Mzg1ht4xhUt9YxSykdKKQdKKQdOnDjxWg5jWZwfn17uIQAAAKwqNxKCDyc5XGv9RvP5C+mE\n4mPNUuY0fx5vzh9Jcue86/c0bUu171mgPUs84yK11k/WWvfXWvfv2LHjur5JAAAA1o7rDsG11qNJ\nXi6l3Nc0vTvJU0keSXKhwvPDSb7YHD+S5ENNlei3JznbLGn+UpL3llK2NgWx3pvkS825c6WUtzdV\noT90yb0WekarPX9iOMfPjy/3MAAAAFasnhu8/p8m+Q+llL4kzyf51XSC9edLKR9O8mKSX276/lmS\nX0xyMMlo0ze11tOllN9L8njT73drraeb419P8sdJBpP8efOVJL+/yDNa4W8Onlyw/fkTI0mS99w/\ncCuHAwAAsGrcUAiutf5dkv0LnHr3An1rkt9Y5D6fTvLpBdoPJHnLAu2nFnpGW4xOziz3EAAAAFal\nG90nGAAAAFYNIRgAAIDWEIIBAABoDSF4lZudvXnbMM/M1pt6PwAAgJVGCGbO154+nr9+buHK0wAA\nAGuBELzGzczWDI1MXnX/ianZ13A0AAAAy0sIXuOePnouT7w4lNHJ6eUeCgAAwLITgte4kYnOnsJT\nM971BQAAEIIBAABoDSEYAACA1hCCV7nj5ycyNaOYFQAAwNUQgle57x85myd/dG65hwEAALAqCMFr\nwPjUTL781LHlHgYAAMCKJwSvAVXhZwAAgKsiBLeFoAwAACAEAwAA0B5C8Bo0PjWTeuka6bI8YwEA\nAFhJhOA16K+ePZlDp0aXexgAAAArjhC8BtQFXvg9PTJ5aaclzcx6aRgAAFj7epZ7ANy40YmZG7r+\n/PhUvvH86Zs0GgAAgJXLTPAaNjk9m/GpTkB+/NDpy98Tbpwfn76VwwIAAFg2ZoLXsP/87ImL9hCe\nrUm3AlkAAECLmQlewxaZ+AUAAGgtIRgAAIDWEIIBAABoDSG4xZ49dj6jk4piAQAA7SEEt9To5HRe\nPDWav3v5zHIPBQAA4JYRgltqrmiW4lkAAECLCMFr1ELLnBfbJxgAAKAthOA1amJq9qr6zQjGAABA\niwjBLXe1YRkAAGAtEIJbyvwvAADQRkJwi/zFMyeWewgAAADLSggGAACgNYRgAAAAWkMIbqmy3AMA\nAABYBj3LPQBuvVPDE/n2S2eWexgAAAC3nJngFjp0amS5hwAAALAshOCWqdXmSAAAQHsJwS0zOjmz\n3EMAAABYNt4Jbpm/fe7Ucg8BAABg2ZgJBgAAoDWEYAAAAFpDCAYAAKA1hGAUywIAAFpDCCaHTto3\nGAAAaAchGAAAgNa44RBcSukupXy7lPL/Np/vLqV8o5RysJTyp6WUvqa9v/l8sDm/d949fqdpf6aU\n8r557Q82bQdLKR+d177gMwAAAGApN2Mm+J8l+cG8z/8mycdrrfckGUry4ab9w0mGmvaPN/1SSrk/\nyQeTvDnJg0n+sAnW3Un+IMn7k9yf5Feavks9AwAAABZ1QyG4lLInyX+R5N81n0uSdyX5QtPlM0k+\n0Bw/1HxOc/7dTf+Hknyu1jpRa30hycEkDzRfB2utz9daJ5N8LslDV3gGAAAALOpGZ4L/9yT/Isls\n8/m2JGdqrdPN58NJdjfHu5O8nCTN+bNN/7n2S65ZrH2pZ3ATPH9ieLmHAAAA8Jq47hBcSvlHSY7X\nWp+4ieO5qUopHymlHCilHDhx4sRyD2fVeP6EatEAAMDadCMzwT+b5B+XUg6ls1T5XUn+jyRbSik9\nTZ89SY40x0eS3JkkzfnNSU7Nb7/kmsXaTy3xjIvUWj9Za91fa92/Y8eO6/9OW+jE+YnMztblHgYA\nAMBNdd0huNb6O7XWPbXWvekUtvpqrfWfJPlakl9quj2c5IvN8SPN5zTnv1prrU37B5vq0Xcn2Zfk\nm0keT7KvqQTd1zzjkeaaxZ7BTfKdl8/kefsHAwAAa8xrsU/wbyf5rVLKwXTe3/1U0/6pJLc17b+V\n5KNJUmt9MsnnkzyV5D8l+Y1a60zzzu9vJvlSOtWnP9/0XeoZ3ETjUzPLPQQAAICbqnQmVte+/fv3\n1wMHDiz3MBb15aeOLfcQLnP75oG8ZffmJJ3l0bet70tXV1nmUQEAAFyulPJErXX/lfq9FjPBXIef\n27d9uYdwmZGJTgHu0yOTzfJoVaMBAIDVTQheIQZ6u5d7CJc5Pz6dg8fPZ6xZFn1yeHKZRwQAAHBj\neq7chTY7dHJ07nh4fDpTM7Pp7e787uTpo+dy+PRY3nP/ruUaHgAAwDUxE8w1eebo+bnjw6fHlnEk\nAAAA104I5ppMzcwu9xAAAACumxAMAABAawjBAAAAtIYQzDU5tUCF6PPjUzk7NrUMowEAALg2QjA3\n7BvPn87jL5xe7mEAAABckS2SuGZnR6fS1+P3JwAAwOojBHPNHj9k1hcAAFidTOcBAADQGkIwAAAA\nrSEEc9NNTs/m+RPDyz0MAACAywjBK8id29Zd9Hmwr/uyPm/ZvflWDee6/eCVc3n+xEiGRi7fTgkA\nAGA5CcEryK5N/Rd9Lgv06VqocYWZnq1JkrrM4wAAALiUELzarIIQfEGtYjAAALCyCMEr2D07Nyz3\nEK7Jpe8Bj07O5Nz41DKNBgAA4HJC8ArSfcla552bBi7rM9h7+XvCK8UrZ8cv+vzM0fP55vP2FAYA\nAFYOIXgF2TjQmx/fs3Thq40DvbdoNNdubHImP3jl3HIPAwAAYFFC8Aqz0OzvanJkaCyz3gUGAABW\nqJ7lHgCX2793a2aaCssPvGFbZmZqTg5P5Oi5znLj2zcP5OglS49XkrOjC78H/NWnj2X3lnW57/aN\nt3hEAAAAHULwCrRlXd/c8aZm+fPW9X3Zt6sTHt98x6YVHYIXMzubvHx6NLdvGsiJ4YlVV/gLAABY\n/SyHXoVKKSmraKuk4+cvDuyPHzqdQydH5j7XWvPCyZFMzcze6qEBAAAtIwSvUvtfv225h3DVzo1N\nL3n++PmJPHd8OD88dv4WjQgAAGgrIXiV6utZO391zx7r7C98/NzEMo8EAABY69ZOkmqZ3u5VtB46\nC1eLPni8M/M7OTOTJHPFwAAAAF4rQvAq1dPdlXe9aWfu2DK43EO5okMnR3Pg0OkF2wEAAG4lIXgV\n6+oq6Volf4NnFtk26VnvAQMAALfQKolQLOaNOzZk02DvZe2b113ethK9eGo0JatpaTcAALCaCcGr\nXG93V37izs2Xtf/03tVTPRoAAOBWEYLXgP6e7rzn/l1znzcM9CzjaAAAAFYuaWkN+YX7diTpzA4D\nAABwOWlpDent7lqVAXiprZHOjk5ldHL6pj7v+PnxTE7P3tR7AgAAq8PqS0ysabVeHIgfP3Q6f3Pw\n1E27/9TMbL778tl8+6Whm3ZPAABg9bAcmhXlxPBESkq2rOt9TWa1L2TssamZm35vAABg5TMTzIoy\nPjmb77x8Jk+8aKYWAAC4+YRgVpTDQ6NJkuHxm/seMAAAQCIEr2lvff3W/Ow925d7GNdkdPLVZcrj\nliwDAAA3mRC8hm1b35fBvu4kSW/P6vur/qtnT84dL1UhenpmNrNLVJgGAAC4YPUlI67Z/r1b87a7\nty33MG7IUhWi/+KZE/nGC6dzdnTK7DEAALAkIbgFtqzry0Bv93IP44ZNzSy+t+/IxHRnO6XnTi7a\nBwAAQAhuke6ukr6ervzEnVvm2vZsG0yS3Hf7xuUa1lV77sRwjp0bz5efOpYvP3Xssj2Fk2R28ZwM\nAABgn+AdP4t0AAAgAElEQVQ2ecd9O5IkpZQkuWx2+I07N+S548O3fFxX6/DpsRw+PTb3eWh0KufG\npi7rNzNb88LJkbxh+/p0dZXUWvP00fPZe9v6dHeVWzlkAABghRGCW+RC+E2Sn7xrS9b39+TQqZFl\nHNGN+dYiewn/zXMnMzE1m77urtx127qcHZvKkaGxHBkay8/fu+MWjxIAAFhJLIduqds29C/6nvCu\nTQO3eDQ318RUZ030bLNc+siZV2ePT41MJEmmZ2qGRiZv/eAAAIBlJQQz53WbB7Kuvzv7dm3I296w\nuqtJJ8lYUyn61PCrYffJI+fmjp9YZCYZAABYu647BJdS7iylfK2U8lQp5clSyj9r2reVUh4tpTzb\n/Lm1aS+llE+UUg6WUr5bSnnrvHs93PR/tpTy8Lz2nyqlfK+55hOlWc+72DO4dlvX9SVJNg70ZKC3\nOz/zxu0Z6O3OxoHe9HR3lk+v1iXER4Y6M8DFa8AAAEDjRmaCp5P881rr/UnenuQ3Sin3J/lokq/U\nWvcl+UrzOUnen2Rf8/WRJH+UdAJtko8leVuSB5J8bF6o/aMkvzbvugeb9sWewTXatWkgP3/vjmxp\nwvBidm7qv0UjurleOTs2tzz6Wh0/N77ktkwAAMDqc90huNb6Sq31W83x+SQ/SLI7yUNJPtN0+0yS\nDzTHDyX5bO14LMmWUsrrkrwvyaO11tO11qEkjyZ5sDm3qdb6WO3shfPZS+610DO4Dn09V/4xeMsd\nm2/BSG6++cufr2R8aibjzRLq8amZfPfw2XzvyNnXamisABPTM/nKD47lzKj3wwEA2uKmvBNcStmb\n5CeTfCPJrlrrK82po0l2Nce7k7w877LDTdtS7YcXaM8Sz7h0XB8ppRwopRw4ceLEtX9jzOnqKnnr\n67fmji2DF+0zvNrN32v4r549mb969mSSV4tqjU/OLMu4uDXOjk6l1uSl06PLPRQAAG6RGw7BpZQN\nSf6vJP9DrfWiabdmBrcueOFNstQzaq2frLXur7Xu37Fjdb7Xupw2DfYmSS5srbttfV/uv2NTdmxc\nnUujF/K3z53K2Uv2Gj47+urn1/SHFwAAuOVuKASXUnrTCcD/odb6H5vmY81S5jR/Hm/ajyS5c97l\ne5q2pdr3LNC+1DO4iX589+b89N5t6em+/Mek9yqWUK8Go5MzefyF03lx3n7Jp0cnM968R3xheTQA\nALA23Eh16JLkU0l+UGv93+adeiTJhQrPDyf54rz2DzVVot+e5GyzpPlLSd5bStnaFMR6b5IvNefO\nlVLe3jzrQ5fca6FncBP1dHdl87reBc+96faNSToFs95z/4Kr0VeVZ48Nzx0/d3w432q2T6q1E4RP\n21MYAADWhJ4buPZnk/xXSb5XSvm7pu1fJvn9JJ8vpXw4yYtJfrk592dJfjHJwSSjSX41SWqtp0sp\nv5fk8abf79ZaTzfHv57kj5MMJvnz5itLPINl0tNdMj2zNhcP/+1zpzIzW/PW12/NtvVLV9EGAABW\ntusOwbXWv0qy2A6s716gf03yG4vc69NJPr1A+4Ekb1mg/dRCz2D5vOO+nTl4/Hx6u7vy7LHh9Pd2\nXffWRCvNzGwn3H/rxaG8+8d2pth4GAAAVq0bmQmmxQb7upMkWwZfnRm9Z2dnifT2Df3p6+nK9EzN\nXx88uSzje6088eJQ9u/ddlHbifMTOTM6mX27Ni7TqAAAgKslBHNdNg305mfv2T4Xhudb39/5sert\nTt5z/67MztZ0dZV8+aljt3qYN92ZpnL0+fGpvHhqNHu2DuY7L59Jkrxxx4Y8c+x8Ng32ZveWweUc\nJgAAsAghmOu2UABeSFezx9KWdb1zIXI1mx/mj54dnzv+9stDGRqZypGhsUVDcK01h06N5s6tgwtW\n3QYAAF5b/iucW2bfro3ZNNibd75pZ9bia7VDI5cH/FrrXGXpqZnZPH5oKM8dH84P51WjTpJz41MZ\nm7QdEwAAvNbMBHPLbB7szQN3d96nfdebdubc2HQeP3T6CletThdmi19/27q8eGr0svMXim1d8M3n\nO/8c1sJ2UytRrTWHmxn6CysTAABoJyGYZVFKSX/v2l+IsFAATpJj58Zz4unx3La+P5sGF96LmZvn\nyJmxPHP0fKZna+7evj5fe+Z4tq/vz65N/cs9NAAAbrG1n0JYsQZ6u/Mz99x2Udvmdb15x307lmlE\nt9bsbKey9HPHX10aPTY5k85uYq/65gun88rZsVs9vDXlwh7W0zOdbbtmZmqOnRvPqCXoAACtYyaY\nZbWurycPvGFbJqZm091VsnGgJz3dXWumiNa1+uuDJ9PVlbzj3p0ZnZrJ+fGpnBubypNHprJ9Q396\nm2JaIxPT6e/pUlzrGl36LroQDADQPkIwy27TQG8ycHHb7q2DF4Xg++/YlNs3DeSrTx+/xaO79WZn\ns+D3+TfPncpP7Nmc3u6u/O1zp9LdVfLON+3MzGxNt/dclzR/bv3SmfarNT41k5PDE9mzdd1F7VMz\nsxken87W9X2LXAkAwEoiBLMiDfZevP1SVynp6ip5z/27cuL8RLas681fPnNimUa3PKamZ3Pg0NDc\n55nZOleA621v2JZnjp6f+8VBf29Xfnz3lmxed/n7xheqUF/tFlerVa01E9OzGZj3s3To5Oii72lf\nybdeGsroxEx2bhxIX8+rM/DfeflMzoxO5Z1v2umXEQAAq4AQzIq0ZV1fHnjDtgz2dufFU6MXFTDa\nsbFz/M437czXnj6ev3/XlnSVkm+9OLTY7da8bzx/cZXtianZucrb3V0lM7M1t23oy/YN/Xnm6Pkk\ni1eifuHkSHq6Su7c1pnxPH5uPLM1uX3zwIL9V6LJ6dl8/YedX5K89fVbLzp3rRPBx8+PZ/v6/kw1\n7xXXXHyD8+PTzX1rklsbgocnplOSrO/3r3IAgKvlv5xYsTYNdGYx79m5YcHz3c3M8AX33b4xr9s8\nkLGpmctCYZtd2I7p1PBkTg1PzrVfmEVe19edu3esz+bB3nSVMleoa2xqJl2l5NDJkSRJf09Xzo5N\n5eDx4ezdvi6bBnpTSsmOjf2ZmJ7J8ydGct+ujcu2BdH0TOe98lLKXABOkm+9OJQ92wYXvOZHZ5Yu\nOHZmdDLfffls9mwbvMXx9uo89typJLbWAgC4FkIwa8aFmcuN3V158+5NefLIubxuy0Amp2fnwt+F\ndl41Ojmz4D+Tly5ZNvzEvJn2QydfPTe/iFlvd8n0bM3h02P5+Xt35PTIZDYM9OTc2FTW9/dkQ39P\nRien091VMjIxk++8fCb7927NlnWLv0/7wsmR7N4ymL6ertRaM1s7vwA5enY83z9yNj++Z3N2bOzP\nXzTL49/9Yzsvu8fh09dWXXtoZDI93WVu9nd8anbu3GUzySsxHQMAsCghmDXpdZsHs3VdX/q6u9LV\nVXLg0OmcGZ3K6zYPZvuG/szWmlqTg8eHc8eWwVYvpb5R8wuYzQ/H82djlzL/Pef57t21MT881lm6\n/dzx4fzkXVvy3cNnMzNbc/vmgRw9O54k+e7hs3nz7k1z133lB9dePG1yejZPvHg6QyOLVyS/tLL0\n1Mxszo1NJddXZwsAgGVSrrdS6mqzf//+euDAgeUeBstkdrbzJufVFC66sEz4XW/amYnp2bw8NHrZ\nrCjt0t1VktLZX/j2zQPZvqE/h4dGL/oFwM5N/Tl+bmLRpcm11gxPTGfjwOXFyq5WrTVHz43n9k0D\nKaXM/axaDg0AkJRSnqi17r9SPzPBtMK1vKdaSmfJa1dXyWBfd/bt3JDbNw9k00BvJqdn8/0fnc3p\nZnn11vV9GRp59T3b3p6u/P07t+TxF7yTvJZceK86SY6eHZ+bhZ7v+LmJJJ1fomxZ15v779iUgZ7u\nnByZSF93V04OT+bQyZG8/Y23ZUNTyOrImbHMzNTcddu6uc8/+NG53L55IG/ZvTmnRyazZbB37uf3\n5dNj+eGx83nhxIg9jgEArpMQDJf4+Xt3ZHbeColSylyRrr6errz1rk614fPjU1nf15Op2dn85x+e\nTJK89a4t2TjQm3t2bsiRM2N5611b09/TNbfv78/t256Riel8+6Uzt/i74lY6MzqVvzl4aq4y93wj\nE9Pp7+nKzGzND37UeRd7+8a+jEzMzH0+enY858amMjo5kzu3rct9t29MkkzOdILvtQbg2dma8xPT\n2TzYm5GJ6Qz0dtvOCQBoLcuh4SaYna05Nz61aIGn8+NTGejtTm93Z3/ZscmZuWXWP71329x+vmdH\np/L4odMZ7OvOXdvWzW1ntHldb/a/fmuePzmSHRv7882m+nV/b1cm5hVtYu3ZtqEvt63vy5GhsUzN\n1kxNX/73/c437cyxc+M5PTKZienZ3Ltrw0XLrr/2zPHMzNTs37s1Bw4NZbCvO5PTs3nbG7ZlXd+t\n/V1obd7HX64q4klna6nHnjt10f/2AIDV72qXQwvBsMKcHZ3KxoGedHWVzM7WnBmbyrb1i1dPTjqF\nnXq7O9sDPXP0fLZv6MtgX3fGJmfmZv2GJ6YzPjWTvp6uDI1M5eXTo9m7fV26u7rS213y9Cvn5+7X\n011y/+s25fCZsfy93Zvzdy+fydnRqfzsPdvz1wdPzvXr6+nK6zYPpObyatLcOnu2DS5YAXv31sHc\ntW1d/rbZSmnr+t6Lin/t3b5+0S3IboaxyZl0dSX9Pd1zbU8fPZfDp8fy7h/bmXJptbFb5MVTI3n2\n2HDuum1d7t21cVnGAADcfN4JhlVq/sxUV1e5YgBOOmH0ggtLZ5NcNMs30PtqENm+vj+bB3tz++aB\nubY9W9dddt+dmzrnf3rvtrm2xYowvXHHhoxNzWRDf09qrTkxPJGdGwfy1I/OZffWwWwe7HxfUzOd\nmcwLs+IX2nq7uzLbLB2u6ew8ND1bc2ZsMsfOTmTP1sGLtmniVYttAXVkaCzHzr36/vJS1a/PjE7m\nWy8N5af3brtoFnlsciaztWZ9/7X/38WFX5jM/5k5MrT4dlUjE9MZmZie+7kDAHgtCMHQQl1d5aIA\nfDN0d5W5gk+llOzc2Ln//Xdsuqjf/PB7adulS2T7ujr3uXCvd71pZ841S8tHJqZz8Phwzo9P39Tv\nY62Znll8tc+hkyM5fm78oneMD50cza7N/XP/zC8E2V+4b0dKkpGJmUWXEI9PzaS3uyvdXSUvnByZ\nax+bnMlgX/dFfWu9UISuM75SytyM9Xvuv/xn88tPHcsbd27I3dvXX/bMl0+P5p6dGzIzWzM5M3vF\nJd7F5s4A0GpCMLBqdHWVufeuB3q7c9uG/iTJs8fO50XLsa/LpUW2jp0bz7Fz47nv9tm5X2okyV8+\n8+q+z/fdvjHPHD0/V+l6amY2r5wZn9vX+T3378pzx4fn+n/75aH8zBu3J+mE3yT5+rMnsmmwN6MT\nMxmfmsnP7ds+1/+Vs2N58si5/IM33nbRDPRzx4cvC8HfP3I2Z0ansnNTZ9XByMR03vWmnXnshVOZ\nnJ7NL9y7I0+8OJQzo1NzM9JTs1f3Hv2FrdWGRiezaaD3ohUXS5mZrSlZ3veeAYDFCcHAqrdv18bc\nuW1dzo9P5zsvn0lvT9eCBaS4eheKsi117vFDp/O2u7flbw6euuj8hf2LLxidmMnk9Gy+/sNXg/T0\nTJ3baixJ/urZV981f/JIp0r2S6dH82Ov25T5tStGJ6fT09U1F0gvnJmYmsnIRGdVwFOvnMvoRCfc\nz9bM7ef85aeO5Y4tg/nRmVeXZM/O1vzlsydy923rs/eSgH2hqnvSeU1h/msB8x09O55Ngz1zM9Bf\ne/p4erpL3nHfzgX7zzc+1Rnn/NcVAIDXlhAMrAkDvd0Z6O3OL9y3I73dXXNBbMNAT3ZvGczmdb35\nzstnsr6/J6eHJ/P39mzO9w6fTdIpGLVn67pMTM1m24a+nBubylPNdkUsbmamXhaAFzM/AF+t0cnp\nvHJ2LOPzKqBfeN4bd264aLb5u83fZZKL9nGemL54pnt+AJ6Ymp0LugePD6e/tysvnBjJlnV9OX7+\n4r2gz45OpdY6V8xremY2NUlPV8n3j5xNd1fJO9/0aui9sAz9lbNjOTU8mbfs3jx3bmpmNi+dHs2m\ngc7PZNJ5bWB8qlPI7o4tg9fwTwkAuFZCMLCmXHi/+B/euz1/89ypPLB329yy1H+4b8dFfXct8O5p\nkmzo78mOjf0Zney8bzpba86NTc/N2t2zc0O2b+zPoZMjFwWuCy4sF55vXX/33OwkV2doZGrRYl7z\nA/BSlgrp84uGJa/OQI9OLly8a3q2pre75Pz4VL7RbFO2qynidel+0Jfec/eWwWxd35cnXjy94Pc0\n/5cud2wZTK01wxPTFxUpAwBuDiEYWJP6e7rzzqtYjrqY3u6ubB7syuZ5M3iXesvuzRfN8M23vr8n\n0zOzF1U6PjU8kdmmINTfvXTmov4/c89tVz2ryvI4PTKZUnLRLzPmB+nvHzmb+1/3aiG48+Ovht0n\nXhzKu39s55IVuud74eRInj8xkgfesC2bBGEAuKnsEwywjOZXR046y2z/oilCtXldb86OTuWenRty\n8CpnPln9OlWzO8f7dm3I629bv/QFizg1PJHNg73pWaAi+9UYnZzOqeHJ3Lnt8u3TXgvTM7N59vhw\n9u3ccN1jBqDd7BMMsApcCL8X9HR35e1vvC293SX9Pa8WSzp6bjzD49N5z/278tjzpzJsa6g1a/7v\npp89NpwjZ8bmqmtfcGp4IqOTM9mxsVMh/bHnT+XObevyxh0bMjUzm5GJ6Xy7WW1w766N2bmpP7O1\nXrZ91OxsTSmdn8Ozo1Pp6+ma287qwsqEjQM9c1XZFx5vzejkzDXtJV1rzcT07EUFwV46PZojQ2Pp\n7+nKG3ZsuOp7XYsnXhzKxoGe3Ltr45U7A7BmCcEAK8yGBcLEW+/amtHJTvB9+xtuS9IJEi+dHs2z\nx4azY2N/fuLOLUkur87M6jY6MTP3d9rVlfzcPTvmAu78d89fODGSF06MXHb9D4+dn9u+6sfu2JRd\nG/szMjGTTYM9+erTx7NzU3/27dyYxw913nP+8T2bL1rG/8Njw3ng7oUrY09Mz+Q///DVyt5//64t\n2bquL91X2B7q0KnRPHd8OP/gjZ2f5cHe7rlK3/PXp03PzGam1vR0dWVqZjbdXWXBvcaXMjNb8/yJ\nzvZaQyOTGRqZvK4QPDo5fcU9qAFYHSyHBljFaq159vhwXn/burmZ4wv/Xj92biLfP3I2r9sykOPn\nJrJ/79aMTc7k6aPnMzk9e1GFbJjvJ+/aMhe0L/i5fdvzw2Pnc3J4Iu+8b2deOTu+aBX1e3dtTCnJ\n2bGp3Lm1WU5dOqF202Dv3L7TP3bHpvzgR+eydX1f+nu6cvTseLas683+vdty/Px4vvty5+dzoLc7\n41Mz6epK7rt9Uzb092Tz4MLvSp8ankhvT9fcu9TPnRi+7JcDb969KQM93TkzNpXbNw3MzX7PNz/0\nHj07nu8fOZuf/P/bu7MYx678vuPfP/cii8Xaq5fqfVVprNFoJE+PpZFnJNmecQIrD36QEcCTDQay\nAHHyEIxhIIAfEwRBYsSIYdgO4iDx2Jk48cBwMJktCTKe0WiZ1tJSt1Td6u7qrXZWkcXidu/Jw71k\nk7V0dUlqkdX8fQCiyUPWrcv777rk/9xz/ufwYHN98o183/HO7VVOjPVvub2PKl+qkuuLbxo90st8\n32k9bhFpc7/DoZUEi4j0GOcczkEkYuRLVV69ugzAYDreXFP33IkRiuU6b9/cOUnWusyyG5EI+Dv8\nd3niyBCvX1u+52s+d3yYfKlGpe6xL9eH5zsqNa+5XNbnjg+TTsS4Ml/k2mLpntt67uw4kYg117Oe\nGEgxu1puFiZ7b7bA9cXSlnO0a55PPBphsVjhJ9fzRKPGowcGGM9uXX1+N3zf4YBiuc4rV5c4Npbh\nxBZDxVdKNfoS0eb62fdSqtapeY5YxHY1hP3GcomLtwvNY9Xg+W7LK//5UpVipc7k0IOZU768VuW1\na8s8cWSI4cz2w/V3o+753Myvc3g4rc4GkT1Kc4JFRGRLZkbj+91gOsHzj4w3v/CVax4RMxKxCP3J\nGPOFCgN9MSaH0lxbXGM4kyAejfDDy4uc2ZdtFk26cGuF2/kyR0fTXF0o8cUzY5TrPkvFanMobsNo\nNslCofKJvmfpHjslwMCOCTDQXKYK4OrC5iS39fmdfO/iHC9MTTTXs25U/X7n1irnjo/QSIdqXvvO\nf7CwxuW5IudOjFAPl8nyPMebMyu8MJXiu+/OEosGnUTZVIxCuc5jkznGskku3FrlzkqZZ06NkopH\nOT+TZzybZCybpO45CuVaM6F/5EBQdXxmqRQs4RZejY5EDN93zaHsnzs+TDwaaQ4bL9c8ap5PLBLh\n6uIax0YzbVXof+bkSPNqt+c7Plgocny0f8urq5fDq+kr6zWGwqSz5vn8n0vzHB/LNOdxN9bTbnSu\n7ZQET88VKVbqPB5O57hfy6Vq89+PKwl+b7bIrfw6fYkomUSMuu+2HXEgD4ftOnH2otYaD7IzXQkW\nEZFdq9b9TVed1qvelsNAV8s1yjWPQrnOoaE0EYP/fWmeRw8OkIxFOT+zzKMHckQjxq38Or6DXF98\n01rAjx4caK67u1EyHqFS09Vo6R6fOz68bSI+0Bdndf3ucllj2STzH6Jj6MhImvGBFK98sPn37GaE\nxtMnR7m1ss4H82sMZeI8NhkkpNGww2xmaZ3LC0U8L/jOeHK8n5H+BL6j+bufOTXKynqNt26stA2n\nf2Fqovl7vvPOLLl0nCePDLG6XieXjjfnu3/60CClap25QoXhTGLLK96tGsPcc+k42VSMs/sGtnxd\nsVInFrG2ImxbuZVf5+KdVXyftqkirfvfaqVUY73msS9394p/pe4xPVfkkX0DOw7Tzpeq3MqXmTqw\n9X4/CL7vuHinwInxTFvhxe0sFCttc/z9sKPn4xqC7pyj6vn3tS9wdxTDh+2Y2Dh8vjG64emTow9k\nCsODdHtlnQs3V5udaBD8fY0PJJt/v71Kw6E3UBIsIrK3zCyVSMWjZFMxYhFrLpvz9s0V7qyUGcok\nWF6rNr8EzBXKuDCBjkaM9ZrH8lqVm/n15tq+UwcGWC5VuZ0v3+tXi0iocQX7fmxM7gGeOjpMJhnl\n4p0Cd1aCv7t0Mtq23vZWTk9kmSuUyZdqPHd2vDl6xcwoVur86HL7uuqNZLXm+ZSqHrOrZbKpWLPj\n7NnTY7w3G+zD0ydH8Z0jk4zhnOPyfLFtNEFrB8KTR4dIxaOk4lGm5wpMDKTIpuJtBQifPT1GLGK8\nczu4uj91YIBamNxNDCSb009afffdWZyj+d7udfVuverxg+mFZtG6V68ukS/V2hJ05xxXF0tMDvVt\nWzyuMbd9Xy7FSH+CCzdXefb02JbD6AvlGi9fWWJfLsXJ8X5S8SjfeWeWdDK6qVr9h/VX0wuUql5b\nItdQ83zWax4DqTjnZ/JMDCSbsXz+kXHqvrvvInmNY3N5rsiTR4fI9cVxDs7fyLNUrDKaTfL4oUHy\npSqe7zbN+695PtcWS5wYy7TFyTm3q/2A4Mrzes3D8xylWp39uT4g6CzOJmPN7fu+Y65QaetkafXa\ntWWW16pt0wEa/ye367jZeEwe1ivGSoI3UBIsIvJwqHs+c4UK+3OpTcvsbKdc86jUfHLp4ArC69eX\nqXuO4UycvkSMQrlGIhrh2mIJz7/7uZhORClVPSaH+yjXfA3jFulS/anYrpeOe+bUKO/PFpvD3z+K\n4f4ES8VgiPah4TQzS+1D9J9/ZJzFtSrnNxSc2z+YanbK/eyZMZbXqoxlk6ys1/BdsFpAvlRtm+ve\nGGFweCTN9cUSh4bTzK6WqdZ9JgaCpPUH0ws8dTSo6l71fN6YybfVfWisQ392f5aZpXXO7MtyK7/O\n/lyK2ytlqp7ffD+NY/X/3l9o3i9W6mQSseYV1HLNY75QYSybxPMdb9zIU6p4zYRsZqnUVs2+1VNH\nh5vn5vlChTdm7h6j0xPZTVNq9uVSzc6MhWKFuu84ONjXlsw3al/czK9zfanEenVzp0s0as3RDa1e\nmJqg5vnNEUyNjtNPHcyxL5eiVK1TKAe3qwtr/OyZMX4wvcCx0QxDmQS1ut+WSC8WKywUqxweTnNp\nttD2OfKF06PNCvuZZKxZMb9Rh6BRjK9c86h6PqWKR38qxqU7BZbXqhwfy3BsNMN8sdIsJNiaBJdr\nweiEqf0DFMp1Vss1Dg2n+c47s0wMpPipyVzbe2/Mtd9Y/+DOSpnxbHJPFKJTEryBkmAREblfa5Xg\ny0Kjl76h0dP+hdOjJGNRfN+xXKoy0p/kg4U1xrJJStU6F26tEo8EczKPjWWYXS3veOVLRGQvevrk\nKD+YXtj5hdsY6ItTqtapb5GQ7sbZ/Vku3t460d6NTDLGWmXrDpUXpiY2LUP42SNDvLahjsFQJk4y\nFuXkeH+z82ArG6dCbLVqw5fOjvP9i3PbbmPjaI1GR8mZfVkuzxc3Hdfnzo7zvXB7jx3KMZ5NsbRW\n5cKtlU3Til6YmuCVq0ushJ0n96qQ3y2UBG+gJFhERDqpVK1Tqnr4zjV77D9/YoRipU7ErO3qh4iI\ndJ8Dg33cyq93ejc+Vo0RBffrfoZbd5KqQ4uIiHSRdCLWrMT7wtTdeV6NZWqeOTVKqepx8c4qpYrH\nuRMjvD9bYLFlSCIEiXPdd1sWQ9ooFrWPfHVFREQCD1sCDOwqAX6YKAkWERHpAo3iO61FZz5zeIhC\nucadlTKnJrJtr29U4D0x3s9AKsZPrufZP5jCOXCOtrleS2vVtmWHGnOdN2rUSemRQWIiIrJLhXKN\nbGrvLx2mJFhERKSLZVPxLb9wjPQn24al3WuI2lA6zvhAkqOjGQZatlWq1qnWffoSUVbX64xlg7le\njflfA31BtdJq3W+uofvc2XF+eGWxWWjm+UfGOT+T33TFGoICNvtzKRbXqpRrHnOrKiwmIrKX5UtK\ngo2C7VcAAAyjSURBVEVERGQPMLMt144MhmgH98eyd6tsNyq1NiRiET59aBCzYKmXp0+Osl71qPs+\nZsbjhwabS4VculPg8HC6bd3N7QqpNCq4rlXrzYq3rdVgH5vMUa751Hyfw8Np4tEIc4UyyViUXF+w\nTE0mGSMRM5bXanzp7DgRg/lihUrNb6tGu1XF3obxgeS2CXo0YhwfyzBfqDQr64qIyN6mwlgiIiLS\ncZ7vKFXru7rCUPd8Imb3XLbDOcdyqcZwJoFzjkKlTiIa4WZ+ncmhPmqeoz+cl+2Hy2NV6j7nZ/Ik\nYhE+e2Soua25Qpk3Z1aIRozJoT6uLZb40tlxouHvr3k+r19bJhWP8ulDg5RrHq9eXebYWIZcX7y5\ntu3R0TS38mUSsch9L+vzxTNjrFU8Xrm69VzwaMTalve6X61L+4iI7OSnJnNMDGy9fnE3UHXoDZQE\ni4iISCetVeqsVeqMh18gnXOsluvk+jYn/r7vcOFrdkr0N3LOUfMcfvgd768uLzC1P1jjdGapRM3z\nOTDYRyIawQzenytyaChNJALJWJSltSqpeIRoxChXgyVTKp7HjeV1lopVnjgy1Jxj/jMnR5o/c/HO\natsSK43EfDAd5/BwmjdvrBCPRajVg9e0zk03C15/7vjIpiVlTo73Mz1XbD7ebk57636JyIPx/CPj\nmHXvesFKgjdQEiwiIiLyYJWqdV67tsxTR4dJxaM45+75hdn3HZ4LhtJvbF8qVUnFo80r9QDVuk88\navfcZs3z27a3Wq7x4ytLHB3NsD+XalZkB3jrxgoTuSSDfQmuL5WYHOpjeq7I1P4BLs0WODjUx2Kx\nyoHBFMlYtO33NN6bc44rC2vk+uKcvx4sdbYvF3R0VOoen54cJBYN1g1fWqtSrNS5vljizL4sh4bT\nzK2WKVU9RvoTJGNRbuXXg6kHZlxZWOPZU6OYWXN92k8dzJHri5OMRbi2VOJySwfBSH+CvkSUG0vr\nPHVsGIAr80UWi1XMgqJ3G9fT/cLpUS7dKVCt+3zqYI5oxHjr5goDqTipeIRCuY4Z9MWjvD8b/K6d\nltV57FCuuRTcRkdHM1xdWNvU/ujBAcb6k8wXK+wbSOH5QSfRhVsrHB/rJxmLNI9vQ6Nafixi/DAc\naQHB2sPZVIyby5urOZ+eyJJORplZKjVrGUwO93FjaevKz0dH01TrbsfK0N26fFIkAr6/8+vux+OH\nBxnVOsF7i5JgEREREXmQGlfwo7u4cn+/ap6Pc8Ec/Y9re7HIvTsUdlKq1olGjGQsSqEczJlvTGko\n1zzi0WBEQd3zqXmurVaAc46q52/qXLiXtUqdG8vrzCyV+PyJkbYODecclbpPKn53e41OltX1GvFY\npK0w4L1MzxVJxSNMDqWb215aq5JJxtq2v53Wzh/nHPPFCuPZFPmwYydfqlGueRwdzWz58yulGtlU\nbFcjQGD7TiUIpo9Ew3jPrpZJJ6LczK/Tn4wxkkmyUKyQS8dJRCMkYxEqdZ/1qsdQJrGrfeg0JcEb\nKAkWEREREdnbnHMUK7urHyC9436T4I+nK6lDzOzLZnbJzKbN7Gud3h8REREREXlwzEwJsHxkezYJ\nNrMo8DvAV4Ap4FfMbKqzeyUiIiIiIiLdbM8mwcBPA9POuSvOuSrwdeDFDu+TiIiIiIiIdLG9nAQf\nBGZaHt8I20RERERERES2tJeT4B2Z2a+Z2atm9ur8/Hynd0dEREREREQ6bC8nwTeBQy2PJ8O2Jufc\n7znnnnTOPTk2NvaJ7pyIiIiIiIh0n72cBL8CnDKzY2aWAF4CvtnhfRIREREREZEuFtv5Jd3JOVc3\ns38EfAuIAn/onLvQ4d0SERERERGRLrZnk2AA59xfAn/Z6f0QERERERGRvWEvD4cWERERERER2RUl\nwSIiIiIiItIzlASLiIiIiIhIz1ASLCIiIiIiIj1DSbCIiIiIiIj0DCXBIiIiIiIi0jOUBIuIiIiI\niEjPUBIsIiIiIiIiPUNJsIiIiIiIiPQMJcEiIiIiIiLSM5QEi4iIiIiISM9QEiwiIiIiIiI9w5xz\nnd6HT4SZzQPXOr0fOxgFFjq9E9JGMelOikv3UUy6k+LSfRST7qS4dB/FpPvshZgccc6N7fSinkmC\n9wIze9U592Sn90PuUky6k+LSfRST7qS4dB/FpDspLt1HMek+D1NMNBxaREREREREeoaSYBERERER\nEekZSoK7y+91egdkE8WkOyku3Ucx6U6KS/dRTLqT4tJ9FJPu89DERHOCRUREREREpGfoSrCIiIiI\niIj0DCXBXcDMvmxml8xs2sy+1un9eRiZ2R+a2ZyZvd3SNmxm3zaz98N/h8J2M7PfDuPxppk90fIz\nXw1f/76ZfbWl/bNm9lb4M79tZvbJvsO9x8wOmdn3zewdM7tgZv84bFdcOsTMUmb2YzN7I4zJb4Xt\nx8zs5fA4/omZJcL2ZPh4Onz+aMu2fiNsv2Rmv9DSrvPdh2BmUTP7iZn9RfhYMekwM7sanl/Om9mr\nYZvOXx1mZoNm9g0zu2hm75rZ5xWXzjGzM+HfSOO2ama/rph0lpn9k/Bz/m0z+2MLPv9763PFOadb\nB29AFLgMHAcSwBvAVKf362G7Ac8CTwBvt7T9S+Br4f2vAf8ivP+LwP8EDDgHvBy2DwNXwn+HwvtD\n4XM/Dl9r4c9+pdPvudtvwH7gifB+FngPmFJcOhoTA/rD+3Hg5fD4/SnwUtj+u8DfD+//A+B3w/sv\nAX8S3p8Kz2VJ4Fh4jovqfPeRYvNPgf8C/EX4WDHpfEyuAqMb2nT+6nxc/iPw98L7CWBQcemOW3i+\nuQMcUUw6GoeDwAdAX/j4T4G/1WufK7oS3Hk/DUw7564456rA14EXO7xPDx3n3P8FljY0v0jwYUn4\n799oaf8jF/gRMGhm+4FfAL7tnFtyzi0D3wa+HD434Jz7kQvOCn/Usi3ZhnPutnPu9fB+AXiX4MSs\nuHRIeGyL4cN4eHPAc8A3wvaNMWnE6hvA82EP/IvA151zFefcB8A0wblO57sPwcwmgb8G/H742FBM\nupXOXx1kZjmCTu8/AHDOVZ1zeRSXbvE8cNk5dw3FpNNiQJ+ZxYA0cJse+1xREtx5B4GZlsc3wjZ5\n8Cacc7fD+3eAifD+djG5V/uNLdrlPoVDaz5DcOVRcekgC4bdngfmCL5kXAbyzrl6+JLW49g89uHz\nK8AIu4+V3Nu/Af4Z4IePR1BMuoED/peZvWZmvxa26fzVWceAeeA/WDB94PfNLIPi0i1eAv44vK+Y\ndIhz7ibwr4DrBMnvCvAaPfa5oiRYhOAKGMEXGvmEmVk/8N+AX3fOrbY+p7h88pxznnPucWCSoDf3\nbId3qaeZ2V8H5pxzr3V6X2STZ5xzTwBfAf6hmT3b+qTOXx0RI5j69O+dc58B1giG2jYpLp0Rzi/9\nJeC/bnxOMflkhfOvXyToNDoAZIAvd3SnOkBJcOfdBA61PJ4M2+TBmw2H0RD+Oxe2bxeTe7VPbtEu\nOzCzOEEC/J+dc38WNisuXSAcQvh94PMEw9Fi4VOtx7F57MPnc8Aiu4+VbO9p4JfM7CrBkLLngH+L\nYtJx4dUUnHNzwH8n6DTS+auzbgA3nHMvh4+/QZAUKy6d9xXgdefcbPhYMemcF4APnHPzzrka8GcE\nnzU99bmiJLjzXgFOhRXZEgRDRb7Z4X3qFd8EGtUFvwr8eUv7r4YVCs8BK+GQnW8BP29mQ2Ev2s8D\n3wqfWzWzc+EciV9t2ZZsIzxWfwC865z71y1PKS4dYmZjZjYY3u8Dfo5grvb3gV8OX7YxJo1Y/TLw\nvbBH/5vAS2FFyWPAKYLCJTrf7ZJz7jecc5POuaMEx+t7zrm/iWLSUWaWMbNs4z7BeedtdP7qKOfc\nHWDGzM6ETc8D76C4dINf4e5QaFBMOuk6cM7M0uExa/yd9NbniuuC6ly9fiOohPcewdy73+z0/jyM\nN4IT722gRtBT/HcJ5jN8F3gf+A4wHL7WgN8J4/EW8GTLdv4OwcT/aeBvt7Q/SfAF6DLw7wDr9Hvu\n9hvwDMHwpzeB8+HtFxWXjsbkMeAnYUzeBv552H6c4INtmmAoWzJsT4WPp8Pnj7ds6zfD436Jlkqd\nOt99pPh8kbvVoRWTzsbiOEHF0zeAC43jpvNX52/A48Cr4XnsfxBUElZcOhuTDMGVw1xLm2LS2Zj8\nFnAxPG7/iaDCc099rli4oyIiIiIiIiIPPQ2HFhERERERkZ6hJFhERERERER6hpJgERERERER6RlK\ngkVERERERKRnKAkWERERERGRnqEkWERERERERHqGkmARERERERHpGUqCRUREREREpGf8f1JNUpq8\ncECsAAAAAElFTkSuQmCC\n", 691 | "text/plain": [ 692 | "" 693 | ] 694 | }, 695 | "metadata": {}, 696 | "output_type": "display_data" 697 | } 698 | ], 699 | "source": [ 700 | "plt.figure(figsize=(16,9))\n", 701 | "plt.plot(advi.hist, alpha=.3);" 702 | ] 703 | }, 704 | { 705 | "cell_type": "markdown", 706 | "metadata": {}, 707 | "source": [ 708 | "### Important to note\n", 709 | "Calling `lasagne.layers.get_output` we in fact get symbolic output of the model. The inferred approximation is not yet applied. Good news are that `pymc3.variatonal` is designed to meet the needs of Bayesian deep learning. It is pretty easy to make it work with minimal efforts and lines of code." 710 | ] 711 | }, 712 | { 713 | "cell_type": "code", 714 | "execution_count": 43, 715 | "metadata": {}, 716 | "outputs": [], 717 | "source": [ 718 | "from theano.configparser import change_flags\n", 719 | "\n", 720 | "# The right way to compile a function without changing important pymc3 flag `compute_test_value='raise'`\n", 721 | "with change_flags(compute_test_value='ignore'):\n", 722 | " # create symbolic input image\n", 723 | " inpimg = tt.tensor4('input')\n", 724 | " # number of samples for posterior predictive distribution\n", 725 | " it = tt.iscalar('i')\n", 726 | " # posterior predictive probability\n", 727 | " _prediction = gelato.layers.get_output(network)\n", 728 | " # then replacements follow\n", 729 | " prediction = advi.approx.apply_replacements(\n", 730 | " _prediction, deterministic=True, \n", 731 | " # you can replace minibatch tensor with symbolic input\n", 732 | " more_replacements={input_var:inpimg})\n", 733 | " \n", 734 | " predictions = advi.approx.sample_node(\n", 735 | " _prediction, it, \n", 736 | " more_replacements={input_var:inpimg})\n", 737 | " # That is it, finally we compile both functions\n", 738 | " predictions_f = theano.function([inpimg, theano.In(it, 's', 10)], predictions)\n", 739 | " prediction_f = theano.function([inpimg], prediction)" 740 | ] 741 | }, 742 | { 743 | "cell_type": "markdown", 744 | "metadata": {}, 745 | "source": [ 746 | "## Making predictions\n", 747 | "\n", 748 | "There are different approaches to make decisions under Bayesian setup. One can use __MAP__ esimation for prediction, other options are to use mean of the posterior or predictive distribution and integrate out any statistics, usually mean or mode. I'll compare MAP and posterior predictive mode" 749 | ] 750 | }, 751 | { 752 | "cell_type": "code", 753 | "execution_count": 44, 754 | "metadata": { 755 | "collapsed": true 756 | }, 757 | "outputs": [], 758 | "source": [ 759 | "from scipy.stats import mode\n", 760 | "y_pred_MAP = np.argmax(prediction_f(X_test), axis=1)\n", 761 | "error_under_MAP = y_pred_MAP != y_test\n", 762 | "error_rate_under_MAP = error_under_MAP.mean()\n", 763 | "\n", 764 | "# distribution for probabilistic predictions\n", 765 | "y_preds_posterior = predictions_f(X_test, 100)\n", 766 | "# take distribution for modes\n", 767 | "# than integrate out mode for modes\n", 768 | "y_pred_posterior = mode(np.argmax(y_preds_posterior, axis=-1), axis=0).mode[0]\n", 769 | "error_under_posterior = y_pred_posterior != y_test\n", 770 | "error_rate_under_posterior = error_under_posterior.mean()" 771 | ] 772 | }, 773 | { 774 | "cell_type": "code", 775 | "execution_count": 45, 776 | "metadata": {}, 777 | "outputs": [ 778 | { 779 | "name": "stdout", 780 | "output_type": "stream", 781 | "text": [ 782 | "MAP : 0.095900\n", 783 | "predictive posterior mode: 0.074800\n" 784 | ] 785 | } 786 | ], 787 | "source": [ 788 | "print('MAP : %f' % error_rate_under_MAP)\n", 789 | "print('predictive posterior mode: %f' % error_rate_under_posterior)" 790 | ] 791 | }, 792 | { 793 | "cell_type": "markdown", 794 | "metadata": {}, 795 | "source": [ 796 | "Seems like MAP estimation may be not the best way to predict. On the other hand it is much faster" 797 | ] 798 | }, 799 | { 800 | "cell_type": "code", 801 | "execution_count": 46, 802 | "metadata": { 803 | "collapsed": true 804 | }, 805 | "outputs": [], 806 | "source": [ 807 | "def check_the_error_at(idx):\n", 808 | " print('true:', y_test[error_under_posterior][idx],'prediction:', y_pred_posterior[error_under_posterior][idx])\n", 809 | " plt.gray();plt.matshow(X_test[error_under_posterior][idx][0]);plt.show();" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 47, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "name": "stdout", 819 | "output_type": "stream", 820 | "text": [ 821 | "true: 3.0 prediction: 2\n" 822 | ] 823 | }, 824 | { 825 | "data": { 826 | "text/plain": [ 827 | "" 828 | ] 829 | }, 830 | "metadata": {}, 831 | "output_type": "display_data" 832 | }, 833 | { 834 | "data": { 835 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADldJREFUeJzt3V+MVeW5x/HfY/VcOHqBOocQK3pqjDO1SREH0ws0muY0\n2BuZqOQQFXrhwZCaIDkXh+CFRi00J8VjrxR6JAVDPTGBqaYxKcY0wtwYcDAKDD02DULJODhyoQaT\nE+XpxSyf7tKZ9917rz17rQ3fT0L23uvZfx7WDD/Wn3e/y9xdACBJl1TdAID6IBAABAIBQCAQAAQC\nAUAgEACESgLBzJaZ2R/N7E9mtqGKHlLM7LiZfWBm75nZwRr0s93MTpvZ4YZlV5nZm2b2YXE7r2b9\nPWVmp4p1+J6Z/bjC/q4zsz+Y2VEzO2Jm64rltViHif66vg6t2+MQzOxbkv5P0r9K+oukA5JWuvvR\nrjaSYGbHJQ25+1TVvUiSmd0p6QtJO939e8Wy/5J0xt1/XoTqPHf/zxr195SkL9z9F1X01MjMFkha\n4O5jZnalpHclLZf0E9VgHSb6W6Eur8MqthBul/Qnd/+zu/+/pP+VdG8FffQMd98n6cx5i++VtKO4\nv0PTv0CVmKW/2nD3CXcfK+5/Lmlc0rWqyTpM9Nd1VQTCtZJONjz+iyr6yye4pL1m9q6Zram6mVnM\nd/eJ4v7HkuZX2cwsHjOz94tdisp2aRqZ2Q2SbpX0jmq4Ds/rT+ryOuSg4syWuvtiSfdI+mmxSVxb\nPr3fV7cx6C9IulHSIkkTkrZU245kZldI2i3pcXf/rLFWh3U4Q39dX4dVBMIpSdc1PP52saw23P1U\ncXta0oimd3PqZrLY9/xmH/R0xf38HXefdPev3f2cpF+p4nVoZpdp+h/bLnffUyyuzTqcqb8q1mEV\ngXBA0k1m9i9m9k+S/k3S6xX0MSMz6ysO7MjM+iT9SNLh9Ksq8bqk1cX91ZJeq7CXf/DNP7TCsCpc\nh2Zmkl6SNO7uzzWUarEOZ+uvinXY9bMMklScPnle0rckbXf3n3W9iVmY2Xc0vVUgSZdK+k3V/ZnZ\nK5LuknSNpElJT0r6raRXJS2U9JGkFe5eyYG9Wfq7S9Obui7puKRHG/bXu93fUkn7JX0g6VyxeKOm\n99MrX4eJ/laqy+uwkkAAUE8cVAQQCAQAgUAAEAgEAIFAABAqDYQaDwuWRH9l1bm/OvcmVddf1VsI\ntf6hiP7KqnN/de5Nqqi/qgMBQI2UGphkZssk/VLTIw7/x91/nnk+o6CAiri75Z7TdiC0M9EJgQBU\np5lAKLPLwEQnwAWmTCD0wkQnAFpw6Vx/QHH6pO5HdAGoXCA0NdGJu2+TtE3iGAJQd2V2GWo90QmA\n1rW9heDuX5nZY5J+r79NdHKkY50B6LquTpDCLgNQnbk+7QjgAkMgAAgEAoBAIAAIBAKAQCAACAQC\ngEAgAAgEAoBAIAAIBAKAQCAACAQCgEAgAAgEAoBAIAAIBAKAQCAACAQCgEAgAAgEAoBAIAAIBAKA\nQCAACAQCgEAgAAgEAoBAIAAIBAKAQCAACJdW3cCFZGBgIFm/5557kvWbb7651OcPDw8n68eOHUvW\nc/2bpa8mvmfPnmQ9Z3BwMFkfHx8v9f7PP/98sp5bPxeDUoFgZsclfS7pa0lfuftQJ5oCUI1ObCHc\n7e5THXgfABXjGAKAUDYQXNJeM3vXzNZ0oiEA1Sm7y7DU3U+Z2T9LetPMjrn7vsYnFEFBWAA9oNQW\ngrufKm5PSxqRdPsMz9nm7kMccATqr+1AMLM+M7vym/uSfiTpcKcaA9B95u7tvdDsO5reKpCmdz1+\n4+4/y7ymvQ+ridx5/p07dybrl19+ebKe+1nkxgHw+vTrv/zyy2R9ZGQkWV+1alWyXnfunl6BKnEM\nwd3/LOn77b4eQP1w2hFAIBAABAIBQCAQAAQCAUAgEACEtschtPVhNR+H0N/fn6xPTk4m67l1efLk\nyWR969atyfqnn36arOd88sknpV6fWz9VW7p0abL+8MMPJ+vnzp1L1u++++5kfd++fcl61ZoZh8AW\nAoBAIAAIBAKAQCAACAQCgEAgAAgEAoDAdRka5K6LkBtnkKsPDaUnjZqaYvLqMnLXlciNM8j9/I4e\nPdpyT72GLQQAgUAAEAgEAIFAABAIBACBQAAQCAQAgXEIDUZHR0vVc+fBGWdQzjPPPJOsr1u3Llm/\n5JL0/3/PPvtssn4x/PzYQgAQCAQAgUAAEAgEAIFAABAIBACBQAAQuC5DCxYuXFjq9SdOnOhQJxem\nJ554Ill/+umnk/Xc73LuuhZLlixJ1nv959eR6zKY2XYzO21mhxuWXWVmb5rZh8XtvLLNAqheM7sM\nv5a07LxlGyS95e43SXqreAygx2UDwd33STpz3uJ7Je0o7u+QtLzDfQGoQLsHFee7+0Rx/2NJ8zvU\nD4AKlf5yk7t76mChma2RtKbs5wCYe+1uIUya2QJJKm5Pz/ZEd9/m7kPunp5yGEDl2g2E1yWtLu6v\nlvRaZ9oBUKXsOAQze0XSXZKukTQp6UlJv5X0qqSFkj6StMLdzz/wONN79fQ4BKT19/cn62+//Xay\nnrsuhln6NPr4+HiyfssttyTrF7pmxiFkjyG4+8pZSj9suSMAtcbQZQCBQAAQCAQAgUAAEAgEAIFA\nABC4LgOadueddybrW7ZsSdZz4wxyY2I2bdqUrG/evDlZRx5bCAACgQAgEAgAAoEAIBAIAAKBACAQ\nCAAC4xAuIn19fcn6hg3pybNz103IjSM4efJksr5+/fpkfWRkJFlHeWwhAAgEAoBAIAAIBAKAQCAA\nCAQCgEAgAAiMQ7iADAwMJOu7d+9O1svOV7Bnz55kfe3atcn61NRUso65xxYCgEAgAAgEAoBAIAAI\nBAKAQCAACAQCgMA4hB5y2223JetvvPFGst7f35+s58YZ3H///ck68xX0vuwWgpltN7PTZna4YdlT\nZnbKzN4r/vx4btsE0A3N7DL8WtKyGZb/t7svKv6k/2sC0BOygeDu+ySd6UIvACpW5qDiY2b2frFL\nMa9jHQGoTLuB8IKkGyUtkjQhadarfJrZGjM7aGYH2/wsAF3SViC4+6S7f+3u5yT9StLtieduc/ch\ndx9qt0kA3dFWIJjZgoaHw5IOz/ZcAL3DcueezewVSXdJukbSpKQni8eLJLmk45IedfeJ7IeZpT8M\nSS+++GKy/sgjjyTrZpas534XDh06lKzPtf379yfrufkYRkdHO9lOz3H39C+AmhiY5O4rZ1j8Ulsd\nAag1hi4DCAQCgEAgAAgEAoBAIAAIBAKAkB2H0NEPu8jHIfT19SXrGzZsSNbvu+++ZD13XYWxsbFk\nfa7l5nNoYkxMqddv2rQpWd+8eXOyfvbs2WS97poZh8AWAoBAIAAIBAKAQCAACAQCgEAgAAgEAoDA\nOIQuOnLkSLKeG0eQ+z5/7jz73r17k/W5tnjx4mR9cHAwWR8eHk7Wly9fnqznxjGsWrUqWd+1a1ey\nXneMQwDQEgIBQCAQAAQCAUAgEAAEAgFAIBAABMYhdNDLL7+crD/44IPJ+tBQ+uJWVc9nUHcHDhxI\n1nPzMWzdujVZX7t2bcs91QnjEAC0hEAAEAgEAIFAABAIBACBQAAQCAQAIXs5ePzNwMBAsp77Pv74\n+HiyfuzYsZZ7upjk1n+u3s0xN70qu4VgZteZ2R/M7KiZHTGzdcXyq8zsTTP7sLidN/ftAphLzewy\nfCXpP9z9u5J+IOmnZvZdSRskveXuN0l6q3gMoIdlA8HdJ9x9rLj/uaRxSddKulfSjuJpOySlt5cB\n1F5LBxXN7AZJt0p6R9J8d58oSh9Lmt/RzgB0XdMHFc3sCkm7JT3u7p81Tljp7j7bF5fMbI2kNWUb\nBTD3mtpCMLPLNB0Gu9x9T7F40swWFPUFkk7P9Fp33+buQ+6e/iofgMo1c5bBJL0kadzdn2sovS5p\ndXF/taTXOt8egG7KzodgZksl7Zf0gaRzxeKNmj6O8KqkhZI+krTC3c9k3uuCPhGc+z5+br6Dhx56\nKFnv9esCXH/99cn6HXfckaxv3LgxWc9d1+Ls2bPJ+pIlS5L1Xh8n0sx8CNljCO4+Kmm2N/phq00B\nqC+GLgMIBAKAQCAACAQCgEAgAAgEAoDAdRk6aOfOncl67roMuZ/FoUOHkvXcfAuNw83b+fzR0dFk\nfXh4OFlfvHhxsn711Vcn62X7f+CBB5L1kZGRZL3XcV0GAC0hEAAEAgFAIBAABAIBQCAQAAQCAUBg\nHEIX5eZLyF1XoK+vL1lvYm6Lnn792NhYsr5+/fpkPTeO4kLHOAQALSEQAAQCAUAgEAAEAgFAIBAA\nBAIBQGAcQo3kxiEsW7YsWc9dl2BwcDBZz82nUFZuHEDu80+cOJGsT01NtdzTxYRxCABaQiAACAQC\ngEAgAAgEAoBAIAAIBAKAkB2HYGbXSdopab4kl7TN3X9pZk9J+ndJnxRP3ejub2Tei3EIQEWaGYfQ\nTCAskLTA3cfM7EpJ70paLmmFpC/c/RfNNkQgANVpJhAubeJNJiRNFPc/N7NxSdeWbw9A3bR0DMHM\nbpB0q6R3ikWPmdn7ZrbdzOZ1uDcAXdZ0IJjZFZJ2S3rc3T+T9IKkGyUt0vQWxJZZXrfGzA6a2cEO\n9AtgDjX15SYzu0zS7yT93t2fm6F+g6Tfufv3Mu/DMQSgIh35cpNNT5X7kqTxxjAoDjZ+Y1jS4Xaa\nBFAfzZxlWCppv6QPJJ0rFm+UtFLTuwsu6bikR4sDkKn3YgsBqEhHTjt2EoEAVIf5EAC0hEAAEAgE\nAIFAABAIBACBQAAQCAQAgUAAEAgEAIFAABAIBACBQAAQCAQAgUAAEAgEACE763KHTUn6qOHxNcWy\nuqK/curcX517kzrf3/XNPKmrE6T8w4ebHXT3ocoayKC/curcX517k6rrj10GAIFAABCqDoRtFX9+\nDv2VU+f+6tybVFF/lR5DAFAvVW8hAKgRAgFAIBAABAIBQCAQAIS/AkJCov02bLkcAAAAAElFTkSu\nQmCC\n", 836 | "text/plain": [ 837 | "" 838 | ] 839 | }, 840 | "metadata": {}, 841 | "output_type": "display_data" 842 | } 843 | ], 844 | "source": [ 845 | "check_the_error_at(0)" 846 | ] 847 | }, 848 | { 849 | "cell_type": "code", 850 | "execution_count": 48, 851 | "metadata": {}, 852 | "outputs": [ 853 | { 854 | "name": "stdout", 855 | "output_type": "stream", 856 | "text": [ 857 | "true: 9.0 prediction: 4\n" 858 | ] 859 | }, 860 | { 861 | "data": { 862 | "text/plain": [ 863 | "" 864 | ] 865 | }, 866 | "metadata": {}, 867 | "output_type": "display_data" 868 | }, 869 | { 870 | "data": { 871 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADOlJREFUeJzt3W+IXXV+x/HPx40F/6EzhB1DaqsGB1x8EMughYSSqo3p\nPok+UMyDNcV1kwcrJLDCah5onhSkqNtVRBibsLPqTgloGpXQbtQ1dhU0UaJG09ZliTZhzDQoGBEs\nmm8fzMm3Y3bmd+/M/XPOZN8vCHPv+Z57zzcnySe/37m/OeOIEABI0ll1NwCgOQgEAIlAAJAIBACJ\nQACQCAQAqZZAsL3G9n/a/p3te+roocT2Ydvv2T5ge38D+tlue9L2wWnbBm3vsf1h9XWgYf1ttX20\nOocHbH+/xv4usf0b2x/Yft/2pmp7I85hob++n0P3ex2C7e9I+i9JfyPpiKR9ktZFxAd9baTA9mFJ\nIxFxvO5eJMn2X0n6QtIvI+Kqats/SPo0Ih6oQnUgIn7aoP62SvoiIh6so6fpbC+RtCQi3rZ9gaS3\nJN0k6e/UgHNY6O9W9fkc1jFCuEbS7yLi9xHxv5L+WdLaGvpYMCLiVUmfnrZ5raSx6vGYpv4C1WKW\n/hojIiYi4u3q8QlJhyQtVUPOYaG/vqsjEJZK+u9pz4+opt98QUj6te23bG+ou5lZDEXERPX4E0lD\ndTYzi7tsv1tNKWqb0kxn+1JJV0t6Qw08h6f1J/X5HHJRcWYrI+IvJP2tpB9XQ+LGiql5X9PWoD8u\naZmk5ZImJD1UbzuS7fMlPSNpc0R8Pr3WhHM4Q399P4d1BMJRSZdMe/6n1bbGiIij1ddJSTs1Nc1p\nmmPV3PPUHHSy5n6+JSKORcQ3EXFS0hOq+RzaPltT/9iejohnq82NOYcz9VfHOawjEPZJusL2Zbb/\nRNJtkp6roY8Z2T6vurAj2+dJWi3pYPlVtXhO0vrq8XpJu2rs5Q+c+odWuVk1nkPblrRN0qGIeHha\nqRHncLb+6jiHff+UQZKqj0/+UdJ3JG2PiL/vexOzsH25pkYFkrRI0q/q7s/2uKRVkhZLOibpfkn/\nImmHpD+T9JGkWyOilgt7s/S3SlND3ZB0WNLGafP1fve3UtK/S3pP0slq8xZNzdNrP4eF/tapz+ew\nlkAA0ExcVASQCAQAiUAAkAgEAIlAAJBqDYQGLwuWRH+danJ/Te5Nqq+/ukcIjf5DEf11qsn9Nbk3\nqab+6g4EAA3S0cIk22sk/VxTKw7/KSIeaLE/q6CAmkSEW+0z70CYz41OCASgPu0EQidTBm50Apxh\nOgmEhXCjEwBzsKjXB6g+Pmn6FV0A6iwQ2rrRSUSMShqVuIYANF0nU4ZG3+gEwNzNe4QQEV/bvkvS\nv+n/b3Tyftc6A9B3fb1BClMGoD69/tgRwBmGQACQCAQAiUAAkAgEAIlAAJAIBACJQACQCAQAiUAA\nkAgEAIlAAJAIBACJQACQCAQAiUAAkAgEAIlAAJAIBACJQACQCAQAiUAAkAgEAIlAAJAIBACJQACQ\nCAQAiUAAkAgEAIlAAJAIBABpUScvtn1Y0glJ30j6OiJGutEUgHp0FAiVv46I4114HwA1Y8oAIHUa\nCCHp17bfsr2hGw0BqE+nU4aVEXHU9ncl7bH9HxHx6vQdqqAgLIAFwBHRnTeyt0r6IiIeLOzTnYMB\nmLOIcKt95j1lsH2e7QtOPZa0WtLB+b4fgPp1MmUYkrTT9qn3+VVE/GtXugJQi65NGdo6GFMGoDY9\nnTIAOPMQCAASgQAgEQgAEoEAIBEIAFI3vtsRC8RFF11UrI+Ojhbrt9xyS7G+b9++Yv3GG28s1j/7\n7LNiHb3HCAFAIhAAJAIBQCIQACQCAUAiEAAkAgFA4tufzyArV64s1p9//vli/cILLyzWjx49Wqwv\nXbq0WH/qqaeK9dtvv71YR2f49mcAc0IgAEgEAoBEIABIBAKARCAASAQCgMT9EBaQgYGBYn3Hjh3F\nevUzNGZ13XXXFeuvv/56sb5+/fpifcuWLcU66scIAUAiEAAkAgFAIhAAJAIBQCIQACQCAUBiHcIC\ncu+99xbr5557brG+du3aYn3v3r1z7mm6bdu2FetXXXVVR++P3ms5QrC93fak7YPTtg3a3mP7w+pr\necUMgAWhnSnDLyStOW3bPZJeiogrJL1UPQewwLUMhIh4VdKnp21eK2msejwm6aYu9wWgBvO9qDgU\nERPV408kDXWpHwA16viiYkRE6eaptjdI2tDpcQD03nxHCMdsL5Gk6uvkbDtGxGhEjETEyDyPBaBP\n5hsIz0k69b2u6yXt6k47AOrUcspge1zSKkmLbR+RdL+kByTtsP1DSR9JurWXTf6xWLFiRbF+5513\nFut33313sd7pOoNWTp48Way3+rkNw8PDxfqaNad/2PVtjzzySLGO1loGQkSsm6V0fZd7AVAzli4D\nSAQCgEQgAEgEAoBEIABIBAKA5IhZVx13/2CFJc5o/Tn6tddeW6zfcMMNxfqJEyfm3FM3vfzyy8X6\nZZddVqyfc845xfrFF188557+mERE+QdziBECgGkIBACJQACQCAQAiUAAkAgEAIlAAJD4uQwNsnjx\n4mJ9fHy8WK97ncFtt91WrF955ZXF+tBQ+dack5Oz3pgLXcIIAUAiEAAkAgFAIhAAJAIBQCIQACQC\nAUBiHUKDXH99+c729913X0+P3+p+A1u3bi3WN23aVKwfOXKkWP/444+L9RdffLFYR+cYIQBIBAKA\nRCAASAQCgEQgAEgEAoBEIABIrENYQJYtW1asf/nll8X68PBwsf7YY48V6wMDA8X6xo0bi/UXXnih\nWH/yySeL9Va/P3Su5QjB9nbbk7YPTtu21fZR2weqX9/vbZsA+qGdKcMvJK2ZYfvPImJ59Wt3d9sC\nUIeWgRARr0r6tA+9AKhZJxcV77L9bjWlKE8uASwI8w2ExyUtk7Rc0oSkh2bb0fYG2/tt75/nsQD0\nybwCISKORcQ3EXFS0hOSrinsOxoRIxExMt8mAfTHvALB9pJpT2+WdHC2fQEsHC3XIdgel7RK0mLb\nRyTdL2mV7eWSQtJhSeUPoNEVY2NjxfqiReU/zsHBwWJ9586dxfqjjz5arL/yyivFeitvvvlmsd6q\nf3SuZSBExLoZNm/rQS8AasbSZQCJQACQCAQAiUAAkAgEAIlAAJC4H0KDbN68uVi/4447ivXXXnut\nWN+9u/xNqe+8806x/tVXXxXrnVq9enWxvn8/q997jRECgEQgAEgEAoBEIABIBAKARCAASAQCgMQ6\nhAYZHx/vqA50ihECgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQg\nAEgEAoBEIABILQPB9iW2f2P7A9vv295UbR+0vcf2h9XXgd63C6CX2hkhfC3pJxHxPUl/KenHtr8n\n6R5JL0XEFZJeqp4DWMBaBkJETETE29XjE5IOSVoqaa2ksWq3MUk39apJAP0xp2sIti+VdLWkNyQN\nRcREVfpE0lBXOwPQd23fZNX2+ZKekbQ5Ij63nbWICNsxy+s2SNrQaaMAeq+tEYLtszUVBk9HxLPV\n5mO2l1T1JZImZ3ptRIxGxEhEjHSjYQC9086nDJa0TdKhiHh4Wuk5Seurx+sl7ep+ewD6qZ0pwwpJ\nP5D0nu0D1bYtkh6QtMP2DyV9JOnW3rQIoF9aBkJE/FaSZylf3912ANSJlYoAEoEAIBEIABKBACAR\nCAASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAILV9CzWgbnv37q27hTMeIwQAiUAAkAgE\nAIlAAJAIBACJQACQCAQAiXUI6Juzzir//9OqPjw83M12MANGCAASgQAgEQgAEoEAIBEIABKBACAR\nCABSy3UIti+R9EtJQ5JC0mhE/Nz2Vkk/kvQ/1a5bImJ3rxrFwjc4OFisX3755cX6kiVLutkOZtDO\nwqSvJf0kIt62fYGkt2zvqWo/i4gHe9cegH5qGQgRMSFponp8wvYhSUt73RiA/pvTNQTbl0q6WtIb\n1aa7bL9re7vtgS73BqDP2g4E2+dLekbS5oj4XNLjkpZJWq6pEcRDs7xug+39tvd3oV8APdRWINg+\nW1Nh8HREPCtJEXEsIr6JiJOSnpB0zUyvjYjRiBiJiJFuNQ2gN1oGgm1L2ibpUEQ8PG379Eu+N0s6\n2P32APRTO58yrJD0A0nv2T5QbdsiaZ3t5Zr6KPKwpI096RBA37TzKcNvJXmGEmsOMCfHjx8v1nfv\nLv+V2rVrVzfbwQxYqQggEQgAEoEAIBEIABKBACARCAASgQAgOSL6dzC7fwcD8C0RMdN6om9hhAAg\nEQgAEoEAIBEIABKBACARCAASgQAgtXODlG46Lumjac8XV9uaiv460+T+mtyb1P3+/rydnfq6MOkP\nDm7vb/K9FumvM03ur8m9SfX1x5QBQCIQAKS6A2G05uO3Qn+daXJ/Te5Nqqm/Wq8hAGiWukcIABqE\nQACQCAQAiUAAkAgEAOn/AJgQk4TqlfWrAAAAAElFTkSuQmCC\n", 872 | "text/plain": [ 873 | "" 874 | ] 875 | }, 876 | "metadata": {}, 877 | "output_type": "display_data" 878 | } 879 | ], 880 | "source": [ 881 | "check_the_error_at(1)" 882 | ] 883 | }, 884 | { 885 | "cell_type": "markdown", 886 | "metadata": {}, 887 | "source": [ 888 | "# Uncertainty\n", 889 | "From a glance uncertainty is not important. But I'll show that it can used for making decisions.\n", 890 | "\n", 891 | "\n", 892 | "## Simple variance \n", 893 | "Let's calculate average variance for probability predictions in correct pedictions and wrong predictions. I don't use Chi squared statistics to illustrate the difference as [@twiecki](https://github.com/twiecki) did. But still, this picture gives an evidence that mistakes are done with more quantity of uncertainty and thus can be potentially treated as problebatic examples." 894 | ] 895 | }, 896 | { 897 | "cell_type": "code", 898 | "execution_count": 49, 899 | "metadata": { 900 | "scrolled": true 901 | }, 902 | "outputs": [ 903 | { 904 | "data": { 905 | "text/plain": [ 906 | "" 907 | ] 908 | }, 909 | "execution_count": 49, 910 | "metadata": {}, 911 | "output_type": "execute_result" 912 | }, 913 | { 914 | "data": { 915 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VFX6wPHvSU9IDwklhYTea+gg2Cj2hmIFBcsPu66K\nuys2VsVFFEFFULFLs7EqYEV6Cb0FCCGQhJJGQnqZOb8/7hASCCGBJNPez/PMk8nMmTvvDOG+93Sl\ntUYIIYRwsXYAQgghbIMkBCGEEIAkBCGEEBaSEIQQQgCSEIQQQlhIQhBCCAFIQhBCCGEhCUEIIQQg\nCUEIIYSFm7UDqI3GjRvr6Ohoa4chhBB2ZdOmTRla69DzlbOrhBAdHU1cXJy1wxBCCLuilDpUk3LS\nZCSEEAKQhCCEEMJCEoIQQgjAzvoQqlJaWkpKSgpFRUXWDsWmeHl5ERERgbu7u7VDEULYCbtPCCkp\nKfj5+REdHY1Sytrh2AStNZmZmaSkpBATE2PtcIQQdsLum4yKiooICQmRZFCBUoqQkBCpNQkhasXu\nEwIgyaAK8p0IIWrLIRKCEEI4qsT0PP67LB6Tuf63O5aEIIQQNiojr5ixczfyzYZk0nLrvwlYEkID\nMJlM1f5+LmVlZfURjhDCDhSWmBj/WRxpuUV8PCaWZgHe9f6ekhDqwJdffkmfPn3o3r07Dz74ICaT\nCV9fX55++mm6devG2rVriY6O5rnnnqNnz54sXLiQrVu30q9fP7p27cqNN97IiRMnABg6dChPPPEE\nsbGxTJ8+nYULF9K5c2e6devGJZdcYuVPKoRoCCaz5vF5W9iWks300T3oERXUIO9r98NOK3r5f7vY\nfeRknR6zY3N/Xry20zmf37NnD/Pnz2f16tW4u7szYcIEvvrqK/Lz8+nbty9vvfVWedmQkBA2b94M\nQNeuXZkxYwZDhgxh0qRJvPzyy7zzzjsAlJSUlK/Z1KVLF5YtW0Z4eDjZ2dl1+tmEELZHa82rP+3m\n193Heenajgzv1LTB3tuhEoI1/PHHH2zatInevXsDUFhYSFhYGK6urtx8882Vyt52220A5OTkkJ2d\nzZAhQwAYM2YMo0aNOqscwMCBAxk7diy33norN910U31/HCGElX286iCfrkli/KAYxg5s2HlEDpUQ\nqruSry9aa8aMGcPrr79e6fGpU6fi6upa6bFGjRrV6JgVy82aNYv169fz888/06tXLzZt2kRISMjF\nBy6EsDm/7DjKf37Zw8jOTfnnVR0a/P1r1IeglBqhlNqrlEpQSk2s4nlPpdR8y/PrlVLRlsevVEpt\nUkrtsPy8rMJrlluOudVyC6urD9WQLr/8chYtWkRaWhoAWVlZHDpU/UqzAQEBBAUFsXLlSgC++OKL\n8trCmQ4cOEDfvn155ZVXCA0NJTk5uW4/gBDCJmw6lMUT87fSMyqIt2/rjotLw88lOm8NQSnlCrwH\nXAmkABuVUou11rsrFBsHnNBat1ZKjQamALcBGcC1WusjSqnOwDIgvMLr7tRa2/UGBx07dmTy5MkM\nGzYMs9mMu7s777333nlf99lnn/HQQw9RUFBAy5YtmTt3bpXlnnnmGfbv34/Wmssvv5xu3brV9UcQ\nQlhZYnoe4z+LIzzQmzn3xOLl7nr+F9UDpXX1kx2UUv2Bl7TWwy2/Pw+gtX69QpllljJrlVJuwDEg\nVFc4uDKmzmYCzbTWxUqp5cA/apMQYmNj9Zkb5OzZs4cOHRq+amUP5LsRwvZl5BVz0/tryC8u47sJ\nA2gRUrOm5dpQSm3SWseer1xNmozCgYrtFClUvsqvVEZrXQbkAGc2dN8MbNZaF1d4bK6luegFJWst\nCCGcTMW5Bh+Nia2XZFAbDTIPQSnVCaMZ6cEKD9+pte4CDLbc7j7Hax9QSsUppeLS09PrP1ghhGgA\n1pprUJ2aJIRUILLC7xGWx6osY2kyCsBoHkIpFQF8D9yjtT5w6gVa61TLz1zga6BPVW+utZ6ttY7V\nWseGhp53j2ghhLB5FecavHhNw841qE5NEsJGoI1SKkYp5QGMBhafUWYxMMZy/xbgT621VkoFAj8D\nE7XWq08VVkq5KaUaW+67A9cAOy/uowghhH2w5lyD6pw3IVj6BB7BGCG0B1igtd6llHpFKXWdpdjH\nQIhSKgF4Cjg1NPURoDUw6YzhpZ7AMqXUdmArRg1jTl1+MCGEsEXWnmtQnRpNTNNa/wL8csZjkyrc\nLwJGVfG6ycDkcxy2V83DFEII+2cLcw2qI4vb2bFPP/2UI0eOWDsMIUQN2Mpcg+pIQrCSM5e2vpCl\nriUhCGEfTu1r4KIUn97bm+BGHtYOqUoOtZaRtXz++edMnToVpRRdu3bl1Vdf5b777iMjI4PQ0FDm\nzp1LVFQUY8eOxcvLiy1btjBw4ED8/f05cOAAiYmJREVF8eWXXzJx4kSWL19OcXExDz/8MA8+aIzU\nnTJlCl9++SUuLi6MHDmS2NhY4uLiuPPOO/H29mbt2rV4e9f/eulCiNqpONfgm/v7WX2uQXUcKyEs\nmQjHdtTtMZt2gZFvnPPpXbt2MXnyZNasWUPjxo3JyspizJgx5bdPPvmExx57jB9++AGAlJQU1qxZ\ng6urKy+99BK7d+9m1apVeHt7M3v2bAICAti4cSPFxcUMHDiQYcOGER8fz48//sj69evx8fEhKyuL\n4OBgZs6cydSpU4mNPe8ERCGEFVScazDrrl42MdegOo6VEKzgzz//ZNSoUTRu3BiA4OBg1q5dy3ff\nfQfA3XffzbPPPlteftSoUZVWQb3uuuvKr+x//fVXtm/fzqJFiwBjmez9+/fz+++/c++99+Lj41P+\nHkII22bNfQ0ulGMlhGqu5G3FmUtgV/xda82MGTMYPnx4pTLLli1rkNiEEHXHVucaVEc6lS/SZZdd\nxsKFC8nMzASM5a8HDBjAvHnzAPjqq68YPHhwjY41fPhwPvjgA0pLSwHYt28f+fn5XHnllcydO5eC\ngoLy9wDw8/MjNze3rj+SEOIi2fJcg+o4Vg3BCjp16sS//vUvhgwZgqurKz169GDGjBnce++9/Pe/\n/y3vVK6J8ePHk5SURM+ePdFaExoayg8//MCIESPYunUrsbGxeHh4cNVVV/Haa68xduxYHnroIelU\nFsKG2Ppcg+qcd/lrWyLLX9eOfDdCNKzE9Dxu/mANgT4efPt/A2xmeGldLn8thBDiPOxlrkF1pMlI\nCCEukj3NNaiOJAQhhLgI9jbXoDrSZCScnslsP/1owrbY6r4GF0oSgnBayVkF3D57HQPe+IP4Yyet\nHY6wQ/Y416A6khCE09Fa8+W6Qwx/ZwU7UnMwa7jtw3VsTc62dmjCjtjrXIPqSEKoJ9HR0WRkZADg\n6+tr5WjEKUeyC7nnkw38+4ed9IwKYtmTl/Dd/w3A39uNO+esY31iprVDFHbAnucaVEcSgnAKWmsW\nxCUz/O0VxCWd4NUbOvPFuD6EB3oTGezDwgcH0DTAizFzN/D3vnRrhytsmD3sa3ChJCHUgRtuuIFe\nvXrRqVMnZs+ebe1wxBnSThYx/rM4nl20nQ7N/Fn6xGDu7tcCpU5f1TUN8GLBg/1p2diX8Z9tZOnO\no1aMWNgqR5hrUB2HGnY6ZcMU4rPi6/SY7YPb81yf56ot88knnxAcHExhYSG9e/fm5ptvrtMYxIXR\nWrN42xEm/biLolIT/766A/cNjDln9T7E15NvHujHvXM38PDXW5g6ysSNPSIaOGphqxxlrkF1HCoh\nWMu7777L999/D0BycjL79++3ckQiI6+YF37YyZKdx+geGchbt3ajVej5+3ICvN35Ylxf7v88jqcW\nbCO/2MRd/Vo0QMTClpnMmifmO8Zcg+o4VEI435V8fVi+fDm///47a9euxcfHh6FDh1JUVNTgcYjT\nluw4yr9/2EluURnPjmjHA4Nb4uZa89bRRp5ufDK2Nw9/tZl//7CTgpIyHrikVT1GLGzd5J93s2yX\n/exrcKEcKiFYQ05ODkFBQfj4+BAfH8+6deusHZLTyi4o4cXFu/hx6xE6h/vz9ajutGvqd0HH8nJ3\nZdbdvXhy/lZe+yWevGITT17RplK/g3AOH686yNzVjjPXoDqSEC7SiBEjmDVrFh06dKBdu3b069fP\n2iE5pT/jjzPx2x1k5Zfw5BVtmXBpK9xrUSuoirurC9NH98DHw5V3/9hPfnEZ/766gyQFJ7Jkx1Em\n/7zboeYaVEcSwkXy9PRkyZIlZz2elJRUfj8vL68BI3IuJ4tKefV/u1m4KYV2Tfz4ZGxvOocH1Nnx\nXV0Ub9zUFR8PNz5edZCCkjIm39AFVwcZdy7O7eftR3lqgePNNaiOJARht1bsS+e5b7dz/GQRE4a2\n4vEr2uDpVvdjwl1cFC9e2xFfTzdm/pVAQYmJqaO6XXQNRNimolITr/y0m6/XH6Z7ZKDDzTWojiQE\nYXfyist47Zc9fL3+MC1DG/Ht/w2o91EfSin+MbwdPp6uvLl0LwUlJmbe0aNeEpCwnn3Hc3nk683s\nO57HQ0Na8fSwtk6V+B0iIWitpV33DPa0E15trEvM5JlF20g5Ucj9g2N4eli7Br16mzC0Nb6ebkz6\ncRfjP4vjw7t74ePhEP+NnJrWmvkbk3npf7vw9XTj8/v6cEnbUGuH1eDs/i/Zy8uLzMxMQkJCJClY\naK3JzMzEy8vL2qHUmcISE28ui2fu6iRahPiw4MH+9I4Otkos9/SPxtvdlee+3c6YTzbw8dje+Hu5\nWyUWcfFOFpXyz+928NP2owxq3Zhpt3UjzM9x/u/Uht0nhIiICFJSUkhPl/VnKvLy8iIiwjFm2W46\ndIJ/LNzGwYx8xvRvwXMj21v9qnxUbCQ+Hm48Pm8Ld85Zz2f39XG4ZQycwdbkbB79ZjNHsot4dkQ7\nHrqklVN0Hp+L3ScEd3d3YmIce2ywsyoqNfH27/uYsyKRZgHefD2+LwNaN7Z2WOWu7toMHw9XHvpy\nE6Nnr+XLcX0J83fOK0t7YzZrPlqVyJtL99LE34sFD/ajVwvr1DhtifP0lgi7sj0lm2tnrOLDvxO5\nNTaSpU8MtqlkcMql7cOYe29vUk4UMurDtaScKLB2SOI8MvKKuffTjbz2SzxXdGjCL48NlmRgIQlB\n2JSSMjPTft3Lje+v4WRRKXPv7c0bN3fFz4bb6Ae0asyX4/tyIr+EW2etJTFd5p3YqjUJGVw1fSVr\nEzN59YbOfHBXTwJ8bPdvq6FJQhA2Y8/Rk9zw3mre/TOB67s359cnhnBpuzBrh1UjPaOCmPdAf4rL\nzNz64TrZktPGlJnMvPXrXu78eD1+Xm78+PDAs5ZAF5IQhA0oM5mZ+ed+rpu5irTcYmbf3Ytpt3a3\nuyu3js39mf9gf9xclGzJaUNSswsZPXsdM/5MYFSvCP736CA6NPO3dlg2qUYJQSk1Qim1VymVoJSa\nWMXznkqp+Zbn1yuloi2PX6mU2qSU2mH5eVmF1/SyPJ6glHpXSap2Sglpudz8wRqm/rqP4Z2a8uuT\nlzDMjleTbB3my8KH+hPg7S5bctqAZbuOcdX0lcQfy2X66O68eUs3q49Qs2XnTQhKKVfgPWAk0BG4\nXSnV8Yxi44ATWuvWwNvAFMvjGcC1WusuwBjgiwqv+QC4H2hjuY24iM8h7IjZrDmWU8TsFQe46t1V\nHM4qYOYdPZh5R0+HGLoZGWzMk2gW6M2YuRtYvjfN2iE5naJSEy/+uJMHv9hEVLAPPz06iOu7h1s7\nLJtXk1TZB0jQWicCKKXmAdcDuyuUuR54yXJ/ETBTKaW01lsqlNkFeCulPIFgwF9rvc5yzM+BG4Cz\nV4kTdkdrTU5hKclZhSSfKCA5q4DkEwUcziokJauAlOxCSsrMAFzZsQmv3diFUD9PK0ddt5oGeDH/\ngX7c88kG7v88jhm392BE52bWDsspHEjP49Gvt7D76EnGDYrhuRHt8XCT1vGaqElCCAeSK/yeAvQ9\nVxmtdZlSKgcIwaghnHIzsFlrXayUCrccp+IxJX3bkaJSEyknCjicVWCc+C0n/VNJILeorFL5AG93\nIoO9ad/Mjys7NiEi2Ie2Yb70iQl22I69EF9Pvr7/9Jac/73FxE09HWOyoK36dlMKL/y4E083Fz4Z\nG8tl7ZtYOyS70iCNaUqpThjNSMMu4LUPAA8AREVF1XFk4lzKTGaO5hSRfKKAlKxC48RffrVfSHpu\ncaXynm4uRAb7EBnkTWx0EJFBPsbvwd5EBvs47dIOZ27JWVAiW3LWh7ziMib9sJPvtqTSNyaY6aN7\n0DRAJgnWVk0SQioQWeH3CMtjVZVJUUq5AQFAJoBSKgL4HrhHa32gQvmKl0pVHRMArfVsYDZAbGys\nY67YZgVaazLySspP8iknjKv8Uyf+o9lFlJlPf90uCpoFeBMZ7M2l7UIrn/CDfAj183TYK/2LdeaW\nnPnFZTw4RLbkrCs7U3N49JstHMrM58kr2vLIZa1lv4oLVJOEsBFoo5SKwThpjwbuOKPMYoxO47XA\nLcCfWmutlAoEfgYmaq1XnyqstT6qlDqplOoHrAfuAWZc9KcR53X8ZBETvtrM7iMnKSw1VXqusa8H\nEUE+dI8M4tquxpV9VLAPkUE+NAv0cqplgOtaxS05X18ST36JbMl5sbTWfLomidd/iSe4kQff3N+P\nvi1DrB2WXTtvQrD0CTwCLANcgU+01ruUUq8AcVrrxcDHwBdKqQQgCyNpADwCtAYmKaUmWR4bprVO\nAyYAnwLeGJ3J0qHcAN76dS87UnK4s19U+ck+MtiHiCBvGnnKcLz6JFty1p0T+SU8s2gbv+9J44oO\nYfz3lm4EOcAINWtT9rRufmxsrI6Li7N2GHZr3/FcRryzgnsHxvDCNWeOHBYNxWzWvPLTbj5dk8Tt\nfSJlS85aWp+YyRPzt5KZV8LzV7Vn7IBoSarnoZTapLWOPV85uSR0IlOWxNPI041HLm1t7VCcmmzJ\neWFMZs3MPxOY/sc+ooJ9+G7CgDrdP1tIQnAa6xMz+SM+jWdHtJOqtQ2oakvOGbf3cJq9e2vr+Mki\nHp+3hXWJWdzQvTmTb+yCrzRx1jn5Rp2A1prXl8TT1N+L+wbK3hG2pOKWnGPnbuD9O3s5xGztuvRn\n/HH+sXA7hZaa1M09w6WJqJ5IHdUJLN15jK3J2Tx5ZRu5ArVB9/SP5p3burP5cDbXzVzF7iOyUioY\nS6FP/mk3930aRxN/L356bBC39IqQZFCPJCE4uFKTmTeX7aVNmC83yyxZm3VDj3AWPtifMpPmpg9W\n879tR6wdklUlZeRzy6w1fLTqIGP6t+D7CQNoFepr7bAcniQEBzdvYzIHM/J5bkR73KTT0qZ1iwxk\n8aMD6dw8gEe/2cIbS+Ixme1nFGBd+WFLKtfMWMWhzAI+vLsXL1/fWWq2DUTOEA4sv7iM6b/vp090\nMJd3sI+NZpxdmJ8XX9/fjzv7RjHr7wPc9+lGcgpKrR1Wg8gtKuXJ+Vt5Yv5WOjTz45fHBzPcjpdC\nt0eSEBzYnJWJZOQVM/Gq9tLuakc83Fz4z41deO3GLqw5kMH1761i3/Fca4dVrzYfPsFV765k8bYj\nPHVlW765vx/hgd7WDsvpSEJwUOm5xcxZkciITk3pGRVk7XDEBbijbxTf3N+PvGITN763mmW7jlk7\npDpnzC3Yz6hZa9EaFjzYj8cubyPNm1Yi37qDmvHnforKzDwzop21QxEXITY6mJ8eHUTrMF8e/GIT\nb/+2D7OD9CscyS7k9jnrmPrrPq7q0oxfHh9MrxbB1g7Lqck8BAd0MCOfr9cfZnTvSBmZ4QCaBngx\n/8H+/Ov7nUz/Yz+7jpzk7du64WfHS4ov2XGUid/toMxk5q1R3bhJ5hbYBKkhOKCpy/bi4ebC41e0\nsXYooo54ubsydVRXXry2I3/tTePG99eQmJ5n7bBqraCkjOe/287/fbWZ6BAffn5sMDfL3AKbIQnB\nwWxNzubnHUcZP7glYX6yQYgjUUpx78AYvhjXh8y8Yq5/bzV/xdvPfs07U3O4ZsYq5m1MZsLQViz6\nvwFEN25k7bBEBZIQHIjWmtd/2UNIIw8euKSltcMR9WRAq8YsfmQQkUE+3PfZRt5fnoAtr1psNms+\nWpnIje+vJr+4jK/G9eXZEe1lMT8bJP8iDmT53nTWH8ziscvbyMJfDi4y2Idv/28A13RtzptL9/LI\nN1soKCk7/wsbWFpuEWPmbmDyz3u4tF0YSx+/hAGtG1s7LHEOctZwECaz5o0l8bQI8eH2PrL3tDPw\n9nDl3dHd6dzcnylL4zmQlsece2KJDPaxdmiAsSjdMwu3k19Sxn9u7MwdfaKkr8DGSQ3BQXy3OYW9\nx3N5Zng7PNzkn9VZKKV4cEgr5t7bhyPZhVw7cxWrEzKsGlNRqYmXFu/ivk/jCPXz5H+PDOLOvi0k\nGdgBOXM4gKJSE9N+20e3iACu7tLM2uEIKxjSNpTFjwwizM+Tez7ZwMerDlqlX2Hf8VxueG81n65J\n4r6BMfzw8EDaNPFr8DjEhZGE4AA+XZPE0ZwiJo6U/XmdWXTjRnw3YSBXdAjj1Z928/SCbRSVmhrk\nvbXWfLHuENfOWEVGXjFz7+3NpGs7yqJ0dkb6EOxcdkEJ7/+VwNB2ofRvFWLtcISV+Xq68cGdvZj5\nVwLTfttHQnoes+7qRfN6XBcoK7+EZxdt5/c9xxnSNpSpo7oR6udZb+8n6o/UEOzc+8sPkFtcxnMj\n2ls7FGEjXFwUj13ehjn3xJKYns91M1ex4WBWvbzX6oQMRryzghX70nnhmo7MHdtbkoEdk4Rgx1Kz\nC/l0TRI39YigQzN/a4cjbMyVHZvww8MD8PNy54456/hi3aE661coKTPzxpJ47vp4PX5ebnz/8ADG\nDYrBxUWaLO2ZJAQ79tavewF4alhbK0cibFXrMD9+eHggg9s05oUfdvLP73dQXHZx/QoHLbuZzfr7\nAKN7R/HTo4Pp1DygjiIW1iR9CHZq95GTfL8llQcGt5R140W1Arzd+WhMb97+bR8z/0pg77FcZt3V\nizD/2i1torVm0aYUXly8C3dXF2bd1ZMRnWVUmyORGoKdenNZPH6ebkwY2traoQg74Oqi+Mfwdrx/\nZ0/2HM3lmhmr2HL4RI1fn1NYyqPfbOGZRdvpGhHA0icGSzJwQJIQ7NCaAxks35vOw5e2JsDHfpdA\nFg3vqi7N+G7CADzdXbjtw3UsiEs+72s2JmVx1fSVLNl5jGeGt+Or8f1oFiC1UkckCcHOmC1LVDQP\n8GLMgGhrhyPsUIdm/ix+eBC9Y4J4dtF2XvxxJ6Um81nlykxm3v5tH7d9uBZXF8Wih/rz8KWtcZWO\nY4clfQh25ucdR9meksPUUd1k0o+4YEGNPPjs3j5MWRrPnJUHiT+Wy/t39iTE1xgympxVwBPzt7Lp\n0Alu6hnOy9d1susNeUTNSEKwIyVlZv67bC/tm/pxY49wa4cj7Jybqwv/urojHZv7M/HbHVw3czUf\n3t2LxIx8/vXdDgCmj+7O9d3lb81ZSEKwI99sOMzhrALm3ttbqu2iztzYI4LWoX48+EUcN7y3mjKz\npkdUIO+O7mEzK6eKhiEJwU7kFpXy7h/76dcymKFtQ60djnAwXSICWPzoICb9uJO2Tfx45NLWuMkG\nNk5HEoKdmLMikcz8Ej6RBexEPWns68n7d/aydhjCiuQSwA6knSxizsqDXN21Gd0iA60djhDCQUlC\nsAPv/LGfUpOZZ4a1s3YoQggHVqOEoJQaoZTaq5RKUEpNrOJ5T6XUfMvz65VS0ZbHQ5RSfyml8pRS\nM894zXLLMbdabmF18YEczYH0POZvTObOvlFEN25k7XCEEA7svH0ISilX4D3gSiAF2KiUWqy13l2h\n2DjghNa6tVJqNDAFuA0oAl4AOltuZ7pTax13kZ/Bof136V683Fx49PI21g5FCOHgalJD6AMkaK0T\ntdYlwDzg+jPKXA98Zrm/CLhcKaW01vla61UYiUHU0qZDJ1i66xgPXNKKxr6yxrwQon7VJCGEAxUX\nPEmxPFZlGa11GZAD1GT7rrmW5qIX1DmGziilHlBKxSml4tLT02twSMegteaNJXto7OvJ+MEx1g5H\nCOEErNmpfKfWugsw2HK7u6pCWuvZWutYrXVsaKjzjL//fU8aG5NO8MQVbWjkKaODhRD1ryYJIRWI\nrPB7hOWxKssopdyAACCzuoNqrVMtP3OBrzGapgTGomJvLo2nZeNG3NY78vwvEEKIOlCThLARaKOU\nilFKeQCjgcVnlFkMjLHcvwX4U1ezV59Syk0p1dhy3x24BthZ2+Ad1bebU9iflsezI9rhLrNFhRAN\n5LxtEVrrMqXUI8AywBX4RGu9Syn1ChCntV4MfAx8oZRKALIwkgYASqkkwB/wUErdAAwDDgHLLMnA\nFfgdmFOnn8xOFZaYmPbbPnpEBTK8U1NrhyOEcCI1apzWWv8C/HLGY5Mq3C8CRp3jtdHnOKzMka/C\nJ6sPcvxkMTNu7ylLVAghGpS0R9iQrPwSZi0/wBUdwugTE2ztcIQQTkYSgg15768E8kvKeG5Ee2uH\nIoRwQpIQbERyVgFfrD3EqF6RtGniZ+1whBBOSBKCjXjr170oBU9e2dbaoQghnJQkBBuwMzWHH7Ye\n4b5BMTQN8LJ2OEIIJyUJwQZMWRpPoI87Dw1pZe1QhBBOzCkSQkZeMdXMk7OqlfvTWbk/g0cubU2A\nt7u1wxFCODGnWCRn7NwNpJ0sZkjbUIa0C2Vw61ACfKx/8jWbNW8siSciyJu7+7ewdjhCCCfn8AlB\na83YATH8tTeNZbuOsXBTCi4KekQFMdSSIDo3D8DFpeEngf1v+xF2HTnJO7d1x9PNtcHfXwghKlK2\n2pRSldjYWB0Xd+H76ZSZzGxLyWb53nT+3pfO9pQcAEIaeXBJ21CGtgtlcJtQght51FXI51RcZuLy\nt/7G38udnx4dZJWEJIRwDkqpTVrr2POVc/gaQkVuri70ahFMrxbBPD2sHRl5xazcn87yveks35vG\n91tSUQq6RgQyxJIgukUE4loPJ+sv1x0m5UQhn9/XRZKBEMImOFUNoToms2ZHag5/701n+b40tiVn\nY9YQ6OPrdlGzAAAgAElEQVTO4DahRv9D21BC/S5+57KTRaUMefMvOjUP4MvxfesgeiGEODepIdSS\nq4uie2Qg3SMDefyKNpzIL2FlQgZ/W5qX/rftCACdw/0ttYcwekQG4nYBy1N/+PcBThSUMnGkLFEh\nhLAdUkOoAbNZs/voSf7el87fe9PZdPgEJrPGz8uNQa0bM7RdKEPahtVoUtmxnCKGTv2L4Z2aMn10\njwaIXgjh7KSGUIdcXBSdwwPoHB7Aw5e2JqewlDUJGeWd00t2HgOgfVM/hrQzmpZiWwTj4XZ27eGd\n3/dhMmv+MaxdQ38MIYSoliSECxDg7c7ILs0Y2aUZWmv2Hs81+h72pvPJqoN8+HcijTxcGVBeewgl\nIsiH/cdzWRCXzJgB0UQG+1j7YwghRCWSEC6SUor2Tf1p39SfB4e0Iq+4jDUJGfy9z0gQv+0+DkDr\nMF9cFDTycOPRy9pYOWohhDibJIQ65uvpxrBOTRnWqSlaaw6k57N8bxp/70tn/cEsnh3erkHmOQgh\nRG1JQqhHSilah/nSOsyX8YNborWWbTGFEDbLKRa3sxWSDIQQtkwSghBCCEASghBCCAtJCEIIIQBJ\nCEIIISwkIQghhAAkIQghhLCQhCCEEAKQhCCEEMJCEoIQQghAEoIQQggLSQhCCCEASQhCCCEsJCEI\nIYQAapgQlFIjlFJ7lVIJSqmJVTzvqZSab3l+vVIq2vJ4iFLqL6VUnlJq5hmv6aWU2mF5zbtKlgIV\nQohy6QXp/Jr0K1M2TGH8r+Mxa3O9v+d590NQSrkC7wFXAinARqXUYq317grFxgEntNatlVKjgSnA\nbUAR8ALQ2XKr6APgfmA98AswAlhycR9HCCHsj1mbSchOYGvaVrakbWFL2hZS81IB8HL1onPjzuSW\n5BLgGVCvcdRkg5w+QILWOhFAKTUPuB6omBCuB16y3F8EzFRKKa11PrBKKdW64gGVUs0Af631Osvv\nnwM3IAlBCOEECssK2Zmxky1pW9ictpntadvJLc0FIMQrhB5hPbij/R30COtB++D2uLu6N0hcNUkI\n4UByhd9TgL7nKqO1LlNK5QAhQEY1x0w545jhNQlYCCHsTXpBevmV/9a0rcRnxVOmywBoHdia4THD\n6RHWgx6hPYjwi7DaZlo2v4WmUuoB4AGAqKgoK0cjhBDVM2szB7IPlJ/8N6dtLm/+8XT1pHPjzozt\nPJYeYT3oFtqt3puBaqMmCSEViKzwe4TlsarKpCil3IAAIPM8x4w4zzEB0FrPBmYDxMbG6hrEK4QQ\nDaZi88+WtC1sS99Gbknl5p/b299Oj7AedAju0GDNPxeiJglhI9BGKRWDcdIeDdxxRpnFwBhgLXAL\n8KfW+pwnb631UaXUSaVUP4xO5XuAGRcQvxBCNKiMwoxKzT97MveUN/+0CmjFsBbDjOafsB5E+kXa\n1V7q500Ilj6BR4BlgCvwidZ6l1LqFSBOa70Y+Bj4QimVAGRhJA0AlFJJgD/goZS6ARhmGaE0AfgU\n8MboTJYOZSGETTFrM4nZiWxO21w+Aiglz+j+9HT1pFNIJ5tt/rkQqpoLeZsTGxur4+LirB2GEMIJ\nLDm4hNfWv0Z2cTYAwV7B5Vf+3cO60zG4o003/1SklNqktY49Xzmb71QWoj5prTFrM64urtYORdgI\nk9nEjC0z+Hjnx3QP7c7NbW+mZ1hPu2v+uRCSEITTySvJY/3R9aw+sprVqas5XnCc5r7NifKLItIv\nkij/KKL8oojyjyLcNxwPVw9rhywaSG5JLs+teI6VqSu5te2tTOwz0W5qAXVBEoJweGZtJj4rntWp\nq1mVuort6dsp02X4uPnQt1lfRsSM4EjeEQ7nHmZb+jbySvPKX+uiXGjWqJmRKCxJ4tT9CL8IvNy8\nrPjJRF06mHOQx/58jJTcFF7o9wK3tru14YPQGgoyIWMfpO+FjP3G/exDMGEd1HNNVhKCcEiZhZms\nObKm/JZVlAVAh+AOjO08lgHNB9A9tPtZV39aa7KLszmce5jDJw+X/0zOTWbZoWXkFOdUKt/Epwkt\n/FtUqllE+kUS6ReJj7tPg31ecXFWpKzguRXP4eHqwUfDP6JXk171+4ZmE5xIOn3Cr3grPHG6nJs3\nNG4NTbtCST54+ddrWJIQhEMoNZeyLW0ba46sYVXqKvZk7QEgyDOI/s37Myh8EP2b96exd+Nqj6OU\nIsgriCCvILqFdjvr+ZziHJJzk8uTxan7fyX/VZ50Tgn1Di1PFOVJw5IwfD186+7DiwumtebjnR/z\n7uZ3aR/cnumXTqeZb7O6e4OSfMtJfz9k7LWc9PdDZgKYSk6XaxQKjdtBxxugcVvjFtoW/CPApeEW\npZZRRsJupealsjrV6AdYf2w9+aX5uCpXuoV2Y2D4QAY2H0iHkA64qIb5D5VbkmskiNzDJJ9MrlS7\nSC9Mr1Q22Cv4rCaoXk160aRRkwaJVRgTyiatnsTSpKWMjB7JywNfxtvNu/YH0hry0iqf8DP2Qfo+\nOFlhhR7lAkExlhN+G8tJvx2EtAaf4Lr7YFWo6SgjSQjCbhSWFRJ3LK68FpB0MgmAZo2aMaD5AAaF\nD6JPsz74e9RvtfpCFJQWlCeLU0ni1P3jBccB8HX35T+D/sNlUZdZOVrHdzTvKI//9TjxWfE83vNx\n7ut83/lHEJlKLc08+06f8E8lgIpNie6NKpzw256+4g9uCW6e9fq5zkUSgrB7WmsScxJZlbqKNUfW\nEHcsjhJzCZ6unsQ2iS2vBcQExNj1cMCisiIO5Bzg1bWvsitzF+M6j+ORHo/g5iItuvUh7lgcT//9\nNKWmUt645A0uibik6oIFWbB+FhzfZZz0sxLBXHr6ed+mxok/tF3lq37/cLCxv0dJCMIunSw5yboj\n68prAaeunlsGtCyvBfRq0sshR/eUmEp4Y8MbLNy3kL5N+zLlkimEeIdYOyyHsmDvAl5f/zoRfhG8\ne9m7xATEVF3w+G6YdztkH4bgVpaTfpvTV/uN24CX/cxKloQg7ILJbGJ35u7yOQE7MnZg0iZ83X3p\n16xfeS2gTjv6bNwPCT8wed1kAj0DmTZ0Gl1Du1o7JLtXairl9Q2vs3DfQgaHD2bKJVPw8/CrunD8\nz/DdA+DRCG77CiJ7N2yw9UASgrBZWmvijsexcN9C1h5ZW740QKeQTuW1gC6hXXB3qccJQSUFkLIB\nklYbV4HeQeATAj6Wn97BRkffqfvuDVsj2ZO5hyeXP8nxguNM7D2RW9vdatfNYtaUWZjJU8ufYnPa\nZsZ3Gc8j3R+pema61rByKvw5GZr3gNFfg3/zhg+4HsjSFcLmmLWZFSkrmLNjDtvTtxPoGcjg8MEM\nDB9I/+b9Cfaqx5EWxXmQvB4OrTaSQOomoz1YuYBfcyjKAcuSxVVy97Ekh1OJI9iSNCreD678uEej\nC25L7hDSgfnXzOf5lc8zef1ktqVv44X+L1zYKBgntjtzN4//9TjZRdm8ecmbjIwZWXXBkgL48WHY\n9R10uRWuexfcne+7lhqCqHdl5jKWJS3jox0fkZCdQLhvOGM7jeWG1jfUX19AcS4cXg9JK40kcGQL\nmMtAuUJ4T2gxEKIHQWTf05N9ykqgMMvoTCzMMmaMFlh+Fp6ocD/r9P2i7HPH4OpxRm0j+OyaR/n9\nIAiMgjMmypm1mdnbZ/P+1vdpE9SGt4e+TZS/bBRVE0sOLmHS6kkEegUy/dLpdAzpWHXB7GSYdwcc\n2wFXvAQDH7e5TuGLJU1GwuqKTcX8mPAjc3fOJSUvhVYBrRjXZRwjYkbUfXNQUQ4cXgdJq4zb0W2g\nTeDifnYC8KzDSWFmExRmn50oKiaUwhMV7lvKaNPZxwpuCbd+AU07n/XU6tTVPLfyOcxmM/8Z9B8u\njbq07j6Dg6m4OF3PsJ5MGzrt3J3zh9fB/LugrBhu/hjaDmvYYBuIJARhNfml+SzYu4DPd39ORmEG\nXRp3YXyX8QyNHFp3k8QKs+Hw2tMJ4Nh20Gbjqjw8FqIHGkkgso/RdGNLzGYoPlkhgWRB7lFY/rrx\nua59B7qNPutlqXmpPLX8KXZn7ub+LvfzcPeHZZXWM9RqcbrNn8NPT0FgJNw+zxhJ5KAkIYgGd6Lo\nBF/t+Yqv478mtySXvs36cn+X++nTtM/Fd4gWZMGhNZY+gJVwbCegwdUTInobV//RA4379tr2m5cG\ni+4zPl/sOBjx+lkTmYpNxby+/nW+3f8t/Zr1481L3iTIK8hKAduWiovTPd/3+XMvTmcqg2X/hA0f\nQqvL4JZPjCY7ByYJQTSYY/nH+GzXZ3y7/1sKywq5POpyxncZT+fGZzd91Fh+puXkv8r4eXwXoMHN\ny7jqbzHISALhvRp8BFC9MpXBHy/DmneNzzbqM+MK9gzf7/+eyesmE+wdzLQh0+gS2sUKwdqOiovT\nTRs67dyL0xVkwcKxcPBv6P8IXPEyuDr+2BpJCKLeJeUkMXfXXBYfWIzWmqtbXs19ne+jVWCr2h8s\nLx0OrTJGACWtgnRjcTrcfYwEED3ISALhPa02/b9B7V4MP0wwOplv+di4kj2zSOZunlr+FGkFaUzs\nM5FRbUc53dDUWi1Ol7YHvrkdTqbCNe9AjzsbNlgrkoQg6s2ezD18tOMjfjv0Gx6uHtzY+kbGdh5L\nuG94zQ9SWgh7f7H0Aaw2FgYDYx2YqH5G80/0YGjWHdycdIOajARYcLdxIrvsXzDo6bNWvswpzuG5\nlc+xOnU117W6jn/3+7fTDE2t1eJ0e5fAt+Mtk82+NC4ynIgkBFHnNh3fxJwdc1iduhpfd19ua3cb\nd3W867xLSldSWghxc2H1O5B3HDz8oEX/06OAmnU7a+ilUyvJh/89DjsWQtuRcOMHZ7V3m7WZWdtm\nMWvbLNoGteXtoW8T6X92M5Mjqbg43RO9nuDeTvdWXTvSGlZNgz9eNf62Rn8NAbW4cHEQkhBEndBa\nszJ1JR/t+IgtaVsI9grmrg53cVv722q3qmhpIWz6FFa9bSSCmEtg8D+MROAEbbgXRWvYMMfoCA0I\nN4amNjt7OYuVKSuZuHIiWmteH/w6QyKHWCHY+lfjxelKCmDxI7DzW+h8C1w/034HHFwkSQjiopjM\nJn499Csf7/iYvSf20qxRM8Z2GsuNbW6sXZNEaVGFRHDMaAYa+rzRJCRqJ3kDLBhjDFe9elqVbeAp\nuSk8tfwp9mTt4YGuDzCh2wSHGppa48XpclKMyWZHt8MVL8LAJxxuslltSEIQF6TEVMLiA4uZu3Mu\nh3MPExMQw7jO47iq5VW1m0xWWmSM8141zRhj32IQDJ0IMYPrL3hnkJcO394HB1dAr7EwYspZo6yK\nyop4bf1rfJ/wPQOaD+CNwW/Y/dDUWi1Od3i9MdmstBBu/gjajWjYYG2QJARRKwWlBSzct5DPd31O\nWmEaHUM6cn+X+7ks6rLaTSYrLYItX8DKaZB7xGgSGvq8JIK6ZCqDvyYbta7mPeDWz41lL87w7b5v\neW39a4R4hzBt6LSLGwZsRTVenA5g8xfw81MQEAGjv4Gw9g0brI2ShCBqJKc4h6/3fM1X8V+RU5xD\nn6Z9GNdlHP2b9a/dEMayYqNGcCoRRA2AS583moicuKper+J/hu8fAhdX40q49RVnFdmVsYunlj9F\nemE6z/d9nlva3GJXQ1MrLk73ysBXzr04nakMfv03rP8AWg6FW+bW+7aU9kQSgqhWWkEan+/6nAX7\nFlBYVsjQyKGM7zK+yo3lq1VWfLpGcDIVIvsZiSBmiCSChpB5ABbcY0zcG/o8XPLMWUNTs4uymbhy\nIquPrOb6Vtfz737/tosNhmq8OF1BFiy6FxKXQ78JcOWrMlDhDJIQRJUyCzN5b+t7/JDwAyZtYmTM\nSMZ1HkeboDa1O1BZSYVEkGIsGjf0eePqTBJBwyopgJ+ehO3zoPWVcNPss66OTWYTH2z7gA+3f0j7\n4PZMGzqNSD/bGpqqtSazKJPE7ET+Sv6LL/d8ef7F6dL3wjejjRVLr3kbet7dsEHbCUkIohKtNT8l\n/sSUjVPIL83nptY3Mbbz2NqfFMpKYOuXRiLISYaIPkaNoOWlkgisSWuI+xiWTAT/ZsbQ1Obdzyq2\nImUFE1dOBOCNwdUM2axHWmuO5R/jQM4BErMTScxJ5ED2ARJzEjlZcrK83HkXp9u71Jhs5u5tTDaL\n6ttAn8D+SEIQ5VLzUnl17ausPrKabqHdeHnAy7VfXqKsBLZ+BSvfsiSC3kaNoNVlkghsScomowkp\nPx2ungo97zmrSHJuMk8tf4r4rHge6vYQD3V9qF6GpprMJlLyUkjMTuRAzgEO5hwsP/EXlhWWlwv2\nCqZlQEvjFtiSVoGtaBXQilCf0KoPrLUxsfH3ly2Tzb4yOpHFOUlCEJjMJr6J/4Z3t7yLQvF4z8cZ\n3X507UYNmUph69ewYirkHDaWlr70eWh1uSQCW5WfaQxNTVwOPe6Cq6aeNSGrqKyIV9e9yuIDixnY\nfCBvDH6DQK/AC3q7UlMph04eMq74cxLLr/qTcpIoMZeUlwvzCaNVQCtaBbYiJiCGVoGtaBnQsnZD\nYksLYfGjxsztzjfDdTPBw+eC4nYmkhCc3P4T+3lpzUtsz9jO4PDBvNDvhdptVG8qhW3fwIr/GnsO\nh/eCof+E1pII7ILZBH+9ZuwR3KybMTQ1KLpSEa01C/ct5I0NbxDqHcq0odPo1LjTOQ9ZWFZIUk7S\nWU09ybnJmCwb/igUzX2bl1/lnzrxxwTEnHveQE3lpFomm22Dy1+AQU/J32INSUJwUiWmEubsmMNH\nOz7Cz92PiX0mMjJmZM2HGppKYds8SyI4BM17wqX/NIY0yn8++7N3KXz/AKDgpjlV7gi2M2MnTy5/\nkszCTP7V918Mix5W6Ur/VDPPkbwjaIzzhatyJco/qryp59TVfnRAdP0srpe8EebfaXSg3zwH2p1j\n+KmokiQEJ7Q1bSsvrnmRxJxErml5Dc/2frbm1XFTKWyfbySCE0nGhKehz0ObYZII7F3WQZh/Nxzf\nAZc8a8wYP6PP4ETRCZ5b8Rxrj66t9LiHiwfRAdGn2/cDjBN/C/8W5+7srWtbvoKfngD/cGNnM5ls\nVmuSEJxIfmk+0zdPZ178PJo2asqk/pMYFD6oZi82lVVIBAeN5aaHPg9th0sicCSlhfDz08bAgFaX\nGfsHVzE0ddG+ReSW5pa39Yf7hltvLSRTGfw2Cda9Z8xrGfWpTDa7QHWaEJRSI4DpgCvwkdb6jTOe\n9wQ+B3oBmcBtWusky3PPA+MAE/CY1nqZ5fEkINfyeFlNgpWEcLYVKSt4dd2rHM8/zh0d7uCxHo/h\n416DTjZTGexYAH+/aUkE3SyJYIQkAkelNWz+DH55BnybwK2fGX1DtqjwhLGd6IE/oe9DMOw/Mtns\nItQ0IZz3G1ZKuQLvAVcCKcBGpdRirfXuCsXGASe01q2VUqOBKcBtSqmOwGigE9Ac+F0p1VZrSw8U\nXKq1zqjVJxMAZBVlMWXDFH45+AutA1sz9aqpNZtlbDYZIzT+ngJZidC0q7HmS7uRkggcnVLGgnhN\nuxqrpn4yAka+aTxmjX97sxny04yVSXOSLT8tt9TNxtDZ62ZUOXRW1I+apNw+QILWOhFAKTUPuB6o\nmBCuB16y3F8EzFRGL+b1wDytdTFwUCmVYDle5YZKUWOnJpi9ufFN8krzmNBtAuO7jD9/e67WkPC7\nUQVP2w1NuxibhbS7ShKBswnvCQ/+bUzq+ukJY1ntq9+q++GbJflVn+zLH0sFc2nl13j4GXtIN+sK\ng540ds8TDaYmCSEcSK7wewpw5pTA8jJa6zKlVA4QYnl83RmvPbVdkQZ+VUpp4EOt9ezah+9cjuQd\n4ZV1r7A6tZYTzI5sMRLBwRUQFGMs/NXpRkkEzswnGO601BT/ngLHdsBtn0Nwy5q93mw2Njqq8oRv\n+b0wq/JrlAv4NTdO+BG9jb/BgAgIiLT8jACvgLr/rKLGrNkoN0hrnaqUCgN+U0rFa61XnFlIKfUA\n8ABAVNTZS/w6A5PZxLy985i+eToAE/tMZHS70efv7DuRZGwduHMR+ITAyP8azQPOukexqMzF1RhS\nHB4L390PHw6Fmz40mg+L84zFCnOSjXWCzjzhnzxy9tW9p//pk3tE77NP9n7NpB/AxtXkXycVqLjg\nTYTlsarKpCil3IAAjM7lc75Wa33qZ5pS6nuMpqSzEoKl5jAbjE7lGsTrUBJOJPDi2hfZnr6dQeGD\nmNRv0vknmBVkGTOLN8wGFzdjq8qBj4NXLba8FM6j7TCjCWnBPcZCcV6BUJRduYxyBf/mxok9ss/p\nk7xc3TuUmiSEjUAbpVQMxsl8NHDHGWUWA2Mw+gZuAf7UWmul1GLga6XUNIxO5TbABqVUI8BFa51r\nuT8MeKVOPpGDKDGV8NGOj5izYw6+7r68Pvh1ro65uvoJZqWFsH4WrHwbSnKNZQuGPm/8RxaiOkHR\ncN+vxlpVBZmVT/aBkeDbVK7uncB5/4UtfQKPAMswhp1+orXepZR6BYjTWi8GPga+sHQaZ2EkDSzl\nFmB0QJcBD2utTUqpJsD3lpObG/C11nppPXw+u7Q1bSsvrXmJAzkHuLrl1Tzb+1mCvaoZf202GXMJ\n/pxsVPPbjoArXoKwDg0VsnAE7l5w2b+sHYWwIqeYmKa1totdoipOMGvSqAmT+k1icEQ1W09qDQl/\nwO8vwvGdxjITV74i21UKISqps3kIjmDs0rGcLDlJtH80LfxbEB0QTbR/NDEBMQR42ka7Z8UJZre3\nv53Hej5GI/dG537Bka2WkUN/G9V9GTkkhLhITpEQ+jXrx+6s3SRkJ7A8eTlluqz8uUDPQKL9o4kO\nMJJFjH8M0QHRRPpF4uFa/6NxKk4waxnQks9Hfk73sLM3Nil34pDRNLRjAXgHw4gpEHufjBwSQlw0\np2gyqqjUXEpqbiqHTh4i6WQSB3MOlt/PKDw9adpFudC8UfPy2sSppBHtH02YT9hFN0GdOcHsgS4P\nMK7LuHMnoYIso8Nvw2xjPHe/CTDoCRnZIYQ4L2kyOgd3F3fjxB4QzRCGVHoutyS3PDkk5SSRdDKJ\nQycPsen4pko7PHm7eVdKEhWboapt5rE4kneEV9e9yqrUVXQN7crL/V+mdVDrqguXFsGGD41kUHQS\netxp7EsQEF51eSGEuEBOlxCq4+fhR+fGnencuHOlx83aTFpBWqXaRFJOEtsztrM0aWn5GvEAYd5h\ntAhocVatorlvcxSq5hPMzGajWejPycZEoDbDjJFDTc69gYkQQlwMSQg14KJcaNqoKU0bNaV/8/6V\nnis2FXP45OHy2sTBnIMknUxiWdKyShuGu7m4EegZSEZhBgPDBzKp3ySa+55jfkDCH/Dbi8b69c26\nww3vQ0zDb4YuhHAuzpEQlk8BU4mxTsupm29YnYzI8XT1pE1QG9oEtan0uNaa7OLsSs1PqXmpDI0c\neu4JZke3GyOHEv+CwBbGmvWdbgKXWuyBLIQQF8g5EkLiX8aKjuWrbgPujSzJIaZyoghuaay5cpEn\nYaUUQV5BBHkF0SOsR/WFsw/Dn/8xJpd5B8Lw16H3OHDzvKgYhBCiNpwjIdy31NgiMvuwsZ1gVuLp\nW9oe2Luk8kJdbl7GqqBVJYyAiLO2H7xghSeMzuL1s43aysDHjSV/vQPr5vhCCFELzpEQAFzdIaSV\ncTuT2WSs4lgxUZxKHAf+gLKi02Vd3I2JYGfWKoJjIDDKeJ/zKS2CjXOMBeiKcqD7HcaqkwERdfZx\nhRCitpwnIVTHxRWCWhi3VpdWfs5shtyjZyQLS8JIWgWl+afLKlcjKZyVLFoax3ZxN3Yr+/NVY+RQ\n6yuNkUNNK49qEkIIa5CEcD4uLsaY/4Dws9cI0hry0qpIFomQshGKT1YorIymoMITxv7F18+ElkMb\n8IMIIUT1JCFcDKXAr4lxa1F5OCpaG7OLKyaJnGRoeSl0vllGDgkhbI4khPqiFDQKMW6Rva0djRBC\nnJdcpgohhAAkIQghhLCQhCCEEAKQhCCEEMJCEoIQQghAEoIQQggLSQhCCCEASQhCCCEs7GpPZaVU\nOnDoAl/eGMg4bynnId/HafJdVCbfx2mO8l200FqHnq+QXSWEi6GUiqvJJtPOQr6P0+S7qEy+j9Oc\n7buQJiMhhBCAJAQhhBAWzpQQZls7ABsj38dp8l1UJt/HaU71XThNH4IQQojqOVMNQQghRDUcPiEo\npUYopfYqpRKUUhOtHY81KaUilVJ/KaV2K6V2KaUet3ZMtkAp5aqU2qKU+snasViTUipQKbVIKRWv\nlNqjlOp//lc5LqXUk5b/JzuVUt8opbysHVN9c+iEoJRyBd4DRgIdgduVUh2tG5VVlQFPa607Av2A\nh538+zjlcWCPtYOwAdOBpVrr9kA3nPg7UUqFA48BsVrrzoArMNq6UdU/h04IQB8gQWudqLUuAeYB\n11s5JqvRWh/VWm+23M/F+A8fbt2orEspFQFcDXxk7VisSSkVAFwCfAygtS7RWmdbNyqrcwO8lVJu\ngA9wxMrx1DtHTwjhQHKF31Nw8hPgKUqpaKAHsN66kVjdO8CzgNnagVhZDJAOzLU0n32klGpk7aCs\nRWudCkwFDgNHgRyt9a/Wjar+OXpCEFVQSvkC3wJPaK1PWjsea1FKXQOkaa03WTsWG+AG9AQ+0Fr3\nAPIBp+1zU0oFYbQmxADNgUZKqbusG1X9c/SEkApEVvg9wvKY01JKuWMkg6+01t9ZOx4rGwhcp5RK\nwmhOvEwp9aV1Q7KaFCBFa32qxrgII0E4qyuAg1rrdK11KfAdMMDKMdU7R08IG4E2SqkYpZQHRqfQ\nYivHZDVKKYXRRrxHaz3N2vFYm9b6ea11hNY6GuNv40+ttcNfBVZFa30MSFZKtbM8dDmw24ohWdth\noJ9Sysfy/+ZynKCT3c3aAdQnrXWZUuoRYBnGKIFPtNa7rByWNQ0E7gZ2KKW2Wh77p9b6FyvGJGzH\no7iXW8EAAABcSURBVMBXlounROBeK8djNVrr9UqpRcBmjNF5W3CCWcsyU1kIIQTg+E1GQgghakgS\nghBCCEASghBCCAtJCEIIIQBJCEIIISwkIQghhAAkIQghhLCQhCCEEAKA/weG827iR0+IkwAAAABJ\nRU5ErkJggg==\n", 916 | "text/plain": [ 917 | "" 918 | ] 919 | }, 920 | "metadata": {}, 921 | "output_type": "display_data" 922 | } 923 | ], 924 | "source": [ 925 | "plt.plot(y_preds_posterior.var(0)[error_under_posterior].mean(0), label='errors')\n", 926 | "plt.plot(y_preds_posterior.var(0)[~error_under_posterior].mean(0), label='correct')\n", 927 | "plt.plot(y_preds_posterior.var(0).mean(0), label='all')\n", 928 | "plt.legend()" 929 | ] 930 | }, 931 | { 932 | "cell_type": "markdown", 933 | "metadata": {}, 934 | "source": [ 935 | "Wow, variance seems to be much higher for pictures where error was done" 936 | ] 937 | }, 938 | { 939 | "cell_type": "markdown", 940 | "metadata": {}, 941 | "source": [ 942 | "## Expected error rate (Inconsistency in predictions)\n", 943 | "\n", 944 | "We can get the posterior distribution for predictions. That gives a lot of information for us. When we constructed `y_pred_posterior` we minimized that expected rate of error, taking the mode of the predictive distribution. Note that is true for categorical predictions. For regression problem with L2 loss one preferes to integrate out mean of the posterior.\n", 945 | "\n", 946 | "__Disclamer__: that's not exactly the expected error for object classification but for estimating the right statistic of distribution. In out case we estimate mode for categorical distribution. This error is about predicting wrong mode and thus taking another desision." 947 | ] 948 | }, 949 | { 950 | "cell_type": "code", 951 | "execution_count": 50, 952 | "metadata": {}, 953 | "outputs": [ 954 | { 955 | "data": { 956 | "text/plain": [ 957 | "" 958 | ] 959 | }, 960 | "execution_count": 50, 961 | "metadata": {}, 962 | "output_type": "execute_result" 963 | }, 964 | { 965 | "data": { 966 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHF5JREFUeJzt3XuYHmV9//H3h4STEEggkdIksIABDP4oh+VUaBtAAqIS\n/IEQL9CEovmJeOilrQSkJaL8AGtBqBaNJc3BAwSoEhGhIRAEbQiJBJCTBExKIoeEJJxBEr/9Y+6F\ncd3DzGbneWazn9d1zbUz99wz830mm/0+98w99ygiMDMzK2qzZgdgZmZ9ixOHmZmV4sRhZmalOHGY\nmVkpThxmZlaKE4eZmZXixGHWyyRNlHR3s+Mwq4oThzWdpGWSXpP0cm76ZhPjmS/p4xXtu0VStPus\nL0s6tYrjdRHHFElvpmOvk/RLSYeV2L6yc2T1N7DZAZglH4yI25odRAMNjoj13VWSNCAiNnRX1s0+\nBnZyrGsj4nRJA4EvA9cBI4ru1/ovtzis1iRdJemG3PKlkuYpM0bSCknnSVqdWi6n5epuKenrkv5H\n0rOSvi1p69z6cZKWSHpR0hOSjpN0EfBXwDfzLR9Je0uaK2mNpMcknZLbz46S5qT9LAT22IjPOz19\n5pslvQIc2UnZ9pJmSlolabmk8yVtlvYxUdIvJF0u6XlgSlfHTEnl+8BwScPSPoZIuintf22aH5HW\nlT5HtomJCE+emjoBy4D3drLuHcBvgIlkf6xWAyPSujHAeuAyYEvgb4BXgL3S+suBOcAOwCDgJ8DF\nad3BwAvAMWRfoIYDe6d184GP52LYBngKOIOslb5/imN0Wn8NMDvVew+wEri7k8/TAgQwsJP101Nc\nh6e4tuqkbCZwY/pcLekcnZn2MTGdl8+keLfu4DhTgO+l+S2AS9JnGpjKdgROSud/EFlr5Me57Uud\nI0+b1tT0ADx5SonjZWBdbvpEbv0hwBpgOfCRXHlb4tgmVzYb+EdAKYnskVt3GPDbNP8d4PJO4mn/\nR/FU4K52db4DXAAMAN5sSzpp3f8vkDjWtZvendZPB2a22+aPytIxf5//owz8P2B+mp8I/E8353xK\n2sc6YAPwPDCmi/r7AWt7co6a/fvlqfcn3+OwujgxOrnHERH3SHoSeCdZYshbGxGv5JaXA38ODCP7\ntrxYUts6kf3RBRgJ3Fwwtl2BQySty5UNBGal4wwk+7adj6E7Q6PzexxPdVM2FNi83XGWk7WautpH\ne7Mju8cxFLgBOJAsISDpHWQttuOAIan+oC7ur3R1jmwT43scVnuSzia7FPU74IvtVg+RtE1ueZdU\nbzXwGrBPRAxO0/YRsW2q9xSd34toP2T0U8Cduf0MjohtI+IsYBVZq2dkuxg2RkdDVufLVpO1cnZt\nd8yV3eyj44NFrAYmAVMk7ZyKvwDsBRwSEdsBf53K27JwmXNkmxgnDqs1SXsCXwVOBz4KfFHSfu2q\nfVnSFpL+CvgAcF1E/AH4LnC5pHemfQ2XdGza5mrgDElHS9osrds7rXsW2D23/5uAPSV9VNLmaTpI\n0rvTt+//JPuj+w5Jo4EJvX8m3paOORu4SNIgSbsCnwe+txH7fAy4lbcT8yCyxLtO0g5kl+XyCp+j\nnsZk9eXEYXXxk3bPNfwodRP9HnBpRNwfEY8D5wGzJG2ZtnsGWEvWyvg+8MmIeDStOwdYCiyQ9CJw\nG9m3aCJiIdmN3MvJbjzfydvf4K8ATk69ia6MiJeAscD4dJxngEvJWkEAnwa2TeXTgf8o8HnXtfu8\nny93uvgM2T2cJ4G7gR8A00ruo71/BialRPsNYGuy1s0C4JZ2dcueI9uEKMIvcrK+SdIYsp5BfvbA\nrIHc4jAzs1KcOMzMrBRfqjIzs1Lc4jAzs1I2yQcAhw4dGi0tLc0Ow8ysT1m8ePHqiBjWXb1NMnG0\ntLSwaNGiZodhZtanSCoy6oEvVZmZWTlOHGZmVooTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV\n4sRhZmalVJo4JC2T9KCkJZIWpbIdJM2V9Hj6OSSVS9KVkpZKekDSAbn9TEj1H5dU6UtyzMysa414\ncvzI9GrKNpOBeRFxiaTJafkc4H3AqDQdAlxF9g7jtrePtZK9rnKxpDkRsbaqgFsm/7TH2y675P29\nGImZWf0041LVOGBGmp8BnJgrnxmZBcDg9P7jY4G5EbEmJYu5wHGNDtrMzDJVJ44A/kvSYkmTUtlO\nEfF0mn8G2CnNDyd74X2bFamss/I/ImmSpEWSFq1atao3P4OZmeVUfanqiIhYmd5hPFfSo/mVERGS\neuWFIBExFZgK0Nra6peMmJlVpNIWR0SsTD+fA34EHAw8my5BkX4+l6qvBEbmNh+RyjorNzOzJqgs\ncUjaRtKgtnlgLPBrYA7Q1jNqAnBjmp8DfCz1rjoUeCFd0roVGCtpSOqBNTaVmZlZE1R5qWon4EeS\n2o7zg4i4RdK9wGxJZwLLgVNS/ZuB44GlwKvAGQARsUbSV4B7U70LI2JNhXGbmVkXKkscEfEk8Bcd\nlD8PHN1BeQBnd7KvacC03o7RzMzK85PjZmZWihOHmZmV4sRhZmalOHGYmVkpThxmZlaKE4eZmZXi\nxGFmZqU4cZiZWSlOHGZmVooTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV4sRhZmalOHGYmVkp\nThxmZlaKE4eZmZXixGFmZqU4cZiZWSlOHGZmVooTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV\n4sRhZmalOHGYmVkpThxmZlaKE4eZmZXixGFmZqU4cZiZWSmVJw5JAyTdJ+mmtLybpHskLZV0raQt\nUvmWaXlpWt+S28e5qfwxScdWHbOZmXWuES2OzwGP5JYvBS6PiHcBa4EzU/mZwNpUfnmqh6TRwHhg\nH+A44N8kDWhA3GZm1oFKE4ekEcD7gX9PywKOAq5PVWYAJ6b5cWmZtP7oVH8ccE1EvBERvwWWAgdX\nGbeZmXWu6hbHN4AvAn9IyzsC6yJifVpeAQxP88OBpwDS+hdS/bfKO9jmLZImSVokadGqVat6+3OY\nmVlSWeKQ9AHguYhYXNUx8iJiakS0RkTrsGHDGnFIM7N+aWCF+z4cOEHS8cBWwHbAFcBgSQNTq2IE\nsDLVXwmMBFZIGghsDzyfK2+T38bMzBqsshZHRJwbESMiooXs5vbtEXEacAdwcqo2Abgxzc9Jy6T1\nt0dEpPLxqdfVbsAoYGFVcZuZWdeqbHF05hzgGklfBe4Drk7lVwOzJC0F1pAlGyLiIUmzgYeB9cDZ\nEbGh8WGbmRk0KHFExHxgfpp/kg56RUXE68CHO9n+IuCi6iI0M7Oi/OS4mZmV4sRhZmalOHGYmVkp\nThxmZlaKE4eZmZXixGFmZqU4cZiZWSlOHGZmVooTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV\n4sRhZmalOHGYmVkpThxmZlZKocQh6WuStpO0uaR5klZJOr3q4MzMrH6KtjjGRsSLwAeAZcC7gH+o\nKigzM6uvoomj7RWz7weui4gXKorHzMxqrug7x2+S9CjwGnCWpGHA69WFZWZmdVWoxRERk4G/BFoj\n4k3gFWBclYGZmVk9FW1xAOwNtEjKbzOzl+MxM7OaK5Q4JM0C9gCWABtSceDEYWbW7xRtcbQCoyMi\nqgzGzMzqr2ivql8Df1ZlIGZm1jcUbXEMBR6WtBB4o60wIk6oJCozM6utooljSpVBmJlZ31EocUTE\nnZJ2Ag5KRQsj4rnqwjIzs7oqOlbVKcBC4MPAKcA9kk6uMjAzM6unopeqvgQc1NbKSE+O3wZcX1Vg\nZmZWT0V7VW3W7tLU8yW2NTOzTUjRFsctkm4FfpiWTwVuriYkMzOrs6JjVf0DMBXYN01TI+KcrraR\ntJWkhZLul/SQpC+n8t0k3SNpqaRrJW2RyrdMy0vT+pbcvs5N5Y9JOrZnH9XMzHpD4bGqIuIG4IYS\n+34DOCoiXpa0OXC3pJ8Bnwcuj4hrJH0bOBO4Kv1cGxHvkjQeuBQ4VdJoYDywD/DnwG2S9oyIDR0d\n1MzMqtVli0PS3ennS5JezE0vSXqxq20j83Ja3DxNARzF2zfVZwAnpvlxaZm0/mhJSuXXRMQbEfFb\nYClwcKlPaWZmvabLxBERR6SfgyJiu9w0KCK2627nkgZIWgI8B8wFngDWRcT6VGUFMDzNDweeSsdb\nD7wA7Jgv72Cb/LEmSVokadGqVau6C83MzHqo6HMcs4qUtRcRGyJiP2AEWSth79IRFhQRUyOiNSJa\nhw0bVtVhzMz6vaJdavfJL6R3chxY9CARsQ64AzgMGJx7p8cIYGWaXwmMzO1/e7Juv2+Vd7CNmZk1\nWHf3OM6V9BKwb/7+BvAscGM32w6TNDjNbw0cAzxClkDanjqfkNvPnLRMWn97GsZ9DjA+9braDRhF\n9hS7mZk1QZe9qiLiYuBiSRdHxLkl970zMEPSALIENTsibpL0MHCNpK8C9wFXp/pXA7MkLQXWkPWk\nIiIekjQbeBhYD5ztHlVmZs1TdJDDcyUNIfu2v1Wu/OddbPMAsH8H5U/SQa+oiHidbCysjvZ1EXBR\nkVjNzKxaRV8d+3Hgc2T3F5YAhwL/Tda11szM+pGiN8c/Rzak+vKIOJKsJbGusqjMzKy2iiaO19Ol\nJCRtGRGPAntVF5aZmdVV0SFHVqQeUj8G5kpaCyyvLiwzM6urojfHP5Rmp0i6g+wZi1sqi8rMzGqr\n6M3xK8nGi/plRNxZcUxmZlZjRe9xLAbOl/SEpK9Laq0yKDMzq6+i7+OYERHHk/Wsegy4VNLjlUZm\nZma1VPb1r+8iG6hwV+DR3g/HzMzqrujouF9LLYwLgQeB1oj4YKWRmZlZLXV7czy9TOkl4LCIWF19\nSGZmVmfdtjjSCLWnOGmYmRkUv8fxK0kHVRqJmZn1CUWfHD8EOE3ScuAVQGSNkX0ri8zMzGqpaOI4\nttIozMyszyj6HMdyste3HpXmXy26rZmZbVqKdse9ADgHaHsL4ObA96oKyszM6qtoq+FDwAlk9zeI\niN8Bg6oKyszM6qto4vh96pYbAJK2qS4kMzOrs6KJY7ak7wCDJX0CuA34bnVhmZlZXRV9H8fXJR0D\nvEj25r9/ioi5lUZmZma1VPR9HNsAt0fEXEl7AXtJ2jwi3qw2PDMzq5uil6p+DmwpaTjZm/8+Ckyv\nKigzM6uvoolDEfEq8H+BqyLiw8A+1YVlZmZ1VThxSDoMOA34aSobUE1IZmZWZ0UTx9+RPfz3o4h4\nSNLuwB3VhWVmZnVVtFfVncCdkraTNCgingQ+W21oZmZWR0WHHGmV9CDwAPBrSfdLOrDa0MzMrI6K\njo47DfhURNwFIOkI4D8AD6tuZtbPFL3HsaEtaQBExN3A+mpCMjOzOuuyxSHpgDR7Zxpy5Idk41Wd\nCsyvNjQzM6uj7i5V/Uu75Qty89HLsZiZWR/QZeKIiCN7umNJI4GZwE5kSWZqRFwhaQfgWqAFWAac\nEhFrJQm4Ajie7EVREyPiV2lfE4Dz066/GhEzehqXmZltnKK9qraXdJmkRWn6F0nbd7PZeuALETEa\nOBQ4W9JoYDIwLyJGAfPSMsD7gFFpmgRclY69A1lL5xDgYOACSUNKfUozM+s1RW+OTwNeAk5J04tk\nvao6FRFPt7UYIuIl4BFgODAOaGsxzABOTPPjgJmRWUA2hPvOZO87nxsRayJiLTAXOK5g3GZm1suK\ndsfdIyJOyi1/WdKSogeR1ALsD9wD7BQRT6dVz5BdyoIsqTyV22xFKuusvP0xJpG1VNhll12KhmZm\nZiUVbXG8lp7dAEDS4cBrRTaUtC1wA/B3EfFifl3+rYIbKyKmRkRrRLQOGzasN3ZpZmYdKNri+CQw\nM93XELAGmNDdRpI2J0sa34+I/0zFz0raOSKeTpeinkvlK4GRuc1HpLKVwJh25fMLxm1mZr2saIvj\ngxHxF2RPiv+fiNg/Ih7oaoPUS+pq4JGIuCy3ag5vJ50JwI258o8pcyjwQrqkdSswVtKQdFN8bCoz\nM7Mm6DJxSDonDad+MkC61DS/4L4PJ3vh01GSlqTpeOAS4BhJjwPvTcsANwNPAkvJ3mf+qXTMNcBX\ngHvTdGEqMzOzJujuUtWjwIeB3SXdlZZ3lLRXRDzW1YZpWBJ1svroDuoHcHYn+5pG1rPLzMyarLtL\nVeuA88haAWPIHtADmCzplxXGZWZmNdVdi+NY4J+APYDLyIZVfyUizqg6MDMzq6cuWxwRcV5EHE02\nNMgsstfFDpN0t6SfNCA+MzOrmaLdcW+NiEXAIklnRcQRkoZWGZiZmdVToe64EfHF3OLEVLa6ioDM\nzKzeij7H8ZaIuL+KQMzMrG8onTjMzKx/c+IwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzM\nrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPM\nzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMyslIHNDmBT0zL5pz3edtkl7+/FSMzM\nquEWh5mZleLEYWZmpVSWOCRNk/ScpF/nynaQNFfS4+nnkFQuSVdKWirpAUkH5LaZkOo/LmlCVfGa\nmVkxVbY4pgPHtSubDMyLiFHAvLQM8D5gVJomAVdBlmiAC4BDgIOBC9qSjZmZNUdliSMifg6saVc8\nDpiR5mcAJ+bKZ0ZmATBY0s7AscDciFgTEWuBufxpMjIzswZq9D2OnSLi6TT/DLBTmh8OPJWrtyKV\ndVb+JyRNkrRI0qJVq1b1btRmZvaWpt0cj4gAohf3NzUiWiOiddiwYb21WzMza6fRiePZdAmK9PO5\nVL4SGJmrNyKVdVZuZmZN0ujEMQdo6xk1AbgxV/6x1LvqUOCFdEnrVmCspCHppvjYVGZmZk1S2ZPj\nkn4IjAGGSlpB1jvqEmC2pDOB5cApqfrNwPHAUuBV4AyAiFgj6SvAvanehRHR/oa7mZk1UGWJIyI+\n0smqozuoG8DZnexnGjCtF0MzM7ON4CfHzcysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvF\nicPMzErxq2NrxK+dNbO+wC0OMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFvao2Ee6RZWaN4haH\nmZmV4sRhZmalOHGYmVkpThxmZlaKE4eZmZXiXlW2UT2ywL2yzPobtzjMzKwUJw4zMyvFl6pso/nh\nQ7P+xYnDmspJx6zv8aUqMzMrxS0O67PcWinH58t6ixOH9UvN7IK8scc2azZfqjIzs1Lc4jDrAbca\nrD9zi8PMzEpxi8PMuuVhaSzPicPMKuceXZuWPnOpStJxkh6TtFTS5GbHY2bWX/WJFoekAcC3gGOA\nFcC9kuZExMPNjczMqtasjghu6XSuTyQO4GBgaUQ8CSDpGmAc4MRhZpXw5bXO9ZXEMRx4Kre8Ajgk\nX0HSJGBSWnxZ0mMbcbyhwOqN2L5Kjq1nHFvPOLYe0KX1jY2uz9uuRXbQVxJHtyJiKjC1N/YlaVFE\ntPbGvnqbY+sZx9Yzjq1nNvXY+srN8ZXAyNzyiFRmZmYN1lcSx73AKEm7SdoCGA/MaXJMZmb9Up+4\nVBUR6yV9GrgVGABMi4iHKjxkr1zyqohj6xnH1jOOrWc26dgUEb0RiJmZ9RN95VKVmZnVhBOHmZmV\n0m8TR3dDmEjaUtK1af09klpqFNtfS/qVpPWSTm5UXAVj+7ykhyU9IGmepEL9whsU2yclPShpiaS7\nJY2uS2y5eidJCkkN68pZ4LxNlLQqnbclkj5el9hSnVPS79xDkn5Ql9gkXZ47Z7+RtK5Gse0i6Q5J\n96X/q8eXOkBE9LuJ7Ab7E8DuwBbA/cDodnU+BXw7zY8Hrq1RbC3AvsBM4OSanbcjgXek+bNqdt62\ny82fANxSl9hSvUHAz4EFQGtdYgMmAt9s1O9ZydhGAfcBQ9LyO+sSW7v6nyHr1FOL2MhukJ+V5kcD\ny8oco7+2ON4awiQifg+0DWGSNw6YkeavB46WpDrEFhHLIuIB4A8NiKdsbHdExKtpcQHZMzd1ie3F\n3OI2QKN6hhT5fQP4CnAp8HqD4ioTWzMUie0TwLciYi1ARDxXo9jyPgL8sCGRFYstgO3S/PbA78oc\noL8mjo6GMBneWZ2IWA+8AOxYk9iapWxsZwI/qzSitxWKTdLZkp4AvgZ8ti6xSToAGBkRjR7Rr+i/\n6Unpksb1kkZ2sL4KRWLbE9hT0i8kLZB0XI1iAyBdrt0NuL0BcUGx2KYAp0taAdxM1iIqrL8mDquY\npNOBVuCfmx1LXkR8KyL2AM4Bzm92PACSNgMuA77Q7Fg68ROgJSL2Bebydku8DgaSXa4aQ/at/ruS\nBjc1oj81Hrg+IjY0O5CcjwDTI2IEcDwwK/0eFtJfE0eRIUzeqiNpIFlz7vmaxNYshWKT9F7gS8AJ\nEfFGnWLLuQY4sdKI3tZdbIOA9wDzJS0DDgXmNOgGebfnLSKez/07/jtwYAPiKhQb2bfpORHxZkT8\nFvgNWSKpQ2xtxtO4y1RQLLYzgdkAEfHfwFZkgx8W04ibNXWbyL6lPEnWfGy7ebRPuzpn88c3x2fX\nJbZc3ek09uZ4kfO2P9mNuVE1/DcdlZv/ILCoLrG1qz+fxt0cL3Leds7NfwhYUKPYjgNmpPmhZJdo\ndqxDbKne3sAy0sPWNTpvPwMmpvl3k93jKBxjQz5IHSey5tlv0h+5L6WyC8m+JUOWga8DlgILgd1r\nFNtBZN+0XiFrBT1Uo9huA54FlqRpTo1iuwJ4KMV1R1d/vBsdW7u6DUscBc/bxem83Z/O2941ik1k\nl/keBh4ExtcltrQ8BbikUTGVOG+jgV+kf9MlwNgy+/eQI2ZmVkp/vcdhZmY95MRhZmalOHGYmVkp\nThxmZlaKE4eZmZXixGFmZqU4cVi/JWlDbtjrJW3DT0taloZfb9jQ5j0h6byC9e6Q9HLdP4/1HX6O\nw/otSS9HxLYdlC8jewBvdYl9DYxsMMwOl4tuV3RdWt9h/J3UnQ/8fUQsKlLfrCtucZh1Q9KBku6U\ntFjSrZJ2TuXzJX1D0iLgc5KmS/q2pHuAr0naQdKP06iyCyTtm7abImmWpF8As9oda4ykuyTNIXsa\nmrSPxelFRZNS2SXA1qml9P1UdrqkhansO5IGNOwkWb8ysNkBmDXR1pKW5JYvjohr8xUkbQ78KzAu\nIlZJOhW4CPjbVGWLiGhNdaeTDSj3lxGxQdK/AvdFxImSjiJ78dZ+abvRwBER8VoHcR0AvCeyQfsA\n/jYi1kjaGrhX0g0RMVnSpyNiv3TsdwOnAodHxJuS/g04LR3TrFc5cVh/9lrbH94u7EU2cu3c9B6v\nAcDTufXXtqt/Xbw9fPYRwEkAEXG7pB0ltb08Z04nSQNgYS5pAHxW0ofS/Eiy0V/bj9R8NNmotfem\nOLcGGvVSI+tnnDjMuiayQSQP62T9K90sd6arem+tkzQGeC9wWES8mu5VbNVJnDMi4tyCxzfrMd/j\nMOvaY8AwSYdBdulK0j4Ft72L7HJRWwJYHX/8+toitgfWpqSxN9m7Otq8mS6lAcwDTpb0znS8HdKb\n58x6nVsc1p+1v8dxS0RMzleIiN9LOhm4UtL2ZP9nvkE2zHh3pgDTJD0AvApM6EGMtwCflPQIWRJb\nkFs3FXhA0q8i4jRJ5wP/ld7k9ibZO2WW9+CYZl1yd1yzdnrSHbfu3B3XepMvVZn9qVXAvE3lgTlJ\ndwC7k7VCzDaaWxxmZlaKWxxmZlaKE4eZmZXixGFmZqU4cZiZWSn/C5ycdPEq4MohAAAAAElFTkSu\nQmCC\n", 967 | "text/plain": [ 968 | "" 969 | ] 970 | }, 971 | "metadata": {}, 972 | "output_type": "display_data" 973 | } 974 | ], 975 | "source": [ 976 | "y_preds_labels = np.argmax(y_preds_posterior, axis=-1)\n", 977 | "prediction_expected_error_rate = (y_preds_labels != y_pred_posterior).mean(0)\n", 978 | "plt.hist(prediction_expected_error_rate, bins=20)\n", 979 | "plt.title('Expected Error Rate');\n", 980 | "plt.xlabel('E[error rate]')\n", 981 | "plt.ylabel('#observations')" 982 | ] 983 | }, 984 | { 985 | "cell_type": "markdown", 986 | "metadata": {}, 987 | "source": [ 988 | "Seems like our model is not always confident in predictions and is aware of it. Let's check how often it happens so" 989 | ] 990 | }, 991 | { 992 | "cell_type": "code", 993 | "execution_count": 51, 994 | "metadata": {}, 995 | "outputs": [ 996 | { 997 | "data": { 998 | "text/plain": [ 999 | "0.377" 1000 | ] 1001 | }, 1002 | "execution_count": 51, 1003 | "metadata": {}, 1004 | "output_type": "execute_result" 1005 | } 1006 | ], 1007 | "source": [ 1008 | "((prediction_expected_error_rate != 0) == error_under_posterior).mean()" 1009 | ] 1010 | }, 1011 | { 1012 | "cell_type": "markdown", 1013 | "metadata": {}, 1014 | "source": [ 1015 | "In 37% cases our model is not confident enough and it leeds to mistake. \n", 1016 | "\n", 1017 | "## High confident errors\n", 1018 | "Next. How often is out model overconfident? In other words it seems all to be okay but we have an error." 1019 | ] 1020 | }, 1021 | { 1022 | "cell_type": "code", 1023 | "execution_count": 52, 1024 | "metadata": {}, 1025 | "outputs": [ 1026 | { 1027 | "data": { 1028 | "text/plain": [ 1029 | "0.0053475935828877002" 1030 | ] 1031 | }, 1032 | "execution_count": 52, 1033 | "metadata": {}, 1034 | "output_type": "execute_result" 1035 | } 1036 | ], 1037 | "source": [ 1038 | "(prediction_expected_error_rate[error_under_posterior] == 0).mean()" 1039 | ] 1040 | }, 1041 | { 1042 | "cell_type": "markdown", 1043 | "metadata": {}, 1044 | "source": [ 1045 | "Good news, It happens pretty rare, exactly 4 times." 1046 | ] 1047 | }, 1048 | { 1049 | "cell_type": "code", 1050 | "execution_count": 53, 1051 | "metadata": {}, 1052 | "outputs": [ 1053 | { 1054 | "data": { 1055 | "text/plain": [ 1056 | "4" 1057 | ] 1058 | }, 1059 | "execution_count": 53, 1060 | "metadata": {}, 1061 | "output_type": "execute_result" 1062 | } 1063 | ], 1064 | "source": [ 1065 | "(prediction_expected_error_rate[error_under_posterior] == 0).sum()" 1066 | ] 1067 | }, 1068 | { 1069 | "cell_type": "code", 1070 | "execution_count": 54, 1071 | "metadata": { 1072 | "collapsed": true 1073 | }, 1074 | "outputs": [], 1075 | "source": [ 1076 | "houston_we_have_a_problem = prediction_expected_error_rate[error_under_posterior] == 0" 1077 | ] 1078 | }, 1079 | { 1080 | "cell_type": "code", 1081 | "execution_count": 55, 1082 | "metadata": { 1083 | "collapsed": true 1084 | }, 1085 | "outputs": [], 1086 | "source": [ 1087 | "def problem_display():\n", 1088 | " for i in range(houston_we_have_a_problem.sum()):\n", 1089 | " print('true:', y_test[error_under_posterior][houston_we_have_a_problem][i],\n", 1090 | " 'prediction:', y_pred_posterior[error_under_posterior][houston_we_have_a_problem][i])\n", 1091 | " plt.gray();plt.matshow(X_test[error_under_posterior][houston_we_have_a_problem][i][0]);plt.show();" 1092 | ] 1093 | }, 1094 | { 1095 | "cell_type": "code", 1096 | "execution_count": 56, 1097 | "metadata": {}, 1098 | "outputs": [ 1099 | { 1100 | "name": "stdout", 1101 | "output_type": "stream", 1102 | "text": [ 1103 | "true: 7.0 prediction: 2\n" 1104 | ] 1105 | }, 1106 | { 1107 | "data": { 1108 | "text/plain": [ 1109 | "" 1110 | ] 1111 | }, 1112 | "metadata": {}, 1113 | "output_type": "display_data" 1114 | }, 1115 | { 1116 | "data": { 1117 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADoBJREFUeJzt3X2MVGWWx/HfAV2DoAbp2AIruryYOEJsTYeMBjeYcYlO\nTARjyOA/mIyDf4zJmGziEhN8iRliNoO78xfKrGSYOOOGxGFRNBmkQyIGJYNKEGRXCHarvHQLqOC7\n0mf+6MvZHu1+bnXd6rq3m+8nIVV1T1Xd0xf48dxbD0+ZuwsAJGlM2Q0AqA4CAUAgEAAEAgFAIBAA\nBAIBQCglEMzsFjP7PzM7YGbLy+ghxcw6zextM9tlZjsr0M9aM+sxsz39tl1sZi+b2f7sdmLF+nvE\nzA5lx3CXmf20xP4uM7OtZvaOme01s19l2ytxDBP9Nf0YWrPnIZjZWEnvSvoXSR9K+qukJe7+TlMb\nSTCzTknt7n6s7F4kycz+WdJnkv7g7rOzbf8u6YS7P56F6kR3/7cK9feIpM/c/Tdl9NSfmU2WNNnd\n3zSzCyS9IWmhpLtVgWOY6G+xmnwMyxghzJV0wN0Puvs3kv5b0u0l9DFiuPsrkk58b/PtktZl99ep\n7w9QKQbprzLc/Yi7v5ndPyVpn6SpqsgxTPTXdGUEwlRJH/R7/KFK+uETXNJmM3vDzJaV3cwgWt39\nSHb/qKTWMpsZxH1mtjs7pSjtlKY/M7tC0rWSdqiCx/B7/UlNPoZcVBzYPHe/TtKtkn6ZDYkry/vO\n+6o2B321pBmS2iQdkbSq3HYkM5sg6TlJ97v7yf61KhzDAfpr+jEsIxAOSbqs3+N/zLZVhrsfym57\nJG1Q32lO1XRn555nzkF7Su7n77h7t7ufdvdeSb9TycfQzM5V31+2P7r7n7PNlTmGA/VXxjEsIxD+\nKmmWmf2Tmf2DpJ9Jer6EPgZkZuOzCzsys/GSFkjak35VKZ6XtDS7v1TSxhJ7+YEzf9Eyi1TiMTQz\nk/S0pH3u/kS/UiWO4WD9lXEMm/4pgyRlH5/8p6Sxkta6+6+b3sQgzGy6+kYFknSOpD+V3Z+ZPStp\nvqQWSd2SHpb0P5LWS5omqUvSYncv5cLeIP3NV99Q1yV1Srq33/l6s/ubJ2mbpLcl9WabH1TfeXrp\nxzDR3xI1+RiWEggAqomLigACgQAgEAgAAoEAIBAIAEKpgVDhacGS6K+oKvdX5d6k8vore4RQ6d8U\n0V9RVe6vyr1JJfVXdiAAqJBCE5PM7BZJv1XfjMP/cvfHc57PLCigJO5uec+pOxDqWeiEQADKU0sg\nFDllYKETYJQpEggjYaETAENwznDvIPv4pOpXdAGoWCDUtNCJu6+RtEbiGgJQdUVOGSq90AmAoat7\nhODu35nZfZL+ov9f6GRvwzoD0HRNXSCFUwagPMP9sSOAUYZAABAIBACBQAAQCAQAgUAAEAgEAIFA\nABAIBACBQAAQCAQAgUAAEAgEAIFAABAIBACBQAAQCAQAgUAAEAgEAIFAABAIBACBQAAQCAQAgUAA\nEAgEAIFAABAIBACBQAAQCAQAgUAAEM4pu4EqefTRR5P1O+64I1nv6OhI1o8ePZqsr1+/Plnv6upK\n1idMmJCsn3/++cn6uHHjkvWDBw8Wen1ef3mvf//995N1FFcoEMysU9IpSaclfefu7Y1oCkA5GjFC\nuMndjzXgfQCUjGsIAELRQHBJm83sDTNb1oiGAJSn6CnDPHc/ZGaXSHrZzP7X3V/p/4QsKAgLYAQo\nNEJw90PZbY+kDZLmDvCcNe7ezgVHoPrqDgQzG29mF5y5L2mBpD2NagxA85m71/dCs+nqGxVIface\nf3L3X+e8pr6dNcnGjRuT9dtuu61JnQxs8+bNyfrUqVOT9UsvvTRZnzRpUrL+4osvFnr9lClTkvWW\nlpZkffXq1cn6Aw88kKyf7dzd8p5T9zUEdz8o6Zp6Xw+gevjYEUAgEAAEAgFAIBAABAIBQCAQAISz\naj2ECy+8MFmfNm1akzqpz4IFC0rdf948jHrntJzx1VdfJetbt24t9P7IxwgBQCAQAAQCAUAgEAAE\nAgFAIBAABAIBQKh7PYS6dlbyegjTp09P1vfv31/o/Xt7e5P1LVu2JOttbW3J+vjx4wvVP/nkk2Q9\nb57G2LFjC73/mDHpf3++/vrrZP3yyy9P1vPmMZztalkPgRECgEAgAAgEAoBAIAAIBAKAQCAACAQC\ngHBWrYcw3I4dS38J9q233lro/a+++upkfc6cOcl63noCc+f+4Iu3/k7ePIdt27Yl63k//1NPPZWs\nP/nkk8n63XffnawjHyMEAIFAABAIBACBQAAQCAQAgUAAEAgEAIF5CCPI3r17C9XzvPDCC4VeP9yu\nv/76ZH3SpEnJ+vHjxxvZzqiUO0Iws7Vm1mNme/ptu9jMXjaz/dntxOFtE0Az1HLK8HtJt3xv23JJ\nHe4+S1JH9hjACJcbCO7+iqQT39t8u6R12f11khY2uC8AJaj3omKrux/J7h+V1NqgfgCUqPBFRXf3\n1OKpZrZM0rKi+wEw/OodIXSb2WRJym57Bnuiu69x93Z3b69zXwCapN5AeF7S0uz+UkkbG9MOgDLl\nnjKY2bOS5ktqMbMPJT0s6XFJ683s55K6JC0eziZHitWrV5fdwqg2c+bMQnXmIeTLDQR3XzJI6ScN\n7gVAyZi6DCAQCAACgQAgEAgAAoEAIBAIAMJZtR7CRx99lKw/9thjyfqsWbOS9ZUrVw65J6BKGCEA\nCAQCgEAgAAgEAoBAIAAIBAKAQCAACOY+6Opnjd9ZYqk1jH5TpkxJ1g8cOJCsn3feecn6DTfckKzv\n2LEjWR/t3N3ynsMIAUAgEAAEAgFAIBAABAIBQCAQAAQCAUA4q9ZDQLkOHz6crJ8+fbpJnWAwjBAA\nBAIBQCAQAAQCAUAgEAAEAgFAIBAABOYhIFx00UXJ+qRJk5L17u7uZP3zzz9P1k+dOpWsjx8/PllH\ncbkjBDNba2Y9Zran37ZHzOyQme3Kfv10eNsE0Ay1nDL8XtItA2z/D3dvy3691Ni2AJQhNxDc/RVJ\nJ5rQC4CSFbmoeJ+Z7c5OKSY2rCMApak3EFZLmiGpTdIRSasGe6KZLTOznWa2s859AWiSugLB3bvd\n/bS790r6naS5ieeucfd2d2+vt0kAzVFXIJjZ5H4PF0naM9hzAYwcufMQzOxZSfMltZjZh5IeljTf\nzNokuaROSfcOY4+jxrRp05L1Sy65JFnfvXt3sv7NN98k63feeWey/tBDDyXrs2fPTtZXrFiRrL/+\n+uvJekdHR7J+1113Jev33HNPsj5hwoRkvajt27cn619++eWw7r8RcgPB3ZcMsPnpYegFQMmYugwg\nEAgAAoEAIBAIAAKBACAQCACCuXvzdmbWvJ2VYNy4ccn6Sy+l/1Noe3t6MucHH3yQrO/cmZ4dvnDh\nwmQ9b70BM0vWm/lnqYpmzJiRrHd2djankUG4e/o3UIwQAPRDIAAIBAKAQCAACAQCgEAgAAgEAoDA\nPIQGam1tTdYPHz7cpE6Gx5gx6X8/ent7m9RJNTEPAcCoQiAACAQCgEAgAAgEAoBAIAAIBAKAkLsM\nO2r36aefJusbNmxI1hctWtTIdhoub87KyZMnk/WPP/44WT9+/Hiyft111yXrRb333nvJek9PT7Ke\n970YIwEjBACBQAAQCAQAgUAAEAgEAIFAABAIBACB9RCaaObMmcn69u3bk/W8z+nzXHnllYVe/8UX\nXyTrixcvTtbfeuutZD3v59u0aVOyfvPNNyfr69atS9aXL1+erOfNQ6i6hqyHYGaXmdlWM3vHzPaa\n2a+y7Reb2ctmtj+7ndiIpgGUp5ZThu8k/au7/0jSjyX90sx+JGm5pA53nyWpI3sMYATLDQR3P+Lu\nb2b3T0naJ2mqpNslnRmDrZOU/p4wAJU3pIuKZnaFpGsl7ZDU6u5HstJRSekFBQFUXs3/ucnMJkh6\nTtL97n6y/xd/ursPdsHQzJZJWla0UQDDr6YRgpmdq74w+KO7/znb3G1mk7P6ZEkDXoJ19zXu3u7u\n6a82BlC6Wj5lMElPS9rn7k/0Kz0vaWl2f6mkjY1vD0Az5c5DMLN5krZJelvSmYX3H1TfdYT1kqZJ\n6pK02N1P5LzXWT0PIc/EielPbr/99ttkva2tLVl/9913C71+1apVyfqcOXOS9aJuuummZH3Lli3J\n+sqVK5P1FStWDLmnkaSWeQi51xDc/VVJg73RT4baFIDqYuoygEAgAAgEAoBAIAAIBAKAQCAACHwv\nQ4XkfW9BnldffbXQ6zdv3pysD/c8gzx56yXkrdfQ0tLSyHZGJUYIAAKBACAQCAACgQAgEAgAAoEA\nIBAIAALfy4BR47XXXkvWr7nmmmT9qquuSta7urqG3FOVNOR7GQCcPQgEAIFAABAIBACBQAAQCAQA\ngUAAEFgPAaPGrl27kvW5c+cm688880yyfuONNw65p5GGEQKAQCAACAQCgEAgAAgEAoBAIAAIBAKA\nkLsegpldJukPkloluaQ17v5bM3tE0i8kfZQ99UF3fynnvVgPAShJLesh1BIIkyVNdvc3zewCSW9I\nWihpsaTP3P03tTZEIADlqSUQcmcquvsRSUey+6fMbJ+kqcXbA1A1Q7qGYGZXSLpW0o5s031mttvM\n1prZxAb3BqDJag4EM5sg6TlJ97v7SUmrJc2Q1Ka+EcSqQV63zMx2mtnOBvQLYBjVtMiqmZ0raZOk\nv7j7EwPUr5C0yd1n57wP1xCAkjRkkVUzM0lPS9rXPwyyi41nLJK0p54mAVRHLZ8yzJO0TdLbknqz\nzQ9KWqK+0wWX1Cnp3uwCZOq9GCEAJWnIx46NRCAA5eF7GQAMCYEAIBAIAAKBACAQCAACgQAgEAgA\nAoEAIBAIAAKBACAQCAACgQAgEAgAAoEAIBAIAELuqssNdkxSV7/HLdm2qqK/YqrcX5V7kxrf3+W1\nPKmpC6T8YOdmO929vbQGctBfMVXur8q9SeX1xykDgEAgAAhlB8Kakvefh/6KqXJ/Ve5NKqm/Uq8h\nAKiWskcIACqEQAAQCAQAgUAAEAgEAOFvcTNMUzR1/bkAAAAASUVORK5CYII=\n", 1118 | "text/plain": [ 1119 | "" 1120 | ] 1121 | }, 1122 | "metadata": {}, 1123 | "output_type": "display_data" 1124 | }, 1125 | { 1126 | "name": "stdout", 1127 | "output_type": "stream", 1128 | "text": [ 1129 | "true: 7.0 prediction: 4\n" 1130 | ] 1131 | }, 1132 | { 1133 | "data": { 1134 | "text/plain": [ 1135 | "" 1136 | ] 1137 | }, 1138 | "metadata": {}, 1139 | "output_type": "display_data" 1140 | }, 1141 | { 1142 | "data": { 1143 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADUhJREFUeJzt3V+IHXWaxvHn2cSAGEMSxLZNoq7/LsIIcWhkccLiIju6\nA2r0IhhkiDDYXkSMILIiiPFiMCxJdgOCkllDMjLjojiz0Tg4IzrgLIomRonRdtewJE5CJ9mgEL1x\nib570ZV3ejX9q9N9Tp+qTr4fCF2n3upTbyqdh/rV+XWVI0IAIEl/1XQDANqDQACQCAQAiUAAkAgE\nAIlAAJAaCQTbN9v+T9v7bT/cRA8ltg/Y/tD2B7Z3t6CfrbaP2d43bt1C26/Z/rT6uqBl/a2zfbg6\nhh/Y/kmD/S2x/UfbH9v+yPbaan0rjmGhv74fQ/d7HoLtWZL+S9LfSzokaZekVRHxcV8bKbB9QNJQ\nRBxvuhdJsv23kr6S9MuI+EG17p8kfR4R66tQXRAR/9ii/tZJ+ioiNjTR03i2ByUNRsQe2+dLek/S\nCkl3qwXHsNDfSvX5GDZxhnCdpP0R8d8R8b+S/k3SbQ30MWNExJuSPv/O6tskba+Wt2vsB6gRE/TX\nGhExGhF7quUvJY1IWqSWHMNCf33XRCAskvTnca8PqaG/fEFI+oPt92wPN93MBAYiYrRaPiJpoMlm\nJnCf7b3VkKKxIc14ti+TdK2kd9TCY/id/qQ+H0MuKp7e8oj4oaR/kLSmOiVurRgb97VtDvpTkq6Q\ntEzSqKSNzbYj2Z4r6UVJD0TEifG1NhzD0/TX92PYRCAclrRk3OvF1brWiIjD1ddjkn6rsWFO2xyt\nxp6nxqDHGu7n/4mIoxHxTUR8K+kXavgY2j5HY//ZfhURv6lWt+YYnq6/Jo5hE4GwS9JVtv/a9hxJ\nd0p6qYE+Tsv2edWFHdk+T9KPJe0rf1cjXpK0ulpeLWlHg718z6n/aJXb1eAxtG1Jz0gaiYhN40qt\nOIYT9dfEMez7pwySVH188i+SZknaGhE/73sTE7B9ucbOCiRptqRfN92f7eck3SDpAklHJT0m6d8l\nPS/pEkkHJa2MiEYu7E3Q3w0aO9UNSQck3TtuvN7v/pZL+pOkDyV9W61+RGPj9MaPYaG/VerzMWwk\nEAC0ExcVASQCAUAiEAAkAgFAIhAApEYDocXTgiXRX7fa3F+be5Oa66/pM4RW/6OI/rrV5v7a3JvU\nUH9NBwKAFulqYpLtmyVt1tiMw3+NiPU12zMLCmhIRLhumykHwlRudEIgAM3pJBC6GTJwoxPgDNNN\nIMyEG50AmITZ072D6uOTtl/RBaDuAqGjG51ExBZJWySuIQBt182QodU3OgEweVM+Q4iIk7bvk/R7\n/eVGJx/1rDMAfdfXG6QwZACaM90fOwI4wxAIABKBACARCAASgQAgEQgAEoEAIBEIABKBACARCAAS\ngQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEIABKB\nACARCAASgQAgEQgAEoEAIM3u5pttH5D0paRvJJ2MiKFeNAWgGV0FQuXvIuJ4D94HQMMYMgBI3QZC\nSPqD7fdsD/eiIQDN6XbIsDwiDtu+UNJrtj+JiDfHb1AFBWEBzACOiN68kb1O0lcRsaGwTW92BmDS\nIsJ120x5yGD7PNvnn1qW9GNJ+6b6fgCa182QYUDSb22fep9fR8SrPekKQCN6NmToaGcMGYDGTOuQ\nAcCZh0AAkAgEAIlAAJAIBACJQACQevHbjmeNO++8s1i///77i/U77rijWD9y5MikewJ6iTMEAIlA\nAJAIBACJQACQCAQAiUAAkAgEAIlff56E999/v1i/+OKLi/WBgYFetgNMCr/+DGBSCAQAiUAAkAgE\nAIlAAJAIBACJQACQuB/COEuXLu2q/vTTT/eyHfTZ4sWLi/WTJ08W62fC/Sw4QwCQCAQAiUAAkAgE\nAIlAAJAIBACJQACQzqp5CPPmzSvW6+YRzJkzp1jfuXPnpHtC7yxcuLBYX79+fbG+atWqYv3EiRPF\n+qJFi4r1maD2DMH2VtvHbO8bt26h7ddsf1p9XTC9bQLoh06GDNsk3fyddQ9Lej0irpL0evUawAxX\nGwgR8aakz7+z+jZJ26vl7ZJW9LgvAA2Y6kXFgYgYrZaPSOJmgcAZoOuLihERpZun2h6WNNztfgBM\nv6meIRy1PShJ1ddjE20YEVsiYigihqa4LwB9MtVAeEnS6mp5taQdvWkHQJNqhwy2n5N0g6QLbB+S\n9Jik9ZKet/0zSQclrZzOJjs1a9asYn3Pnj3F+uWXX16s1z3D4pVXXinWt23bVqxv2LChWD948GCx\n/vXXXxfrM13dPJBHH320WL/nnnu62v+mTZu6+v6ZoDYQImKi2Ro39rgXAA1j6jKARCAASAQCgEQg\nAEgEAoBEIABIrvtsvac7K0xx7oU1a9YU608++WSxvnnz5mK97n4Hd911V7F+9913F+t1jh2bcEKo\nJOntt98u1nfsKM8fGxkZKdYvueSSYv3AgQPF+uzZ5U+5h4bKk1nXrl1brNfNI6mzb9++Yv2aa67p\n6v2bFhGu24YzBACJQACQCAQAiUAAkAgEAIlAAJAIBADpjJqHUPc5eN1zGa6++upi/fjx48W6Xf6Y\nt+79H3rooWJ98eLFxfqNN5Z/I71uHkDdz0Ld36/bn6UvvviiWH/11VeL9bfeeqtY37hxY7F+3XXX\nFet79+4t1tuOeQgAJoVAAJAIBACJQACQCAQAiUAAkAgEAOmMmofwxBNPFOt1z2V44YUXetlO382f\nP79Ynzt3brH+4IMPFuv79++fdE/jHTp0qFh/4403ivW6eRzvvvtusf7yyy8X6ytWnNnPLGYeAoBJ\nIRAAJAIBQCIQACQCAUAiEAAkAgFAOqPmIZztnn322WK97n4JF154YbF++PDhSffUS3PmzCnWL7ro\nomJ9165dxXrd/RBmup7MQ7C91fYx2/vGrVtn+7DtD6o/P+m2WQDN62TIsE3SzadZ/88Rsaz687ve\ntgWgCbWBEBFvSvq8D70AaFg3FxXvs723GlIs6FlHABoz1UB4StIVkpZJGpU04d0rbQ/b3m179xT3\nBaBPphQIEXE0Ir6JiG8l/ULShJdnI2JLRAxFRPnRvgAaN6VAsD047uXtksrP0QYwI9TOQ7D9nKQb\nJF0g6aikx6rXyySFpAOS7o2I0dqdMQ9hWt16663F+uDgYLF+6aWX9rKd71m+fHmxXne/g4GBgWL9\ns88+K9ZvuummYv2TTz4p1me6TuYhlJ/cMfYmq06z+pkpdQSg1Zi6DCARCAASgQAgEQgAEoEAIBEI\nABL3Q0Br3HLLLcX68PBwsf74448X67t3n92z53kuA4BJIRAAJAIBQCIQACQCAUAiEAAkAgFAqv31\nZ6BfrrzyymL93HPPLdbP9nkGvcAZAoBEIABIBAKARCAASAQCgEQgAEgEAoDEPAS0xvz584v1efPm\n9amTsxdnCAASgQAgEQgAEoEAIBEIABKBACARCAAS8xDQGkuWLGm6hbNe7RmC7SW2/2j7Y9sf2V5b\nrV9o+zXbn1ZfF0x/uwCmUydDhpOSHoyIpZL+RtIa20slPSzp9Yi4StLr1WsAM1htIETEaETsqZa/\nlDQiaZGk2yRtrzbbLmnFdDUJoD8mdVHR9mWSrpX0jqSBiBitSkckDfS0MwB91/FFRdtzJb0o6YGI\nOGH/5bmRERETPcjV9rCk8lM6AbRCR2cIts/RWBj8KiJ+U60+anuwqg9KOna6742ILRExFBFDvWgY\nwPTp5FMGS3pG0khEbBpXeknS6mp5taQdvW8PQD91MmT4kaSfSvrQ9gfVukckrZf0vO2fSTooaeX0\ntAigX2oDISL+Q5InKN/Y23YANImpywASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEI\nABKBACARCAASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIHX8KDdgum3btq1Yv/766/vT\nyFmMMwQAiUAAkAgEAIlAAJAIBACJQACQCAQAyRFR3sBeIumXkgYkhaQtEbHZ9jpJ90j6n2rTRyLi\ndzXvVd4ZgGkTEa7bppNAGJQ0GBF7bJ8v6T1JKyStlPRVRGzotCECAWhOJ4FQO1MxIkYljVbLX9oe\nkbSo+/YAtM2kriHYvkzStZLeqVbdZ3uv7a22F/S4NwB91nEg2J4r6UVJD0TECUlPSbpC0jKNnUFs\nnOD7hm3vtr27B/0CmEa11xAkyfY5knZK+n1EbDpN/TJJOyPiBzXvwzUEoCGdXEOoPUOwbUnPSBoZ\nHwbVxcZTbpe0bypNAmiPTj5lWC7pT5I+lPRttfoRSas0NlwISQck3VtdgCy9F2cIQEN68rFjLxEI\nQHN6MmQAcPYgEAAkAgFAIhAAJAIBQCIQACQCAUAiEAAkAgFAIhAAJAIBQCIQACQCAUAiEAAkAgFA\nqr3rco8dl3Rw3OsLqnVtRX/daXN/be5N6n1/l3ayUV9vkPK9ndu7I2KosQZq0F932txfm3uTmuuP\nIQOARCAASE0HwpaG91+H/rrT5v7a3JvUUH+NXkMA0C5NnyEAaBECAUAiEAAkAgFAIhAApP8DA4Pa\njpzM5L4AAAAASUVORK5CYII=\n", 1144 | "text/plain": [ 1145 | "" 1146 | ] 1147 | }, 1148 | "metadata": {}, 1149 | "output_type": "display_data" 1150 | }, 1151 | { 1152 | "name": "stdout", 1153 | "output_type": "stream", 1154 | "text": [ 1155 | "true: 6.0 prediction: 0\n" 1156 | ] 1157 | }, 1158 | { 1159 | "data": { 1160 | "text/plain": [ 1161 | "" 1162 | ] 1163 | }, 1164 | "metadata": {}, 1165 | "output_type": "display_data" 1166 | }, 1167 | { 1168 | "data": { 1169 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADm1JREFUeJzt3V+MFWWexvHnEVEMEmkxi+g2sqgXjBOFtWOMko2r7oSV\nCzUaXS8MGhOMGRNNNnGNF2pMTCYrurtXhh4lsoZR+TP+CawOhJj458IMoEGUVQfErG0LCgZaBRf1\ntxddvPZi91vdXadPVeP3k3T6nPqdU/WjunlSVefttxwRAgBJOq7uBgA0B4EAICEQACQEAoCEQACQ\nEAgAkloCwfYC2x/Y/ovte+voIcf2Ltvv2n7H9qYG9LPM9h7b2wYsO9X2BtsfFd87Gtbfg7Z7in34\nju2rauyv0/artt+3/Z7tu4rljdiHmf7avg/d7nEItidI+lDSP0j6VNKfJd0UEe+3tZEM27skdUXE\nl3X3Ikm2/07S15L+MyJ+XSz7V0n7IuJ3Rah2RMS/NKi/ByV9HRFL6uhpINszJM2IiC22p0jaLOka\nSbeoAfsw098NavM+rOMI4SJJf4mInRHxv5KelXR1DX2MGxHxmqR9Ry2+WtLy4vFy9f8C1WKI/hoj\nInojYkvxuE/SdklnqiH7MNNf29URCGdK+p8Bzz9VTf/4jJC03vZm24vrbmYI0yOit3j8uaTpdTYz\nhDttby1OKWo7pRnI9ixJ8yS9pQbuw6P6k9q8D7moOLj5EfG3kv5R0m+LQ+LGiv7zvqaNQX9c0tmS\n5krqlfRove1Itk+WtEbS3RFxYGCtCftwkP7avg/rCIQeSZ0Dnv91sawxIqKn+L5H0vPqP81pmt3F\nueeRc9A9Nffz/0TE7oj4ISJ+lPR71bwPbU9U/3+2FRHxx2JxY/bhYP3VsQ/rCIQ/SzrX9t/YPkHS\nP0l6qYY+BmV7cnFhR7YnS/qNpG35d9XiJUmLiseLJL1YYy8/c+Q/WuFa1bgPbVvSk5K2R8RjA0qN\n2IdD9VfHPmz7pwySVHx88u+SJkhaFhEPt72JIdierf6jAkk6XtIf6u7P9jOSLpN0mqTdkh6Q9IKk\nlZJmSvpE0g0RUcuFvSH6u0z9h7ohaZek2wecr7e7v/mSXpf0rqQfi8X3qf88vfZ9mOnvJrV5H9YS\nCACaiYuKABICAUBCIABICAQACYEAIKk1EBo8LFgS/VXV5P6a3JtUX391HyE0+oci+quqyf01uTep\npv7qDgQADVJpYJLtBZL+Q/0jDp+IiN+VvJ5RUEBNIsJlrxl1IIxmohMCAajPcAKhyikDE50Ax5gq\ngTAeJjoBMALHj/UGio9Pmn5FF4CqBcKwJjqJiG5J3RLXEICmq3LK0OiJTgCM3KiPECLie9t3SvqT\nfpro5L2WdQag7do6QQqnDEB9xvpjRwDHGAIBQEIgAEgIBAAJgQAgIRAAJAQCgIRAAJAQCAASAgFA\nQiAASAgEAAmBACAhEAAkYz6FGn5y3XXXZeurVq3K1pcuXZqt33HHHSPuCRiIIwQACYEAICEQACQE\nAoCEQACQEAgAEgIBQMI07G00a9asbH3Hjh3Z+uHDh7P1zs7ObP2LL77I1nFsYxp2ACNCIABICAQA\nCYEAICEQACQEAoCEQACQMB9CG11xxRWV3n/CCSdk68cdR76jmkqBYHuXpD5JP0j6PiK6WtEUgHq0\n4gjh7yPiyxasB0DNOMYEkFQNhJC03vZm24tb0RCA+lQ9ZZgfET22/0rSBtv/HRGvDXxBERSEBTAO\nVDpCiIie4vseSc9LumiQ13RHRBcXHIHmG3Ug2J5se8qRx5J+I2lbqxoD0H5VThmmS3re9pH1/CEi\nXmlJV8eojo6OSu/v6enJ1g8dOlRp/cCoAyEidkq6oIW9AKgZHzsCSAgEAAmBACAhEAAkBAKAhEAA\nkDAfQhtdcskl2XoxpmNIGzduzNb3798/4p6AgThCAJAQCAASAgFAQiAASAgEAAmBACAhEAAkjENo\nofPOOy9bX7hwYbYeEdn66tWrR9zTQBMnTszWy/q79NJLs/Wy/tetW5etb9myJVvv6+vL1lEdRwgA\nEgIBQEIgAEgIBAAJgQAgIRAAJAQCgMRlnx23dGN2+zZWgwsuyM9KX/Y5e5kJEyZk6zNnzszWV6xY\nka1Xna+h6u9S2XwPy5cvz9ZfeSV/W5C9e/eOuKdjSUTkf4DiCAHAAAQCgIRAAJAQCAASAgFAQiAA\nSAgEAAnzIRxDXnjhhWy9bJzEzp07s/UlS5Zk61OnTs3W58yZk63Pnj07W3/66aez9QMHDmTry5Yt\ny9YfeeSRbL23tzdbPxaUHiHYXmZ7j+1tA5adanuD7Y+K7x1j2yaAdhjOKcNTkhYctexeSRsj4lxJ\nG4vnAMa50kCIiNck7Ttq8dWSjowjXS7pmhb3BaAGo72oOD0ijpxQfS5peov6AVCjyhcVIyJyf7Rk\ne7GkxVW3A2DsjfYIYbftGZJUfN8z1AsjojsiuiKia5TbAtAmow2ElyQtKh4vkvRia9oBUKfS+RBs\nPyPpMkmnSdot6QFJL0haKWmmpE8k3RARR194HGxdv+j5EN5+++1K63/ooYey9fvvvz9b/+6777L1\nsvsuVJ3PoaqVK1dm69dff32l9ZeNw7jxxhuz9c2bN1fa/lgbznwIpdcQIuKmIUpXjLgjAI3G0GUA\nCYEAICEQACQEAoCEQACQEAgAEu7L0EK33nprtv7EE09UWv+hQ4ey9UmTJmXrt912W7b+1FNPjbSl\ntjrxxBOz9csvvzxbLxtHcPPNN2frhw8fztZvueWWbP3ZZ5/N1sca92UAMCIEAoCEQACQEAgAEgIB\nQEIgAEgIBAAJ92UYR0466aRs/YMPPsjWn3vuuVa203Zl8zm8/PLL2fobb7yRrU+bNi1bX7hwYbZ+\nzz33ZOt1j0MYDo4QACQEAoCEQACQEAgAEgIBQEIgAEgIBAAJ4xBaqO77FqxduzZbP3jwYJs6aaa+\nvr5sffHi/B0He3p6svWOjo5K9a+++ipbbweOEAAkBAKAhEAAkBAIABICAUBCIABICAQACeMQ2sgu\nnRa/kiVLlozp+o91vb29ld5/1llnZeudnZ3Z+rgYh2B7me09trcNWPag7R7b7xRfV41tmwDaYTin\nDE9JWjDI8n+LiLnF13+1ti0AdSgNhIh4TdK+NvQCoGZVLireaXtrcUqRH6QNYFwYbSA8LulsSXMl\n9Up6dKgX2l5se5PtTaPcFoA2GVUgRMTuiPghIn6U9HtJF2Ve2x0RXRHRNdomAbTHqALB9owBT6+V\ntG2o1wIYP0rHIdh+RtJlkk6z/amkByRdZnuupJC0S9LtY9jjMSMiKr1/rMcx/NJdeeWVld7/zTff\nZOuHDh2qtP52KA2EiLhpkMVPjkEvAGrG0GUACYEAICEQACQEAoCEQACQEAgAEuZDaKEPP/wwW9+6\ndWu2fv7557eyHRzlnHPOydZXr15daf1l93Uo+/1oAo4QACQEAoCEQACQEAgAEgIBQEIgAEgIBAAJ\n4xBa6ODBg9n6ihUrsnXGIVRTNp9B2TiDKVOmZOurVq3K1tesWZOtjwccIQBICAQACYEAICEQACQE\nAoCEQACQEAgAEle9V8CINma3b2MNdPrpp2frn332WbZedl+GHTt2ZOvz5s3L1hcsGOwm3z958803\ns/Wy/succcYZ2fqiRYuy9YcffrjS9nt6erL1zs7OSuuvW0SU3tiDIwQACYEAICEQACQEAoCEQACQ\nEAgAEgIBQMI4hDaaPHlytr5hw4Zs/eKLL87Wy36We/fuzdbL5gP4+OOPs/W1a9dm63PmzMnW58+f\nn62fcsop2XrZv7/svghl+3f//v3ZetO1ZByC7U7br9p+3/Z7tu8qlp9qe4Ptj4rvHa1oGkB9hnPK\n8L2kf46IX0m6WNJvbf9K0r2SNkbEuZI2Fs8BjGOlgRARvRGxpXjcJ2m7pDMlXS1pefGy5ZKuGasm\nAbTHiC4q2p4laZ6ktyRNj4jeovS5pOkt7QxA2w17klXbJ0taI+nuiDgw8A9tIiKGumBoe7Gk/F0w\nATTCsI4QbE9UfxisiIg/Fot3255R1GdI2jPYeyOiOyK6IqKrFQ0DGDvD+ZTBkp6UtD0iHhtQeknS\nkb9HXSTpxda3B6CdSsch2J4v6XVJ70r6sVh8n/qvI6yUNFPSJ5JuiIh9Jev6RY9DKHPhhRdm6+vX\nr8/Wp06d2sp2fqZsPoaqY1q+/fbbbL2vry9b7+7uztaXLl2arff29mbr491wxiGUXkOIiDckDbWi\nK0baFIDmYugygIRAAJAQCAASAgFAQiAASAgEAAnzIYwj06ZNy9bL7kswadKkbH327NnZetl8CGW/\nS9u3b8/W161bl61v27YtW0ce92UAMCIEAoCEQACQEAgAEgIBQEIgAEgIBAAJ4xCAXwjGIQAYEQIB\nQEIgAEgIBAAJgQAgIRAAJAQCgIRAAJAQCAASAgFAQiAASAgEAAmBACAhEAAkBAKApDQQbHfaftX2\n+7bfs31XsfxB2z223ym+rhr7dgGMpdIJUmzPkDQjIrbYniJps6RrJN0g6euIWDLsjTFBClCb4UyQ\ncvwwVtIrqbd43Gd7u6Qzq7cHoGlGdA3B9ixJ8yS9VSy60/ZW28tsd7S4NwBtNuxAsH2ypDWS7o6I\nA5Iel3S2pLnqP4J4dIj3Lba9yfamFvQLYAwNa5JV2xMlrZX0p4h4bJD6LElrI+LXJevhGgJQk5ZM\nsmrbkp6UtH1gGBQXG4+4VhK35gXGueF8yjBf0uuS3pX0Y7H4Pkk3qf90ISTtknR7cQEyty6OEICa\nDOcIgfsyAL8Q3JcBwIgQCAASAgFAQiAASAgEAAmBACAhEAAkBAKAhEAAkBAIABICAUBCIABICAQA\nCYEAICEQACSlsy632JeSPhnw/LRiWVPRXzVN7q/JvUmt7++s4byorROk/Gzj9qaI6KqtgRL0V02T\n+2tyb1J9/XHKACAhEAAkdQdCd83bL0N/1TS5vyb3JtXUX63XEAA0S91HCAAahEAAkBAIABICAUBC\nIABI/g+xM1mfMV0J2QAAAABJRU5ErkJggg==\n", 1170 | "text/plain": [ 1171 | "" 1172 | ] 1173 | }, 1174 | "metadata": {}, 1175 | "output_type": "display_data" 1176 | }, 1177 | { 1178 | "name": "stdout", 1179 | "output_type": "stream", 1180 | "text": [ 1181 | "true: 3.0 prediction: 7\n" 1182 | ] 1183 | }, 1184 | { 1185 | "data": { 1186 | "text/plain": [ 1187 | "" 1188 | ] 1189 | }, 1190 | "metadata": {}, 1191 | "output_type": "display_data" 1192 | }, 1193 | { 1194 | "data": { 1195 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADVRJREFUeJzt3X+o1XWex/HXa7UI1D8MWTEr3Q0LpoW14RJLhhjDDuU/\n/YAu6x+LG5JGU0xhkPSHSbIgi+UORZHtlXHQaSnKqT9sV4uhpohQQ9RyN6fJmuymK/5h9Udb+t4/\n7tf33HXu/Zxz7zn3fL83nw+Qe873/T3n++6bvvh8vudzv8cRIQCQpL+ouwEAzUEgAEgEAoBEIABI\nBAKARCAASLUEgu2bbf+37d/bXlNHDyW2j9o+aHu/7b0N6GeL7RO2Dw3bdqnt3baPVD9nNqy/dbaP\nVedwv+2lNfZ3he3f2v7Q9ge2f15tb8Q5LPTX83PoXq9DsD1F0keS/l7S55L2SFoWER/2tJEC20cl\n9UXEybp7kSTbiyV9LelXEfE31bZ/kXQqIjZUoTozIh5uUH/rJH0dERvr6Gk423MkzYmI923PkLRP\n0m2S/kkNOIeF/vrV43NYxwjhekm/j4g/RMT/Svp3SbfW0MekERFvSTp13uZbJW2tHm/V0F+gWozS\nX2NExGBEvF89/krSYUlz1ZBzWOiv5+oIhLmS/jjs+eeq6T++ICTtsr3P9sq6mxnF7IgYrB5/KWl2\nnc2M4j7bB6opRW1TmuFsz5d0naT31MBzeF5/Uo/PIRcVR3ZjRPxY0i2SflYNiRsrhuZ9TVuD/oyk\nqyQtlDQo6fF625FsT5f0kqQHIuL08FoTzuEI/fX8HNYRCMckXTHs+eXVtsaIiGPVzxOSdmhomtM0\nx6u557k56Ima+/l/IuJ4RJyJiLOSnlPN59D2RRr6x7Y9Il6uNjfmHI7UXx3nsI5A2CNpge2/sn2x\npH+Q9GoNfYzI9rTqwo5sT5P0U0mHyq+qxauSllePl0t6pcZe/sy5f2iV21XjObRtSQOSDkfEE8NK\njTiHo/VXxzns+acMklR9fPKvkqZI2hIR/9zzJkZh+681NCqQpKmSfl13f7afl7RE0ixJxyU9Kuk3\nkl6QdKWkTyX1R0QtF/ZG6W+Jhoa6IemopFXD5uu97u9GSb+TdFDS2WrzIxqap9d+Dgv9LVOPz2Et\ngQCgmbioCCARCAASgQAgEQgAEoEAINUaCA1eFiyJ/jrV5P6a3JtUX391jxAa/T9F9NepJvfX5N6k\nmvqrOxAANEhHC5Ns3yzpFxpacfhvEbGhxf6sggJqEhFutc+4A2E8NzohEID6tBMInUwZuNEJ8APT\nSSBMhhudABiDqRN9gOrjk6Zf0QWgzgKhrRudRMRmSZslriEATdfJlKHRNzoBMHbjHiFExPe275P0\nn/rTjU4+6FpnAHqupzdIYcoA1GeiP3YE8ANDIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKARCAA\nSAQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABI\nBAKARCAASAQCgEQgAEgEAoA0tZMX2z4q6StJZyR9HxF93WgKQD06CoTKTRFxsgvvA6BmTBkApE4D\nISTtsr3P9spuNASgPp1OGW6MiGO2/1LSbtv/FRFvDd+hCgrCApgEHBHdeSN7naSvI2JjYZ/uHAzA\nmEWEW+0z7imD7Wm2Z5x7LOmnkg6N9/0A1K+TKcNsSTtsn3ufX0fEf3Slq0mqr6/8qeuePXuK9Vaj\ntQMHDhTrBw8eLNY/+eSTYv3dd98t1l9//fVi/bvvvivW0XzjDoSI+IOkv+1iLwBqxseOABKBACAR\nCAASgQAgEQgAEoEAIHVtpWJbB5vkKxVnzpxZrA8ODhbrn332WbH+4osvFus33XRTsX7ttdcW69On\nTy/WqzUlozpy5Eixvn79+mJ927ZtxTom1oSuVATww0MgAEgEAoBEIABIBAKARCAASAQCgMQ6hDGY\nMmVKsb5jx45i/Z577inWv/jiizH3NNzll19erF9yySXF+kMPPVSs33nnncX6tGnTivVdu3YV62vX\nri3W9+/fX6yjjHUIAMaEQACQCAQAiUAAkAgEAIlAAJAIBACJdQhd1Op7Ga655ppiffv27d1sp+vm\nzZtXrN9xxx3F+oYNG4r1VusMFi9eXKx/++23xfqFjnUIAMaEQACQCAQAiUAAkAgEAIlAAJAIBACJ\ndQg9dNlllxXrnd4PoelWrVpVrD/99NPF+pYtW4r1u+++e8w9XUi6sg7B9hbbJ2wfGrbtUtu7bR+p\nfpa/wQTApNDOlOGXkm4+b9saSW9ExAJJb1TPAUxyLQMhIt6SdOq8zbdK2lo93irpti73BaAG472o\nODsizn2R4ZeSZnepHwA1mtrpG0RElC4W2l4paWWnxwEw8cY7Qjhue44kVT9PjLZjRGyOiL6IKP8q\nIIDajTcQXpW0vHq8XNIr3WkHQJ1arkOw/bykJZJmSTou6VFJv5H0gqQrJX0qqT8izr/wONJ7XdDr\nEFC2c+fOjl6/dOnSLnXyw9TOOoSW1xAiYtkopZ+MuSMAjcbSZQCJQACQCAQAiUAAkAgEAIlAAJA6\nXroMtGvq1PJftxkzZhTrp0+f7mY7GAEjBACJQACQCAQAiUAAkAgEAIlAAJAIBACJdQjomXvvvbdY\nv+GGG4r1tWvXdrMdjIARAoBEIABIBAKARCAASAQCgEQgAEgEAoDU8nsZunowvpdhUps2bVqx/thj\njxXrK1asKNafffbZYv3hhx8u1lHWzvcyMEIAkAgEAIlAAJAIBACJQACQCAQAiUAAkLgfAtKiRYuK\n9YGBgWJ9wYIFxfrGjRuLddYZ1K/lCMH2FtsnbB8atm2d7WO291d/lk5smwB6oZ0pwy8l3TzC9k0R\nsbD6s7O7bQGoQ8tAiIi3JJ3qQS8AatbJRcX7bB+ophQzu9YRgNqMNxCekXSVpIWSBiU9PtqOtlfa\n3mt77ziPBaBHxhUIEXE8Is5ExFlJz0m6vrDv5ojoi4i+8TYJoDfGFQi25wx7erukQ6PtC2DyaLkO\nwfbzkpZImmX7c0mPSlpie6GkkHRU0qoJ7BFdMnfu3GL97bffLtbffPPNYr2/v79YP3DgQLGO+rUM\nhIhYNsLm8goVAJMSS5cBJAIBQCIQACQCAUAiEAAkAgFA4n4IF5CnnnqqWG/1HR1r1qwp1llnMPkx\nQgCQCAQAiUAAkAgEAIlAAJAIBACJQACQWIdwAbn44os7ev1HH33UpU7QVIwQACQCAUAiEAAkAgFA\nIhAAJAIBQCIQACTWIVxAWt2v4JZbbinWH3zwwWJ93bp1xfqZM2eKddSPEQKARCAASAQCgEQgAEgE\nAoBEIABIBAKA5Fb34u/qwezeHQxj9vHHHxfr8+fPL9Zbfe/D+vXri/WTJ08W6+hMRLjVPi1HCLav\nsP1b2x/a/sD2z6vtl9rebftI9XNmN5oGUJ92pgzfS1odET+S9HeSfmb7R5LWSHojIhZIeqN6DmAS\naxkIETEYEe9Xj7+SdFjSXEm3Stpa7bZV0m0T1SSA3hjTRUXb8yVdJ+k9SbMjYrAqfSlpdlc7A9Bz\nbf9yk+3pkl6S9EBEnLb/dH0iImK0C4a2V0pa2WmjACZeWyME2xdpKAy2R8TL1ebjtudU9TmSToz0\n2ojYHBF9EdHXjYYBTJx2PmWwpAFJhyPiiWGlVyUtrx4vl/RK99sD0Est1yHYvlHS7yQdlHS22vyI\nhq4jvCDpSkmfSuqPiFMt3ot1CA22aNGiYn1gYKBYv/rqq4v11atXF+ubNm0q1tGZdtYhtLyGEBFv\nSxrtjX4y1qYANBdLlwEkAgFAIhAAJAIBQCIQACQCAUDiexmQ3nnnnWJ98eLFxfq+ffuK9Xnz5o25\nJ/QWIwQAiUAAkAgEAIlAAJAIBACJQACQCAQAiXUIaNv9999frM+aNatY37t3bzfbwQRghAAgEQgA\nEoEAIBEIABKBACARCAASgQAgsQ4BadWqVcX6ihUrivUnn3yyWN+2bduYe0JvMUIAkAgEAIlAAJAI\nBACJQACQCAQAiUAAkBwR5R3sKyT9StJsSSFpc0T8wvY6SXdL+p9q10ciYmeL9yofDBPqrrvuKtYH\nBgaK9ddee61Y7+/vL9a/+eabYh0TKyLcap92FiZ9L2l1RLxve4akfbZ3V7VNEbGxkyYBNEfLQIiI\nQUmD1eOvbB+WNHeiGwPQe2O6hmB7vqTrJL1XbbrP9gHbW2zP7HJvAHqs7UCwPV3SS5IeiIjTkp6R\ndJWkhRoaQTw+yutW2t5rmxvqAQ3XViDYvkhDYbA9Il6WpIg4HhFnIuKspOckXT/SayNic0T0RURf\nt5oGMDFaBoJtSxqQdDginhi2fc6w3W6XdKj77QHopXY+ZVgk6R8lHbS9v9r2iKRlthdq6KPIo5LK\nvzsLoPFarkPo6sFYhwDUpp11CKxUBJAIBACJQACQCAQAiUAAkAgEAIlAAJAIBACJQACQCAQAiUAA\nkAgEAIlAAJAIBACJQACQ2rlBSjedlPTpsOezqm1NRX+daXJ/Te5N6n5/89rZqac3SPmzg9t7m3yv\nRfrrTJP7a3JvUn39MWUAkAgEAKnuQNhc8/Fbob/ONLm/Jvcm1dRfrdcQADRL3SMEAA1CIABIBAKA\nRCAASAQCgPR/TtnkiaARutMAAAAASUVORK5CYII=\n", 1196 | "text/plain": [ 1197 | "" 1198 | ] 1199 | }, 1200 | "metadata": {}, 1201 | "output_type": "display_data" 1202 | } 1203 | ], 1204 | "source": [ 1205 | "problem_display()" 1206 | ] 1207 | }, 1208 | { 1209 | "cell_type": "markdown", 1210 | "metadata": {}, 1211 | "source": [ 1212 | "I would rather agree with my network sometimes :)" 1213 | ] 1214 | }, 1215 | { 1216 | "cell_type": "markdown", 1217 | "metadata": {}, 1218 | "source": [ 1219 | "## Low confidence predictions\n", 1220 | "Let's see where model estimates the distribution over classes worst" 1221 | ] 1222 | }, 1223 | { 1224 | "cell_type": "code", 1225 | "execution_count": 57, 1226 | "metadata": { 1227 | "collapsed": true 1228 | }, 1229 | "outputs": [], 1230 | "source": [ 1231 | "top_three = np.argsort(prediction_expected_error_rate)[-3:][::-1]" 1232 | ] 1233 | }, 1234 | { 1235 | "cell_type": "code", 1236 | "execution_count": 58, 1237 | "metadata": {}, 1238 | "outputs": [ 1239 | { 1240 | "data": { 1241 | "text/plain": [ 1242 | "array([9182, 9254, 7991])" 1243 | ] 1244 | }, 1245 | "execution_count": 58, 1246 | "metadata": {}, 1247 | "output_type": "execute_result" 1248 | } 1249 | ], 1250 | "source": [ 1251 | "top_three" 1252 | ] 1253 | }, 1254 | { 1255 | "cell_type": "code", 1256 | "execution_count": 59, 1257 | "metadata": { 1258 | "collapsed": true 1259 | }, 1260 | "outputs": [], 1261 | "source": [ 1262 | "def low_confidence_examples_display():\n", 1263 | " for i in top_three:\n", 1264 | " print('true:', y_test[i],\n", 1265 | " 'prediction:', y_pred_posterior[i],\n", 1266 | " 'expected error rate:', prediction_expected_error_rate[i]\n", 1267 | " )\n", 1268 | " plt.gray();plt.matshow(X_test[i][0]);plt.show();" 1269 | ] 1270 | }, 1271 | { 1272 | "cell_type": "code", 1273 | "execution_count": 60, 1274 | "metadata": {}, 1275 | "outputs": [ 1276 | { 1277 | "name": "stdout", 1278 | "output_type": "stream", 1279 | "text": [ 1280 | "true: 5.0 prediction: 7 expected error rate: 0.78\n" 1281 | ] 1282 | }, 1283 | { 1284 | "data": { 1285 | "text/plain": [ 1286 | "" 1287 | ] 1288 | }, 1289 | "metadata": {}, 1290 | "output_type": "display_data" 1291 | }, 1292 | { 1293 | "data": { 1294 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADJNJREFUeJzt3V+MFXWaxvHnWejFRDRizJDWhYUlZAyZC5zgaJRs2Gx2\n4qoJmhiyeANXGLMmcjfGG7mZxGzU3bkiMiuBTRZXE53RkM06ZmTVeEEG0GgDO+u4aR1ICyFcAMY/\nAd696OKdHqa7zumuOudXwPeTdLq63uqq1xKe/H5VRR1HhABAkv6sdAMAuoNAAJAIBACJQACQCAQA\niUAAkIoEgu37bP/W9u9sP1Wihzq2x21/Yvsj2wc60M9O2ydtj01Zd7Ptt21/Wn1f1LH+ttk+Xp3D\nj2zfX7C/Jbb32T5i+7DtJ6v1nTiHNf0N/Rx62M8h2J4n6X8l/Z2kY5J+I2ljRBwZaiM1bI9LWhMR\np0r3Ikm2/1rSOUn/FhE/qNb9k6TTEfFsFaqLIuInHepvm6RzEfFciZ6msj0qaTQiDtm+QdJBSQ9J\n2qwOnMOa/jZoyOewxAjhR5J+FxH/FxHfSfoPSesL9HHFiIj3JJ2+bPV6Sbur5d2a/ANUxAz9dUZE\nTETEoWr5rKSjkm5TR85hTX9DVyIQbpP0+yk/H1Oh//gaIelXtg/a3lK6mRksjoiJavlLSYtLNjOD\nJ2x/XE0pik1pprK9TNIdkvarg+fwsv6kIZ9DLipOb21E/FDS30v6x2pI3FkxOe/r2jPo2yWtkLRa\n0oSk58u2I9leKOk1SVsj4szUWhfO4TT9Df0clgiE45KWTPn5L6p1nRERx6vvJyX9QpPTnK45Uc09\nL81BTxbu549ExImIuBARFyX9XIXPoe0RTf5l+/eIeL1a3ZlzOF1/Jc5hiUD4jaSVtpfb/nNJ/yDp\nzQJ9TMv29dWFHdm+XtKPJY3V/1YRb0raVC1vkvRGwV7+xKW/aJWHVfAc2raklyQdjYgXppQ6cQ5n\n6q/EORz6XQZJqm6f/IukeZJ2RsRPh97EDGz/lSZHBZI0X9Ke0v3ZflnSOkm3SDoh6RlJv5T0qqSl\nkj6XtCEiilzYm6G/dZoc6oakcUmPTZmvD7u/tZLel/SJpIvV6qc1OU8vfg5r+tuoIZ/DIoEAoJu4\nqAggEQgAEoEAIBEIABKBACAVDYQOPxYsif6a6nJ/Xe5NKtdf6RFCp/+niP6a6nJ/Xe5NKtRf6UAA\n0CGNHkyyfZ+kn2nyicN/jYhne2zPU1BAIRHhXtvMORDm8qITAgEop59AaDJl4EUnwFWmSSBcCS86\nATAL8wd9gOr2Sdev6AJQs0Do60UnEbFD0g6JawhA1zWZMnT6RScAZm/OI4SIOG/7CUlv6Q8vOjnc\nWmcAhm6oL0hhygCUM+jbjgCuMgQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKA\nRCAASAQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBE\nIABIBAKARCAASPOb/LLtcUlnJV2QdD4i1rTRFIAyGgVC5W8i4lQL+wFQGFMGAKlpIISkX9k+aHtL\nGw0BKKfplGFtRBy3/T1Jb9v+n4h4b+oGVVAQFsAVwBHRzo7sbZLORcRzNdu0czAAsxYR7rXNnKcM\ntq+3fcOlZUk/ljQ21/0BKK/JlGGxpF/YvrSfPRHxX610BaCI1qYMfR2MKQNQzECnDACuPgQCgEQg\nAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKARCAASAQCgNTGW5eBVixbtqy2vnz58tr6\ngw8+2Oj4+/fvr62PjdW//+fIkSONjt8FjBAAJAIBQCIQACQCAUAiEAAkAgFAIhAAJJ5DuIYsWLCg\ntv7tt9/W1kdGRmrrq1atqq3v2rWrtr506dLa+k033VRbH7Tvvvuutr5169ba+osvvthmOwPBCAFA\nIhAAJAIBQCIQACQCAUAiEAAkAgFA4uPgryK33357bX3Pnj219Xfffbe2ftdddzWqNzU+Pl5b7/U+\nhUHr1d+KFSuG08gMWvk4eNs7bZ+0PTZl3c2237b9afV9UdNmAZTXz5Rhl6T7Llv3lKRfR8RKSb+u\nfgZwhesZCBHxnqTTl61eL2l3tbxb0kMt9wWggLleVFwcERPV8peSFrfUD4CCGv/jpoiIuouFtrdI\n2tL0OAAGb64jhBO2RyWp+n5ypg0jYkdErImINXM8FoAhmWsgvClpU7W8SdIb7bQDoKSezyHYflnS\nOkm3SDoh6RlJv5T0qqSlkj6XtCEiLr/wON2+eA5hgO6+++7a+gcffNBo/wsXLmz0+01duHChtj5v\n3rwhdTK9Xv31ep/CoPXzHELPawgRsXGG0t/OuiMAncajywASgQAgEQgAEoEAIBEIABKBACDxuQzo\n29dff126BQwYIwQAiUAAkAgEAIlAAJAIBACJQACQCAQAiecQriKnTp2qrZ85c6a2fuONN9bWH3/8\n8dr69u3ba+voPkYIABKBACARCAASgQAgEQgAEoEAIBEIAFLPz2Vo9WB8LkNR69atq62/8847tXW7\n/rX+d955Z239wIEDtXUMVj+fy8AIAUAiEAAkAgFAIhAAJAIBQCIQACQCAUDiOYRryPz59a+/uPfe\ne2vr+/btq61/8cUXtfUHHnigtn748OHaOppp5TkE2zttn7Q9NmXdNtvHbX9Ufd3ftFkA5fUzZdgl\n6b5p1v9zRKyuvv6z3bYAlNAzECLiPUmnh9ALgMKaXFR8wvbH1ZRiUWsdAShmroGwXdIKSaslTUh6\nfqYNbW+xfcA2/7IF6Lg5BUJEnIiICxFxUdLPJf2oZtsdEbEmItbMtUkAwzGnQLA9OuXHhyWNzbQt\ngCtHz89lsP2ypHWSbrF9TNIzktbZXi0pJI1LemyAPXZGr/cBzJs3r7Z+/vz5RsdfsGBBo+OvX7++\ntv7II4/MuqeplixZUlvfu3dvbX358uWNjo/megZCRGycZvVLA+gFQGE8ugwgEQgAEoEAIBEIABKB\nACARCABSz9uOV5NXXnml0e/3us8/OjpaWz927Fij499zzz219VtvvbXR/gftm2++Kd0CemCEACAR\nCAASgQAgEQgAEoEAIBEIABKBACBdU88hrFy5sra+atWq2vrIyEib7Vxxzp49W1v/8MMPa+ubN29u\nsRsMAiMEAIlAAJAIBACJQACQCAQAiUAAkAgEAMkRMbyD2cM72Bz0eg7huuuuq60/+uijbbbTuq++\n+qq2/vrrr9fWz507V1v/7LPPZt0Thici6j9YRIwQAExBIABIBAKARCAASAQCgEQgAEgEAoDEcwjA\nNaKV5xBsL7G9z/YR24dtP1mtv9n227Y/rb4vaqNpAOX0HCHYHpU0GhGHbN8g6aCkhyRtlnQ6Ip61\n/ZSkRRHxkx77YoQAFNLKCCEiJiLiULV8VtJRSbdJWi9pd7XZbk2GBIAr2KwuKtpeJukOSfslLY6I\niar0paTFrXYGYOj6fsmq7YWSXpO0NSLO2H8YfUREzDQdsL1F0pamjQIYvL7uMtgekbRX0lsR8UK1\n7reS1kXERHWd4b8j4vs99sM1BKCQtu4yWNJLko5eCoPKm5I2VcubJL0xlyYBdEc/dxnWSnpf0ieS\nLlarn9bkdYRXJS2V9LmkDRFxuse+GCEAhfQzQuDBJOAawQtSAMwKgQAgEQgAEoEAIBEIABKBACAR\nCAASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEI\nABKBACARCAASgQAgEQgAEoEAIBEIABKBACD1DATbS2zvs33E9mHbT1brt9k+bvuj6uv+wbcLYJAc\nEfUb2KOSRiPikO0bJB2U9JCkDZLORcRzfR/Mrj8YgIGJCPfaZn4fO5mQNFEtn7V9VNJtzdsD0DWz\nuoZge5mkOyTtr1Y9Yftj2zttL2q5NwBD1ncg2F4o6TVJWyPijKTtklZIWq3JEcTzM/zeFtsHbB9o\noV8AA9TzGoIk2R6RtFfSWxHxwjT1ZZL2RsQPeuyHawhAIf1cQ+jnLoMlvSTp6NQwqC42XvKwpLG5\nNAmgO/q5y7BW0vuSPpF0sVr9tKSNmpwuhKRxSY9VFyDr9sUIASiknxFCX1OGthAIQDmtTBkAXDsI\nBACJQACQCAQAiUAAkAgEAIlAAJAIBACJQACQCAQAiUAAkAgEAIlAAJAIBACJQACQer51uWWnJH0+\n5edbqnVdRX/NdLm/Lvcmtd/fX/az0VBfkPInB7cPRMSaYg30QH/NdLm/LvcmleuPKQOARCAASKUD\nYUfh4/dCf810ub8u9yYV6q/oNQQA3VJ6hACgQwgEAIlAAJAIBACJQACQ/h/7M7SWMdYWPgAAAABJ\nRU5ErkJggg==\n", 1295 | "text/plain": [ 1296 | "" 1297 | ] 1298 | }, 1299 | "metadata": {}, 1300 | "output_type": "display_data" 1301 | }, 1302 | { 1303 | "name": "stdout", 1304 | "output_type": "stream", 1305 | "text": [ 1306 | "true: 4.0 prediction: 6 expected error rate: 0.75\n" 1307 | ] 1308 | }, 1309 | { 1310 | "data": { 1311 | "text/plain": [ 1312 | "" 1313 | ] 1314 | }, 1315 | "metadata": {}, 1316 | "output_type": "display_data" 1317 | }, 1318 | { 1319 | "data": { 1320 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADGRJREFUeJzt3V+IXOUdxvHnqdob9SIiDcGutRVvRGgsi0YaSopUrDcq\nuGtzUVIoxAsF4x+oeKM3BSkmtldCrMEU1O4uahUpVRFBK9lglGCiaatIdF3WBPFCvSqaXy/25NdR\nd8+Z3TMz79n4/UCYM+edmfPzmDy875l33uOIEABI0ndKFwCgOwgEAIlAAJAIBACJQACQCAQAqUgg\n2L7a9r9tv2v7rhI11LF91PYh2wdtH+hAPXtsH7d9uGffObZfsP1O9biuY/Xda3u+OocHbV9TsL4x\n2y/Zftv2W7ZvrfZ34hzW1Dfyc+hRz0OwfZqk/0j6haQPJb0maWtEvD3SQmrYPippPCI+Ll2LJNn+\nmaTPJf0lIi6p9v1B0icRcV8Vqusi4ncdqu9eSZ9HxP0lauple4OkDRHxhu2zJb0u6TpJv1EHzmFN\nfZMa8Tks0UO4TNK7EfFeRPxX0l8lXVugjjUjIl6W9MnXdl8raW+1vVeLf4GKWKa+zoiIhYh4o9r+\nTNIRSeepI+ewpr6RKxEI50ma63n+oQr9x9cISc/bft329tLFLGN9RCxU2x9JWl+ymGXcYvvNakhR\nbEjTy/YFki6VtF8dPIdfq08a8TnkouLSNkfETyT9UtLNVZe4s2Jx3Ne1OegPSrpQ0kZJC5J2li1H\nsn2WpCck7YiIT3vbunAOl6hv5OewRCDMSxrref79al9nRMR89Xhc0lNaHOZ0zbFq7HlyDHq8cD1f\nERHHIuLLiDgh6SEVPoe2z9DiP7ZHI+LJandnzuFS9ZU4hyUC4TVJF9n+oe3vSvqVpGcK1LEk22dW\nF3Zk+0xJV0k6XP+uIp6RtK3a3ibp6YK1fMPJf2iV61XwHNq2pIclHYmIXT1NnTiHy9VX4hyO/FsG\nSaq+PvmjpNMk7YmI34+8iGXY/pEWewWSdLqkx0rXZ/txSVsknSvpmKR7JP1N0rSk8yW9L2kyIopc\n2Fumvi1a7OqGpKOSbuoZr4+6vs2SXpF0SNKJavfdWhynFz+HNfVt1YjPYZFAANBNXFQEkAgEAIlA\nAJAIBACJQACQigZCh6cFS6K+trpcX5drk8rVV7qH0On/KaK+trpcX5drkwrVVzoQAHRIq4lJtq+W\n9Cctzjj8c0Tc1/B6ZkEBhUSEm16z6kBYzUInBAJQTj+B0GbIwEInwCmmTSCshYVOAKzA6cM+QPX1\nSdev6AJQu0Doa6GTiNgtabfENQSg69oMGTq90AmAlVt1DyEivrB9i6Tn9P+FTt4aWGUARm6kC6Qw\nZADKGfbXjgBOMQQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoBEIABIBAKARCAASAQCgEQg\nAEgEAoBEIABIBAKARCAASAQCgEQgAEgEAoA09Fu5ASdNT0/Xtk9MTNS2X3HFFbXts7OzK64JX0UP\nAUAiEAAkAgFAIhAAJAIBQCIQACQCAUDidvAYmLGxsdr2Dz74oNXnz8zM1LZPTk62+vxTXT+3g281\nMcn2UUmfSfpS0hcRMd7m8wCUNYiZij+PiI8H8DkACuMaAoDUNhBC0vO2X7e9fRAFASin7ZBhc0TM\n2/6epBds/ysiXu59QRUUhAWwBrTqIUTEfPV4XNJTki5b4jW7I2KcC45A9606EGyfafvsk9uSrpJ0\neFCFARi9NkOG9ZKesn3ycx6LiH8MpCqsSVNTU6VLQEurDoSIeE/SjwdYC4DC+NoRQCIQACQCAUAi\nEAAkAgFAIhAAJO7LgL5t2rSptr3pvgnoPnoIABKBACARCAASgQAgEQgAEoEAIBEIABLzENC3Xbt2\nlS4BQ0YPAUAiEAAkAgFAIhAAJAIBQCIQACQCAUBiHgLS5ORkbXvTegdzc3O17bOzs7XtExMTte0Y\nPnoIABKBACARCAASgQAgEQgAEoEAIBEIABLzEJB27NjR6v133nlnbfsNN9zQ6vOb5jmgvcYegu09\nto/bPtyz7xzbL9h+p3pcN9wyAYxCP0OGRyRd/bV9d0l6MSIukvRi9RzAGtcYCBHxsqRPvrb7Wkl7\nq+29kq4bcF0ACljtRcX1EbFQbX8kaf2A6gFQUOuLihERtmO5dtvbJW1vexwAw7faHsIx2xskqXo8\nvtwLI2J3RIxHxPgqjwVgRFYbCM9I2lZtb5P09GDKAVBS45DB9uOStkg61/aHku6RdJ+kadu/lfS+\npPof0qMT2q53sG/fvtr26enp2vapqana9ib79+9v9X40awyEiNi6TNOVA64FQGFMXQaQCAQAiUAA\nkAgEAIlAAJAIBACJ9RC+Rdqud3D77bcPqBJ0FT0EAIlAAJAIBACJQACQCAQAiUAAkAgEAIl5CKeQ\nnTt31rY3rXcwMzNT2z47O1vbvmnTptr2tprWY0B79BAAJAIBQCIQACQCAUAiEAAkAgFAIhAAJOYh\nrCFjY2O17U3rFczNzdW233HHHSuuqdfExESr9zfV19SO9ughAEgEAoBEIABIBAKARCAASAQCgEQg\nAEjMQ+iQpnkGr776aqvPn5ycrG1v+z1/U/1NHnjggVbvR3uNPQTbe2wft324Z9+9tudtH6z+XDPc\nMgGMQj9DhkckXb3E/gciYmP15++DLQtACY2BEBEvS/pkBLUAKKzNRcVbbL9ZDSnWDawiAMWsNhAe\nlHShpI2SFiQtu7qn7e22D9g+sMpjARiRVQVCRByLiC8j4oSkhyRdVvPa3RExHhHjqy0SwGisKhBs\nb+h5er2kw8u9FsDa0TgPwfbjkrZIOtf2h5LukbTF9kZJIemopJuGWOMpo+l7+qmpqVbvb1oPoem+\nCm21vS9D030jmtqb5lmgWWMgRMTWJXY/PIRaABTG1GUAiUAAkAgEAIlAAJAIBACJQACQHBGjO5g9\nuoN10PT0dG17030NZmZmatub7qvQ9D3+5Zdf3ur9Te3DZrvo8bsuIhpPED0EAIlAAJAIBACJQACQ\nCAQAiUAAkAgEAIl5CAN022231bbv2rVrRJWsTTfeeGNte9M8DtRjHgKAFSEQACQCAUAiEAAkAgFA\nIhAAJAIBQGIewgAN+1zu27evtr1pvYT5+fna9qbv+dvOs5ibm6ttP//882vb0Q7zEACsCIEAIBEI\nABKBACARCAASgQAgEQgAUuPt4NG/pt/zN+n67/3b3ndhbGxsQJVgWBp7CLbHbL9k+23bb9m+tdp/\nju0XbL9TPa4bfrkAhqmfIcMXku6IiIslbZJ0s+2LJd0l6cWIuEjSi9VzAGtYYyBExEJEvFFtfybp\niKTzJF0raW/1sr2SrhtWkQBGY0UXFW1fIOlSSfslrY+IharpI0nrB1oZgJHr+6Ki7bMkPSFpR0R8\n2ntjzYiI5X64ZHu7pO1tCwUwfH31EGyfocUweDQinqx2H7O9oWrfIOn4Uu+NiN0RMR4R44MoGMDw\n9PMtgyU9LOlIRPT+vvUZSduq7W2Snh58eQBGqZ8hw08l/VrSIdsHq313S7pP0rTt30p6X9LkcEpc\nO7o+j6C0pvUQUF5jIETEPyUtt7DClYMtB0BJTF0GkAgEAIlAAJAIBACJQACQCAQAifUQMDKzs7Ol\nS0ADeggAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEIABKBACARCAASgQAgEQgAEoEAIBEIAJIj\nlrwD23AOtszt3gAMX0QsdzuFRA8BQCIQACQCAUAiEAAkAgFAIhAAJAIBQGoMBNtjtl+y/bbtt2zf\nWu2/1/a87YPVn2uGXy6AYWqcmGR7g6QNEfGG7bMlvS7pOkmTkj6PiPv7PhgTk4Bi+pmY1HjnpohY\nkLRQbX9m+4ik89qXB6BrVnQNwfYFki6VtL/adYvtN23vsb1uwLUBGLG+A8H2WZKekLQjIj6V9KCk\nCyVt1GIPYucy79tu+4DtAwOoF8AQ9fXjJttnSHpW0nMRsWuJ9gskPRsRlzR8DtcQgEIG8uMm25b0\nsKQjvWFQXWw86XpJh1dTJIDu6Odbhs2SXpF0SNKJavfdkrZqcbgQko5Kuqm6AFn3WfQQgEL66SGw\nHgLwLcF6CABWhEAAkAgEAIlAAJAIBACJQACQCAQAiUAAkAgEAIlAAJAIBACJQACQCAQAiUAAkAgE\nAKlx1eUB+1jS+z3Pz632dRX1tdPl+rpcmzT4+n7Qz4tGukDKNw5uH4iI8WIFNKC+drpcX5drk8rV\nx5ABQCIQAKTSgbC78PGbUF87Xa6vy7VJheoreg0BQLeU7iEA6BACAUAiEAAkAgFAIhAApP8B5W+k\nso0Uu+QAAAAASUVORK5CYII=\n", 1321 | "text/plain": [ 1322 | "" 1323 | ] 1324 | }, 1325 | "metadata": {}, 1326 | "output_type": "display_data" 1327 | }, 1328 | { 1329 | "name": "stdout", 1330 | "output_type": "stream", 1331 | "text": [ 1332 | "true: 2.0 prediction: 9 expected error rate: 0.74\n" 1333 | ] 1334 | }, 1335 | { 1336 | "data": { 1337 | "text/plain": [ 1338 | "" 1339 | ] 1340 | }, 1341 | "metadata": {}, 1342 | "output_type": "display_data" 1343 | }, 1344 | { 1345 | "data": { 1346 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADiVJREFUeJzt3VuMVWWaxvHnEVATTwENpFTQ0ehFp5PBSYWoKSZ4mPaQ\nGPWGNIkKiQkaMQEzF4PGAxdqiAGc0QuEHqAZDg4k0mI6k+lGY6ITjTSQElCYsSWQgZSUSDwRBZV3\nLmrxdY1Nfauq9q69VlH/X0Jq13o2e79s4XGttb9a2xEhAJCks6oeAEB9UAgAEgoBQEIhAEgoBAAJ\nhQAgqaQQbN9u+79t/9n2/CpmyLG93/Yu2522t9VgnpW2u23v7rVtnO0ttj8pvo6t2XwLbB8qXsNO\n23dWON9E22/b/tj2R7bnFttr8Rpm5mv5a+hWr0OwPUrS/0j6B0kHJf1J0oyI+Lilg2TY3i+pPSKO\nVD2LJNn+e0nfSvq3iPhlse0FSUcjYmFRqmMj4p9qNN8CSd9GxKIqZurNdpuktojYYfsCSdsl3SNp\nlmrwGmbmm64Wv4ZV7CFMkfTniNgXESck/bukuyuYY9iIiHckHf3Z5rslrS5ur1bPX6BK9DFfbURE\nV0TsKG5/I2mPpMtUk9cwM1/LVVEIl0n6317fH1RFf/iMkPRH29ttz656mD5MiIiu4vZnkiZUOUwf\nHrW9szikqOyQpjfbV0q6TtIHquFr+LP5pBa/hpxUPL2OiPg7SXdImlPsEtdW9Bz31W0N+lJJV0ua\nLKlL0uJqx5Fsny/pNUnzIuLr3lkdXsPTzNfy17CKQjgkaWKv7y8vttVGRBwqvnZL+p16DnPq5nBx\n7HnqGLS74nn+n4g4HBE/RcRJSb9Rxa+h7THq+ce2LiI2FZtr8xqebr4qXsMqCuFPkq6x/Te2z5b0\na0lvVDDHadk+rzixI9vnSfqVpN3531WJNyTNLG7PlLS5wln+yql/aIV7VeFraNuSVkjaExFLekW1\neA37mq+K17Dl7zJIUvH2yT9LGiVpZUQ81/Ih+mD7KvXsFUjSaEnrq57P9quSpkm6RNJhSc9Iel3S\nRkmTJB2QND0iKjmx18d809SzqxuS9kt6qNfxeqvn65D0rqRdkk4Wm59Qz3F65a9hZr4ZavFrWEkh\nAKgnTioCSCgEAAmFACChEAAkFAKApNJCqPGyYEnM16g6z1fn2aTq5qt6D6HW/1HEfI2q83x1nk2q\naL6qCwFAjTS0MMn27ZL+RT0rDv81IhaW3J9VUEBFIsJl9xl0IQzmQicUAlCd/hRCI4cMXOgEOMM0\nUgjD4UInAAZg9FA/QfH2Sd3P6AJQY4XQrwudRMRyScslziEAddfIIUOtL3QCYOAGvYcQET/aflTS\nH/SXC5181LTJALRcSy+QwiEDUJ2hftsRwBmGQgCQUAgAEgoBQEIhAEgoBAAJhQAgoRAAJBQCgIRC\nAJBQCAASCgFAQiEASCgEAAmFACChEAAkFAKAhEIAkFAIABIKAUBCIQBIKAQACYUAIKEQACQUAoCE\nQgCQUAgAEgoBQEIhAEgoBAAJhQAgGd3Ib7a9X9I3kn6S9GNEtDdjKADVaKgQCjdFxJEmPA6AinHI\nACBptBBC0h9tb7c9uxkDAahOo4cMHRFxyPZ4SVts742Id3rfoSgKygIYBhwRzXkge4GkbyNiUeY+\nzXkyAAMWES67z6APGWyfZ/uCU7cl/UrS7sE+HoDqNXLIMEHS72yfepz1EfGfTZkKGITbbrstm196\n6aXZfOrUqdl8/fr12fzNN9/M5sPBoAshIvZJ+tsmzgKgYrztCCChEAAkFAKAhEIAkFAIABIKAUDS\ntJWK/XoyViqe0caNG5fNJ06cmM3L1gm88sorDf3+UaNGZfMyx44dy+YdHR3Z/MMPP2zo+Rs1pCsV\nAZx5KAQACYUAIKEQACQUAoCEQgCQUAgAEtYhILn44ouz+X333ZfNH3/88Wx+0UUXZfPRo/M/jV+2\njuD777/P5lu3bs3ml19+eTa/6qqrsvmqVauy+YMPPpjNhxrrEAAMCIUAIKEQACQUAoCEQgCQUAgA\nEgoBQNKMT3/GMDFmzJhs/uSTT2bzefPmZfNG17QcPHgwmy9btiybb968OZvv27cvm2/atCmbl61D\nOH78eDYfDthDAJBQCAASCgFAQiEASCgEAAmFACChEAAkrEMYQebPn5/N586dm83L1hl0d3dn8w0b\nNmTzl156KZt/+umn2bxMW1tbNr/22muz+Q8//JDNy+YfDkr3EGyvtN1te3evbeNsb7H9SfF17NCO\nCaAV+nPI8FtJt/9s23xJb0XENZLeKr4HMMyVFkJEvCPp6M823y1pdXF7taR7mjwXgAoM9qTihIjo\nKm5/JmlCk+YBUKGGTypGROQunmp7tqTZjT4PgKE32D2Ew7bbJKn42ufp5YhYHhHtEdE+yOcC0CKD\nLYQ3JM0sbs+UlP+5UwDDQukhg+1XJU2TdIntg5KekbRQ0kbbD0o6IGn6UA6J5jhy5Eg2P3ToUDZf\nsmRJNl+zZk1Dz9+oss91WLt2bTafNGlSNi/78+3duzebDwelhRARM/qIbmnyLAAqxtJlAAmFACCh\nEAAkFAKAhEIAkFAIABI3ei39AT1ZZokzUOaKK67I5rNmzcrmTz31VDZ///33s/n06fnlNl1dXdm8\nahHhsvuwhwAgoRAAJBQCgIRCAJBQCAASCgFAQiEASFiHgNq4/vrrs/nrr7+ezcePH5/Nt27dms3v\nuuuubP75559n87pjHQKAAaEQACQUAoCEQgCQUAgAEgoBQEIhAEga/ig3oL+ee+65bP7YY49l83PP\nPTebb9y4MZs//PDD2fzLL7/M5iMBewgAEgoBQEIhAEgoBAAJhQAgoRAAJBQCgITrIQzAHXfckc3P\nOeecbL5ly5ZsfuzYsQHP1Ert7e3ZfOHChdn85ptvzuZl6wCef/75bL5o0aJsPtI15XoItlfa7ra9\nu9e2BbYP2e4sft3Z6LAAqtefQ4bfSrr9NNtfjIjJxa//aO5YAKpQWggR8Y6koy2YBUDFGjmp+Kjt\nncUhxdimTQSgMoMthKWSrpY0WVKXpMV93dH2bNvbbG8b5HMBaJFBFUJEHI6InyLipKTfSJqSue/y\niGiPiPwpagCVG1Qh2G7r9e29knb3dV8Aw0fp9RBsvyppmqRLbB+U9IykabYnSwpJ+yU9NIQztszL\nL7+czR955JFsbuff5j1+/Hg2L1sT8t1332XzuXPnZvNdu3Zl8ylT+tzRkyQtXbo0m591Vv7/Ly+8\n8EI2X79+fTbfuXNnNkfjSgshImacZvOKIZgFQMVYugwgoRAAJBQCgIRCAJBQCAASCgFAMqKuhzBr\n1qxsvmJF/t3U9957L5s//fTT2byzszObP/DAA9n8xRdfzOZlTpw4kc3PPvvsbL53795sXvbn37Rp\nUzY/efJkNkdjmnI9BAAjB4UAIKEQACQUAoCEQgCQUAgAEgoBQDKi1iHs2bMnm1944YXZfMOGDdn8\niy++yOZjxozJ5jfeeGM2v/XWW7P5UOvo6MjmZes0UC3WIQAYEAoBQEIhAEgoBAAJhQAgoRAAJBQC\ngGRErUOYP39+Nn/22WezednnDjSqu7s7mx84cCCbL17c5yfqSZKmTp2azefMmZPN161bl83vv//+\nbI5qsQ4BwIBQCAASCgFAQiEASCgEAAmFACChEAAkI2odQplbbrklm0+bNq2hx//qq6+y+Zo1a7L5\n4cOHG3r+sWPHZvNVq1Zl85tuuimbl61jWLt2bTbH0GrKOgTbE22/bftj2x/ZnltsH2d7i+1Piq/5\nv20Aaq8/hww/SvrHiPiFpOslzbH9C0nzJb0VEddIeqv4HsAwVloIEdEVETuK299I2iPpMkl3S1pd\n3G21pHuGakgArTGgk4q2r5R0naQPJE2IiK4i+kzShKZOBqDlRvf3jrbPl/SapHkR8bX9l/MTERF9\nnTC0PVvS7EYHBTD0+rWHYHuMespgXUSc+gjfw7bbirxN0ml/VC8ilkdEe0S0N2NgAEOnP+8yWNIK\nSXsiYkmv6A1JM4vbMyVtbv54AFqpdB2C7Q5J70raJelksfkJ9ZxH2ChpkqQDkqZHxNGSx6r1OoSR\nbvLkydl8x44d2Xznzp3ZfMqUKdn8xIkT2RyN6c86hNJzCBHxX5L6eqD8Sh4AwwpLlwEkFAKAhEIA\nkFAIABIKAUBCIQBI+r10GWe+zs7ObL5s2bJsfsMNNzRzHFSAPQQACYUAIKEQACQUAoCEQgCQUAgA\nEgoBQMLnMqDfxo8fn80nTpyYzbdv397McTBATflcBgAjB4UAIKEQACQUAoCEQgCQUAgAEgoBQMI6\nBGCEYB0CgAGhEAAkFAKAhEIAkFAIABIKAUBCIQBISgvB9kTbb9v+2PZHtucW2xfYPmS7s/h159CP\nC2AolS5Mst0mqS0idti+QNJ2SfdImi7p24hY1O8nY2ESUJn+LEwq/eSmiOiS1FXc/sb2HkmXNT4e\ngLoZ0DkE21dKuk7SB8WmR23vtL3S9tgmzwagxfpdCLbPl/SapHkR8bWkpZKuljRZPXsQi/v4fbNt\nb7O9rQnzAhhC/frhJttjJP1e0h8iYslp8isl/T4iflnyOJxDACrSlB9usm1JKyTt6V0GxcnGU+6V\ntHswQwKoj/68y9Ah6V1JuySdLDY/IWmGeg4XQtJ+SQ8VJyBzj8UeAlCR/uwhcD0EYITgeggABoRC\nAJBQCAASCgFAQiEASCgEAAmFACChEAAkFAKAhEIAkFAIABIKAUBCIQBIKAQACYUAICm96nKTHZF0\noNf3lxTb6or5GlPn+eo8m9T8+a7oz51aeoGUv3pye1tEtFc2QAnma0yd56vzbFJ183HIACChEAAk\nVRfC8oqfvwzzNabO89V5Nqmi+So9hwCgXqreQwBQIxQCgIRCAJBQCAASCgFA8n9NJD73xJlUCwAA\nAABJRU5ErkJggg==\n", 1347 | "text/plain": [ 1348 | "" 1349 | ] 1350 | }, 1351 | "metadata": {}, 1352 | "output_type": "display_data" 1353 | } 1354 | ], 1355 | "source": [ 1356 | "low_confidence_examples_display()" 1357 | ] 1358 | }, 1359 | { 1360 | "cell_type": "markdown", 1361 | "metadata": {}, 1362 | "source": [ 1363 | "That's done, I hope model confidence does not seem such vague now. Thank you for reading." 1364 | ] 1365 | } 1366 | ], 1367 | "metadata": { 1368 | "kernelspec": { 1369 | "display_name": "Python 3", 1370 | "language": "python", 1371 | "name": "python3" 1372 | }, 1373 | "language_info": { 1374 | "codemirror_mode": { 1375 | "name": "ipython", 1376 | "version": 3 1377 | }, 1378 | "file_extension": ".py", 1379 | "mimetype": "text/x-python", 1380 | "name": "python", 1381 | "nbconvert_exporter": "python", 1382 | "pygments_lexer": "ipython3", 1383 | "version": "3.6.1" 1384 | } 1385 | }, 1386 | "nbformat": 4, 1387 | "nbformat_minor": 1 1388 | } 1389 | -------------------------------------------------------------------------------- /gelato/__init__.py: -------------------------------------------------------------------------------- 1 | from . import random 2 | from .random import * 3 | from gelato.specs import * 4 | from . import layers 5 | from . import specs 6 | from .version import __version__ 7 | -------------------------------------------------------------------------------- /gelato/_compile.py: -------------------------------------------------------------------------------- 1 | from six import exec_ 2 | import inspect 3 | 4 | 5 | def define(name, template, namespace, frame_offset=1): 6 | try: 7 | exec_(template, namespace) 8 | except SyntaxError as e: 9 | raise SyntaxError(str(e) + ':\n' + template) 10 | result = namespace[name] 11 | frm = inspect.stack()[frame_offset] 12 | mod = inspect.getmodule(frm[0]) 13 | result.__name__ = name 14 | if mod is None: 15 | result.__module__ = '__main__' 16 | else: 17 | result.__module__ = mod.__name__ 18 | return result 19 | -------------------------------------------------------------------------------- /gelato/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from .helper import * 2 | from .base import * 3 | from .conv import * 4 | from .dense import * 5 | from .local import * 6 | from .recurrent import * 7 | from .input import * 8 | from .merge import * 9 | from .shape import * 10 | from .embedding import * 11 | from .pool import * 12 | from .special import * 13 | from .stats import * 14 | -------------------------------------------------------------------------------- /gelato/layers/base.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import inspect 3 | import six 4 | import lasagne.layers.base 5 | import pymc3 as pm 6 | from pymc3.memoize import hashable 7 | 8 | from gelato.specs.dist import get_default_spec, FlatSpec 9 | from gelato.specs.base import DistSpec 10 | 11 | __all__ = [ 12 | 'LayerModelMeta', 13 | 'Layer', 14 | 'MergeLayer', 15 | 'bayes' 16 | ] 17 | 18 | 19 | class LayerModelMeta(pm.model.InitContextMeta): 20 | """Magic comes here 21 | """ 22 | 23 | def __init__(cls, what, bases, dic): 24 | from gelato.layers.helper import find_parent 25 | super(LayerModelMeta, cls).__init__(what, bases, dic) 26 | # make flexible property for new class 27 | 28 | def fget(self): 29 | if self._name is None: 30 | return '{}_{}'.format(self.__class__.__name__, self._fingerprint) 31 | else: 32 | return self._name 33 | 34 | def fset(self, value): 35 | if not value: 36 | self._name = None 37 | else: 38 | self._name = str(value) 39 | 40 | cls._name = None 41 | cls.name = property(fget, fset) 42 | 43 | # wrap init for new class 44 | def wrap_init(__init__): 45 | @functools.wraps(__init__) 46 | def wrapped(self, *args, **kwargs): 47 | name = kwargs.get('name') 48 | self._fingerprint = hashable(self.parent) 49 | pm.Model.__init__(self, name) 50 | __init__(self, *args, **kwargs) 51 | return wrapped 52 | 53 | # wrap new for new class 54 | def wrap_new(__new__): 55 | @functools.wraps(__new__) 56 | def wrapped(_cls_, *args, **kwargs): 57 | parent = kwargs.get('model', None) 58 | if parent is None and not issubclass(_cls_, lasagne.layers.InputLayer): 59 | incoming = kwargs.get('incoming', 60 | kwargs.get('incomings', 61 | args[1])) 62 | parent = find_parent(incoming) 63 | kwargs['model'] = parent 64 | instance = __new__(_cls_, *args, **kwargs) 65 | return instance 66 | return classmethod(wrapped) 67 | 68 | cls.__init__ = wrap_init(cls.__init__) 69 | cls.__new__ = wrap_new(cls.__new__) 70 | 71 | def add_param(self, spec, shape, name=None, **tags): 72 | if tags.get('trainable', True): 73 | if tags.get('regularizable', True): 74 | if not isinstance(spec, DistSpec): 75 | # here spec is like test value 76 | # passed to pymc3 distribution 77 | spec = getattr(self, 'default_spec', get_default_spec(spec)) 78 | else: 79 | spec = FlatSpec() 80 | if name is not None: 81 | spec = spec.with_name(name) 82 | return lasagne.layers.base.Layer.add_param( 83 | self, spec, shape, **tags) 84 | cls.add_param = add_param 85 | 86 | # needed for working with lasagne tools 87 | def wrap_getitem(__getitem__): 88 | @functools.wraps(__getitem__) 89 | def wrapped(self, item): 90 | if not isinstance(item, six.string_types): 91 | raise TypeError('%r object accepts only string keys' 92 | % self.__class__) 93 | else: 94 | __getitem__(self, item) 95 | return wrapped 96 | 97 | cls.__getitem__ = wrap_getitem(cls.__getitem__) 98 | 99 | def __repr__(self): 100 | return '{}.{}'.format(self.__module__, self.__name__) 101 | 102 | @classmethod 103 | def __subclasshook__(cls, C): 104 | if lasagne.layers.Layer in C.__mro__ or pm.Model in C.__mro__: 105 | return True 106 | else: 107 | return False 108 | 109 | 110 | def bayes(layercls, stack=1): 111 | try: 112 | issubcls = issubclass(layercls, lasagne.layers.base.Layer) 113 | except TypeError: 114 | raise TypeError('{} needs to be a Layer subclass' 115 | .format(layercls)) 116 | if issubcls: 117 | if type(layercls) is LayerModelMeta: 118 | raise TypeError('{} is already bayesian' 119 | .format(layercls)) 120 | else: 121 | @six.add_metaclass(LayerModelMeta) 122 | class BayesianAnalog(layercls, pm.Model): 123 | pass 124 | frm = inspect.stack()[stack] 125 | mod = inspect.getmodule(frm[0]) 126 | if mod is None: 127 | modname = '__main__' 128 | else: 129 | modname = mod.__name__ 130 | BayesianAnalog.__module__ = modname 131 | BayesianAnalog.__doc__ = layercls.__doc__ 132 | BayesianAnalog.__name__ = layercls.__name__ 133 | return BayesianAnalog 134 | else: 135 | raise TypeError('{} needs to be a Layer subclass' 136 | .format(layercls)) 137 | 138 | Layer = bayes(lasagne.layers.base.Layer) 139 | MergeLayer = bayes(lasagne.layers.base.MergeLayer) 140 | -------------------------------------------------------------------------------- /gelato/layers/conv.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.conv as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/corrmm.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.corrmm as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/cuda_convnet.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.cuda_convnet as __cloned 3 | from .base import bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/dense.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.dense as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | setattr(__module, obj_name, getattr(__cloned, obj_name)) 13 | -------------------------------------------------------------------------------- /gelato/layers/dnn.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.dnn as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/embedding.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.embedding as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/helper.py: -------------------------------------------------------------------------------- 1 | import pymc3 as pm 2 | from lasagne.layers.helper import * 3 | from lasagne.layers.helper import __all__ as __helper__all__ 4 | __all__ = [ 5 | "find_parent", 6 | "find_root", 7 | ] + __helper__all__ 8 | 9 | 10 | def find_parent(layer): 11 | candidates = get_all_layers(layer)[::-1] 12 | found = None 13 | for candidate in candidates: 14 | if isinstance(candidate, pm.Model): 15 | found = candidate 16 | break 17 | return found 18 | 19 | 20 | def find_root(layer): 21 | model = find_parent(layer) 22 | if model is not None: 23 | return model.root 24 | else: 25 | return None 26 | -------------------------------------------------------------------------------- /gelato/layers/input.py: -------------------------------------------------------------------------------- 1 | from lasagne.layers import InputLayer as _InputLayer 2 | import numpy as np 3 | from .base import bayes as _bayes 4 | from theano.compile import SharedVariable 5 | 6 | __all__ = [ 7 | 'InputLayer' 8 | ] 9 | 10 | 11 | @_bayes 12 | class InputLayer(_InputLayer): 13 | def __init__(self, shape, input_var=None, name=None, testval=None, **kwargs): 14 | _InputLayer.__init__(self, shape, input_var=input_var, name=name, **kwargs) 15 | if testval is not None: 16 | self.input_var.tag.test_value = testval 17 | if (not isinstance(self.input_var, SharedVariable) 18 | and not hasattr(self.input_var.tag, 'test_value')): 19 | shape = [s if s is not None else 2 for s in self.shape] 20 | dtype = self.input_var.dtype 21 | self.input_var.tag.test_value = np.random.uniform(size=shape).astype(dtype) 22 | -------------------------------------------------------------------------------- /gelato/layers/local.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.local as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/merge.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.merge as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/pool.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.pool as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/recurrent.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.recurrent as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/shape.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.shape as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/special.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lasagne.layers.special as __cloned 3 | from .base import bayes as __bayes 4 | __module = sys.modules[__name__] 5 | del sys 6 | __all__ = [] 7 | for obj_name in __cloned.__all__: 8 | try: 9 | setattr(__module, obj_name, __bayes(getattr(__cloned, obj_name))) 10 | __all__ += [obj_name] 11 | except TypeError: 12 | pass 13 | # setattr(__module, obj_name, getattr(__cloned, obj_name)) 14 | -------------------------------------------------------------------------------- /gelato/layers/stats.py: -------------------------------------------------------------------------------- 1 | from .base import Layer 2 | 3 | __all__ = [ 4 | 'PosteriorLayer', 5 | 'SamplingLayer' 6 | ] 7 | 8 | 9 | class PosteriorLayer(Layer): 10 | """ 11 | Layer that makes replacements in the graph in the incoming layer 12 | so that `get_output` can be evaluated 13 | """ 14 | def get_output_for(self, incoming, approx=None, deterministic=False, **kwargs): 15 | if approx is None: 16 | raise ValueError('No approximation specified') 17 | return approx.apply_replacements(incoming, deterministic=deterministic) 18 | 19 | 20 | class SamplingLayer(Layer): 21 | """ 22 | Layer that makes replacements in the graph and additionally samples incoming 23 | layer. 24 | """ 25 | def __init__(self, incoming, samples=10, name=None): 26 | super(SamplingLayer, self).__init__(incoming, name=name) 27 | self.samples = samples 28 | 29 | def get_output_for(self, incoming, approx=None, **kwargs): 30 | if approx is None: 31 | raise ValueError('No approximation specified') 32 | return approx.sample_node(incoming, size=self.samples) 33 | 34 | def get_output_shape_for(self, input_shape): 35 | return (None, ) + input_shape 36 | -------------------------------------------------------------------------------- /gelato/random.py: -------------------------------------------------------------------------------- 1 | from pymc3.theanof import set_tt_rng, tt_rng 2 | __all__ = [ 3 | 'set_tt_rng', 4 | 'tt_rng' 5 | ] 6 | -------------------------------------------------------------------------------- /gelato/specs/__init__.py: -------------------------------------------------------------------------------- 1 | from .dist import * 2 | from .base import * 3 | -------------------------------------------------------------------------------- /gelato/specs/base.py: -------------------------------------------------------------------------------- 1 | import six 2 | import copy 3 | import itertools 4 | import functools 5 | import inspect 6 | import pymc3 as pm 7 | from theano import theano 8 | from theano.tensor.basic import _tensor_py_operators 9 | 10 | from lasagne import init 11 | from gelato._compile import define 12 | 13 | __all__ = [ 14 | 'BaseSpec', 15 | 'as_spec_op', 16 | 'get_default_testval', 17 | 'set_default_testval', 18 | 'DistSpec' 19 | ] 20 | 21 | 22 | class BaseSpec(init.Initializer): 23 | _counter = itertools.count(0) 24 | name = None 25 | tag = 'default' 26 | _shape = None 27 | 28 | def auto(self): 29 | if self.name is None: 30 | return 'auto_{}'.format(next(type(self)._counter)) 31 | else: 32 | name = self.name 33 | self.name = None 34 | return name 35 | 36 | def with_name(self, name): 37 | self.name = name 38 | return self 39 | 40 | def with_tag(self, tag): 41 | self.tag = tag 42 | return self 43 | 44 | def with_shape(self, shape): 45 | if callable(shape): 46 | self._shape = shape 47 | elif shape is not None: 48 | self._shape = shape 49 | self.tag = 'custom' 50 | else: 51 | self._shape = None 52 | self.tag = 'default' 53 | return self 54 | 55 | @staticmethod 56 | def _prepare(memo, shape): 57 | if memo is None: 58 | memo = {} 59 | if not isinstance(shape, dict): 60 | shape = {'default': shape} 61 | elif 'default' not in shape: 62 | raise ValueError('default shape not specified, ' 63 | 'please provide it with `default` ' 64 | 'key in input shape dict') 65 | return memo, shape 66 | 67 | def _get_shape(self, shape): 68 | """ 69 | Parameters 70 | ---------- 71 | shape : dict 72 | 73 | Returns 74 | ------- 75 | dict, str 76 | """ 77 | if callable(self._shape): 78 | new_shape, tag = self._shape(shape[self.tag]), self.tag 79 | elif self._shape is not None: 80 | new_shape, tag = self._shape, self.tag 81 | else: 82 | new_shape, tag = shape[self.tag], self.tag 83 | shape = shape.copy() 84 | shape['default'] = new_shape 85 | return shape, tag 86 | 87 | def _call_args(self, args, name, shape, memo): 88 | return [ 89 | self._call(arg, '{}.{}'.format(name, i) 90 | if name is not None and not name.startswith('auto') 91 | else self.auto(), shape, memo) 92 | for i, arg in enumerate(args) 93 | ] 94 | 95 | def _call_kwargs(self, kwargs, name, shape, memo): 96 | return { 97 | key: self._call(arg, '{}:{}'.format(name, next(self._counter), memo) 98 | if name is not None and not name.startswith('auto') 99 | else self.auto(), shape, memo) 100 | for key, arg in kwargs.items() 101 | } 102 | 103 | @staticmethod 104 | def _call(arg, label, shape, memo): 105 | if isinstance(arg, BaseSpec): 106 | return arg(shape, label, memo) 107 | elif isinstance(arg, init.Initializer): 108 | if isinstance(shape, dict): 109 | return arg(shape['default']) 110 | else: 111 | return arg 112 | 113 | def __call__(self, shape, name=None, memo=None): 114 | raise NotImplementedError 115 | 116 | 117 | head = '''\ 118 | class SpecVar(BaseSpec): 119 | """ 120 | Base class that supports delayed tensor operations 121 | """ 122 | def __init__(self, op, *args, **kwargs): 123 | self.op = op 124 | self.args = args 125 | self.kwargs = kwargs 126 | 127 | def __call__(self, shape, name=None, memo=None): 128 | memo, shape = self._prepare(memo, shape) 129 | shape, tag = self._get_shape(shape) 130 | if name is None: 131 | name = self.auto() 132 | if id(self) ^ hash(tag) in memo: 133 | return memo[id(self) ^ hash(tag)] 134 | args = self._call_args(self.args, name, shape, memo) 135 | kwargs = self._call_kwargs(self.kwargs, name, shape, memo) 136 | memo[id(self) ^ hash(tag)] = self.op(*args, **kwargs) 137 | return memo[id(self) ^ hash(tag)] 138 | 139 | def __repr__(self): 140 | if hasattr(self.op, '__name__'): 141 | return 'SpecOp.' + self.op.__name__ 142 | else: 143 | return 'SpecOp.' + type(self.op).__name__ 144 | 145 | __str__ = __repr__ 146 | 147 | def clone(self): 148 | return copy.deepcopy(self) 149 | 150 | def __iter__(self): 151 | raise NotImplementedError 152 | ''' 153 | exclude = { 154 | '__iter__' 155 | } 156 | mth_template = """\ 157 | def {0}{signature}: 158 | '''{doc}''' 159 | return SpecVar{inner_signature} 160 | """ 161 | meths = [] 162 | globs = dict(BaseSpec=BaseSpec, copy=copy) 163 | for key, mth in _tensor_py_operators.__dict__.items(): 164 | if callable(mth): 165 | if six.PY3: 166 | argspec = inspect.getfullargspec(mth) 167 | keywords = argspec.varkw 168 | else: 169 | argspec = inspect.getargspec(mth) 170 | keywords = argspec.keywords 171 | signature = inspect.formatargspec(*argspec) 172 | inner_signature = inspect.formatargspec( 173 | args=['mth{0}'.format(key)] + argspec.args, 174 | varargs=argspec.varargs, 175 | varkw=keywords, 176 | defaults=argspec.args[1:], 177 | formatvalue=lambda value: '=' + str(value) 178 | ) 179 | meths.append(mth_template.format( 180 | key, signature=signature, inner_signature=inner_signature, doc=mth.__doc__)) 181 | globs['mth{0}'.format(key)] = mth 182 | 183 | SpecVar = define('SpecVar', head + '\n'.join(meths), globs, 1) 184 | 185 | 186 | def as_spec_op(func): 187 | return functools.partial(SpecVar, func) 188 | 189 | 190 | class DistSpec(SpecVar): 191 | """Spec based on pymc3 distributions 192 | 193 | All specs support lazy evaluation, see Usage 194 | 195 | Parameters 196 | ---------- 197 | distcls : pymc3.Distribution 198 | args : args for `distcls` 199 | kwargs : kwargs for `distcls` 200 | 201 | Usage 202 | ----- 203 | >>> spec = DistSpec(Normal, mu=0, sd=DistSpec(Lognormal, 0, 1)) 204 | >>> spec += (NormalSpec() + LaplaceSpec()) / 100 - NormalSpec() 205 | >>> with Model(): 206 | ... prior_expr = spec((10, 10), name='silly_prior') 207 | 208 | """ 209 | def __init__(self, distcls, *args, **kwargs): 210 | if not isinstance(distcls, type) and issubclass(distcls, pm.Distribution): 211 | raise ValueError('We can deal with pymc3 ' 212 | 'distributions only, got {!r} instead' 213 | .format(distcls)) 214 | self.testval = kwargs.pop('testval', None) 215 | self.tag = kwargs.get('tag', 'default') 216 | self.args = args 217 | self.kwargs = kwargs 218 | self.distcls = distcls 219 | 220 | def __call__(self, shape, name=None, memo=None): 221 | memo, shape = self._prepare(memo, shape) 222 | if name is None: 223 | name = self.auto() 224 | shape, tag = self._get_shape(shape) 225 | if id(self) ^ hash(tag) in memo: 226 | return memo[id(self) ^ hash(tag)] 227 | model = pm.modelcontext(None) 228 | called_args = self._call_args(self.args, name, shape, memo) 229 | called_kwargs = self._call_kwargs(self.kwargs, name, shape, memo) 230 | called_kwargs.update(shape=shape['default']) 231 | val = model.Var( 232 | name, self.distcls.dist( 233 | *called_args, 234 | dtype=theano.config.floatX, 235 | **called_kwargs 236 | ), 237 | ) 238 | if self.testval is None: 239 | val.tag.test_value = get_default_testval()(shape['default']).astype(val.dtype) 240 | elif isinstance(self.testval, str) and self.testval == 'random': 241 | val.tag.test_value = val.random(size=shape['default']).astype(val.dtype) 242 | else: 243 | val.tag.test_value = self.testval(shape['default']).astype(val.dtype) 244 | memo[id(self) ^ hash(tag)] = val 245 | return memo[id(self) ^ hash(tag)] 246 | 247 | def __repr__(self): 248 | if self._shape != -1: 249 | sh = '; '+str(self._shape) 250 | else: 251 | sh = '' 252 | template = '<{cls}: {args!r}; {kwargs!r}'+sh+'>' 253 | return template.format(cls=self.distcls.__name__, 254 | args=self.args, 255 | kwargs=self.kwargs) 256 | 257 | 258 | def smart_init(shape): 259 | if len(shape) > 1: 260 | return init.GlorotUniform()(shape) 261 | else: 262 | return init.Normal()(shape) 263 | 264 | _default_testval = smart_init 265 | 266 | 267 | def set_default_testval(testval): 268 | global _default_testval 269 | _default_testval = testval 270 | 271 | 272 | def get_default_testval(): 273 | return _default_testval 274 | -------------------------------------------------------------------------------- /gelato/specs/dist.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | import pymc3 as pm 4 | from lasagne import init 5 | from gelato.specs.base import DistSpec, get_default_testval, smart_init 6 | 7 | __all__ = [ 8 | 'get_default_spec', 9 | 'set_default_spec', 10 | 'PartialSpec', 11 | 'UniformSpec', 12 | 'FlatSpec', 13 | 'NormalSpec', 14 | 'BetaSpec', 15 | 'ExponentialSpec', 16 | 'LaplaceSpec', 17 | 'StudentTSpec', 18 | 'CauchySpec', 19 | 'HalfCauchySpec', 20 | 'GammaSpec', 21 | 'WeibullSpec', 22 | 'LognormalSpec', 23 | 'ChiSquaredSpec', 24 | 'HalfNormalSpec', 25 | 'WaldSpec', 26 | 'ParetoSpec', 27 | 'InverseGammaSpec', 28 | 'ExGaussianSpec', 29 | 'VonMisesSpec', 30 | 'SkewNormalSpec', 31 | # 'HalfStudentTSpec', 32 | # 'NormalMixtureSpec' 33 | ] 34 | 35 | _default_spec = DistSpec(pm.Normal, mu=0, sd=10, testval=smart_init) 36 | 37 | 38 | def get_default_spec(testval=None): 39 | # to avoid init collision 40 | cp = copy.deepcopy(_default_spec) 41 | if testval is None and cp.testval is None: 42 | cp.testval = get_default_testval() 43 | elif testval is not None: 44 | cp.testval = testval 45 | else: 46 | pass 47 | return cp 48 | 49 | 50 | def set_default_spec(spec): 51 | global _default_spec 52 | _default_spec = spec 53 | 54 | 55 | class PartialSpec(DistSpec): 56 | spec = None 57 | 58 | def __init__(self, *args, **kwargs): 59 | super(PartialSpec, self).__init__(self.spec, *args, **kwargs) 60 | 61 | 62 | class UniformSpec(PartialSpec): 63 | spec = pm.Uniform 64 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 65 | dist=spec.__name__, 66 | doc=spec.__doc__ 67 | ) 68 | 69 | def __init__(self, lower=0, upper=1): 70 | super(UniformSpec, self).__init__(lower=lower, upper=upper) 71 | 72 | 73 | class FlatSpec(PartialSpec): 74 | spec = pm.Flat 75 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 76 | dist=spec.__name__, 77 | doc=spec.__doc__ 78 | ) 79 | 80 | def __init__(self): 81 | super(FlatSpec, self).__init__(testval=init.Uniform(1)) 82 | 83 | 84 | class NormalSpec(PartialSpec): 85 | spec = pm.Normal 86 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 87 | dist=spec.__name__, 88 | doc=spec.__doc__ 89 | ) 90 | 91 | def __init__(self, mu=0, sd=1): 92 | super(NormalSpec, self).__init__(mu=mu, sd=sd) 93 | 94 | 95 | class BetaSpec(PartialSpec): 96 | spec = pm.Beta 97 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 98 | dist=spec.__name__, 99 | doc=spec.__doc__ 100 | ) 101 | 102 | def __init__(self, alpha=1, beta=1): 103 | super(BetaSpec, self).__init__(alpha=alpha, beta=beta) 104 | 105 | 106 | class ExponentialSpec(PartialSpec): 107 | spec = pm.Exponential 108 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 109 | dist=spec.__name__, 110 | doc=spec.__doc__ 111 | ) 112 | 113 | def __init__(self, lam=1): 114 | super(ExponentialSpec, self).__init__(lam=lam) 115 | 116 | 117 | class LaplaceSpec(PartialSpec): 118 | spec = pm.Laplace 119 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 120 | dist=spec.__name__, 121 | doc=spec.__doc__ 122 | ) 123 | 124 | def __init__(self, mu=0, b=1): 125 | super(LaplaceSpec, self).__init__(mu=mu, b=b) 126 | 127 | 128 | class StudentTSpec(PartialSpec): 129 | spec = pm.StudentT 130 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 131 | dist=spec.__name__, 132 | doc=spec.__doc__ 133 | ) 134 | 135 | def __init__(self, nu, mu=0, sd=1): 136 | super(StudentTSpec, self).__init__(nu=nu, mu=mu, sd=sd) 137 | 138 | 139 | class CauchySpec(PartialSpec): 140 | spec = pm.Cauchy 141 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 142 | dist=spec.__name__, 143 | doc=spec.__doc__ 144 | ) 145 | 146 | def __init__(self, alpha=0, beta=1): 147 | super(CauchySpec, self).__init__(alpha=alpha, beta=beta) 148 | 149 | 150 | class HalfCauchySpec(PartialSpec): 151 | spec = pm.HalfCauchy 152 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 153 | dist=spec.__name__, 154 | doc=spec.__doc__ 155 | ) 156 | 157 | def __init__(self, beta): 158 | super(HalfCauchySpec, self).__init__(beta=beta) 159 | 160 | 161 | class GammaSpec(PartialSpec): 162 | spec = pm.Gamma 163 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 164 | dist=spec.__name__, 165 | doc=spec.__doc__ 166 | ) 167 | 168 | def __init__(self, alpha, beta): 169 | super(GammaSpec, self).__init__(alpha=alpha, beta=beta) 170 | 171 | 172 | class WeibullSpec(PartialSpec): 173 | spec = pm.Weibull 174 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 175 | dist=spec.__name__, 176 | doc=spec.__doc__ 177 | ) 178 | 179 | def __init__(self, alpha, beta): 180 | super(WeibullSpec, self).__init__(alpha=alpha, beta=beta) 181 | 182 | 183 | class LognormalSpec(PartialSpec): 184 | spec = pm.Lognormal 185 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 186 | dist=spec.__name__, 187 | doc=spec.__doc__ 188 | ) 189 | 190 | def __init__(self, mu=0, sd=1): 191 | super(LognormalSpec, self).__init__(mu=mu, sd=sd) 192 | 193 | 194 | class ChiSquaredSpec(PartialSpec): 195 | spec = pm.ChiSquared 196 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 197 | dist=spec.__name__, 198 | doc=spec.__doc__ 199 | ) 200 | 201 | def __init__(self, nu): 202 | super(ChiSquaredSpec, self).__init__(nu=nu) 203 | 204 | 205 | class HalfNormalSpec(PartialSpec): 206 | spec = pm.HalfNormal 207 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 208 | dist=spec.__name__, 209 | doc=spec.__doc__ 210 | ) 211 | 212 | def __init__(self, sd=1): 213 | super(HalfNormalSpec, self).__init__(sd=sd) 214 | 215 | 216 | class WaldSpec(PartialSpec): 217 | spec = pm.Wald 218 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 219 | dist=spec.__name__, 220 | doc=spec.__doc__ 221 | ) 222 | 223 | def __init__(self, mu, lam, alpha=0.): 224 | super(WaldSpec, self).__init__(mu=mu, lam=lam, alpha=alpha) 225 | 226 | 227 | class ParetoSpec(PartialSpec): 228 | spec = pm.Pareto 229 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 230 | dist=spec.__name__, 231 | doc=spec.__doc__ 232 | ) 233 | 234 | def __init__(self, alpha, m): 235 | super(ParetoSpec, self).__init__(alpha=alpha, m=m) 236 | 237 | 238 | class InverseGammaSpec(PartialSpec): 239 | spec = pm.InverseGamma 240 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 241 | dist=spec.__name__, 242 | doc=spec.__doc__ 243 | ) 244 | 245 | def __init__(self, alpha, beta=1): 246 | super(InverseGammaSpec, self).__init__(alpha=alpha, beta=beta) 247 | 248 | 249 | class ExGaussianSpec(PartialSpec): 250 | spec = pm.ExGaussian 251 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 252 | dist=spec.__name__, 253 | doc=spec.__doc__ 254 | ) 255 | 256 | def __init__(self, mu, sd, nu): 257 | super(ExGaussianSpec, self).__init__(mu=mu, sigma=sd, nu=nu) 258 | 259 | 260 | class VonMisesSpec(PartialSpec): 261 | spec = pm.VonMises 262 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 263 | dist=spec.__name__, 264 | doc=spec.__doc__ 265 | ) 266 | 267 | def __init__(self, mu, kappa): 268 | super(VonMisesSpec, self).__init__(mu=mu, kappa=kappa) 269 | 270 | 271 | class SkewNormalSpec(PartialSpec): 272 | spec = pm.SkewNormal 273 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 274 | dist=spec.__name__, 275 | doc=spec.__doc__ 276 | ) 277 | 278 | def __init__(self, mu=0.0, sd=1, alpha=1): 279 | super(SkewNormalSpec, self).__init__(mu=mu, sd=sd, alpha=alpha) 280 | 281 | ''' 282 | class HalfStudentTSpec(PartialSpec): 283 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 284 | dist=pm.HalfStudentT.distribution.__name__, 285 | doc="""Bounded StudentT with support on [0, +inf]\n{doc}""".format( 286 | doc=pm.StudentT.__doc__ 287 | ) 288 | ) 289 | spec = pm.HalfStudentT 290 | 291 | def __init__(self, nu, mu=0, sd=1): 292 | super(HalfStudentTSpec, self).__init__(nu=nu, mu=mu, sd=sd) 293 | ''' 294 | 295 | ''' 296 | class NormalMixtureSpec(PartialSpec): 297 | spec = pm.NormalMixture 298 | __doc__ = """Gelato DistSpec with {dist} prior\n\n{doc}""".format( 299 | dist=spec.__name__, 300 | doc=spec.__doc__ 301 | ) 302 | 303 | def __init__(self, w, mu, sd=None, tau=None): 304 | w = np.asarray(w) 305 | mu = np.asarray(mu) 306 | if sd is not None: 307 | sd = np.asarray(sd) 308 | if tau is not None: 309 | tau = np.asarray(tau) 310 | _, sd = get_tau_sd(tau, sd) 311 | super(NormalMixtureSpec, self).__init__(w=w, mu=mu, sd=sd) 312 | ''' 313 | -------------------------------------------------------------------------------- /gelato/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ferrine/gelato/77828020e5cbfe5e9a753c5add79f49f284fc6a9/gelato/tests/__init__.py -------------------------------------------------------------------------------- /gelato/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import gelato 3 | import pytest 4 | 5 | 6 | @pytest.fixture() 7 | def seeded(): 8 | gelato.set_tt_rng(42) 9 | np.random.seed(42) 10 | -------------------------------------------------------------------------------- /gelato/tests/datasets.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def generate_linear_regression(intercept, slope, sd=.2, size=700): 5 | x = np.random.uniform(-10, 10, size) 6 | y = intercept + x * slope 7 | return x, y + np.random.normal(size=size, scale=sd) 8 | 9 | 10 | def generate_sinus_regression(intercept, slope, sd=.2, size=700): 11 | x = np.random.uniform(-10, 10, size) 12 | y = intercept + (x + np.sin(x)) * slope 13 | return x, y + np.random.normal(size=size, scale=sd) 14 | -------------------------------------------------------------------------------- /gelato/tests/test_layers.py: -------------------------------------------------------------------------------- 1 | from theano import theano 2 | from .datasets import generate_linear_regression 3 | from gelato.layers import * 4 | import pymc3 as pm 5 | 6 | 7 | def test_stats_layers(): 8 | intercept = .2 9 | slope = 2 10 | x_, _ = generate_linear_regression(intercept, slope) 11 | x = theano.shared(x_[None].T) 12 | inp = InputLayer((None, 1), input_var=x) 13 | nnet = DenseLayer(inp, 3) 14 | nnet = DenseLayer(nnet, 2) 15 | nnet_post = PosteriorLayer(nnet) 16 | nnet_sample = SamplingLayer(nnet, samples=10) 17 | with nnet_post.root: 18 | approx = pm.MeanField() 19 | out_ = get_output(nnet_post, approx=approx) 20 | out = out_.eval() # should work 21 | assert out.shape == (x_.shape[0], 2) 22 | with nnet_sample.root: 23 | approx = pm.MeanField() 24 | out_ = get_output(nnet_sample, approx=approx) 25 | out = out_.eval() # should work 26 | assert out.shape == (10, x_.shape[0], 2) 27 | -------------------------------------------------------------------------------- /gelato/tests/test_magic.py: -------------------------------------------------------------------------------- 1 | import lasagne.nonlinearities as to 2 | import numpy as np 3 | import pymc3 as pm 4 | from lasagne import layers as llayers 5 | 6 | from gelato.layers import DenseLayer, InputLayer 7 | from gelato.layers import get_output, find_parent, find_root 8 | from gelato.specs.dist import NormalSpec, LognormalSpec 9 | from .datasets import generate_linear_regression 10 | 11 | 12 | class TestWorkflow(object): 13 | @classmethod 14 | def setup_class(cls): 15 | cls.intercept = 1 16 | cls.slope = 3 17 | cls.sd = .1 18 | cls.x, cls.y = generate_linear_regression(cls.intercept, cls.slope, sd=cls.sd) 19 | cls.x = np.matrix(cls.x).T 20 | cls.y = np.matrix(cls.y).T 21 | 22 | def test_workflow(self): 23 | inp = InputLayer(self.x.shape) 24 | out = DenseLayer(inp, 1, W=NormalSpec(sd=LognormalSpec()), nonlinearity=to.identity) 25 | out = DenseLayer(out, 1, W=NormalSpec(sd=LognormalSpec()), nonlinearity=to.identity) 26 | assert out.root is inp 27 | with out: 28 | pm.Normal('y', mu=get_output(out), 29 | sd=self.sd, 30 | observed=self.y) 31 | 32 | def test_find_parent_and_root(self): 33 | inp = InputLayer(self.x.shape) 34 | middle = llayers.DenseLayer(inp, 10) 35 | middle1 = DenseLayer(middle, 10) 36 | middle2 = llayers.DenseLayer(middle1, 4) 37 | 38 | assert find_parent(middle2) is middle1 39 | assert find_root(middle2) is inp 40 | -------------------------------------------------------------------------------- /gelato/tests/test_spec.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | from pymc3 import Normal, Lognormal, Model, fit 4 | import theano.tensor as tt 5 | from gelato.specs import * 6 | 7 | 8 | class TestSpec(object): 9 | def test_shape(self): 10 | spec = DistSpec(Normal, mu=0, sd=1) 11 | spec2 = DistSpec(Normal, mu=0, sd=DistSpec(Lognormal, 0, 1)) 12 | 13 | with Model('layer'): 14 | var = spec((100, 100), 'var') 15 | var2 = spec2((100, 100), 'var2') 16 | assert (var.init_value.shape == (100, 100)) 17 | assert (var.name.endswith('var')) 18 | assert (var2.init_value.shape == (100, 100)) 19 | assert (var2.name.endswith('var2')) 20 | 21 | 22 | _for_test = dict( 23 | UniformSpec=[ 24 | dict(), 25 | dict(lower=0, upper=1) 26 | ], 27 | NormalSpec=[ 28 | dict(), 29 | dict(mu=0, sd=1), 30 | ], 31 | BetaSpec=[ 32 | dict(), 33 | dict(alpha=1, beta=2) 34 | ], 35 | ExponentialSpec=[ 36 | dict(), 37 | dict(lam=10) 38 | ], 39 | LaplaceSpec=[ 40 | dict(), 41 | dict(mu=2, b=3) 42 | ], 43 | StudentTSpec=[ 44 | dict(nu=10), 45 | dict(nu=10, mu=5, sd=10), 46 | ], 47 | CauchySpec=[ 48 | dict(), 49 | dict(alpha=2, beta=5) 50 | ], 51 | HalfCauchySpec=[ 52 | dict(beta=5) 53 | ], 54 | GammaSpec=[ 55 | dict(alpha=2, beta=5), 56 | ], 57 | WeibullSpec=[ 58 | dict(alpha=2, beta=5) 59 | ], 60 | HalfStudentTSpec=[ 61 | dict(nu=10), 62 | dict(nu=10, mu=5, sd=10), 63 | ], 64 | LognormalSpec=[ 65 | dict(), 66 | dict(mu=0, sd=1), 67 | ], 68 | ChiSquaredSpec=[ 69 | dict(nu=10) 70 | ], 71 | HalfNormalSpec=[ 72 | dict(), 73 | dict(sd=1), 74 | ], 75 | WaldSpec=[ 76 | dict(mu=1, lam=1, alpha=2), 77 | ], 78 | ParetoSpec=[ 79 | dict(alpha=.5, m=2) 80 | ], 81 | InverseGammaSpec=[ 82 | dict(alpha=2), 83 | dict(alpha=.5, beta=2) 84 | ], 85 | ExGaussianSpec=[ 86 | dict(mu=0, sd=1, nu=2), 87 | ], 88 | VonMisesSpec=[ 89 | dict(mu=2, kappa=1) 90 | ], 91 | SkewNormalSpec=[ 92 | dict(), 93 | dict(mu=0, sd=1, alpha=2), 94 | ], 95 | NormalMixtureSpec=[ 96 | dict(w=[.1, .9], mu=[0, 0], sd=[.1, 10]), 97 | ] 98 | ) 99 | _skip = [ 100 | 'HalfStudentTSpec', 101 | 'NormalMixtureSpec' 102 | ] 103 | 104 | 105 | def setup_specs_kwargs(): 106 | from gelato.specs.dist import __all__ 107 | from gelato.specs import dist 108 | specs = [ 109 | getattr(dist, s) 110 | for s in __all__ 111 | if s in set(_for_test.keys()) - set(_skip) 112 | ] 113 | specs_kwargs = [] 114 | for spec in specs: 115 | specs_kwargs.extend([(spec, d) for d in _for_test[spec.__name__]]) 116 | return specs_kwargs 117 | 118 | 119 | @pytest.mark.parametrize( 120 | ['spec', 'kwargs'], 121 | setup_specs_kwargs() 122 | ) 123 | def test_spec(spec, kwargs): 124 | with Model(): 125 | assert ( 126 | spec(**kwargs)((1, 1)).tag.test_value.shape 127 | == 128 | (1, 1) 129 | ) 130 | assert ( 131 | spec(**kwargs)((10, 1)).tag.test_value.shape 132 | == 133 | (10, 1) 134 | ) 135 | assert ( 136 | spec(**kwargs)((10, 1, 10)).tag.test_value.shape 137 | == 138 | (10, 1, 10) 139 | ) 140 | 141 | 142 | def pseudo_random(shape): 143 | np.random.seed(sum(shape)) 144 | return np.random.randint(1, 5, size=shape).astype('int32') 145 | 146 | 147 | def _case4(): 148 | ss = NormalSpec().with_shape(lambda s: (s[0], 1)).dot(NormalSpec().with_shape(lambda s: (1, s[1]))) + LaplaceSpec() 149 | ss = ss*ss 150 | return ss 151 | 152 | 153 | def _case5(): 154 | lap = LaplaceSpec().with_shape(()) 155 | ss = NormalSpec().with_shape(lambda s: (s[0], 1)).dot(NormalSpec().with_shape(lambda s: (1, s[1]))) + lap 156 | ss = ss*ss 157 | return ss 158 | 159 | 160 | @pytest.mark.parametrize( 161 | 'expr', 162 | [ 163 | (NormalSpec() + LaplaceSpec()) / 100 - NormalSpec(), 164 | as_spec_op(tt.nlinalg.matrix_power)(NormalSpec() * LaplaceSpec(), 2) / 100 - NormalSpec(), 165 | NormalSpec().with_shape(lambda s: (s[0], 1)).dot(NormalSpec().with_shape(lambda s: (1, s[1]))) + LaplaceSpec(), 166 | _case4(), _case5() 167 | 168 | ] 169 | ) 170 | def test_expressions(expr): 171 | with Model() as model: 172 | var = expr((10, 10)) 173 | Normal('obs', observed=var) 174 | assert var.tag.test_value.shape == (10, 10) 175 | assert len(model.free_RVs) == 3 176 | fit(1) 177 | -------------------------------------------------------------------------------- /gelato/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.1' 2 | -------------------------------------------------------------------------------- /img/gelato.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ferrine/gelato/77828020e5cbfe5e9a753c5add79f49f284fc6a9/img/gelato.jpg -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | usefixtures = seeded -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | coveralls 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | git+git://github.com/Lasagne/Lasagne.git 2 | git+git://github.com/pymc-devs/pymc3.git 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import codecs 3 | import re 4 | import os 5 | 6 | here = os.path.abspath(os.path.dirname(__file__)) 7 | 8 | REQUIREMENTS = [ 9 | # use requirements.txt 10 | 'lasagne', 11 | # 'pymc3' 12 | ] 13 | REQUIREMENTS_DEV = [ 14 | 'pep8', 15 | 'coverage', 16 | 'nose' 17 | ] 18 | 19 | 20 | try: 21 | with codecs.open(os.path.join(here, 'README.md'), encoding='utf-8') as readme_file: 22 | LONG_DESCRIPTION = readme_file.read() 23 | 24 | with codecs.open(os.path.join(here, 'gelato', 'version.py'), encoding='utf-8') as version_file: 25 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file.read(), re.M) 26 | VERSION = version_match.group(1) 27 | except Exception: 28 | LONG_DESCRIPTION = '' 29 | VERSION = '' 30 | 31 | if __name__ == '__main__': 32 | setup( 33 | name='gelato', 34 | version=VERSION, 35 | packages=find_packages(), 36 | description='Bayesian dessert for Lasagne', 37 | long_description=LONG_DESCRIPTION, 38 | author='Maxim Kochurov', 39 | author_email='maxim.v.kochurov@gmail.com', 40 | download_url='https://github.com/ferrine/gelato', 41 | install_requires=REQUIREMENTS, 42 | tests_require=REQUIREMENTS_DEV 43 | ) 44 | --------------------------------------------------------------------------------