├── 2_notation_and_theory ├── img │ ├── .DS_Store │ ├── TwoDice.avif │ ├── HMMSequences.png │ ├── ProbMLCOVID.png │ └── InverseProblems.png ├── rise.css ├── Notation.md ├── State_Space_Model_Basics.md ├── Notation.ipynb ├── State_Space_Model_Basics.ipynb └── Bayesian_Basics.md ├── 1_kickoff_session └── README.md ├── 4_lgssms ├── Notes.md ├── Vectorization and Parallization.ipynb └── rise.css ├── environment.yml ├── SymbolList.md ├── 3_hmms ├── stockexample.py ├── HMMStateEstimation_Simple.py ├── rise.css ├── notes.md ├── HMMStateEstimation.py └── Discrete HMMs.ipynb ├── README.md ├── .gitignore ├── LICENSE └── JAX └── JaxGradient.ipynb /2_notation_and_theory/img/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/canyon289/ssm_book_club/HEAD/2_notation_and_theory/img/.DS_Store -------------------------------------------------------------------------------- /2_notation_and_theory/img/TwoDice.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/canyon289/ssm_book_club/HEAD/2_notation_and_theory/img/TwoDice.avif -------------------------------------------------------------------------------- /2_notation_and_theory/img/HMMSequences.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/canyon289/ssm_book_club/HEAD/2_notation_and_theory/img/HMMSequences.png -------------------------------------------------------------------------------- /2_notation_and_theory/img/ProbMLCOVID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/canyon289/ssm_book_club/HEAD/2_notation_and_theory/img/ProbMLCOVID.png -------------------------------------------------------------------------------- /1_kickoff_session/README.md: -------------------------------------------------------------------------------- 1 | # Kick off Session 2 | 3 | * [Dynamax Colab](https://colab.research.google.com/drive/1A2qCtUqjykoFWbawVtBAb9aEy0KpX8fr) 4 | -------------------------------------------------------------------------------- /2_notation_and_theory/img/InverseProblems.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/canyon289/ssm_book_club/HEAD/2_notation_and_theory/img/InverseProblems.png -------------------------------------------------------------------------------- /4_lgssms/Notes.md: -------------------------------------------------------------------------------- 1 | # Lesson Plan 2 | 3 | ## Prior Plan 4 | 1. Talk about gh filters 5 | 2. Relation to state space 6 | 3. Kalman filters 7 | * Cover notation of z, P, Q, and R 8 | 4. Implement dog example in dynamax 9 | 5. JAX jit and vmap 10 | * Show examples of each -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: ssm_book_club 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - jupyter 6 | - jupyterlab 7 | - jupytext 8 | - matplotlib 9 | - numpy 10 | - pandas 11 | - pip 12 | - pymc 13 | - python=3.10.4 14 | - scipy 15 | - seaborn 16 | - statsmodels 17 | - pip: 18 | - dynamax @ git+https://github.com/probml/dynamax 19 | - preliz 20 | - isort 21 | - rise 22 | 23 | 24 | -------------------------------------------------------------------------------- /SymbolList.md: -------------------------------------------------------------------------------- 1 | # Symbol List 2 | 3 | ## State Space Model 4 | * $y$ - Observations or emissions 5 | * $z$ - Hidden states 6 | * $t$ - Current timestep 7 | * $T$ - Final timestep 8 | * $A$ - Transition Matrix 9 | * $B$ - Emission Probability 10 | 11 | ## Nipunbatra Article 12 | * $x$ - Observations 13 | * $B$ - Emission Probability 14 | * $B$ - Emission Probability 15 | * $\phi$ - Transition Matrix 16 | * $\pi$ - Initial, or prior, Probability 17 | * $i$ and $j$ are states 18 | * $K$ is the set of all states 19 | 20 | ## Dynamax 21 | * $\alpha$ - Prior concentration 22 | -------------------------------------------------------------------------------- /3_hmms/stockexample.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from matplotlib import cm, pyplot as plt 3 | from matplotlib.dates import YearLocator, MonthLocator 4 | import yfinance as ys 5 | import pandas as pd 6 | 7 | # INTC = yf.Ticker("INTC") 8 | # INTC.get_shares_full() 9 | # hist = INTC.history(start="1995-01-01", end="2012-01-06") 10 | hist = pd.read_csv("stockprices.csv") 11 | emissions = hist["Close"].to_numpy() 12 | 13 | emissions = np.atleast_2d(emissions).T 14 | print(emissions.ndim) 15 | 16 | from dynamax.hidden_markov_model import DiagonalGaussianHMM 17 | 18 | import jax.numpy as jnp 19 | import jax.random as jr 20 | 21 | true_num_states = 4 22 | emission_dim = 1 23 | hmm = DiagonalGaussianHMM(true_num_states, emission_dim) 24 | 25 | key = jr.PRNGKey(0) 26 | hmm = DiagonalGaussianHMM(4, emission_dim, transition_matrix_stickiness=10.) 27 | params, props = hmm.initialize(key=key, method="kmeans", emissions=emissions) 28 | params, lps = hmm.fit_em(params, props, emissions, num_iters=100) -------------------------------------------------------------------------------- /3_hmms/HMMStateEstimation_Simple.py: -------------------------------------------------------------------------------- 1 | 2 | from functools import partial 3 | 4 | import jax.numpy as jnp 5 | import jax.random as jr 6 | import matplotlib.pyplot as plt 7 | from jax import vmap 8 | from jax.nn import one_hot 9 | 10 | from dynamax.hidden_markov_model import CategoricalHMM 11 | 12 | 13 | initial_probs = jnp.array([0.5, 0.5]) 14 | 15 | transition_matrix = jnp.array([[1.0, 0.0], 16 | [0.0, 1.0]]) 17 | 18 | # transition_matrix = jnp.array([[.5, .5], 19 | # [0.0, 1.0]]) 20 | 21 | emission_probs = jnp.array([[1/2, 1/2], # fair die 22 | [1/10, 9/10]]) # loaded die 23 | 24 | num_states = 2 # two types of dice (fair and loaded) 25 | num_emissions = 1 26 | num_classes = 2 27 | 28 | # Construct the HMM 29 | hmm = CategoricalHMM(num_states, num_emissions, num_classes) 30 | 31 | # Initialize the parameters struct with known values 32 | params, _ = hmm.initialize(initial_probs=initial_probs, 33 | transition_matrix=transition_matrix, 34 | emission_probs=emission_probs.reshape(num_states, num_emissions, num_classes)) 35 | 36 | num_timesteps =3 37 | true_states, emissions = hmm.sample(params, jr.PRNGKey(42), num_timesteps) 38 | print(true_states, emissions) 39 | 40 | num_batches = 5 41 | 42 | batch_states, batch_emissions = \ 43 | vmap(partial(hmm.sample, params, num_timesteps=num_timesteps))( 44 | jr.split(jr.PRNGKey(0), num_batches)) 45 | 46 | p0 = jnp.mean(emissions[true_states==0] + 1 == 6) # fair 47 | p1 = jnp.mean(emissions[true_states==1] + 1 == 6) # loaded 48 | 49 | posterior = hmm.filter(params, emissions) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # State Space Model Book Club 2 | 3 | ## Resources 4 | * **Community** - https://community.intuitivebayes.com/ 5 | * **Kick off Blog Post** - https://ravinkumar.com/ssm-book-club.html 6 | * **Livestreams** - https://www.youtube.com/@ravink/streams 7 | * **ProbML Book** - https://probml.github.io/pml-book/book2.html 8 | * Chapter 8 9 | * Chapter 9 10 | * Chapter 29 11 | 12 | ## Environment Setup 13 | The best way to learn is to get hands on with the code. 14 | Installing an environment gives you the most control. 15 | 16 | There are many python environment management tools, learn whatever tool you want, 17 | we'll be using conda by default. 18 | 19 | ### Create a Workspace 20 | 21 | Clone this repo 22 | ```bash 23 | git clone git@github.com:canyon289/ssm_book_club.git 24 | ``` 25 | 26 | Move into the repo: 27 | ```bash 28 | cd ssm_book_club 29 | ``` 30 | 31 | ### Create conda environment (not for Windows users - see below) 32 | 33 | #### Unix/Mac Users 34 | 35 | 1. Create the Conda Environment: 36 | ```bash 37 | conda env create -f environment.yml 38 | ``` 39 | 40 | 2. Activate the new environment: 41 | ```bash 42 | conda activate ssm_book_club 43 | ``` 44 | 45 | #### Windows Users 46 | 47 | The installation of the `dynamax` dependency `jaxlib` runs some issues on Windows. 48 | One possible workaround is the following: 49 | 50 | 1. Remove the pip dependencies at the bottom of the `environment.yml` file. 51 | ```bash 52 | - pip: 53 | - dynamax @ git+https://github.com/probml/dynamax 54 | ``` 55 | 56 | 2. Create the conda environment using: 57 | ```bash 58 | conda env create -f environment.yml 59 | ``` 60 | 61 | 3. Activate the new environment 62 | ```bash 63 | conda activate ssm_book_club 64 | ``` 65 | 66 | 4. Pip install jaxlib with, for example: 67 | ```bash 68 | pip install "jax[cpu]===0.3.25" -f https://whls.blob.core.windows.net/unstable/index.html --use-deprecated legacy-resolver 69 | ``` 70 | Note that this will install the CPU only version of jax (and jaxlib). 71 | If you are interested in adding GPU support see additional information here `https://github.com/cloudhan/jax-windows-builder#unstable-builds`. 72 | Version `0.3.25` is the latest available as of 2023-01-21. 73 | For more versions see the list at `https://whls.blob.core.windows.net/unstable/index.html` and release information at `https://github.com/google/jax/releases`. 74 | 75 | 5. Pip install dynamax with: 76 | ```bash 77 | pip install dynamax[notebooks] 78 | ``` 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | .DS_Store 131 | -------------------------------------------------------------------------------- /4_lgssms/Vectorization and Parallization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "3effc136-00e7-435f-92bd-ca2c7027ac50", 6 | "metadata": {}, 7 | "source": [ 8 | "# Jax Vectorization and Parallelization" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "74c329bf-b690-459b-a98f-bb1bac41467f", 14 | "metadata": {}, 15 | "source": [ 16 | "## Two styles\n", 17 | "* Vmap\n", 18 | "* Pmap" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "id": "dc4dbeae-96cf-4f84-b4cd-31c357ab2e08", 24 | "metadata": {}, 25 | "source": [ 26 | "## What if we had a lot of dogs?\n", 27 | "and a lot of sensors?" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "d6e06960", 33 | "metadata": {}, 34 | "source": [ 35 | "## Parallel Inference In Dynamax\n", 36 | "\n", 37 | "* https://probml.github.io/dynamax/notebooks/linear_gaussian_ssm/lgssm_parallel_inference.html\n", 38 | "* https://github.com/probml/dynamax/blob/main/dynamax/linear_gaussian_ssm/parallel_inference.py#L89" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "id": "aa2afc38-7e2c-42be-8020-e57d7e757ed8", 44 | "metadata": {}, 45 | "source": [ 46 | "## Docs\n", 47 | "* https://jax.readthedocs.io/en/latest/jax-101/03-vectorization.html\n", 48 | "* https://jax.readthedocs.io/en/latest/jax-101/06-parallelism.html" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "id": "21e5f8de-764a-4bbc-804c-6d1471797165", 54 | "metadata": {}, 55 | "source": [ 56 | "Conceptually, this is not very different from vectorisation, where the same operations occur in parallel in different parts of memory on the same device. We have already seen that vectorisation is supported in JAX as a program transformation, jax.vmap. JAX supports device parallelism analogously, using jax.pmap to transform a function written for one device into a function that runs in parallel on multiple devices. This colab will teach you all about it." 57 | ] 58 | } 59 | ], 60 | "metadata": { 61 | "kernelspec": { 62 | "display_name": "Python 3 (ipykernel)", 63 | "language": "python", 64 | "name": "python3" 65 | }, 66 | "language_info": { 67 | "codemirror_mode": { 68 | "name": "ipython", 69 | "version": 3 70 | }, 71 | "file_extension": ".py", 72 | "mimetype": "text/x-python", 73 | "name": "python", 74 | "nbconvert_exporter": "python", 75 | "pygments_lexer": "ipython3", 76 | "version": "3.9.16" 77 | } 78 | }, 79 | "nbformat": 4, 80 | "nbformat_minor": 5 81 | } 82 | -------------------------------------------------------------------------------- /3_hmms/rise.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,400&display=swap'); 2 | 3 | 4 | /* Fonts & Colors 5 | ------ 6 | * Black #2F3645 7 | * Accent Blue #0078EE 8 | * Callout Yellow #FBC02D 9 | * Highlight Blue #9BDDFB 10 | * Error Red #E76E62 11 | * Highlight Green #4CD5C5 12 | */ 13 | .reveal h2, .reveal h3, .reveal h4, .reveal h5, .reveal h6 { 14 | font-family: "Poppins", "Open Sans", sans-serif; 15 | font-weight: 700; 16 | margin: 0.2rem 0 0 0 ; 17 | color: #00524b; 18 | } 19 | 20 | .reveal p { 21 | font-family: "Open Sans", sans-serif; 22 | margin: 0.2rem 0 0 0 ; 23 | color: #12221D; 24 | } 25 | 26 | .reveal h1 { 27 | font-family: "Poppins", "Open Sans", sans-serif; 28 | margin: 0.2rem 0 0 0 ; 29 | color: #00524b; 30 | } 31 | .reveal h1:before { /* Remove if buggy with slide with H1 (alignment or size) */ 32 | content: ''; 33 | float: right; 34 | width: 30%; 35 | height: calc(30vh); 36 | } 37 | .reveal p strong { 38 | color: #00524b; 39 | } 40 | ::selection { 41 | background-color: #9BDDFB; 42 | color: #FFFFFF; 43 | } 44 | ul li::marker { 45 | color: #56BB92; 46 | } 47 | 48 | /* Links */ 49 | .rise-enabled .reveal a { 50 | color: #2F3645; 51 | text-decoration: none; 52 | } 53 | .reveal .present a { 54 | border: 1px #56BB92 solid; 55 | border-radius: 4rem; 56 | background-color: #FFFFFF !important; 57 | padding: 0.4rem 4rem; 58 | max-width: 100%; 59 | font-size: 0.6em; 60 | } 61 | .reveal .present a:hover { 62 | border: 1px #FBC02D solid; 63 | background-color: #FBC02D !important; 64 | color: #FFFFFF; 65 | -webkit-transition: 0.15s ease-in; 66 | transition: 0.15s ease-in; 67 | } 68 | .reveal .present a::selection { 69 | background-color: #FFFFFF; 70 | color: #2F3645; 71 | } 72 | 73 | /* Format cell */ 74 | div.text_cell_render.rendered_html { 75 | background-color: rgba(255, 255, 255, 0.6); 76 | } 77 | 78 | Background 79 | .reveal .slide-background-content { 80 | opacity: 20%; 81 | background-image: url("img/background_slide.png"); 82 | } 83 | 84 | /* Arrow */ 85 | .controls-arrow { 86 | opacity: 0; 87 | } 88 | 89 | /* Slide Number */ 90 | .slide-number-a {} 91 | .slide-number-delimiter { 92 | visibility: hidden; 93 | } 94 | .slide-number-b { 95 | visibility: hidden; 96 | } 97 | 98 | /*Center output images*/ 99 | .rise-enabled .reveal section img { 100 | display: block; 101 | margin-left: auto; 102 | margin-right: auto; 103 | } 104 | 105 | -------------------------------------------------------------------------------- /4_lgssms/rise.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,400&display=swap'); 2 | 3 | 4 | /* Fonts & Colors 5 | ------ 6 | * Black #2F3645 7 | * Accent Blue #0078EE 8 | * Callout Yellow #FBC02D 9 | * Highlight Blue #9BDDFB 10 | * Error Red #E76E62 11 | * Highlight Green #4CD5C5 12 | */ 13 | .reveal h2, .reveal h3, .reveal h4, .reveal h5, .reveal h6 { 14 | font-family: "Poppins", "Open Sans", sans-serif; 15 | font-weight: 700; 16 | margin: 0.2rem 0 0 0 ; 17 | color: #00524b; 18 | } 19 | 20 | .reveal p { 21 | font-family: "Open Sans", sans-serif; 22 | margin: 0.2rem 0 0 0 ; 23 | color: #12221D; 24 | } 25 | 26 | .reveal h1 { 27 | font-family: "Poppins", "Open Sans", sans-serif; 28 | margin: 0.2rem 0 0 0 ; 29 | color: #00524b; 30 | } 31 | .reveal h1:before { /* Remove if buggy with slide with H1 (alignment or size) */ 32 | content: ''; 33 | float: right; 34 | width: 30%; 35 | height: calc(30vh); 36 | } 37 | .reveal p strong { 38 | color: #00524b; 39 | } 40 | ::selection { 41 | background-color: #9BDDFB; 42 | color: #FFFFFF; 43 | } 44 | ul li::marker { 45 | color: #56BB92; 46 | } 47 | 48 | /* Links */ 49 | .rise-enabled .reveal a { 50 | color: #2F3645; 51 | text-decoration: none; 52 | } 53 | .reveal .present a { 54 | border: 1px #56BB92 solid; 55 | border-radius: 4rem; 56 | background-color: #FFFFFF !important; 57 | padding: 0.4rem 4rem; 58 | max-width: 100%; 59 | font-size: 0.6em; 60 | } 61 | .reveal .present a:hover { 62 | border: 1px #FBC02D solid; 63 | background-color: #FBC02D !important; 64 | color: #FFFFFF; 65 | -webkit-transition: 0.15s ease-in; 66 | transition: 0.15s ease-in; 67 | } 68 | .reveal .present a::selection { 69 | background-color: #FFFFFF; 70 | color: #2F3645; 71 | } 72 | 73 | /* Format cell */ 74 | div.text_cell_render.rendered_html { 75 | background-color: rgba(255, 255, 255, 0.6); 76 | } 77 | 78 | Background 79 | .reveal .slide-background-content { 80 | opacity: 20%; 81 | background-image: url("img/background_slide.png"); 82 | } 83 | 84 | /* Arrow */ 85 | .controls-arrow { 86 | opacity: 0; 87 | } 88 | 89 | /* Slide Number */ 90 | .slide-number-a {} 91 | .slide-number-delimiter { 92 | visibility: hidden; 93 | } 94 | .slide-number-b { 95 | visibility: hidden; 96 | } 97 | 98 | /*Center output images*/ 99 | .rise-enabled .reveal section img { 100 | display: block; 101 | margin-left: auto; 102 | margin-right: auto; 103 | } 104 | 105 | -------------------------------------------------------------------------------- /2_notation_and_theory/rise.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,400&display=swap'); 2 | 3 | 4 | /* Fonts & Colors 5 | ------ 6 | * Black #2F3645 7 | * Accent Blue #0078EE 8 | * Callout Yellow #FBC02D 9 | * Highlight Blue #9BDDFB 10 | * Error Red #E76E62 11 | * Highlight Green #4CD5C5 12 | */ 13 | .reveal h2, .reveal h3, .reveal h4, .reveal h5, .reveal h6 { 14 | font-family: "Poppins", "Open Sans", sans-serif; 15 | font-weight: 700; 16 | margin: 0.2rem 0 0 0 ; 17 | color: #00524b; 18 | } 19 | 20 | .reveal p { 21 | font-family: "Open Sans", sans-serif; 22 | margin: 0.2rem 0 0 0 ; 23 | color: #12221D; 24 | } 25 | 26 | .reveal h1 { 27 | font-family: "Poppins", "Open Sans", sans-serif; 28 | margin: 0.2rem 0 0 0 ; 29 | color: #00524b; 30 | } 31 | .reveal h1:before { /* Remove if buggy with slide with H1 (alignment or size) */ 32 | content: ''; 33 | float: right; 34 | width: 30%; 35 | height: calc(30vh); 36 | } 37 | .reveal p strong { 38 | color: #00524b; 39 | } 40 | ::selection { 41 | background-color: #9BDDFB; 42 | color: #FFFFFF; 43 | } 44 | ul li::marker { 45 | color: #56BB92; 46 | } 47 | 48 | /* Links */ 49 | .rise-enabled .reveal a { 50 | color: #2F3645; 51 | text-decoration: none; 52 | } 53 | .reveal .present a { 54 | border: 1px #56BB92 solid; 55 | border-radius: 4rem; 56 | background-color: #FFFFFF !important; 57 | padding: 0.4rem 4rem; 58 | max-width: 100%; 59 | font-size: 0.6em; 60 | } 61 | .reveal .present a:hover { 62 | border: 1px #FBC02D solid; 63 | background-color: #FBC02D !important; 64 | color: #FFFFFF; 65 | -webkit-transition: 0.15s ease-in; 66 | transition: 0.15s ease-in; 67 | } 68 | .reveal .present a::selection { 69 | background-color: #FFFFFF; 70 | color: #2F3645; 71 | } 72 | 73 | /* Format cell */ 74 | div.text_cell_render.rendered_html { 75 | background-color: rgba(255, 255, 255, 0.6); 76 | } 77 | 78 | Background 79 | .reveal .slide-background-content { 80 | opacity: 20%; 81 | background-image: url("img/background_slide.png"); 82 | } 83 | 84 | /* Arrow */ 85 | .controls-arrow { 86 | opacity: 0; 87 | } 88 | 89 | /* Slide Number */ 90 | .slide-number-a {} 91 | .slide-number-delimiter { 92 | visibility: hidden; 93 | } 94 | .slide-number-b { 95 | visibility: hidden; 96 | } 97 | 98 | /*Center output images*/ 99 | .rise-enabled .reveal section img { 100 | display: block; 101 | margin-left: auto; 102 | margin-right: auto; 103 | } 104 | 105 | -------------------------------------------------------------------------------- /3_hmms/notes.md: -------------------------------------------------------------------------------- 1 | # General Notes 2 | 3 | 1. HMM Evidence Likelihood 4 | 2. Forward algorithm - Used to calculate the probability were in a state. Assume we know prior, transition, and emission probabilities 5 | 3. Backward algorithm 6 | 4. Viterbi algorithm 7 | 5. Parameter learning 8 | 9 | ## HMM Evidence Likelihood 10 | * The website uses this notation $L(X \vert \theta)$ 11 | * By the time we get here $P(HHH \vert z_{3}=F) = \Big[P(HH \vert z_{2}=F)*A_{FF}+P(HH \vert z_{2}=B)*A_{BF}\Big] \phi(H \vert F)$ we only care about two things. 12 | 1. The probability of the sequences that got us the last two heads 13 | 2. The emission probability at that time 14 | * The thing is to calculate the probability of how we got those two heads we recurse down into that probability 15 | 16 | * $\alpha_t(i) = P(X_{1:t}\vert z_{t}=i)$ is the probability of being in state 'i' at the time 't' given the 'observations till time t'. 17 | * We can figure this out by summing a bunch of branches 18 | * I think dot means dot product? 19 | 20 | 21 | # Book CLub 22 | * Understand the point of SSMs like filtering etc 23 | * Be able to name the algorithms 24 | * Identify them in the Dynamax codebase 25 | * Detail specific things like dirichlet priors 26 | 27 | 28 | ## State Estimation Filtering 29 | 30 | ### Instantiate an HMM class with initial parameters 31 | https://github.com/probml/dynamax/blob/9650ee9940f229f6ea8349cb6288a3f4f41fec02/dynamax/hidden_markov_model/models/categorical_hmm.py#L30 32 | 33 | Those parameters are 34 | * Initial probability 35 | * Transition Matrix 36 | * Emission probabilities 37 | * Prior Concentration (This one is optional) 38 | 39 | In this model we're not inferring or estimating any of these paramters. 40 | It is important to note though that the initial probability is 41 | mutated a bit under the hood with a dirichlet prior. 42 | 43 | ### Call initialize 44 | https://github.com/probml/dynamax/blob/9650ee9940f229f6ea8349cb6288a3f4f41fec02/dynamax/hidden_markov_model/models/categorical_hmm.py#L59 45 | 46 | This constructs the priors from the MLE provided. 47 | Currently implements Dirichlet for sampling 48 | This then samples from the prior to get the values. 49 | 50 | ## Then calls filter 51 | This is what I need to learn. 52 | * Get transition matrix 53 | * The log likelihoods seem to be time varying 54 | * Figure out how these are calculated 55 | * Learn how condition on is implemented 56 | * Predict 57 | 58 | * Read more on JAX jit 59 | * https://jax.readthedocs.io/en/latest/jax-101/02-jitting.html 60 | 61 | * Do the first couple by hand to really get an idea of whats going on 62 | * This will be a great use of the book club 63 | * There's a lot of probabilities going on 64 | * Emissions, transition, state 65 | * In this filtering function we are trying to estimate prob of state 66 | -------------------------------------------------------------------------------- /3_hmms/HMMStateEstimation.py: -------------------------------------------------------------------------------- 1 | 2 | from functools import partial 3 | 4 | import jax.numpy as jnp 5 | import jax.random as jr 6 | import matplotlib.pyplot as plt 7 | from jax import vmap 8 | from jax.nn import one_hot 9 | 10 | from dynamax.hidden_markov_model import CategoricalHMM 11 | 12 | 13 | initial_probs = jnp.array([0.5, 0.5]) 14 | transition_matrix = jnp.array([[0.95, 0.05], 15 | [0.10, 0.90]]) 16 | 17 | # transition_matrix = jnp.array([[1.0, 0.0], 18 | # [0.0, 1.0]]) 19 | 20 | emission_probs = jnp.array([[1/6, 1/6, 1/6, 1/6, 1/6, 1/6], # fair die 21 | [1/10, 1/10, 1/10, 1/10, 1/10, 5/10]]) # loaded die 22 | 23 | emission_probs = jnp.array([[1/2, 1/2], # fair die 24 | [1/10, 9/10]]) # loaded die 25 | 26 | print(f"A.shape: {transition_matrix.shape}") 27 | print(f"B.shape: {emission_probs.shape}") 28 | 29 | num_states = 2 # two types of dice (fair and loaded) 30 | num_emissions = 1 # only one die is rolled at a time 31 | num_classes = 2 # each die has six faces 32 | 33 | # Construct the HMM 34 | hmm = CategoricalHMM(num_states, num_emissions, num_classes) 35 | 36 | # Initialize the parameters struct with known values 37 | params, _ = hmm.initialize(initial_probs=initial_probs, 38 | transition_matrix=transition_matrix, 39 | emission_probs=emission_probs.reshape(num_states, num_emissions, num_classes)) 40 | 41 | num_timesteps =3 42 | true_states, emissions = hmm.sample(params, jr.PRNGKey(42), num_timesteps) 43 | print(true_states, emissions) 44 | # print(f"true_states.shape: {true_states.shape}") 45 | # print(f"emissions.shape: {emissions.shape}") 46 | # print("") 47 | # print("First few states: ", true_states[:5]) 48 | # print("First few emissions: ", emissions[:5, 0]) 49 | 50 | # To sample multiple sequences, just use vmap 51 | # In this case 5 sequences were modeled 52 | num_batches = 5 53 | 54 | batch_states, batch_emissions = \ 55 | vmap(partial(hmm.sample, params, num_timesteps=num_timesteps))( 56 | jr.split(jr.PRNGKey(0), num_batches)) 57 | 58 | # print(f"batch_states.shape: {batch_states.shape}") 59 | # print(f"batch_emissions.shape: {batch_emissions.shape}") 60 | 61 | # count fraction of times we see 6 in each state 62 | # remember that python is zero-indexed, so we have to add one! 63 | p0 = jnp.mean(emissions[true_states==0] + 1 == 6) # fair 64 | p1 = jnp.mean(emissions[true_states==1] + 1 == 6) # loaded 65 | # print("empirical frequencies: ", jnp.array([p0, p1])) 66 | # print("expected frequencies: ", emission_probs[:, -1]) 67 | 68 | posterior = hmm.filter(params, emissions) 69 | # print(f"marginal likelihood: {posterior.marginal_loglik: .2f}") 70 | # print(f"posterior.filtered_probs.shape: {posterior.filtered_probs.shape}") 71 | 72 | 73 | # Manual first step calcu 74 | # Probability of being on the loaded state -------------------------------------------------------------------------------- /2_notation_and_theory/Notation.md: -------------------------------------------------------------------------------- 1 | --- 2 | jupyter: 3 | jupytext: 4 | formats: ipynb,md 5 | text_representation: 6 | extension: .md 7 | format_name: markdown 8 | format_version: '1.3' 9 | jupytext_version: 1.14.4 10 | kernelspec: 11 | display_name: Python 3 (ipykernel) 12 | language: python 13 | name: python3 14 | --- 15 | 16 | 17 | # How I Learn Notation 18 | 19 | 20 | 21 | ## Why Notation 22 | 23 | 24 | 1. It's the most compact and precise way to represent math 25 | 2. There's no way to avoid it with the texts we've picked 26 | 27 | 28 | ## Full State Space Model Equation 29 | 30 | 31 | $$p(y_{1:T}, z_{1:T} | u_{1:T}) = p(z_1 | u_1) p(y_1 | z_1, u_1) \prod_{t=1}^T p(z_t | z_{t-1}, u_t) p(y_t | z_t, u_t)$$ 32 | 33 | 34 | ProbML Equation 29.5 35 | 36 | 37 | ## Bayes Theorem 38 | 39 | 40 | $$ \underbrace{p(\boldsymbol{\theta} \mid \boldsymbol{Y})}_{\text{posterior}} = \frac{\overbrace{p(\boldsymbol{Y} \mid \boldsymbol{\theta})}^{\text{likelihood}}; \overbrace{p(\boldsymbol{\theta})}^{\text{prior}}}{\underbrace{{{\int_{\boldsymbol{\Theta}} p(\boldsymbol{Y} \mid \boldsymbol{\theta})p(\boldsymbol{\theta}) d\boldsymbol{\theta}}}}_{\text{marginal likelihood}}} $$ 41 | 42 | 43 | ## My steps 44 | 45 | 46 | 1. Understand the fundamental philosophy 47 | 2. Find the simplest applied example 48 | * Typically in code 49 | 3. Implement it myself 50 | 4. Add complexity one step at a time 51 | 52 | 53 | This is what we did just did for Bayes Theorem 54 | 55 | 56 | 57 | ## Bayes Theorem Simplified 58 | 59 | 60 | $$ 61 | \text{Posterior} = \frac{\text{Likelihood} ; * \text{Prior}}{\text{Marginal-Likelihood}} 62 | $$ 63 | 64 | 65 | From there we built back up to Linear Regression 66 | 67 | 68 | 69 | ## Things I looks for 70 | 71 | 72 | 1. What type of math am I dealing with? 73 | * Probability 74 | * Distributions 75 | * Integrals etc 76 | 2. Figure out what's a scalar, vector, matrix 77 | * Figure out whats a real number, indicator, random variable 78 | 3. Pay attention to shapes 79 | 80 | 81 | ## State Space Model Simplified Version 82 | 83 | 84 | $$ 85 | \begin{align} 86 | p(y_{1:T}, z_{1:T} \mid \theta) 87 | &= \mathrm{Cat}(z_1 \mid \pi) 88 | \prod_{t=2}^T \mathrm{Cat}(z_t \mid A_{z_{t-1}}) 89 | \prod_{t=1}^T \mathrm{Cat}(y_t \mid B_{z_t}) 90 | \end{align} 91 | $$ 92 | 93 | 94 | ## What I suggest for you 95 | 96 | 97 | 1. Develop a strategy that works for you 98 | * Use mine, or find the combination that works for you 99 | 2. Utilize code, examples, notecards whatever 100 | * Use the SSM community 101 | 3. Don't get frustrated, you can do it 102 | * It's like learning a new language 103 | 104 | 105 | ## Recap 106 | * Notation can be challenging but is important 107 | * Break it down one step at a time 108 | * Learn the meaning not just the symbols 109 | * Building things in code really help me 110 | 111 | 112 | -------------------------------------------------------------------------------- /2_notation_and_theory/State_Space_Model_Basics.md: -------------------------------------------------------------------------------- 1 | --- 2 | jupyter: 3 | jupytext: 4 | formats: ipynb,md 5 | text_representation: 6 | extension: .md 7 | format_name: markdown 8 | format_version: '1.3' 9 | jupytext_version: 1.14.4 10 | kernelspec: 11 | display_name: Python 3 (ipykernel) 12 | language: python 13 | name: python3 14 | --- 15 | 16 | 17 | # State Space Models 18 | 19 | 20 | 21 | ## Two Parts 22 | 23 | 24 | 25 | 1. State 26 | 27 | 28 | 29 | 2. Space 30 | 31 | 32 | 33 | * State - The hidden thing we can't observe 34 | * Also called transition model 35 | * Space - Where we see outcomes 36 | * or emissions 37 | 38 | 39 | 40 | ## Simplified State Space Model 41 | 42 | 43 | $$ 44 | \begin{align} 45 | p(y_{1:T}, z_{1:T} \mid \theta) 46 | &= \mathrm{Cat}(z_1 \mid \pi) 47 | \prod_{t=2}^T \mathrm{Cat}(z_t \mid A_{z_{t-1}}) 48 | \prod_{t=1}^T \mathrm{Cat}(y_t \mid B_{z_t}) 49 | \end{align} 50 | $$ 51 | 52 | 53 | ## Parts of the equation 54 | 55 | 56 | $$ 57 | \begin{align} 58 | p(y_{1:T}, z_{1:T} \mid \theta) 59 | &= \overbrace{\mathrm{Cat}(z_1 \mid \pi)}^{Prior for Initial State} 60 | \prod_{t=2}^T \mathrm{Cat}(z_t \mid A_{z_{t-1}}) 61 | \prod_{t=1}^T \mathrm{Cat}(y_t \mid B_{z_t}) 62 | \end{align} 63 | $$ 64 | 65 | 66 | ## Parts of the equation 67 | 68 | 69 | $$ 70 | \begin{align} 71 | p(y_{1:T}, z_{1:T} \mid \theta) 72 | &= \overbrace{\mathrm{Cat}(z_1 \mid \pi)}^{Prior for Initial State} 73 | \underbrace{\prod_{t=2}^T \mathrm{Cat}(z_t \mid A_{z_{t-1}})}_{Transition Model} 74 | \prod_{t=1}^T \mathrm{Cat}(y_t \mid B_{z_t}) 75 | \end{align} 76 | $$ 77 | 78 | 79 | Transmission model or dynamics model 80 | 81 | 82 | ## Parts of the equation 83 | 84 | 85 | $$ 86 | \begin{align} 87 | p(y_{1:T}, z_{1:T} \mid \theta) 88 | &= \overbrace{\mathrm{Cat}(z_1 \mid \pi)}^{Prior for Initial State} 89 | \underbrace{\prod_{t=2}^T \mathrm{Cat}(z_t \mid A_{z_{t-1}})}_{Transition Model} 90 | \overbrace{\prod_{t=1}^T \mathrm{Cat}(y_t \mid B_{z_t})}^{Observation Model} 91 | \end{align} 92 | $$ 93 | 94 | 95 | Observation model or emissions model 96 | 97 | 98 | ## Hidden Markov Model Notation 99 | $$ 100 | \begin{align} 101 | p(y_{1:T}, z_{1:T} \mid \theta) 102 | &= \overbrace{\mathrm{Cat}(z_1 \mid \pi)}^{Prior for Initial State} 103 | \underbrace{\prod_{t=2}^T \mathrm{Cat}(z_t \mid A_{z_{t-1}})}_{Transition Model} 104 | \overbrace{\prod_{t=1}^T \mathrm{Cat}(y_t \mid B_{z_t})}^{Observation Model} 105 | \end{align} 106 | $$ 107 | 108 | 109 | 110 | 111 | $$\theta = (\pi, A, B)$$ 112 | 113 | $$A - \text{Transition Matrix}$$ 114 | $$B - \text{Emission Probability}$$ 115 | $$\pi - \text{Initial Probability}$$ 116 | 117 | 118 | ## State Space Sequence 119 |
120 | 121 |
122 | 123 | 124 | 125 | ## Dishonest Casino 126 | 127 | 128 |
129 | 130 |
131 | 132 | 133 | ## Dishonest Casino Setup 134 | 135 | 136 | The dealer has two dice 137 | * One dice is fair 138 | * One dice is biased 139 | * The dealer swaps the dice at random 140 | * **Does not mean uniform random** 141 | 142 | 143 | ## Dishonest Casino HMM Questions 144 | 145 | 146 | * How can we tell which dice is in use based on the outcomes? 147 | * While were watching live (online filtering) 148 | * A replay after the fact (offline smoothing) 149 | * What will the next dice rolls be? (Future observations forecasting/prediction) 150 | * what will the next dice dice in use?(Future state forecasting/prediction) 151 | 152 | 153 | ## HMM questions generalized 154 | 155 | 156 | If we see a bunch of outcomes can we 157 | * What will happen? 158 | * Estimate what state we are in, or were in, or will be in 159 | * Estimate what we will see next 160 | 161 | * What's the underlying truth of the world? 162 | * Estimate the transition matrix parameters 163 | * Estimate the emission model parameter 164 | * Estimate model the starting point 165 | 166 | 167 | ## State Space Model Recap 168 | 169 | 170 | * State and Space are primary terms 171 | * There are other equally valid terms terms like 172 | * Emissions 173 | * Observation 174 | * Transition 175 | * Dynamics 176 | 177 | * In using SSMs we may be more interested in 178 | * The hidden system state 179 | * Either what's next or what's to come 180 | * After the fact or during the sequencing process 181 | * What we well see next for observations 182 | * EStimating the model parameters 183 | * We're starting with discrete HMM 184 | * No autoregressive dependency 185 | * No covariate 186 | * Discrete states 187 | 188 | 189 | ## Reading Next Time 190 | Full post on Discourse 191 | 192 | * Casino HMMs from Dynamax 193 | * Complete reading of https://nipunbatra.github.io/hmm/ 194 | * Filtering (forwards algorithm) 195 | * Smoothing (forwards-backwards algorithm) 196 | * Most likely state sequence (Viterbi algorithm) 197 | 198 | -------------------------------------------------------------------------------- /2_notation_and_theory/Notation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "cba7c90b", 6 | "metadata": { 7 | "slideshow": { 8 | "slide_type": "slide" 9 | } 10 | }, 11 | "source": [ 12 | "# How I Learn Notation" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "d1f20832", 18 | "metadata": { 19 | "slideshow": { 20 | "slide_type": "slide" 21 | } 22 | }, 23 | "source": [ 24 | "## Why Notation" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "9012973e", 30 | "metadata": {}, 31 | "source": [ 32 | "1. It's the most compact and precise way to represent math\n", 33 | "2. There's no way to avoid it with the texts we've picked" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "id": "994f707c", 39 | "metadata": { 40 | "slideshow": { 41 | "slide_type": "slide" 42 | } 43 | }, 44 | "source": [ 45 | "## Full State Space Model Equation" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "id": "9cb901e5", 51 | "metadata": {}, 52 | "source": [ 53 | "$$p(y_{1:T}, z_{1:T} | u_{1:T}) = p(z_1 | u_1) p(y_1 | z_1, u_1) \\prod_{t=1}^T p(z_t | z_{t-1}, u_t) p(y_t | z_t, u_t)$$" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "id": "df58f3ca", 59 | "metadata": {}, 60 | "source": [ 61 | "ProbML Equation 29.5" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "id": "9c0a2cab", 67 | "metadata": { 68 | "slideshow": { 69 | "slide_type": "slide" 70 | } 71 | }, 72 | "source": [ 73 | "## Bayes Theorem" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "id": "bc55a43f", 79 | "metadata": {}, 80 | "source": [ 81 | "$$ \\underbrace{p(\\boldsymbol{\\theta} \\mid \\boldsymbol{Y})}_{\\text{posterior}} = \\frac{\\overbrace{p(\\boldsymbol{Y} \\mid \\boldsymbol{\\theta})}^{\\text{likelihood}}; \\overbrace{p(\\boldsymbol{\\theta})}^{\\text{prior}}}{\\underbrace{{{\\int_{\\boldsymbol{\\Theta}} p(\\boldsymbol{Y} \\mid \\boldsymbol{\\theta})p(\\boldsymbol{\\theta}) d\\boldsymbol{\\theta}}}}_{\\text{marginal likelihood}}} $$" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "8648ac10", 87 | "metadata": { 88 | "slideshow": { 89 | "slide_type": "slide" 90 | } 91 | }, 92 | "source": [ 93 | "## My steps" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "id": "48d89271", 99 | "metadata": {}, 100 | "source": [ 101 | "1. Understand the fundamental philosophy\n", 102 | "2. Find the simplest applied example\n", 103 | " * Typically in code\n", 104 | "3. Implement it myself\n", 105 | "4. Add complexity one step at a time" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "id": "609a0515", 111 | "metadata": { 112 | "slideshow": { 113 | "slide_type": "fragment" 114 | } 115 | }, 116 | "source": [ 117 | "This is what we did just did for Bayes Theorem" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "id": "d55ebc65", 123 | "metadata": { 124 | "slideshow": { 125 | "slide_type": "slide" 126 | } 127 | }, 128 | "source": [ 129 | "## Bayes Theorem Simplified" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "id": "92beccd8", 135 | "metadata": {}, 136 | "source": [ 137 | "$$\n", 138 | "\\text{Posterior} = \\frac{\\text{Likelihood} ; * \\text{Prior}}{\\text{Marginal-Likelihood}}\n", 139 | "$$" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "id": "3515b654", 145 | "metadata": { 146 | "slideshow": { 147 | "slide_type": "fragment" 148 | } 149 | }, 150 | "source": [ 151 | "From there we built back up to Linear Regression" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "id": "a1687474", 157 | "metadata": { 158 | "slideshow": { 159 | "slide_type": "slide" 160 | } 161 | }, 162 | "source": [ 163 | "## Things I looks for" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "id": "c786a801", 169 | "metadata": {}, 170 | "source": [ 171 | "1. What type of math am I dealing with?\n", 172 | " * Probability\n", 173 | " * Distributions\n", 174 | " * Integrals etc\n", 175 | "2. Figure out what's a scalar, vector, matrix\n", 176 | " * Figure out whats a real number, indicator, random variable\n", 177 | "3. Pay attention to shapes" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "id": "823f0460", 183 | "metadata": { 184 | "slideshow": { 185 | "slide_type": "slide" 186 | } 187 | }, 188 | "source": [ 189 | "## State Space Model Simplified Version" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "id": "cdeabe14", 195 | "metadata": {}, 196 | "source": [ 197 | "$$\n", 198 | "\\begin{align}\n", 199 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 200 | "&= \\mathrm{Cat}(z_1 \\mid \\pi) \n", 201 | "\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}}) \n", 202 | "\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})\n", 203 | "\\end{align}\n", 204 | "$$" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "id": "8ad8f343", 210 | "metadata": { 211 | "slideshow": { 212 | "slide_type": "slide" 213 | } 214 | }, 215 | "source": [ 216 | "## What I suggest for you" 217 | ] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "id": "2322c5c9", 222 | "metadata": {}, 223 | "source": [ 224 | "1. Develop a strategy that works for you\n", 225 | " * Use mine, or find the combination that works for you\n", 226 | "2. Utilize code, examples, notecards whatever\n", 227 | " * Use the SSM community\n", 228 | "3. Don't get frustrated, you can do it\n", 229 | " * It's like learning a new language" 230 | ] 231 | }, 232 | { 233 | "cell_type": "markdown", 234 | "id": "72ab1c5a", 235 | "metadata": { 236 | "slideshow": { 237 | "slide_type": "slide" 238 | } 239 | }, 240 | "source": [ 241 | "## Recap\n", 242 | "* Notation can be challenging but is important\n", 243 | "* Break it down one step at a time\n", 244 | "* Learn the meaning not just the symbols\n", 245 | "* Building things in code really help me\n" 246 | ] 247 | } 248 | ], 249 | "metadata": { 250 | "celltoolbar": "Slideshow", 251 | "jupytext": { 252 | "formats": "ipynb,md" 253 | }, 254 | "kernelspec": { 255 | "display_name": "Python 3 (ipykernel)", 256 | "language": "python", 257 | "name": "python3" 258 | }, 259 | "language_info": { 260 | "codemirror_mode": { 261 | "name": "ipython", 262 | "version": 3 263 | }, 264 | "file_extension": ".py", 265 | "mimetype": "text/x-python", 266 | "name": "python", 267 | "nbconvert_exporter": "python", 268 | "pygments_lexer": "ipython3", 269 | "version": "3.10.4" 270 | }, 271 | "rise": { 272 | "auto_select": "none", 273 | "enable_chalkboard": true, 274 | "scroll": true 275 | } 276 | }, 277 | "nbformat": 4, 278 | "nbformat_minor": 5 279 | } 280 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /2_notation_and_theory/State_Space_Model_Basics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "6b960713", 6 | "metadata": { 7 | "slideshow": { 8 | "slide_type": "slide" 9 | } 10 | }, 11 | "source": [ 12 | "# State Space Models" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "215a2196", 18 | "metadata": { 19 | "slideshow": { 20 | "slide_type": "slide" 21 | } 22 | }, 23 | "source": [ 24 | "## Two Parts" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "id": "e475fda8", 30 | "metadata": { 31 | "slideshow": { 32 | "slide_type": "fragment" 33 | } 34 | }, 35 | "source": [ 36 | "1. State" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "c0684769", 42 | "metadata": { 43 | "slideshow": { 44 | "slide_type": "fragment" 45 | } 46 | }, 47 | "source": [ 48 | "2. Space" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "id": "e7e21007", 54 | "metadata": { 55 | "slideshow": { 56 | "slide_type": "skip" 57 | } 58 | }, 59 | "source": [ 60 | "* State - The hidden thing we can't observe\n", 61 | " * Also called transition model\n", 62 | "* Space - Where we see outcomes\n", 63 | " * or emissions" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "id": "69c5b489", 69 | "metadata": { 70 | "slideshow": { 71 | "slide_type": "slide" 72 | } 73 | }, 74 | "source": [ 75 | "## Simplified State Space Model" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "id": "8f13b3b7", 81 | "metadata": {}, 82 | "source": [ 83 | "$$\n", 84 | "\\begin{align}\n", 85 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 86 | "&= \\mathrm{Cat}(z_1 \\mid \\pi) \n", 87 | "\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}}) \n", 88 | "\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})\n", 89 | "\\end{align}\n", 90 | "$$" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "id": "5c880603", 96 | "metadata": { 97 | "slideshow": { 98 | "slide_type": "slide" 99 | } 100 | }, 101 | "source": [ 102 | "## Parts of the equation" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "id": "458ae128", 108 | "metadata": {}, 109 | "source": [ 110 | "$$\n", 111 | "\\begin{align}\n", 112 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 113 | "&= \\overbrace{\\mathrm{Cat}(z_1 \\mid \\pi)}^{Prior for Initial State}\n", 114 | "\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}})\n", 115 | "\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})\n", 116 | "\\end{align}\n", 117 | "$$" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "id": "c0c8e809", 123 | "metadata": { 124 | "slideshow": { 125 | "slide_type": "slide" 126 | } 127 | }, 128 | "source": [ 129 | "## Parts of the equation" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "id": "8149c98c", 135 | "metadata": {}, 136 | "source": [ 137 | "$$\n", 138 | "\\begin{align}\n", 139 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 140 | "&= \\overbrace{\\mathrm{Cat}(z_1 \\mid \\pi)}^{Prior for Initial State}\n", 141 | "\\underbrace{\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}})}_{Transition Model}\n", 142 | "\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})\n", 143 | "\\end{align}\n", 144 | "$$" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "id": "f0392352", 150 | "metadata": {}, 151 | "source": [ 152 | "Transmission model or dynamics model" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "id": "d7f41b09", 158 | "metadata": { 159 | "slideshow": { 160 | "slide_type": "slide" 161 | } 162 | }, 163 | "source": [ 164 | "## Parts of the equation" 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "id": "baef781b", 170 | "metadata": {}, 171 | "source": [ 172 | "$$\n", 173 | "\\begin{align}\n", 174 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 175 | "&= \\overbrace{\\mathrm{Cat}(z_1 \\mid \\pi)}^{Prior for Initial State}\n", 176 | "\\underbrace{\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}})}_{Transition Model}\n", 177 | "\\overbrace{\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})}^{Observation Model}\n", 178 | "\\end{align}\n", 179 | "$$" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "id": "fa76d413", 185 | "metadata": {}, 186 | "source": [ 187 | "Observation model or emissions model" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "id": "8df79812", 193 | "metadata": { 194 | "slideshow": { 195 | "slide_type": "slide" 196 | } 197 | }, 198 | "source": [ 199 | "## Hidden Markov Model Notation\n", 200 | "$$\n", 201 | "\\begin{align}\n", 202 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 203 | "&= \\overbrace{\\mathrm{Cat}(z_1 \\mid \\pi)}^{Prior for Initial State}\n", 204 | "\\underbrace{\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}})}_{Transition Model}\n", 205 | "\\overbrace{\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})}^{Observation Model}\n", 206 | "\\end{align}\n", 207 | "$$\n", 208 | "\n" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "id": "8f1290f4", 214 | "metadata": {}, 215 | "source": [ 216 | "$$\\theta = (\\pi, A, B)$$\n", 217 | "\n", 218 | "$$A - \\text{Transition Matrix}$$\n", 219 | "$$B - \\text{Emission Probability}$$\n", 220 | "$$\\pi - \\text{Initial Probability}$$" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "id": "d4ab0fe1", 226 | "metadata": { 227 | "slideshow": { 228 | "slide_type": "slide" 229 | } 230 | }, 231 | "source": [ 232 | "## State Space Sequence\n", 233 | "
\n", 234 | " \n", 235 | "
" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "id": "d5802174", 241 | "metadata": { 242 | "slideshow": { 243 | "slide_type": "slide" 244 | } 245 | }, 246 | "source": [ 247 | "## Dishonest Casino" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "id": "2c345251", 253 | "metadata": {}, 254 | "source": [ 255 | "
\n", 256 | " \n", 257 | "
" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "id": "fccb7cd0", 263 | "metadata": { 264 | "slideshow": { 265 | "slide_type": "slide" 266 | } 267 | }, 268 | "source": [ 269 | "## Dishonest Casino Setup" 270 | ] 271 | }, 272 | { 273 | "cell_type": "markdown", 274 | "id": "72523a3d", 275 | "metadata": {}, 276 | "source": [ 277 | "The dealer has two dice\n", 278 | "* One dice is fair\n", 279 | "* One dice is biased\n", 280 | "* The dealer swaps the dice at random\n", 281 | " * **Does not mean uniform random**" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "id": "5e4b6fb8", 287 | "metadata": { 288 | "slideshow": { 289 | "slide_type": "slide" 290 | } 291 | }, 292 | "source": [ 293 | "## Dishonest Casino HMM Questions" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "id": "b8e57572", 299 | "metadata": {}, 300 | "source": [ 301 | "* How can we tell which dice is in use based on the outcomes?\n", 302 | " * While were watching live (online filtering)\n", 303 | " * A replay after the fact (offline smoothing)\n", 304 | "* What will the next dice rolls be? (Future observations forecasting/prediction)\n", 305 | "* what will the next dice dice in use?(Future state forecasting/prediction)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "id": "24ce9b0b", 311 | "metadata": { 312 | "slideshow": { 313 | "slide_type": "slide" 314 | } 315 | }, 316 | "source": [ 317 | "## HMM questions generalized" 318 | ] 319 | }, 320 | { 321 | "cell_type": "markdown", 322 | "id": "f894c45e", 323 | "metadata": {}, 324 | "source": [ 325 | "If we see a bunch of outcomes can we\n", 326 | "* What will happen?\n", 327 | " * Estimate what state we are in, or were in, or will be in\n", 328 | " * Estimate what we will see next\n", 329 | " \n", 330 | "* What's the underlying truth of the world?\n", 331 | " * Estimate the transition matrix parameters\n", 332 | " * Estimate the emission model parameter\n", 333 | " * Estimate model the starting point" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "id": "b1d4871b", 339 | "metadata": { 340 | "slideshow": { 341 | "slide_type": "slide" 342 | } 343 | }, 344 | "source": [ 345 | "## State Space Model Recap" 346 | ] 347 | }, 348 | { 349 | "cell_type": "markdown", 350 | "id": "9ffe405d", 351 | "metadata": {}, 352 | "source": [ 353 | "* State and Space are primary terms\n", 354 | "* There are other equally valid terms terms like\n", 355 | " * Emissions\n", 356 | " * Observation\n", 357 | " * Transition\n", 358 | " * Dynamics\n", 359 | "\n", 360 | "* In using SSMs we may be more interested in\n", 361 | " * The hidden system state\n", 362 | " * Either what's next or what's to come\n", 363 | " * After the fact or during the sequencing process\n", 364 | " * What we well see next for observations\n", 365 | " * EStimating the model parameters\n", 366 | "* We're starting with discrete HMM\n", 367 | " * No autoregressive dependency\n", 368 | " * No covariate\n", 369 | " * Discrete states" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "id": "60bc0737", 375 | "metadata": { 376 | "slideshow": { 377 | "slide_type": "slide" 378 | } 379 | }, 380 | "source": [ 381 | "## Reading Next Time\n", 382 | "Full post on Discourse\n", 383 | "\n", 384 | "* Casino HMMs from Dynamax\n", 385 | "* Complete reading of https://nipunbatra.github.io/hmm/\n", 386 | "* Filtering (forwards algorithm)\n", 387 | "* Smoothing (forwards-backwards algorithm)\n", 388 | "* Most likely state sequence (Viterbi algorithm)" 389 | ] 390 | } 391 | ], 392 | "metadata": { 393 | "celltoolbar": "Slideshow", 394 | "jupytext": { 395 | "formats": "ipynb,md" 396 | }, 397 | "kernelspec": { 398 | "display_name": "Python 3 (ipykernel)", 399 | "language": "python", 400 | "name": "python3" 401 | }, 402 | "language_info": { 403 | "codemirror_mode": { 404 | "name": "ipython", 405 | "version": 3 406 | }, 407 | "file_extension": ".py", 408 | "mimetype": "text/x-python", 409 | "name": "python", 410 | "nbconvert_exporter": "python", 411 | "pygments_lexer": "ipython3", 412 | "version": "3.10.4" 413 | }, 414 | "rise": { 415 | "auto_select": "none", 416 | "enable_chalkboard": true, 417 | "scroll": true 418 | } 419 | }, 420 | "nbformat": 4, 421 | "nbformat_minor": 5 422 | } 423 | -------------------------------------------------------------------------------- /2_notation_and_theory/Bayesian_Basics.md: -------------------------------------------------------------------------------- 1 | --- 2 | jupyter: 3 | jupytext: 4 | formats: ipynb,md 5 | text_representation: 6 | extension: .md 7 | format_name: markdown 8 | format_version: '1.3' 9 | jupytext_version: 1.14.4 10 | kernelspec: 11 | display_name: Python 3 (ipykernel) 12 | language: python 13 | name: python3 14 | --- 15 | 16 | 17 | # State Space Model Book Club 18 | 19 | 20 | ```python slideshow={"slide_type": "skip"} 21 | import numpy as np 22 | import pymc as pm 23 | import arviz as az 24 | import pandas as pd 25 | import preliz as pz 26 | 27 | import matplotlib.pyplot as plt 28 | ``` 29 | 30 | 31 | ## Agenda 32 | 33 | 34 | Three notebooks 35 | 1. Bayesian Basics (This one) 36 | 2. Notation 37 | 3. State Space Model Basics 38 | 39 | 40 | ## Bayes Formula 41 | 42 | 43 | $$ \underbrace{p(\boldsymbol{\theta} \mid \boldsymbol{Y})}_{\text{posterior}} = \frac{\overbrace{p(\boldsymbol{Y} \mid \boldsymbol{\theta})}^{\text{likelihood}}; \overbrace{p(\boldsymbol{\theta})}^{\text{prior}}}{\underbrace{{{\int_{\boldsymbol{\Theta}} p(\boldsymbol{Y} \mid \boldsymbol{\theta})p(\boldsymbol{\theta}) d\boldsymbol{\theta}}}}_{\text{marginal likelihood}}} $$ 44 | 45 | 46 | ## Bayesian Update Intuition 47 | 48 | 49 | Much simpler than the formula 50 | 51 | 52 | 1. You have some prior belief 53 | * It may be opinionated/informmed 54 | * It may not 55 | 56 | 57 | 58 | 2. You get some data 59 | 60 | 61 | 62 | 63 | 3. You update your beliefs 64 | 65 | 66 | 67 | ## Simplified Bayes Formula 68 | 69 | 70 | $$ 71 | \text{Posterior} = \frac{\text{Likelihood} * \text{Prior}}{\text{Marginal-Likelihood}} 72 | $$ 73 | 74 | 75 | * **Prior** - Your beliefs prior to seeing the data 76 | * **Likelihood** - How believable the data is given a set of model parameters 77 | * **Posterior** - Your beliefs after combining the two 78 | 79 | * **Marginal Likelihood** - A term you need to calculate proper probabilities but in practice you largely ignore 80 | 81 | 82 | ## The typical problems Bayes 101 problems 83 | 84 | 85 | * Coin Flips 86 | * COVID 87 | * Monty Hall Problem 88 | 89 | 90 | ## Example from ProbML 91 | 92 | 93 | 94 |
95 | 96 |
97 | 98 | 99 | 100 | ## Covid Prevalance 101 | 102 | 103 | $$ 104 | P(SF) = \text{10%} \\ 105 | P(\text{~}SF) = \text{90%} 106 | $$ 107 | 108 | 109 | ## Likelihood of positive test 110 | 111 | 112 | $$ 113 | P(PT | SF) = \text{Chance of a positive test given the person has space-flu.} \\ 114 | P(PT | \text{~}SF) = \text{Chance of a positive test given the person doesn’t have space flu.} 115 | $$ 116 | 117 | 118 | $$ 119 | P(PT \mid SF) = \text{90%} \\ 120 | P(PT \mid \text{~}SF) = \text{20%} 121 | $$ 122 | 123 | 124 | 125 | ## Covid in code 126 | 127 | 128 | ```python slideshow={"slide_type": "fragment"} 129 | prior = [.1, .9] # P(SF), P(~SF) 130 | likelihood = [.9, .2] # P(PT | SF), p(PT | ~SF) 131 | ``` 132 | 133 | ```python slideshow={"slide_type": "fragment"} 134 | unnormalized_posterior = [None, None] 135 | unnormalized_posterior[0] = likelihood[0]*prior[0] 136 | unnormalized_posterior[1] = likelihood[1]*prior[1] 137 | ``` 138 | 139 | ```python slideshow={"slide_type": "fragment"} 140 | marginal_likelihood = likelihood[0]*prior[0] + likelihood[1]*prior[1] 141 | ``` 142 | 143 | ```python slideshow={"slide_type": "fragment"} 144 | posterior = [None, None] 145 | posterior[0] = unnormalized_posterior[0] / marginal_likelihood 146 | posterior[1] = unnormalized_posterior[1] / marginal_likelihood 147 | posterior 148 | ``` 149 | 150 | 151 | ## COVID Example Visualized 152 | 153 | 154 | ```python hide_input=true 155 | from IPython.display import HTML, IFrame 156 | 157 | HTML('') 158 | ``` 159 | 160 | 161 | ## Inverse Problems 162 | 163 | 164 | 165 |
166 | 167 |
168 | 169 | 170 | 171 | ## We see something, what do we learn? 172 | Not, we know something (probability values) what is the probability of some subevent occurring? 173 | 174 | 175 | 176 | ## User Conversion Probability on a website? 177 | 178 | 179 | * A 100 visitors visit us 180 | * 8 Convert 181 | 182 | What do we believe about unobservable conversion rate? 183 | 184 | 185 | ## What are **possible conversion rates**? 186 | 187 | 188 | Possible conversion rates 189 | 190 | * 0% 191 | * 8% 192 | * 20% 193 | * 31% 194 | * 88% 195 | * 99% 196 | 197 | 198 | Anything between 0% and 1 % is possible 199 | 200 | 201 | 202 | ## What is the plausibility of the conversion rates? 203 | 204 | 205 | 206 | ### Lets start with priors 207 | 208 | 209 | ```python 210 | pz.Beta(1, 1).plot_pdf(figsize=(20,8)); 211 | ``` 212 | 213 | 214 | ## Bayesian Update 215 | 216 | 217 | ```python 218 | num_conversions = 8 219 | num_non_conversions = 100 - num_conversions 220 | 221 | # The bayesian update is happening right here 222 | pz.Beta(1+num_conversions, 1+num_non_conversions).plot_pdf(); 223 | ``` 224 | 225 | 226 | Made possible through "pen and paper" mathematics and people that are smart at math 227 | 228 | 229 | 230 | ## Relative Plausibility of all possible beliefs 231 | 232 | 233 | ```python 234 | num_conversions = 8 235 | num_visits = 100 236 | num_non_conversions = num_visits - num_conversions 237 | pz.Beta(2+8, 2+num_non_conversions).plot_pdf(); 238 | ``` 239 | 240 | 241 | ## Bayesian Update with a PPL 242 | 243 | 244 | ```python 245 | with pm.Model() as model: 246 | θ = pm.Beta("θ", 1, 1) 247 | y = pm.Binomial("y", n=num_visits, p=θ, observed=num_conversions) 248 | trace = pm.sample() 249 | ``` 250 | ```python slideshow={"slide_type": "-"} 251 | az.plot_trace(trace); 252 | ``` 253 | 254 | Made possible through computers and other people who are smart at math 255 | 256 | 257 | ## What's the difference 258 | 259 | 260 | 261 | 262 | * Conjugate Model - Pure "pen on paper math" 263 | * No computer needed 264 | * Exact 265 | * Very restricted to specific prior likelihood combinations 266 | 267 | 268 | 269 | * Markov Chain Monte Carlo algorithms 270 | * Not very practical without computers 271 | * Enables 272 | * Generally applicable 273 | 274 | 275 | 276 | ## Relation to this book club 277 | 278 | 279 | * Various SSMs have "pen and paper" solutions 280 | * With tools like Dynamax different and more complex models may be solvable 281 | * Dynamax supports MCMC through blackjax so we might see this ater 282 | 283 | 284 | We want to learn both the traditional techniques **and** what newer tools like JAX and Dynamax let us solve 285 | 286 | 287 | 288 | ## Bayesian Linear Regression of Penguins 289 | 290 | 291 | ```python 292 | penguins_url = "https://gist.githubusercontent.com/slopp/ce3b90b9168f2f921784de84fa445651/raw/4ecf3041f0ed4913e7c230758733948bc561f434/penguins.csv" 293 | penguins = pd.read_csv(penguins_url) 294 | # Subset to the columns needed 295 | missing_data = penguins.isnull()[ 296 | ["bill_length_mm", "flipper_length_mm", "sex", "body_mass_g"] 297 | ].any(axis=1) 298 | # Drop rows with any missing data 299 | penguins = penguins.loc[~missing_data] 300 | 301 | adelie_mask = (penguins["species"] == "Adelie") 302 | adelie_mass_obs = penguins.loc[adelie_mask, "body_mass_g"].values 303 | adelie_flipper_length_obs = penguins.loc[adelie_mask, "flipper_length_mm"] 304 | 305 | ``` 306 | 307 | 308 | ## Make a plot 309 | 310 | 311 | ```python 312 | fig, ax = plt.subplots() 313 | 314 | 315 | ax.scatter(adelie_flipper_length_obs, adelie_mass_obs) 316 | ax.set_xlabel('Flipper Length') 317 | ax.set_ylabel('Mass'); 318 | ``` 319 | 320 | 321 | ## Bayesian Regression 322 | 323 | 324 | ```python 325 | 326 | with pm.Model() as model_adelie_flipper_regression: 327 | # pm.Data allows us to change the underlying value in a later code block 328 | adelie_flipper_length = pm.Data("adelie_flipper_length", 329 | adelie_flipper_length_obs) 330 | σ = pm.HalfStudentT("σ", 100, 2000) 331 | β_0 = pm.Normal("β_0", 0, 4000) 332 | β_1 = pm.Normal("β_1", 0, 4000) 333 | μ = pm.Deterministic("μ", β_0 + β_1 * adelie_flipper_length) 334 | 335 | mass = pm.Normal("mass", mu=μ, sigma=σ, observed = adelie_mass_obs) 336 | 337 | inf_data_adelie_flipper_regression = pm.sample(return_inferencedata=True) 338 | 339 | ``` 340 | 341 | 342 | ## Regression with Uncertainty bounds 343 | 344 | 345 | ```python 346 | fig, ax = plt.subplots() 347 | alpha_m = inf_data_adelie_flipper_regression.posterior.mean().to_dict()["data_vars"]["β_0"]["data"] 348 | beta_m = inf_data_adelie_flipper_regression.posterior.mean().to_dict()["data_vars"]["β_1"]["data"] 349 | 350 | flipper_length = np.linspace(adelie_flipper_length_obs.min(), adelie_flipper_length_obs.max(), 100) 351 | 352 | flipper_length_mean = alpha_m + beta_m * flipper_length 353 | ax.plot(flipper_length, flipper_length_mean, c='g', lw=4, 354 | label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x') 355 | 356 | ax.scatter(adelie_flipper_length_obs, adelie_mass_obs) 357 | 358 | # Figure out how to do this from inference data 359 | az.plot_hdi(adelie_flipper_length_obs, inf_data_adelie_flipper_regression.posterior['μ'], hdi_prob=0.94, color='k', ax=ax) 360 | 361 | ax.set_xlabel('Flipper Length') 362 | ax.set_ylabel('Mass'); 363 | ``` 364 | 365 | 366 | ## Dynamax Book Club Takeaway 367 | 368 | 369 | * Bayes theorem is a philosophy for how we can update our beliefs given observations 370 | * ProbML calls outcome -> belief mapping inverse probability 371 | * What we care about is the relative plausibiilty 372 | * For the same model there can be different estimators 373 | * Each as their tradeoffs 374 | * Computer enables newer ones not possible in the past 375 | -------------------------------------------------------------------------------- /3_hmms/Discrete HMMs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1f7b8444", 6 | "metadata": { 7 | "slideshow": { 8 | "slide_type": "slide" 9 | } 10 | }, 11 | "source": [ 12 | "# Discrete HMMs" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "1d828aa9", 18 | "metadata": { 19 | "slideshow": { 20 | "slide_type": "slide" 21 | } 22 | }, 23 | "source": [ 24 | "## Agenda\n", 25 | "* Key ideas and words\n", 26 | "* HMM Overview\n", 27 | " * What confused me\n", 28 | "* How I'm approaching this\n", 29 | "* Going through examples\n", 30 | " * State estimation\n", 31 | " * Parameter estimation\n", 32 | "* Where I would use Dynamax in my past career\n", 33 | " * Practical considerations\n", 34 | "* Updated Book Club Focus" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "6bb17537", 40 | "metadata": { 41 | "slideshow": { 42 | "slide_type": "slide" 43 | } 44 | }, 45 | "source": [ 46 | "## Summary\n", 47 | "* HMM can be confusing, especially between implementations and texts\n", 48 | "* Dynamax's implementation is quite good\n", 49 | " * Their docs are great for practitioners\n", 50 | "* JAX is used liberally throughout the library for speedups\n", 51 | "* There's many different implementations\n", 52 | " * More than I had originally expected" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "id": "1b06f411", 58 | "metadata": { 59 | "slideshow": { 60 | "slide_type": "slide" 61 | } 62 | }, 63 | "source": [ 64 | "## References\n", 65 | "* This repo https://github.com/canyon289/ssm_book_club\n", 66 | "* Casino HMMs from Dynamax\n", 67 | " * https://probml.github.io/dynamax/notebooks/hmm/casino_hmm_inference.html\n", 68 | " * https://probml.github.io/dynamax/notebooks/hmm/casino_hmm_learning.html\n", 69 | "* Complete reading of https://nipunbatra.github.io/hmm/\n" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "id": "b182026c", 75 | "metadata": { 76 | "slideshow": { 77 | "slide_type": "slide" 78 | } 79 | }, 80 | "source": [ 81 | "# Key ideas and words" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "6ce62e8a", 87 | "metadata": { 88 | "slideshow": { 89 | "slide_type": "slide" 90 | } 91 | }, 92 | "source": [ 93 | "## Discrete HMM\n", 94 | "\n", 95 | "$$\n", 96 | "\\begin{align}\n", 97 | "p(y_{1:T}, z_{1:T} \\mid \\theta) \n", 98 | "&= \\overbrace{\\mathrm{Cat}(z_1 \\mid \\pi)}^{Prior for Initial State}\n", 99 | "\\underbrace{\\prod_{t=2}^T \\mathrm{Cat}(z_t \\mid A_{z_{t-1}})}_{Transition Model}\n", 100 | "\\overbrace{\\prod_{t=1}^T \\mathrm{Cat}(y_t \\mid B_{z_t})}^{Observation Model}\n", 101 | "\\end{align}\n", 102 | "$$\n" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "id": "75f8e583", 108 | "metadata": { 109 | "slideshow": { 110 | "slide_type": "slide" 111 | } 112 | }, 113 | "source": [ 114 | "## Terminology\n", 115 | "* Forward Filter - Estimating state probability using only \"seen\" data\n", 116 | "* Prediction - Estimating the next time step\n", 117 | "* Smoothing Estimating state probability using all data\n", 118 | " * Forward Backward Pass\n", 119 | "* Viterbi Algorithm - Max\n", 120 | " * Sequence Assignment" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "id": "249e08c0", 126 | "metadata": { 127 | "slideshow": { 128 | "slide_type": "slide" 129 | } 130 | }, 131 | "source": [ 132 | "## Symbols\n", 133 | "\n", 134 | "https://github.com/canyon289/ssm_book_club/blob/hmms/SymbolList.md#nipunbatra-article" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "id": "986aced5", 140 | "metadata": { 141 | "slideshow": { 142 | "slide_type": "slide" 143 | } 144 | }, 145 | "source": [ 146 | "## Things that were challenging to me\n", 147 | "* Change of symbols between sources and withing texts\n", 148 | " * Duplication of symbols in same text\n", 149 | " * Python variable names that correlated to these terms\n", 150 | "* Differing usage of words\n", 151 | "* Ambiguous terms\n", 152 | " * Especially in the code\n", 153 | "* Abstractions in Dynamax\n", 154 | " * Required lots of tracing" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "353cf75a", 160 | "metadata": { 161 | "slideshow": { 162 | "slide_type": "slide" 163 | } 164 | }, 165 | "source": [ 166 | "## Various Time dependencies ( or Lack thereof)\n", 167 | "* Things that are independent of time\n", 168 | " * Transition Matrix\n", 169 | " * Prior Probability\n", 170 | " * Emission Prob assuming state\n", 171 | " \n", 172 | "* Things dependent on a particular time window but independent of others\n", 173 | " * Observations\n", 174 | " * Log likelihood\n", 175 | "\n", 176 | "* Things that change over time\n", 177 | " * State estimation\n", 178 | " * Emission probability after updating state probability\n" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "id": "f636c911", 184 | "metadata": { 185 | "slideshow": { 186 | "slide_type": "slide" 187 | } 188 | }, 189 | "source": [ 190 | "## How I'm learning\n", 191 | "Referencing\n", 192 | "* External articles\n", 193 | "* ProbML book\n", 194 | "* Diving into the code\n", 195 | "* Writing my own mini examples" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "id": "1d12b415", 201 | "metadata": { 202 | "slideshow": { 203 | "slide_type": "slide" 204 | } 205 | }, 206 | "source": [ 207 | "## Applied Example, Filtering, Smoothing - Biased coin toss" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 1, 213 | "id": "0aab565e", 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "import jax.numpy as jnp\n", 218 | "import jax.random as jr\n", 219 | "# import matplotlib.pyplot as plt\n", 220 | "from jax import vmap" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 2, 226 | "id": "30c9d44f", 227 | "metadata": {}, 228 | "outputs": [ 229 | { 230 | "name": "stdout", 231 | "output_type": "stream", 232 | "text": [ 233 | "This a modified version of dynamax for the HMM Book club session\n", 234 | "Code can be found here https://github.com/canyon289/dynamax/tree/hmm_session\n" 235 | ] 236 | }, 237 | { 238 | "name": "stderr", 239 | "output_type": "stream", 240 | "text": [ 241 | "No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" 242 | ] 243 | } 244 | ], 245 | "source": [ 246 | "%load_ext autoreload\n", 247 | "%autoreload 2\n", 248 | "import dynamax\n", 249 | "from dynamax.hidden_markov_model import CategoricalHMM" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 3, 255 | "id": "4f44d370", 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [ 259 | "initial_probs = jnp.array([0.5, 0.5])\n", 260 | "\n", 261 | "transition_matrix = jnp.array([[1.0, 0.0],\n", 262 | " [0.0, 1.0]])" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 4, 268 | "id": "7500f474", 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "initial_probs = jnp.array([0.5, 0.5])\n", 273 | "\n", 274 | "transition_matrix = jnp.array([[1.0, 0.0],\n", 275 | " [0.0, 1.0]])" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": 5, 281 | "id": "11124313", 282 | "metadata": {}, 283 | "outputs": [], 284 | "source": [ 285 | "num_states = 2 # two types of dice (fair and loaded)\n", 286 | "num_emissions = 1\n", 287 | "num_classes = 2" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 6, 293 | "id": "00162056", 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "emission_probs = jnp.array([[1/2, 1/2], # fair die\n", 298 | " [1/10, 9/10]]) # loaded di" 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": 7, 304 | "id": "ee2fd561", 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "# A bunch of stuff happens here, like log likelihood calculations and such\n", 309 | "hmm = CategoricalHMM(num_states, num_emissions, num_classes)\n", 310 | "\n", 311 | "# Initialize the parameters struct with known values\n", 312 | "params, _ = hmm.initialize(initial_probs=initial_probs,\n", 313 | " transition_matrix=transition_matrix,\n", 314 | " emission_probs=emission_probs.reshape(num_states, num_emissions, num_classes))" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 8, 320 | "id": "a0fde3cc", 321 | "metadata": {}, 322 | "outputs": [ 323 | { 324 | "data": { 325 | "text/plain": [ 326 | "(Array([1, 1, 1], dtype=int32),\n", 327 | " Array([[1],\n", 328 | " [1],\n", 329 | " [1]], dtype=int32))" 330 | ] 331 | }, 332 | "execution_count": 8, 333 | "metadata": {}, 334 | "output_type": "execute_result" 335 | } 336 | ], 337 | "source": [ 338 | "num_timesteps = 3\n", 339 | "true_states, emissions = hmm.sample(params, jr.PRNGKey(42), num_timesteps)\n", 340 | "true_states, emissions" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": 9, 346 | "id": "7b8a8c56", 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "name": "stdout", 351 | "output_type": "stream", 352 | "text": [ 353 | "log_probs; [[-0.6931472 -0.10536057]\n", 354 | " [-0.6931472 -0.10536057]\n", 355 | " [-0.6931472 -0.10536057]]\n", 356 | "Iteration: 0\n", 357 | "predicted_probs: [0.5 0.5]\n", 358 | "Log Likelihood of emission given state: [-0.6931472 -0.10536057]\n", 359 | "Filtered probs: [0.35714287 0.64285713]\n", 360 | "Predicted probs: [0.35714287 0.64285713]\n", 361 | "\n", 362 | "\n", 363 | "Iteration: 1\n", 364 | "predicted_probs: [0.35714287 0.64285713]\n", 365 | "Log Likelihood of emission given state: [-0.6931472 -0.10536057]\n", 366 | "Filtered probs: [0.23584908 0.7641509 ]\n", 367 | "Predicted probs: [0.23584908 0.7641509 ]\n", 368 | "\n", 369 | "\n", 370 | "Iteration: 2\n", 371 | "predicted_probs: [0.23584908 0.7641509 ]\n", 372 | "Log Likelihood of emission given state: [-0.6931472 -0.10536057]\n", 373 | "Filtered probs: [0.14637005 0.85362995]\n", 374 | "Predicted probs: [0.14637005 0.85362995]\n", 375 | "\n", 376 | "\n" 377 | ] 378 | } 379 | ], 380 | "source": [ 381 | "posterior = hmm.filter(params, emissions)" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "id": "0584dd97", 387 | "metadata": { 388 | "slideshow": { 389 | "slide_type": "slide" 390 | } 391 | }, 392 | "source": [ 393 | "## Log Likelihoods Verification\n", 394 | "Verify what we're seeing above" 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "execution_count": 13, 400 | "id": "e3b7616b", 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "data": { 405 | "text/plain": [ 406 | "array([-0.69314718, -0.10536052])" 407 | ] 408 | }, 409 | "execution_count": 13, 410 | "metadata": {}, 411 | "output_type": "execute_result" 412 | } 413 | ], 414 | "source": [ 415 | "# Log likelihood verification\n", 416 | "from scipy import stats\n", 417 | "stats.bernoulli([.5, .9]).logpmf(1)" 418 | ] 419 | }, 420 | { 421 | "cell_type": "markdown", 422 | "id": "b9b209e6", 423 | "metadata": { 424 | "slideshow": { 425 | "slide_type": "slide" 426 | } 427 | }, 428 | "source": [ 429 | "## Bayesian Update for Filtering\n", 430 | "Calculating Probability that we're in biased coin state after one coin toss\n", 431 | "\n", 432 | "$$p(Biased | one heads) = p(Biased | x=H) $$" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 17, 438 | "id": "e39cd73f", 439 | "metadata": {}, 440 | "outputs": [ 441 | { 442 | "data": { 443 | "text/plain": [ 444 | "0.6428571428571429" 445 | ] 446 | }, 447 | "execution_count": 17, 448 | "metadata": {}, 449 | "output_type": "execute_result" 450 | } 451 | ], 452 | "source": [ 453 | "# (Likelihood of heads assumed biased) * prior/(total probability of heads)\n", 454 | "p_biased_state_one_heads = (.9*.5)/(.9*.5 + .5*.5)\n", 455 | "p_biased_state_one_heads" 456 | ] 457 | }, 458 | { 459 | "cell_type": "code", 460 | "execution_count": 18, 461 | "id": "67423ce5", 462 | "metadata": {}, 463 | "outputs": [ 464 | { 465 | "data": { 466 | "text/plain": [ 467 | "0.7641509433962265" 468 | ] 469 | }, 470 | "execution_count": 18, 471 | "metadata": {}, 472 | "output_type": "execute_result" 473 | } 474 | ], 475 | "source": [ 476 | "(.9*p_biased_state_one_heads)/(.9*p_biased_state_one_heads + .5*(1-p_biased_state_one_heads))" 477 | ] 478 | }, 479 | { 480 | "cell_type": "markdown", 481 | "id": "d7ba259a", 482 | "metadata": {}, 483 | "source": [ 484 | "Predicted probs stays the same because of filtering" 485 | ] 486 | }, 487 | { 488 | "cell_type": "markdown", 489 | "id": "6d7108a8", 490 | "metadata": { 491 | "slideshow": { 492 | "slide_type": "slide" 493 | } 494 | }, 495 | "source": [ 496 | "### Non identity transition matrix" 497 | ] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": 26, 502 | "id": "218be38c", 503 | "metadata": {}, 504 | "outputs": [], 505 | "source": [ 506 | "transition_matrix = jnp.array([[.5, .5],\n", 507 | " [0.0, 1.0]])" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 27, 513 | "id": "4775dbe9", 514 | "metadata": {}, 515 | "outputs": [], 516 | "source": [ 517 | "# Initialize the parameters struct with known values\n", 518 | "params, _ = hmm.initialize(initial_probs=initial_probs,\n", 519 | " transition_matrix=transition_matrix,\n", 520 | " emission_probs=emission_probs.reshape(num_states, num_emissions, num_classes))" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": 28, 526 | "id": "eeb32d11", 527 | "metadata": {}, 528 | "outputs": [ 529 | { 530 | "name": "stdout", 531 | "output_type": "stream", 532 | "text": [ 533 | "log_probs; [[-0.6931472 -0.10536057]\n", 534 | " [-0.6931472 -0.10536057]\n", 535 | " [-0.6931472 -0.10536057]]\n", 536 | "Iteration: 0\n", 537 | "predicted_probs: [0.5 0.5]\n", 538 | "Log Likelihood of emission given state: [-0.6931472 -0.10536057]\n", 539 | "Filtered probs: [0.35714287 0.64285713]\n", 540 | "Predicted probs: [0.17857143 0.82142854]\n", 541 | "\n", 542 | "\n", 543 | "Iteration: 1\n", 544 | "predicted_probs: [0.17857143 0.82142854]\n", 545 | "Log Likelihood of emission given state: [-0.6931472 -0.10536057]\n", 546 | "Filtered probs: [0.10775863 0.8922414 ]\n", 547 | "Predicted probs: [0.05387932 0.94612074]\n", 548 | "\n", 549 | "\n", 550 | "Iteration: 2\n", 551 | "predicted_probs: [0.05387932 0.94612074]\n", 552 | "Log Likelihood of emission given state: [-0.6931472 -0.10536057]\n", 553 | "Filtered probs: [0.03066732 0.96933264]\n", 554 | "Predicted probs: [0.01533366 0.9846663 ]\n", 555 | "\n", 556 | "\n" 557 | ] 558 | } 559 | ], 560 | "source": [ 561 | "posterior = hmm.filter(params, emissions)" 562 | ] 563 | }, 564 | { 565 | "cell_type": "markdown", 566 | "id": "5dfb988e", 567 | "metadata": { 568 | "slideshow": { 569 | "slide_type": "slide" 570 | } 571 | }, 572 | "source": [ 573 | "## Things I noticed in the code\n", 574 | "* Architecture of Dynamax\n", 575 | "* What is coming next\n", 576 | "* JAX usage" 577 | ] 578 | }, 579 | { 580 | "cell_type": "markdown", 581 | "id": "085fcb19", 582 | "metadata": { 583 | "slideshow": { 584 | "slide_type": "slide" 585 | } 586 | }, 587 | "source": [ 588 | "## Computational complexity\n", 589 | "* Forward Algorithm $O(TK^{2})$\n", 590 | " * Linear with time\n", 591 | " * Quadratic with number of states" 592 | ] 593 | }, 594 | { 595 | "cell_type": "markdown", 596 | "id": "3b1019c6", 597 | "metadata": { 598 | "slideshow": { 599 | "slide_type": "slide" 600 | } 601 | }, 602 | "source": [ 603 | "## JAX Speedups" 604 | ] 605 | }, 606 | { 607 | "cell_type": "markdown", 608 | "id": "c2e52251", 609 | "metadata": {}, 610 | "source": [ 611 | "* Parallelization over independent time series\n", 612 | "* Gradient" 613 | ] 614 | }, 615 | { 616 | "cell_type": "markdown", 617 | "id": "a18417bc", 618 | "metadata": { 619 | "slideshow": { 620 | "slide_type": "slide" 621 | } 622 | }, 623 | "source": [ 624 | "## Real World Example: Parts from a machine shop\n", 625 | "https://www.youtube.com/watch?v=OCc2F8KccD4\n" 626 | ] 627 | }, 628 | { 629 | "cell_type": "markdown", 630 | "id": "9e863d53", 631 | "metadata": { 632 | "slideshow": { 633 | "slide_type": "slide" 634 | } 635 | }, 636 | "source": [ 637 | "### Questions asked of me\n", 638 | "* What state is the machine in? \n", 639 | " * Are we going to get good parts or bad parts?\n", 640 | "* How often does it tend to switch?\n", 641 | " * Once its good does it stay good, or is totally unreliable?\n", 642 | "* What is the probability of bad parts coming off this line?" 643 | ] 644 | }, 645 | { 646 | "cell_type": "markdown", 647 | "id": "ffd20821", 648 | "metadata": { 649 | "slideshow": { 650 | "slide_type": "slide" 651 | } 652 | }, 653 | "source": [ 654 | "### Data\n", 655 | "* The order parts were produced from the machine\n", 656 | "* Which ones were good and which ones weren't\n", 657 | "* Parts from many machines" 658 | ] 659 | }, 660 | { 661 | "cell_type": "markdown", 662 | "id": "c3b92282", 663 | "metadata": {}, 664 | "source": [ 665 | "### Use of HMM\n", 666 | "* **Smoothing** - What state(s) was the machine in over the last 7 days and when?\n", 667 | " * Look for correlation such as shift changes time etc\n", 668 | "* **Parameter Estimation** - How faulty was the machine and when did it tend to switch?\n", 669 | "* **Filtering** - What state is the machine in now?" 670 | ] 671 | }, 672 | { 673 | "cell_type": "markdown", 674 | "id": "c434e94c", 675 | "metadata": { 676 | "slideshow": { 677 | "slide_type": "slide" 678 | } 679 | }, 680 | "source": [ 681 | "## Useful Extensions for the above case\n", 682 | "* Covariate HMM\n", 683 | "* Autoregressive HMMs" 684 | ] 685 | }, 686 | { 687 | "cell_type": "markdown", 688 | "id": "5571a2d3", 689 | "metadata": { 690 | "slideshow": { 691 | "slide_type": "slide" 692 | } 693 | }, 694 | "source": [ 695 | "## Other HMMs in dynamax" 696 | ] 697 | }, 698 | { 699 | "cell_type": "markdown", 700 | "id": "8660fae7", 701 | "metadata": {}, 702 | "source": [ 703 | "* There are many more present\n", 704 | " * https://probml.github.io/dynamax/api.html#high-level-models" 705 | ] 706 | }, 707 | { 708 | "cell_type": "markdown", 709 | "id": "7ed5887e", 710 | "metadata": { 711 | "slideshow": { 712 | "slide_type": "slide" 713 | } 714 | }, 715 | "source": [ 716 | "## Takeaways" 717 | ] 718 | }, 719 | { 720 | "cell_type": "markdown", 721 | "id": "81fa79c2", 722 | "metadata": {}, 723 | "source": [ 724 | "* For state estimation smoothing, filtering, and viterbi are all supported\n", 725 | "* Multiple methods for parameter estimation\n", 726 | " * SGD\n", 727 | " * Minibatch\n", 728 | " * Expectation Maximization" 729 | ] 730 | }, 731 | { 732 | "cell_type": "markdown", 733 | "id": "9fe96d0e", 734 | "metadata": { 735 | "slideshow": { 736 | "slide_type": "slide" 737 | } 738 | }, 739 | "source": [ 740 | "## (Refined) Book Club Focus\n", 741 | "* Practitioner that's looking to use the library\n", 742 | " * Understand what exists\n", 743 | " * How it works from \"the drivers seat\"" 744 | ] 745 | }, 746 | { 747 | "cell_type": "markdown", 748 | "id": "d7703802", 749 | "metadata": { 750 | "slideshow": { 751 | "slide_type": "slide" 752 | } 753 | }, 754 | "source": [ 755 | "### What you folks said\n", 756 | "* How do i convert a well estimated state space model into a compelling case for action or decision?\n", 757 | "* Using this as a forcing function to help learn more about state space models and work up to structural time series and causal impact\n", 758 | "* Fluency in expressing diverse state space models in python\n" 759 | ] 760 | }, 761 | { 762 | "cell_type": "markdown", 763 | "id": "6559bd38", 764 | "metadata": {}, 765 | "source": [ 766 | "## Next Weeks Agenda\n", 767 | "* Finish off last two notebooks on State Space Models" 768 | ] 769 | } 770 | ], 771 | "metadata": { 772 | "celltoolbar": "Slideshow", 773 | "kernelspec": { 774 | "display_name": "Python 3 (ipykernel)", 775 | "language": "python", 776 | "name": "python3" 777 | }, 778 | "language_info": { 779 | "codemirror_mode": { 780 | "name": "ipython", 781 | "version": 3 782 | }, 783 | "file_extension": ".py", 784 | "mimetype": "text/x-python", 785 | "name": "python", 786 | "nbconvert_exporter": "python", 787 | "pygments_lexer": "ipython3", 788 | "version": "3.10.4" 789 | }, 790 | "rise": { 791 | "auto_select": "none", 792 | "enable_chalkboard": true, 793 | "scroll": true 794 | } 795 | }, 796 | "nbformat": 4, 797 | "nbformat_minor": 5 798 | } 799 | -------------------------------------------------------------------------------- /JAX/JaxGradient.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8587a3ac", 6 | "metadata": {}, 7 | "source": [ 8 | "# Jax Gradients" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 3, 14 | "id": "5297e1d7", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import matplotlib.pyplot as plt\n", 19 | "import numpy as np\n", 20 | "import jax.numpy as jnp\n", 21 | "import jax" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "e94c7a9a", 27 | "metadata": {}, 28 | "source": [ 29 | "## A parabola" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 22, 35 | "id": "4154c54a", 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "x = np.linspace(-20, 20, 1000)\n", 40 | "\n", 41 | "def y(x):\n", 42 | " return x**2" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 5, 48 | "id": "d129dfbc", 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "[]" 55 | ] 56 | }, 57 | "execution_count": 5, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | }, 61 | { 62 | "data": { 63 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABRxklEQVR4nO3deXhTVcIG8DdLm+6hpbTpRlvWAmUplK2AIAjIpogKiCKMy6iADuqMijqf6IwgjKKjjCKOg7iCCigqClWhyF52KDsUKF0opbTpmjTJ+f5IGy0UaEvSk+X9PU+eZ0huynu9neTl3HPPVQghBIiIiIiciFJ2ACIiIqLLsaAQERGR02FBISIiIqfDgkJEREROhwWFiIiInA4LChERETkdFhQiIiJyOiwoRERE5HTUsgM0hsViQU5ODgIDA6FQKGTHISIionoQQqCkpASRkZFQKq89RuKSBSUnJwcxMTGyYxAREVEjZGVlITo6+prbuGRBCQwMBGDdwaCgIMlpiIiIqD70ej1iYmJs3+PX4pIFpea0TlBQEAsKERGRi6nP9AxOkiUiIiKnw4JCRERETocFhYiIiJwOCwoRERE5HRYUIiIicjosKEREROR0WFCIiIjI6bCgEBERkdNhQSEiIiKn0+CCsnHjRowZMwaRkZFQKBT45ptvar0uhMDs2bMRGRkJX19fDBo0CBkZGbW2MRgMePzxxxEaGgp/f3/cdtttOHfu3A3tCBEREbmPBheUsrIydO3aFQsXLqzz9fnz52PBggVYuHAh0tPTodPpMHToUJSUlNi2mTlzJlatWoVly5Zh06ZNKC0txejRo2E2mxu/J0REROQ2FEII0eg3KxRYtWoVxo4dC8A6ehIZGYmZM2fi2WefBWAdLQkPD8e8efPwyCOPoLi4GC1atMAnn3yCCRMmAPj97sRr1qzB8OHDr/v36vV6aLVaFBcX8148RERELqIh3992nYOSmZmJvLw8DBs2zPacRqPBwIEDsWXLFgDArl27UFVVVWubyMhIJCYm2ra5nMFggF6vr/VwhPP6Svz75+N47ccjDvn5REREzq6yyoyHlu7Esh1nUWW2SMth14KSl5cHAAgPD6/1fHh4uO21vLw8eHt7Izg4+KrbXG7u3LnQarW2R0xMjD1j25zXV+LNn49h6ZbTKDeaHPJ3EBERObONxy7g58Pn8e9fjkNVj7sOO4pDruK5/DbKQojr3lr5WtvMmjULxcXFtkdWVpbdsv5R5ygtWob4oaLKjF8O5zvk7yAiInJm3+/PBQCM7BwBpdJNCopOpwOAK0ZC8vPzbaMqOp0ORqMRly5duuo2l9NoNAgKCqr1cASFQoHRXSIAAN/ty3HI30FEROSsKoxm/Hz4PADYvg9lsWtBiY+Ph06nQ2pqqu05o9GItLQ0pKSkAAB69OgBLy+vWtvk5ubi4MGDtm1kGtM1EgCw4dgFlFRWSU5DRETUdDYczUe50YzoYF90i2kmNYu6oW8oLS3FiRMnbH/OzMzE3r17ERISgpYtW2LmzJmYM2cO2rZti7Zt22LOnDnw8/PDpEmTAABarRYPPvggnn76aTRv3hwhISH461//is6dO+OWW26x3541UoIuEK1b+OPkhTKkHjqPcd2jZUciIiJqEjWnd0Z1ibju1AxHa3BB2blzJ26++Wbbn5966ikAwJQpU/DRRx/hmWeeQUVFBaZNm4ZLly6hd+/eWLduHQIDA23vefPNN6FWqzF+/HhUVFRgyJAh+Oijj6BSqeywSzfGeponEv/+5Ti+25fDgkJERB6hzGDCL0eqT+90jpSc5gbXQZHF0eugnMgvwS0LNkKtVGDni7egmZ+33f8OIiIiZ7J6Xw6e+GIPYpv7YcNfBzlkBEXaOijuok1YIBJ0gTBZBNZm1H3pMxERkTv5Yb/14pDRTnB6B2BBuaqaybI15+OIiIjcVUllFdYfvQAAGOUEp3cAFpSrqrm8avOJAhSUGiSnISIicpyfD5+H0WRBqxb+6BAReP03NAEWlKuIbe6PLtFaWATw40Ge5iEiIvf1Q/XZgtFdIp3i9A7AgnJNNaMo33PRNiIiclPFFVVIO2Y9vSN7cbY/YkG5hlFdrOfhdpwuxHl9peQ0RERE9rcuIw9VZoF24QFoF+4cp3cAFpRrimrmi+4tm0GI34e/iIiI3Mn3fzi940xYUK7j96t5eJqHiIjcy8VSAzadKADgXKd3ABaU6xrZOQIKBbD7bBHOXSqXHYeIiMhu1hzMg9ki0DlKi1YtAmTHqYUF5TrCg3zQKy4EAE/zEBGRe/lur/XswG1dnev0DsCCUi+juWgbERG5mZyiCuw4XQiFAhjd1blO7wAsKPUyIlEHlVKBA9nFOF1QJjsOERHRDauZW9kzLgQRWl/Jaa7EglIPoQEapLRuDgD44QBHUYiIyPWt3ue8p3cAFpR6q5nd/B0XbSMiIhd38kIpDmbroVYqMLKz853eAVhQ6m14Jx28VAocySvB8fMlsuMQERE12urqybH924YixN9bcpq6saDUUzM/bwxs1wLA78NiRERErkYIYTsb4KyndwAWlAa5rVsUAODbvTkQQkhOQ0RE1HAZOXqcKiiDRq3EsE462XGuigWlAW7pEAY/bxXOFpZjT1aR7DhEREQNVnMW4JYO4QjQqCWnuToWlAbw81ZjWMdwAL+fvyMiInIVFsvvp3fGOPHpHYAFpcFurz7N8/3+HJjMFslpiIiI6m/nmUvILa5EoEaNQe1byI5zTSwoDVQz47mg1IjNJy/KjkNERFRvq/dlAwCGJ+rg46WSnObaWFAayEulxKjqa8a/3ZstOQ0REVH9VJktWHMgD4BzX71TgwWlEW7vZj2waw/mobLKLDkNERHR9W0+UYDCMiNCA7xtq6M7MxaURujeMhhRzXxRZjTjl8P5suMQERFdV83VOyM7R0Ctcv6vf+dP6ISUSoVtFOUbnuYhIiInV1llxrqM8wB+Pwvg7FhQGqnmap4NR/NRXF4lOQ0REdHV/XokH6UGE6Ka+aJ7y2DZceqFBaWR2usCkaALRJVZ4MeDvMMxERE5r5q1u8Z0jYRCoZCcpn5YUG7AbdXDZN9y0TYiInJS+soq/HrUOl/SFa7eqcGCcgNqDvS2zIvIK66UnIaIiOhKPx3Ig9FkQbvwAHSICJQdp95YUG5AdLAfesYFQwjYlg4mIiJyJiv3nAMAjE2KcpnTOwALyg2z3eF4H6/mISIi55JdVIFtpwoB/H5xh6tgQblBozpHQK1U4GC2HifyS2XHISIisqmZHNunVQiimvlKTtMwLCg3KMTfGze1s95waTVP8xARkZMQQmBV9emdO5Jca/QEYEGxi9ttV/NkQwghOQ0RERFwOLcEx86XwlutxK2JEbLjNBgLih3c0iEcvl4qnLlYjr1ZRbLjEBER2VY6v6VDGLS+XpLTNBwLih34a9QY1ikcANdEISIi+cwWgW+rC8pYF5scW4MFxU5qzu+t3peDKrNFchoiIvJkW09exHm9Ac38vDCofZjsOI3CgmIn/duEIjRAg8IyI9KOXpAdh4iIPNiqPdbRk1GdI+Ctds2vetdM7YTUKqVtsmzNojhERERNrcJoxk/V94hzxat3arCg2NG47tZfhJ8P56O4gnc4JiKippd6+DzKjGbEhPiiR6xr3Lm4LiwodtQxIgjtwwNhNFmw5gDvcExERE3vm+rTO3d0c62l7S/HgmJHCoUCd1SPoqzczdM8RETUtApKDUg7Zp0HebsLn94BWFDsbmy3KCgUQPrpS8gqLJcdh4iIPMj3+3Jgtgh0jdaidYsA2XFuCAuKnem0PujXOhTA77OoiYiImsKq6rW4xrr46AnAguIQ4/5wmodL3xMRUVM4daEU+7KKoFIqMLpLpOw4N4wFxQGGd9LB10uF0xfLsYdL3xMRURP4pnr0ZEDbULQI1EhOc+NYUBzAX6PGrYk6AMCq3TzNQ0REjiWE+P3qHTc4vQOwoDhMzWme7/bnwGji0vdEROQ4u85cwtnCcvh5qzC0Y7jsOHbBguIgKa1DER6kQVF5FdYfzZcdh4iI3NiK6qUtRiRGwM9bLTmNfbCgOIhKqcDt3bgmChEROVZllRnf77MuDnpXj2jJaeyHBcWBak7z/HokH0XlRslpiIjIHa3NyEOJwYSoZr7oHR8iO47dsKA4UIIuCB0iglBlFvh+P5e+JyIi+1tRfTHGnd2joFS67tL2l2NBcbA7ufQ9ERE5SF5xJTYdty5tf6cbnd4BWFAc7raukVAqgN1ni3C6oEx2HCIiciOr9mTDIoCeccGIbe4vO45dsaA4WFiQD/q3bQEAWMml74mIyE6EELard+7s7l6jJwALSpOoOc2zYtc5WCxc+p6IiG7c/nPFOJFfCh8vJUZ2iZAdx+5YUJrA8E46BPqokV1UgW2ZF2XHISIiN/D1LuvoyfBOOgT5eElOY392Lygmkwkvvvgi4uPj4evri1atWuGVV16BxfL7aqpCCMyePRuRkZHw9fXFoEGDkJGRYe8oTsPHS4UxXa03bvp6JyfLEhHRjTGYzFi9z3rvHXda++SP7F5Q5s2bh0WLFmHhwoU4fPgw5s+fj3/961945513bNvMnz8fCxYswMKFC5Geng6dToehQ4eipKTE3nGcRs0v0JqDuSiprJKchoiIXNmvh/NRXFEFXZAPUlqHyo7jEHYvKFu3bsXtt9+OUaNGIS4uDnfddReGDRuGnTt3ArCOnrz11lt44YUXMG7cOCQmJmLp0qUoLy/H559/bu84TiMpphlat/BHZZUFaw5wTRQiImq8mtM7d3SPgsqN1j75I7sXlP79++OXX37BsWPHAAD79u3Dpk2bMHLkSABAZmYm8vLyMGzYMNt7NBoNBg4ciC1bttT5Mw0GA/R6fa2Hq1EoFLirRwwA4Cue5iEioka6UGLAhmPVa5+44dU7NexeUJ599lncc889SEhIgJeXF5KSkjBz5kzcc889AIC8vDwAQHh47bsthoeH21673Ny5c6HVam2PmJgYe8duEuO6R0GpAHaeuYRMrolCRESN8O3ebJgtAt1imqFNWIDsOA5j94KyfPlyfPrpp/j888+xe/duLF26FK+//jqWLl1aazuFovaQlBDiiudqzJo1C8XFxbZHVlaWvWM3ifAgHwxsZ10T5etdrrkPREQkV83pHXdbOfZydi8of/vb3/Dcc89h4sSJ6Ny5MyZPnownn3wSc+fOBQDodDoAuGK0JD8//4pRlRoajQZBQUG1Hq6q5jTPil3WBkxERFRfGTnFOJJXAm+VErd1iZQdx6HsXlDKy8uhVNb+sSqVynaZcXx8PHQ6HVJTU22vG41GpKWlISUlxd5xnM4tHcOg9fVCnr4Sm08UyI5DREQuZMUu64rkQzuGQ+vnfmuf/JHdC8qYMWPw6quv4ocffsDp06exatUqLFiwAHfccQcA66mdmTNnYs6cOVi1ahUOHjyIqVOnws/PD5MmTbJ3HKejUatwezdr6/1qFyfLEhFR/RhNFny711pQ3HXtkz9S2/sHvvPOO/j73/+OadOmIT8/H5GRkXjkkUfwf//3f7ZtnnnmGVRUVGDatGm4dOkSevfujXXr1iEwMNDecZzS3T1i8PHWM1ibkYfi8iq3b8FERHTjfj1yHhfLjAgL1GBAW/dc++SPFEIIl5sIodfrodVqUVxc7JLzUYQQGPHv33AkrwT/HJuI+/rEyo5ERERO7oGP0vHrkXw8Nqg1nr01QXacRmnI9zfvxSOBdU0U6/AcT/MQEdH15BVXYsPRfADA3R5wegdgQZFmbFIU1EoF9mUV4fh5913in4iIbtyK3edgEUCvuBC0auG+a5/8EQuKJKEBGgxqHwbg92vaiYiILieEwFc7rWtn3Z3sGaMnAAuKVDW/aCv3ZMNktlxnayIi8kTbMwtx+mI5AjRqjOoSITtOk2FBkWhwQhia+3vjQokBadX3VSAiIvqjL6tHT8Z0jYCft90vvnVaLCgSeamUGJsUBQBYns6l74mIqDZ9ZRXWHMgFANyd7Jr3oWssFhTJJvS0/sL9ciQf+SWVktMQEZEz+X5fLiqrLGgbFoCkmGay4zQpFhTJ2oUHonvLZjBbhG0JYyIiIgBYXn16Z0LPmKveUNddsaA4gYk9WwIAlqefhQuum0dERA5wNK8E+7KKoFYqbNMBPAkLihMY1SUC/t4qnL5Yju2ZhbLjEBGRE6iZHHtLh3CEBmgkp2l6LChOwF+jxm3VNxDkZFkiIjKaLFi1x3raf3xPz1n75I9YUJzEhOrTPGsO5KK4vEpyGiIikunnw+dRWGZEeJAGN7VtITuOFCwoTqJrtBYJukAYTBZ8u4+TZYmIPFnN6Z27ekRDrfLMr2rP3GsnpFAobJccf7Eji5NliYg8VG5xBTZWL955dw/PWvvkj1hQnMgdSVHwVitxOFePg9l62XGIiEiCFbusNwbsHR+CuFB/2XGkYUFxIs38vHFrJx0AYFn6WclpiIioqVkswrb2yXgPWzn2ciwoTmZi9Wme1XtzUG40SU5DRERNadOJAmQVViDIx7NuDFgXFhQn06dVc7QM8UOJwYQ1B/JkxyEioib0xQ7r6Pm47tHw8VJJTiMXC4qTUSp/nyy7nKd5iIg8Rn5JJVIPnQcATOzl2ad3ABYUp3RXj2goFUD66Us4kV8qOw4RETWBr3edg8ki0L1lMyTogmTHkY4FxQmFB/lgcEIYgN+vhSciIvdlsQgs22H9vL+nV0vJaZwDC4qTqllZdsWuczCaLJLTEBGRI205eRFnC8sR6KPG6C6RsuM4BRYUJ3Vz+xYIC9TgYpkR6w5xsiwRkTv7fMcZAMC4pCj4env25NgaLChOSq1S2ibLfr6dk2WJiNzVhRID1mVYJ8fe05und2qwoDixib1aQqmwDv2dusDJskRE7qhmcmwSJ8fWwoLixKKa+WJQe+tk2Zpr44mIyH1YLMK2cjgnx9bGguLk7q0e7vtq1zlUVpklpyEiInvaeuoizlwsR6BGjdEevnLs5VhQnNyg9mGI1PqgqLwKPx7MlR2HiIjs6PPq0fGxSVHw81ZLTuNcWFCcnEqpwMTqYT9OliUich8FpQasy7BepcnTO1diQXEBE3rGQKVUIP30JRw7XyI7DhER2cHXu86hyizQNaYZOkZycuzlWFBcQHiQD4Z2CAfAURQiIndgXTnW+nk+iffdqRMLiouYVD1ZdsXucyg3miSnISKiG7Ht1EWcvliOAI0aY7py5di6sKC4iP5tQtEyxA8llSZ8v4+TZYmIXNln1aPht3eL5OTYq2BBcRFKpcI2ivIZ10QhInJZ5/WVWFs9Ofa+PrGS0zgvFhQXclePaHipFNiXVYSD2cWy4xARUSMs25EFk0WgZ1wwOkRwcuzVsKC4kNAADW5NtC7k8xknyxIRuZwqs8V2Y0COnlwbC4qLqVlZdvXebJQaOFmWiMiV/HL4PM7rDQgN8MatiTrZcZwaC4qL6R0fgtYt/FFmNOObPdmy4xARUQN8ss06ejKhZww0apXkNM6NBcXFKBQKTOptHRb8bPtZCCEkJyIiovo4kV+KzScuQqngyrH1wYLigu7sHgWNWonDuXrsOnNJdhwiIqqHz7ZbR08GJ4QjOthPchrnx4Ligpr5eWNstygAwMdbz0hOQ0RE11NuNOHrXecAAJP7cnJsfbCguKiaX/AfD+Yiv6RSchoiIrqW7/bloKTShNjmfhjQJlR2HJfAguKiEqO06BEbjCqzwLIdWbLjEBHRVQghbKPd9/ZuCaVSITmRa2BBcWH3962ZLHsGVWaL5DRERFSXvVlFyMjRw1utxN09eGPA+mJBcWEjEiMQGqDBeb0B6zLOy45DRER1qLm0eEyXSAT7e0tO4zpYUFyYt1qJe6pv0/3x1tNywxAR0RUKy4z4fr/1Bq+cHNswLCgublLvllApFdieWYgjeXrZcYiI6A++2pkFo8mCzlFadI3Wyo7jUlhQXFyE1hfDOoYDAD7hJcdERE7DYhH4tHrtk8l9YqFQcHJsQ7CguIH7+8YBAFbtyYa+skpuGCIiAgCkHbuArMIKBPmoMaZrpOw4LocFxQ30aRWCduEBKDeasaJ6ISAiIpJryZbTAIDxyTHw9eZ9dxqKBcUNKBQKTK4eRflk6xlYLLw/DxGRTCcvlGLjsQtQKH4f5aaGYUFxE+OSohCoUeNUQRk2nyyQHYeIyKN9XD16MiQhHC2b8747jcGC4ib8NWrc2SMaALB0CyfLEhHJoq+sst1350/94uSGcWEsKG7kvj7Wa+x/PXIeWYXlktMQEXmmr3eeQ5nRjLZhAUhp3Vx2HJfFguJG2oQFoH+bUFgEbJe2ERFR07FYBJZWL5w5tV8cLy2+ASwobqbm/jzLdmShwmiWnIaIyLNsOJaPMxfLEeSjxh1JUbLjuDSHFJTs7Gzcd999aN68Ofz8/NCtWzfs2rXL9roQArNnz0ZkZCR8fX0xaNAgZGRkOCKKxxnSIRwxIb4orqjCqj3ZsuMQEXmUJZtPAwAm9moJP2+13DAuzu4F5dKlS+jXrx+8vLzw448/4tChQ3jjjTfQrFkz2zbz58/HggULsHDhQqSnp0On02Ho0KEoKSmxdxyPo1IqMDUlHgCwZHMmhOAlx0RETeFEfgl+O14AhcK6cizdGLsXlHnz5iEmJgZLlixBr169EBcXhyFDhqB169YArKMnb731Fl544QWMGzcOiYmJWLp0KcrLy/H555/bO45HGp8cjQCNGsfzS7HpBC85JiJqCjVXUN7SIRwxIby0+EbZvaCsXr0aycnJuPvuuxEWFoakpCR88MEHttczMzORl5eHYcOG2Z7TaDQYOHAgtmzZUufPNBgM0Ov1tR50dYE+Xrir+pLj/23KlJyGiMj9FVdUYcXu6kuLU+LkhnETdi8op06dwnvvvYe2bdti7dq1ePTRR/HEE0/g448/BgDk5eUBAMLDw2u9Lzw83Pba5ebOnQutVmt7xMTE2Du225maEgeFAlh/9AJOXiiVHYeIyK19tTML5UYz2oUHoC8vLbYLuxcUi8WC7t27Y86cOUhKSsIjjzyChx9+GO+9916t7S6/9EoIcdXLsWbNmoXi4mLbIysry96x3U5cqD+GJIQBAJZWr2hIRET2Z7YIfFx9N/mpKfG8tNhO7F5QIiIi0LFjx1rPdejQAWfPngUA6HQ6ALhitCQ/P/+KUZUaGo0GQUFBtR50fQ/0s06W/XrXORRX8C7HRESOsP5IPs4WlkPr64WxSbxrsb3YvaD069cPR48erfXcsWPHEBtrndEcHx8PnU6H1NRU2+tGoxFpaWlISUmxdxyP1rd1c7QPD0S50Ywv0znqRETkCEu2WOf6TegZw0uL7cjuBeXJJ5/Etm3bMGfOHJw4cQKff/45Fi9ejOnTpwOwntqZOXMm5syZg1WrVuHgwYOYOnUq/Pz8MGnSJHvH8WgKhQIP9I8DAHy05TRMZovcQEREbuZQjh6bT1yESqmwLZRJ9mH3gtKzZ0+sWrUKX3zxBRITE/GPf/wDb731Fu69917bNs888wxmzpyJadOmITk5GdnZ2Vi3bh0CAwPtHcfj3d4tCiH+3sguqsDPh8/LjkNE5FY+rL5SckSiDtHBvLTYnhTCBVfy0uv10Gq1KC4u5nyUenh97VEsXH8CveJC8OWjfWXHISJyC/n6SvSb9yuqzALfTO+HbjHNZEdyeg35/ua9eDzA5L6xUCsV2HG6EAezi2XHISJyCx9vPYMqs0BybDDLiQOwoHiA8CAfjOoSAQD432Yu3EZEdKMqjGbbXeMfGhAvOY17YkHxEDWXHH+3Lwfn9ZWS0xARubYVu8+hqLwKMSG+GNpRJzuOW2JB8RBdY5qhZ1wwqsyCC7cREd0Ai0XYbiPyQL94qJRcmM0RWFA8yEMDWgEAPtt+FmUGk+Q0RESuaf3RfJwqKEOgjxp3J/PWK47CguJBbukQjvhQfxRXVOGrnVy4jYioMf77m3X0ZFKvlgjQcGE2R2FB8SAqpQIP9rfORflwcyYXbiMiaqCD2cXYesq6MNsU3rXYoVhQPMyd3aMR4u+NrMIKrM3gwm1ERA1RM/dkVOcIRDbzlZzGvbGgeBhfbxXu62Ndjnnxb6fgguv0ERFJkVdcidX7cgDw0uKmwILige7vGwtvtRL7soqw88wl2XGIiFzCx1tPw2QR6BUXgi7RzWTHcXssKB4oNECDO7tHAwAWbzwlOQ0RkfMrM5jw2fazAIAHOXrSJFhQPFTN8OTPh8/j5IVSyWmIiJzb8vQsFFdUIT7UH7d0CJcdxyOwoHio1i0CcEuHcAjx+904iYjoSlVmi+1z8uEBrbgwWxNhQfFgD1ePoqzYdQ4XSw2S0xAROafv9+cgu6gCoQHeGNc9SnYcj8GC4sF6xYega7QWBpMFn2w7IzsOEZHTEULg/TTrXL0/9YuHj5dKciLPwYLiwRQKBR6+ybr8/cdbz6Cyyiw5ERGRc9lw7AKO5JXA31uF+3rHyo7jUVhQPNytnXSIDvZFYZkRX+06JzsOEZFTeT/tJADgnl4tofXzkpzGs7CgeDi1SomHq28iuHjjSS5/T0RUbW9WEbadKoRaqcAD/XlpcVNjQSGMT46xLX//w4Fc2XGIiJxCzejJ7d2iuKy9BCwoBF9vFf5UfdOr9zac5PL3ROTxMgvK8FNGHgDgz9Vz9ahpsaAQAOD+vnHw91bhSF4JNhy9IDsOEZFUH/x2CkIAgxPC0F4XKDuOR2JBIQCA1s8L91bfRPC9DSclpyEikie/pBJfV1808OjA1pLTeC4WFLJ5sH88vFVK7DhdiJ2nC2XHISKSYumW0zCaLEhq2Qw944Jlx/FYLChkEx7kgzt7WFdJ5CgKEXmiUoMJn2y1Llz5yE2toVBwWXtZWFColj/f1BoKBfDLkXwcydPLjkNE1KQ+23YG+koTWoX6Y1hH3hRQJhYUqiU+1B8jEyMAAIs4ikJEHqSyyowPfrPeFPDRQa2h5E0BpWJBoSs8Nsg6Key7/bnIKiyXnIaIqGl8uTMLBaUGRDXzxR1JvCmgbCwodIXEKC0GtA2F2SKweOMp2XGIiBzOaLLYRo0fHdgKXip+PcrGI0B1qhlF+XJnFi6UGCSnISJyrG/2ZCOnuBItAjW4OzlGdhwCCwpdRd9WzdE1phkMJgv+tzlTdhwiIocxWwTeq17W/uEB8fDxUklORAALCl2FQqHA9OpRlE+2nkFRuVFyIiIix/jhQC4yC8rQzM8L9/aOlR2HqrGg0FXd0iEcCbpAlBpMWLL5tOw4RER2Z7EIvLv+BADgTynx8NeoJSeiGiwodFVKpQKPD24LAFiyORMllVWSExER2Zd1zacSBGjUmFp901RyDiwodE0jEnVoExYAfaUJH1evrkhE5A6EEFhYPXpyX59YaP28JCeiP2JBoWtSKhWYcXMbAMB/fzuFMoNJciIiIvvYfOIi9mUVwcdLiYcGxMuOQ5dhQaHrGt0lAnHN/XCpvAqfbuMoChG5h3d+PQ4AmNizJUIDNJLT0OVYUOi61ColplePonzw2ylUGM2SExER3ZidpwuxPbMQXioFHhnYSnYcqgMLCtXL2KQoRAf7oqDUiC92nJUdh4johrz9q3XuyZ3doxGh9ZWchurCgkL14qVSYtog6yjKorSTqKziKAoRuaZdZy5h47ELUCsVts81cj4sKFRvd/aIQqTWB/klBny1M0t2HCKiRvn3L9a5J3d2j0bL5n6S09DVsKBQvWnUKjxavbrsextOwmiySE5ERNQwfxw9qZlbR86JBYUaZHxyDMICNcgprsTK3edkxyEiahCOnrgOFhRqEB8vFf58k3XG+382nECVmaMoROQaOHriWlhQqMHu7R2L0AANsgor8PUujqIQkWvg6IlrYUGhBvP1VuGx6rkoC389AYOJV/QQkXPj6InrYUGhRrm3d0uEB2mQXVSBL9N5RQ8ROTeOnrgeFhRqFB8vle1fIQvXn+C6KETktDh64ppYUKjRJvSMQaTWB+f1Bny+navLEpFz4uiJa2JBoUbTqFV4fEhbAMC7G07yHj1E5HQ4euK6WFDohtzVIxoxIb4oKDXgk22nZcchIqrljXVHAXD0xBWxoNAN8VIp8cRg6yjKorRTKDWYJCciIrLafKIAW05ehLdKiceHcPTE1bCg0A27IykK8aH+KCwzYumW07LjEBFBCIF/rbWOnkzq3RLRwRw9cTUsKHTD1Col/lI9F2XxxlPQV1ZJTkREnu6Xw/nYm1UEHy8lpt3cWnYcagQWFLKLMV0j0SYsAMUVVfjfpkzZcYjIg1ksAq9Xzz2ZmhKPsEAfyYmoMVhQyC5USgVm3mIdRfnwt0wUlRslJyIiT/XDgVwcyStBoEaNRwe2kh2HGokFhexmZGIEEnSBKDGYsCjtlOw4ROSBTGYL3kw9BgB4+KZWaObnLTkRNZbDC8rcuXOhUCgwc+ZM23NCCMyePRuRkZHw9fXFoEGDkJGR4ego5GBKpQJ/G94eALBkcybyiislJyIiT7NydzZOFZQhxN8bD/SPlx2HboBDC0p6ejoWL16MLl261Hp+/vz5WLBgARYuXIj09HTodDoMHToUJSUljoxDTWBwQhiSY4NhMFnw9q/HZcchIg9iMJltq8Y+NrA1AjRqyYnoRjisoJSWluLee+/FBx98gODgYNvzQgi89dZbeOGFFzBu3DgkJiZi6dKlKC8vx+eff+6oONREFAoFnh2RAABYnp6FUxdKJSciIk+xbEcWsosqEB6kweS+sbLj0A1yWEGZPn06Ro0ahVtuuaXW85mZmcjLy8OwYcNsz2k0GgwcOBBbtmyp82cZDAbo9fpaD3JePeNCMDghDGaLwBvV54KJiByp3GjCO7+eAADMGNwWPl4qyYnoRjmkoCxbtgy7d+/G3Llzr3gtLy8PABAeHl7r+fDwcNtrl5s7dy60Wq3tERMTY//QZFd/G94eCgXww/5cHDhXLDsOEbm5JZtPo6DUgOhgX0xI5neEO7B7QcnKysJf/vIXfPrpp/Dxufq15wqFotafhRBXPFdj1qxZKC4utj2ysrLsmpnsr0NEEMZ2iwIAzF97RHIaInJnhWVGLNpwEgDw12Ht4a3mBaruwO5HcdeuXcjPz0ePHj2gVquhVquRlpaGt99+G2q12jZycvloSX5+/hWjKjU0Gg2CgoJqPcj5PXlLO3ipFPjteAG2nCiQHYeI3NQ7vx5HicGETpFBuK1rpOw4ZCd2LyhDhgzBgQMHsHfvXtsjOTkZ9957L/bu3YtWrVpBp9MhNTXV9h6j0Yi0tDSkpKTYOw5J1LK5Hyb1agkAmLf2KIQQkhMRkbvJKizHp9vOAACeG5EApbLukXhyPXa/BiswMBCJiYm1nvP390fz5s1tz8+cORNz5sxB27Zt0bZtW8yZMwd+fn6YNGmSveOQZDMGt8VXu85hX1YR1mbk4dbECNmRiMiNvL7uKKrMAgPahmJA2xay45AdSTlR98wzz2DmzJmYNm0akpOTkZ2djXXr1iEwMFBGHHKgFoEaPFi9WNL8tUdhMlskJyIid3Ewuxjf7s0BADx7a4LkNGRvCuGC4+56vR5arRbFxcWcj+IC9JVVGDh/PS6VV2HOHZ0xqXdL2ZGIyA3c99/t2HSiALd3i8S/JybJjkP10JDvb051JocL8vHCE0OsNxJckHoMpQaT5ERE5Oo2HruATScK4K1S4q/D2suOQw7AgkJN4t7esYhr7oeCUgMWp52UHYeIXJjFIvDaj9blC+7rE4uYED/JicgRWFCoSXirlXiuegn8xb+d4o0EiajRVu/LwaFcPQI1aswY3EZ2HHIQFhRqMsM76ZAcG4zKKgveWHdUdhwickEGkxmvV39+PDqoNUL8vSUnIkdhQaEmo1Ao8MKoDgCAr3efw6Ec3lOJiBpmyebTOHfJekPAB/rFy45DDsSCQk0qqWUwRneJgBDAnDWHuXgbEdVbQakBC6tvCPi34Qnw9eYNAd0ZCwo1uWdvTYC3SolNJwqQduyC7DhE5CJqrgLsHKXFuKQo2XHIwVhQqMnFhPhhSkosAOsoChdvI6LrOZKnx7IdZwEAfx/dkUvaewAWFJJixs1tofX1wrHzpfh61znZcYjIiQkh8OoPh2ERwIhEHXrFh8iORE2ABYWk0Pr9vnjbG1y8jYiuYf3RfPx23Loo26wRHWTHoSbCgkLSTO5jXbztQokB/1l/QnYcInJCVWYL/vnDYQDAn/rFoWVzLsrmKVhQSBpvtRIvjuoIAPjwt0ycLiiTnIiInM1n287g1IUyNPf3xnQuyuZRWFBIqiEdwjCgbSiMZgteXXNYdhwiciLF5VV465fjAIAnh7ZDkI+X5ETUlFhQSCqFQoGXxnSESqlA6qHz+O04LzsmIqu3fz2OovIqtAsPwMSeMbLjUBNjQSHp2oQF4v6+1suOX/nuEKp42TGRxzuRX4qlW04DAF4Y1RFqFb+uPA2PODmFmUPaIcTfG8fzS/HptjOy4xCRREIIvPxdBkwWgcEJYRjYroXsSCQBCwo5Ba2fF54e1g4A8GbqMRSWGSUnIiJZ1mact11W/NKYjrLjkCQsKOQ0JvZsiQ4RQdBXmni3YyIPVWE04x/fHwIA/PmmVoht7i85EcnCgkJOQ6VUYHb1v5a+2HGWdzsm8kCL0k4iu6gCkVofTLu5tew4JBELCjmV3q2aY1SXCFgEMPu7DN7tmMiDZBWW4720kwCsE2P9vNWSE5FMLCjkdGaNSICPlxI7Mgvx7d4c2XGIqIn84/tDMJosSGndHCM762THIclYUMjpRAf74fHB1vv0/POHwyiuqJKciIgcLe3YBaw7dB5qpQIv39YJCgXvVuzpWFDIKT00IB6tWvijoNSAN1OPyY5DRA5kNFnw8uoMAMCUlDi0DQ+UnIicAQsKOSWNWoV/3J4IAPh462kczC6WnIiIHOV/mzNxqqAMoQEa/OWWtrLjkJNgQSGn1a9NKEZXT5h98ZuDsFg4YZbI3WQXVeDt6vvtzBqRwPvtkA0LCjm1v4/uiACNGnuzirB8Z5bsOERkZ7NXZ6DcaEbPuGDckRQlOw45ERYUcmrhQT6YWT3kO++nI1xhlsiNrMvIQ2r1xNhX7+gMpZITY+l3LCjk9KamxCFBF4ii8irM/+mI7DhEZAdlBhNmV0+MffimVmjHibF0GRYUcnpqlRL/HGudMLssPQu7zlySnIiIbtRbPx9DTnElooN98cRgToylK7GgkEtIjgvB3T2iAQAvrDqAKrNFciIiaqxDOXr8b/NpAMA/bk+Er7dKbiBySiwo5DKeG5GAYD8vHMkrweKNp2THIaJGsFgEXvjmAMwWgZGddbg5IUx2JHJSLCjkMpoHaPDiKOvNBP/9y3FkFpRJTkREDfVF+lnsOVuEAI0a/ze6k+w45MRYUMiljOsehQFtQ2E0WfD8ygO8mSCRC7lQYsC8H60T3Z8e1g46rY/kROTMWFDIpSgUCrw6tjN8vJTYeuoivtp5TnYkIqqn2aszoK80ITEqCJP7xMqOQ06OBYVcTsvmfnjylnYAgFfXHMaFEoPkRER0PT8dzMMPB3KhUirw2rguUKv49UPXxt8QckkP9o9Hp8ggFFdU4eXvMmTHIaJrKC6vwt+/PQgAeOSmVkiM0kpORK6ABYVcklqlxLw7u0ClVOD7/bn45fB52ZGI6CrmVI90tgr1xxNDuOYJ1Q8LCrmsxCgtHuwfD8B6M8FSg0lyIiK63OYTBbb7aM27qwt8vLjmCdUPCwq5tCdvaYeYEF/kFldyGXwiJ1NuNOG5lfsBAPf3jUXPuBDJiciVsKCQS/P1VmHuHV0AAB9vPYMtJwskJyKiGm+sO4aswgpEan3wzK0JsuOQi2FBIZfXv20o7unVEgDwzNf7UcZTPUTS7Tl7CUs2ZwIAXh3XGQEateRE5GpYUMgtPD8yAVHNfHHuUgVe+5Gneohkqqwy429f74dFAOOSonBzey5nTw3HgkJuIdDHC/PutJ7q+WTbGWw5wVM9RLIsSD2GE/mlaBGowd9Hd5Qdh1wUCwq5jf5tQzGpd/WpnhX7eVUPkQQ7MgvxwW/Wm3m+Nq4zgv29JSciV8WCQm7l+ZEdbKd65q45LDsOkUcpM5jw16/2QQhgfHI0hnQIlx2JXBgLCrmVAI0a/7rLeqrns+1nsZmneoiazNwfD+NsYTmimvny1A7dMBYUcjspbUJxX5/fr+rRV1ZJTkTk/jYeu4BPt50FAMy/qwsCfbwkJyJXx4JCbmnWiA6ICfFFdlEFZn/Le/UQOVJxRRWeXWFdkG1K31j0axMqORG5AxYUckv+GjXeHN8NSgWwck82vt+fIzsSkdt6+bsM5BZXIq65H54dwQXZyD5YUMhtJceFYPrNbQAAL6w6iNziCsmJiNzPmgO5WLk7G0oF8Mb4rvDz5oJsZB8sKOTWnhjSFl2itSiuqMJfv9oHi0XIjkTkNnKKKjBr5QEAwCMDW6NHLO+1Q/bDgkJuzUulxFsTusHXS4XNJy7if9VLbxPRjTFbBJ76ci+KK6rQJVqLJ29pJzsSuRkWFHJ7rVoE4MXRHQAA8386iiN5esmJiFzf+xtPYtupQvh5q/DviUnwVvPrhOyLv1HkESb1aokhCWEwmi2YuWwvKqvMsiMRuax9WUVYsO4YAGD2mE6ID/WXnIjcEQsKeQSFQoF5d3VBaIA3juSV8IaCRI1UZjBh5vK9MFkERnbW4e7kaNmRyE2xoJDHCA3Q4F93dQUAfLTlNNZm5ElOROR6Xv4uA5kFZYjQ+mDuHV2gUChkRyI3ZfeCMnfuXPTs2ROBgYEICwvD2LFjcfTo0VrbCCEwe/ZsREZGwtfXF4MGDUJGBhfTIse7OSEMDw+IB2BdZTa7iJceE9XXD/tz8eXOc1AogDcndIPWj6vFkuPYvaCkpaVh+vTp2LZtG1JTU2EymTBs2DCUlZXZtpk/fz4WLFiAhQsXIj09HTqdDkOHDkVJSYm94xBd4W/DE9A1phmKK6rwxBd7UGW2yI5E5PTOXCzDc9WrxT42sDX6tGouORG5O4UQwqELQ1y4cAFhYWFIS0vDTTfdBCEEIiMjMXPmTDz77LMAAIPBgPDwcMybNw+PPPLIdX+mXq+HVqtFcXExgoKCHBmf3FRWYTlGvv0bSipNeGxQazx7K1e/JLqayioz7nxvCzJy9EiODcYXf+4DLxVnCFDDNeT72+G/YcXFxQCAkBDrAj6ZmZnIy8vDsGHDbNtoNBoMHDgQW7ZsqfNnGAwG6PX6Wg+iGxET4od5d1rvevzehpNIO3ZBciIi5/XqD4eRkaNHsJ8X3pmUxHJCTcKhv2VCCDz11FPo378/EhMTAQB5edaJieHh4bW2DQ8Pt712ublz50Kr1doeMTExjoxNHmJk5wjc29t61+Onlu9Fvr5SciIi5/Pdvhx8su0MAGDBhG6I0PpKTkSewqEFZcaMGdi/fz+++OKLK167fOa3EOKqs8FnzZqF4uJi2yMrK8shecnz/H10RyToAnGxzIi/LNsLE+ejENlkFpTZlrKfNqg1bm4fJjkReRKHFZTHH38cq1evxvr16xEd/ft18jqdDgCuGC3Jz8+/YlSlhkajQVBQUK0HkT34eKmwcFJ3+HmrsPXURbxevfgUkaerrDJj2me7UWowoVd8CJ4ayqXsqWnZvaAIITBjxgysXLkSv/76K+Lj42u9Hh8fD51Oh9TUVNtzRqMRaWlpSElJsXccoutqExaA+XdZ56MsSjuJnw7mSk5EJN/L3x3C4Vw9mvt74517kqDmvBNqYnb/jZs+fTo+/fRTfP755wgMDEReXh7y8vJQUWFdb0KhUGDmzJmYM2cOVq1ahYMHD2Lq1Knw8/PDpEmT7B2HqF5Gd4nEQ/2tZfqvX+3HyQulkhMRyfNleha+2HEWCgXw1sRuCA/ykR2JPJDdLzO+2jySJUuWYOrUqQCsoywvv/wy3n//fVy6dAm9e/fGf/7zH9tE2uvhZcbkCFVmC+7973bsyCxE27AAfDO9H/w1atmxiJrUvqwi3L1oK4xmC54a2g5PDGkrOxK5kYZ8fzt8HRRHYEEhR8kvqcTotzchv8SA0V0i8M49SVzKmzxGQakBY97ZhNziSgztGI737+sBpZK//2Q/TrUOCpErCQv0wbv3dodaqcD3+3Pxv82nZUciahJVZgumf7YbucWVaNXCHwvGd2U5IalYUIgukxwXghdHdQAAzFlzGFtOFkhOROR4c9ccwfbMQvh7q7B4cg8E+vA+OyQXCwpRHaakxGFst0iYLQLTPtuNsxfLZUcicphv9mTjf5szAQBvjO+GNmGBkhMRsaAQ1UmhUOC1O7uga7QWReVVeHBpOkoqq2THIrK7vVlFeLb6JoAzbm6DWxN1khMRWbGgEF2Fj5cKi+9PRniQBsfzSzFz2V6YLS43p5zoqnKLK/DwxzthMFkwOCEMT3IxNnIiLChE1xAe5IPFk5OhUSvxy5F8/GvtUdmRiOyi3GjCQ0t34kKJAe3DA/Hvid2g4qRYciIsKETX0TWmWa2VZlftOSc5EdGNsVgEnly+Fxk51pVi/zslmZNiyemwoBDVw+3dojD95tYAgGdXHMDO04WSExE13uvrjmJtxnl4q5R4f3IPxIT4yY5EdAUWFKJ6enpoewzrGA6jyYKHP96JzIIy2ZGIGmzl7nN4d8NJAMC8uzojOS5EciKiurGgENWTUqnAvycmoWtMM1wqr8LUJTtwsdQgOxZRvW05UWC7Ymf6za1xR1L0dd5BJA8LClED+Hqr8OGUZMSE+OLMxXI89PFOVFaZZcciuq7DuXo88skuVJkFRnWOwNND28uORHRNLChEDRQaoMFHf+oFra8X9pwt4uXH5PRyiirwpyXpKDGY0Cs+BG9wGXtyASwoRI3QukUAPrg/Gd4qJX7KyMOrPxyWHYmoTsUV1tORefpKtA0LwAeTk+HjpZIdi+i6WFCIGqnmX6IA8L/NmXg/7aTkRES1GUxm/PnjnTh2vhThQRp89EAvaP14OTG5BhYUohswpmskXhhpvbHg3B+PYNmOs5ITEVmZLQJPLd+H7ZmFCNCosWRqL0Q185Udi6jeWFCIbtDDN7XCY4Osa6Q8v+oAfjyQKzkReTohBF5YdQA/HMiFl0qB9yf3QMfIINmxiBqEBYXIDp4Z3h739IqBRQB/WbYXm44XyI5EHkoIgVd/OIxl6VlQKoB/T0xCvzahsmMRNRgLCpEdKBQK/HNsZ4zsrIPRbMGfP9mJPWcvyY5FHujtX07gv5syAQCv3dkFIztHSE5E1DgsKER2olIq8OaEbhjQNhTlRjP+9FE6DuXoZcciD/K/TZl48+djAID/G90R45NjJCciajwWFCI70qhVWHRfDyS1bIai8irc9+F2HM0rkR2LPMCX6Vl45ftDAIAnb2mHB/rHS05EdGNYUIjszF+jxkd/6oUu0VoUlhkx6YNtOH6eJYUc58udWXh2pXUJ+4f6x+OJIW0kJyK6cSwoRA6g9fXCJw/0RqfIIFwsM+KeD7bjRH6p7Fjkhr5Mz8KzK/ZDCGByn1i8MKoDFAquEkuujwWFyEG0fl749MHe6BARhIJSAyZ9sA2nLrCkkP18mW4dORECuL9vLF65vRPLCbkNFhQiBwr298ZnD/VGgi4Q+SUG3PPBNo6kkF0sTz+LZ6pHTqb0jcXLt7GckHthQSFysBB/b3z6UG+0Cw/Aeb0BE97fioycYtmxyIV9seMsnl1xAAAwNSUOs1lOyA2xoBA1gdAADZb9uS8So6rnpCzeht1cJ4UaYfHGk5i18vdy8tKYjiwn5JZYUIiaSIi/Nz5/uA+SY4OhrzThvv9ux5YTXHGW6kcIgfk/HcGcNUcAAI/c1IrlhNwaCwpREwry8cLHD/ZC/zbWxdymfpSOXw6flx2LnJzZIvDiNwfx7gbrHbOfvTUBs0byah1ybywoRE3Mz1uN/05JxtCO4TCaLHjkk134ameW7FjkpIwmC2Yu34vPtp+FQgHMuaOz7eaURO6MBYVIAh8vFd69tzvGJUXBZBH429f78fYvxyGEkB2NnEhJZRUe+ngnvtuXAy+VAm9PTMKk3i1lxyJqEiwoRJJ4qZR4Y3xXTKv+1/CC1GOYtfIATGaL5GTkDHKLK3D3oq3YeOwCfL1U+OD+ZIzpGik7FlGTYUEhkkihUOCZWxPwj7GJUCqAZelZePjjnSgzmGRHI4kycoox9j+bcSSvBKEBGix/pA8GtQ+THYuoSbGgEDmByX1isei+HvDxUmL90QuYsHgr8oorZcciCTYczcf4RVtxXm9Am7AArJqWgi7RzWTHImpyLChETmJYJx2+eLgPQvy9cTBbjzELN2EP10rxGEIILNmciQeX7kSZ0Yy+rZpjxWMpiAnxkx2NSAoWFCInktQyGN9O74f24YG4UGLAhMXbsGLXOdmxyMEqq8z429f78fJ3h2C2CIzrHoWlD/SC1tdLdjQiaVhQiJxMTIgfVkxLwbDqy5Cf/mof5qw5DLOFV/i4o/P6SkxYvA1f7zoHpQJ4cVQHvHF3V3ir+fFMno3/DyByQgEaNRbd1wNPDG4DAFi88RTu/992XCgxSE5G9rTrzCWMfmcT9mUVQevrhaUP9MJDA1pxATYisKAQOS2lUoGnhrXHwklJ8PNWYfOJixj19m/YkVkoOxrdICEEPth4ChPe34oLJQa0Cw/A6hn9MKBtC9nRiJwGCwqRkxvdJRKrZ/RD27AA5JcYcM8H27Ao7SQsPOXjki6VGfHQ0p14dc1hmCwCo7pEYOW0foht7i87GpFTYUEhcgFtwgLx7Yx+uCMpCmaLwGs/HsFDH+9EQSlP+biSXWcuYdTbv+GXI/nwVivxz7GJWHhPEgI0atnRiJyOQrjg2tp6vR5arRbFxcUICgqSHYeoyQghsCw9Cy+tzoDRZEFogDfm39UFgxPCZUeja6gyW/Du+pN459fjMFkE4kP9sXBSEjpFamVHI2pSDfn+ZkEhckGHc/WYuWwvjp4vAQDc16clXhjZEb7eKsnJ6HInL5TiqeV7se9cMQBgTNdIzLkjEYE+vISYPA8LCpEHqKwy419rj+LDTZkAgFah/nh9fFd0bxksORkBgMUi8PHW05j74xEYTBYE+ajxj7GJuK1rJK/SIY/FgkLkQTYdL8Bfv9qHPH0lFApgakoc/jqsPfw5r0GaUxdK8fyqA9h2ynrF1YC2oZh/VxdEaH0lJyOSiwWFyMMUlRvxyveHsHJ3NgAgqpkvXr0jkTeYa2JGkwWL0k5i4foTMJos8PFSYtaIDpjcJxZKJUdNiFhQiDzUxmMX8PyqAzh3qQIAcHu3SDw/sgPCg3wkJ3N/O08XYtbKAzieXwoAuKldC/zz9kS0bM576RDVYEEh8mDlRhMWrDuG/23OhEUAft4qTL+5DR7sHw8fL06itbfc4grM/+koVu2xjl419/fG/43pyLkmRHVgQSEi7D9XhNmrM7D7bBEAoGWIH14c1QFDO4bzi9MOyo0mvJ92Cu9vPInKKgsAYEJyDGaNTEAzP2/J6YicEwsKEQGwXkny7b5svPbjEZzXWxd16x0fgr8Nb4/kuBDJ6VyTyWzBN3tz8Prao8jTVwIAesYF4++jO6JLdDO54YicHAsKEdVSZjDhP+tP4L+bMmE0Wf+1PzghDE8Pa8fFwurJbBH4fn8O/v3zcZwqKANgnYz8/MgOGNlZx1EponpgQSGiOuUUVeCdX4/jy53nYK6+l8/IzjpMG9QGiVEsKnUxWwTWZuThzdRjtgmwzfy88MhNrfGnfnGc10PUACwoRHRNmQVleOvnY1i9Lwc1nwD924TikYGt0L9NKEcDYF0I7+td5/DhpkxkVo+YBPmo8fCAVpjaL44rwRI1AgsKEdXLkTw9Fm04ie/259pGVDpFBuFP/eIxukuER44OnNdX4vPtZ/HJtjMoLDMCsBaTqf3i8WD/eGh9WUyIGosFhYga5Nylcvz3t0wsT89CRZUZAKD19cKd3aMxqXdLtAkLkJzQscwWgY3HL+CL7Wfxy5F8W1mLDvbFg/3jMT45hivzEtkBCwoRNcqlMiM+33EWn28/i+yiCtvzPWKDcXu3SIzsHIHQAI3EhPZ1JE+P1Xtz8O3enFr72zMuGPf3jcOIRB3UKqXEhETuhQWFiG5IzYjCZ9vO4tcj51E9oACVUoH+bUIxsrMONyeEISzQtVaoFULg2PlSpB7Kw+p9OTh2vtT2mtbXC+O6R2FSr5ZoGx4oMSWR+3KZgvLuu+/iX//6F3Jzc9GpUye89dZbGDBgwHXfx4JC1HTO6yvx/f5crN6bjX3nimu91jVai8EJ4RjQLhSdo7TwcsLRhpLKKuzILMSvR/Kx4eiFWiMl3iolBrZvgdu6RmJox3CPnHND1JRcoqAsX74ckydPxrvvvot+/frh/fffx3//+18cOnQILVu2vOZ7WVCI5MgsKMN3+3Lwy+HzV5QVP28VesQGo1dcCLrHBqNTZFCTr6gqhEB2UQUOnCvGjtOFSD9diEM5etsIEABo1EqktG6OEZ0jMLyTjpNeiZqQSxSU3r17o3v37njvvfdsz3Xo0AFjx47F3Llzr/leFhQi+fJLKrHhyAX8cuQ8tmcWoqi86optopr5omNkENqFByA2xB8tm/shrrk/wgI1N3R338oqM7KLKnC2sBznCstx8kIZDufqcThXD32l6YrtY0J8MbBdCwxOCEPfVqHw9eZICZEMDfn+ljIt3Wg0YteuXXjuuedqPT9s2DBs2bLliu0NBgMMBoPtz3q93uEZiejawgJ9ML5nDMb3jIHFInAsvwTbTxViR2YhDmQX42xhObKLKpBdVIHUQ+drvVelVCDYzxuhAd4I8fdGkI8XNF5KaNRKaNQqKBWA0SxQZbbAaLKg3GjGpXIjLpUZUVhurLMM1VArFWgbHojk2GD0jA9Br7gQ6LSuNVeGiCQVlIKCApjNZoSHh9d6Pjw8HHl5eVdsP3fuXLz88stNFY+IGkipVCBBF4QEXRCmpMQBAIorqnA4V4+MHD1OXSjF2cJynLloLS1mi0BBqQEFpYZr/+Br8PdWISbEDzEhfogN8UNCRBA6RASiTVgANGqOkBC5OqkX9l++WqUQos4VLGfNmoWnnnrK9me9Xo+YmBiH5yOixtP6eqFPq+bo06p5reerzBZcLDXiYpkBF0uNKCwzoqSyCgaTBQaTdcTEIgS8VUp4qZXwUinh46VEc39vBPt5I9jfG839rSMvXPGWyH1JKSihoaFQqVRXjJbk5+dfMaoCABqNBhqN+6y9QOTJvFRK6LQ+PO1CRNck5ZpAb29v9OjRA6mpqbWeT01NRUpKioxIRERE5ESkneJ56qmnMHnyZCQnJ6Nv375YvHgxzp49i0cffVRWJCIiInIS0grKhAkTcPHiRbzyyivIzc1FYmIi1qxZg9jYWFmRiIiIyElwqXsiIiJqEg35/na+damJiIjI47GgEBERkdNhQSEiIiKnw4JCRERETocFhYiIiJwOCwoRERE5HRYUIiIicjosKEREROR0WFCIiIjI6Uhb6v5G1Cx+q9frJSchIiKi+qr53q7PIvYuWVBKSkoAADExMZKTEBERUUOVlJRAq9VecxuXvBePxWJBTk4OAgMDoVAo7Pqz9Xo9YmJikJWV5Zb3+XH3/QPcfx+5f67P3feR++f6HLWPQgiUlJQgMjISSuW1Z5m45AiKUqlEdHS0Q/+OoKAgt/3FA9x//wD330fun+tz933k/rk+R+zj9UZOanCSLBERETkdFhQiIiJyOiwol9FoNHjppZeg0WhkR3EId98/wP33kfvn+tx9H7l/rs8Z9tElJ8kSERGRe+MIChERETkdFhQiIiJyOiwoRERE5HRYUIiIiMjpeFxBefXVV5GSkgI/Pz80a9aszm3Onj2LMWPGwN/fH6GhoXjiiSdgNBqv+XMNBgMef/xxhIaGwt/fH7fddhvOnTvngD1omA0bNkChUNT5SE9Pv+r7pk6desX2ffr0acLk9RcXF3dF1ueee+6a7xFCYPbs2YiMjISvry8GDRqEjIyMJkrcMKdPn8aDDz6I+Ph4+Pr6onXr1njppZeu+zvpzMfw3XffRXx8PHx8fNCjRw/89ttv19w+LS0NPXr0gI+PD1q1aoVFixY1UdKGmzt3Lnr27InAwECEhYVh7NixOHr06DXfc7X/nx45cqSJUtff7Nmzr8ip0+mu+R5XOn5A3Z8pCoUC06dPr3N7Zz9+GzduxJgxYxAZGQmFQoFvvvmm1uuN/TxcsWIFOnbsCI1Gg44dO2LVqlV2ze1xBcVoNOLuu+/GY489VufrZrMZo0aNQllZGTZt2oRly5ZhxYoVePrpp6/5c2fOnIlVq1Zh2bJl2LRpE0pLSzF69GiYzWZH7Ea9paSkIDc3t9bjoYceQlxcHJKTk6/53ltvvbXW+9asWdNEqRvulVdeqZX1xRdfvOb28+fPx4IFC7Bw4UKkp6dDp9Nh6NChtvs8OZMjR47AYrHg/fffR0ZGBt58800sWrQIzz///HXf64zHcPny5Zg5cyZeeOEF7NmzBwMGDMCIESNw9uzZOrfPzMzEyJEjMWDAAOzZswfPP/88nnjiCaxYsaKJk9dPWloapk+fjm3btiE1NRUmkwnDhg1DWVnZdd979OjRWserbdu2TZC44Tp16lQr54EDB666rasdPwBIT0+vtX+pqakAgLvvvvua73PW41dWVoauXbti4cKFdb7emM/DrVu3YsKECZg8eTL27duHyZMnY/z48di+fbv9ggsPtWTJEqHVaq94fs2aNUKpVIrs7Gzbc1988YXQaDSiuLi4zp9VVFQkvLy8xLJly2zPZWdnC6VSKX766Se7Z78RRqNRhIWFiVdeeeWa202ZMkXcfvvtTRPqBsXGxoo333yz3ttbLBah0+nEa6+9ZnuusrJSaLVasWjRIgcktL/58+eL+Pj4a27jrMewV69e4tFHH631XEJCgnjuuefq3P6ZZ54RCQkJtZ575JFHRJ8+fRyW0Z7y8/MFAJGWlnbVbdavXy8AiEuXLjVdsEZ66aWXRNeuXeu9vasfPyGE+Mtf/iJat24tLBZLna+70vEDIFatWmX7c2M/D8ePHy9uvfXWWs8NHz5cTJw40W5ZPW4E5Xq2bt2KxMREREZG2p4bPnw4DAYDdu3aVed7du3ahaqqKgwbNsz2XGRkJBITE7FlyxaHZ26I1atXo6CgAFOnTr3uths2bEBYWBjatWuHhx9+GPn5+Y4P2Ejz5s1D8+bN0a1bN7z66qvXPP2RmZmJvLy8WsdLo9Fg4MCBTne8rqa4uBghISHX3c7ZjqHRaMSuXbtq/bcHgGHDhl31v/3WrVuv2H748OHYuXMnqqqqHJbVXoqLiwGgXscrKSkJERERGDJkCNavX+/oaI12/PhxREZGIj4+HhMnTsSpU6euuq2rHz+j0YhPP/0UDzzwwHVvTusqx++PGvt5eLXjas/PUBaUy+Tl5SE8PLzWc8HBwfD29kZeXt5V3+Pt7Y3g4OBaz4eHh1/1PbJ8+OGHGD58OGJiYq653YgRI/DZZ5/h119/xRtvvIH09HQMHjwYBoOhiZLW31/+8hcsW7YM69evx4wZM/DWW29h2rRpV92+5phcfpyd8XjV5eTJk3jnnXfw6KOPXnM7ZzyGBQUFMJvNDfpvX9f/J8PDw2EymVBQUOCwrPYghMBTTz2F/v37IzEx8arbRUREYPHixVixYgVWrlyJ9u3bY8iQIdi4cWMTpq2f3r174+OPP8batWvxwQcfIC8vDykpKbh48WKd27vy8QOAb775BkVFRdf8R50rHb/LNfbz8GrH1Z6foS55N+PLzZ49Gy+//PI1t0lPT7/unIsadbVkIcR127M93lNfjdnnc+fOYe3atfjyyy+v+/MnTJhg+9+JiYlITk5GbGwsfvjhB4wbN67xweupIfv35JNP2p7r0qULgoODcdddd9lGVa7m8mPjyONVl8Ycw5ycHNx66624++678dBDD13zvbKP4bU09L99XdvX9byzmTFjBvbv349NmzZdc7v27dujffv2tj/37dsXWVlZeP3113HTTTc5OmaDjBgxwva/O3fujL59+6J169ZYunQpnnrqqTrf46rHD7D+o27EiBG1RtUv50rH72oa83no6M9QtygoM2bMwMSJE6+5TVxcXL1+lk6nu2KSz6VLl1BVVXVFW/zje4xGIy5dulRrFCU/Px8pKSn1+nsbqjH7vGTJEjRv3hy33XZbg/++iIgIxMbG4vjx4w1+b2PcyDGtuVLlxIkTdRaUmisO8vLyEBERYXs+Pz//qsfYERq6jzk5Obj55pvRt29fLF68uMF/X1Mfw7qEhoZCpVJd8a+sa/231+l0dW6vVquvWUBle/zxx7F69Wps3LgR0dHRDX5/nz598OmnnzogmX35+/ujc+fOV/29ctXjBwBnzpzBzz//jJUrVzb4va5y/Br7eXi142rPz1C3KCihoaEIDQ21y8/q27cvXn31VeTm5toO1rp166DRaNCjR48639OjRw94eXkhNTUV48ePBwDk5ubi4MGDmD9/vl1yXa6h+yyEwJIlS3D//ffDy8urwX/fxYsXkZWVVesX2JFu5Jju2bMHAK6aNT4+HjqdDqmpqUhKSgJgPc+clpaGefPmNS5wIzRkH7Ozs3HzzTejR48eWLJkCZTKhp+dbepjWBdvb2/06NEDqampuOOOO2zPp6am4vbbb6/zPX379sV3331X67l169YhOTm5Ub/LjiaEwOOPP45Vq1Zhw4YNiI+Pb9TP2bNnj9RjVV8GgwGHDx/GgAED6nzd1Y7fHy1ZsgRhYWEYNWpUg9/rKsevsZ+Hffv2RWpqaq0R7HXr1tn3H+V2m27rIs6cOSP27NkjXn75ZREQECD27Nkj9uzZI0pKSoQQQphMJpGYmCiGDBkidu/eLX7++WcRHR0tZsyYYfsZ586dE+3btxfbt2+3Pffoo4+K6Oho8fPPP4vdu3eLwYMHi65duwqTydTk+1iXn3/+WQAQhw4dqvP19u3bi5UrVwohhCgpKRFPP/202LJli8jMzBTr168Xffv2FVFRUUKv1zdl7OvasmWLWLBggdizZ484deqUWL58uYiMjBS33XZbre3+uH9CCPHaa68JrVYrVq5cKQ4cOCDuueceERER4XT7J4T1irA2bdqIwYMHi3Pnzonc3Fzb449c5RguW7ZMeHl5iQ8//FAcOnRIzJw5U/j7+4vTp08LIYR47rnnxOTJk23bnzp1Svj5+Yknn3xSHDp0SHz44YfCy8tLfP3117J24Zoee+wxodVqxYYNG2odq/Lycts2l+/jm2++KVatWiWOHTsmDh48KJ577jkBQKxYsULGLlzT008/LTZs2CBOnToltm3bJkaPHi0CAwPd5vjVMJvNomXLluLZZ5+94jVXO34lJSW27zoAts/MM2fOCCHq93k4efLkWlfabd68WahUKvHaa6+Jw4cPi9dee02o1Wqxbds2u+X2uIIyZcoUAeCKx/r1623bnDlzRowaNUr4+vqKkJAQMWPGDFFZWWl7PTMz84r3VFRUiBkzZoiQkBDh6+srRo8eLc6ePduEe3Zt99xzj0hJSbnq6wDEkiVLhBBClJeXi2HDhokWLVoILy8v0bJlSzFlyhSn2p8au3btEr179xZarVb4+PiI9u3bi5deekmUlZXV2u6P+yeE9dK6l156Seh0OqHRaMRNN90kDhw40MTp62fJkiV1/s5e/u8LVzqG//nPf0RsbKzw9vYW3bt3r3UJ7pQpU8TAgQNrbb9hwwaRlJQkvL29RVxcnHjvvfeaOHH9Xe1Y/fH37/J9nDdvnmjdurXw8fERwcHBon///uKHH35o+vD1MGHCBBERESG8vLxEZGSkGDdunMjIyLC97urHr8batWsFAHH06NErXnO141dzGfTljylTpggh6vd5OHDgQNv2Nb766ivRvn174eXlJRISEuxeyBRCVM9WIiIiInISvMyYiIiInA4LChERETkdFhQiIiJyOiwoRERE5HRYUIiIiMjpsKAQERGR02FBISIiIqfDgkJEREROhwWFiIiInA4LChERETkdFhQiIiJyOiwoRERE5HT+HxnXmNYBurDrAAAAAElFTkSuQmCC\n", 64 | "text/plain": [ 65 | "
" 66 | ] 67 | }, 68 | "metadata": {}, 69 | "output_type": "display_data" 70 | } 71 | ], 72 | "source": [ 73 | "plt.plot(x, y(x));" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "id": "11b3290c", 79 | "metadata": {}, 80 | "source": [ 81 | "## What if we want the gradient?" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 6, 87 | "id": "934d4b85", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "grad_y = jax.grad(y)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 10, 97 | "id": "43cb2895", 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "data": { 102 | "text/plain": [ 103 | "(Array(0., dtype=float32, weak_type=True),\n", 104 | " Array(4., dtype=float32, weak_type=True))" 105 | ] 106 | }, 107 | "execution_count": 10, 108 | "metadata": {}, 109 | "output_type": "execute_result" 110 | } 111 | ], 112 | "source": [ 113 | "grad_y(0.0), grad_y(2.0)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "04696eaa", 119 | "metadata": {}, 120 | "source": [ 121 | "## Plot Gradients" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 25, 127 | "id": "6163d8d5", 128 | "metadata": {}, 129 | "outputs": [ 130 | { 131 | "data": { 132 | "text/plain": [ 133 | "[]" 134 | ] 135 | }, 136 | "execution_count": 25, 137 | "metadata": {}, 138 | "output_type": "execute_result" 139 | }, 140 | { 141 | "data": { 142 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGdCAYAAADnrPLBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABYvElEQVR4nO3dd3hUVeLG8e9MekISEgIJgUACBARCM/QOUnQFRNeKBeyFYlZxXXV/a9kVFBUbAlawgquCuhYElSpFWqjSWyCEEEp6nbm/Py7EROkkuTOZ9/M88zzrmZvwyriZN2fuOcdmGIaBiIiIiJuyWx1ARERE5GKozIiIiIhbU5kRERERt6YyIyIiIm5NZUZERETcmsqMiIiIuDWVGREREXFrKjMiIiLi1rytDlAVnE4nqampBAcHY7PZrI4jIiIi58AwDLKzs4mOjsZuP/38i0eUmdTUVGJiYqyOISIiIhcgJSWF+vXrn/Z5jygzwcHBgPmXERISYnEaERERORdZWVnExMSUvo+fjkeUmZMfLYWEhKjMiIiIuJmz3SKiG4BFRETEranMiIiIiFtTmRERERG3pjIjIiIibk1lRkRERNyayoyIiIi4NZUZERERcWsqMyIiIuLWVGZERETErVVZmRk/fjw2m42kpKTSMcMweOqpp4iOjiYgIIDevXuzadOmcl9XWFjI6NGjiYiIICgoiCFDhrB///6qii0iIiIurkrKzMqVK3nrrbdo3bp1ufEJEyYwceJEJk2axMqVK4mKiqJ///5kZ2eXXpOUlMTs2bOZOXMmS5YsIScnh0GDBuFwOKoiuoiIiLi4Si8zOTk53Hzzzbz99tuEhYWVjhuGwSuvvMITTzzBNddcQ0JCAu+//z55eXl88sknAGRmZvLuu+/y0ksv0a9fP9q1a8dHH33Ehg0b+PHHHys7uoiIiLiBSi8zI0eO5Morr6Rfv37lxnfv3k1aWhoDBgwoHfPz86NXr14sXboUgNWrV1NcXFzumujoaBISEkqvOZXCwkKysrLKPTzV6r3HGD1jLav3HrM6ioiIVEMLtx3m4f+uY+OBTMsyVOqp2TNnzmTNmjWsXLnyT8+lpaUBEBkZWW48MjKSvXv3ll7j6+tbbkbn5DUnv/5Uxo8fz9NPP32x8auFT1fu43/rUrHbILFh2Nm/QERE5Dy8t2Q3C7cdJjzIh4R6oZZkqLSZmZSUFB588EE++ugj/P39T3vdH4/1NgzjrEd9n+2axx57jMzMzNJHSkrK+YWvRm7rEgvAdxsOkp5dYG0YERGpVnZn5LJw22FsNrilc0PLclRamVm9ejXp6ekkJibi7e2Nt7c3Cxcu5LXXXsPb27t0RuaPMyzp6emlz0VFRVFUVMSxY8dOe82p+Pn5ERISUu7hqRLqhZLYMIxih8HMXz231ImISMX7cJn5SUrvprVpWCvIshyVVmYuu+wyNmzYQHJycumjffv23HzzzSQnJ9OoUSOioqKYN29e6dcUFRWxcOFCunbtCkBiYiI+Pj7lrjl48CAbN24svUbO7rYuZlv+eMVeih1Oi9OIiEh1kFdUwmerzV+Sb+saa2mWSrtnJjg4mISEhHJjQUFB1KpVq3Q8KSmJcePGER8fT3x8POPGjSMwMJBhw4YBEBoayp133snDDz9MrVq1CA8PZ+zYsbRq1epPNxTL6V2RUJd/1/iNQ1mFzN10iCtb17U6koiIuLkv16aSXVBCw1qB9IqvbWmWSr0B+Gz+/ve/k5+fzwMPPMCxY8fo1KkTc+fOJTg4uPSal19+GW9vb66//nry8/O57LLLmD59Ol5eXhYmdy++3naGdWrAaz9t5/2le1RmRETkohiGwQfL9gBwa+eG2O1nvte1stkMwzAsTVAFsrKyCA0NJTMz02PvnzmUVUC3536mxGnw3ZgetIj2zL8HERG5eCt2HeGGt5YT4OPF8scuIzTQp1L+nHN9/9bZTB4iMsSfgQlRAHy4fI+1YURExK19cOLG36HtoiutyJwPlRkPMvzEMu3Zaw+QmVdsbRgREXFLaZkFzNlkrkS+tXOstWFOUJnxIB1iw7gkKpiCYmfpHegiIiLn45MVe3E4DTrGhrvMLQsqMx7EZrMx/MTyuQ+W7cXprPa3S4mISAUqKnHyya8nl2Nbt0neH6nMeJir2kYT4u/NvqN5LNx22Oo4IiLiRr7feJCMnELqBPsxsGWU1XFKqcx4mEBfb65vHwPA+yeW1YmIiJyLkzf+3typIT5erlMhXCeJVJlbOjfEZoMFWw+zJyPX6jgiIuIGNh7IZPXeY3jbbdzUMcbqOOWozHig2Iggejc1d2v8cPlei9OIiIg7OLlJ3hWt6lIn5PQHSFtBZcZDnTxH47+rUsgrKrE2jIiIuLQjOYV8mZwKwPAurnPj70kqMx6qV3xtGtYKJLughNlrD1gdR0REXNiMX/dRVOKkVb1QEhuGWR3nT1RmPJTdbuO2E5voTf9lDx5wqoWIiFyAohJn6Y2/d3SPxWaz9hymU1GZ8WDXta9PkK8X29NzWLIjw+o4IiLigr7feJD07EJqB/txZatoq+OcksqMBwvx9+G6E8u031uy2+I0IiLiagzDKH1/uLVzQ3y9XbM2uGYqqTIjusZis8H8rYfZeTjH6jgiIuJC1uw7zrr9mfh62RnWqYHVcU5LZcbDxUYEcdklkYB574yIiMhJ034xZ2WGtI0mooafxWlOT2VGuKN7LACfr96v07RFRASA1OP5fL/RPB379m6x1oY5C5UZoUujWlwSFUx+sYOZK/dZHUdERFzAh8vN07E7xYXTMjrU6jhnpDIj2Gw27ugeB8D7S/dQ4nBanEhERKyUX+TgkxXmL7cn3x9cmcqMADCkTTS1gnxJzSzgh02HrI4jIiIWmr32AJn5xcSEB9CveaTVcc5KZUYA8Pfx4ubO5hbV7/2iZdoiIp7KMIzSG3+Hd4nFy+56m+T9kcqMlLqlcwN8vGys3nuM5JTjVscRERELLNmRwfb0HIJ8vbi+g2udjn06KjNSqk6wP4Nbm7s7TtPsjIiIR5p2YpuO69rHEOLvY22Yc6QyI+Xc3s280evb9QdJyyywOI2IiFSlXYdz+HlLOjYbDO8aa3Wcc6YyI+W0qh9Kx9hwSpwGHy7fY3UcERGpQu8v3QNAn2Z1iIsIsjbMeVCZkT85uYneJyv2kV/ksDaMiIhUieN5Rfx31X4A7ujm+suxy1KZkT/p3yKK+mEBHMsrZvbaA1bHERGRKvDxin3kFzu4JCqYbk1qWR3nvKjMyJ942W2MOPFZ6TtLduF0GtYGEhGRSlVY4ij9iOmeno2w2Vx/OXZZKjNySjd0iCHYz5tdh3P5eUu61XFERKQSfZ2cSnp2IZEhfgw6sarVnajMyCkF+/uUHvf+1uJdFqcREZHKYhgG7yw2t+MY0TUOX2/3qwbul1iqzIhusXjbbfy6+6g20RMRqaYWbc9g66Fsgny9Sn+JdTcqM3JadUMDGNLWnG58W7MzIiLV0jsnfr5f3yGG0AD32CTvj1Rm5Izu7tEIgO83HCTlaJ7FaUREpCL9djCLxdszsNvcbzl2WSozckbN64bQIz4CpwHvLtERByIi1cnJWfcrWtUlJjzQ4jQXTmVGzuqenubszH9XpXA8r8jiNCIiUhHSMgv437pU4PdZeHelMiNn1b1JBJdEBZNX5ODjFfusjiMiIhVg+tI9FDsMOsaG0zamptVxLorKjJyVzWYrnZ2ZvnQPhSU64kBExJ3lFpbwyYq9ANzVw33vlTlJZUbOyaDW0USF+HM4u5CvklOtjiMiIhfhv6tSyCooIS4iiH7NI62Oc9FUZuSc+Hrbub1bLABvL9qFYeiIAxERd1TicJYu6Lizexx2u3sdXXAqKjNyzm7q1IAaft5sT89hwbbDVscREZELMGdTGvuP5RMW6MNfL61vdZwKoTIj5yzE34ebOsYA5uyMiIi4F8MwmLJgJwC3dYklwNfL4kQVQ2VGzsvt3eLwtttYuvMIG/ZnWh1HRETOw5IdGWxKzSLAx4sRXWOtjlNhVGbkvETXDGBQ67oAvLlop8VpRETkfJyclbmxYwxhQb4Wp6k4KjNy3u7t1RiA7zYcZE9GrsVpRETkXKxLOc7SnUfwttu4y803yfsjlRk5b83rhtCnWW2cBrype2dERNzC1IXmrMyQttHUqxlgcZqKpTIjF+SBPk0A+GL1ftKzCixOIyIiZ7LzcA5zNqUBcN+J2fXqRGVGLkiH2HDaNwyjqMx+BSIi4prM/cGgX/M6NI0MtjpOhVOZkQt2f2+z3X+0fC+ZecUWpxERkVM5lFXArDUHgN9/blc3KjNywfpeUodmkcHkFjn4cPkeq+OIiMgpvLdkN0UOJx1jw0lsGG51nEqhMiMXzGazlbb8ab/sIb9IB1CKiLiSzLxiPlpuHih5X+/qtYKpLJUZuSiDWtelflgAR3KL+Gx1itVxRESkjI9W7CW3yEGzyGD6NKtjdZxKozIjF8Xby869Pc22/+bCXRQ7nBYnEhERgIJiB9N+MRdo3N+7MTab+x8oeTqVWmamTJlC69atCQkJISQkhC5duvD999+XPm8YBk899RTR0dEEBATQu3dvNm3aVO57FBYWMnr0aCIiIggKCmLIkCHs37+/MmPLebqufQwRNXw5cDyfb9anWh1HRESAz1bvJyOniHpldm6vriq1zNSvX5/nnnuOVatWsWrVKvr27ctVV11VWlgmTJjAxIkTmTRpEitXriQqKor+/fuTnZ1d+j2SkpKYPXs2M2fOZMmSJeTk5DBo0CAcDt2f4Sr8fby4vVscYG6V7XQaFicSEfFsJQ4nb504cuaeno3w9qreH8TYDMOo0nee8PBwXnjhBe644w6io6NJSkri0UcfBcxZmMjISJ5//nnuvfdeMjMzqV27Nh9++CE33HADAKmpqcTExPDdd98xcODAc/ozs7KyCA0NJTMzk5CQkEr7d/NkmfnFdHvuZ3IKS3jntvb0axFpdSQREY81a81+HvrvOmoF+bLk0b5uezr2ub5/V1lVczgczJw5k9zcXLp06cLu3btJS0tjwIABpdf4+fnRq1cvli5dCsDq1aspLi4ud010dDQJCQml15xKYWEhWVlZ5R5SuUIDfLi5cwMAJi/YQRV3ZBEROcHhNHhj/g4A7urRyG2LzPmo9DKzYcMGatSogZ+fH/fddx+zZ8+mRYsWpKWZ2ypHRpb/DT4yMrL0ubS0NHx9fQkLCzvtNacyfvx4QkNDSx8xMTEV/G8lp3Jn9zh8ve2s2XecZbuOWB1HRMQjzdmYxs7DuYQG+HDLiV8yq7tKLzPNmjUjOTmZ5cuXc//99zN8+HA2b95c+vwf7642DOOsd1yf7ZrHHnuMzMzM0kdKipYMV4U6wf7c0N4sjq//tMPiNCIinsfpNHj95+0A3N4tlmB/H4sTVY1KLzO+vr40adKE9u3bM378eNq0acOrr75KVFQUwJ9mWNLT00tna6KioigqKuLYsWOnveZU/Pz8SldQnXxI1bivd2N8vGws23WEVXuOWh1HRMSj/LQlnS1p2dTw82ZE11ir41SZKr+92TAMCgsLiYuLIyoqinnz5pU+V1RUxMKFC+natSsAiYmJ+Pj4lLvm4MGDbNy4sfQacS31agbw10vrA/Daz5qdERGpKoZhMOnErMytXRpSM9DX4kRVx7syv/njjz/OFVdcQUxMDNnZ2cycOZMFCxYwZ84cbDYbSUlJjBs3jvj4eOLj4xk3bhyBgYEMGzYMgNDQUO68804efvhhatWqRXh4OGPHjqVVq1b069evMqPLRXigdxM+W72fRdsOk5xynLYxNa2OJCJS7S3ensG6/Zn4+9i5q3uc1XGqVKWWmUOHDnHrrbdy8OBBQkNDad26NXPmzKF///4A/P3vfyc/P58HHniAY8eO0alTJ+bOnUtw8O/Hk7/88st4e3tz/fXXk5+fz2WXXcb06dPx8qr+d2e7qwa1ArmqbTSz1hxg0s/beWd4B6sjiYhUe5NOzIbf3KkhtWr4WZymalX5PjNW0D4zVW/n4Rz6TVyIYcC3Y7rTMjrU6kgiItXWil1HuOGt5fh62Vn8aB8iQ/ytjlQhXG6fGfEsjWvXYFDraOD33xZERKRyTDqxr8z1HepXmyJzPlRmpNKM6tMEgO83prHtUPZZrhYRkQuxdt8xFm/PwNtu496eja2OYwmVGak0zaKCubyluQRfszMiIpXj5G6/V7erR0x4oMVprKEyI5VqVF9zduab9ansOpxjcRoRkeplU2omP/6Wjt0G9/f2zFkZUJmRSpZQL5S+l9TBacDkBTutjiMiUq289pO5r8yVraNpVLuGxWmsozIjlW70idmZ2WsPkHI0z+I0IiLVw6bUTH7YdAibDcac+DnrqVRmpNK1axBGj/iIcie5iojIxXnlR3NWZnDraOIjg89ydfWmMiNV4sHL4gH4fPV+9h3R7IyIyMXYeCCTeZsPYbfBmBM/Xz2ZyoxUifax4fSIj6CkzImuIiJyYV75cRsAQ9pE06SO594rc5LKjFSZv/VvCsCstQfYk5FrcRoREfe0Yf/vK5g0K2NSmZEqc2mDMPo0q43DafCaZmdERC7IyVmZoW3refQKprJUZqRKnZyd+XLtAXZq3xkRkfOyLuU4P21Jx8tuY7RmZUqpzEiVal2/Jv2am/vOnNwfQUREzk3ZWZm4iCCL07gOlRmpckn9zNmZr9elsl1nNomInJO1+44xf+thvOw2xlzm2fvK/JHKjFS5hHqhDGwZiWHAK5qdERE5Jyf3lbmmXT0a1tKsTFkqM2KJk7Mz364/yJa0LIvTiIi4ttV7j7FwmzkrM8rDd/s9FZUZsUTzuiFc2aouAK/M0+yMiMiZnLxX5q+XalbmVFRmxDIP9ovHZoM5m9LYlJppdRwREZf06+6jLN6egbfdxui+WsF0KiozYpmmkcEMbh0N/P5ZsIiI/M4wDF74YQsAN3SIISY80OJErkllRiw15rJ47DaYt/kQ61KOWx1HRMSlLNx2mJV7juHnbdeszBmozIilmtSpwdB29QB44YetFqcREXEdTqdR+nNxeNdYokL9LU7kulRmxHJ/69cUHy8bS3ZksHRHhtVxRERcgnk/YRY1/Ly5r1djq+O4NJUZsVxMeCDDOjYA4PkftmIYhsWJRESsVeJw8tJcc1bmrh5xhAf5WpzItanMiEsY1TeeAB8v1qUcZ+7mQ1bHERGx1Oy1B9h5OJewQB/u7B5ndRyXpzIjLqF2sB93dI8F4MUftuJwanZGRDxTYYmjdIXnA72bEOzvY3Ei16cyIy7jnp6NCQ3wYXt6Dl+uPWB1HBERS8z8NYUDx/OJDPHj1i4NrY7jFlRmxGWEBvhwf2/zJreXf9xGYYnD4kQiIlUrr6iE13/eAcDovvH4+3hZnMg9qMyISxneJZY6wX7sP5bPjBX7rI4jIlKlpi/dQ0ZOIQ3CA7m+fYzVcdyGyoy4lABfL8ZcZm4MNWn+DnILSyxOJCJSNTLzipm6YCcAf+sfj6+33qLPlf6mxOXc0CGGhrUCycgpYtovu62OIyJSJSYv2EFWQQlNI2swpE09q+O4FZUZcTk+XnYe6t8UgDcX7eJ4XpHFiUREKteB4/lMW7oHgH9ccQledpu1gdyMyoy4pMGto7kkKpjsghKmnJh2FRGpribO3UZRiZNOceH0aVbH6jhuR2VGXJLdbuPvlzcDYNrSPRw4nm9xIhGRyvHbwSxmrd0PwGN/aY7NplmZ86UyIy6rT7M6dG4UTlHJ79t6i4hUN8/P2YJhwJWt6tI2pqbVcdySyoy4LJvNxuN/aQ6YW3tvSs20OJGISMVaujODBVsP42238cjAZlbHcVsqM+LSWtevyZA20RgGjP9uiw6hFJFqw+k0eO77LQDc3KkBsRFBFidyXyoz4vIeGdgMXy87S3ZksHDbYavjiIhUiG82HGT9/kyCfL0YfWJ/LbkwKjPi8mLCA7ntxPkkz32/RYdQiojbKypx8uIP5r2A9/ZqTEQNP4sTuTeVGXELo/o2IcTfmy1p2XyxZr/VcURELsrHK/ay72getYP9uKtHnNVx3J7KjLiFmoG+jO5rTsO+NHcr+UU6hFJE3FN2QXHpYZJJ/eIJ9PW2OJH7U5kRt3Fb14bUDwvgUFYh7y7ZZXUcEZEL8sb8nRzNLaJR7SBu0GGSFUJlRtyGn7dX6dLFqQt3kZFTaHEiEZHzk3I0j/eWmGfOPX5Fc7y99DZcEfS3KG5lcOtoWtULJaewhNd+2m51HBGR8/Lc91socjjp1qQWlzXXsQUVRWVG3Ird/vtGeh+v2Mf2Q9kWJxIROTcr9xzl2w0Hsdvgn1e20LEFFUhlRtxOl8a16N8iEofT4D/f/mZ1HBGRs3I6Df79zWYAbugQQ/O6IRYnql5UZsQtPfGX5vh42Vi47TDzt6RbHUdE5Iy+TD7A+v2Z1PDz5qH+OragoqnMiFuKjQji9m7m3gz//nYzxQ6nxYlERE4tr6iECXPMDfIe6NOY2sHaIK+iqcyI2xrVtwm1gnzZdTiXD5fttTqOiMgpvbVoF2lZBdQPC+CObtogrzKozIjbCvH34eEB5nTtKz9u41hukcWJRETKS8ss4M2F5r5Y/7jiEvx9vCxOVD2pzIhbu6FDDJdEBZNVUMLLP26zOo6ISDkTfthCfrGD9g3DuLJVXavjVFsqM+LWvOw2/jW4BWAu1d6mpdoi4iLW7z/OrDUHAPi/QVqKXZkqtcyMHz+eDh06EBwcTJ06dRg6dChbt24td41hGDz11FNER0cTEBBA79692bRpU7lrCgsLGT16NBEREQQFBTFkyBD279dhg2Lq2jiCgS3Npdr//mYzhqFTtUXEWk6nwVNfm+9lV7erR5uYmtYGquYqtcwsXLiQkSNHsnz5cubNm0dJSQkDBgwgNze39JoJEyYwceJEJk2axMqVK4mKiqJ///5kZ//+G3ZSUhKzZ89m5syZLFmyhJycHAYNGoTDocMGxfT4X5rj62Vn8fYM5m/VUm0RsdbstQdYs+84gb5ePHr5JVbHqfZsRhX+Gnv48GHq1KnDwoUL6dmzJ4ZhEB0dTVJSEo8++ihgzsJERkby/PPPc++995KZmUnt2rX58MMPueGGGwBITU0lJiaG7777joEDB571z83KyiI0NJTMzExCQrRRUXU1/vvfeHPhLhpFBDEnqSe+3voUVUSqXnZBMX1eXEhGTiGPXn4J9/dubHUkt3Wu799V+tM+MzMTgPDwcAB2795NWloaAwYMKL3Gz8+PXr16sXTpUgBWr15NcXFxuWuio6NJSEgoveaPCgsLycrKKveQ6m9UnyZE1PBjV0Yu7/2y2+o4IuKhXv1xOxk5hcRFBHFH91ir43iEKiszhmHw0EMP0b17dxISEgBIS0sDIDIysty1kZGRpc+lpaXh6+tLWFjYaa/5o/HjxxMaGlr6iInREeueINjfh39cYU7nvvbTdg5m5lucSEQ8zY70bKYv3QPAvwa3wM9bS7GrQpWVmVGjRrF+/XpmzJjxp+f+eIe3YRhnvev7TNc89thjZGZmlj5SUlIuPLi4lWva1SOxYRh5RQ6e1blNIlKFDMPgqa83U+I06Ne8Dn2a6VTsqlIlZWb06NF8/fXXzJ8/n/r165eOR0VFAfxphiU9Pb10tiYqKoqioiKOHTt22mv+yM/Pj5CQkHIP8Qx2u41nrmqJ3QbfrD/I0p0ZVkcSEQ/xw6Y0luzIwNfbzv8NamF1HI9SqWXGMAxGjRrFrFmz+Pnnn4mLK7+Nc1xcHFFRUcybN690rKioiIULF9K1a1cAEhMT8fHxKXfNwYMH2bhxY+k1ImW1jA7lls4NAXjyq006t0lEKl1+kYN/f2POBt/ToxENawVZnMizeFfmNx85ciSffPIJX331FcHBwaUzMKGhoQQEBGCz2UhKSmLcuHHEx8cTHx/PuHHjCAwMZNiwYaXX3nnnnTz88MPUqlWL8PBwxo4dS6tWrejXr19lxhc39nD/Zny7/iDb03N4f+ke7urRyOpIIlKNTV24kwPH84kO9eeBPlq9VNUqtcxMmTIFgN69e5cbnzZtGiNGjADg73//O/n5+TzwwAMcO3aMTp06MXfuXIKDg0uvf/nll/H29ub6668nPz+fyy67jOnTp+PlpRur5NRCA3149PJL+PsX63nlx+0MaRNNnRB/q2OJSDWUcjSPqQt3AvD4lc0J9K3Ut1Y5hSrdZ8Yq2mfGMzmdBtdMWUpyynGGto3mlRvbWR1JRKqhuz9YxbzNh+jSqBaf3N1JxxZUIJfcZ0akKp28Gdhmgy+TU1mx64jVkUSkmpm7KY15mw/hbbfx9FUtVWQsojIj1Vrr+jW5qWMDAJ78ehMluhlYRCpIbmFJ6flLd/dsRNPI4LN8hVQWlRmp9h4Z0IywQB+2pP2+mZWIyMV67aftpGYWUD8sgDF9462O49FUZqTaCwvy5bErmgPw0txt7D+WZ3EiEXF3W9KyeGeJeWzKM1e1JMBXC1KspDIjHuG69vXpGBdOfrGDJ7/ahAfc9y4ilcTpNPjn7I04nAaXt4yi7yWn3sBVqo7KjHgEm83GuKsT8PGy8dOWdH7YdOpzvUREzuaz1Sms2nuMIF8vnhyinX5dgcrMxcg/DhtngdNhdRI5B03qBHNfL3Mzqye/3kR2QbHFiUTE3RzJKWT891sA+Fv/ptQNDbA4kYDKzMVZ+TZ8fju80QmSPwGH3hxd3cg+TYitFcihrEJemrvN6jgi4mbGf7+F43nFNK8bwoiusVbHkRNUZi6GTxD414Qj2+HL++H1S2Hlu1BcYHUyOQ1/Hy/+M7QVAO8v28O6lOPWBhIRt7F81xE+X70fmw3GXZ2At5feQl2FXomL0eUB+NtG6Pc0BNWG4/vg24fg1Taw7A0oyrU6oZxC9/gIhraNxjDg8dkbtPeMiJxVQbGDJ2ZvAOCmjg1o1yDM4kRSlsrMxfILhu5JkLQBrpgAIfUgJw1+eBxeaQWLXoSCTKtTyh/8c1ALQgN82JSapb1nROSs3pi/g52Hc6kd7MejAy+xOo78gcpMRfEJgE73wphkGPwahMVB3hH4+d/wciv4+T+Qq+30XUVEDT/+cYX5A2nivG0cOJ5vcSIRcVW/HcxiygLzIMlnhrQkNNDH4kTyRyozFc3bFxKHw6hVcM07UPsSKMyERS/AKwnwwxOQrWXBruCG9jG0bxhGXpE5fay9Z0TkjxxOg398sZ4Sp8HAlpFc0aqu1ZHkFFRmKouXN7S+Du5fBjd8BHXbQHEeLJsEr7SGbx6CY3utTunR7HYbz/21Fb5edhZsPcysNQesjiQiLmbaL7tZtz+TYH9vnrkqweo4choqM5XNbofmg+GehXDzFxDTGRyFsOpdc/XTlw9Axg6rU3qsJnWCebCfeabKM99sJj1bK9FExLTvSB4vzt0KwON/aU5kiL/FieR0VGaqis0G8f3gjjkw4lto1AecJZD8MUxqD5/dDmkbrU7pke7p2YiW0SFk5hfz5FebrI4jIi7AMAwen72BgmInnRuFc2OHGKsjyRmozFQ1mw1iu8NtX8JdP0OzvwAGbJoFU7vBjJtg/2qrU3oUHy87E65tjbfdxvcb0/h+w0GrI4mIxT5bvZ8lOzLw87bz3DWtsdlsVkeSM1CZsVL9RLhpBtz3C7S8BrDB1u/gnb7wwVDY84vVCT1Gy+jQ0qMO/u+rTRzPK7I4kYhYJT27gP98sxkwjyyIjQiyOJGcjcqMK4hKgOumwaiV0PZmsHvDrvkw/S/w3uWw/UfQSptKN/qyJjSpU4OMnEKeOfGDTEQ8i2GYJ2JnFZSQUC+Eu7rHWR1JzoHKjCuJiIehk2H0Gmh/J3j5wb5l8PFf4a3e8Nv/wKndaiuLn7cXz/+1NTYbzFpzgPlb062OJCJV7KvkVOZuPoS33cbzf22tIwvchF4lVxTWEAZNhAfXQZdR4BMIB5Ph01tgSldY/xk4SqxOWS0lNgzj9q7mb2JPzNpAlk7WFvEYh7IKePJrcxHAmMviaRkdanEiOVcqM64spC4MfBaSNkLPR8AvBA7/BrPuMldArfkASnRvR0UbO7ApDcIDSc0s4N//08dNIp7AMAwem7WBzPxiWtUL5f7eja2OJOdBZcYdBNWCvv80D7Xs+38QEA7HdsPXo+G1drDiLSjWdvwVJdDXmxeva4PNZq5o+HHzIasjiUgl+3z1fn7eko6vl52Xrm+Djz5ecit6tdyJfyj0HGuWmoHjoEYUZO2H7x8xdxX+5VUozLY6ZbXQMS6cu3s0AuAfszZwNFczYCLVVerxfJ753++rl5pGBlucSM6Xyow78g2CLiPNe2qunAg1G0BuOsz7F7ycAAueg/xjVqd0ew/1b0r8idVN//xSZzeJVEeGYfDoF+vJLiyhbUxN7u6h1UvuSGXGnfn4Q4c7zdVPQ6dArXgoOA4Lxpsndc97EnIOW53Sbfn7eDHx+rZ42218tyGNr9elWh1JRCrYjF9TWLzd3BzvpevbaPWSm9KrVh14+UDbYTByBVw7DSIToCgbfnkFXmkF3z8KmTpE8UK0qh/KqL5NAPjXV5s4lKWzm0Sqi5SjeTz7rfnx0iMDm9G4dg2LE8mFUpmpTuxekHAN3LcEbpoJ9dpDST6smAqvtoGvx8DRXVandDsj+zShVb1QMvOL+fvn6/Vxk0g1UOJw8rdPk8ktctAhNozbu+njJXemMlMd2WzQ7Aq460e49UuI7QHOYljzPryeCLPugfQtVqd0Gz5ediZe3wZfbzsLtx1mxq8pVkcSkYs0deFOVu09Rg0/byZe3xYvu85ecmcqM9WZzQaN+8CIb+COH6BJfzCcsP5TmNwZPr0VDq6zOqVbiI8M5pEBzQD4z7eb2Z2Ra3EiEblQ61KO88qP2wF4ekhLYsIDLU4kF0tlxlM06Ay3fA73LIDmgwEDfvsa3uwJH10L+1ZYndDl3dk9ji6NapFX5ODBmWspKtHREiLuJq+ohKRPkylxGlzZui7XXFrP6khSAVRmPE10O7jhI3hgObS6Hmx22DEP3hsA0wfBrgU61PI07HYbE29oQ2iAD+v3ZzJx3jarI4nIefr3N7+xOyOXuqH+jBvaCptNHy9VByoznqpOc/jr2zBqFVx6G9h9YM9i+OAqeKcfbJ2jUnMKdUMDeP6vrQB4c9FOlu7MsDiRiJyruZvSmPHrPmw2eOn6NoQG+lgdSSqIyoynq9UYhrwODyZDp/vA2x8OrIIZN8DUHrBxFjgdVqd0KZcn1OXGDjEYBjz06TqOaXdgEZeXnl3AP2ZtAODuHo3o2jjC4kRSkVRmxBRaH654HpI2QLck8K0BhzbA57fDG50geQY4dIL0Sf8a3IJGEUGkZRXw2CztDiziypxOg79/vp6juUW0qBvCwwOaWh1JKpjKjJRXow70f9osNb0fA/+acGQ7fHkfvH4prHoPSgqtTmm5QF9vXr2xHT5eNuZsSuPTlVquLeKq3l2ymwVbD+PnbeeVG9vi5+1ldSSpYCozcmqB4dD7H+ahlv2ehqDacHwffPM3cwO+ZW9AkWcvT25VP5SxJ5ZrP/2/zew8nGNxIhH5o7X7jvH8HHNfrX8NbqFDJKsplRk5M79g6J5kztRcMQFC6kH2QfjhcfOohEUvQkGm1Sktc3ePRnRrUov8YgejPllLQbHuLxJxFZl5xYz6ZG3pMuxhHRtYHUkqicqMnBufAOh0L4xJhsGvQVgc5B2Bn/9tHmr5838g94jVKauc3W5j4vVtqRXky28Hs3j6f5utjiQi/H4a9oHj+TQID2T8NVqGXZ2pzMj58faFxOHmku5r3oHal0BhJix6wZyp+eEJyE6zOmWVigzx55Ub22KzwYxf9/FVsg71FLHah8v3MmdTGj5eNiYNa0eIv5ZhV2cqM3JhvLyh9XVw/zJzE766baA4F5ZNgldaw7cPm/fYeIge8bUZ3cc8XfuxWRt0/4yIhTYeyOQ/3/wGwD+uaE7r+jWtDSSVTmVGLo7dbh6PcM9CuPkLiOkMjkJY+Q681g6+HAkZO6xOWSUe7NeUzo3CyStyMPLjNeQX6f4ZkaqWU1jC6BlrKXI46dc8kju6xVodSaqAyoxUDJsN4vvBHXNgxLfQqDc4SyD5I3ijA3x2O6RttDplpfKy23jtxnZE1PBlS1o2T/9vk9WRRDyKYRg8PmsDuzNyiQ7158XrWus+GQ+hMiMVy2aD2O5w21dw10/Q9ArzpO5Ns2BqN5hxE+xfbXXKSlMnxJ9Xb2yHzQYzV6Ywe+1+qyOJeIz3l+7h63Wp5i8WN7WjZqCv1ZGkiqjMSOWp3x6GzYT7lkDLqwEbbP0O3ukLHwyFPb9YnbBSdGsSwZi+8QA8Pmsj2w9lW5xIpPpbvfco//nWvE/msSsuoX1suMWJpCqpzEjli2oF102HUSuh7c1g84Jd82H6X+C9y2H7j9XuUMsxl8XTtbG5/8y9H64mq0BHQYhUlsPZhTzw8ZrS/WTu7B5ndSSpYiozUnUi4mHoZBizFtrfCV6+sG8ZfPxXeKs3/PY/cDqtTlkhTk5z1w31Z1dGLg99ug6ns3oVNhFXUOJwMuqTNRzKKqRJnRpM+Kvuk/FEKjNS9cIawqCJ8OB66DIKfALhYDJ8egtM6QrrPwNHidUpL1pEDT+m3pKIr7edH387xKT5nrGqS6QqTfhhKyt2HyXI14uptyQS5OdtdSSxgMqMWCekLgx8FpI2Qo+x4BcCh3+DWXfBpPaw5gMoKbI65UVpE1OT/1yVAMDLP25j/pZ0ixOJVB/fbzjIW4t2AfDCdW1oUqeGxYnEKiozYr2gWnDZ/5nnP/X9JwSEw7Hd8PVoc6+aFW9Bcb7VKS/Y9R1iGNapAYYBD85cy54Mzz6gU6QibDuUzSOfrwfgnp6N+EuruhYnEiupzIjrCKgJPR8xT+oe8CzUiIKs/fD9I+auwr+8CoXuuTLoycEtaNegJlkFJdz30Wryitz/YzQRqxzPK+LuD1aRU1hC50bh/H1gM6sjicUqtcwsWrSIwYMHEx0djc1m48svvyz3vGEYPPXUU0RHRxMQEEDv3r3ZtKn8RmOFhYWMHj2aiIgIgoKCGDJkCPv3a++Oas03CLqOggfXwZUTIbQB5KbDvH/Bywmw4HnIP2Z1yvPi521+nh9Rw48tadn8/fP1GNVsBZdIVShxOBn5yRr2HsmjflgAk29OxNtLv5d7ukr9LyA3N5c2bdowadKkUz4/YcIEJk6cyKRJk1i5ciVRUVH079+f7Ozff/tOSkpi9uzZzJw5kyVLlpCTk8OgQYNwOLRVfLXn4w8d7oQxa2DoFKjVBAqOw4Jx5kndPz4FOYetTnnOIkP8mXLLpXjbbXyz/iBv6IZgkfP2n29/45cdRwj09eKd4e0JD9LGeAI2o4p+PbTZbMyePZuhQ4cC5qxMdHQ0SUlJPProo4A5CxMZGcnzzz/PvffeS2ZmJrVr1+bDDz/khhtuACA1NZWYmBi+++47Bg4ceE5/dlZWFqGhoWRmZhISElIp/35SBZwO2PwVLH4JDp04GsE7ABJHQNfREFrP0njn6uMVe3litpl/ys2XcoU+6xc5J5+u3MejX2wAYOotiVyeEGVxIqls5/r+bdnc3O7du0lLS2PAgAGlY35+fvTq1YulS5cCsHr1aoqLi8tdEx0dTUJCQuk1p1JYWEhWVla5h1QDdi9IuMbcUfimmVCvPZTkw4op8Gob+HoMHN1tdcqzurlTQ0Z0jQXgof+uY+OBTGsDibiBVXuO8s8vzV8CHurfVEVGyrGszKSlpQEQGRlZbjwyMrL0ubS0NHx9fQkLCzvtNacyfvx4QkNDSx8xMTEVnF4sZbNBsyvgrh/h1i8htgc4i2HN+/B6Isy6B9K3WJ3yjP55ZXN6Nq1NfrGDu95fRXpWgdWRRFzWgeP53PfRaoodBle2qsvovk2sjiQuxvK7pv64U6NhGGfdvfFs1zz22GNkZmaWPlJSUiokq7gYmw0a94ER38AdP0CT/mA4YP2nMLkzfHorHFxndcpT8vayM2lYOxrXDiItq4C7P1hFQbHuAxP5o6yCYu6YtpKMnCJa1A3hBZ2ELadgWZmJijKnCP84w5Kenl46WxMVFUVRURHHjh077TWn4ufnR0hISLmHVHMNOsMtn8M9C6D5YMCA376GN3vCx9fBvhVWJ/yTEH8f3h3egZqBPqzbn8nYz9ZphZNIGcUOJyM/XsPWQ9nUCfbj7eHtCfTVDr/yZ5aVmbi4OKKiopg3b17pWFFREQsXLqRr164AJCYm4uPjU+6agwcPsnHjxtJrRMqJbgc3fAQPLIdW14PNDtvnwnsDYPog2LXApQ61jI0IYsrNiaUrnCbO22Z1JBGXYBgG//flRhZvzyDQ14v3RnSgXs0Aq2OJi6rUMpOTk0NycjLJycmAedNvcnIy+/btw2azkZSUxLhx45g9ezYbN25kxIgRBAYGMmzYMABCQ0O58847efjhh/npp59Yu3Ytt9xyC61ataJfv36VGV3cXZ3m8Ne3YdQquPQ2sPvAnsXwwVXwbn/YOsdlSk2XxrV49mrzyIPXf97BjF/3WZxIxHpTFu5k5soU7DZ4/aZ2JNQLtTqSuLBKXZq9YMEC+vTp86fx4cOHM336dAzD4Omnn+bNN9/k2LFjdOrUiTfeeIOEhITSawsKCnjkkUf45JNPyM/P57LLLmPy5MnndVOvlmYLmfvhl9fMm4RLTtxsG9kKej4MzYeYK6UsNnHuVl77eQdedhtv35ZI30tO/1GqSHX2v3WpjJ6xFoCnh7Rk+InVf+J5zvX9u8r2mbGSyoyUykmHZZNg5btQlGOO1YqHHg9Dq2vBy8eyaIZhMPaz9XyxZj8BPl58em9nWtevaVkeESus2nOUYe+soKjEyR3d4vjX4BZWRxILqcyUoTIjf5J3FFa8CSummrsKA9RsAN3/Bm1vBm8/S2IVO5zcMX0li7dnEFHDl1n3d6NBrUBLsohUtW2Hsrlu6jIy84vp3yKSqbck4mXXyiVPpjJThsqMnFZhtjlLs2wS5J44GiG4rrmjcOII85yoKpZdUMwNby5n88EsGkUE8cX9XQnTlu1SzR04ns9fJy8lLauAtjE1+eTuTlq5JK6/A7CIS/ALhu5J8OB6uGIChNSD7IPww+PwSitY9CIUVO0OvcH+Pky73Vy5sSsjlzveX0luoU7ZlurraG4Rt727grSsAprUqcG0ER1UZOS8qMyIAPgGQqd7YUwyDH4NwuIg7wj8/G/zUMuf/wO5R6osTmSIP9Nv70BogA9r9x3nvo9WU1iiTfWk+sktLOH26SvZeTiX6FB/Prijo2Yi5bypzIiU5e0LicPNJd3XvA21L4HCTFj0gjlT88MTkH36ozQqUnxkMNNv70CgrxeLt2fw4IxkShzOKvmzRapCUYmT+z9ew7qU44QF+vDBnZ2I1l4ycgFUZkROxcsbWl8P9y+D6z+Eum2gONe8t+aV1vDtw3C88veDadcgjLdva4+vl505m9J4bNYGnM5qf5ubeACH02DsZ+tYtO0wAT7mpnhN6tSwOpa4KZUZkTOx26HFELhnIdz8OcR0BkchrHwHXmsHX46EjB2VGqFbkwheu6kddht8tno/z373m449ELfmdBo8MXsDX69LxdtuY+qtibRrEHb2LxQ5DZUZkXNhs0F8f7hjDoz4Fhr1BmcJJH8Eb3SAz++AQ5sq7Y+/PCGKCde2AeDdJbt5/efKLVAilcUwDJ7636bS3X1fubEtvZrWtjqWuDmVGZHzYbNBbHe47Su46ydoegUYTtj4BUzpCjOGwYHVlfJHX5tYn38NMjcQmzhvG1MX7qyUP0ekshiGwbPf/sYHy/Zis8FL17dhUOtoq2NJNaAyI3Kh6reHYTPhviXQ8mrABlu/hbf7wodXw55fKvyPvKN7HGMHNAXgue+38NYiFRpxHy/N3cY7S3YDMP7qVlzdrr7FiaS6UJkRuVhRreC66TBqpbl7sM0Ldv4M0/8C710OO36s0EMtR/WN52/9zEIz7rstvLN4V4V9b5HK8vpP25k03/x49JmrWnJjxwYWJ5LqRGVGpKJExMPQyTBmLbS/E7x8Yd8y+Oiv8FZv+O0bcFbM0uoH+8Uz5rJ4AP7z7W+8d+K3XRFX9Mb8Hbw0bxsA/7yyObd1ibU2kFQ7Os5ApLJkHTSXcq96D4rzzLHazaHnWPNjqYs8qdswDCbO21Z6M/BTg1swolvcxaYWqTCGYfDyj9t57aftADwysBkj+zSxOJW4E53NVIbKjFgq9wgsnwy/vgWFWeZYeCPzUMvWN5ob9V0gwzB4ce5W3phv3jvzzyubc1ePRhWRWuSiGIbB83O2lt6o/o8rLuG+Xo0tTiXuRmWmDJUZcQn5x2Hl27BsMuQfNcdC6kO3B+HSW8HnwnY+NQyDF37YyuQF5pvG3/o1ZcxlTbDZdNqwWMMwDP79zW+894v58ef/DWrBnd01ayjnT2WmDJUZcSlFubBqGix9HXJOHI0QVAe6joL2d5iHX54nwzB4Y/4OXpxr3pdwd484Hv9LcxUaqXJOp8GTX2/iw+V7Afj30ARu7dzQ4lTirlRmylCZEZdUXGBuurfkVcg8cTSCf03o/AB0ugcCzn9H1PeW7OaZbzYDcFPHBvxnaAJedhUaqRrFDiePfr6eWWsPYLPBc9e04oYOWrUkF05lpgyVGXFpjmJY/19YMhGOnNjZ1zcYOt4FnUdCjfPbHfW/K1P4x6z1OA24qm00L17XBh8vLVyUypVf5OCBj1czf+thvOw2XryutfaRkYumMlOGyoy4BacDNn8JiyfCoY3mmHcAJI6ArqMhtN45f6tv1qeSNDOZEqdB30vqMGlYOwJ9vSsltsjxvCLufH8Vq/cew9/HzuSbL6XvJZFWx5JqQGWmDJUZcSuGAdvmwKIXfj8awe4DbYeZK6DCz+1Gyvlb0rnvo9UUljhpUz+Ud0d0IKKGXyUGF0+UllnAbe+tYNuhHEL8vZl2ewcSG4ZbHUuqCZWZMlRmxC0ZBuxaAItehL1LzDGbF7S6Fro/BHUuOeu3WL33GHe9v5JjecU0rBXI+7d3JDYiqHJzi8fYfiibEdNWcuB4PpEhfnxwRyeaRZ3/Dewip6MyU4bKjLi9vctg8Yvm0QgA2KD5YHMDvrptzviluw7nMHzar6QczSc8yJd3h7enXYPzv7lYpKwl2zO4/+PVZBeU0CgiiA/u7Ej9sECrY0k1ozJThsqMVBupa2HxS/Db/34fix8APcZCg06n/bLD2YXcMX0lGw5k4u9j59Ub2zGwZVQVBJbq6NOV+3hi9kZKnAYdYsN489b2hAdd+OaPIqejMlOGyoxUO+m/mTcKb/wcjBPnPcX2MGdq4nrBKfaXyS0sYeQna1iw9TA2G4wd0IwHejfWXjRyzpxOgxfmbmXKiQ0ar2obzYRrW+PnfXFHc4icjspMGSozUm0d2Qm/vALJM8BZbI7V72DO1DQd+KdSU+xw8u9vNvPBMnNDs6Fto3nur63x99GbkZxZXlEJYz9bx3cbzI0eH7wsnqR+8SrDUqlUZspQmZFqL3M//PIarHkfSgrMsahW0ONhaD7kT4dafrh8L099vQmH06BNTE3evjWROiH+FgQXd5ByNI+7P1jFlrRsfLxsPP/X1lxzqfaQkcqnMlOGyox4jJx086Tule9CUY45FtHUXP3U6lrw8im9dOmODO7/eA2Z+cVEhfjz1m2JtK5f05rc4rIWbz/M6BlrOZ5XTEQNXybfnEjHOC29lqqhMlOGyox4nLyjsOJNWDEVCo6bYzUbQvckaHszeJv7zezJyOWuD1axIz0HXy87/xrcgps7NdBHB4JhGLy5aBcT5mzBaUCbmJpMveVS6oZe2IGoIhdCZaYMlRnxWIXZ5izNskmQe9gcC64LXcdA4nDwDSKroJix/13H3M2HALi6XT2evTpBOwZ7sJzCEh79Yj3frj8IwPXt6/PMVQm6t0qqnMpMGSoz4vGK8mDth/DLq5B1wBwLrAVdRkKHuzD8Qnhr0S4m/LAVh9OgaWQNptySSOPaNazNLVVu44FMRn2yhj1H8vC223hycAtu6dxQs3ViCZWZMlRmRE4oKYJ1M2DJy3BstznmFwqd7oXO97MizWDUjLUczi6khp83z16dwFVtz/1MKHFfhmHw/tI9jPtuC0UOJ9Gh/rw+rJ2OJhBLqcyUoTIj8geOEtg0y9yA7/AWc8wnCNrfTkbre3jg61R+3X0UMD92evqqloT4+5zhG4o7y8wr5pHPf/+osX+LSF64tjU1A7URnlhLZaYMlRmR03A6Ycs35lEJB9eZY15+ONvdwjSu4tlfcnAaUD8sgFduaEv7WP2WXt0s3ZHB2M/WkZpZgK+Xncf+cgkjusbqYyVxCSozZajMiJyFYZjnPi16AVJWmGN2bzIaX82YlF4sPR6O3Qaj+8Yzqm8TfLzs1uaVi5Zf5OD5OVuYvnQPAA1rBTLppktpVT/U2mAiZajMlKEyI3KODAP2LDFnanYtMIdsdtYG9+axwwPYajSgZXQIE65tTctovem5qzX7jvHwf9exOyMXgJs7NeDxvzQnyE8r2MS1qMyUoTIjcgH2r4JFL8K270uH5tOBlwuHsNnWhPt7N2ZU3yY6l8eNFBQ7ePWn7by5cCdOA6JC/Hn+2tb0alrb6mgip6QyU4bKjMhFSNtg3ii86UvA/HGxyNGKSSVDOVa7AxOubU27BmGWRpSzW7z9ME/M3si+o3kAXNOuHk8ObklooG7sFtelMlOGyoxIBTi8zVzSvf5TMBwA/OpsxhslQ6nffhCPXH6JVr+4oIycQv7zzWa+TE4FzNmYZ65qyYCWURYnEzk7lZkyVGZEKtCxPfDLqxhrP8LmKAJgvTOOaV7X0vHyW7ihQ0Psdq2EsVqJw8mMX/fx4txtZOYXY7PB8C6xjB3YjBq6N0bchMpMGSozIpUg6yAsfR3HqvfwKskHYKuzPv8LvYl+195P24a1LA7ouZZsz+Df32xm66FsAFrUDWH8Na1oE1PT2mAi50llpgyVGZFKlJuBY9lkHMvexNdhntS92xnJ8ujb6HbNKBrUqWltPg+yJyOXZ7/7jXknNr+rGejDQ/2bMqxjA7y1nF7ckMpMGSozIlUg/zg5S6ZgWz6FIEcmAKlGLdY2GEHna8ZQK6ymtfmqsYOZ+bz+8w7+uzKFEqeBl93GrZ0bktQvXvcxiVtTmSlDZUakChXmcPDnqfiveoMwh3kkQoYRyua44bQZ+hChNbXyqaIcySlk8oKdfLh8L0UlTgB6N6vNE39pTnxksMXpRC6eykwZKjMiFiguYOe8N6mxahKRznQAjhs12BBzEy2HPkJ4RKTFAd3XoawC3luym4+W7yW3yFxZ1jE2nLEDm9ExTkdOSPWhMlOGyoyIdZzFRWyY8za11r5BfecBAHKMANbVvZa4wX8nul4DixO6j90Zuby1aCdfrD5AkcOciWlVL5SxA5vRMz5C5ylJtaMyU4bKjIj1nCUlbPjxA0JWvkacYzcA+YYvy2oOInzAWNq0aKE341MwDINlO4/wwbK9/LA5jZM/sTvEhnFfr8b0vaSO/t6k2lKZKUNlRsR1GE4nGxZ8SuCyl2lSvBWAIsOLn/z74+z6IL06d9Q+KEBWQTGzVu/nw+V72Xk4t3S8X/M63NersU4wF4+gMlOGyoyICzIM9q3+jpL5E2iUmwxAiWHnW7qxLf5uenXrQYfYMI+adXA4DZbuzGD22gPM2ZhG3on7YYJ8vbjm0vrc2qUhTXVjr3gQlZkyVGZEXFvW1kUc/2E8DY4uBcBp2Pje2YHZQTfSrF03rkioS8vokGpZbAzDYFNqFl+vS+Wr5AMcyiosfa5pZA1u7dyQoe3qEeyvM5TE86jMlKEyI+IejANrOT73OcL2zikd+9nRlkklQ8kIa8sVraIY0CKSNvVruvUmcMUOJyt2HWXe5jR+/C2dA8fzS5+rGejDoNZ1ubpdPS5t4FkzUyJ/VO3KzOTJk3nhhRc4ePAgLVu25JVXXqFHjx7n9LUqMyJuJv03Sha+iNemWdgwV+0sdbTgdcfVLHO2INjfh26NI+jRNIIeTWoTEx7g0m/6hmGw7VAOy3ZmsHTnEZbtOkJ2QUnp8/4+dvo0q8PQdvXo06wOvt7uW9REKlK1KjOffvopt956K5MnT6Zbt268+eabvPPOO2zevJkGDc6+rFNlRsRNHdkJS17GWDcTm7MYgHXE82rRVfzsbAeYBaZ2sB+XNqjJpQ3CuLRhGC2jQwj0te4m4sy8YjYcyGT9geOsT8lk1d6jZOQUlbumVpAvlzWvQ/8WUXRvEkGAr5dFaUVcV7UqM506deLSSy9lypQppWPNmzdn6NChjB8//qxfrzIj4uYy98Mvr8Ga96GkAIDDQU350Odapqa3pMj551mZmPAAmtYJJj4ymPg6NagfFkB0zQAiQ/wrZOajqMTJ4ZxCUo7msetwLrszctidkcv29Bz2Hsn70/X+PnY6xIbTuVEtujSuRZv6NfHS6eIiZ1RtykxRURGBgYF89tlnXH311aXjDz74IMnJySxcuPCs30NlRqSayEmHZZNg5btQZB5q6awVz57m9/KTVw9W7c9h7b7jpGcXnvZb2GxQu4YfETX8CAnwJsTfh5AAH2r4eeNtt2G327DbbNht5r0teUUO8osc5BU5yC4sJiO7iPTsAo7lFZ8xasNagbSqF0rr+qG0jQmjTUwoft6afRE5H+f6/u3ymzlkZGTgcDiIjCy/9XlkZCRpaWmn/JrCwkIKC3//YZaVlVWpGUWkitSoA/2fgW5JsOJNWDEF+5HtNFoylkY1G3J39yS46WaOFtrYdiib7Yey2Xoom12Hc0k9nk9qZgFFJU7SswvPWHjOlY+XjeiaATSKCCIuogZxtYNoHBFEi+gQHfAoUoVcvsyc9Meb+wzDOO0Nf+PHj+fpp5+uilgiYoXAcOjzGHQZCavehWVvwPG98M3fYOEEwruOoXPicDo3qlXuywzD4EhuEQePF3A0r4is/GIy84vJKigmp6AEpwFOw8DpNHAYBr5edgJ8vQj09SLA15safl5E1PCjTrA/dYL9CA3wwa6PikQsVy0/ZjrVzExMTIw+ZhKproryYM0HsPQ1yDLPfyKwlll2OtwF/qHW5hORC3KuHzO5/Po/X19fEhMTmTdvXrnxefPm0bVr11N+jZ+fHyEhIeUeIlKN+QZC5/tgzFoY/CqExULeEfjpGXi5Ffz8LOQdtTqliFQSl5+Zgd+XZk+dOpUuXbrw1ltv8fbbb7Np0yYaNmx41q/XDcAiHsZRAptmwaIXIcM8/wmfIOhwB3QZDcGRZ/56EXEJ1WY100mTJ09mwoQJHDx4kISEBF5++WV69ux5Tl+rMiPioZxO2PINLHoB0tabY15+cOlt0O1BqBljbT4ROaNqV2YuhsqMiIczDNjxo1lqUlaYY3ZvaHMjdH8IajW2Np+InJLKTBkqMyICmKVmzxJY/CLsWmCO2ezQ8mro8TBEtrQ0noiUpzJThsqMiPzJ/lXmPTXbvv99rNmV0PNhqJdoXS4RKaUyU4bKjIicVtoGWPwSbPoSOPHjsHFf6DEWYrtZmUzE46nMlKEyIyJndXgbLHkZ1n8KhsMca9DVnKlpfJl5DoKIVCmVmTJUZkTknB3bA7+8Cms/AseJk66j25kzNc3+AnaX355LpNpQmSlDZUZEzlvWQVj6OqyeBsUnTsGu08K8Ubjl1WDXoZEilU1lpgyVGRG5YLkZsHwy/Po2FJ44tDa8kbmku/UN4K0DJUUqi8pMGSozInLR8o+bhWb5ZMg/cTRCSH3ongTtbgGfACvTiVRLKjNlqMyISIUpzDE/elr6OuQcMseC6kDX0dD+DvCrYW0+kWpEZaYMlRkRqXDFBZD8ESx5BTJTzLGAMOh0P3S6x/zfInJRVGbKUJkRkUrjKIb1/zX3qjm60xzzDYaOd0HnkVCjtrX5RNyYykwZKjMiUumcDtj8JSx6CdI3mWPeAZA4wvwIKrSelelE3JLKTBkqMyJSZZxO2DbHPP/pwGpzzO4D7W6GbkkQHmdpPBF3ojJThsqMiFQ5w4Bd882Zmr1LzDGbF7S6Dno8BLWbWZtPxA2ozJShMiMiltq7zJyp2fHjiQEbtBhibsBXt42l0URcmcpMGSozIuISUteaJ3Vv+eb3sfiB0HMsxHS0LpeIi1KZKUNlRkRcyqHNsGQibPwCDKc5FtfTPP8prqcOtRQ5QWWmDJUZEXFJR3aaJ3WvmwnOYnOsfkdzpiZ+gEqNeDyVmTJUZkTEpWXuh19egzXvQ0mBORbVypypaT5EJ3WLx1KZKUNlRkTcQk46LJsEK9+FohxzLKKpeaNwwrXg5W1tPpEqpjJThsqMiLiVvKOw4k1YMQUKMs2xmg3NQy3b3gzefpbGE6kqKjNlqMyIiFsqyIJV78KyNyD3sDkWHG3uKJw4AnwDLY0nUtlUZspQmRERt1aUB2s+gKWvQdYBcywwAro8AB3uBn/9XJPqSWWmDJUZEakWSgph3QxzBdSxPeaYfyh0vBc63w+B4ZbGE6loKjNlqMyISLXiKDH3qFn8EmRsNcd8gqDDHdBlNARHWptPpIKozJShMiMi1ZLTCVv+Z+4qnLbeHPPyg0tvg24PQs0Ya/OJXCSVmTJUZkSkWjMM2D7PPP8pZYU5ZveGNjdC94egVmNr84lcIJWZMlRmRMQjGAbsWQKLXoDdC80xmx1aXm3uVRPZ0tp8IudJZaYMlRkR8TgpK82Zmm1zfh9rdiX0fBjqJVqXS+Q8qMyUoTIjIh4rbYN5o/CmL4ETP+4b9zWPSojtZmUykbNSmSlDZUZEPN7hbeaS7vWfguEwxxp0NWdqGl+mQy3FJanMlKEyIyJywrE98MursPYjcBSZY9HtzJmaZn/RoZbiUlRmylCZERH5g6xUWDoJVk+D4jxzrE4L80bhlleD3cvafCKozJSjMiMichq5GbB8Mvz6NhRmmWPhjaH736D1DeDta20+8WgqM2WozIiInEX+cbPQLJ8M+UfNsdAYc/O9dreAT4Cl8cQzqcyUoTIjInKOCnPMj56Wvg45h8yxGpHQZRS0vwP8alibTzyKykwZKjMiIuepuACSP4Ilr0BmijkWEAadH4CO90BATSvTiYdQmSlDZUZE5AI5imH9f829ao7uNMd8g6Hj3dBlJARFWJtPqjWVmTJUZkRELpLTAZu/hEUvQfomc8w7ANrfDl1HQ0i0pfGkelKZKUNlRkSkgjid5hEJi1+EA6vNMS9faDsMuiVBeJyl8aR6UZkpQ2VGRKSCGQbsmm/O1OxdYo7ZvKDVddDjIajdzNp8Ui2ozJShMiMiUon2LjNnanb8eGLABi2GmLsK121taTRxbyozZajMiIhUgQNrzBuFt3zz+1j8QOg5FmI6WpdL3JbKTBkqMyIiVejQZlgyETZ+AYbTHIvrac7UxPXUoZZyzlRmylCZERGxwJGd5knd62aCs9gcq9/RnKmJH6BSI2elMlOGyoyIiIWOp8DS12DNB1BSYI5FtTJnapoP0UndcloqM2WozIiIuIDsQ7BsEqx6D4pyzLGIpuZJ3QnXgpe3tfnE5ajMlKEyIyLiQvKOwoqp5qMg0xyr2RC6J0Hbm8Hbz9J44jpUZspQmRERcUEFWbDqXVg6CfIyzLHgaHNH4cQR4BtoaTyxnspMGSozIiIurCjPvJ/ml1chO9UcC4wwz37qcBf46+e2p1KZKUNlRkTEDZQUwroZ5gqoY3vMMf9Q6HgvdL4fAsMtjSdV71zfvyv1FvJnn32Wrl27EhgYSM2aNU95zb59+xg8eDBBQUFEREQwZswYioqKyl2zYcMGevXqRUBAAPXq1eOZZ57BAzqYiIhn8fYzP14atRqufgsimpn31CyaAC8nwNx/mjcRi/xBpZaZoqIirrvuOu6///5TPu9wOLjyyivJzc1lyZIlzJw5ky+++IKHH3649JqsrCz69+9PdHQ0K1eu5PXXX+fFF19k4sSJlRldRESs4uUNbW6AB5bD9R9AVGsozoWlr8MrreDbseZyb5ETquRjpunTp5OUlMTx48fLjX///fcMGjSIlJQUoqPN4+NnzpzJiBEjSE9PJyQkhClTpvDYY49x6NAh/PzMO9yfe+45Xn/9dfbv34/tHDZd0sdMIiJuzDBg+zzz/KeUFeaY3Rva3AjdH4Jaja3NJ5XGJT5mOptly5aRkJBQWmQABg4cSGFhIatXry69plevXqVF5uQ1qamp7Nmz55Tft7CwkKysrHIPERFxUzYbNB0Ad/wAw7+BuF7gLIG1H8Gk9vD5neYRCuKxLC0zaWlpREZGlhsLCwvD19eXtLS0015z8p9PXvNH48ePJzQ0tPQRExNTCelFRKRK2WwQ1wOGfw13/ghNLzfPftr4OUzpAjNvNg+7FI9z3mXmqaeewmaznfGxatWqc/5+p/qYyDCMcuN/vObkJ2On+4jpscceIzMzs/SRkqLPVkVEqpWYDjDsU7hvCbS8GrCZp3W/3Qc+vAb2LrU6oVSh8947etSoUdx4441nvCY2NvacvldUVBQrVqwoN3bs2DGKi4tLZ1+ioqL+NAOTnp4O8KcZm5P8/PzKfSwlIiLVVFQruG469N5mLule/yns/Ml8NOhqHmrZuK8OtazmzrvMREREEBERUSF/eJcuXXj22Wc5ePAgdevWBWDu3Ln4+fmRmJhYes3jjz9OUVERvr6+pddER0efc2kSEZFqrnZTuHoK9H7U3Hxv7Uewbyl8dA1Et4Oej0DTK3SoZTVVqa/qvn37SE5OZt++fTgcDpKTk0lOTiYnxzxgbMCAAbRo0YJbb72VtWvX8tNPPzF27Fjuvvvu0ruWhw0bhp+fHyNGjGDjxo3Mnj2bcePG8dBDD53TSiYREfEgYbEw6GV4cB10HgneAZC6FmYOg6ndYMPn4HRYnVIqWKUuzR4xYgTvv//+n8bnz59P7969AbPwPPDAA/z8888EBAQwbNgwXnzxxXIfE23YsIGRI0fy66+/EhYWxn333ce//vWvcy4zWpotIuKhcjNg+WT49W0oPLGyNbwxdP8btL4BvH2tzSdnpOMMylCZERHxcPnHzUKz/A3IP2aOhcZAtweh3S3gE2BpPDk1lZkyVGZERASAwhxYPc3cTTjnxNEINSKhyyhofwf41bA2n5SjMlOGyoyIiJRTXABrPzRvFs48sX1HQBh0fgA63gMBNS2NJyaVmTJUZkRE5JQcxeZy7sUT4ehOc8w3GDreDV1GQlDFrN6VC6MyU4bKjIiInJHTAZtmm6UmfZM55h0A7W+HrqMhJPrMXy+VQmWmDJUZERE5J04nbJsDi16A1BNHI3j5Qtth0C0JwuMsjedpVGbKUJkREZHzYhiwaz4sehH2/mKO2byg1XXQ4yGo3czafB5CZaYMlRkREblge5fB4hdhx48nBmzQ4iro8TDUbW1ptOpOZaYMlRkREbloB9bA4pfMAy1Pih9onv8U09G6XNWYykwZKjMiIlJhDm2GJRNh4xdgOM2xuJ7m+U+xPXSoZQVSmSlDZUZERCrckZ3mSd3rZoKz2Byr39GcqYkfoFJTAVRmylCZERGRSnM8BZa+Bms+gJICcyyqtXlPTfMhOqn7IqjMlKEyIyIilS77ECybBKveg6Iccyyimbn6KeFa8PK2Np8bUpkpQ2VGRESqTN5RWDHVfBRkmmNhseY+NW2HgbeflencispMGSozIiJS5QqyYNW7sHQS5GWYY8HR0G0MXDocfAOtzecGVGbKUJkRERHLFOXBmvfhl9cgO9UcC4wwz37qcBf4633pdFRmylCZERERy5UUQvIn5gqo43vNMf9Q6HgvdL4fAsOtzeeCVGbKUJkRERGX4Sgx96hZ/BJkbDXHfIKgwx3QZTQER1qbz4WozJShMiMiIi7H6YQt/zMPtUzbYI55+cGlt0G3B6FmjLX5XIDKTBkqMyIi4rIMA7bPM0vN/l/NMbs3tLkRuj8EtRpbm89CKjNlqMyIiIjLMwzYs9g8qXv3QnPMZoeW15gb8EW2sDafBc71/VvbEoqIiLgCm80842n413Dnj9D0cvPsp42fmxvxyWlpO0IRERFXE9MBhn0KB9ebq5+6jbE6kUtTmREREXFVdVvDddOsTuHy9DGTiIiIuDWVGREREXFrKjMiIiLi1lRmRERExK2pzIiIiIhbU5kRERERt6YyIyIiIm5NZUZERETcmsqMiIiIuDWVGREREXFrKjMiIiLi1lRmRERExK2pzIiIiIhb84hTsw3DACArK8viJCIiInKuTr5vn3wfPx2PKDPZ2dkAxMTEWJxEREREzld2djahoaGnfd5mnK3uVANOp5PU1FSCg4Ox2WxWx6lyWVlZxMTEkJKSQkhIiNVx5AS9Lq5Jr4vr0mvjmirzdTEMg+zsbKKjo7HbT39njEfMzNjtdurXr291DMuFhIToB4AL0uvimvS6uC69Nq6psl6XM83InKQbgEVERMStqcyIiIiIW1OZ8QB+fn48+eST+Pn5WR1FytDr4pr0urguvTauyRVeF4+4AVhERESqL83MiIiIiFtTmRERERG3pjIjIiIibk1lRkRERNyaykw1tWfPHu68807i4uIICAigcePGPPnkkxQVFZW7bt++fQwePJigoCAiIiIYM2bMn66Rivfss8/StWtXAgMDqVmz5imv0WtjjcmTJxMXF4e/vz+JiYksXrzY6kgeZ9GiRQwePJjo6GhsNhtffvlluecNw+Cpp54iOjqagIAAevfuzaZNm6wJ60HGjx9Phw4dCA4Opk6dOgwdOpStW7eWu8aq10ZlpprasmULTqeTN998k02bNvHyyy8zdepUHn/88dJrHA4HV155Jbm5uSxZsoSZM2fyxRdf8PDDD1uY3DMUFRVx3XXXcf/995/yeb021vj0009JSkriiSeeYO3atfTo0YMrrriCffv2WR3No+Tm5tKmTRsmTZp0yucnTJjAxIkTmTRpEitXriQqKor+/fuXnsMnlWPhwoWMHDmS5cuXM2/ePEpKShgwYAC5ubml11j22hjiMSZMmGDExcWV/vN3331n2O1248CBA6VjM2bMMPz8/IzMzEwrInqcadOmGaGhoX8a12tjjY4dOxr33XdfubFLLrnE+Mc//mFRIgGM2bNnl/6z0+k0oqKijOeee650rKCgwAgNDTWmTp1qQULPlZ6ebgDGwoULDcOw9rXRzIwHyczMJDw8vPSfly1bRkJCAtHR0aVjAwcOpLCwkNWrV1sRUU7Qa1P1ioqKWL16NQMGDCg3PmDAAJYuXWpRKvmj3bt3k5aWVu518vPzo1evXnqdqlhmZiZA6fuKla+NyoyH2LlzJ6+//jr33Xdf6VhaWhqRkZHlrgsLC8PX15e0tLSqjihl6LWpehkZGTgcjj/9vUdGRurv3IWcfC30OlnLMAweeughunfvTkJCAmDta6My42aeeuopbDbbGR+rVq0q9zWpqalcfvnlXHfdddx1113lnrPZbH/6MwzDOOW4nNmFvDZnotfGGn/8+9XfuWvS62StUaNGsX79embMmPGn56x4bbwr9btLhRs1ahQ33njjGa+JjY0t/d+pqan06dOHLl268NZbb5W7LioqihUrVpQbO3bsGMXFxX9q1nJ25/vanIlem6oXERGBl5fXn36DTE9P19+5C4mKigLMWYC6deuWjut1qjqjR4/m66+/ZtGiRdSvX7903MrXRmXGzURERBAREXFO1x44cIA+ffqQmJjItGnTsNvLT8R16dKFZ599loMHD5b+hzd37lz8/PxITEys8OzV3fm8Nmej16bq+fr6kpiYyLx587j66qtLx+fNm8dVV11lYTIpKy4ujqioKObNm0e7du0A836nhQsX8vzzz1ucrnozDIPRo0cze/ZsFixYQFxcXLnnrXxtVGaqqdTUVHr37k2DBg148cUXOXz4cOlzJ9vzgAEDaNGiBbfeeisvvPACR48eZezYsdx9992EhIRYFd0j7Nu3j6NHj7Jv3z4cDgfJyckANGnShBo1aui1schDDz3ErbfeSvv27UtnM/ft21fuXjOpfDk5OezYsaP0n3fv3k1ycjLh4eE0aNCApKQkxo0bR3x8PPHx8YwbN47AwECGDRtmYerqb+TIkXzyySd89dVXBAcHl85ihoaGEhAQgM1ms+61qdS1UmKZadOmGcApH2Xt3bvXuPLKK42AgAAjPDzcGDVqlFFQUGBRas8xfPjwU7428+fPL71Gr4013njjDaNhw4aGr6+vcemll5YuO5WqM3/+/FP+/2P48OGGYZhLgJ988kkjKirK8PPzM3r27Gls2LDB2tAe4HTvKdOmTSu9xqrXxnYioIiIiIhb0momERERcWsqMyIiIuLWVGZERETEranMiIiIiFtTmRERERG3pjIjIiIibk1lRkRERNyayoyIiIi4NZUZERERcWsqMyIiIuLWVGZERETEranMiIiIiFv7f7f7mTvf72+EAAAAAElFTkSuQmCC\n", 143 | "text/plain": [ 144 | "
" 145 | ] 146 | }, 147 | "metadata": {}, 148 | "output_type": "display_data" 149 | } 150 | ], 151 | "source": [ 152 | "plt.plot(x, y(x));\n", 153 | "\n", 154 | "grad_x = -4.0\n", 155 | "x + grad_x\n", 156 | "plt.plot(x + grad_x, (grad_y(grad_x)*x + y(grad_x)))" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "id": "c67862c2", 162 | "metadata": {}, 163 | "source": [ 164 | "## Gradients for complex things" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 35, 170 | "id": "52717aff", 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "def y(x):\n", 175 | " return 20*jnp.sin(x) + x**2\n", 176 | "\n", 177 | "grad_y = jax.grad(y)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 44, 183 | "id": "a6d02fca", 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "data": { 188 | "text/plain": [ 189 | "[]" 190 | ] 191 | }, 192 | "execution_count": 44, 193 | "metadata": {}, 194 | "output_type": "execute_result" 195 | }, 196 | { 197 | "data": { 198 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGdCAYAAADnrPLBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABfEklEQVR4nO3deViU5eLG8e+wDYuAIAqiuGOpmCUuqbnllrlkaZtlmmaZS8fUFltO1jlq2XrSUlvUdstc2zUz09RcUkPNXQQFRFFBdph5f39M8otywQTeGeb+XNdc57wzz8zcOOTcvsvzWAzDMBARERFxUR5mBxARERG5HCozIiIi4tJUZkRERMSlqcyIiIiIS1OZEREREZemMiMiIiIuTWVGREREXJrKjIiIiLg0L7MDlAe73U5SUhKBgYFYLBaz44iIiEgJGIbBmTNniIyMxMPj/Ptf3KLMJCUlERUVZXYMERER+QcSExOpWbPmeR93izITGBgIOP4wgoKCTE4jIiIiJZGRkUFUVFTR9/j5uEWZOXtoKSgoSGVGRETExVzsFBGdACwiIiIuTWVGREREXJrKjIiIiLg0lRkRERFxaSozIiIi4tJUZkRERMSlqcyIiIiIS1OZEREREZemMiMiIiIuTWVGREREXJrKjIiIiLg0lRkRERFxaSozIiIiJliy9ShPLYnj4PFMs6O4PLdYNVtERMSZGIbBrNUH2J1yhlqh/txftZLZkVya9syIiIiUs5/3p7E75Qz+Pp7c3rKW2XFcnsqMiIhIOXt37UEAbmsRRbCft8lpXJ/KjIiISDk6nJbFqj3HsVhgSNs6ZsepEFRmREREytFHvyQA0LFhVeqEBZicpmIotzIzdepULBYLY8eOLbrPMAwmTZpEZGQkfn5+dOrUiZ07dxZ7Xl5eHmPGjCEsLIyAgAD69u3LkSNHyiu2iIhIqcktsPHZ5kQABl1b2+Q0FUe5lJlNmzbx1ltvcdVVVxW7f9q0abzyyivMmDGDTZs2ERERQbdu3Thz5kzRmLFjx7J48WLmz5/P2rVryczMpHfv3thstvKILiIiUmq+/C2Z09kF1KjsR6crqpkdp8Io8zKTmZnJXXfdxdtvv01ISEjR/YZh8Nprr/Hkk09yyy23EBMTw3vvvUd2djYff/wxAOnp6bz77ru8/PLLdO3alWuuuYYPP/yQuLg4vv/++7KOLiIiUqo+2HAYgLuurYWnh8XkNBVHmZeZUaNG0atXL7p27Vrs/kOHDpGSkkL37t2L7rNarXTs2JF169YBsGXLFgoKCoqNiYyMJCYmpmjMueTl5ZGRkVHsJiIiYqbfjpxme+JpfDw9uK1FlNlxKpQynTRv/vz5/Prrr2zatOlvj6WkpAAQHh5e7P7w8HAOHz5cNMbHx6fYHp2zY84+/1ymTp3Ks88+e7nxRURESs2Hf+yVubFpBGGVrCanqVjKbM9MYmIi//rXv/jwww/x9fU97ziLpfhuNsMw/nbfX11szMSJE0lPTy+6JSYmXlp4ERGRUpSeU8Cy7UkADGqjE39LW5mVmS1btpCamkpsbCxeXl54eXmxevVqXn/9dby8vIr2yPx1D0tqamrRYxEREeTn53Pq1KnzjjkXq9VKUFBQsZuIiIhZlm07Sm6BnSsjAmleK+TiT5BLUmZlpkuXLsTFxbFt27aiW4sWLbjrrrvYtm0b9erVIyIighUrVhQ9Jz8/n9WrV9O2bVsAYmNj8fb2LjYmOTmZHTt2FI0RERFxdp9tdkwpcluLqIsefZBLV2bnzAQGBhITE1PsvoCAAKpUqVJ0/9ixY5kyZQrR0dFER0czZcoU/P39GThwIADBwcEMGzaM8ePHU6VKFUJDQ5kwYQJNmzb92wnFIiIizmhXUgZxR9Px9rTQ75oaZsepkExdNfvRRx8lJyeHkSNHcurUKVq3bs3y5csJDAwsGvPqq6/i5eXFbbfdRk5ODl26dGHevHl4enqamFxERKRkFmxxnLfZrXE4oQE+JqepmCyGYRhmhyhrGRkZBAcHk56eXurnz9jtBh6aK0BERM4hr9BG6ykrOZ1dwNx7W9JZE+VdkpJ+f2ttpssQdySdnv9bw46j6WZHERERJ/T9rlROZxcQEeRLh+iqZsepsFRmLsOsnw6w59gZHvpkK1l5hWbHERERJ3N2HaYBsTU1428ZUpm5DP+9KYaIIF8Onsji2S92XvwJIiLiNpJO5/DTvuOAo8xI2VGZuQwhAT68dsfVWCyOy+6WbjtqdiQREXESC7ccwTCgdd1Q6oQFmB2nQlOZuUzX1qvCmM4NAHj089/47chpcwOJiIjp7HaDBVv+f24ZKVsqM6XgX10bcv2V1cgrtDP8/c0cy8g1O5KIiJjol0MnSTiZTSWrFz2bRpgdp8JTmSkFnh4W/nfH1URXq8SxjDwGz9nI6ex8s2OJiIhJzp7426dZJP4+pk7p5hZUZkpJoK837w5uSbVAK7tTzjB4zkYydYWTiIjbycgt4Ou4ZABua6ETf8uDykwpqlXFnw/va02Ivzfbj6QzdO4mMnILzI4lIiLl6IvtSeQV2omuVomroyqbHcctqMyUsobhgbw/tDWBVi82xp/kjtkbOH4mz+xYIiJSThb8sajkrS1qalHJcqIyUwaa1gzmk/uvJaySD7uSMxgwax0JadlmxxIRkTK2P/UM2xJP4+mhRSXLk8pMGYmpEcyCEW2pGeLH4bRs+r35M5viT5odS0REytDZy7E7X1GVaoG+JqdxHyozZahuWAALH2xLk8ggTmblM/DtDXy6KcHsWCIiUgYKbXYW/eqYPFUz/pYvlZkyFh7ky4IRbbixaQQFNoPHFsbxny93UWizmx1NRERK0Zp9Jzh+Jo/QAB+uvzLc7DhuRWWmHPj7eDHjzuaM7RoNwLtrDzH0vc260klEpAJZsMUxt8xNV0fi46Wv1/KkP+1y4uFhYWzXhrx5V3N8vT34ae9xBsxcR+JJnRgsIuLqTmXl8/2uVABujdXyBeVNZaac3di0Op+PaEt4kJW9xzK5+c2f2ZZ42uxYIiJyGZZtTyLfZqdx9SAaRwaZHcftqMyYIKZGMEtGtaNR9SBOZOZz++z1fLsjxexYIiLyD509xHSrZvw1hcqMSaoH+7FgRJuiBSpHfrSFJVuPmh1LREQu0e/JGew4moG3p4WbrtbcMmZQmTFRJasXbw2K5dbYmtgNePizbXy2KdHsWCIicgk+/2Numa6NwgkN8DE5jXtSmTGZl6cHL/S/irta18Iw4NGFv2kPjYiIiyiw2Yv+ztbcMuZRmXECHh4W/tsvhsFtagMwYcF2ftp73ORUIiJyMT/sTiUtK5+qgVY6Nqxqdhy3pTLjJCwWC8/0aULfZpEU2g1GfLiF346cNjuWiIhcwNlTA265pgZenvpKNYv+5J2Ih4eFl25txnUNwsjOt3H/+1u04raIiJNKOp3Dqj2OuWVub6m5ZcykMuNkfLw8mHl3c+pXDSAlI5dRH/9KgZY+EBFxOp9tTsRuwLX1QqlXtZLZcdyayowTCvT15q17WlDJ6sXGQyeZ9u1usyOJiMifFNrsfPrHIaY7W9UyOY2ozDip+lUr8fJtzQB4e80h1u47YXIiERE5a/Xe4ySn5xLi780NMRFmx3F7KjNOrEeTCO5q7Wj84xds43R2vsmJREQE4JONCYDjcmyrl6fJaURlxsk91asx9aoGcCwjj2eW7TQ7joiI20tOz+GH3Y4Tf+/QISaHglxT315lxsn5+Xjy6m1X42GBpduSNP+MiIjJPtt0BLsBreuGUl8n/sLOxfC/q+CYef/gVplxAc2iKjO4bR0Anlqyg9wCm7mBRETcVIHNXnSIaWBrN98rk58Fy8bAgiGQeQzWTTctisqMixjf/QoignxJOJnN6yv3mR1HRMQtfbsjhZSMXMIqWd37xN+UOHirE/z6PmCB68ZBX5UZuYhKVi+evakJAG/9dJBDJ7JMTiQi4n7m/nwIgLuvreWeJ/4aBmyYCW9fDyf2QmB1uGcpdH0GPL1Ni6Uy40J6NImg0xVVKbQbPP/N72bHERFxK9sTT/Nrwmm8PS3ueYgp6wR8fDt8+zjY8qFhTxjxM9TraHYylRlX88SNjfCwwHc7j7Hx0Emz44iIuI156+IB6HNVJNUCfc0NU94O/AAz28K+78DTCje+BHd+AgFVzE4GlHGZmTlzJldddRVBQUEEBQXRpk0bvvnmm6LHDcNg0qRJREZG4ufnR6dOndi5s/jZ0Hl5eYwZM4awsDACAgLo27cvR44cKcvYTq1heGDRpYCTv9qF3W6YnEhEpOJLPZPLl78lAXBvu7ompylHhfmw/Gn44GbHSb5Vr4ThP0Cr4WCxmJ2uSJmWmZo1a/L888+zefNmNm/ezPXXX89NN91UVFimTZvGK6+8wowZM9i0aRMRERF069aNM2fOFL3G2LFjWbx4MfPnz2ft2rVkZmbSu3dvbDb3vaLn4a4NCfDxZPuRdL6MSzY7johIhffhhgQKbAaxtUNoWjPY7DjlI+0AzOkO6153bLcYCsNXQUSMubnOwWIYRrn+0z40NJQXX3yRoUOHEhkZydixY3nssccAx16Y8PBwXnjhBR544AHS09OpWrUqH3zwAbfffjsASUlJREVF8fXXX9OjR48SvWdGRgbBwcGkp6cTFBRUZj9beXp95T5eWbGXelUDWPFwRzw9nKchi4hUJJl5hbR7/gfScwp4867m3Ni0utmRyt72+fDVeMjPBN/KcNMMaNSn3GOU9Pu73M6ZsdlszJ8/n6ysLNq0acOhQ4dISUmhe/fuRWOsVisdO3Zk3bp1AGzZsoWCgoJiYyIjI4mJiSkacy55eXlkZGQUu1U097arQ2V/bw4ez2LZ9qNmxxERqbA+/uUw6TkF1AsLoEeTCn45dm4GLBwOix9wFJna7eDBn00pMpeizMtMXFwclSpVwmq1MmLECBYvXkzjxo1JSUkBIDw8vNj48PDwosdSUlLw8fEhJCTkvGPOZerUqQQHBxfdoqKiSvmnMl+grzfD29cD4PWV+ym02U1OJCJS8eQW2Hh7jeNy7BGd6lfsveBHNsPs9hD3GVg8ofNTMPgLCK5pdrKLKvMyc8UVV7Bt2zY2bNjAgw8+yODBg9m1a1fR45a/nEBkGMbf7vuri42ZOHEi6enpRbfExMTL+yGc1OC2dQjx9+bQiSyWbksyO46ISIXz+ZYjHD+TR2SwL/2urmF2nLJht8OaV2BODzgVD8G14N5voOMj4OEac+mUeZnx8fGhQYMGtGjRgqlTp9KsWTP+97//ERHh2FX31z0sqampRXtrIiIiyM/P59SpU+cdcy5Wq7XoCqqzt4qoktWL+zvUB2DGqv3YdGWTiEipyS+0M/unAwAM71APH68KOJtJRjJ8cBOsfBbshdDkZhixBmq1NjvZJSn3T8YwDPLy8qhbty4RERGsWLGi6LH8/HxWr15N27ZtAYiNjcXb27vYmOTkZHbs2FE0xt3d06Y2wX6OvTPf7Tz/oTcREbk0n25OJPFkDmGVfLijZQWcJG/PN465Yw79BN7+0HcGDJgLfpXNTnbJvMryxZ944gl69uxJVFQUZ86cYf78+fz44498++23WCwWxo4dy5QpU4iOjiY6OpopU6bg7+/PwIEDAQgODmbYsGGMHz+eKlWqEBoayoQJE2jatCldu3Yty+guI8DqxeA2tXn9h/3MWn2AnjERFz1MJyIiF5aTb2P6H+vgjbk+Gj8f1zjcUiIFubDi37BxtmM7oqmjxIRFm5vrMpRpmTl27BiDBg0iOTmZ4OBgrrrqKr799lu6desGwKOPPkpOTg4jR47k1KlTtG7dmuXLlxMYGFj0Gq+++ipeXl7cdttt5OTk0KVLF+bNm4enZwX6xbpMg9vW4a01B/ntSDrrDqTRrkGY2ZFERFzavHXxpJ7Jo2aIH3e2qkB7ZVJ3w8JhcGyHY/vaUY51lbys5ua6TOU+z4wZKuI8M3/176U7eH/9YdpHh/HBMNc61iki4kxOZuXT6cVVZOQW8vKtzegf6/xX81yUYcCWefDtRCjMAf8wuHkWRHczO9kFOd08M1K2hrevh6eHhTX7TrDjaLrZcUREXNaL3+0hI7eQRtWD6HdNBbiCKfskfDYIvhzrKDL1r4cH1zl9kbkUKjMVRFSoP72vcsxKOfungyanERFxTXFH0pm/KQGAZ/s2cf15ZeJ/hlnXwe9fgIc3dPsP3LUQAs9/RbArUpmpQM5OovdNXDIp6bkmpxERcS12u8Ezy3ZgGHDT1ZG0qhtqdqR/zlYIq6bAe70h4yiE1of7VkC7h8Cj4n31V7yfyI3F1AimVd1QCu0GH2yINzuOiIhLmbsunl8TThPg48nEno3MjvPPnU6Aeb1g9Qtg2OHqu+CBnyDyGrOTlRmVmQpmaLs6AHz8SwK5Be67sriIyKU4dCKLF7/bDcATvRoREexrcqJ/aOdimHkdJG4AaxD0fxf6vQnWSmYnK1MqMxVMt8YR1Kjsx6nsApZs1QKUIiIXU2CzM2HBdnIL7FzXIIyBrngpdn4WLB0NC4ZAXjrUbOmYybfpALOTlQuVmQrG08PC4La1AZj7czxucOW9iMhlefG7PWw5fIpKVi+e79/U9SYeTd4OszvC1g8AC7Qf71hbKaSO2cnKjcpMBXR7i1r4+3iy59gZ1h9IMzuOiIjT+nZHMm/9cQXoS7deRc0Qf5MTXQLDgPVvwjtdIW0fBFaHwcugy7/B09vsdOVKZaYCCvb3pn9zxyRPc34+ZHIaERHntONoOuM/2w7A8PZ1uSGmusmJLkHmcfj4NvhuItjy4YobYcTPULeD2clMoTJTQQ3540TglbtTiT+RZW4YEREnE38iiyFzN5KVb6Nt/So8esOVZkcquQM/OBaI3LccPK1w40twx8cQUMXsZKZRmamg6letRMeGVTEMeH/9YbPjiIg4jYS0bAbN+YUTmfk0rh7E7EGxeHu6wNdhYT4sfxo+uBmyUqFqI7h/FbQaDq52nk8pc4FPT/6ps3tnFmxOJCuv0NwwIiJOYO+xMwyYtY7EkznUruLPvKEtCfR1gfNL0g7AnO6w7nXHdothjiIT3sTcXE5CZaYC6xhdlbphAZzJK2TRr0fMjiMiYqpVe1IZMHMdqWfyuCI8kAUPtKFaoJPPJ2MYsO0TmNUekraCb2W4/SPo/Qp4+5mdzmmozFRgHh4WBrdxXKY9b108drsu0xYR91Nos/Pa93sZOm8TGbmFNK9VmU8fuJZqQU5eZHIzYNFwWDICCrKg9nWOBSIb9TY7mdNRmang+sfWpJLViwPHs1i7/4TZcUREytXulAxufnMdr32/D8OAu1rX4pP7r6Wyv4/Z0S7syGbHApFxC8DiCZ2fclx2HVwBVvEuA15mB5CyFejrzYDYmsxbF8+8dfF0aFjV7EgiImUuPbuAGav2MW9dPAU2g2A/b57t24R+1zh5GbDb4OfXHItE2gshuBYMeBeiWpmdzKmpzLiBwW3rMG9dPD/sTuXQiSzqhgWYHUlEpEzkF9r5YMNhXl+5j/ScAgC6NQ5ncr8Y5z+slJEMi++HQz85tpvcAr1fBb/KpsZyBSozbqBuWACdr6jKqj3HeX99PM/00dnvIlKx2O0GX8Yl8/LyPRxOywagYXglnrixER0bVnX+JQr2fANLRkLOSfD2hxtfdKx27ey5nYTKjJsY0q4uq/Yc5/PNRxjf/QoqWfXRi0jFsGbfcZ7/Zjc7kzIACKtkZXz3htwaWxMvZ58/piDHMXfMprcd2xFXwYA5EBZtbi4Xo280N9G+QRj1wgI4eCKLRb8e4Z42dcyOJCJyWQ4ez+SZZTtZs89xcUMlqxcPdKjH0OvqEuAK/2BL3Q2fD4XUnY7tNqMd6yp5Wc3N5YJc4NOW0uDhYWFw2zo8s2wn89bFc3fr2nh4aPeliLievEIbb6w6wKwfD5Bvs+PtaeHua2szunMDqlRygSJgGLB5Dnz3BBTmQkBV6DcLoruancxlqcy4kf6xNXnxuz0cPJ7Fmv0n6Kgrm0TExRw8nsmYT7YWHVLq2LAqz93UhNpVXOTChuyTsGwM7P7SsV2/C9w8CypVMzeXi1OZcSOVrF7c2qImc3+OZ97Ph1RmRMSlLN12lImL4sjOtxEa4MN/+8XQMybC+U/uPSt+LSy6HzKOgoc3dJ0E144EDyc/r8cFqMy4mcFtHJdpr9pzXJdpi4hLMAyD/63cx2vf7wOgTb0qvHbH1YQ7+6XWZ9kKYfULsOYlMOwQWt8xd0zkNWYnqzBUB91MnbAAOl/h2J353rp4c8OIiFyE3W7w+MK4oiIzomN9PryvtesUmVOHYd6N8NM0R5G5+m544CcVmVKmMuOGhrStA8DnW46QqdW0RcRJ2e0GExfF8enmRDw9LEy5uSmP97wST1e5eGHHIscCkYm/gDUI+r8L/d4AayWzk1U4KjNuqH10GPWrBpCZV8jCLVpNW0Scj2EY/HvZDj7dnIiHBf53x9UMbF3L7Fglk58FS0fB5/dCXjrUbAkj1kDTAWYnq7BUZtyQxeK4TBsch5q0mraIOJu31xzkww0JWCzw6u1X0/uqSLMjlUzydpjdEbZ+CFig/QS49xsIqWN2sgpNZcZN3dK8JoFWLw6eyOKnfcfNjiMiUuS7nSlM/WY3AE/3asxNVzv54pAAdjusfwPe6Qpp+yAw0rHKdZenwdPb7HQVnsqMm3Jcph0FwDydCCwiTuLg8UzGfboNw4BB19bm3nZ1zI50cZnH4ePbHJPg2fLhil7w4M9Qt4PZydyGyowbu6dNbSwW+HHPcfYdO2N2HBFxc7kFNkZ9vJWsfBut64byTJ/Gzj+HzP6VMLMt7F8BXr5w40twx0fgH2p2MreiMuPG6oQF0L1xOAAzfzxgchoRcXeTv/qd35MzqBLgw+t3XuPci0QW5sPyp+DDWyArFao2guGroNVwrXRtAif+TZHyMKpzAwCWbk8i8WS2yWlExF2t3XeCDzYcBuCV2518Qry0A/BuN1g33bHd8j64fxWENzY3lxtTmXFzV9WsTPvoMGx2g9k/ae+MiJS/rLxCHl/0G+A4/O20S60YBmz72DF3TPI28AuB2z+CXi+Dt5/Z6dyayowU7Z35bPMRUjNyTU4jIu7mxe/2cORUDjUq+/HoDVeaHefcctNh4X2w5EEoyII67WHEz9Cot9nJhDIuM1OnTqVly5YEBgZSrVo1+vXrx549e4qNMQyDSZMmERkZiZ+fH506dWLnzp3FxuTl5TFmzBjCwsIICAigb9++HDmiyd5KS+u6ocTWDiG/0M47aw+ZHUdE3MiWw6eKrqh8vn9TKlmdcMnAxE2OvTE7PgeLJ1z/FNyzFIJd4JJxN1GmZWb16tWMGjWKDRs2sGLFCgoLC+nevTtZWVlFY6ZNm8Yrr7zCjBkz2LRpExEREXTr1o0zZ/7/6pqxY8eyePFi5s+fz9q1a8nMzKR3797YbLayjO82LBYLo//YO/PB+sMcP5NnciIRcQc2u8GkZY5/vA6IrUn7aCc7vGS3wU8vwZwecPowVK4FQ7+FDo+Ah6fZ6eRPLIZhlNv0r8ePH6datWqsXr2aDh06YBgGkZGRjB07lsceewxw7IUJDw/nhRde4IEHHiA9PZ2qVavywQcfcPvttwOQlJREVFQUX3/9NT169Ljo+2ZkZBAcHEx6ejpBQUFl+jO6KsMw6PfmOrYnnmZI2zpM6tvE7EgiUsHN35jA44viCLR6seqRToRVspod6f9lJMGi+yF+jWM7pj/0fhV8g83N5WZK+v1drufMpKenAxAa6rj+/tChQ6SkpNC9e/eiMVarlY4dO7Ju3ToAtmzZQkFBQbExkZGRxMTEFI2Ry2exWHi0xxUAfPTLYV3ZJCJlKj2ngGnfOU47GNutoXMVmd1fO+aOiV8D3gFw05uORSJVZJxWuZUZwzAYN24c1113HTExMQCkpKQAEB4eXmxseHh40WMpKSn4+PgQEhJy3jF/lZeXR0ZGRrGbXFy7BmG0a1CFApvBa9/vMzuOiFRg//t+Hyez8omuVol72tQ2O45DQQ58NQHm3wk5p6B6M3jgJ7jmLs0d4+TKrcyMHj2a3377jU8++eRvj/11hkfDMC466+OFxkydOpXg4OCiW1RU1D8P7mYe6eG4kmDx1iPsSdGswCJS+hJPZvPBhngA/t2nMd7OMDle6u/w9vWw6W3HdpvRMGwFhDUwN5eUSLn8Bo0ZM4Zly5axatUqatasWXR/REQEwN/2sKSmphbtrYmIiCA/P59Tp06dd8xfTZw4kfT09KJbYmJiaf44FdrVUZXpGROB3YDnvtxJOZ5SJSJu4tXv91JgM2gfHWb+Sb+GAZvehbc6QeouCKgKdy+EHpPBy4kOfckFlWmZMQyD0aNHs2jRIn744Qfq1q1b7PG6desSERHBihUriu7Lz89n9erVtG3bFoDY2Fi8vb2LjUlOTmbHjh1FY/7KarUSFBRU7CYl98SNjfDx8uDn/Wl8u+Pch/JERP6JPSlnWLz1KACP/HGenmmyT8Knd8NX46AwFxp0hQfXOf5XXEqZXtA/atQoPv74Y5YuXUpgYGDRHpjg4GD8/PywWCyMHTuWKVOmEB0dTXR0NFOmTMHf35+BAwcWjR02bBjjx4+nSpUqhIaGMmHCBJo2bUrXrvqFKwtRof6M6FCP13/Yz3+/+p1OV1TDz0eXIYrI5Xvxuz0YBtzYNIKralY2L0j8Wlg4HM4kgYc3dHsWWj8IHk5wyEsuWZmWmZkzZwLQqVOnYvfPnTuXIUOGAPDoo4+Sk5PDyJEjOXXqFK1bt2b58uUEBgYWjX/11Vfx8vLitttuIycnhy5dujBv3jw8PfUFW1Ye7NSAz7cc4ejpHF77fi8Tb2xkdiQRcXFbDp/k+9+P4elhYXx3k/bK2Aph9fOO+WMwoEoDx5VKkVebk0dKRbnOM2MWzTPzz6z8/RjD3tuMhwU+f7AtzWuFXPxJIiLncfvs9fxy6CR3tIzi+f5XlX+AU4cdSxIc2ejYvvpu6PkCWCuVfxYpEaecZ0ZcS5dG4dzSvAZ2AyYs2E5ugWZcFpF/ZsPBNH45dBIfTw8e6hJd/gF2LIRZ1zmKjDUIBsyBfm+oyFQQKjNyQc/0bkK1QCsHj2fx7Be7zI4jIi5q+g+Ouatua1mTyMrluMJ0XiYsGQWfD4W8DKjZCkascczoKxWGyoxcULC/Ny/f1gyLBT7ZmMDCLVrgU0QuzZbDp/h5fxpeHhZGdKxffm+ctA3e6gjbPgQsjjWV7v0GQuqUXwYpFyozclHto6vyrz92Cz+5JI64I+kmJxIRVzLjj70ytzSvQc0Q/7J/Q7sd1s2Ad7pC2n4IjITBXzhWu/Z0wlW55bKpzEiJPHR9NB0bViW3wM698zYSfyLr4k8SEbcXdySdVXuO42GBkZ3KYTbdzFT4+FZY/iTYC+DK3vDgz1C3fdm/t5hGZUZKxMPDwoyB19C4ehAnMvMZNOcXktNzzI4lIk7u7LkyN11dgzphAWX7Zvu/h5ntHP/r5Qu9XoHbPwT/0LJ9XzGdyoyUWKCvN/OGtqRWqD+JJ3Po/+Y69qdq/SYRObfdKRks33UMiwVGdS7Dc2UK8+C7J+HD/pCVCtUaw/BV0HKYFoh0EyozckmqBfryyf3XUr9qAEnpuQyYtZ4f96SaHUtEnNCMH/YDcGNMdRpUC7zI6H/oxH54txusn+HYbjkchv8A4Y3L5v3EKanMyCWrUdmPBSPacnVUZU5nFzBk7ib+++UucvI1D42IOOxPzeSruGQARnUug3NlDAO2fgSzO0DydvALgTs+hl4vgXc5XvotTkFlRv6R0AAf5t9/LYPb1AbgnbWHuP7lH1my9Sg2e4WfVFpELuLNH/djGNC1UTiNI0t55vXcdFg4DJaOhIIsqNPesUDklb1K933EZWg5A7ls3+86xjPLdnL0tOOE4JohfgxuU4c+zSKJCPY1OZ2IlLeEtGw6v/wjNrvB0lHtaBZVufRePHETLBwKpxPA4gmdn4DrHgYPrdVXEZX0+1tlRkpFboGNd9ce4u01BzmdXVB0f2ztEDo1rMq19atwVc1grF76C0ekont84W/M35RIh4ZVeX9oq9J5UbsN1r4Cq6aCYYPKtR0LREa1LJ3XF6ekMvMnKjPlJyffxpJtR1mwOZFfE04Xe8zq5UFs7RDa1q9Cm/phXFUzGG9PHekUqUiOns6h04urKLAZfD6iDS3qlMJl0elHYfEDEL/GsR0zAHq/Ar7Bl//a4tRK+v2tqRClVPn5eHJnq1rc2aoWKem5rPj9GBsOpLHhYBppWfmsO5DGugNpwF4CfDxp1yCMvldH0rVROL7e2msj4upm/XiAAptBm3pVSqfI7P4Klo6CnFPgHeA4wbfZnbrkWorRnhkpF4ZhcOB4JusOpLH+QBrrD6YVOxxVyerFHS2jGNa+LtWDdSWCiCs6lpFL+2mryC+08/Hw1rStH/bPX6wgB5Y/BZvecWxXv9pxWCmsHGYRFqehPTPiVCwWCw2qBdKgWiD3tKmD3W6wKzmDr+OSWbY9iSOncnhn7SHeWx/P0OvqMub6aCpZ9esp4kpmrz5IfqGdFrVDaFOvyj9/oWO7HFcrpe5ybLcdA9f/G7x8SieoVDjaMyOmMwyDH/ceZ+aPB9h46CQA4UFWXhzQjA4Nq5qcTkRK4kRmHte98AO5BXbeG9qKjv/kv13DcOyJWf4UFOZCQDW4eSY06Fr6gcUllPT7W2dfiuksFgudr6jGZw+04d3BLagV6s+xjDwGz93Ii9/tptBmNzuiiFzEO2sOkVtgp1nNYDpE/4PDS9knYf5d8PUER5Fp0NUxd4yKjJSAyow4lS6Nwln+cAfual0Lw4A3Vh1gxIe/anZhESd2KiufD9bHAzDm+mgsl3py7qE1jgUi93wFHt7QYwoMXACVtGdWSkZlRpyOr7cnk29uyvQ7r8Hq5cH3vx/j7nd/If1PJwyLiPOY+/MhsvJtNK4eRJdG1Ur+RFsBrPwPvNcHziRBlWgYvhLajAIPfT1Jyem3RZxWn2aRfDCsNUG+Xmw5fIp7520kO7/Q7Fgi8iens/OZuy4egDHXNyj5XplT8TC3J6x5CTDgmkHwwGqo3qysokoFpjIjTq1V3VDm39+GIF8vfk04zQMfbCGvUIecRJzF7J8Ocia3kCsjAunRJKJkT4r7HGa1hyObwBoMA+bCTTPAJ6Bsw0qFpTIjTq9xZBDzhrbC38eTNftO8NTiHbjBRXgiTi/1TC5zfz4EwITuV+DhcZG9MnmZsGSk47LrvAyo2QpGrIGYW8ohrVRkKjPiEprXCmHm3bF4WGDBliO898dubRExzxs/7Ce3wM41tSpf/FyZpG0wuwNs+wgsHtDhUbj3GwipXS5ZpWJTmRGX0bFhVSb2bATAf776nQ0H00xOJOK+jpzK5uONCQA80v2K858rY7fDuunwTlc4eQCCasDgL+H6J8FTE2NK6VCZEZdyX/u69Ls6Epvd4OFPt3E6O9/sSCJu6dUV+yiwGbRrUIW2Dc4zr0xmKnw0wDEJnr0AruwNI9ZCnXblG1YqPJUZcSkWi4XJNzelThV/ktNzeWJxnM6fESlncUfSWbT1COA4V+ac9n0PM9vCgZXg5Qu9X4XbPwT/Ulh8UuQvVGbE5QRYvfjfHdfg5WHh67gUFmw5YnYkEbdhGAbPfbkTw4B+V0dyTa2Q4gMK8+C7J+Gj/pB1HKo1gft/hBZDtdK1lBmVGXFJzaIq83C3hgA898UuUtJzTU4k4h6+jkthU/wpfL09ePSGK4s/eGKf49yY9TMc263ud0yCV61R+QcVt6IyIy5rRMf6XFOrMpl5hTy1RJdri5S13AIbU77+HXD89xdZ2c/xgGHA1g8dVyul/AZ+oXDHJ3Dji+DtZ2JicRcqM+KyPD0svND/Krw9LXz/+zG+2ZFidiSRCu217/dx9HQO1YN9eaBDfcedOafh86GwdBQUZEOd9vDgz3DljaZmFfeiMiMurWF4IA92agDAv5fu1PpNImVkV1IGb685CMCzfZvg5+MJCb84ZvLduQgsntDl33DPUgiKNDmtuBuVGXF5ozrXp0G1SpzIzOPV7/eaHUekwim02Zm46DdsdoOeMRF0b1QVVr/oWFspPQEq14Zhy6H9ePDwNDuuuCGVGXF5Vi9Pnu3bBIAPNhxmd0qGyYlEKpY3Vh1g+5F0An29+E/nEHivL6z6Lxg2aHqrY0mCmi3MjiluTGVGKoR2DcLoGROBzW4wadlOnQwsUko2x5/kfysdezzfaZVC2IfXw+G14B0A/WbBLW+Db7DJKcXdqcxIhfFkr0ZYvTzYcPAkX8fpZGCRy3UqK59/zd+Gt5HPRxHzab3xIcg5BdWvduyNufpOzR0jTqFMy8xPP/1Enz59iIyMxGKxsGTJkmKPG4bBpEmTiIyMxM/Pj06dOrFz585iY/Ly8hgzZgxhYWEEBATQt29fjhzRJGnydzVD/Hmwk+MKi8lf7SI7v9DkRCKuK7/QzoMfbSEgfS/f+P2bdqeXOR5o+xAMWwFV6psbUORPyrTMZGVl0axZM2bMmHHOx6dNm8Yrr7zCjBkz2LRpExEREXTr1o0zZ84UjRk7diyLFy9m/vz5rF27lszMTHr37o3NZivL6OKiRnSsT43KfiSl5zLrxwNmxxFxSYZh8MzSOKIPz+cLn6eoZyRAQDW4exF0/w94+ZgdUaQYi1FOJxdYLBYWL15Mv379AMd/LJGRkYwdO5bHHnsMcOyFCQ8P54UXXuCBBx4gPT2dqlWr8sEHH3D77bcDkJSURFRUFF9//TU9evQo0XtnZGQQHBxMeno6QUFBZfLzifP4Ji6ZBz/6FauXBz9M6ESNypq0S6SkDMPgf8t+ocnmJ+jmucVxZ3R3uOlNqFTV3HDidkr6/W3aOTOHDh0iJSWF7t27F91ntVrp2LEj69atA2DLli0UFBQUGxMZGUlMTEzRmHPJy8sjIyOj2E3cxw0xEbSuG0peoZ3nv9ltdhwRl2G3G3z4yYfc8esddPPcgs3iDTc8DwM/U5ERp2ZamUlJcZygGR4eXuz+8PDwosdSUlLw8fEhJCTkvGPOZerUqQQHBxfdoqKiSjm9ODOLxcK/+zTGYoEvtiexOf6k2ZFEnF5mdg7fvT6Ku/aMIcJyinT/OnjevxKufVAn+YrTM/1qJstf/iMxDONv9/3VxcZMnDiR9PT0oltiYmKpZBXX0SQymDtaOkrss1/swm7Xpdoi57N1+zYOv9SBnqc/wsNiEF+7P8Fj10H1ZmZHEykR08pMREQEwN/2sKSmphbtrYmIiCA/P59Tp06dd8y5WK1WgoKCit3E/YzvfgWVrF7EHU1n4a+6Ak7kzwzDYFviad5+cxr1F91AE/tezuDPgU4zqHPvHPAJMDuiSIl5mfXGdevWJSIighUrVnDNNdcAkJ+fz+rVq3nhhRcAiI2NxdvbmxUrVnDbbbcBkJyczI4dO5g2bZpZ0cVFhFWyMub6Bkz9ZjfTvtvDjU2rE2A17VdepFRl5BaQmpHLyawCbHYDwzDw9LDg7eWBj6cH3p4eeHta8Pb0wMfLAy8PCycy84lPy2LToZNs3JvAoJNvMNzrJ7DAYf+mVBn8PvXD65n9o4lcsjL9mz0zM5P9+/cXbR86dIht27YRGhpKrVq1GDt2LFOmTCE6Opro6GimTJmCv78/AwcOBCA4OJhhw4Yxfvx4qlSpQmhoKBMmTKBp06Z07dq1LKNLBTGkXR0+3pjA4bRs3vxxP4/0uNLsSCL/SEZuAd/tSGHVnlS2J6Zz9HTOP36tGMtBXveeQT2vFOx4cLrFQ9Tu+TR4quyLayrT39zNmzfTuXPnou1x48YBMHjwYObNm8ejjz5KTk4OI0eO5NSpU7Ru3Zrly5cTGBhY9JxXX30VLy8vbrvtNnJycujSpQvz5s3D01OLmcnFWb08efLGRtz/wRbeXnOIO1rWIirU3+xYIiV26EQWM3/cz5JtSeQX2os9FujrRVglK14eFiwWsNkNCmwGBTY7BTY7+YV28m12CmwGNrtBkK8H//JbzuDc9/EyCrEHRuLR/x1C67Qz6acTKR3lNs+MmTTPjHszDIO73/2Fn/encWPTCN68K9bsSCIXlZNv47WVe3lnzSFsf5zAHl2tEr2viqRV3VBiagQR6Otd4tezZ6TgsfRBOPCD445GfaDP6+AfWhbxRUpFSb+/tU9RKjyLxcLTvRtz4//W8HVcChsOpnFtvSpmxxI5rx1H0xn50a8knMwGoPMVVRl9fTTNa1W+6NWe57RvBR5LHoSs4+DlBzdMgdh7dcm1VBimX5otUh6ujAhiYOtaADz3xa6if+mKOJtFvx6h/8x1JJzMJjLYl3fuacHce1sRWzvk0otMYR58+wR8NMBRZKo1gft/hBZDVWSkQlGZEbfxcNeGBPp6sSs5g8+3aO4hcT4zftjHuM+2k1dop8uV1fhmbAe6Nj7/NBQXdGIfvNMVNrzh2G71AAz/AarpJHipeFRmxG1UqWTlX12iAXjxuz2cyS0wOZGIg2EYvPjdbl5avheAUZ3r8/Y9LQj2K/k5MX96Mfj1A5jdAVJ+A79QuHM+3DgNvH1LObmIc1CZEbdyT5s61AsL4ERmPjNW7b/4E0TKwRur9vPGKscq70/ceCWP9LgSD49/cBgo5zR8fi8sGw0F2VC3Azy4Dq7oWbqBRZyMyoy4FR8vD57q3QiAuWvjOZyWZXIicXefbzlStEfm6d6Nub9D/X/2Qgm/wKz2sHMxeHhBl2dg0BIIql56YUWclMqMuJ3OV1SjQ8Oq5NvsTP7qd7PjiBtbs+84jy/8DYAHOtZj2HV1L/1F7DZYPQ3m9oT0BAipA0OXQ/tx4KH5uMQ9qMyI27FYLDzdqxGeHhaW7zrG2n0nzI4kbighLZtRH/1Kod2g39WRPPZPZqdOPwLv9YVVk8GwQdPb4IE1UFNzKYl7UZkRtxQdHsiga2sDMHHxb2TlFZqcSNxJboGNER9uISO3kKujKvPCgKsu/RyZ37+Ame3g8FrwqQQ3z4b+b4OvJgYV96MyI25rfPeG1KjsR+LJHKZ9u9vsOOJGnlqyg13JGVQJ8GHm3c2xel3C4aD8bPjyYfj0bsg9DZHXwAM/QbM7yiyviLNTmRG3FejrzQv9rwLgvfWHWX8gzeRE4g6+2J7E51uO4GGB6XdeQ/Vgv5I/+dhOeLszbJ7j2G73L8f5MVX+4UnDIhWEyoy4teuiw4pmBp6wYDvp2Zp7RspOakYuTy/dAcDo66Np2yCsZE80DNj4NrzVGY7vhkrhMGgxdHsOvHzKMLGIa1CZEbf3xI2NqBXqz9HTOUz4fDtusPaqmMAwDB5fFMfp7AKaRAYx5voGJXtiVhrMHwhfTwBbHkR3d8wdU//6sg0s4kJUZsTtVbJ68cbA5vh4erBi1zHeXXvI7EhSAS3YfIQfdqfi4+nBK7ddjbdnCf76PbgaZrWDPV+Dpw/c8DwM/AwCSrhHR8RNqMyIAE1rBhdNpjf1m92s2pNqciKpSI6cyua5L3cBMK57Q66ICLzwE2wF8P2z8P5NcCYZwhrCfSvh2ge1QKTIOajMiPxh0LW1GRBbE5vdYPRHv7IzKd3sSFIB2O0Gjyz4jcy8QmJrhzC8fb0LP+HkIZhzA6x9BTCg+T2Ola6rX1UecUVcksqMyB8sFgtTbm5K2/pVyMq3MXjOJvaknDE7lri499fHs/5gGn7enrx8azM8LzSfzG8LHEsSHN0MvsFw63vQdzr4BJRfYBEX5GV2ABFn4uPlwcy7Y7nzrQ3sSs7gjrfW8/7Q1jStGVzq75WeU8COo+nsTEonOT2XtMx8AAKsntQKDaBJZBCt6obi660p6V3VweOZPP/HHEZP3HgldcLOU0ryzsDXj8D2TxzbUdc6JsCrXKuckoq4NpUZkb8I9vPmk+HXcs/cjWxPPM1ts9fzwoCr6Nss8rJf+1hGLt/tTOHbHSn8cugkNvuFr5yyenlwQ0wE97SpTWzt0Mt+fyk/hTY74xdsJ7fAznUNwrirde1zDzz6KywcBicPgsUDOjwKHR4BT/31LFJSFsMNrkPNyMggODiY9PR0goI01beUzJncAh788FfW7nes3XTLNTV4slcjqlSyXtLrJKRl8+3OZL7dkcKvCaeLPRYV6kfTGsHUCg0grJIPHhYL6TkFHDqRxab4kySn5xaN7diwKk/2akTD8IucPCpO4c0f9zPt2z0EWr347uEORFb+y+R4djusnw4r/wP2Agiq6dgbU7utOYFFnFBJv79VZkQuwGY3eHn5HmauPoBhQKDViztb1+L2llHUr1rpnM8psNmJO5rOyt+PsfL3VHb/5byb2Noh3NAkgh5NIqhVxf+8720YBnFH0/lww2EW/XqUQruBj6cH47s35L729S587oWY6vfkDPrOWEuBzeClW5sxILZm8QFnjsHiB+DgKsd2o77Q93XwCyn/sCJOTGXmT1Rm5HJtTTjFU0t2sDMpo+i+WqH+XBkRSLUgK54WCyey8kk6ncOupAzyCu1F4zw9LLSuG0rPmAi6N4kgPMj3kt8//kQWz325ix92Oy4Zb1u/Cm/e1ZzK/pr91dnkF9q56Y2f+T05g66Nwnn7nlgsf76ceu9yWPIgZJ8ALz/o+Tw0H6xLrkXOQWXmT1RmpDTY7Qar9qT+sY7TCQps5/9PJ8jXi/YNq9K1UTU6NaxGSMDllw7DMFiw+QjPfrGTrHwbdar48+6QlufdQyTmePG73byx6gAh/t4sf7gjVQP/OCxZmAffT4INbzq2w2Og/7tQ7UrTsoo4O5WZP1GZkdKWkVvAtoTT7E/NJD2ngEK7nRB/HyKCfWlcPYg6VQLwKKPDQLtTMhg2bzNHT+cQGuDDR/e1plF1/V47g83xJ7lt9nrsBrx5V3NubFrd8cDxvbBwKKTEObZbj4Cuz4L3pe+lE3EnKjN/ojIjFc2JzDzunbuJuKPpVPb35sNhrYmpUfqXj0vJZeYVcuP/1pBwMptbmtfglduudiwQufUD+OYxKMgG/ypw05twxQ1mxxVxCSX9/takeSIuKKySlQ/va83VUZU5nV3AwLc3sDsl4+JPlDLzny92kXAymxqV/ZjUtwnknIYFQ2DZGEeRqdsRRvysIiNSBlRmRFxUsJ83HwxrRfNalcnILWTwnI0cPZ1jdiy39N3OFD7dnIjFAq/c1oyg1C2OmXx3LQEPL+g6CQYtgaDqJicVqZh0mEnExaVnF3Dr7HXsPZZJ/aoBfD6ibamccGwmwzCIT8tmx9F0jp/JI6fARoCPJxHBfjSJDKJmiF/xK4RMlHgym97T15KeU8CIDrV53P8rWP08GHYIqQP950DNWLNjirgknTPzJyozUtElp+dwy5vrSE7PJbZ2CB/d19oll0E4dCKLDzcc5uu45GITBv5V7Sr+9GgSwa2xNYk2cRLB3AIbt85aT9zRdLpGFvBWpdl4JKxzPHjV7XDjS+Crv3NE/imVmT9RmRF3sO/YGfrPXEdGbiE3X1ODV25r5jR7Ly7mcFoWLy3fy1e/JXF2hQerlweNqjv2wgT4eJGZV0h8WhZ7j50pdll8++gwRnduQOt6Vco99xOL4/j4lwT6+/3Kiz7v4JF3GnwqQa+Xodkd5Z5HpKJRmfkTlRlxF+v2n2DQnI3Y7AaP3XAlD3aqb3akC8ovtDN79QFmrNpfNNFg5yuqcmerWnRoWPWce5ey8gpZvfc4S7cdZcWuY0Xlp8uV1Xi855XltqfmvXXxTF32K097f8hdnisdd0Y2h/7vQBXn/nMXcRUqM3+iMiPu5IMNh3l6yQ4sFnhrUAu6NQ43O9I5JZ7MZvTHv7L9SDoA1zUIY+KNV9IksuSXmCeezGbm6gN8uikRm93Ay8PCfe3r8a8u0fj5lN1htu93HeOVDxfxmtd0GnocddzZbix0fhK8XPt8JRFnojLzJyoz4m6eXrKDDzYcxt/Hk4UPtnW6SfW+33WMcZ9tIyO3kCBfL/7TL4a+zSL/8WGxA8czef6b3azYdQyAmiF+/KdfDJ2vqFaasQFYv/8EK9//D49YPsJqKcCoFI7l5tlQv3Opv5eIu1OZ+ROVGXE3BTY7g+dsZN2BNGpU9mPp6HaEXeJq32Xlg/Xx/HvZTgwDro6qzIyB11Az5PwLbl6K73cd45llO4suUe/VtDpP925MRHDpzLS7ccdeMheM4HrLFgDs0d3x6DcTAsJK5fVFpDiVmT9RmRF3dDo7n35v/Ex8WjYtaofwoclXOBmGwUvL9/DGqgMA3NmqFs/2bYKPV+lOd5WVV8hr3+9lzs/x2OwGAT6ePNytIYPb1sHb85+/17oVC2mwdjzVLKcowBu6P4d3mwe1QKRIGapwMwC/+eab1K1bF19fX2JjY1mzZo3ZkUScWmV/H94Z3JJAXy82Hz7FYwt/w6x/uxTY7Dz6+W9FRWZct4ZMuTmm1IsMQIDViyd7NWbZ6HY0r1WZrHwb//3qd3q/vpaf9h6/5D+D3Nxcfp41mmvXDqOa5RTJ3rUwhq/Eu+1IFRkRJ+ESZebTTz9l7NixPPnkk2zdupX27dvTs2dPEhISzI4m4tQaVKvErLtj8fKwsHRbEq9+v6/cM2TnF3L/+5tZsOUIHhZ4/pamPNQluswvG28SGcznI9ryQv+mhPh7s+fYGe6Zs5EBs9azancqNvuFS41hGKzduIlD066jXcoHeFgMtlbtR7UJG/Cp0axMs4vIpXGJw0ytW7emefPmzJw5s+i+Ro0a0a9fP6ZOnXrR5+swk7i7Tzcl8NhCx4rNL9/ajP6xNcvlfdMy8xj63ma2J57G19uDGXc2p6sJV1edyspn+g/7+eiXw0WXgEcE+dL7qupcW68KDapVIiTAh7xCG/EnstkUf5IzGz9iVPZMAi05ZBBAfJupXNVjcLlnF3FnFeacmfz8fPz9/VmwYAE333xz0f3/+te/2LZtG6tXr/7bc/Ly8sjLyyvazsjIICoqSmVG3NoL3+5m5o8H8Pa08N69rWjboGxPWk08mc09czZy6EQWlf29eXdwS2Jrh5Tpe15MakYub/10kM9/PcLp7IJzjgkgh+e859Hf03EoOzGwGUF3zSM4ol55RhURKtA5MydOnMBmsxEeXvxfc+Hh4aSkpJzzOVOnTiU4OLjoFhUVVR5RRZzaI92voFfT6hTYDO57fzNbDp8ss/eKO5LOLTPXcehEFjUq+/H5iLamFxmAakG+PNW7Mb880YWZdzXnjpZRRFerhN8fJ0ZfZTnAt35P0d9zDXY8yG33KFFjf1CREXFyXmYHKKm/Hl83DOO8x9wnTpzIuHHjirbP7pkRcWceHhZevq0ZGbkFrNl3giFzNvHx8GtpWrPkk9SVxA+7jzHqo63kFNi4MiKQ94a2IjyodC6NLi1WL096Nq1Oz6aOVawNuw3bz6/jueq/WOyFEByFxy1v41u7jclJRaQknH7PTFhYGJ6enn/bC5Oamvq3vTVnWa1WgoKCit1EBHy9PXlrUAta1Q3lTF4hA9/ZwC8H00rltQ3D4IP18dz33mZyCmy0jw5jwYg2Tldk/uZMCpYP++O1cpKjyDS+CUasARUZEZfh9GXGx8eH2NhYVqxYUez+FStW0LZtW5NSibguPx9P5gxpSas6oZzJLWTQnI18HZd8Wa+ZnV/IuM+28/TSndgNuDW2JnOGtCTQ17uUUpeRvcthZls4uAq8/KDP63Dre+Bn/iExESk5py8zAOPGjeOdd95hzpw5/P777zz88MMkJCQwYsQIs6OJuKRKVi/eH9aK7o3DyS+0M/KjX3nui13k/3Glz6XYnniam2b8zOKtR/H0sDCx55VMG3DVZU1QV+YK8+Cbx+HjWyE7DcKbwgOrIXaw5o4RcUEucc7M7bffTlpaGs899xzJycnExMTw9ddfU7t2bbOjibgsX29P3ryrOc9/s5t31h5izs+HWH8wjeduakLLOqEXfX5aZh7Tf9jPe+vjMQyoFmhl+p3X0LpelXJIfxmO74HPh8Exx6XqtB4BXZ8Fbyc/HCYi5+X0l2aXBs0zI3JhK3Yd45HPtxddrtw+Ooy7WtfmuugwKln//988hTY724+ks3TbURZsPkJOgQ2AfldH8lTvxk6z/tM5GQb8+j588xgU5oB/Feg3Exr2MDuZiJxHhZlnpjSozIhcXFpmHi8t38OnmxI5Ozmut6eFmiH+hPh7k51vIz4ti9yC/z8U1bRGMI/dcCXXRTv5Qos5p+CLf8GupY7tep3g5tkQGGFqLBG5MJWZP1GZESm5xJPZfLDhMN/tTOFwWvbfHg+0etG1cTi3NK/BdQ3CynxZgsuWsAEW3gfpieDhBdc/DW0fAg8nPqdHRACVmWJUZkT+maOnc0hIyyY9Jx9fb09qhvhTLywADw8nLzAAtkJY8xKsfgEMO4TUhQHvQo1Ys5OJSAmV9PvbJU4AFhFz1KjsR43KfmbHuHSnE2HRcEhY79i+6g7o9RJYA83NJSJlQmVGRCqWXUth2RjITQefStDrFWh2u9mpRKQMqcyISMWQnw3fPg6/vufYrhEL/d+BUK2rJFLRqcyIiOtLiXPMHXNiD2CB68ZC5yfB08lnIBaRUqEyIyKuyzBg41uw/Gmw5UGlCLhltuPSaxFxGyozIuKask7A0lGw91vHdsMb4KY3IMDJ57wRkVKnMiMirufgj7DoAchMAU8rdP8vtBqudZVE3JTKjIi4DlsB/PBf+Pl/gAFhV8CAORARY3YyETGRyoyIuIaTBx0n+Sb96tiOHQI9poKPv6mxRMR8KjMi4vy2fwpfjYf8M+AbDH1nQOO+ZqcSESehMiMizivvDHw1AX6b79iu1RZueQsqR5mbS0ScisqMiDino1sch5VOHQKLB3R8HDpMAA9Ps5OJiJNRmRER52K3w7r/OU70tRdCcBTc8jbUbmN2MhFxUiozIuI8zqTA4gccl14DNO4HfV4DvxATQ4mIs1OZERHnsOdbWDoSstPA2x9ueB6a36O5Y0TkolRmRMRcBbnw/TPwyyzHdkRT6D8HqjY0N5eIuAyVGRExz/E9jpN8j8U5tq8dCV0ngZfV1Fgi4lpUZkSk/BkG/PoefPM4FOaAfxj0mwkNu5udTERckMqMiJSvnFOw7CH4fZlju14nuHk2BEaYGktEXJfKjIiUn8PrYeF9kHEEPLygy7+hzRjw8DA7mYi4MJUZESl7tkL46UX4aRoYdgitB/3fhRrNzU4mIhWAyoyIlK3TibBoOCSsd2w3uxNufBGsgebmEpEKQ2VGRMrOziXwxUOQmw4+gdD7VbjqVrNTiUgFozIjIqUvPwu+nei4YgmgRqzjsFJoXXNziUiFpDIjIqUrJQ4+Hwon9gIWuO5h6PwEeHqbnUxEKiiVGREpHYYBv8yGFU+DLR8qRcAtb0G9jmYnE5EKTmVGRC5f1glYMhL2fefYbtgTbnoDAqqYm0tE3ILKjIhcngOrHCtdZx4DTyt0/y+0Gq4FIkWk3KjMiMg/U5gPq/4LP78OGFD1SsdJvhExZicTETejMiMily7tACwcBklbHdux90KPKeDjb24uEXFLKjMicmm2z4evxkN+JvhWhr7ToXFfs1OJiBtTmRGRksnNcJSYuM8c27XbOa5WCq5pbi4RcXtlurrb5MmTadu2Lf7+/lSuXPmcYxISEujTpw8BAQGEhYXx0EMPkZ+fX2xMXFwcHTt2xM/Pjxo1avDcc89hGEZZRheRPzuyBWa3dxQZiyd0fhIGf6EiIyJOoUz3zOTn53PrrbfSpk0b3n333b89brPZ6NWrF1WrVmXt2rWkpaUxePBgDMNg+vTpAGRkZNCtWzc6d+7Mpk2b2Lt3L0OGDCEgIIDx48eXZXwRsdvh59dg1WSwF0JwFPR/B2pda3YyEZEiZVpmnn32WQDmzZt3zseXL1/Orl27SExMJDIyEoCXX36ZIUOGMHnyZIKCgvjoo4/Izc1l3rx5WK1WYmJi2Lt3L6+88grjxo3Doss/RcpGRrLjkutDqx3bTW6G3q+BX2UzU4mI/E2ZHma6mPXr1xMTE1NUZAB69OhBXl4eW7ZsKRrTsWNHrFZrsTFJSUnEx8ef83Xz8vLIyMgodhORS7DnW5jVzlFkvP2h7wwYMFdFRkSckqllJiUlhfDw8GL3hYSE4OPjQ0pKynnHnN0+O+avpk6dSnBwcNEtKiqqDNKLVEAFufD1o/DJ7ZCdBhFN4f7V0HyQJsETEad1yWVm0qRJWCyWC942b95c4tc712EiwzCK3f/XMWdP/j3fIaaJEyeSnp5edEtMTCxxHhG3lbob3ukCG2c7tq8dBfethKoNzc0lInIRl3zOzOjRo7njjjsuOKZOnToleq2IiAh++eWXYvedOnWKgoKCor0vERERf9sDk5qaCvC3PTZnWa3WYoelROQCDAO2zINvJ0JhDviHwc2zILqb2clERErkkstMWFgYYWFhpfLmbdq0YfLkySQnJ1O9enXAcVKw1WolNja2aMwTTzxBfn4+Pj4+RWMiIyNLXJpE5DyyT8IXD8HvXzi263WGm2dD4Ln/oSAi4ozK9JyZhIQEtm3bRkJCAjabjW3btrFt2zYyMzMB6N69O40bN2bQoEFs3bqVlStXMmHCBIYPH05QUBAAAwcOxGq1MmTIEHbs2MHixYuZMmWKrmQSuVyH18Gs9o4i4+EN3f4Ddy9SkRERl2MxynD2uSFDhvDee+/97f5Vq1bRqVMnwFF4Ro4cyQ8//ICfnx8DBw7kpZdeKnaYKC4ujlGjRrFx40ZCQkIYMWIE//73v0tcZjIyMggODiY9Pb2oJIm4LVsh/DQNfnoRDDuE1nMsEFmjudnJRESKKen3d5mWGWehMiPyh9MJsHA4JG5wbDcbCDdOA2ugublERM6hpN/fWptJxF3sXAzL/gV56eATCL1fhatuNTuViMhlU5kRqejys+Dbx+HX9x3bNVo4liQIrWtuLhGRUqIyI1KRJf8GC4fBib2ABdqPg04TwdPb7GQiIqVGZUakIjIM2DATvn8GbPkQWN1xyXW9jmYnExEpdSozIhVN5nFYOhL2LXdsX3GjY22lgCrm5hIRKSMqMyIVyYEfYPEIyDwGnlboMRla3qd1lUSkQlOZEakICvPhh//Autcd21UbwYB3IbyJublERMqByoyIq0s74DjJN2mrY7vFUOg+GXz8zc0lIlJOVGZEXNn2+fDVeMjPBN/KcNMMaNTH7FQiIuVKZUbEFeVmOEpM3GeO7drt4Ja3ILimublEREygMiPiao5sdhxWOhUPFk/HvDHtx4GHp9nJRERMoTIj4irsdvj5NVg1GeyFEFzLMZNvrdZmJxMRMZXKjIgryEiGxffDoZ8c201ucayt5FfZ1FgiIs5AZUbE2e35BpaMhJyT4O0PPafBNXdr7hgRkT+ozIg4q4JcWPE0bHzLsR1xFQyYA2HR5uYSEXEyKjMizih1N3w+FFJ3OravHQVdnwEvq7m5RESckMqMiDMxDNgyF76dCIW5EFAV+s2C6K5mJxMRcVoqMyLOIvskfPEQ/P6FY7t+F7h5FlSqZm4uEREnpzIj4gzif4ZFwyHjKHh4Ow4pXTsKPDzMTiYi4vRUZkTMZCuE1S/AmpfAsENofccCkZHXmJ1MRMRlqMyImOXUYcfemMRfHNtX3+W47NpaydxcIiIuRmVGxAw7FsEXYyEvHaxBjgnwmg4wO5WIiEtSmREpT/lZ8M1jsPUDx3bNlo4lCULqmBpLRMSVqcyIlJfk7fD5MEjbB1ig/Xjo9Dh4epudTETEpanMiJQ1w4ANM+H7Z8CWD4HV4Za3oG4Hs5OJiFQIKjMiZSnzOCx5EPavcGxf0QtumgH+oebmEhGpQFRmRMrK/pWweARkpYKnFXpMhpb3aYFIEZFSpjIjUtoK8+GH52DddMd21UaOuWPCm5ibS0SkglKZESlNaQccC0Qmb3Nstxjm2CPj7WdqLBGRikxlRqQ0GAZs/wS+mgAFWeAXAn1nQKPeZicTEanwVGZELlduBnw1DuIWOLZrX+e4Wim4hrm5RETchMqMyOVI3AQLh8Hpw2DxhM4T4bpx4OFpdjIREbehMiPyT9htsPZVWDUFDBtUrgX934WoVmYnExFxOyozIpcqIwkW3Q/xaxzbTW6BPq+Bb7CpsURE3JXKjMil2P01LB0FOSfBOwBunOZY7Vpzx4iImMajrF44Pj6eYcOGUbduXfz8/Khfvz7PPPMM+fn5xcYlJCTQp08fAgICCAsL46GHHvrbmLi4ODp27Iifnx81atTgueeewzCMsoou8ncFOY4rlebf6Sgy1ZvBAz/BNXeryIiImKzM9szs3r0bu93O7NmzadCgATt27GD48OFkZWXx0ksvAWCz2ejVqxdVq1Zl7dq1pKWlMXjwYAzDYPp0x4RjGRkZdOvWjc6dO7Np0yb27t3LkCFDCAgIYPz48WUVX+T/pf7umDsmdZdju81o6PJv8LKam0tERACwGOW4i+PFF19k5syZHDx4EIBvvvmG3r17k5iYSGRkJADz589nyJAhpKamEhQUxMyZM5k4cSLHjh3DanV8eTz//PNMnz6dI0eOYCnBv4ozMjIIDg4mPT2doKCgsvsBpWIxDNg8B757AgpzIaAq9JsF0V3NTiYi4hZK+v1dZoeZziU9PZ3Q0P9fYG/9+vXExMQUFRmAHj16kJeXx5YtW4rGdOzYsajInB2TlJREfHx8uWUXN5N9Ej692zF/TGEu1O8CD65TkRERcULldgLwgQMHmD59Oi+//HLRfSkpKYSHhxcbFxISgo+PDykpKUVj6tSpU2zM2eekpKRQt27dv71XXl4eeXl5RdsZGRml9WOIO4hf67haKeMoeHhD10lw7UjwKNfuLyIiJXTJfztPmjQJi8VywdvmzZuLPScpKYkbbriBW2+9lfvuu6/YY+c6TGQYRrH7/zrm7JGx8x1imjp1KsHBwUW3qKioS/0xxR3ZCuGHyTCvt6PIVGkA930PbUeryIiIOLFL3jMzevRo7rjjjguO+fOelKSkJDp37kybNm146623io2LiIjgl19+KXbfqVOnKCgoKNr7EhERUbSX5qzU1FSAv+3VOWvixImMGzeuaDsjI0OFRi7s1GFYNBwS//h9vPpu6PkCWCuZm0tERC7qkstMWFgYYWFhJRp79OhROnfuTGxsLHPnzsXjL/+6bdOmDZMnTyY5OZnq1asDsHz5cqxWK7GxsUVjnnjiCfLz8/Hx8SkaExkZ+bfDT2dZrdZi59iIXNCOhfDFw5CXDtYg6P0qNB1gdioRESmhMtt3npSURKdOnYiKiuKll17i+PHjpKSkFNvL0r17dxo3bsygQYPYunUrK1euZMKECQwfPrzorOWBAwditVoZMmQIO3bsYPHixUyZMoVx48aV6EomkfPKz3JMgPf5UEeRqdkSRqxRkRERcTFldgLw8uXL2b9/P/v376dmzZrFHjt7zounpydfffUVI0eOpF27dvj5+TFw4MCieWgAgoODWbFiBaNGjaJFixaEhIQwbty4YoeRRC5Z0jbHApFp+wELtB8PnR4HT2+zk4mIyCUq13lmzKJ5ZqSI3Q6/zIQVz4C9AAIj4Za3oG57s5OJiMhflPT7W2szifvITIUlD8L+7x3bV/aGvtPBP/TCzxMREaemMiPuYf9KWDwCslLByxd6TIYWw7SukohIBaAyIxVbYT6sfBbWz3BsV2sM/d+F8Mbm5hIRkVKjMiMV14n9sHAoJG93bLe8D7r/F7z9zM0lIiKlSmVGKh7DgG0fw9ePQEEW+IXATW/Alb3MTiYiImVAZUYqltx0+PJhx0R4AHXaw82zIbiGublERKTMqMxIxZG4yXFY6XQCWDyh8xNw3cPg4Wl2MhERKUMqM+L67DZY+yqsmgKGDSrXgv5zIKql2clERKQcqMyIa8tIgkX3Q/wax3bMAOj9CvgGm5tLRETKjcqMuK7dXznWVso5Bd4B0OslaHan5o4REXEzKjPiegpyYPlTsOkdx3b1Zo7DSmENzM0lIiKmUJkR13Jsl2OByNRdju02o6HLM+DlY24uERExjcqMuAbDgM3vwndPQmEuBFSDm2dCg65mJxMREZOpzIjzyz4Jy8bA7i8d2w26Qr+ZUKmaublERMQpqMyIczu0xnG10pkk8PCGbs9C6wfBw8PsZCIi4iRUZsQ52Qrgx+dhzcuAAVUaOBaIjLza7GQiIuJkVGbE+Zw6DAvvgyMbHdvX3A03vADWSubmEhERp6QyI84l7nPH2kp5GWANgj6vQUx/s1OJiIgTU5kR55CXCd88Bts+dGzXbAX934GQ2ubmEhERp6cyI+ZL2uaYOyZtP2CBDo9Ax8fAU7+eIiJycfq2EPPY7bDhTfh+EtgLIKgG3PIW1LnO7GQiIuJCVGbEHJmpsHgEHFjp2L6yN/SdDv6h5uYSERGXozIj5W//944ik3UcvHyhxxRoMVQLRIqIyD+iMiPlpzAPVj4H62c4tqs1gQHvQrVG5uYSERGXpjIj5ePEflg4FJK3O7ZbDofu/wFvP3NziYiIy1OZkbJlGLDtY/j6ESjIAr9QuOkNuPJGs5OJiEgFoTIjZSc33TEB3o6Fju067R1XKwVFmptLREQqFJUZKRuJGx1zx5xOAIsnXP8ktBsLHp5mJxMRkQpGZUZKl90Ga1+BVVPBsEHl2o4FIqNamp1MREQqKJUZKT3pR2HxAxC/xrEdMwB6vwK+webmEhGRCk1lRkrH71/CstGQcwq8A6DXS9DsTs0dIyIiZU5lRi5PQQ589yRsftexXf1qGDAHqtQ3NZaIiLgPlRn5547tgs+HwvHfHdttx8D1/wYvH3NziYiIW1GZkUtnGLDpHVj+FBTmQkA1uHkWNOhidjIREXFDKjNyabJPwtLRsOcrx3aDbtBvJlSqam4uERFxWyozUnKH1sCi++FMEnj6QNdnofUI8PAwO5mIiLixMv0W6tu3L7Vq1cLX15fq1aszaNAgkpKSio1JSEigT58+BAQEEBYWxkMPPUR+fn6xMXFxcXTs2BE/Pz9q1KjBc889h2EYZRld/sxW4Fgg8r0+jiJTJRru+x7ajFSRERER05XpnpnOnTvzxBNPUL16dY4ePcqECRMYMGAA69atA8Bms9GrVy+qVq3K2rVrSUtLY/DgwRiGwfTp0wHIyMigW7dudO7cmU2bNrF3716GDBlCQEAA48ePL8v4AnAqHhbeB0c2ObavGQQ9XwCfAFNjiYiInGUxynEXx7Jly+jXrx95eXl4e3vzzTff0Lt3bxITE4mMdKzXM3/+fIYMGUJqaipBQUHMnDmTiRMncuzYMaxWKwDPP/8806dP58iRI1hKMI9JRkYGwcHBpKenExQUVKY/Y4US97ljbaW8DLAGQ5/XIOYWs1OJiIibKOn3d7kdIzh58iQfffQRbdu2xdvbG4D169cTExNTVGQAevToQV5eHlu2bCka07Fjx6Iic3ZMUlIS8fHx53yvvLw8MjIyit3kEuRlwpKRjrWV8jIgqjWMWKMiIyIiTqnMy8xjjz1GQEAAVapUISEhgaVLlxY9lpKSQnh4eLHxISEh+Pj4kJKSct4xZ7fPjvmrqVOnEhwcXHSLiooqzR+pYkvaCrM7wLaPwOIBHR+DIV9DSG2zk4mIiJzTJZeZSZMmYbFYLnjbvHlz0fhHHnmErVu3snz5cjw9PbnnnnuKnbx7rsNEhmEUu/+vY84+/3yHmCZOnEh6enrRLTEx8VJ/TPdjt8O66fBONzh5AIJqwOAvofMT4KmL3kRExHld8rfU6NGjueOOOy44pk6dOkX/PywsjLCwMBo2bEijRo2Iiopiw4YNtGnThoiICH755Zdizz116hQFBQVFe18iIiL+tgcmNTUV4G97bM6yWq3FDkvJRZw5BktGwIEfHNuN+kCf18E/1NxcIiIiJXDJZeZsOfknzu5RycvLA6BNmzZMnjyZ5ORkqlevDsDy5cuxWq3ExsYWjXniiSfIz8/Hx8enaExkZGSx0iSXYfXzjiLj5Qs3TIXYe7VApIiIuIwyO2dm48aNzJgxg23btnH48GFWrVrFwIEDqV+/Pm3atAGge/fuNG7cmEGDBrF161ZWrlzJhAkTGD58eNFZywMHDsRqtTJkyBB27NjB4sWLmTJlCuPGjSvRlUxSAl2egYY94f4focVQFRkREXEpZVZm/Pz8WLRoEV26dOGKK65g6NChxMTEsHr16qJDQJ6ennz11Vf4+vrSrl07brvtNvr168dLL71U9DrBwcGsWLGCI0eO0KJFC0aOHMm4ceMYN25cWUV3P36VYeB8qNbI7CQiIiKXrFznmTGL5pkRERFxPU43z4yIiIhIWVCZEREREZemMiMiIiIuTWVGREREXJrKjIiIiLg0lRkRERFxaSozIiIi4tJUZkRERMSlqcyIiIiIS1OZEREREZemMiMiIiIuTWVGREREXJqX2QHKw9m1NDMyMkxOIiIiIiV19nv7Ymtiu0WZOXPmDABRUVEmJxEREZFLdebMGYKDg8/7uMW4WN2pAOx2O0lJSQQGBmKxWEr1tTMyMoiKiiIxMfGCy5NL+dDn4Vz0eTgXfR7ORZ/HxRmGwZkzZ4iMjMTD4/xnxrjFnhkPDw9q1qxZpu8RFBSkX0Ynos/DuejzcC76PJyLPo8Lu9AembN0ArCIiIi4NJUZERERcWkqM5fJarXyzDPPYLVazY4i6PNwNvo8nIs+D+eiz6P0uMUJwCIiIlJxac+MiIiIuDSVGREREXFpKjMiIiLi0lRmRERExKWpzPwD8fHxDBs2jLp16+Ln50f9+vV55plnyM/PLzYuISGBPn36EBAQQFhYGA899NDfxkjpmTx5Mm3btsXf35/KlSufc4w+k/Lz5ptvUrduXXx9fYmNjWXNmjVmR3IbP/30E3369CEyMhKLxcKSJUuKPW4YBpMmTSIyMhI/Pz86derEzp07zQlbwU2dOpWWLVsSGBhItWrV6NevH3v27Ck2Rp/H5VOZ+Qd2796N3W5n9uzZ7Ny5k1dffZVZs2bxxBNPFI2x2Wz06tWLrKws1q5dy/z581m4cCHjx483MXnFlp+fz6233sqDDz54zsf1mZSfTz/9lLFjx/Lkk0+ydetW2rdvT8+ePUlISDA7mlvIysqiWbNmzJgx45yPT5s2jVdeeYUZM2awadMmIiIi6NatW9E6dlJ6Vq9ezahRo9iwYQMrVqygsLCQ7t27k5WVVTRGn0cpMKRUTJs2zahbt27R9tdff214eHgYR48eLbrvk08+MaxWq5Genm5GRLcxd+5cIzg4+G/36zMpP61atTJGjBhR7L4rr7zSePzxx01K5L4AY/HixUXbdrvdiIiIMJ5//vmi+3Jzc43g4GBj1qxZJiR0L6mpqQZgrF692jAMfR6lRXtmSkl6ejqhoaFF2+vXrycmJobIyMii+3r06EFeXh5btmwxI6Lb02dSPvLz89myZQvdu3cvdn/37t1Zt26dSankrEOHDpGSklLs87FarXTs2FGfTzlIT08HKPq+0OdROlRmSsGBAweYPn06I0aMKLovJSWF8PDwYuNCQkLw8fEhJSWlvCMK+kzKy4kTJ7DZbH/7sw4PD9efsxM4+xno8yl/hmEwbtw4rrvuOmJiYgB9HqVFZeZPJk2ahMViueBt8+bNxZ6TlJTEDTfcwK233sp9991X7DGLxfK39zAM45z3y7n9k8/kQvSZlJ+//pnqz9m56PMpf6NHj+a3337jk08++dtj+jwuj5fZAZzJ6NGjueOOOy44pk6dOkX/Pykpic6dO9OmTRveeuutYuMiIiL45Zdfit136tQpCgoK/tbA5fwu9TO5EH0m5SMsLAxPT8+//asyNTVVf85OICIiAnDsEahevXrR/fp8ytaYMWNYtmwZP/30EzVr1iy6X59H6VCZ+ZOwsDDCwsJKNPbo0aN07tyZ2NhY5s6di4dH8Z1cbdq0YfLkySQnJxf9gi5fvhyr1UpsbGypZ6+oLuUzuRh9JuXDx8eH2NhYVqxYwc0331x0/4oVK7jppptMTCYAdevWJSIighUrVnDNNdcAjvOcVq9ezQsvvGByuorHMAzGjBnD4sWL+fHHH6lbt26xx/V5lA6VmX8gKSmJTp06UatWLV566SWOHz9e9NjZlt29e3caN27MoEGDePHFFzl58iQTJkxg+PDhBAUFmRW9QktISODkyZMkJCRgs9nYtm0bAA0aNKBSpUr6TMrRuHHjGDRoEC1atCjac5mQkFDsvDIpO5mZmezfv79o+9ChQ2zbto3Q0FBq1arF2LFjmTJlCtHR0URHRzNlyhT8/f0ZOHCgiakrplGjRvHxxx+zdOlSAgMDi/ZYBgcH4+fnh8Vi0edRGky8ksplzZ071wDOefuzw4cPG7169TL8/PyM0NBQY/To0UZubq5JqSu+wYMHn/MzWbVqVdEYfSbl54033jBq165t+Pj4GM2bNy+6FFXK3qpVq87538LgwYMNw3BcDvzMM88YERERhtVqNTp06GDExcWZG7qCOt93xdy5c4vG6PO4fBbDMIzyLE8iIiIipUlXM4mIiIhLU5kRERERl6YyIyIiIi5NZUZERERcmsqMiIiIuDSVGREREXFpKjMiIiLi0lRmRERExKWpzIiIiIhLU5kRERERl6YyIyIiIi5NZUZERERc2v8BTyE1AasXqigAAAAASUVORK5CYII=\n", 199 | "text/plain": [ 200 | "
" 201 | ] 202 | }, 203 | "metadata": {}, 204 | "output_type": "display_data" 205 | } 206 | ], 207 | "source": [ 208 | "plt.plot(x, y(x));\n", 209 | "\n", 210 | "grad_x = 5.0\n", 211 | "plt.plot(x + grad_x, (grad_y(grad_x)*x + y(grad_x)))" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "id": "391c6d97", 217 | "metadata": {}, 218 | "source": [ 219 | "## Gradients in 2D" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 56, 225 | "id": "a209a0bb", 226 | "metadata": {}, 227 | "outputs": [ 228 | { 229 | "data": { 230 | "text/plain": [ 231 | "(Array(320., dtype=float32, weak_type=True), 320)" 232 | ] 233 | }, 234 | "execution_count": 56, 235 | "metadata": {}, 236 | "output_type": "execute_result" 237 | } 238 | ], 239 | "source": [ 240 | "def z(x,y):\n", 241 | " return 20*x*(4*y)\n", 242 | "\n", 243 | "grad_z = jax.grad(z)\n", 244 | "\n", 245 | "grad_z(1.0, 4.0), 4*20*4" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 59, 251 | "id": "7bcf69da", 252 | "metadata": {}, 253 | "outputs": [ 254 | { 255 | "data": { 256 | "text/plain": [ 257 | "(Array(80., dtype=float32, weak_type=True), 80)" 258 | ] 259 | }, 260 | "execution_count": 59, 261 | "metadata": {}, 262 | "output_type": "execute_result" 263 | } 264 | ], 265 | "source": [ 266 | "grad_z_res_y = jax.grad(z, argnums=1)\n", 267 | "\n", 268 | "grad_z_res_y(1.0, 4.0), 20*4" 269 | ] 270 | }, 271 | { 272 | "cell_type": "markdown", 273 | "id": "c71582fb", 274 | "metadata": {}, 275 | "source": [ 276 | "## Value and Grad" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 61, 282 | "id": "083a533c", 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "data": { 287 | "text/plain": [ 288 | "(Array(25., dtype=float32, weak_type=True),\n", 289 | " Array(10., dtype=float32, weak_type=True))" 290 | ] 291 | }, 292 | "execution_count": 61, 293 | "metadata": {}, 294 | "output_type": "execute_result" 295 | } 296 | ], 297 | "source": [ 298 | "x = np.linspace(-20, 20, 1000)\n", 299 | "\n", 300 | "def y(x):\n", 301 | " return x**2\n", 302 | "\n", 303 | "jax.value_and_grad(y)(5.0)" 304 | ] 305 | } 306 | ], 307 | "metadata": { 308 | "kernelspec": { 309 | "display_name": "Python 3 (ipykernel)", 310 | "language": "python", 311 | "name": "python3" 312 | }, 313 | "language_info": { 314 | "codemirror_mode": { 315 | "name": "ipython", 316 | "version": 3 317 | }, 318 | "file_extension": ".py", 319 | "mimetype": "text/x-python", 320 | "name": "python", 321 | "nbconvert_exporter": "python", 322 | "pygments_lexer": "ipython3", 323 | "version": "3.10.4" 324 | } 325 | }, 326 | "nbformat": 4, 327 | "nbformat_minor": 5 328 | } 329 | --------------------------------------------------------------------------------