├── .gitignore ├── README.md ├── notebooks ├── Makefile ├── README.md ├── img │ ├── .DS_Store │ ├── 1d_conv.jpeg │ ├── aae_dim_reduc_2.png │ ├── ai_ml_dl.png │ ├── alexnet.png │ ├── algorithms1.png │ ├── algorithms10.png │ ├── algorithms11.png │ ├── algorithms12.png │ ├── algorithms13.png │ ├── algorithms2.png │ ├── algorithms3.png │ ├── algorithms4.png │ ├── algorithms5.png │ ├── algorithms6.png │ ├── algorithms7.png │ ├── algorithms8.png │ ├── algorithms9.png │ ├── alphago.jpeg │ ├── approximation.png │ ├── breakthrough.jpg │ ├── cartoonLVmodel.png │ ├── celebA.png │ ├── cnns │ │ ├── .DS_Store │ │ ├── 2dconv.gif │ │ ├── alexnet.png │ │ ├── cnn-filters-all.png │ │ ├── cnn-filters1.png │ │ ├── cnn-filters2.png │ │ ├── cnn-filters4.png │ │ ├── cnn.png │ │ ├── convlayer.png │ │ ├── flybrain.gif │ │ ├── hierarchy.jpg │ │ ├── hog.jpg │ │ ├── hubel.ppm │ │ ├── imagecaption.png │ │ ├── imagenet.png │ │ ├── layer1.png │ │ ├── layer2.png │ │ ├── layer3.png │ │ ├── layer45.png │ │ ├── lenet.png │ │ ├── levels.png │ │ ├── pathway.png │ │ ├── pooling.jpg │ │ ├── resnet-block-im.png │ │ ├── resnet-block.png │ │ ├── sift.jpg │ │ ├── stylegan.png │ │ └── unet.png │ ├── coordinate_descent.png │ ├── cornell_tech1.png │ ├── cornell_tech2.svg │ ├── cornell_tech3.png │ ├── cornell_tech4.jpg │ ├── cornell_tech5.png │ ├── darts.png │ ├── dcgan_feats.png │ ├── decision_tree.png │ ├── dl_data_curve.png │ ├── dna_map.jpg │ ├── functional_gradient.png │ ├── functional_gradient.pptx │ ├── google.png │ ├── google_assistant.png │ ├── image_captioning.jpeg │ ├── kmeans_convergence.gif │ ├── l1-vs-l2-annotated.png │ ├── l1-vs-l2.png │ ├── learning.png │ ├── learning_curve1.png │ ├── learning_curve2.png │ ├── learning_curve3.png │ ├── learning_curve4.png │ ├── learning_curve5.png │ ├── learning_curve6.png │ ├── learning_curve7.png │ ├── learning_curve8.png │ ├── life3.0.jpg │ ├── loss_curve1.png │ ├── loss_curve2.png │ ├── loss_curve3.png │ ├── loss_curve4.png │ ├── margin.png │ ├── mogdata.png │ ├── mogdata2.png │ ├── mogdata2_v2.png │ ├── mogdata_v2.png │ ├── mogdensity.png │ ├── mogdensity1d.png │ ├── mogdensity1d_v2.png │ ├── mogdensity_v2.png │ ├── mogdensity_v3.png │ ├── nns │ │ ├── aneuron.jpeg │ │ ├── backprop │ │ │ ├── backprop-abstract-figure0.png │ │ │ ├── backprop-abstract-figure1.png │ │ │ ├── backprop-abstract-figure2.png │ │ │ ├── backprop-abstract-figure3.png │ │ │ ├── backprop-abstract-figure4.png │ │ │ ├── backprop-abstract-figure5.png │ │ │ ├── backprop-abstract-figure6.png │ │ │ ├── backprop-twolayers-figure0.png │ │ │ ├── backprop-twolayers-figure1.png │ │ │ ├── backprop-twolayers-figure10.png │ │ │ ├── backprop-twolayers-figure11.png │ │ │ ├── backprop-twolayers-figure2.png │ │ │ ├── backprop-twolayers-figure3.png │ │ │ ├── backprop-twolayers-figure4.png │ │ │ ├── backprop-twolayers-figure5.png │ │ │ ├── backprop-twolayers-figure6.png │ │ │ ├── backprop-twolayers-figure7.png │ │ │ ├── backprop-twolayers-figure8.png │ │ │ ├── backprop-twolayers-figure9.png │ │ │ └── backprop-twolayers-legend.png │ │ ├── bio-neuron.png │ │ ├── bp1.png │ │ ├── layers-1.png │ │ ├── layers-2.png │ │ ├── layers-3.png │ │ ├── layers-4.png │ │ ├── layers-L.png │ │ ├── net-intro.png │ │ └── nn-example.png │ ├── obama.jpg │ ├── obama2.jpg │ ├── oldfMOG_v2.png │ ├── oldfSingle_v2.png │ ├── oldfaithful_v2.png │ ├── pca_decomposition.png │ ├── pca_example.png │ ├── pca_example_plane.png │ ├── pca_projection1.png │ ├── pca_projection2.png │ ├── pca_projection_data.png │ ├── pca_reconstruction.png │ ├── pca_two_views.png │ ├── rl.png │ ├── siri.png │ ├── spam.png │ ├── svm-margin-fig.key │ ├── tesla.jpg │ ├── tesla_data.png │ ├── tesla_zoom.jpg │ └── waymo.jpg ├── lecture1-introduction.ipynb ├── lecture10-svm-dual.ipynb ├── lecture11-kernels.ipynb ├── lecture12-decision-trees.ipynb ├── lecture12-neural-networks.ipynb ├── lecture13-boosting.ipynb ├── lecture14-neural-networks.ipynb ├── lecture15-deep-learning.ipynb ├── lecture16-unsupervised-learning.ipynb ├── lecture17-density-estimation.ipynb ├── lecture18-clustering.ipynb ├── lecture19-dimensionality-reduction.bak.ipynb ├── lecture19-dimensionality-reduction.ipynb ├── lecture2-supervised-learning.ipynb ├── lecture20-evaluation.ipynb ├── lecture21-model-iteration.ipynb ├── lecture22-evaluation-tools.ipynb ├── lecture23-overview.ipynb ├── lecture3-linear-regression.ipynb ├── lecture4-foundations-supervised-learning.ipynb ├── lecture5-maximum-likelihood.ipynb ├── lecture5b-probabilistic-perspectives.ipynb ├── lecture6-classification.ipynb ├── lecture7-generative-models.ipynb ├── lecture8-naive-bayes.ipynb └── lecture9-support-vector-machines.ipynb └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Applied Machine Learning (Cornell CS5785) 2 | 3 | This repo contains executable course notes and slides for the Applied ML course at Cornell and Cornell Tech. 4 | 5 | 8 | 9 | These materials accompany a set of Youtube [lecture videos](https://www.youtube.com/watch?v=vcE9WGbi4QY&list=PL2UML_KCiC0UlY7iCQDSiGDMovaupqc83) from the Fall 2020 edition of the course. 10 | 11 | ## Contents 12 | 13 | This repo is organized as follows. 14 | 15 | ``` 16 | . 17 | ├── README.md 18 | ├── notebooks # Notebooks and slides 19 | └── requirements.txt # Packages needed for your virtualenv 20 | ``` 21 | 22 | ## Setup 23 | 24 | ### Requirements 25 | 26 | You should be able to run all the contents of this repo using the packages provided in `requirements.txt`. 27 | 28 | In a new `virtualenv`, run this: 29 | ``` 30 | pip install -r requirements.txt 31 | ``` 32 | 33 | ## Feedback 34 | 35 | Please send feedback to [Volodymyr Kuleshov](https://www.cs.cornell.edu/~kuleshov/) 36 | -------------------------------------------------------------------------------- /notebooks/Makefile: -------------------------------------------------------------------------------- 1 | #LECTURES:=lecture1 lecture2 lecture3 lecture4 lecture5b lecture6 2 | 3 | NOTEBOOKS:=$(wildcard *.ipynb) 4 | SLIDES:=$(subst ipynb,slides.html,$(NOTEBOOKS)) 5 | HTML:=$(subst ipynb,html,$(NOTEBOOKS)) 6 | PDFS:=$(subst ipynb,pdf,$(NOTEBOOKS)) 7 | 8 | $(SLIDES): %.slides.html: %.ipynb 9 | jupyter nbconvert $< --to slides --reveal-prefix "http://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0" 10 | 11 | $(HTML): %.html: %.ipynb 12 | jupyter nbconvert $< --to html_embed --reveal-prefix "http://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0" 13 | 14 | $(PDFS): %.pdf: %.ipynb 15 | jupyter nbconvert $< --to pdf --reveal-prefix "http://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0" 16 | 17 | notes: 18 | make $(HTML) 19 | 20 | slides: 21 | make $(SLIDES) 22 | 23 | pdfs: 24 | make $(PDFS) 25 | 26 | clean: 27 | rm *.html *.pdf 28 | 29 | all: 30 | make notes slides pdfs 31 | -------------------------------------------------------------------------------- /notebooks/README.md: -------------------------------------------------------------------------------- 1 | # Applied Machine Learning (Cornell CS5785): Notebooks and Slides 2 | 3 | This repo contains executable course notes and slides for the Applied ML course at Cornell and Cornell Tech. 4 | 5 | There are four types of files you can obtain from this folder. These contain all the materials shown in the video lectures. 6 | * Exectuable notebooks in `*.ipynb` format. 7 | * Slides used in the video lectures in HTML format and powered by Reveal.js `*.slides.html` 8 | * Portable course notes in `*.html` format; images are embedded in the notes. 9 | * Portable `*.pdf` files 10 | 11 | All three types of files have the same content, they're just in a different format. 12 | 13 | The Jupyter notebooks are fully interactive and you can use them to regenerate all the materials for the course. 14 | 15 | ## Setup 16 | 17 | In order to generate all the files, use the provded Makefile. 18 | ``` 19 | make all 20 | ``` 21 | 22 | If you want to generate each type of file separately, you can type 23 | ``` 24 | make notes 25 | make slides 26 | make pdfs 27 | ``` 28 | 29 | You can also generate specific files, e.g.: 30 | ``` 31 | make lecture15-deep-learning.slides.html 32 | ``` 33 | 34 | To reset the repo and remove the generated files, do this: 35 | ``` 36 | make clean 37 | ``` 38 | 39 | ### Requirements 40 | 41 | You should be able to run all the contents of this repo using the packages provided in `requirements.txt` at the root of the repo. 42 | 43 | In a new `virtualenv`, run this: 44 | ``` 45 | pip install -r requirements.txt 46 | ``` 47 | 48 | ## Issues 49 | 50 | Currently, there are issues in linking images to the contents: 51 | * The `*.slides.html` must be located in the same folder as the `img` folder in order to display images. 52 | * The `*.pdf` files do not contain images due a limitation of `nbconvert`. 53 | 54 | However the `*.html` notes are fully portable. 55 | 56 | ## Feedback 57 | 58 | Please send feedback to [Volodymyr Kuleshov](https://www.cs.cornell.edu/~kuleshov/) 59 | -------------------------------------------------------------------------------- /notebooks/img/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/.DS_Store -------------------------------------------------------------------------------- /notebooks/img/1d_conv.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/1d_conv.jpeg -------------------------------------------------------------------------------- /notebooks/img/aae_dim_reduc_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/aae_dim_reduc_2.png -------------------------------------------------------------------------------- /notebooks/img/ai_ml_dl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/ai_ml_dl.png -------------------------------------------------------------------------------- /notebooks/img/alexnet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/alexnet.png -------------------------------------------------------------------------------- /notebooks/img/algorithms1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms1.png -------------------------------------------------------------------------------- /notebooks/img/algorithms10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms10.png -------------------------------------------------------------------------------- /notebooks/img/algorithms11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms11.png -------------------------------------------------------------------------------- /notebooks/img/algorithms12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms12.png -------------------------------------------------------------------------------- /notebooks/img/algorithms13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms13.png -------------------------------------------------------------------------------- /notebooks/img/algorithms2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms2.png -------------------------------------------------------------------------------- /notebooks/img/algorithms3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms3.png -------------------------------------------------------------------------------- /notebooks/img/algorithms4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms4.png -------------------------------------------------------------------------------- /notebooks/img/algorithms5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms5.png -------------------------------------------------------------------------------- /notebooks/img/algorithms6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms6.png -------------------------------------------------------------------------------- /notebooks/img/algorithms7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms7.png -------------------------------------------------------------------------------- /notebooks/img/algorithms8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms8.png -------------------------------------------------------------------------------- /notebooks/img/algorithms9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/algorithms9.png -------------------------------------------------------------------------------- /notebooks/img/alphago.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/alphago.jpeg -------------------------------------------------------------------------------- /notebooks/img/approximation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/approximation.png -------------------------------------------------------------------------------- /notebooks/img/breakthrough.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/breakthrough.jpg -------------------------------------------------------------------------------- /notebooks/img/cartoonLVmodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cartoonLVmodel.png -------------------------------------------------------------------------------- /notebooks/img/celebA.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/celebA.png -------------------------------------------------------------------------------- /notebooks/img/cnns/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/.DS_Store -------------------------------------------------------------------------------- /notebooks/img/cnns/2dconv.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/2dconv.gif -------------------------------------------------------------------------------- /notebooks/img/cnns/alexnet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/alexnet.png -------------------------------------------------------------------------------- /notebooks/img/cnns/cnn-filters-all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/cnn-filters-all.png -------------------------------------------------------------------------------- /notebooks/img/cnns/cnn-filters1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/cnn-filters1.png -------------------------------------------------------------------------------- /notebooks/img/cnns/cnn-filters2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/cnn-filters2.png -------------------------------------------------------------------------------- /notebooks/img/cnns/cnn-filters4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/cnn-filters4.png -------------------------------------------------------------------------------- /notebooks/img/cnns/cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/cnn.png -------------------------------------------------------------------------------- /notebooks/img/cnns/convlayer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/convlayer.png -------------------------------------------------------------------------------- /notebooks/img/cnns/flybrain.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/flybrain.gif -------------------------------------------------------------------------------- /notebooks/img/cnns/hierarchy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/hierarchy.jpg -------------------------------------------------------------------------------- /notebooks/img/cnns/hog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/hog.jpg -------------------------------------------------------------------------------- /notebooks/img/cnns/hubel.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/hubel.ppm -------------------------------------------------------------------------------- /notebooks/img/cnns/imagecaption.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/imagecaption.png -------------------------------------------------------------------------------- /notebooks/img/cnns/imagenet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/imagenet.png -------------------------------------------------------------------------------- /notebooks/img/cnns/layer1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/layer1.png -------------------------------------------------------------------------------- /notebooks/img/cnns/layer2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/layer2.png -------------------------------------------------------------------------------- /notebooks/img/cnns/layer3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/layer3.png -------------------------------------------------------------------------------- /notebooks/img/cnns/layer45.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/layer45.png -------------------------------------------------------------------------------- /notebooks/img/cnns/lenet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/lenet.png -------------------------------------------------------------------------------- /notebooks/img/cnns/levels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/levels.png -------------------------------------------------------------------------------- /notebooks/img/cnns/pathway.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/pathway.png -------------------------------------------------------------------------------- /notebooks/img/cnns/pooling.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/pooling.jpg -------------------------------------------------------------------------------- /notebooks/img/cnns/resnet-block-im.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/resnet-block-im.png -------------------------------------------------------------------------------- /notebooks/img/cnns/resnet-block.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/resnet-block.png -------------------------------------------------------------------------------- /notebooks/img/cnns/sift.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/sift.jpg -------------------------------------------------------------------------------- /notebooks/img/cnns/stylegan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/stylegan.png -------------------------------------------------------------------------------- /notebooks/img/cnns/unet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cnns/unet.png -------------------------------------------------------------------------------- /notebooks/img/coordinate_descent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/coordinate_descent.png -------------------------------------------------------------------------------- /notebooks/img/cornell_tech1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cornell_tech1.png -------------------------------------------------------------------------------- /notebooks/img/cornell_tech2.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | red 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /notebooks/img/cornell_tech3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cornell_tech3.png -------------------------------------------------------------------------------- /notebooks/img/cornell_tech4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cornell_tech4.jpg -------------------------------------------------------------------------------- /notebooks/img/cornell_tech5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/cornell_tech5.png -------------------------------------------------------------------------------- /notebooks/img/darts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/darts.png -------------------------------------------------------------------------------- /notebooks/img/dcgan_feats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/dcgan_feats.png -------------------------------------------------------------------------------- /notebooks/img/decision_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/decision_tree.png -------------------------------------------------------------------------------- /notebooks/img/dl_data_curve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/dl_data_curve.png -------------------------------------------------------------------------------- /notebooks/img/dna_map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/dna_map.jpg -------------------------------------------------------------------------------- /notebooks/img/functional_gradient.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/functional_gradient.png -------------------------------------------------------------------------------- /notebooks/img/functional_gradient.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/functional_gradient.pptx -------------------------------------------------------------------------------- /notebooks/img/google.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/google.png -------------------------------------------------------------------------------- /notebooks/img/google_assistant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/google_assistant.png -------------------------------------------------------------------------------- /notebooks/img/image_captioning.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/image_captioning.jpeg -------------------------------------------------------------------------------- /notebooks/img/kmeans_convergence.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/kmeans_convergence.gif -------------------------------------------------------------------------------- /notebooks/img/l1-vs-l2-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/l1-vs-l2-annotated.png -------------------------------------------------------------------------------- /notebooks/img/l1-vs-l2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/l1-vs-l2.png -------------------------------------------------------------------------------- /notebooks/img/learning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve1.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve2.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve3.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve4.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve5.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve6.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve7.png -------------------------------------------------------------------------------- /notebooks/img/learning_curve8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/learning_curve8.png -------------------------------------------------------------------------------- /notebooks/img/life3.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/life3.0.jpg -------------------------------------------------------------------------------- /notebooks/img/loss_curve1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/loss_curve1.png -------------------------------------------------------------------------------- /notebooks/img/loss_curve2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/loss_curve2.png -------------------------------------------------------------------------------- /notebooks/img/loss_curve3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/loss_curve3.png -------------------------------------------------------------------------------- /notebooks/img/loss_curve4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/loss_curve4.png -------------------------------------------------------------------------------- /notebooks/img/margin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/margin.png -------------------------------------------------------------------------------- /notebooks/img/mogdata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdata.png -------------------------------------------------------------------------------- /notebooks/img/mogdata2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdata2.png -------------------------------------------------------------------------------- /notebooks/img/mogdata2_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdata2_v2.png -------------------------------------------------------------------------------- /notebooks/img/mogdata_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdata_v2.png -------------------------------------------------------------------------------- /notebooks/img/mogdensity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdensity.png -------------------------------------------------------------------------------- /notebooks/img/mogdensity1d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdensity1d.png -------------------------------------------------------------------------------- /notebooks/img/mogdensity1d_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdensity1d_v2.png -------------------------------------------------------------------------------- /notebooks/img/mogdensity_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdensity_v2.png -------------------------------------------------------------------------------- /notebooks/img/mogdensity_v3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/mogdensity_v3.png -------------------------------------------------------------------------------- /notebooks/img/nns/aneuron.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/aneuron.jpeg -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure0.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure1.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure2.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure3.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure4.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure5.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-abstract-figure6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-abstract-figure6.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure0.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure1.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure10.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure11.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure2.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure3.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure4.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure5.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure6.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure7.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure8.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-figure9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-figure9.png -------------------------------------------------------------------------------- /notebooks/img/nns/backprop/backprop-twolayers-legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/backprop/backprop-twolayers-legend.png -------------------------------------------------------------------------------- /notebooks/img/nns/bio-neuron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/bio-neuron.png -------------------------------------------------------------------------------- /notebooks/img/nns/bp1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/bp1.png -------------------------------------------------------------------------------- /notebooks/img/nns/layers-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/layers-1.png -------------------------------------------------------------------------------- /notebooks/img/nns/layers-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/layers-2.png -------------------------------------------------------------------------------- /notebooks/img/nns/layers-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/layers-3.png -------------------------------------------------------------------------------- /notebooks/img/nns/layers-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/layers-4.png -------------------------------------------------------------------------------- /notebooks/img/nns/layers-L.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/layers-L.png -------------------------------------------------------------------------------- /notebooks/img/nns/net-intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/net-intro.png -------------------------------------------------------------------------------- /notebooks/img/nns/nn-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/nns/nn-example.png -------------------------------------------------------------------------------- /notebooks/img/obama.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/obama.jpg -------------------------------------------------------------------------------- /notebooks/img/obama2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/obama2.jpg -------------------------------------------------------------------------------- /notebooks/img/oldfMOG_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/oldfMOG_v2.png -------------------------------------------------------------------------------- /notebooks/img/oldfSingle_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/oldfSingle_v2.png -------------------------------------------------------------------------------- /notebooks/img/oldfaithful_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/oldfaithful_v2.png -------------------------------------------------------------------------------- /notebooks/img/pca_decomposition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_decomposition.png -------------------------------------------------------------------------------- /notebooks/img/pca_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_example.png -------------------------------------------------------------------------------- /notebooks/img/pca_example_plane.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_example_plane.png -------------------------------------------------------------------------------- /notebooks/img/pca_projection1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_projection1.png -------------------------------------------------------------------------------- /notebooks/img/pca_projection2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_projection2.png -------------------------------------------------------------------------------- /notebooks/img/pca_projection_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_projection_data.png -------------------------------------------------------------------------------- /notebooks/img/pca_reconstruction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_reconstruction.png -------------------------------------------------------------------------------- /notebooks/img/pca_two_views.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/pca_two_views.png -------------------------------------------------------------------------------- /notebooks/img/rl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/rl.png -------------------------------------------------------------------------------- /notebooks/img/siri.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/siri.png -------------------------------------------------------------------------------- /notebooks/img/spam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/spam.png -------------------------------------------------------------------------------- /notebooks/img/svm-margin-fig.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/svm-margin-fig.key -------------------------------------------------------------------------------- /notebooks/img/tesla.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/tesla.jpg -------------------------------------------------------------------------------- /notebooks/img/tesla_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/tesla_data.png -------------------------------------------------------------------------------- /notebooks/img/tesla_zoom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/tesla_zoom.jpg -------------------------------------------------------------------------------- /notebooks/img/waymo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuleshov/cornell-cs5785-2020-applied-ml/30cd86f9de63ff1baf970dcdf631465a897d2087/notebooks/img/waymo.jpg -------------------------------------------------------------------------------- /notebooks/lecture12-neural-networks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "colab_type": "text", 7 | "id": "i_f5u2x9nn6I", 8 | "slideshow": { 9 | "slide_type": "slide" 10 | } 11 | }, 12 | "source": [ 13 | "\n", 14 | "\n", 15 | "# Lecture 13: Neural Networks\n", 16 | "\n", 17 | "### Applied Machine Learning\n", 18 | "\n", 19 | "__Volodymyr Kuleshov, Jin Sun__
Cornell Tech" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "slideshow": { 26 | "slide_type": "slide" 27 | } 28 | }, 29 | "source": [ 30 | "# Part 1: An Artifical Neuron\n", 31 | "\n", 32 | "In this lecture, we will learn about a new class of machine learning algorithms inspired by the brain.\n", 33 | "\n", 34 | "We will start by defining a few building blocks for these algorithms, and draw connections to neuroscience." 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "# Review: Components of A Supervised Machine Learning Problem\n", 42 | "\n", 43 | "At a high level, a supervised machine learning problem has the following structure:\n", 44 | "\n", 45 | "$$ \\underbrace{\\text{Training Dataset}}_\\text{Attributes + Features} + \\underbrace{\\text{Learning Algorithm}}_\\text{Model Class + Objective + Optimizer } \\to \\text{Predictive Model} $$\n", 46 | "\n", 47 | "Where does the dataset come from?" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": { 53 | "slideshow": { 54 | "slide_type": "slide" 55 | } 56 | }, 57 | "source": [ 58 | "# Review: Binary Classification\n", 59 | "\n", 60 | "In supervised learning, we fit a model of the form\n", 61 | "$$ f : \\mathcal{X} \\to \\mathcal{Y} $$\n", 62 | "that maps inputs $x \\in \\mathcal{X}$ to targets $y \\in \\mathcal{Y}$." 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": { 68 | "slideshow": { 69 | "slide_type": "fragment" 70 | } 71 | }, 72 | "source": [ 73 | "In classification, the space of targets $\\mathcal{Y}$ is *discrete*. Classification is binary if $\\mathcal{Y} = \\{0,1\\}$" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": { 79 | "slideshow": { 80 | "slide_type": "fragment" 81 | } 82 | }, 83 | "source": [ 84 | "Each value of $y$ value is a *class* and we are interested in finding a hyperplane that separates the different classes." 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": { 90 | "slideshow": { 91 | "slide_type": "slide" 92 | } 93 | }, 94 | "source": [ 95 | "# Review: Logistic Regression\n", 96 | "\n", 97 | "Logistic regression fits a model of the form\n", 98 | "$$ f(x) = \\sigma(\\theta^\\top x) = \\frac{1}{1 + \\exp(-\\theta^\\top x)}, $$\n", 99 | "where\n", 100 | "$$ \\sigma(z) = \\frac{1}{1 + \\exp(-z)} $$\n", 101 | "is known as the *sigmoid* or *logistic* function." 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": { 107 | "slideshow": { 108 | "slide_type": "subslide" 109 | } 110 | }, 111 | "source": [ 112 | "Here is how the logistic function looks like." 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 1, 118 | "metadata": { 119 | "slideshow": { 120 | "slide_type": "-" 121 | } 122 | }, 123 | "outputs": [ 124 | { 125 | "data": { 126 | "text/plain": [ 127 | "[]" 128 | ] 129 | }, 130 | "execution_count": 1, 131 | "metadata": {}, 132 | "output_type": "execute_result" 133 | }, 134 | { 135 | "data": { 136 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfGElEQVR4nO3deXxU5d338c+P7HuAJGwJOwJhFQLuSl1aFIVqq0Klrjc83r1t7a22dSta26ett61WK8qN1rUWikuVKkrdQJ9aEZA1rAkgISxJCNnJfj1/JNqIIANMcmb5vl8vXjNz5pD5jobv65przjmXOecQEZHg18nrACIi4h8qdBGREKFCFxEJESp0EZEQoUIXEQkRkV69cFpamuvbt69XLy8iEpRWrlxZ4pxLP9xznhV63759WbFihVcvLyISlMzssyM9pykXEZEQoUIXEQkRKnQRkRChQhcRCRFHLXQze8rMisxs/RGeNzN7xMzyzGytmY3xf0wRETkaX0bozwATv+b5C4FBrX9mAo+feCwRETlWRy1059wHQOnX7DIFeM61+BhINbMe/gooIiK+8cdx6L2AgjaPd7Vu23PojmY2k5ZRPL179/bDS4uIBAbnHHWNzVTUNlBZ20hlbSM1dY1U1TVSU9/UettIVV0T5w3JYFRWqt8zdOiJRc65ucBcgJycHF2IXUQCUlOzo6ymnpKqekqq6iipqqO0up6ymgbKauopO9jAgZoGylvvtxR4Aw1NvtVaRlJMwBZ6IZDV5nFm6zYRkYBzsL6JXQdq2FNey97yWvZW1LKnvJZ9rbfFlXWUVtfRfIRuTo6NpHNCNKlxUaTGR9OnawLJcZEkxUaRFNtymxwbSVJsJAnRkSTEtP6JjiA+JpL4qAg6dbJ2eW/+KPSFwE1mNh84BSh3zn1lukVEpKOU1zSQV1xJfnE1BaU1FJTWsLO0hoIDBymurPvK/mmJ0XRLjqVnSiyjs1JIS4yha0I0aUkxdE2IIT0pmi4JMaTERRHRTmXsD0ctdDObB0wA0sxsF3APEAXgnJsDLAIuAvKAGuC69gorItJW+cEGcneXs3lvJXlFVeQVVZFfXEVJVf0X+3Qy6JkaR1bneM4dnEFWlziyusTTMzWO7smxZCTHEBMZ4eG78J+jFrpzbtpRnnfAf/ktkYjIYZRU1bFuVzm5u8tZX1hB7p5yCkoPfvF8SlwUAzMSOXdIBgMzEhmYkUj/tER6dY4jKiI8zqH07GqLIiJH4pxje0k1K3YcYPmOUlZ8doDtJdVfPN+3azwje6UybXxvhvVMYWiPJNITYzAL3OmQjqBCF5GAUFh2kKWbi/lgSzHLd5Syv7pl2iQ1PoqcPl24clwWo7NSye6ZTHJslMdpA5MKXUQ8UdvQxCfbS1m6pZilW4rJK6oCoGdKLBMGZzCub2dy+namf1piux0VEmpU6CLSYWobmnh/UxGvr93De5uKONjQRHRkJ07p14Wp47KYMDidAemJYT91crxU6CLSrmobmvhgSzGvr93DOxv3UVPfRNeEaC4b04vzh3bjlP5diI9WFfmD/iuKSLvI3V3OC8t28vfVu6msa6RzfBRTRvfk4pE9OaVfFyLD5MiTjqRCFxG/OVjfxOtrd/PCsp2sLigjJrITk0b2YMroXpw+oGvYHD7oFRW6iJyw7SXVPPvRDl75dBcVtY0MzEjknkuyuezkTFLidURKR1Ghi8hx27ingtnv57Fo3R4iO3Vi4vDuXHVKb8b366IvNj2gQheRY7a6oIxH38vjnY37SIyJZObZA7jhzH6kJ8V4HS2sqdBFxGcrdpTy8Ltb+XBrCSlxUfz3+Sdx7el9Na0SIFToInJUBaU1/ObNjSxat5e0xGjuuHAIV53ah8QYVUgg0f8NETmiytoGHluSz58+3E5EJ+OWC05ixln9iYsOjasThhoVuoh8RVOz46WVBTyweAslVXVcNqYXP/3WELqnxHodTb6GCl1EviR3dzk/fWktubsrGNunM09ek8PodlguTfxPhS4iADQ0NTP7/TwefS+PzgnRPDLtZC4Z2UOHHwYRFbqIsHFPBbe9uIbc3RV8e3RP7p08jNT4aK9jyTFSoYuEsYamZuYsyeeR97aSEhfFnOljmTi8u9ex5Dip0EXC1PaSan40bxXrCsu5ZFRPfjF5GF0SNCoPZip0kTD0j9y93LpgDRERxmNXjeGiET28jiR+oEIXCSNNzY7f/2Mzjy3JZ2RmCo9dNYbMzvFexxI/UaGLhInS6np+NG8V/y+vhGnjs7jnkmHERukEoVCiQhcJA6sLyvjBn1dSUl3P/d8ZwZXjensdSdqBCl0kxL28chd3vLKOjOQYXr7xdEZkpngdSdqJCl0kRDnnmLN0G/e/tYnTB3Rl9vfG0FlHsYQ0FbpICGpudvzyjQ08/c8dTB7Vk99dPoroSC3/FupU6CIhpq6xiVsWrOGNtXu4/ox+3D1pKJ066fT9cKBCFwkhlbUN/J/nV/JR/n7uvGgIM87qr2uxhBEVukiIKKqs5dqnlrNlXyUPXjGKy8Zkeh1JOpgKXSQEFFXUMnXux+ytqOXJa3KYMDjD60jiARW6SJArqarje08uY29FLc9dP56cvl28jiQe8elrbzObaGabzSzPzG4/zPO9zex9M1tlZmvN7CL/RxWRQx2ormf6k8vYdaCGp64dpzIPc0ctdDOLAGYDFwLZwDQzyz5kt7uBBc65k4GpwGP+DioiX1Ze08D0Py1jW0k1T149jlP7d/U6knjMlxH6eCDPObfNOVcPzAemHLKPA5Jb76cAu/0XUUQOVVnbwNVPf8LWfVXM/f5YzhyU5nUkCQC+FHovoKDN412t29q6F5huZruARcAPD/eDzGymma0wsxXFxcXHEVdEqusaufbp5eQWlvPYVWP0Bah8wV+njk0DnnHOZQIXAc+b2Vd+tnNurnMuxzmXk56e7qeXFgkfdY1N/MezK1hdUMYfp53M+dndvI4kAcSXQi8Esto8zmzd1tYNwAIA59y/gFhAnwFF/Mg5x89eWsu/tu3n95eP4kItSiGH8KXQlwODzKyfmUXT8qXnwkP22QmcB2BmQ2kpdM2piPjRQ29v4dXVu/nJtwbz7ZMPnfUU8aHQnXONwE3AYmAjLUez5JrZfWY2uXW3W4EZZrYGmAdc65xz7RVaJNwsWFHAI+/lcWVOFj+YMMDrOBKgfDqxyDm3iJYvO9tum9Xm/gbgDP9GExGAf+aVcOcr6zhrUBq/unS4rs0iR6TraYoEsC37Krnx+ZUMSE9k9lVjiIrQP1k5Mv12iASoosparnt6ObHRETx13TiSY6O8jiQBToUuEoAO1rccnlhaXc9T14yjV2qc15EkCOjiXCIBxjnHXa+uY11hOXO/n6M1QMVnGqGLBJgXlu3klU8Lufm8QVygE4fkGKjQRQLIqp0H+MXfc5kwOJ0fnTvI6zgSZFToIgGipKqOH7zwKd2SY/nDlaO1DqgcM82hiwSAxqZmfjRvFaXV9bz8n6eTGh/tdSQJQip0kQDw+7e38FH+fh747kiG99KXoHJ8NOUi4rG31u/l8SX5TBvfm8tzso7+F0SOQIUu4qHtJdXc9uIaRmWmcO/kQxcCEzk2KnQRj9Q3NnPz/FVEdDIemz6WmMgIryNJkNMcuohHHnpnC2t3lTNn+hidCSp+oRG6iAc+yi9hztJ8po7LYuJwLVQh/qFCF+lgZTX13PLXNfTrmsCsSzRvLv6jQhfpQM457nhlHfur63h46snER2vWU/xHhS7SgRasKODN9Xu59ZuDddEt8TsVukgH2VZcxb0LN3D6gK7MPKu/13EkBKnQRTpAyyGKq4mJ6sSDV+g6LdI+NIEn0gEefncL6wrLmTN9LN1TYr2OIyFKI3SRdramoIzHl+Rz+dhMJg7v7nUcCWEqdJF2VNfYxG0vriEjKZa7L9YhitK+NOUi0o4efmcrW4uqePq6caTEaZFnaV8aoYu0kzUFZcxZms8VOZl8Y3CG13EkDKjQRdpB26mWuyZpqkU6hqZcRNqBplrECxqhi/iZplrEKyp0ET/SVIt4SVMuIn70yLuaahHvaIQu4icbdlcwZ+k2vjtWUy3iDRW6iB80NTvueGUtqXFR3D1pqNdxJEz5VOhmNtHMNptZnpndfoR9rjCzDWaWa2Z/8W9MkcD23L92sGZXObMuySY1PtrrOBKmjjqHbmYRwGzgAmAXsNzMFjrnNrTZZxBwB3CGc+6AmenzpoSNwrKDPLB4M+eclM7kUT29jiNhzJcR+nggzzm3zTlXD8wHphyyzwxgtnPuAIBzrsi/MUUCk3OOWa+uxzn41beHY6bL4op3fCn0XkBBm8e7Wre1dRJwkpn908w+NrOJh/tBZjbTzFaY2Yri4uLjSywSQBat28u7m4q49ZsnkdUl3us4Eub89aVoJDAImABMA54ws9RDd3LOzXXO5TjnctLT0/300iLeKK9p4J6FuQzvlcy1p/f1Oo6IT4VeCGS1eZzZuq2tXcBC51yDc247sIWWghcJWb99axMHaur57WUjiYzQAWPiPV9+C5cDg8ysn5lFA1OBhYfs8yoto3PMLI2WKZht/ospElg+2V7KvE92csOZ/RjeS4s9S2A4aqE75xqBm4DFwEZggXMu18zuM7PJrbstBvab2QbgfeAnzrn97RVaxEt1jU3c8cpaMjvH8ePz9UFUAodPp/475xYBiw7ZNqvNfQfc0vpHJKQ98cE28ourefq6ccRH6+oZEjg08SdyDHbur+GP7+Vx0YjuOr1fAo4KXcRHzjlmLVxPZCdj1sXDvI4j8hUqdBEfvbV+L0s2F3PLNwfTPSXW6zgiX6FCF/FBVV0jv/j7BrJ7JHPNaX28jiNyWPpGR8QHD729hX2VtTw+fYyOOZeApd9MkaPYsLuCZz7awbTxvTm5d2ev44gckQpd5Gs0NzvuenUdqXFR/OxbQ7yOI/K1VOgiX2P+8gJW7SzjrklDSYnXknIS2FToIkewv6qO+9/axKn9u3DpyYdeYFQk8KjQRY7gN29uoqa+Udc5l6ChQhc5jOU7Snlp5S7+46z+DMxI8jqOiE9U6CKHaGhq5u6/radXahw/PHeg13FEfKZCFznEsx/tYPO+Su65JFsX35KgokIXaWNveS0Pvb2Fc4dkcEF2N6/jiBwTFbpIG798YwONzY57LxmmL0Il6KjQRVp9uLWYN9bu4aZvDKR3Vy34LMFHhS5CyypEs17LpV9aAjPP6e91HJHjom98RIC5S7exvaSa564fT0xkhNdxRI6LRugS9gpKa3j0/TwmjezB2Selex1H5Lip0CWsOeeY9VrLKkQ/n5TtdRyRE6JCl7C2OHcv728u5r8vOEmrEEnQU6FL2Pp8FaKhPZK59vS+XscROWEqdAlbf3h7C3sravm/lw7XKkQSEvRbLGFp454Knv5oB1PH9WaMViGSEKFCl7DT3Oy462+tqxBNHOx1HBG/UaFL2PnrigI+3VnGnRcNJTU+2us4In6jQpewsr+qjt++uYlT+nXhsjFahUhCiwpdwsqvF22iuk6rEEloUqFL2Fi2bT8vf7qLmWf3Z1A3rUIkoUeFLmGhrrGJO/+2jszOcfzw3EFexxFpF7o4l4SFx5fkk19czTPXjSMuWhffktDk0wjdzCaa2WYzyzOz279mv++YmTOzHP9FFDkxeUWVPPZ+PlNG92TC4Ayv44i0m6MWuplFALOBC4FsYJqZfeUqRmaWBNwMLPN3SJHj1dzsuOOVdcRFR/Dzi3XxLQltvozQxwN5zrltzrl6YD4w5TD7/RK4H6j1Yz6REzJ/eQHLdxzgrklDSUuM8TqOSLvypdB7AQVtHu9q3fYFMxsDZDnn3vi6H2RmM81shZmtKC4uPuawIseiqKKW37y5kdP6d+XysZlexxFpdyd8lIuZdQIeBG492r7OubnOuRznXE56uhYSkPZ1799zqWts5teXjdAx5xIWfCn0QiCrzePM1m2fSwKGA0vMbAdwKrBQX4yKl97esI9F6/Zy83mD6JeW4HUckQ7hS6EvBwaZWT8ziwamAgs/f9I5V+6cS3PO9XXO9QU+BiY751a0S2KRo6iqa2TWa+sZ3C2JGWdpwWcJH0ctdOdcI3ATsBjYCCxwzuWa2X1mNrm9A4ocq98t3szeilp+850RREfq3DkJHz6dWOScWwQsOmTbrCPsO+HEY4kcn0+2l/Lsv3Zw9al9dJ1zCTsavkjIOFjfxE9eWkNm5zh+OnGI13FEOpxO/ZeQ8T+LN/HZ/hrmzTiVhBj9akv40QhdQsIn20t55qMdXHNaH04b0NXrOCKeUKFL0NNUi0gLfS6VoKepFpEWGqFLUNNUi8i/qdAlaGmqReTL9PlUgpamWkS+TCN0CUr/zCvh6X9qqkWkLRW6BJ0D1fXcsmA1AzMSuf3CoV7HEQkYKnQJKs45fvbyWkqr63l46mitDyrShgpdgspfPtnJPzbs46ffGsKwnilexxEJKCp0CRp5RZX88vUNnDUojRvO7Od1HJGAo0KXoFDX2MSP5q0mLiqC310+ik6dtAKRyKF0rJcEhd8t3syGPRU8cXUO3ZJjvY4jEpA0QpeA9+HWYp74cDtXndKbC7K7eR1HJGCp0CWg7a+q49YFaxiYkcjdk7K9jiMS0FToErAam5r54bxVlB1s0CGKIj5QoUvA+v3bW/gofz+/+vZwHaIo4gMVugSkxbl7eXxJPtPG9+aKnCyv44gEBRW6BJxtxVXctmANIzNTuOcSzZuL+EqFLgGlpr6RG/+8ksgI47GrxhAbpXlzEV/pOHQJGM45bn95HVuLqnju+vFkdo73OpJIUNEIXQLGsx/tYOGa3dz2zcGcNSjd6zgiQUeFLgHhk+2l/OqNjZw/tBv/ec4Ar+OIBCUVunhue0k1M59fQe8u8fz+Cl2nReR4qdDFU6XV9Vz39Cd0MuPp68aREhfldSSRoKUvRcUztQ1NzHhuBbvLa5k341T6dE3wOpJIUNMIXTzR3Oy47cU1rPzsAA9dMZqxfTp7HUkk6KnQxRMP/GMzr6/dwx0XDmHSyB5exxEJCSp06XDzPtnJ40vy+d4pvZl5dn+v44iEDJ8K3cwmmtlmM8szs9sP8/wtZrbBzNaa2btm1sf/USUULNlcxN2vrueck9K5b/IwzHREi4i/HLXQzSwCmA1cCGQD08zs0AtsrAJynHMjgZeA//F3UAl+H2/bz41/XsngbknMvmoMkRH6gCjiT778ixoP5Dnntjnn6oH5wJS2Ozjn3nfO1bQ+/BjI9G9MCXYrdpRy/TPLyeocz/M3jCcxRgdYifibL4XeCyho83hX67YjuQF483BPmNlMM1thZiuKi4t9TylBbXVBGdc+vZzuybG8MOMUuibGeB1JJCT59TOvmU0HcoAHDve8c26ucy7HOZeTnq5rdYSD9YXlfP9Py+iSEM1fZpxKRpIWeBZpL7587i0E2q4wkNm67UvM7HzgLuAc51ydf+JJMNu4p4Lpf1pGcmwUf5lxCt1TVOYi7cmXEfpyYJCZ9TOzaGAqsLDtDmZ2MvC/wGTnXJH/Y0qw2bqvkqueXEZsZATzZpyqS+GKdICjFrpzrhG4CVgMbAQWOOdyzew+M5vcutsDQCLwopmtNrOFR/hxEgbW7Spn2hMfE9HJ+MuMU+jdVWUu0hF8OtTAObcIWHTItllt7p/v51wSpD7YUsyNf15J5/honrthPP3TE72OJBI2dOyY+M2rqwq57cU1DMxI5Nnrx9MtWXPmIh1JhS5+8eSH2/jVGxs5tX8X5l6dQ3KsLoMr0tFU6HJCmpsdv3lzI098uJ2LRnTnwStGa2FnEY+o0OW41TY08bOX1/La6t1cc1ofZl0yjAitNiTiGRW6HJeC0hp+8MKnrCss5yffGswPJgzQhbZEPKZCl2O2dEsxN89fRVOz48mrczg/u5vXkUQEFbocg+Zmx+z383jwnS0M7pbEnOlj6ZumZeNEAoUKXXxSfrCBW/66mnc3FXHpyb349aUjiIvWl58igUSFLke1aucBfvzX1ewuO8h9U4bx/VP7aL5cJACp0OWIahua+MM7W5n7QT7dk2OZP/M0LeYsEsBU6HJYawrKuO3FNWwtqmLquCzumjSUJJ0sJBLQVOjyJXWNTTzy7lbmLN1GemIMz14/nnNO0rXrRYKBCl2+sPKzA9z5yjo276vk8rGZ3H1xNilxGpWLBAsVurC77CD3v7WJ11bvpltyDE9fO45vDMnwOpaIHCMVehirqW9kztJtzP0gH+fgh+cO5MZzBpCgBZxFgpL+5Yah5mbHq6sLuf+tTeyrqOOSUT352cTBWlVIJMip0MNIc7Pjrdy9PPpeHhv2VDAqM4XZ3xtDTt8uXkcTET9QoYeBxqZmFq7ZzWNL8skrqqJfWgIPXTmKKaN60UlXRxQJGSr0EFbX2MTLKwuZszSfnaU1DOmexB+nncxFI3roMrciIUiFHoIKyw4y/5OdzF9eQHFlHaOyUvn5xdmcNyRDI3KREKZCDxFNzY4lm4t4YdlOlmwuwgHfGJzBdWf05cyBabr2ikgYUKEHuR0l1Sxcs5v5n+xkd3kt6Ukx/Nc3BnLluCwdtSISZlToQaigtIY31u3h9bW7WV9YAcCZA9P4+cXZnJ/djaiITh4nFBEvqNCDgHOOHftreHfjPv6+dg9rCsoAGJWVyt2ThnLhiB70So3zNqSIeE6FHqCq6xr5V/5+lm4pZumWYnaW1gAwolcKt184hEkjepDVRVMqIvJvKvQAcbC+idUFZaz8rJSP8vezfEcpDU2O+OgITh+Qxoyz+zPhpHSVuIgckQrdI0WVtXz6WUuBL99xgPWF5TQ2OwCGdE/i+jP6cc5J6Yzt25mYSC31JiJHp0JvZ845dh04yPrCcnJ3V7B+d8ttcWUdANGRnRidmcqMs/szrm9nxvTuTGp8tMepRSQYqdD9pLGpmZ2lNeQXV5NXVEVeURX5xVXkF1VRWdcIQEQnY1BGImcNSmN4zxRGZqYwIjNFI3AR8QsVuo+amx2lNfXsKaul4EANO0trKCj9921h2UEamtwX+2ckxTAwI5FLx/RiSPdkhvVMZnD3JGKjVN4i0j7CvtAbm5opra6npKqekqo69lfXUVJZT1FlLXsr6thbfpA95bUUVdRR39T8pb/bOT6K3l3iGdYrhYnDezAgPYGBGYkMyEgkWetvikgH86nQzWwi8DAQATzpnPvtIc/HAM8BY4H9wJXOuR3+jXp4zjlqG5qpqmukuq6RytpGKmsbqGi9bXncSNnBespqGiirqedATQPlBxs4UNOy7XBiIjvRIyWWbsmx5PTpTPeUOLonx9A9JY7eXeLJ6hKnRZNFJKActdDNLAKYDVwA7AKWm9lC59yGNrvdABxwzg00s6nA/cCV7RH4r8t38r9Lt1Fd30h1XRPV9Y04d/S/lxQTSUp8FJ3jo0mNjyKrSzypcVF0SYgmLTGatMQY0pJi6JoQTVpSDEkxkbr+iYgEFV9G6OOBPOfcNgAzmw9MAdoW+hTg3tb7LwGPmpk550vVHpsuCTFk90wmITqShJhIEmIiWm6jI4iPjiQpNpLkuKiW29iW28SYSCJ1OryIhDhfCr0XUNDm8S7glCPt45xrNLNyoCtQ0nYnM5sJzATo3bv3cQW+ILsbF2R3O66/KyISyjp02Oqcm+ucy3HO5aSnp3fkS4uIhDxfCr0QyGrzOLN122H3MbNIIIWWL0dFRKSD+FLoy4FBZtbPzKKBqcDCQ/ZZCFzTev+7wHvtMX8uIiJHdtQ59NY58ZuAxbQctviUcy7XzO4DVjjnFgJ/Ap43szyglJbSFxGRDuTTcejOuUXAokO2zWpzvxa43L/RRETkWOhYPhGREKFCFxEJESp0EZEQYV4djGJmxcBnnrz4iUnjkBOmwkC4vedwe7+g9xxM+jjnDnsij2eFHqzMbIVzLsfrHB0p3N5zuL1f0HsOFZpyEREJESp0EZEQoUI/dnO9DuCBcHvP4fZ+Qe85JGgOXUQkRGiELiISIlToIiIhQoV+AszsVjNzZpbmdZb2ZGYPmNkmM1trZn8zs1SvM7UXM5toZpvNLM/Mbvc6T3szsywze9/MNphZrpnd7HWmjmJmEWa2ysxe9zqLv6jQj5OZZQHfBHZ6naUDvA0Md86NBLYAd3icp120WT/3QiAbmGZm2d6maneNwK3OuWzgVOC/wuA9f+5mYKPXIfxJhX78HgJ+CoT8t8rOuX845xpbH35MyyInoeiL9XOdc/XA5+vnhizn3B7n3Ket9ytpKbhe3qZqf2aWCUwCnvQ6iz+p0I+DmU0BCp1za7zO4oHrgTe9DtFODrd+bsiX2+fMrC9wMrDM4ygd4Q+0DMiaPc7hVz5dDz0cmdk7QPfDPHUXcCct0y0h4+ver3PutdZ97qLlI/oLHZlN2p+ZJQIvAz92zlV4nac9mdnFQJFzbqWZTfA4jl+p0I/AOXf+4bab2QigH7DGzKBl+uFTMxvvnNvbgRH96kjv93Nmdi1wMXBeCC8v6Mv6uSHHzKJoKfMXnHOveJ2nA5wBTDazi4BYINnM/uycm+5xrhOmE4tOkJntAHKcc8F41TafmNlE4EHgHOdcsdd52kvrAudbgPNoKfLlwPecc7meBmtH1jIqeRYodc792OM4Ha51hH6bc+5ij6P4hebQxRePAknA22a22szmeB2oPbR+8fv5+rkbgQWhXOatzgC+D5zb+v92devIVYKQRugiIiFCI3QRkRChQhcRCREqdBGREKFCFxEJESp0EZEQoUIXEQkRKnQRkRDx/wEB0fssBlwRowAAAABJRU5ErkJggg==\n", 137 | "text/plain": [ 138 | "
" 139 | ] 140 | }, 141 | "metadata": { 142 | "needs_background": "light" 143 | }, 144 | "output_type": "display_data" 145 | } 146 | ], 147 | "source": [ 148 | "import numpy as np\n", 149 | "from matplotlib import pyplot as plt\n", 150 | "\n", 151 | "z = np.linspace(-5, 5)\n", 152 | "sigma = 1/(1+np.exp(-z))\n", 153 | "\n", 154 | "plt.plot(z, sigma)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "# A Biological Neuron\n", 162 | "\n", 163 | "In order to define an artifical neuron, let's look first a biological one.\n", 164 | "\n", 165 | "TODO: PUT NEURON IMAGE FROM HERE: https://cs231n.github.io/neural-networks-1/\n", 166 | "\n", 167 | "* Each neuron receives input signals from its dendrites\n", 168 | "* It produces output signals along its axon, which connects to the dendrites of other neurons." 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "# An Artificial Neuron: Example\n", 176 | "\n", 177 | "We can imitate this machinery using an idealized artifical neuron.\n", 178 | "* The neuron receives signals $x_j$ at dendrites, which are modulated multiplicatively: $w_j \\cdot x_j$.\n", 179 | "* The body of the neuron sums the modulated inputs: $\\sum_{j=1}^d w_j \\cdot x_j$.\n", 180 | "* These go into the activation function that produces an ouput.\n", 181 | "\n", 182 | "TODO: PUT ARTIFICIAL NEURON IMAGE FROM HERE: https://cs231n.github.io/neural-networks-1/" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "# An Artificial Neuron: Notation\n", 190 | "\n", 191 | "More formally, we say that a neuron is a model $f : \\mathbb{R}^d \\to [0,1]$, with the following components:\n", 192 | "* Inputs $x_1,x_2,...,x_d$, denoted by a vector $x$.\n", 193 | "* Weight vector $w \\in \\mathbb{R}^d$ that modulates input $x$ as $w^\\top x$.\n", 194 | "* An activation function $\\sigma: \\mathbb{R} \\to \\mathbb{R}$ that computes the output $\\sigma(w^\\top x)$ of the neuron based on the sum of modulated features $w^\\top x$." 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "# Logistic Regression as an Artifical Neuron\n", 202 | "\n", 203 | "Logistic regression is a model of the form\n", 204 | "$$ f(x) = \\sigma(\\theta^\\top x) = \\frac{1}{1 + \\exp(-\\theta^\\top x)}, $$\n", 205 | "that can be interpreted as a neuron that uses the *sigmoid* as the activation function." 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "# Perceptron\n", 213 | "\n", 214 | "Another model of a neuron." 215 | ] 216 | }, 217 | { 218 | "cell_type": "markdown", 219 | "metadata": {}, 220 | "source": [ 221 | "# Example\n", 222 | "\n", 223 | "Need to implement a small example. Can probably copy-paste implementation of LR from the LR slides." 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "# Activation Functions\n", 231 | "\n", 232 | "Let's list a few." 233 | ] 234 | }, 235 | { 236 | "cell_type": "markdown", 237 | "metadata": {}, 238 | "source": [ 239 | "\n", 240 | "# Part 2: Artificial Neural Networks\n", 241 | "\n", 242 | "Let's now see how we can connect neurons into networks that form complex models that further mimic the brain." 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "# Review: Artificial Neuron\n", 250 | "\n", 251 | "We say that a neuron is a model $f : \\mathbb{R}^d \\to [0,1]$, with the following components:\n", 252 | "* Inputs $x_1,x_2,...,x_d$, denoted by a vector $x$.\n", 253 | "* Weight vector $w \\in \\mathbb{R}^d$ that modulates input $x$ as $w^\\top x$.\n", 254 | "* An activation function $\\sigma: \\mathbb{R} \\to \\mathbb{R}$ that computes the output $\\sigma(w^\\top x)$ of the neuron based on the sum of modulated features $w^\\top x$." 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "# Review: Logistic Regression as Neuron\n", 262 | "\n", 263 | "Logistic regression is a model of the form\n", 264 | "$$ f(x) = \\sigma(\\theta^\\top x) = \\frac{1}{1 + \\exp(-\\theta^\\top x)}, $$\n", 265 | "that can be interpreted as a neuron that uses the *sigmoid* as the activation function." 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "# Neural Networks: Intuition\n", 273 | "\n", 274 | "A neural network is a directed graph in which a node is a neuron that takes as input the outputs of the neurons that are connected to it.\n", 275 | "\n", 276 | "TODO: Add an image here. Maybe layer image from here: https://cs231n.github.io/neural-networks-1/ (It probably needs some annotations)\n", 277 | "\n", 278 | "Networks are typically organized in layers." 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": { 284 | "slideshow": { 285 | "slide_type": "slide" 286 | } 287 | }, 288 | "source": [ 289 | "# Neural Networks: Layers\n", 290 | "\n", 291 | "A neural network layer is a model $f : \\mathbb{R}^d \\to \\mathbb{R}^p$ that applies $p$ neurons ih parallel to an input $x$.\n", 292 | "$$ f(x) = \\begin{bmatrix}\n", 293 | "\\sigma(w_1^\\top x) \\\\\n", 294 | "\\sigma(w_2^\\top x) \\\\\n", 295 | "\\vdots \\\\\n", 296 | "\\sigma(w_p^\\top x)\n", 297 | "\\end{bmatrix}.\n", 298 | "$$\n", 299 | "where each $w_k$ is the vector of weights for the $k$-th neuron. We refer to $p$ as the *size* of the layer." 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": { 305 | "slideshow": { 306 | "slide_type": "subslide" 307 | } 308 | }, 309 | "source": [ 310 | "By combining the $w_k$ into one matrix $W$, we can write in a more succinct vectorized form:\n", 311 | "$$f(x) = \\sigma(W\\cdot x) = \\begin{bmatrix}\n", 312 | "\\sigma(w_1^\\top x) \\\\\n", 313 | "\\sigma(w_2^\\top x) \\\\\n", 314 | "\\vdots \\\\\n", 315 | "\\sigma(w_p^\\top x)\n", 316 | "\\end{bmatrix},\n", 317 | "$$\n", 318 | "where $\\sigma(W\\cdot x)_k = \\sigma(w_k^\\top x)$ and $W_{kj} = (w_k)_j$. " 319 | ] 320 | }, 321 | { 322 | "cell_type": "markdown", 323 | "metadata": {}, 324 | "source": [ 325 | "# Neural Networks: Notation\n", 326 | "\n", 327 | "A neural network is a model $f : \\mathbb{R} \\to \\mathbb{R}$ that consists of a composition of $L$ neural network layers:\n", 328 | "$$ f(x) = f_L \\circ f_{L-1} \\circ \\ldots f_1 (x). $$\n", 329 | "The final layer $f_L$ has size one (assuming the neural net has one ouput); intermediary layers $f_l$ can have any number of neurons.\n", 330 | "\n", 331 | "The notation $f \\circ g(x)$ denotes the composition $f(g(x))$ of functions" 332 | ] 333 | }, 334 | { 335 | "cell_type": "markdown", 336 | "metadata": {}, 337 | "source": [ 338 | "# Example of a Neural Network\n", 339 | "\n", 340 | "* Let's implement a small neural net in the same that we implemented logistic regression\n", 341 | "* Then we just run it" 342 | ] 343 | }, 344 | { 345 | "cell_type": "markdown", 346 | "metadata": {}, 347 | "source": [ 348 | "# Types of Neural Network Layers\n", 349 | "\n", 350 | "There are many types of neural network layers that can exist. Here are a few:\n", 351 | "* Ouput layer: normally has one neuron and special activation function that depends on the problem\n", 352 | "* Input layer: normally, this is just the input vector $x$.\n", 353 | "* Hidden layer: Any layer between input and output.\n", 354 | "* Dense layer: A layer in which every input is connected ot every neuron.\n", 355 | "* Convolutional layer: A layer in which the operation $w^\\top x$ implements a mathematical [convolution](https://en.wikipedia.org/wiki/Convolution).\n", 356 | "* Anything else?" 357 | ] 358 | }, 359 | { 360 | "cell_type": "markdown", 361 | "metadata": {}, 362 | "source": [ 363 | "# Neuroscience Angle\n", 364 | "\n", 365 | "Annything we should say here?" 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": {}, 371 | "source": [ 372 | "\n", 373 | "# Part 3: Backpropagation\n", 374 | "\n", 375 | "We have defined what is an artificial neural network.\n", 376 | "\n", 377 | "Let's not see how we can train it." 378 | ] 379 | }, 380 | { 381 | "cell_type": "markdown", 382 | "metadata": {}, 383 | "source": [ 384 | "# Review: Neural Network Layers\n", 385 | "\n", 386 | "A neural network layer is a model $f : \\mathbb{R}^d \\to \\mathbb{R}^p$ that applies $p$ neurons ih parallel to an input $x$.\n", 387 | "$$f(x) = \\sigma(W\\cdot x) = \\begin{bmatrix}\n", 388 | "\\sigma(w_1^\\top x) \\\\\n", 389 | "\\sigma(w_2^\\top x) \\\\\n", 390 | "\\vdots \\\\\n", 391 | "\\sigma(w_p^\\top x)\n", 392 | "\\end{bmatrix},\n", 393 | "$$\n", 394 | "where each $w_k$ is the vector of weights for the $k$-th neuron and $W_{kj} = (w_k)_j$. We refer to $p$ as the *size* of the layer." 395 | ] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "metadata": {}, 400 | "source": [ 401 | "# Review: Neural Networks\n", 402 | "\n", 403 | "A neural network is a model $f : \\mathbb{R} \\to \\mathbb{R}$ that consists of a composition of $L$ neural network layers:\n", 404 | "$$ f(x) = f_L \\circ f_{L-1} \\circ \\ldots f_1 (x). $$\n", 405 | "The final layer $f_L$ has size one (assuming the neural net has one ouput); intermediary layers $f_l$ can have any number of neurons.\n", 406 | "\n", 407 | "The notation $f \\circ g(x)$ denotes the composition $f(g(x))$ of functions\n", 408 | "\n", 409 | "TODO: Add some kind of image from the previous part of the lecture" 410 | ] 411 | }, 412 | { 413 | "cell_type": "markdown", 414 | "metadata": {}, 415 | "source": [ 416 | "# Review: The Gradient\n", 417 | "\n", 418 | "The gradient $\\nabla_\\theta f$ further extends the derivative to multivariate functions $f : \\mathbb{R}^d \\to \\mathbb{R}$, and is defined at a point $\\theta_0$ as\n", 419 | "\n", 420 | "$$ \\nabla_\\theta f (\\theta_0) = \\begin{bmatrix}\n", 421 | "\\frac{\\partial f(\\theta_0)}{\\partial \\theta_1} \\\\\n", 422 | "\\frac{\\partial f(\\theta_0)}{\\partial \\theta_2} \\\\\n", 423 | "\\vdots \\\\\n", 424 | "\\frac{\\partial f(\\theta_0)}{\\partial \\theta_d}\n", 425 | "\\end{bmatrix}.$$\n", 426 | "\n", 427 | "In other words, the $j$-th entry of the vector $\\nabla_\\theta f (\\theta_0)$ is the partial derivative $\\frac{\\partial f(\\theta_0)}{\\partial \\theta_j}$ of $f$ with respect to the $j$-th component of $\\theta$." 428 | ] 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": {}, 433 | "source": [ 434 | "# Review: Gradient Descent\n", 435 | "\n", 436 | "If we want to optimize an objective $J(\\theta)$, we start with an initial guess $\\theta_0$ for the parameters and repeat the following update until the function is no longer decreasing:\n", 437 | "$$ \\theta_i := \\theta_{i-1} - \\alpha \\cdot \\nabla_\\theta J(\\theta_{i-1}). $$\n", 438 | "\n", 439 | "As code, this method may look as follows:\n", 440 | "```python\n", 441 | "theta, theta_prev = random_initialization()\n", 442 | "while abs(J(theta) - J(theta_prev)) > conv_threshold:\n", 443 | " theta_prev = theta\n", 444 | " theta = theta_prev - step_size * gradient(theta_prev)\n", 445 | "```" 446 | ] 447 | }, 448 | { 449 | "cell_type": "markdown", 450 | "metadata": {}, 451 | "source": [ 452 | "# Backpropagation\n", 453 | "\n", 454 | "How do we apply gradient descent to a neural network?\n", 455 | "\n", 456 | "Explain backpropgation" 457 | ] 458 | }, 459 | { 460 | "cell_type": "markdown", 461 | "metadata": {}, 462 | "source": [ 463 | "# Review: Chain Rule of Calculus\n", 464 | "\n", 465 | "Probably will need to review this at some point." 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": {}, 471 | "source": [ 472 | "# Example\n", 473 | "\n", 474 | "Let's implement backprop with the simple NN model we had earlier." 475 | ] 476 | }, 477 | { 478 | "cell_type": "markdown", 479 | "metadata": {}, 480 | "source": [ 481 | "\n", 482 | "# Part 4: Stochastic Gradient Descent\n", 483 | "\n", 484 | "In practice, neural networks are often trained on very large datasets.\n", 485 | "\n", 486 | "This requires a mosification to the gradient descent algorithm that we have seen earlier." 487 | ] 488 | }, 489 | { 490 | "cell_type": "markdown", 491 | "metadata": {}, 492 | "source": [ 493 | "Volodymyr will create this section" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": null, 499 | "metadata": {}, 500 | "outputs": [], 501 | "source": [] 502 | } 503 | ], 504 | "metadata": { 505 | "accelerator": "GPU", 506 | "celltoolbar": "Slideshow", 507 | "colab": { 508 | "collapsed_sections": [], 509 | "name": "neural-ode.ipynb", 510 | "provenance": [] 511 | }, 512 | "kernelspec": { 513 | "display_name": "aml", 514 | "language": "python", 515 | "name": "aml" 516 | }, 517 | "language_info": { 518 | "codemirror_mode": { 519 | "name": "ipython", 520 | "version": 3 521 | }, 522 | "file_extension": ".py", 523 | "mimetype": "text/x-python", 524 | "name": "python", 525 | "nbconvert_exporter": "python", 526 | "pygments_lexer": "ipython3", 527 | "version": "3.6.7" 528 | }, 529 | "rise": { 530 | "controlsTutorial": false, 531 | "height": 900, 532 | "help": false, 533 | "margin": 0, 534 | "maxScale": 2, 535 | "minScale": 0.2, 536 | "progress": true, 537 | "scroll": true, 538 | "theme": "simple", 539 | "width": 1200 540 | } 541 | }, 542 | "nbformat": 4, 543 | "nbformat_minor": 1 544 | } 545 | -------------------------------------------------------------------------------- /notebooks/lecture23-overview.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "colab_type": "text", 7 | "id": "i_f5u2x9nn6I", 8 | "slideshow": { 9 | "slide_type": "slide" 10 | } 11 | }, 12 | "source": [ 13 | "\n", 14 | "\n", 15 | "# Lecture 23: Course Overview\n", 16 | "\n", 17 | "### Applied Machine Learning\n", 18 | "\n", 19 | "__Volodymyr Kuleshov__
Cornell Tech" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "slideshow": { 26 | "slide_type": "slide" 27 | } 28 | }, 29 | "source": [ 30 | "# Congratulations on Finishing Applied Machine Learning!\n", 31 | "\n", 32 | "You have made it! This is our last machine learning lecture, in which we will do an overview of the diffrent algorithms seen in the course." 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": { 38 | "slideshow": { 39 | "slide_type": "slide" 40 | } 41 | }, 42 | "source": [ 43 | "# A Map of Applied Machine Learning\n", 44 | "\n", 45 | "We will go through the following map of algorithms from the course.\n", 46 | "
" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "slideshow": { 53 | "slide_type": "slide" 54 | } 55 | }, 56 | "source": [ 57 | "# Supervised Machine Learning\n", 58 | "\n", 59 | "At a high level, a supervised machine learning problem has the following structure:\n", 60 | "\n", 61 | "$$ \\underbrace{\\text{Dataset}}_\\text{Features, Attributes} + \\underbrace{\\text{Learning Algorithm}}_\\text{Model Class + Objective + Optimizer} \\to \\text{Predictive Model} $$\n", 62 | "\n", 63 | "The predictive model is chosen to model the relationship between inputs and targets. For instance, it can predict future targets." 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": { 69 | "slideshow": { 70 | "slide_type": "slide" 71 | } 72 | }, 73 | "source": [ 74 | "# Linear Regression\n", 75 | "\n", 76 | "In linear regression, we fit a model\n", 77 | "$$ f_\\theta(x) := \\theta^\\top \\phi(x) $$\n", 78 | "that is linear in $\\theta$. \n", 79 | "\n", 80 | "The features $\\phi(x) : \\mathbb{R} \\to \\mathbb{R}^p$ are non-linear may non-linear in $x$ (e.g., polynomial features), allowing us to fit complex functions." 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": { 86 | "slideshow": { 87 | "slide_type": "subslide" 88 | } 89 | }, 90 | "source": [ 91 | "
" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": { 97 | "slideshow": { 98 | "slide_type": "slide" 99 | } 100 | }, 101 | "source": [ 102 | "# Overfitting\n", 103 | "\n", 104 | "Overfitting is one of the most common failure modes of machine learning.\n", 105 | "* A very expressive model (a high degree polynomial) fits the training dataset perfectly.\n", 106 | "* The model also makes wildly incorrect prediction outside this dataset, and doesn't generalize." 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": { 112 | "slideshow": { 113 | "slide_type": "slide" 114 | } 115 | }, 116 | "source": [ 117 | "# Regularization\n", 118 | "\n", 119 | "The idea of regularization is to penalize complex models that may overfit the data." 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": { 125 | "slideshow": { 126 | "slide_type": "fragment" 127 | } 128 | }, 129 | "source": [ 130 | "Regularized least squares optimizes the following objective (__Ridge__).\n", 131 | "$$ J(\\theta) = \\frac{1}{2n} \\sum_{i=1}^n \\left( y^{(i)} - \\theta^\\top \\phi(x^{(i)}) \\right)^2 + \\frac{\\lambda}{2} \\cdot ||\\theta||_2^2. $$\n", 132 | "If we use the L1 norm, we have the __LASSO__." 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": { 138 | "slideshow": { 139 | "slide_type": "subslide" 140 | } 141 | }, 142 | "source": [ 143 | "
" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": { 149 | "slideshow": { 150 | "slide_type": "slide" 151 | } 152 | }, 153 | "source": [ 154 | "# Regression vs. Classification\n", 155 | "\n", 156 | "Consider a training dataset $\\mathcal{D} = \\{(x^{(1)}, y^{(1)}), (x^{(2)}, y^{(2)}), \\ldots, (x^{(n)}, y^{(n)})\\}$.\n", 157 | "\n", 158 | "We distinguish between two types of supervised learning problems depnding on the targets $y^{(i)}$. \n", 159 | "\n", 160 | "1. __Regression__: The target variable $y \\in \\mathcal{Y}$ is continuous: $\\mathcal{Y} \\subseteq \\mathbb{R}$.\n", 161 | "2. __Classification__: The target variable $y$ is discrete and takes on one of $K$ possible values: $\\mathcal{Y} = \\{y_1, y_2, \\ldots y_K\\}$. Each discrete value corresponds to a *class* that we want to predict." 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": { 167 | "slideshow": { 168 | "slide_type": "subslide" 169 | } 170 | }, 171 | "source": [ 172 | "
" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": { 178 | "slideshow": { 179 | "slide_type": "slide" 180 | } 181 | }, 182 | "source": [ 183 | "# Parametric vs. Non-Parametric Models\n", 184 | "\n", 185 | "Nearest neighbors is an example of a *non-parametric* model.\n", 186 | "* A parametric model $f_\\theta(x) : \\mathcal{X} \\times \\Theta \\to \\mathcal{Y}$ is defined by a finite set of parameters $\\theta \\in \\Theta$ whose dimensionality is constant with respect to the dataset" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": { 192 | "slideshow": { 193 | "slide_type": "fragment" 194 | } 195 | }, 196 | "source": [ 197 | "* In a non-parametric model, the function $f$ uses the entire training dataset to make predictions, and the complexity of the model increases with dataset size." 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": { 203 | "slideshow": { 204 | "slide_type": "subslide" 205 | } 206 | }, 207 | "source": [ 208 | "* Non-parametric models have the advantage of not loosing any information at training time. \n", 209 | "* However, they are also computationally less tractable and may easily overfit the training set." 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": { 215 | "slideshow": { 216 | "slide_type": "slide" 217 | } 218 | }, 219 | "source": [ 220 | "# Probabilistic vs. Non-Probabilistic Models\n", 221 | "\n", 222 | "A probabilistic model is a probability distribution\n", 223 | "$$P(x,y) : \\mathcal{X} \\times \\mathcal{Y} \\to [0,1].$$\n", 224 | "This model can approximate the data distribution $P_\\text{data}(x,y)$.\n", 225 | "\n", 226 | "If we know $P(x,y)$, we can use the conditional $P(y|x)$ for prediction." 227 | ] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "metadata": { 232 | "slideshow": { 233 | "slide_type": "slide" 234 | } 235 | }, 236 | "source": [ 237 | "# Maximum Likelihood Learning\n", 238 | "\n", 239 | "Maximum likelihood is an objective that can be used to fit any probabilistic model:\n", 240 | "$$ \\theta_\\text{MLE} = \\arg\\max_\\theta \\mathbb{E}_{x, y \\sim \\mathbb{P}_\\text{data}} \\log P(x, y; \\theta). $$\n", 241 | "It minimizes the KL divergence between the model and data distributions:\n", 242 | "$$\\theta_\\text{MLE} = \\arg\\min_\\theta \\text{KL}(P_\\text{data} \\mid\\mid P_\\theta).$$" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": { 248 | "slideshow": { 249 | "slide_type": "slide" 250 | } 251 | }, 252 | "source": [ 253 | "# Discriminative vs. Generative Models\n", 254 | "\n", 255 | "There are two types of probabilistic models: *generative* and *discriminative*.\n", 256 | "\\begin{align*}\n", 257 | "\\underbrace{P_\\theta(x,y) : \\mathcal{X} \\times \\mathcal{Y} \\to [0,1]}_\\text{generative model} & \\;\\; & \\underbrace{P_\\theta(y|x) : \\mathcal{X} \\times \\mathcal{Y} \\to [0,1]}_\\text{discriminative model}\n", 258 | "\\end{align*}\n", 259 | "\n", 260 | "We can obtain predictions from generative models via $\\max_y P_\\theta(x,y)$." 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": { 266 | "slideshow": { 267 | "slide_type": "subslide" 268 | } 269 | }, 270 | "source": [ 271 | "
" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": { 277 | "slideshow": { 278 | "slide_type": "slide" 279 | } 280 | }, 281 | "source": [ 282 | "# The Max-Margin Principle\n", 283 | "\n", 284 | "Intuitively, we want to select linear decision boundaries with high *margin*. \n", 285 | "\n", 286 | "This means that we are as confident as possible for every point and we are as far as possible from the decision boundary." 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": 2, 292 | "metadata": { 293 | "slideshow": { 294 | "slide_type": "subslide" 295 | } 296 | }, 297 | "outputs": [ 298 | { 299 | "data": { 300 | "text/html": [ 301 | "
\n", 302 | "\n", 315 | "\n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | "
sepal length (cm)sepal width (cm)petal length (cm)petal width (cm)target
05.13.51.40.2-1
45.03.61.40.2-1
84.42.91.40.2-1
124.83.01.40.1-1
165.43.91.30.4-1
\n", 369 | "
" 370 | ], 371 | "text/plain": [ 372 | " sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) \\\n", 373 | "0 5.1 3.5 1.4 0.2 \n", 374 | "4 5.0 3.6 1.4 0.2 \n", 375 | "8 4.4 2.9 1.4 0.2 \n", 376 | "12 4.8 3.0 1.4 0.1 \n", 377 | "16 5.4 3.9 1.3 0.4 \n", 378 | "\n", 379 | " target \n", 380 | "0 -1 \n", 381 | "4 -1 \n", 382 | "8 -1 \n", 383 | "12 -1 \n", 384 | "16 -1 " 385 | ] 386 | }, 387 | "execution_count": 2, 388 | "metadata": {}, 389 | "output_type": "execute_result" 390 | } 391 | ], 392 | "source": [ 393 | "import numpy as np\n", 394 | "import pandas as pd\n", 395 | "from sklearn import datasets\n", 396 | "\n", 397 | "# Load the Iris dataset\n", 398 | "iris = datasets.load_iris(as_frame=True)\n", 399 | "iris_X, iris_y = iris.data, iris.target\n", 400 | "\n", 401 | "# subsample to a third of the data points\n", 402 | "iris_X = iris_X.loc[::4]\n", 403 | "iris_y = iris_y.loc[::4]\n", 404 | "\n", 405 | "# create a binary classification dataset with labels +/- 1\n", 406 | "iris_y2 = iris_y.copy()\n", 407 | "iris_y2[iris_y2==2] = 1\n", 408 | "iris_y2[iris_y2==0] = -1\n", 409 | "\n", 410 | "# print part of the dataset\n", 411 | "pd.concat([iris_X, iris_y2], axis=1).head()" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 3, 417 | "metadata": { 418 | "slideshow": { 419 | "slide_type": "subslide" 420 | } 421 | }, 422 | "outputs": [ 423 | { 424 | "data": { 425 | "text/plain": [ 426 | "" 427 | ] 428 | }, 429 | "execution_count": 3, 430 | "metadata": {}, 431 | "output_type": "execute_result" 432 | }, 433 | { 434 | "data": { 435 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtIAAAEGCAYAAABILXIOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA62klEQVR4nO3deXycZbn/8c81SzLZu6V0pQuF0pZuUNaWRZCygwoobmx6OAKyycEfRzkc5RzPUUFQQFmOIKgoYEUEVATZZClgC22hlLZQWro3Tds02ySzXL8/Mq1dkmk6ycxk+b5fr3l18txPnvk2Tye98uR+rtvcHRERERER2TuBfAcQEREREemOVEiLiIiIiGRAhbSIiIiISAZUSIuIiIiIZECFtIiIiIhIBkL5DrC3BgwY4CNHjsx3DBERERHp4ebOnbvR3SvbGu92hfTIkSOZM2dOvmOIiIiISA9nZivSjWtqh4iIiIhIBlRIi4iIiIhkQIW0iIiIiEgGVEiLSLvFEknqm+Mkkp7vKCIiInnX7W42FJHca2iOM3fVFtZsjWJm4DCiXzEHD60gHNTP4yIi0jupkBaRtBpjCZ5evIGmeLJlg7dcjV6+qZ7q+iZOGrsPwYDlMaGIiEh+6FKSiKS1aH0tsW1F9A6SDnVNCZZvbshDKhERkfxTIS0iaS3f1MDuZXSLhDsfbKzLaR4REZGuQoW0iKQV9/Q3FsYSuvFQRER6JxXSIpJWn0jbt1IYMKCkIHdhREREuhAV0iKS1oRB5W3eTBgwY9zAshwnEhER6RpUSItIWkMrihg/sIyAtVyBBggYBA0O27cPFUXhvOYTERHJF7W/E5E9OmhwOSP7FfNhdT31zXEqImFG9y+hKBzMdzQREZG8USEtIu1SWhhi8pCKfMcQERHpMrI+tcPMgmb2tpk91cpYoZk9YmYfmNkbZjYy23lERERERDpDLuZIXwUsamPsK8Bmdx8D3Ab8IAd5REREREQ6LKuFtJkNA04Dft7GLmcBD6aezwJOMDOtNSwiIiIiXV62r0j/GPgmtLkw2lBgJYC7x4EaoH+WM4mIiIiIdFjWCmkzOx3Y4O5zO+FYl5jZHDObU1VV1QnpREREREQ6JptXpKcDZ5rZcuBh4Hgz+/Uu+6wGhgOYWQioAKp3PZC73+vu09x9WmVlZRYji4iIiIi0T9YKaXf/d3cf5u4jgfOA5939S7vs9gRwQer5Oal9PFuZREREREQ6S877SJvZTcAcd38CuA/4lZl9AGyipeAWEREREenyclJIu/uLwIup5zfusD0KnJuLDCIiiaSzsb6JpEO/4jCFIa3MKCIimdPKhiLSKyzeUMuCtVvZ1l8z4c6IvsUcOrwvwYC6boqIyN5TIS0iPd7Sqjrmr9lKYpdbMD7e3EBzIskxowfkKZmIiHRnuVjZUEQkb5LuLFhbs1sRDZBwWLc1Sk00lodkIiLS3amQFpEebXNDjGSaXkDusKYmmrtAIiLSY6iQFpEezXHSzYB2Wq5ai4iI7C0V0iLSo/UpKiBdmRwIGPuUFeYsj4iI9BwqpEWkRwsFjLGVpQRt9+vSAaAiEqJ/cUHug4mISLenrh0i0uNNHFxOcyLJh9X1mBnujmH0LQ5zzOgBWCtFtoiIyJ6okBaRHs/MmDa8LxMGlbOmJkrCncqSAvrqSrSIiHSACmmRHqC+Oc7762v5eEsjDgwqK2T8PuX0KQrnO1qXUhQOst+AknzHEBGRHkKFtEg3t6Uxxt+WbCCe9O031X28uZFVW6LMGNWPIRVFec0nIiLSU+lmQ5Fu7rXl1cR2KKKhpaVbwp1Xl28ika6JsoiIiGRMhbRIN1bTGKOuKZF2nzVbG3OURkREpHdRIS3SjdXHEgTSNJxIulPfnL7QFhERkcyokBbpxkoLgmmXvw6YUVKgWyFERESyQYW0SDdWHglTVth2oWwGQ8ojOUwkIiLSe6iQFunmpo/qRzhoO03xMCAYMGaM6k8w3dwPERERyZh+5yvSzZVHwpw2bhBLqupYsbkBdxhcHuHAgaWUR9RHWkREJFtUSIv0AEXhIJOHVDB5SEW+o2TM3dlQ10R1Q4xwwBjWp4iicDDfsURERNqkQlpE8q6uKc4LH24kGkuQSDoBM95avYUDKkuZMqQCM01PERGRrkdzpEUkr5LuPLe0ivqm+PbVGRPuJB2WbqxnSVVdviOKiIi0SoW0iOTV6ppGmhNJWuvil0g6C9fVknStzigiIl2PCmkRyau1W5uIp2mGHU86DVpURkREuiAV0iKSV8E9TH92nIBa+ImISBekQlpE8mp432JCaQrlkoIQxereISIiXZAKaRHJq8qSAvoWhWmtlg6acfDQ7tvST0REejYV0iKSV2bGcfsNYHifIgIG4YARChiRUIAjR/RlSEVRviOKiIi0Sn2kRXLE3YknnWDACKgv8k5CwQBHjexPUzxBTWOcUNDoWxRW/2gREenSVEiLZJm7s7iqjkXra2mKJ8FgaHmEKUP7UFaot+COCkNBBpZpPrSIiHQPmtohkmWvr9jEgrVbicZbeiW7w6qaKH99fz1bo7F8xxMREZEMqZAWyaLqhmZWbomSaKVPcizpvL26Jg+pREREpDOokBbJomXV9WlX5VtXGyWeTOYwkYiIiHQWFdIiWdQUT7S69PWO4gktfy0iItIdqZAWyaJ+xQVpV+4LBoyCkN6GIiIi3ZH+BxfJov36l0AbLdyCBgdUlqoVnoiISDelQlokiwpDQY4e1T/VO/qf20MBY2BZIQcNKs9fOBEREekQNbEVybLB5RHOGD+IDzbWU13fREEowH79SxhYWqgFR0RERLoxFdIiORAwCAZa5kQHA4FOn86RdGdNTZSPtzSQdGdIeRH79i0mFFChnktbozE+3FhPbXOc8sIQYwaUUqpFd0REeix9hxfJsvW1Uf6+rBp3SKRa4a3Y3MDgskKmj+rf4aK6OZ7kb0s3UN+cIJ7qV712axPz19TwyQMGavXEHFm4bisL120l6eDAGmBJVR2Th1QwdmBZvuOJiEgWZG2OtJlFzOxNM5tvZgvN7Lut7HOhmVWZ2bzU46vZyiOSD03xJH9fVk086duLaIBE0lm7tYn31tV2+DVmr9hEbVN8exENEE860XiSFz+owtP0sZbOsXZrlIXra0mkimho+TPhMH/NVjbWN+UznoiIZEk2bzZsAo5398nAFOBkMzuilf0ecfcpqcfPs5hHJOeWVdfTVh2bcGdxVW3aBVv2pKE5wbraKK0snAhANJ5kQ52KuGx7b/3WVlevhJbz/N76jv/AJCIiXU/WCmlvUZf6MJx66NKY9CrVDU07XYneVTzpNMczX9mwJhojmGZqSNKdzY2xjI8v7bOlMZ52fHODzoGISE+U1fZ3ZhY0s3nABuBZd3+jld3ONrMFZjbLzIa3cZxLzGyOmc2pqqrKZmSRTlUQDO5xn47cEBgOWtqfTgNmFATV5TLbwulW3WnHuIiIdE9Z/R/W3RPuPgUYBhxmZgftssuTwEh3nwQ8CzzYxnHudfdp7j6tsrIym5FFOtXo/sUE0xTKA0sLCXWg0O1fXJC2EHd3hlUUZXx8aZ/R/UvaXMEyaKmFeUREpMfJyaUqd98CvACcvMv2anffNoHz58Ahucgjkiv9iwsYUh5ptZgOBYyDh/Xp0PHNjMP27dvq9I6gGRMHV2gJ8hw4oLKUSDi42zfUgEFJQUiFtIhID5XNrh2VZtYn9bwIOBF4f5d9Bu/w4ZnAomzlEckHM+Ookf2YsE8ZBcEAAWtZMXxweYSTxg6kIhLu8GsMrSji2P3607c4jLGteAty2L59GLeP2q7lQkEwwEljBzKyXzFB29Y33Bjdv4QTxw7s0G8dRESk68pmg9nBwINmFqSlYH/U3Z8ys5uAOe7+BHClmZ0JxIFNwIVZzCPd2Pab5hz6FIXTTpfoagJmTBhUzvh9ymhOJAkGjFCgcwurfcoinDw2QiyRxL1lTm42Vk2sa4rTGEtQWhiiKLzn+d9djbuzpTFGwp0+kXCnFriFoSCHj+jHtOF9iSWSFIQ6f+Edkc6SaI6y9ePFWDBExYgDsUDnvp/dnbrVHxJrqKV0yGgKSis69fgiXYV1tx6z06ZN8zlz5uQ7huSIu7Okqo531m3dqY3c+H3KGL9PmZbYzpGaxhizV2xiazRGwIyEO5WlhRwxoh/F3aSgXr6pnrdW15BIOgYkHfavLGHykAoVvNJreDLJ+7PuYNlffokFAuBOIBRm3Oe/wYhPnNspr1G18HXm/9+NNNVsxAJBkvEYgw+fyeSL/5NQRNOcpHsxs7nuPq2tcS15Jl3aog11vLtu9x69C9fXEkskmTK0T36C9SJ1TXGeXbKBWOocbGvnt6G2iWcWr+e0cYMId/GpCx9tqucfH2/ZrRXh0qp6orEER47sn6dkIrm14IH/YtXfHyfRHN1p+7sP/g/JeJxRJ36+Q8evfn8ub9x8Kcldjr/2jWeoX7eCo7/z25YCXqSH0L9m6bLiyWSrRTS0rAy4uKqOpg70YJb2Wbhu606rJm7jtCxPvqy6Pveh9kLSnbdX17TazzvhzsdbGqltSt8HWqQnaNy0npUvPbZbEQ0tUz0WPXwryXjHep4v/PUPdiuiAZKxJmpXfUDVwtkdOr5IV6NCWrqsDbVNaf+BBsxYu3X3b9jSuVbWNLbZqzrhsGxTQ07z7K0tjbE2Vx3cZtWWxhylEcmf9W+/CJb+v/3NS+dlfPxYQy01K9ruGZCINrD61T9lfHyRrkiFtHRZCd/zUpjpVg2UzrGHGpTknnbIs6S3zIlue5wOLdMu0l0kY83gaX6LZ0Yi3pz58eOxPd63kmjWD63Ss6iQli6rf0lB2gLH3aksKchhot5pQHHbX2MD9ikrzF2YDFREwml/GAgFjMrSrv13EOkM/cYeTEsjrdYlY830GTUh4+MXlPWloKxfm+PBwmIqJ07P+PgiXZEKaemyisNBhlYU0Vqnu4DBgNJCyjuhD7Okd9Dg8lYXfAEIBIwDB3btXtXhYID9BpS0+ncwWnpu6wcy6Q36jJpA2fD9seDufQYCBYUMm346BaV9Mj6+mXHAZy4jWNDaaqpGsKCQoUedlvHxRboiFdLSpR0xoi8DSwsJmmG0FD7BgNGvuIAZo9RpIRcGlhZyyPAKgqlFRqDlKm4oYBwzuj+lhV2/+c/UoRUMrYi0LJaS2hYKGGWREJ8YU6k2itJrHH7d3ZQPP4BgYRFYAAuGCBQUMnDidCZedGOHjz/i+HMZdfKXCYQLCIQLASMYKaGwzwCm/8cvCRW2VmSLdF/qIy3dwubGZtbUtNxYOKgsQn9dQcy55niSj7c0EI0lKS0MMaxPEaFutDAOQG00xqqaKAl3BpYUUllaoCJaeh13Z/PSeWx87w0sEGTQwZ+gbNiYTn2N6OYNrHn9aWKNtVSMGM8+U4/p9EVfRHJBfaSlR+hbVEDfIhXP+VQQCjBmQGm+Y3RIWSTMOE0Hkl7OzOh3wFT6HTA1K8dvrtvCu7/+Ievm/A1PxCnsU8m4hmsYfvSZWXk9ad3yvz3C4t/fSXPtZiwUZuhRpzHx/H/XojidTFekRUREpFM0VK/j+WtmttqPeuj0Mzjk8h/mIVXvM/sHl1A1/+XdtgcLizjxjhcpKC3Pfahuak9XpPc4R9rMPmNmS82sxsy2mlmtmW3t3JgiIiLS3b3+/a+2uajL6lefZPMH83OcqPdZO+e5VotogERTI2/+6LIcJ+rZ2nOz4Q+BM929wt3L3b3M3fWjjIiIiGwXj9ZTt/rDtPu8++ubc5Sm91r0yG1pxzctnpujJL1Dewrp9e7e9lJFIiIi0uvVV63e4z4NVStzkKR3i25av8d94tH6HCTpHdq82dDMPpN6OsfMHgEeB5q2jbv7Y9mNJiIiIt1FUf8he9wn0ndgDpL0bgVlfYk31qXdJ9Bqr2/JRLor0mekHuVAAzBzh22nZz+aiIiIdBcFxaUUDRiadp9xn70yR2l6rwM+fWna8fIRBxIIaBmRztLmFWl3vwjAzKa7+6s7jpmZ1vgUERGRnRx+3d28eP1Z4MndxvqPP4yBk47OQ6reZd9jP82yp3/J1hXv7zZmwRCHXfuzPKTqudrzI8kd7dwmIiIivVj58DEcf+uf6Tf2ELCWEiMYKWHsOVcw/YYH85yu9zjme79nzBlfaVnBErBAgAEHHcmJP3mO4gGD85yuZ2mzj7SZHQkcBVwN7HgLaDnwaXefnPV0rVAfaREREZGeL9HcxIb5f6dp62bKho6m39hDcr4abUdWNiwASlP7lO2wfStwTufEExERERHZ2erZf2b+/90IgCcTEAhQWNaPw795N2VD98tzun/a48qGZjbC3VfkKM8e6Yq0iIiISM+18b03eeOH/0qiObrLiBEuKeeE2/5KQWlFTrJkfEXazJ4EPPV8t3F3P7MzAoqIiIiIbPP+oz9ppYgGcJKxJlb+/Q/sd+qFuY7VqnQ3G94C/Aj4CGgE/i/1qAPSL10kIiIiIrKX3J1NH8xrczzRHGXtm8/mLtAepGt/9xKAmf1ol0vaT5qZ5laIiIiISKczjHQTj60L9cFuT5ISMxu97QMzGwWUZC+SiIiIiPRGZkb/8Ye3OR4sLGLIkafmMFF67SmkrwFeNLMXzewl4AVaWuKJiIiIiHSqcZ+9imBBZLftFggQKipl+NFd5za9dO3vAHD3p81sf+DA1Kb33b0pu7FEREREpDfqO2YSh37jTt6+63oSTY1ghifilA4bw6FX/4RQpOtMjEjXteN4d3/ezD6zy9B+Zoa7P5blbCIiIiLSCw2cNJ2ZP32JTUveorl2M6VDRnep/tHbpLsifSzwPHBGK2MOqJAWERER6WI8mWTFC7/jgyd+TsPG1YQixQybcRZjz76cwvJ++Y7XLs21m1ny+N18/OJjxKP1FPUfxJjTvsLIE8/DAsF8x9su3RLhU4D5vqcVW3JMC7KIiIiItM7deeun32Td3OdapkWkWDBMYXlfjv2fxyis6J/HhHvWXLuZl751Nk01G0nGY9u3BwuLGDhpBtOu/knOlgrf04Is6W42/DlQbWbPmtl3zWymmZWl2V9ERERE8mjT4rm7FdEAnojRtHUTi373kzwla7/Fj91FdJciGiDR1MiGBa9Q9e5reUq2uzYL6VT1PQz4HtAEXAl8YGbzzexnOconIiIiIu20/G8Pk2hqbVVA8EScVa88QRebbLCblS89hu9SRG+TaGpk+bMP5zhR29J27XD3Blpa3/0DeAOYDpwPnJyDbCIiIiKyF6Kb1kOa5UySsRieiGGhgtyF2gvuTjxan3af6JYNOUqzZ+m6dnwBOAqYQssV6W3F9Ax3X5eTdCIiIiLSbhUjx7Np6Tw8EW91vKCsD4EuWkRDy4IskX77pH4gaGU8EKRi3wNbHcuHdHOk7wGOAB4ALnX36939DyqiRURERLqmkTO/gAVbv04aKIiw36kX5jZQBvY77aJWF2QBsFCY0Sd/OceJ2paukO4DXAJEgO+Y2Vwze8rMvm1mx+cknYiIiIi0W+mgEUy84NsECiI7tYkLFhbT/8Bp7HfaRXlM1z6jT/oSAyYeRbCw+J8bA0ECBREmfOE6yoaNyV+4XbTZ/m63Hc32Ac6lZXnwUe6elyZ+an8nIiIikl7t6mUse/qX1Hy0kIKK/oz65HkMnHwMFkh3DbXr8GSSqnde5aNnf0vTlirKR45j9MlfpnzY/jnNsaf2d+n6SE+iZY70tkcB8BowG3jV3fNSzaqQ7n3cnfV1Taze0ogDQ8ojDC6P5KyHpPQM7s7mpfNY8+YzJJqjVB50JIMO/gSBUDjf0dotGW9mzZvPUr3oH4QiJQw98lT6jJ6Q71giPU4y3szSP97LmjefAYzhx5zF6FMuJNBNitBc8GSSqoWzWT/3BRxn0NTjqJw4vdsU6u3VkUL6LeAV/lk4f5ydiHtHhXTv0hxP8vwHVdQ2xYknW/6thgJGUTjICftXUhTuOqsbSdeVaI7yxi2XsnnpfBLNTeBJgpESwsWlTP+PX1Gyz/B8R9yj2tXLeO2/zyfeFCURrQcLEAwXMuCgIzj06h936ZuHRLqTmpVL+Pu3z9mt/VqgIMLxNz9FceXQPCXrOprrtvDaf19I/YaVJKINAAQjxRQPGMJRNzzYbVZPbI+MF2Rx94Pd/Up3/20mRbSZRczszVTf6YVm9t1W9ik0s0fM7AMze8PMRu7t60jP9tryamqise1FNEA86dQ1xXl5WXUek0l3suAX/82mxW+3LFDgSQAS0Xqim6uY/T8X48lknhOml4zHeO17F9JUs6mliAbwJInmRqrenc17v/lRfgOK9CCv3Hheqz2Mk81RXvr2uXlI1PXM+ck11K5etr2IBkhEG6hbu5x/3HpFHpPlXjavvzcBx7v7ZFpa6J1sZkfsss9XgM3uPga4DfhBFvNIN1PXFGd9XRPJVn5p4sCWxhibG5tznku6l1j9Vla/9hTJWNPug56kqXYTVe/Ozn2wvbDurRdS/2Ht/mZINkdZ8fzviO+yipmI7L3Vr/9ltxUBdxSr20z1+3NzmKjrqVu3gk1L3sYTu/+w4Yk4W5YvpHbVB3lIlh9ZK6S9RV3qw3Dqsev/AmcBD6aezwJOME18lZTNjc0E9vDPYVND6ysfiWyzddUHaedBJ5qibFn2Tg4T7b3NS+elXaDAggHq13eJ2Xci3dr6t1/a4z7r3no+B0m6rpqPFrbZXg/ALMiWZe/mMFF+ZXVGuJkFzWwesAF41t3f2GWXocBKAHePAzVA/1aOc4mZzTGzOVVVVdmMLF1IaA83LJi1zJcWSScUKU47dSMQChPascVSFxQqKk37H1cyEe/yfweR7iBcVLrnfUoqcpCk6wpFikl3jcvMCEZ6z/ejNisVM3vSzJ5o69Geg7t7wt2nAMOAw8zsoExCuvu97j7N3adVVlZmcgjphgaWFqYdd2/p4CGSTvm+YwkXl6XdZ/BhM3OUJjNDjzhlp36wuyrqP6Rb3DAp0tXtd8ZX97jPyBO/kIMkXdeACUfgrc25TPFkgoGTZuQwUX6lu+R3C/CjNI92c/ctwAvAybsMrQaGA5hZCKgAdAeZABAMGFOHVhBs5UffYMA4aHAZ4WDParMjnc/MmHTxf7a6SlawoIgRx59LUf9BeUjWfqVDRjH0yFMJFu7+dwgURJh00X/kIZVIz1PcfxCVk49uc3zY0WdSULznq9Y9WbAgwvjPX9vm99QDP3sVoV50RbrdC7Ls9YHNKoGYu28xsyLgGeAH7v7UDvtcDkx096+Z2XnAZ9z9s+mOq/Z3vc+KTfXMW7OVpngCMMJBY+LgcsYM6N3fzGTvbJj/Mu/88n9p3LgGCwQIhAvZ/8x/Yb/TLuoWPck9mWTJ4/fw4Z9/gSfieDJByaARTDz/2wyYcHi+44n0KG/d9e+seuWJ7V1+LBBk5Cc/z8QLv53nZF3HyleeYNHDt9JctwXDCJeUc+Bnr2LfYz+T72idKuM+0jscYH/gf4HxtCwXDoC7j97D502i5UbCIC1Xvh9195vM7CZgjrs/YWYR4FfAVGATcJ67L0t33HwU0rFEkoZYgsJggIj6FueFu9MQS+AOJQXBblH4SNfUWL2OZLyZogFDCKSZd9xVxaP1bFoyj3BJGX33m5SV12jYuIZkcxPFA4eqP7X0Wsl4nE1L38YCQfruP0WLsbTC3WncuAZ3p3jAkB63GAt0TiH9CvCftLSnOwO4CAi4+42dGbS9cllIxxJJ5q7awsebGzAzku70Ly7g0OF9qSjqPquhiUj358kEi3//M5Y9/SDujicSFA8YwsQLb6By4lGd8hpV77zGOw/8Nw0b12DBIGYBRp98PmPPviztHG0RkZ4q4wVZdlDk7s/RUnSvcPfvAKd1VsCuKpF0/rakihWbG0h4yyIgSYeq+maeXbKB2qZ4viOKSC/y9j3f5sM/3U+8sZ5EtIFkrIm6tR/xxo8uZ8M7r3b4+BveeZU3fnQ5dWs/IhlrIhFtIN5Yx4d/up+379Gvs0VEWtOeQrrJzALAUjP7upl9Gujxk1NX1TRS1xxvdTGQeNJZsKYm96FEpFeqXf0ha954mkRzdLexZHOUd+6/iY7c7+LuLLj/JpKtHD/RHGXNG09TuzrtrDsRkV6pPYX0VUAxcCVwCPAl4PxshuoKllXX77Qs9Y6clkJbRCQX1rz+NJ5o+7dgjZs30LBhVcbHb9iwkujmDW2OeyLOmjeezvj4IiI9VXsK6ZHuXufuq9z9Inc/G9g328HyLZZoewEHgKTToStAIiLtFW9qwBOJNscDgSDxpoYOHL+RQJo50J5IpF1ZUUSkt2pPIf3v7dzWowwsi5Bu0byKSEidI0QkJ/rtPyXtSmHuSUoHjcj4+KWDRuDe9sWDYKSEfvtPzfj4IiI9VZu9n8zsFOBUYKiZ3b7DUDnQ4++0O2BACUuq6lqWz9tFMGAcNKg8D6lEpDfa5+BPEC4qJdHUuNv3pEBBhJEnnNfq4gjtFSyIMOKEz7H8bw/vPk/ajHBRCfscfFzGxxcR6anSXZFeA8wBosDcHR5PACdlP1p+FReEOGZ0f0IBI5i6NB2wlseBA0vZt2/vWbVHRPIrEAxx1A0PUFgxgGCkJLUxQLAgwsBJMxh33jUdfo3x532DgZNmtBTkqV6wwUgJhRUDOOqGB7plz20RkWxrTx/pMC1Xrvd198U5SZVGrhdkiSWSrNjcwJbGGJFwkFH9iikp0H8oIpJ7yXiMdXOeo/r9OYQixQw54hQqRo7r1NeoWb6INa//hXi0gf4HTmPQtBMIhNQ3X0R6p85YkOUM4BagwN1HmdkU4CZ3P7NTk7aTlggXERHp3Va99hTvP3o70c0bCEWKGXni5zngU5cSCOlCV64019Xw8UuPseb1pwFn8KEnMuL4cyko7ZPvaJ2qMwrpucDxwIvuPjW17R13n9ipSdtJhbSIiEjv9fotl7LhrRd3215QMYCZdzynZe1zoG7tcl75zueJN0W331cRKIgQDBcy4zsPUTZ0vzwn7DydsbJhzN13XX1Efd9EREQkp1a/9udWi2iA5pqNzLnzutwG6qXevPXrNNfV7HRzcrI5Sqx+K2/eclmvag/cnkJ6oZl9AQia2f5mdgfwWpZziYiIiOxk0aM/Tju+fs5zuQnSi235aCGNG9e02tUMnOiWjWz+YH7Oc+VLewrpK4AJQBPwG6AGuDqLmURERER2E91SlXbck20vXCSdo37tcszSl491a5fnJkwXkK6PdAT4GjAGeAc40t17fP9oERER6ZpCkRKad+11vhMtlJZtBRX9087vtYBRWN4vZ3nyLd2PFA8C02gpok+hpXOHiIiISF6MPumLaccr9jsoR0l6rwHjDiUYbvuGTgsEqTzoyBwmyq90hfR4d/+Su98DnAMck6NMIiIiIrsZc9a/Uth3YKtjFghy6JW35ThR72OBIAdf9sPUaqo7/wYgWBBh6qXf71W959MV0rFtTzSlQ0RERPItEAhw4k+eZcgRp2KBYGqr0WfMJE647a8UVw7Na77eYuDkGUy/8dcMnHIMgVABgVCYyknTOeqGBxh08CfyHS+n2uwjbWYJoH7bh0AR0JB67u5enpOEu1AfaRERERHJhT31kW7zZkN3D7Y1JiIiIiLS27Wn/Z2IiIiIiOxChbSIiIiISAZUSIuIiIiIZECFtIiIiIhIBlRIi4iIiIhkQIW0iIiIiEgGVEiLiIiIiGRAhbSIiIiISAbaXJBFRESkMyUTcVa8MItlf36A6OYNFFYMYPTJX2bkJz9HIFSQ73giIntNhbSIiGSdJxO8cfOlbFo8l0RTIwANG1by3sO3subNZzjqW/epmBaRbkdTO0REJOtWvfannYrobZLNUWqWvcuKFx/LUzIRkcypkBYRkaxb9vSvdiuit0k0R/nor7/OcSIRkY5TIS0iIlnXtKVqD+Mbc5RERKTzqJAWEZGsKx44PP145dAcJRER6TwqpEVEJOvGnHYxwcKiVseChUXsd/rFOU4kItJxKqRFRCTr9jn4OIZOP323YjpYWMSgQ45n6JGn5imZiEjm1P5ORESyzsyY/JXvMuTQmXz4lwdoWL+SogFDGH3K+ewz9TjMLN8RRUT2mgppERHJCTNj4OQZDJw8I99RJMsaq9ey5vW/EmvYSsXIcexz8CcIBFVy5FIi1sy6OX9j66oPKCztw5AjTyHSpzLfsXoc/asWERGRTuHuvPfbW1raGTok482EIiUECyMc+a37KR9+QL4j9gqbP1jA6z+4hGQiTiJaTyBcyHsP/4gDPn0pB3zqa/mO16NkbY60mQ03sxfM7D0zW2hmV7Wyz3FmVmNm81KPG7OVR0RERLJrxXOPsPzZ35KMNZOMNwMQj9bTVFPNq/91AfFoQ54T9nzNtZuZ/b9fIVZfQyJaD0Ay1kQy1szSP97LmtefznPCniWbNxvGgWvdfTxwBHC5mY1vZb+X3X1K6nFTFvOIiIhIlrg7ix/7WZsL7yRjzaye/accp+p9Vrwwi2Qi3upYoqmR92fdkeNEPVvWCml3X+vub6We1wKLADUKFRER6YGaazcTq6tpczzR1EDVgldzmKh32rDgFZLN0TbH69Z+RDIey2Gini0n7e/MbCQwFXijleEjzWy+mf3FzCa08fmXmNkcM5tTVZV+dSwRERHJvUCoAPdk2n2ChcU5StN7tdWvfRuzABYI5ihNz5f1QtrMSoHfA1e7+9Zdht8CRrj7ZOAO4PHWjuHu97r7NHefVlmpO05FRES6mnBxKRWjWpvB2SIYKWbY9NNzmKh32vfoTxGMtPUDi1E5aQYW0DIinSWrX0kzC9NSRD/k7o/tOu7uW929LvX8z0DYzAZkM5OIiIhkx0Ffup5gQWS37YFwIeXDD2DAQUfmIVXvMmjaCRRXDiUQKthtLFgYYdx51+QhVc+Vza4dBtwHLHL3W9vYZ1BqP8zssFSe6mxlEhERkezpd8BUDv9/91AyaATBggiholIC4UKGHnkqR37rfi28kwOBUJgZ//kQg6YdTyBc2HIOCiKUDdufo779ABX7js13xB7F3D07BzabAbwMvANsmzT1LWBfAHe/28y+DlxKS4ePRuAb7v5auuNOmzbN58yZk5XMIiIdlWhuIlZfQ7i0D8Hw7leERHoDd6d+3XJiDXWUDhpBuKQ835F6pea6LdSvX0lBaR9K9hme7zjdkpnNdfdpbY1nbUEWd38FSPujp7vfCdyZrQwiIrnStHUzCx/6AWte/wtgYDDsqNOZ8MVvqoiQXsfMKB08Kt8xer2C0j4UlPbJd4weTSsbioh0UKx+K3+/4RyimzfgO/RvXfnyE1Qvnsux3/s9oTZv/hERke5Kt22KiHTQsr/+mqaa6p2KaABPxIhWr+PjF3+fp2QiIpJNKqRFRDro4xd/TzLW1OpYojnKiucfzXEiERHJBRXSIiIdFI82pB2PNdTlKImIiOSSCmkRkQ6qGHFg24Nm9Bnd6qKtIiLSzamQFhHpoP3PuqTVRSgAguFCxpzx1RwnEhGRXFAhLSLSQZUHHckBZ3+dQLgQC7U0QwqECgiECxn/hevot/+U/AYUEZGsUPs7EZFOsP8ZX2HIYTNZ8fyj1K//mNIhoxlx/LkUDxiS72giIpIlKqRFRDpJyT7DGf/5a/MdQ0REckRTO0REREREMqBCWkREREQkAyqkRUREREQyoEJaRERERCQDutlQREREpAuLxWKsWrWKaDSa7yg9ViQSYdiwYYTD4b36PBXSIiIiIl3YqlWrKCsrY+TIkZhZvuP0OO5OdXU1q1atYtSoUXv1uZraISIiItKFRaNR+vfvryI6S8yM/v37Z3TFX4W0iIiISBenIjq7Mv36amqHiIhILxNrqGPLR+8SCATpM2YywXBBviOJdEsqpEVERHqJZCLOwl//kBXPP0oglLqpyp2x53yd0adcoKuePcTyTfXMX7OVhliC4nCQyUPKGdmvpEPH/N73vsdvfvMbgsEggUCAe+65h8MPP7zVfR944AFmzpzJkCFDOvSa3YEKaRERkV5i/v/dyOrX/0Iy1kQy1rR9+/u/ux0Hxpx6Yd6ySedYvqmeNz/eQsIdgIZYgjc/3gKQcTE9e/ZsnnrqKd566y0KCwvZuHEjzc3Nbe7/wAMPcNBBB/WKQlpzpEVERHqBho1rWD37TySbd7+hKtHUyJJZd5KItV0cSfcwf83W7UX0Ngl35q/ZmvEx165dy4ABAygsLARgwIABDBkyhLlz53LsscdyyCGHcNJJJ7F27VpmzZrFnDlz+OIXv8iUKVNobGzkueeeY+rUqUycOJGLL76YpqaWH+Kuv/56xo8fz6RJk/i3f/s3AJ588kkOP/xwpk6dyic/+UnWr1+fce5cUCEtIiLSC6x/+yXM0vy3b8bmpfNylkeyoyGW2Kvt7TFz5kxWrlzJAQccwGWXXcZLL71ELBbjiiuuYNasWcydO5eLL76Yb3/725xzzjlMmzaNhx56iHnz5mFmXHjhhTzyyCO88847xONx7rrrLqqrq/nDH/7AwoULWbBgATfccAMAM2bM4PXXX+ftt9/mvPPO44c//GHGuXNBUztERER6AU/E8WQy7T7JRCxHaSRbisPBVovm4nAw42OWlpYyd+5cXn75ZV544QU+97nPccMNN/Duu+9y4oknApBIJBg8ePBun7t48WJGjRrFAQccAMAFF1zAT3/6U77+9a8TiUT4yle+wumnn87pp58OtPTM/tznPsfatWtpbm7e677OuaZCWkREpBfoP/4wLND2FelkPEbf0RNzmEiyYfKQ8p3mSAMEzZg8pLxDxw0Ggxx33HEcd9xxTJw4kZ/+9KdMmDCB2bNnZ3S8UCjEm2++yXPPPcesWbO48847ef7557niiiv4xje+wZlnnsmLL77Id77znQ7lzjZN7RAREekFKvYdS5/RB/2zW8cOggUR9j3ubMIlHSu2JP9G9ivhsH37bL8CXRwOcti+fTrUtWPx4sUsXbp0+8fz5s1j3LhxVFVVbS+kY7EYCxcuBKCsrIza2loAxo4dy/Lly/nggw8A+NWvfsWxxx5LXV0dNTU1nHrqqdx2223Mnz8fgJqaGoYOHQrAgw8+mHHmXNEVaRERkV7isGt/yhu3XEbNRwtbpnmk2t0NOvSTHPTl6/OcTjrLyH4lHW53t6O6ujquuOIKtmzZQigUYsyYMdx7771ccsklXHnlldTU1BCPx7n66quZMGECF154IV/72tcoKipi9uzZ/OIXv+Dcc88lHo9z6KGH8rWvfY1NmzZx1llnEY1GcXduvfVWAL7zne9w7rnn0rdvX44//ng++uijTvt7ZIP5Lnd2dnXTpk3zOXPm5DuGiIhIt1WzfBEbF71JIBhi4JRjKRk4LN+RJI1FixYxbty4fMfo8Vr7OpvZXHef1tbn6Iq0iIhIL1MxchwVI1WYiXSU5kiLiIiIiGRAhbSIiIiISAZUSIuIiIiIZECFtIiIiIhIBlRIi4iIiIhkQIW0iIiISA+y6pUnefbKE3jiC+N59soTWPXKkx0+pplx7bXXbv/4lltu2eOqg48//jjvvfdeq2OLFy/muOOOY8qUKYwbN45LLrkk7bGWL1/Ob37zm73OnW0qpEVERER6iFWvPMn8n99I48Y1gNO4cQ3zf35jh4vpwsJCHnvsMTZu3Njuz0lXSF955ZVcc801zJs3j0WLFnHFFVekPZYKaRERERHJqkWP/phEc3SnbYnmKIse/XGHjhsKhbjkkku47bbbdhtbvnw5xx9/PJMmTeKEE07g448/5rXXXuOJJ57guuuuY8qUKXz44Yc7fc7atWsZNuyfCwFNnDixJWsiwXXXXcehhx7KpEmTuOeeewC4/vrrefnll5kyZQq33XYb0WiUiy66iIkTJzJ16lReeOEFABYuXMhhhx3GlClTmDRp0valzT/1qU9xyCGHMGHCBO69994OfS12+rp02pFEpE3uTvV7b1KzYhHh4jIGTTuBgtI++Y4lIiI9TOPGta1vr259+964/PLLmTRpEt/85jd32n7FFVdwwQUXcMEFF3D//fdz5ZVX8vjjj3PmmWdy+umnc8455+x2rGuuuYbjjz+eo446ipkzZ3LRRRfRp08f7rvvPioqKvjHP/5BU1MT06dPZ+bMmXz/+9/nlltu4amnngLgRz/6EWbGO++8w/vvv8/MmTNZsmQJd999N1dddRVf/OIXaW5uJpFIAHD//ffTr18/GhsbOfTQQzn77LPp379/h78mKqRFsqx+/cfM/v5XaaqpJhmPEQiGWfCL/2Ls2Zez/5n/ku94IiLSgxQNGJya1rHL9v6DO3zs8vJyzj//fG6//XaKioq2b589ezaPPfYYAF/+8pd3K7Rbc9FFF3HSSSfx9NNP88c//pF77rmH+fPn88wzz7BgwQJmzZoFQE1NDUuXLqWgoGCnz3/llVe2Twc58MADGTFiBEuWLOHII4/ke9/7HqtWreIzn/kM+++/PwC33347f/jDHwBYuXIlS5cu7ZRCOmtTO8xsuJm9YGbvmdlCM7uqlX3MzG43sw/MbIGZHZytPCL5kGhu4pXvfpGGDatJRBvweIxEUwPJWBNLHvsZK195It8RRUSkBxn32asJFkR22hYsiDDus1d3yvGvvvpq7rvvPurr6zt8rCFDhnDxxRfzxz/+kVAoxLvvvou7c8cddzBv3jzmzZvHRx99xMyZM9t9zC984Qs88cQTFBUVceqpp/L888/z4osv8re//Y3Zs2czf/58pk6dSjQa3fPB2iGbc6TjwLXuPh44ArjczMbvss8pwP6pxyXAXVnMI5Jza17/C/FoA3hyt7FEc5T3H/0J7p6HZCIi0hMNm3EGk796E0UDhoAZRQOGMPmrNzFsxhmdcvx+/frx2c9+lvvuu2/7tqOOOoqHH34YgIceeoijjz4agLKyMmpra1s9ztNPP00sFgNg3bp1VFdXM3ToUE466STuuuuu7WNLliyhvr5+t2MdffTRPPTQQ9v3+fjjjxk7dizLli1j9OjRXHnllZx11lksWLCAmpoa+vbtS3FxMe+//z6vv/56p3wtIItTO9x9LbA29bzWzBYBQ4Edb988C/ilt1QSr5tZHzMbnPpckW5v/by/k4g2tDnetKWK5totFJb3zWEqERHpyYbNOKPTCufWXHvttdx5553bP77jjju46KKLuPnmm6msrOQXv/gFAOeddx7/8i//wu23386sWbPYb7/9tn/OM888w1VXXUUk0nL1/Oabb2bQoEF89atfZfny5Rx88MG4O5WVlTz++ONMmjSJYDDI5MmTufDCC7nsssu49NJLmThxIqFQiAceeIDCwkIeffRRfvWrXxEOhxk0aBDf+ta3KCkp4e6772bcuHGMHTuWI444otO+FpaLq2FmNhL4O3CQu2/dYftTwPfd/ZXUx88B/8/d5+zy+ZfQcsWafffd95AVK1ZkPbNIZ3jrZ/+PVWmmb1gwxMn3vEa4uCyHqUREpDtZtGgR48aNy3eMHq+1r7OZzXX3aW19Ttbb35lZKfB74Oodi+i94e73uvs0d59WWVnZuQFFsmjIEScTjBS3OV42bH8V0SIiIt1UVgtpMwvTUkQ/5O6PtbLLamD4Dh8PS20T6RH2mXIMJQOHEwiFdxsLFESY8KU939ksIiIiXVM2u3YYcB+wyN1vbWO3J4DzU907jgBqND9aehILBJn+H7+kctIMAuECQsVlhCIlFPapZNqVt1I5ofPmaYmISM+lG9OzK9Ovbzb7SE8Hvgy8Y2bzUtu+BewL4O53A38GTgU+ABqAi7KYRyQvwiXlHP5vPyO6eQNbVy4lXFxKn9ETsYAWFhURkT2LRCJUV1fTv39/Wq5TSmdyd6qrq7ff+Lg3stm14xUg7dlOdeu4PFsZRLqSSN+BRPoOzHcMERHpZoYNG8aqVauoqqrKd5QeKxKJ7LRkeXtpZUMRERGRLiwcDjNq1Kh8x5BW6HfLIiIiIiIZUCEtIiIiIpIBFdIiIiIiIhnIycqGncnMqoCeurThAGBjvkNI1uk893w6x72DznPvoPPcO7R1nke4e5urAXa7QronM7M56ZahlJ5B57nn0znuHXSeewed594h0/OsqR0iIiIiIhlQIS0iIiIikgEV0l3LvfkOIDmh89zz6Rz3DjrPvYPOc++Q0XnWHGkRERERkQzoirSIiIiISAZUSIuIiIiIZECFdB6YWdDM3jazp1oZu9DMqsxsXurx1XxklI4zs+Vm9k7qPM5pZdzM7HYz+8DMFpjZwfnIKZlrxzk+zsxqdng/35iPnNIxZtbHzGaZ2ftmtsjMjtxlXO/lHqAd51nv527OzMbucP7mmdlWM7t6l3326v0cympiactVwCKgvI3xR9z96znMI9nzCXdvq5H/KcD+qcfhwF2pP6V7SXeOAV5299Nzlkay4SfA0+5+jpkVAMW7jOu93DPs6TyD3s/dmrsvBqZAy0VNYDXwh11226v3s65I55iZDQNOA36e7yySd2cBv/QWrwN9zGxwvkOJyD+ZWQVwDHAfgLs3u/uWXXbTe7mba+d5lp7lBOBDd991tey9ej+rkM69HwPfBJJp9jk79euEWWY2PDexJAsceMbM5prZJa2MDwVW7vDxqtQ26T72dI4BjjSz+Wb2FzObkMtw0ilGAVXAL1JT8n5uZiW77KP3cvfXnvMMej/3JOcBv21l+169n1VI55CZnQ5scPe5aXZ7Ehjp7pOAZ4EHcxJOsmGGux9My6+JLjezY/IdSDrdns7xW8AId58M3AE8nuN80nEh4GDgLnefCtQD1+c3kmRBe86z3s89RGrqzpnA7zp6LBXSuTUdONPMlgMPA8eb2a933MHdq929KfXhz4FDchtROou7r079uYGWOViH7bLLamDH3zgMS22TbmJP59jdt7p7Xer5n4GwmQ3IeVDpiFXAKnd/I/XxLFoKrh3pvdz97fE86/3co5wCvOXu61sZ26v3swrpHHL3f3f3Ye4+kpZfKTzv7l/acZ9d5uGcSctNidLNmFmJmZVtew7MBN7dZbcngPNTdwgfAdS4+9ocR5UMteccm9kgM7PU88No+Z5bneuskjl3XwesNLOxqU0nAO/tspvey91ce86z3s89yudpfVoH7OX7WV07ugAzuwmY4+5PAFea2ZlAHNgEXJjPbJKxfYA/pL7nhoDfuPvTZvY1AHe/G/gzcCrwAdAAXJSnrJKZ9pzjc4BLzSwONALnuZaT7Y6uAB5K/Tp4GXCR3ss90p7Os97PPUDqwseJwL/usC3j97OWCBcRERERyYCmdoiIiIiIZECFtIiIiIhIBlRIi4iIiIhkQIW0iIiIiEgGVEiLiIiIiGRAhbSISA6ZWcLM5pnZu2b2OzMrTrPvFDM7tR3HPM7Mnmrv9s5iZn3M7LJcvZ6ISFejQlpEJLca3X2Kux8ENANfS7PvFFr6mXZVfYDL9rSTiEhPpUJaRCR/XgbGpFZJvN/M3jSzt83srNSiEDcBn0tdwf6cmR1mZrNT+7y2wypse8XMZqaO81bqqnhpavtyM/tuavs7ZnZganulmT1rZgvN7OdmtiK1NPL3gf1S+W5OHb7UzGaZ2ftm9tC2leBERHoiFdIiInlgZiHgFOAd4NvA8+5+GPAJ4GYgDNwIPJK6gv0I8D5wtLtPTY39TwavOwC4Afikux8MzAG+scMuG1Pb7wL+LbXtP1P5JgCzgH1T268HPkzluy61bSpwNTAeGA1M39uMIiLdhZYIFxHJrSIzm5d6/jJwH/AacKaZbStcI/yzWN1RBfCgme0POC3F9t46gpYi99XUxeICYPYO44+l/pwLfCb1fAbwaYDUMuib0xz/TXdfBZD6e44EXskgp4hIl6dCWkQktxrdfcqOG1LTH85298W7bD98l8/9L+AFd/+0mY0EXszg9Q141t0/38Z4U+rPBJn9H9G0w/NMjyEi0i1oaoeISP79Fbhi23xiM5ua2l4LlO2wXwWwOvX8wgxf63VgupmNSb1WiZkdsIfPeRX4bGr/mUDfNvKJiPQqKqRFRPLvv2iZprHAzBamPgZ4ARi/7WZD4IfA/5rZ27T/Su8JZrZq2wMYQ0sR/lszW0DLtI4D93CM7wIzzexd4FxgHVDr7tW0TBF5d4ebDUVEeg1z93xnEBGRLszMCoGEu8fN7Ejgrl2np4iI9EaauyYiInuyL/ComQVo6X39L3nOIyLSJeiKtIiIiIhIBjRHWkREREQkAyqkRUREREQyoEJaRERERCQDKqRFRERERDKgQlpEREREJAP/H4Djw3TKQm7hAAAAAElFTkSuQmCC\n", 436 | "text/plain": [ 437 | "
" 438 | ] 439 | }, 440 | "metadata": { 441 | "needs_background": "light" 442 | }, 443 | "output_type": "display_data" 444 | } 445 | ], 446 | "source": [ 447 | "# https://scikit-learn.org/stable/auto_examples/neighbors/plot_classification.html\n", 448 | "%matplotlib inline\n", 449 | "import matplotlib.pyplot as plt\n", 450 | "plt.rcParams['figure.figsize'] = [12, 4]\n", 451 | "import warnings\n", 452 | "warnings.filterwarnings(\"ignore\")\n", 453 | "\n", 454 | "# create 2d version of dataset and subsample it\n", 455 | "X = iris_X.to_numpy()[:,:2]\n", 456 | "x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n", 457 | "y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n", 458 | "xx, yy = np.meshgrid(np.arange(x_min, x_max, .02), np.arange(y_min, y_max, .02))\n", 459 | "\n", 460 | "# Plot also the training points\n", 461 | "p1 = plt.scatter(X[:, 0], X[:, 1], c=iris_y2, s=60, cmap=plt.cm.Paired)\n", 462 | "plt.xlabel('Petal Length')\n", 463 | "plt.ylabel('Petal Width')\n", 464 | "plt.legend(handles=p1.legend_elements()[0], labels=['Setosa', 'Not Setosa'], loc='lower right')" 465 | ] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": 4, 470 | "metadata": { 471 | "slideshow": { 472 | "slide_type": "subslide" 473 | } 474 | }, 475 | "outputs": [ 476 | { 477 | "data": { 478 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAADgCAYAAADFcDomAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAABra0lEQVR4nO3ddXxT1//H8ddJ3Z0aLVDcGe422NhgMGfynbu7uwvf+b5z+83dYDAFhjPcXUpbSt0tSXN+fyR0LS3VpEnaz/Px6GPt7c3NCev7nnPvPaK01gghhBBCCCEaz+DsAgghhBBCCOFupBEthBBCCCFEE0kjWgghhBBCiCaSRrQQQgghhBBNJI1oIYQQQgghmkga0UIIIYQQQjSRNKJFLUqpg0qpKc4uR3VKqfuVUu85uxxCuALJqBDuQ/Ladkkj2k0opc5TSq1WSpUopTJt31+vlFKtXI6PlFJaKTXrmO0v2bZf6oj31Vo/rbW+0hHHFsLRbJVomVKqWCmVp5T6RSmV4KD3kowK0QKSV9FY0oh2A0qpO4BXgDlADBANXAuMAbydUKTdwMXVyucJnAvsa87BbK8Xoq07TWsdCMQCGcBrDnwvyagQLSN5FQ2SRrSLU0qFAI8D12utv9VaF2mrDVrrC7XWFUf3U0p9rJTKUkolK6UeVEoZbL8z2H5Ott3F/th23KPvcZHtdzlKqQcaUay5wFilVJjt52nAZuBItWN2VUottB0zWyn1mVIqtNrvDyql7lFKbQZKlFKeSqmLq5XjoeqPwJRSjyqlPrV939l2hX6JUuqQ7fiNKbcQTqe1Lge+Bfoc3aaUmq6U2qCUKlRKpSilHq3+GsmoEM4heRX1kUa06xsF+AA/NbDfa0AIkARMwHpVe5ntd5favibZfh8IvA6glOoDvAlcBMQBEUDHBt6r3Fae82w/Xwx8fMw+CnjGdszeQALw6DH7nA9MB0KBHsAbwIVYr/xDgPgGyjEW6AmcCDyslOrdwP5COJ1Syh+YDayqtrkEa45CsWbiOqXU6bb9JaNCOInkVdRHGtGuLxLI1lqbj25QSq1QSuUra5+t8UopD6zhu892p/og8ALWEIM1RC9qrfdrrYuB+4DzlPWRz9nAPK31Ettd7YcASyPK9TFwse1KeALwY/Vfaq33aq3/0FpXaK2zgBdt+1X3qtY6RWtdZivHXK31Mq21EXgY0A2U4TGtdZnWehOwCRjYiHIL4Sw/KqXygQJgKtbuWQBorRdrrbdorS1a683AF/ybF8moEK1P8ip5bZA0ol1fDhCpqvVx0lqP1lqH2n5nwNrQ9gKSq70umX+vOuPq+J0n1r7VcUBKtWOX2I5bL631MiAKeADrCaOs+u+VUtFKqS+VUmlKqULgU1s5q0up9v2x5ShtRDmOVPu+FOsddiFc1em23PoCNwJ/K6ViAJRSI5RSi5S1O1YB1jEPR/MiGRWi9UleJa8Nkka061sJVACz6tknGzABnaptSwTSbN8fruN3ZqyDJdKxPhYCqh5dRTSybJ8Cd1D7sRPA01ivevtrrYOB/2B9HFVd9avidKo98lJK+TWhHEK4Da11pdb6e6AS6+NTgM+Bn4EErXUI8Bb/5kUyKoSTSF5FfaQR7eK01vnAY8AbSqmzlVJByjpQcBAQYNunEvgaeMr2+07A7VgDCdZHTbcppboopQKxhvErWxeRb4EZSqmxSilvrIMYG/t38SrWx1xL6vhdEFAMFCil4oG7GjjWt8BpSqnRtnI8Su0ThBBuT1nNAsKAHbbNQUCu1rpcKTUcuKDaSySjQjiJ5FXURxrRbkBr/TzWRvHdWO8eZwBvA/cAK2y73YR1sMN+YBnWK+UPbL/7APgEazAPYB3EcJPt2NuAG2z7pwN5QGojy5Wrtf5La11XP6vHgMFY+5P9AnzfwLG22cr0pa0cxUAm1rvwQrQFc5VSxUAh8BRwie3vHuB64HGlVBHWvotfH32RZFQIp5C8igapuv9fCeFctjvm+UB3rfUBJxdHCHEMyagQ7kPy6hhyJ1q4DKXUaUopf6VUAPBfYAtw0LmlEkIcJRkVwn1IXh1PGtHClczCOgjyMNAdOO84j7WEEM4hGRXCfUheHUy6cwghhBBCCNFEcidaCCGEEEKIJpJGtBBCCCGEEE3k2fAuLWNbknotkKa1nnHM7y7FupTm0UVBXtdav1ff8YJCw3VUXEPL0AvRfhzYsSVbax1lj2PZO68gmRXtjyFjz3F/ty+3XPIqhAPUl7uWqC+zDm9EA7dgnaA8+Di//0prfWNjDxYV15GnPptvl4IJ0RZcMDghueG9Gs2ueQXJrGhfAuZMBTof9/ezvtgpeRXCzhrKXUvUl1mHdudQSnUEpgMNXv0KIZxL8ipE8wXMmWqryFuH5FUIq9bM3bEcfSf6Zayr7AXVs89ZSqnxwG7gNq11ioPLJESbYeeTx8tIXoVoEidW4C8jeRXtmDMbz0c57E60UmoGkKm1XlfPbnOBzlrrAcAfwP8d51hXK6XWKqXWFuXlOqC0QrgXe9/1smdebceTzIo2rbXvPFcneRXtnSs0oMGx3TnGADOVUgexrt0+WSn1afUdtNY5Wuuj67i/Bwyp60Ba63e01kO11kODwsIdWGQh2i275dW2r2RWCMeRvArhAhzWnUNrfR9wH4BSaiJwp9b6P9X3UUrFaq3TbT/OxDpAQghxHI66+pa8CtE4rnAHTPIq2jNXyOBRrTE7Rw1KqceBtVrrn4GblVIzATOQC1za2uURwl0448QheRXiX65UeddF8iraMlfMX6s0orXWi4HFtu8frra96mpaCFG31j5xSF6FqMkVK++jJK+iPXDVDMqKhUK4MFc9cQjRXkgGhXAuV85gq3fnEELUz5VPGEK0B5JBIZzPHXIod6KFcCHucNIQoi2TDArhfO6SQ7kTLYQLcJcThhBtlWRQCNfgTlmURrQQTuZOJwwh2hrJnxCuw93yKI1oIZzE3U4WQrQ1kkEhXIc75lEa0UK0Mnc8UQjRVkj+hHA97ppLaUQL0Urc9SQhRFshGRTC9bhzLqURLUQrcOeThBDuTvInhGty92xKI1oIB3L3E4QQ7k4yKITraSu5lEa0EA7SVk4SQrgbyZ4Qrqst5VMa0ULYUVs6OQjhjiSDQriutpZPaUS3MbmZ6ezc8A+BwaH0HTYGD0/5X9xa2trJQTiesaKcLauWYjJW0G/4GAJDwpxdJLcl+ROOprVm39aNZKQepFOPPnTs2tPZRXIrbTGj0sJqI7TWfPfWi/z21Yf0GTqa3Mx0Pnjmfu58+UM6JvVwdvHatLZ4YhCOt2PdKl655zo6JvXA19+f9564hwtve5BJZ5zv7KK5FcmfaA3FBXm8cNsVFORm07lXPz576Ul6njCcG558BU8vb2cXz+W11ZxKI7qN2LhsISv/mMsLP/xNcFgEAIt+/JJX77mO577+E6WUk0sohDiqoqyMV+6+lhuefo3+I8YBcOTQAR69/Ey69T+BhG69nFxCIUR1H//3MRJ79OGSux/HYDBgMlbw8p3X8PNHb3DmVbc6u3jCSQzOLoCwj6XzvmXGRddUNaABJs6aTaXZzMGdW51YsratrV5dC8fauHwhnXv1q2pAA8QkdmHirNksX/Cj8wrmZiR/ojWYjBWsWbiAc66/E4PB2mzy8vbh7OvuYOm875xcOtfXlnMqd6LbiIqyMgKCQ2psU0rhHxRMeWmJk0rVdrXlk4JwvIqy0lp5BQgIDiU347ATSuQ+JHuitVWazVgslfj6BdTYHhAcQkVZqZNK5R7ael7lTnQbMWjsJBb98AUWi6Vq28GdW8lMPUTXfoOcV7A2JmDO1DZ/UhCO12/EWDav/Ju8rCNV24wV5Sz75VsGjZnkxJK5NsmecAZf/wC69h3Esvnf19i+8PvPGTR2spNK5fraQ14dfidaKeUBrAXStNYzjvmdD/AxMATIAWZrrQ86ukxt0YRZs1n1xzyevPpcRk+bRW5GOgu//5xL73kCbx9fZxfP7bWHkwFIXltLeIdYZl1+Iw9dPJMpZ1+EX0Agi3/8koTuvek/aoKzi+dy2kv+mkMy2zouvvNRnr3xIvZv30SX3gPYunope7ds4OH3pTvHsdpTXlvjTvQtwI7j/O4KIE9r3Q14CXiuFcrTJnn7+HLfG58x+YwL2L9tE2aTiQff/ZpRJ890dtHcXns6ISB5bTUzLr6WW59/m4KcLA7t2cnZ193B9U+8UtXnUli1s/w1h2S2FXTu1Y9nvviVsKhodm9cQ7d+J/D0FwuIiI51dtFcSnvLq0PvRCulOgLTgaeA2+vYZRbwqO37b4HXlVJKa60dWa62ytPLm7HTz2Ts9DOdXZQ2ob2dDCSvra9b/xPo1v8EZxfDJbW3/DWHZLZ1hUVFc8aVtzi7GC6rPWbW0bc8XgbuBizH+X08kAKgtTYDBUDEsTsppa5WSq1VSq0tyst1UFGF+Fd7PBlgp7yCZFa0TDvNX3O8jNSxwgW018w67E60UmoGkKm1XqeUmtiSY2mt3wHeAUjqM0CuoIVDtNeTANg3ryCZFU3XnvPXHFLHClfQ3nPryO4cY4CZSqlTAV8gWCn1qdb6P9X2SQMSgFSllCcQgnXwg3Awi8XC8vk/sOLXHzEZjQweP4UpZ/8Hb18/ZxfNKdr7iQDJq8vbv30zv335IZmpyXTq2ZdTLryS6I6dnF2sFpPsNZtk1oUV5uXy25cfsGPtSoLCIph85gUMHD3R2cWyK8muA7tzaK3v01p31Fp3Bs4DFh4TboCfgUts359t20euglvBh88+wG9ffsCEWbOZfvE1bPtnOc/ddDFmk8nZRWtVMmWdleTVtW1avojnb76ExB69OfeGu/ALCOSRS08nbf8eZxetRSR7zSeZdV1F+Xk8cuks8rMzOfPq2zhh7GQ+eOYBfvvyQ2cXzS6k3vxXqy+2opR6HFirtf4ZeB/4RCm1F8jFeiIQDpa2fw9rF/3GSz8txdffOnn8wNETeezyM1n39++MmDLdySVsHXISaJjk1fm01nz+ytNc+9iLVXNI9x4yisCQUL575yVufvYNJ5ew6SR7jiOZdb7fv/qI3oNHcNVDz1dt6z1kJA9dfBoTZp5bVe+6I8luTa3SiNZaLwYW275/uNr2cuCc1iiD+NeujWsYOHpijSAbDAaGTzmVnetXt/lGtJwE6id5dS0lRQVkp6fWehQ8cupp/PLJO84pVDNJ9hxDMutadm74hxkXXV1jW3RCZ6ITOpO8ezs9Bw1zUslaRvJbmyz73Q4Fh0eQmXao1vbM1EOERXZwQolah5wAhDvy9fNHKQMFOVmEVstnZloyIeGRTixZ40n2RHsSUkcdazaZyDly2G0yeyzJcN1kVn83VZCTxR9f/x+/fPI2aQf2Num1g8ZMIjs9rcYy4dv+Wc6q3+cybsbZjiiuU0n/LeFsZpOR1X/+ws8f/o+Nyxdhqaxs9Gs9vbwZN+MsPnz2QcpLSwDIz87ks5eeZMrZFzmqyHYj2RPuaP/2Tcz96E0W/fglpUWFTXrtiWddyM8fvUHq/t2ANf/fvDGH+KQexCR2cURxHUoyfHxyJ9oNrVm4gHcev4vB46fi6x/Ak1efy+QzL+Cc6+5s1Os9vby5+7X/438P3MwP776Ct68fxvIybnz6NSJi4hxc+tYl4RfOlpORztPXnkdYVDRJfQbyzRtz+P6dl7n39U/wDwpu1DHOv+V+Pnj6Pm46dQRRcYlkph3ipHMvZvJZFzq49M0n2RPuyGKx8O7jd7FtzQqGTTqZfds38dVrz3LbC+82uhtG7yGjOPPq23jiynMI7xBDXnYmnXr04canX3Nw6e1Pclw/aUS7mdKiQt55/C4eeOtLOvfqB8BZ19zOg/+ZzsDRE+kxcGijjtMxqQdPf76Awwf2YjJWkNi9NwYPD0cWvVVJ8IWr+Pj5hxl18izOvta6oJzWmnceu5Pv3n6Ri+58tFHH8Pbx5drHXiI/O5OcjMPEJHYhICjEgaVuGcmfcFerfv+ZlL07mfPtQnz8rFO+blj6F/974GZe/nlZo+vJSaefx5hps0jdv5vAkDA6xCc6stgOITlumHTncDObViymx8BhVQ1ogOCwcCafcT6rfp/XpGMppYhP6k7nXv2kAS2EA5iMFWxcsYgZF19TtU0pxWmXXs/qP39p8vFCIzvQte8gl2xAH+02JfkT7mzVH/OYdsGVVQ1ogBPGnYhfQCB7t25s0rG8ff1I6jNQGtBtmDSi3YzWFgwGVWu7wcMDjUz/KYTL0YCqmVmDwYBM1yuE69FaH7eORepYcQxpRLuZAaMnsmP9P6Tu21W1raSogIU/fM7wydOO+7riwnxS9u6kvKy0NYrZ6uQumHBFXt4+9B85jt+++KBqm9aaXz55h2GTTznu68wmE6n7d5OXldEaxWwRyZ1oS4ZNPoVfv/gQk7GiatuW1Uspys+la99Bx31d1uEUDh/c5/YXx5LnppE+0W4mMDiUy+97ksevPJsRU2bgFxDIit9+YtRJp9Fr8Mha+5tNRj7572MsX/AjYVHR5Odkcup/ruL0K25GqdpX2+5IAi9c2cV3PcZT157Pro1rSOozkG3/LKO8rJQH3vqizv1X/PYTn734BD5+/hTl59LrhOFc/cgLBIWGtXLJGybZE23N2FPOYOOyhdw7+ySGn3gqORnpbFy+kFuefRMPz9pNpiOHDvDmw7eRkZqMt7cPXj4+XPngc/QeUrs+dnWS56aTRrQbGj3tdHoOGs7K33/GWFHOnS99UKOPdHVf/28OmYdTeHnuMgJDwshOT+OF268gNKIDk844v5VLbl8SeOEOOsQnMuebP1n953yOpBzglAuvZPD4qXh6edXad8/m9Xz6wuPc+fIHJPUZiLG8jC9fe47X77+R+974zAmlr5tkT7RVBg8Pbnrmf+xcv5pta1bQte9A/nP7QwSHRdTa12wy8dyNF3Hy+Zdz0rmXoAwGNiz9i5fvuoZnv/qNsKgYJ3yC5pFMN480ot1UREwcMy6+tt59Ks1mFv34Bc988RuBIda7WJGx8Vx0xyN88t9H3boRLYEX7sTb149xM85qcL+/vvuE0y65jqQ+A6ted8GtD3Dz9JGkHzpArAvMMSvZE22dUoreQ0Y2eDd588q/CYnswLTzL6/aNnj8FIafeCpL5n7LrMtvdHRR7UIy3XzSiG7DKsrLMJtMteZ+ju3UxS36WtZFwi7asrysDEaeNLPGNk8vLzrEJ5KfleG0RrTkToja8rMz68xkbKcksg6nOKFETSfZbhkZWNiG+QUEEhXXkS2rltTYvmbhr/QY1Lj5pF2FDHYQ7UH3AUNZs/DXGttyMtJJ3b+HTj36OKVMkjsh6tZ9wGA2rVhMRVlZ1TatNWsX/0b3gUOcWLLGkWy3nNyJdnGbVixm3sdvk3HoAAndejHz8hsavWqSUorzbrqPNx++jTOvvpWkPgPYunoZ8z99l/ve/NzBJbcPCblwJ1mHU/jh3VfYtmYFAcGhTDrjPE486z8YDI27X3HS7Et46OKZfPTcQ4yedjrZR1L57u2XmHnpdY1e3dBeJHuirTObjMz/9F2WL/gBY3k5g8adyBlX3lxn/+e6JHTrxYBRE3jmhguZddkN+Pj58/vX/4fZZGJ4PbPvuALJt33InWgX9s9f83n38buYfMb5PPju1wybPI2X7riKHetWNfoYg8dP4bb/vsP2tSv54On7OXLoAA+99w2de/Z1YMntQ0Iu3El+diaPXXEWYR1iuPd/n3LhbQ+ydO63fPHKU40+RnBYBI99+ANe3j58POdhlvz8DbNvuIeZl93gwJLXJtkT7cEbD97C9rUrufLB57njpfexVFby+BVnN2kq2KsfnsO46Wcx7+O3+PLVZ+jcsy/3v/k5nl7eDix588lTXfuSO9EuSmvNN2/8l+sef5m+w8cA1lH+nt4+fP/Oyzzw9peNPlaPgUMbvRy4K5CAC3f0xzcfM3j8FM657k7A2i/y7tf+j1tnjmPGxdcSEhHVqOOERnbgwtsedGRRj0uyJ9qLQ7t3sHvTWl76eRle3j4AXHbvk8y55TKWz/+BE8+6sFHHMXh4cOJZFzZ6f2eSfNufNKJdlKminIzUZPoMG11j+8DRE/nouYecVCrHk5ALd3Vgx5ZaFWlgSBide/bl0N6d9G9kI7q1SeZEe3Rg5xb6DB1d1YA+atCYiRzcudVJpXIcybljSCPaRXn5+BIQHMLhA3uJT+petf3Q7u1ExcY7sWT2J+EWbUFUbEeSd29nyISTqraZTUbSDuwhMrajE0t2fJI90V5FxSVwaM8OtNY1Fh5L3r2DqDjXzGtzSc4dx2GNaKWUL7AE8LG9z7da60eO2edSYA6QZtv0utb6PUeVyZ0opTjlgit446FbGTB6AqWFBQSGhrHq93nMuvwGDu3ewfJff8RYUc7gcSfSb8Q4u6xAWGk2s3bxb2xfu5LgsAjGzTiLDvGJdvhEdZNwuwbJa8tNOecinrx6NqYKI2UlRXj7+pKZeoikPgMJDA7ll0/fITMlmcTuvRlz6hn4+gfY5X0P7tzKyt9+xmQ0MmTCVPoMG93guUBy5/4ksy3Ta/AIDB4evPP4XfgHBVNpMuHj68eahQt4+vNf+eevBWxfu4LA4FDGzTiL6ITOdnnfwrwclsz9hqy0FDr36seoabPw9fO3y7HrIll3rAYHFiqlfJRSFyil7ldKPXz0qxHHrgAma60HAoOAaUqpumYu/0prPcj2JeGu5oTxU8hMO0T6wX3EdenGvq0bKS0uJDfrCM/ccCEeHh6ERUXz8ZxHefvRO9Bat+j9jBXlPHvDf5j/6btEd+xEaVEBD110GhuW/mWnT/QvGdzgOM3MrOS1hWI7JRGd0Jl1i38jKj4BY3k5W1Ytoeegodxz7hQO7d5BXJdubFq5mPvOn0Ze1pEWv+f8T9/l+ZsvwdPbm9DIKD545n4+eOb+es8FkjvXInWscxgMBkZOncHahb/i6elFWFQMq/6YS7f+J/D2Y3cw9//eoEN8ImWlxTx8ySzWLPq14YM2IHn3du45dypp+/cQ16Ubaxf/zoMXTqcgN9sOn6g2ybrjNeZO9E9AAbAOa2gbRVvP4sW2H71sXy1r5bUzn734BGddc1vVakgnn3cZX7z6DHM/fINnv/qdqLgE6/bZl/HgRTPYvPJvBo6e2Oz3W/TDl3h6eXHfm59XTck17MRTee3e63ll3so6lyluDgm2wzU5s5LXllv6y/d4+/jwyPvf4eFpPbWOP+1snrjqXM65/k5OvfAqwJrjz19+im/fepGrHnq+2e+Xk5HOD++9wrNf/UFEdCwAJ82+lPvOn8bO9atrrbYmuXNZUsc6QUFuNj9/9CbPfLGgqi6ddv7l3Hf+yfj4+vPUZ79g8PAAYNRJM3nh9isYNGZSrT7UTfHRsw8y+4a7mXj6eYD1XPDRcw/xw7uvcOk9T7T8Q9lI1ltPYxrRHbXW05pzcKWUB9YTQzfgf1rr1XXsdpZSajywG7hNa+0ey/w4mNlkZPvaldz+Ys0bByfNvpQ/vv6/qtAD+Pj5MWHmOaxf8meLGtEblv7JlHMuqjGnba8ThhMUGs7BnVvp1v+EZh8bJNitqFmZlby2zPolf3DiWf+pakADJPUZSERMHB2TetTYd+q5l/DwJTOPPUSTbF6xmBPGnljVgAbw9Q9g/Glns37Jn1WNaMmdy5M61gm2rFpKv+FjatWlk884n21rV1Y1oAG69T+BiOhY9m3dSK/BI5r1fiVFBSTv3s64GWfX2D713Et47saL7NKIlqy3vsbME71CKdW/OQfXWldqrQcBHYHhSql+x+wyF+istR4A/AH8X13HUUpdrZRaq5RaW5SX25yiuB2lDBg8DBjLy2psLy8prnPhhvLSUrx9mn+FDODl40N5ac35MbXWVJSV4tWCY0vXjVbXrMzaK6/QPjPr5e1Ta37Zo/lRx2S2vLS4RXe0/n2/klrby0tK8PLxkdy5D6ljncDbx4fy0tr5KS0prrVNa22tY319m/1+Hh6eaK0xGWs+bKgoK2lR/XqUZN05jtuIVkptUUptBsYC65VSu5RSm6ttbzStdT6wCJh2zPYcrfXRv6j3gDrXydRav6O1Hqq1HhoUFt6Ut3ZJlspKdm1cw+aVf9cZYgAPT0+Gnzid795+qap/o6Wykm/ffhGtNTvWrazaN+fIYRZ+/xmjp51OcWE+m5YvYt+2jQ32ka40m9mxbhVbVi2hoqyMMdNO55eP36K0qLBqnyVzv8HLx4fE7r2b/DkdVYkn51ew9nAxuWXmqm1mi2ZLRgmbj5RgqrTY/T3dgb0y29K82vZtU5lNP3SADcsWknX4+DfxxpxyBvM/eYfigryqbSt//xljeTkblvxZlcdKs5nv3nmZMaecjqWykp3rV1dlsCFpB/ayYdlCco4cZvD4Kexcv5rdm9ZV/T4z7RB/z/2GEw/PbcGnta/8cjPrDhezP6+8xjnpYH456w4Xk1ctx+2J1LGOU1JUwKbli9i7ZcNx68EBoydyYMeWOurSz0lP3kdxYX7V9hW//ohG07lXfw4f3FeVwYYUF+Sxcfki9m/fhI+fPwNGTeDH91+rKpPZZOK7t63ngpawZz17vLr0aI4PHJPj9q6+7hwzWnJgpVQUYNJa5yul/ICpwHPH7BOrtU63/TgT2NGS93QHybu38/KdV+Pj549/UDCp+3Zx0R2PMm7GWbX2veiOR5hz8yXcO3sqSX0GsmP9KmISunDj06/z0l3X0LXvIPwCAtm8cglnXX0rO9at4qlrz6dLr37kZqbj7ePLbS+8W+fsGnu3buDVe64nMCQUH18/0g7u5bJ7n6L3kFHcfvp4BoyeSHZ6KtmHU7nr1Y/sMvNHSxVWVDJneRqHi4x0DPZhT04ZU5JCGBIXwEurjhDp74kByCgxccuIWAbHBTq7yK2t2ZmVvNatoqyMNx66hd2b1tKpRx/279jM4HEncuWDz9caIzB4/BR2b1rL7WdMYNDoSeRmpnPk0AFuevYNvnztGe6dfRJJfQawfd0q4jp3ZdCYSdw2axz+QcH4+QeQemAPl93zJKNOrt3No6ykmNfuu4GDO7eS2L03+7dvYsSUGVz32Ev899ZL6d5/CN5+fmxZtZTzbrqHxAMft9Y/0XFprfl8Sza/7M6je4Qv6UUmwvw8uWF4NG+vzSSj2Ei8LccndQ3lkkFRLnGeaUVSxzrAb19+yDdvvkCXXv3Iy8rAw9OD2/77LjGJXWrs5+vnz03P/K9WXXrm1beQl5nB7aePZ9DoSeRkHCYjNZmbn32DF267ggM7NldlcNjkU7n8vqdqdOE6au5Hb/LTh/+jS+/+5BxJw9c/kCseeIb3nryHTSsW06lHH7avWUHn3v047ZLrWuufp16bM0p4cWV6jbr05hGx7MguY74tx4eLTET4eXLv2HhC/WSWZNXQFYVS6hOt9UUNbavjdQOwPjrywHrH+2ut9eNKqceBtVrrn5VSz2ANthnIBa7TWu+s77hJfQbopz6b39Dncklmk4nbZ43j3BvvYeypZwCQum8XT15zHve/+Xmdd3u11uxcv4ojhw6S0K0XXfsNQilFeWkJG5ctpKK8jAGjJpC2fw/vPnE3D737DZGx8Witmf/pOyxf8BNPffZLjcrJWF7GrTPHctm9TzJs8imAdZqsZ66/kMf/7ycsWrNr/WqCwsIZOHpis5YvdcQd6OeWpRHm58kVJ3TAw6Aoqqjk0cUppBRUcP/4jgyKsU4Ztj2rlKeXpPL6qUntIuSzvti5TmtdtSRlczLrqLyCe2f2/+Y8QkFOFtc9/lJVd42X77yaHgOHcubVt9b5mozUZHasXUlASGjVQCTr06NVZKQcJKF7LxK69eK2mWO59J4nGH7iqYD1Avvpa8/n0Q9/ILZTUo1jvvP4XVgqK7nywefw9PKirKSY/956GYMnTGXyGRewYdlfmI1GRm14w2X+5pcdKuSrrdk8MTmRUF9PLFrz2eZsFh8sYFRCEJcNsua40JbjU7uHMiUp1NnFdjh75NW2j9Sxx9ixbhVvPHQLD737DR3iE9Fa8/tXH7Hohy945svf6rxIO7YuDYuKBiAj5SA71q0iMDTMtsjZw5hNRq566PmqDL5w2xUMHDOJ0y65tsYxNy1fxEfPPcSD735DRHQsWmvmfvQGaxf/zqMf/sD2NSvISk+lU48+JPUZ0KLPbK+6tthYybVz93PnmLgadekTf6cS7uvJ01MSCbHl+JNNWSTnV/DwxIQGjto2HJvZ6hpztu1b/QfbQIbjPsY9Smu9Gag1Ek1r/XC17+8D7mtEGdqEbf8sIzw6tqoBDdCxa09OPOtCls77lgtvq70SoVKK3kNG0XvIqBrbff0DGHnSaVU/f/byk8y45FoibQuxKKU45cKr+P3rjzm0ZwedevSp2nfDsoUkdOtZ1YAG6NyrH2Onn8nSX77n7GtvJ/aYq/bGclS/rGJjJRuPlPDBrG54GKwnwiAfDy7sH8nr/xypCj1Anyh/hsUHsfRQIaf1dP9Hk83Q5MxKXmuzWCwsmfsNc779q6r/sq+fPxfc8gD/ve3y4zaiozt2IrpjpxrblFL0GTqKPkOtOV6zcAHxXbpXNaABOvXow/jTzmHpL99x7vV3VW03VpSz6ve5NWbI8QsIZPaN9/LuE3dzbvo3VKXORRrQAH/sK+C8fpGE+lrLZFCK8/pFMG93Lqd2C63KcbCPBxf0i+S7HTntohFdB6lj7eTvn79m+kXXVD19VUpx0uxL+f2rjziwYzNJfQbWes2xdelR0Qmdq+aGNpuMrPztJ16eu7xGBs+76V7eeuT2Wo3oRT99xczLb6wa9KuUYsbF1/LHNx+TfnAf/UaMtcvntWd9u+JQEQNi/GvVpUNiA/D2MBBSLcfn94/k8h/3kltmJtyFzjnOUF+f6PuUUkXAAKVUoe2rCMjEOiWPaKKSogJC6lj6NzSyAyWFBS06dmlRIaGRHWpsMxgMhEZE1Tp2aVEhIRE197WWI5qSouaXw5EDG8pMFrw9FL6eNe8khPlZr4yPFe7nSbGx0mHlcUWSWfuyVFZSUVZKcFhEje2hUR1alBOAkqJCQiLrOBdE1T4XmIwVaK0JCAqusT0sqgOlWWm4qhJjJWHHVLBeHgb8vQwYK2tmNkzyKnm1g5KiAkKPqWOVUoTUUQ82hcloxGKpJDA4tMb20Mi6zwWlRYW1ymHw8CA4LKLF546j7F3fFpsqCfOt3SCOCvDi2Bv43h4GArw9KGlnma3LcRvRWutntNZBwBytdbDtK0hrHWG7uhVN1HvIKLavXUF+dmbVNktlJcvn/0D/keNbdOx+w8eydN63NTr8px3YS3ryPpL61rz67jNsNBuXL6Qo/98BUGaTiRW//kj/EeOa9L5HBw86emRwpL8nAV4ebDxSc/aDhQcKMFk0ZaZ/B0BUmC0sO1RY44q6pSwW1x+sKJm1L08vL3oOGsbyBT/W2L5k7jdNzsmx+g4bzeYViymsNhOC2WRi+fwf6T+y5rEDgkKI79Kt1mIPS37+ikExjlvprKUGxgSw8EDNBsPO7DJKTZYag4IBFh0oYKCd8+rqmZW82l//EeNYOu+7GvXgkUMHSNm7k679mj9Fq19AIInderP6z19qbF/6y7d1ngv6jahdH6fs3Ul2ehqdex07gUrTOaK+HRQdwMrUolp16ZLkQgorajaWt2eVUmnRxAU1vavn8bh6Xo/nuPfhlVKDbd9+U+37Klrr9Q4rVRsVFhXN9Iuu4dHLz+CUC64iICiYxT99iY+fX42uFc0x+cwLWL7gB1647XLGnHIGORmHWfDZe5x/ywO1lhSN7tiJyWdcwKOXnc60C67Ax8+fhd9/TmRMHIPGTm70e7bmlDpKKa4c0oGXVh7mlO6hJIb4sPZwCVszS+kd6cdtvx7gzD4RGBT8sCOXhGBvekX6tfh9v9+ew487cymsqMTHU9G/gz/3jovHs45pBp1NMmt/F972IM/fdAlp+/fQrf9gtq9dwT9/zeeBt79q0XGj4hKYcs7FPHrZ6ZxywRX4+gey8PvPCIuKZvC4KbX2/88dj/DSHVdxcOdWuvTuz+alf7Jx0XyenRjdonI40qxe4dz7RzL/XZ7GqIQg0oqMzNuVx/QeYcxZcZgZPcLoHOrD8kNFbMsq5eVpzetCVt3+vHLmLE8jo9gEQGygF3ePiadTWPOnJnMUyav9TZg5m2W/fM+cWy5l7PQzycvKYMFn7zH7xnvwDwxq0bEvvP0hXrz9SpJ3byepzwC2rF7K+r//4OH3vq2175SzL+KJK8/mpTuvYtRJM8lOT2PB5+9x4e0P4e3T/L9FR9a5SeG+DIsL5N4/k5neIwyDgvm78+gV4ce2rBKeWZrK+E7BHCqo4OddedwwLKaqS1ZzGc0WnlmWxvasUirMmhBfD87uE+FW3TCPO7BQKbXI9q0vMBTYBChgANZBC6PqfKGDufOgh6O2rl7GsvnfUV5ayqCxkxl76hnNGrx3rPKyUv7+6Wu2r1lOYGg4k04/77gLpGit2bxiMct//QmTsYIh46cy6uSZdY4yPpYz56M8VFDBb3vzySo10SPcj0mdg7lx/n76dPDDy2BAAxYNm4+U8OHpXfH3bn5/rd/35vPBhkxuGRnLsPhAUgoqeGVVOv5eBp6e0qnhA7SSo4MeJLOOkXU4hT+/+YT0Q/vp2LUnU8+5iLCoGLsce9OKxSxf8CPGinIGj5/C6JNnHXdl0MKHJzB/fzEZ5YougZpTkoJdZhDh8ZQYK/l9Xz47sssI9/Pk5K6hvL8+g5wyM93CfSk3a/w9DaxKLeLywdGc3C202e9VajRzxc/7OalrKGf1iUBrzTfbclh4oIAPTu+Gr6drXPhKXh2roqyMJfO+YevqZQQEhzBx1mx6DKxzTFiTZaQc5M9vPyEjNZnE7n2YcvZ/anWjPKq8tITFP33FjrUrCQqLYNIZ59G176Bmv3dr1Ltaa1anFbP8UBEWrRmdEESF2cJba48wsmMwZWYLvp6KA3kVBHgZeO6kzi16v3v+OIjZAjePiCUuyJvVaUW8tjqda4fGMKlLiH0+lB3UN7CwMbNzfA88orXeYvu5H/Co1vrsel/oIO4ecHfnahO6f7Ipk3WHS3hpWucaI6/v+SOZLqE+XDus+Y2dq+fu4/SeYZza49+r4uxSE9fO2897M7tWDZhytjpG+0tm2xhXy11zFZabufynfbw5I4mogH8vFn7dk8f3O3J5Z2bXZh/7nXUZ7M0p4/lqFbvWmtt/O8jA6AAuPaHuxk5rk7yKpnJm/q+bt49Tu4fVuDucV2bmqp/38dZpXYj0b94NwOxSI9fOPWCtS6vdDPh5Zy4L9ubz5oykel7dulo6O0fPo+EG0FpvVUo1feUN4dZctRLfn1dB/2j/WlMXDYrxZ3NG6XFe1Tilxkr6RNXsChPp70WAl4GDeRUMinWNRnQdJLNtgKtmriUO5lfg72Wo0YAG6NPBn082Z7Xs2HnltfpVK6UYGB3AgfyK47zKJUhexXE5+zxQbLTQ+5h6MMzPkxBfD/blVjS7Eb0np4IQX49aT9N6R/nxzfacZpe3tTXm+dZmpdR7SqmJtq93gSatpiTcm7NDXJ+kMB82HSmttYLShiOldA5t2VKqAV4ebMuq2RDPKjFRYrSQFNbyZVodSDLr5lw5cy3RJcyHMrOFrBJTje3bMksJ8GpZd4ukcF82Hqm5AqzWmo1HSugqeRVuyBXOA4HeHmzPrFkP5paZKSivpFt48/t394jwpaC8staKpduzSgn0do2uV43RmFtplwHXAbfYfl4CvOmwEgmX4ArhbYzZfSP5dW8+b6/N4Nx+kRgUfL8jh5SCCh6dEN+iY5/TL4J312UQ7OPJ8PhAUguNvLo6nV6RvgTX0ZWj0qJZfqiIlalFKAVjEoIYlRCEofVXYZPMuil3yV1zBfl40jvSjyf+TuHmkXEkhnjzT1oxH23M5MrBLRskeUG/KK74eS/vrcvg7D4RaODrbdlklpg4r19kna/Zm1vO73vzyS030yvSj2ndQgn09mhROZpB8ipqcZVzwfn9IvjfmiOE+HowsqN1gPBrq9PpHu5LhH/d4zcaI8Lfi27hPjz+dwo3jYglPsiblalFfL4lmxuO0w2zoNzMr3vz2ZNTRmSAF6d0C6NTC2+WtVSDfaJdjfTXcjxXCW9jpRVW8OzSNA4XmwBNTIAXd46Jp4sdRuT/tDOHH3bkkl9Ria+HgQHR/tw9Nq7W7Bxaa15cmU56kZFTe4Rh0Zp5u/LoEeHH9cPtMwjteOrrr+UKJLMNc7fMtYTFYuG5ZYfZlFFKeaWFUB8PTu8dzum9Ihp+cQOS88p5fkUaR2yzc8QFeXP3mHgSQmpXtMsPFfLOugxm9gwnLsibValF7M4p47mpnQj2cVxXLcmraIirnQ/m7crl2+055JdX4uup6GenWarMFgvPLE1jW2Yp5WZNqG12jhl1zM6RXWri3j+SGRgTwLD4QA7lVzBvdx63jIxlSFxgi8rRkGb1iVZKfa21PlcptQWo1dLWWrdsrUrhclwtuI0VH+zDa9MdMwhhVq8IZjWict+eVcbe3DJeOaUL3h7WE8uYhGBumL+f/bnlJLXgsVdjSWbdj7tmriUMBgP3je/okGN3CvPlf9MbHpxotmjeW5/JA+M70iPCOhXmqIQg/vdPOj/tzOOigbUXwrE3yauoiyueE2b0DK+zYdtSngYDD01o3NLh327PYWxicNUA4ZEdg+gR4cfb6zIYHBtQ55LuraG+y+2jj5ZmtEZBhHO5YnDdyaaMEkYnBFc1oAH8vAyM6hjExoySVmlEI5l1G5I350orNOLraahqQB81sXMIH27M5CIc34hG8iqqkXNC/TYfKeXusXE1tg2M8afUVEl2qbnWYOXWctxGtNY63fbtFGCJ1npP6xRJtBYJrf0EeXuwP6/2DAC5ZeYWD3BsLMmse5DcOV+At4GiikpMlRa8ql345paZCWqlPtGSV3GUnBMaFujjYatP/91WZrZgrNT4tXBQcks0puNXIvC2UqozsA7roIelWuuNDiyXcCAJbNPszytnT045EX6enBAbUOcqTeMSg/lq6wG2ZpbSr4N1OqD16cVszSzlRgf3ia6DZNYFSe5aR0G5mXWHS0DB0LhAgn1qN4oj/b3oHuHLF1uyuXBAFB4GRV6Zma+2ZnN+/7oHITqQ5LUda+/nBbNFsz69mLyySnpG+tI5tO6ntlOTQvh0czbdwv0I9vGg0qL5eGMWg2MDnDEYuEqDjWit9SMASik/4CrgLuBlwHmlFs3W3gPbFJUWzcur0tmeVcqgmABSCo28vyGTRyd2JDqw5tyYoX6eTO4SzGOLU4gJ9KJSQ3aJiTN7hxPQygGXzLoeyV3rWHiggPfWZTAoJgAL8N66DK4dFsP4TsG19j23bwRPLknlz/0FRAd6cTC/goRgH0bEO3aQ0rEkr+1Xez8vpBcZeXRxCuF+nsQFefPV1mwGRPtz04jYWjerJnQK5ueduVzx0166hPpwuMiIUoonJjWuT7WjNNiIVko9CIwBAoENwJ3AUgeXS9hZew9rc8zfk0demZm3ZiRVPfL9YUcOr64+wlMnJtbYN6PYyMIDhcw5qRPFRgsK8PVUPLwohZO7hRHWisszS2Zdg2SudWWWmPhgfQbPn9SJjsHWLlTJ+RXc/1cyfaP8ak3H9cGGTC7oH0W/Dv7klplJCPZmzorDLDxQyEktWH68qSSv7ZOcH+DlVenM6PHvaogVZguPLk7h9335nNI9rMa+v+zJI9zfi4cnJpBaaCQqwJOVKUV8uDGLx5zYkG5MR5IzgQjgT+B74KdqfbmEaLOWJBdyTt+IGn0mT+sZTnJ+ObnHTBC/7FARYzsF0TnUl34d/OnbwZ+u4X4MiwtkZWpRaxddMivaneWHChmTGFzVgAboFOrDiPigWhk8Umwkq8TEtG6hdAr14YTYACIDvDirTzh/Jxe2dtElr6LdySg2WqeErdZY9vE0cFafCJbWkcElyYWc2zeCqAAvTogNoGOwD6f3Cmd3ThmFFeZa+7eWBhvRWuvBWAc+/ANMBbYopZY5umDCfuSKt3lMlRofz5oRMSjwNChMlZaa+1o0Ph614+TjacBU2bpzsUtmnU8y1/qsGaw9XsHHU9XKoKlS4+Vh4NjhDT4eBkwWyatwLDk/gNkCXh6qjgwqjHVksO76WOGhaue7NTXYiFZK9QMuBC4BZgNpwEIHl6vNKykqID87s9Zy1fYSMGdq1Vd9iioqyS9r3FVchdlCdqmJylauZByhsNzM3pwyjGbLcfcZ3jGQX3bn1fh/tDKliBBfTzoEeFFutpBj+/cYER/IkuRCiioqq/bNLzOzIqWIYa3cx1Iya38mYwU5GemYTcZ692tM5lrCWGnNoLkRGbRoTU6piTLT8f/G3YXRbGFvThn55cc/Vw2PD2TJoSIKq2ew3MzyQ9YMVlo02aUmKswW4oO98TRgHYBoY9Ga+XvyGC55dXtaa/KyjlBa3OpPAWuwWCzkZqZTVlIMNHx+KKqorPdv3J2kFVaQVlh7xqqj4oK88PU08E9acdU2rTXz9+RXZbCg3FyV5+HxtevjZYcK6RDo1aKVE1uqMR01n8U6WvhVYI3W2tSYAyulfG2v87G9z7dHB1BU28cH+BgYAuQAs7XWBxtdejeUn53J+0/fx/Y1K/Dw9CQytiOX3vMEPQbabwGrxlTi2aUm3lhzhO2ZZXgYIDbIm2uHxtCtjvmMKy2aTzdn8fu+/KquDbP7RtTqs+QOSo1mHliYQkqBES8PhdaaCZ2CuW54bK19T+8VzkMLU3hwYQrD4wNJKazgn9Ri7h4bx9trM1iSXIiXh8Lbw8BFA6OY3CWE2349wJSkUCq15q/9BZzWM4y4IO86SuJQklk7sVgs/PDuK/z21Yd4enphsVg47dLrOPXCq2pM7u/oO0uVFs1nm7P4rVoGz+0bUeNRaHWrU4v4cEMmpSYLJotmVMcgrhoS7dSpoJrrrTVHWHywAGW745QQ4s0TkxMI9K5ZfXUO9WVK0tEMhqCBP/cXML1HGLtzynlkUQqVFo2xUjOpSwjXDI3hhRWHGZUQRFygF6vTirFomNGj1c9rklc72rzybz7572MU5uVgNhk5YdyJXHbfUwQEhbRqOdb9/QefP/8AJYX5mI0VjOwYyFVDovH3qj1eNLPExBv/HGFndhkGBR2Dvbl2WAxJdlh1t7WtO1zEa6uPUGK7eA/wMnDTiBiGxAXV2E8pxfXDYnh2WRrrDpcQH2xdNbTSojmzdzj3/ZlMcn4FGugW7sslg6L43z9HeHhRCkPjAknOr2Dt4WIemuCYhZsay2HLfitrDROgtS5WSnkBy4BbtNarqu1zPTBAa32tUuo84Ayt9ez6juvOS5JqrXno4tPoO2wMZ1x1C94+vvzz13w+evZBnvpsPhExcQ0fpB6NrcgrLZpbfz3AmMRgzugVjqdBsTS5kA82ZPLqqV0I9a1ZOX2yKYvdOWXcMjKWSH8vDuaV8+yyNP4zMIqxibVHvbuyWxccINTXk1tHxRLq68nunDKe+DuV03uGc1bf2isTmio1K1OK2JVTRoS/J5M7h/DJ5ixKjJVcNyyGEF9PdmaX8fyyNG4dFYuPh4GVKUUoBWMSg+u8KLE3ey0jLJmt7eeP3mDtot+46ZnXiYpL4PDBfbxy9zVMu+BKJp1+HtA6j2Y/25zF9qwybhtlzWByfgXPLkvj/P6RtWae2JNTxpNLUrlzdBz9OvhTbLTw3voMTBbN3WPiHV5We/puew4/7MjhoQkJ9Iz0I7/czKur0skpM/PKKV3qfM2u7LKqDI5OCKKwopI31xzhnrHxdI/wI6/MzP/+SadDoDdn9Q5n0cFC8srM9I7yY2THIDzrmMLSniSvjpO6fzdPXnUu1z7+EgNHT6SspJgvXn2a7MOp3PP6J61Wjv3bNzPn2nO4c1gE/aP9KTFZeH99BmVmzb1ja2aw0qK5cf4BJncJZlavcDyUYvHBAj7ZlMVrpyYRVMcUja6qsNzMVXP3cdGAKKbZLvAX7Mnj081ZvHtaV4J9a9+3zS0zs+hAAbllZnpF+jEwxp9bFhzk3L4RTO0aitbWQf7zdufxyrRO/JNWwp7ccqL8PZnYJaRWe8UR6susw25LaKuj9+m9bF/HtthnAf9n+/5b4ETlrLUbW8GujWuoKCvjvJvuxdfPH4PBwMipMxh18kwW/fhli47dlIp8U0YJ3h4GzusXiY+nAQ+DYmKXEIbGBbLoQEGNfU2Vmt/25nHTCGvlDdA5zJcrBkfz8668FpW5tWWVGEktNHL76Liq4PWI8OPyEzqwYG/dn8XLQzG+czBXDYnmzN4ReBgUK1OKuHFELCG2Y/SK9OP8/pHM251Hz0g/Lj2hA5cM6tAqDWh7kszWpLXm18/f5+pH5hAVZx39Hde5K5fe8yS/vvmUw7tuHGW2aBbsyefmETFVGewU6sOVgzswd1durf1/2Z3Hmb0j6B9tXQo3yMeD64fFsPlIKdmljbrJ6TJ+3ZvH5SdE0zPSurJgqK8nt42KI63QSEZx3V1rqmewe4Qfc3dZl/HubludMMzPkxtHxLL4QAF+XgbO7hPBVUOiGZsY7PAGtD1JXmv785tPmHruJQwaMwmlFP6BQVxy1+Mc2rODtAN7W60cf3z+Dmd0D2JAjDWDgd4eXDs0hm2ZpWSV1MzguvRign08OKdvJN4e1vr4xKRQBkQHsPhgwXHewTV9sjmLHhF+zOhpvTnnaVCc1jOc7uF+fLo5q87XhPt5cpYtg+M6BbMqtZgeEb6c0j0MT4PCy0Mxq1c4cUFerEsvZWKXEK4aEs3pvSNapQHdEIc+21NKeSilNgKZwB9a69XH7BIPpABorc1AAdZRysce52ql1Fql1NqivNqVhrvITk8joVvPWmu8J3bvTXZ6arOO2ZyKPKuk7lX0Oof61Ap4qakSDXQ4ZknNuvZ1dSkFRgK8DbUWX+gc6kN5PX2jq8srNxPq61lrcnd3/Peoi2T2X5VmE4V5OcR36V5je2KP3mQVlrZaOcpMFiq1rjODmXX8zWWVmmrl28fTQEygF9ml7tXfstyk6XTMZwny8SDQ20BKPf0tq7P+e9S8oA319cTfy1Cj/7Q7krzWlJ2eSmL3XjW2eXp5Edele7Pr2GaVI/UgnUNqduPz8TQQG+hF1jEXsk2pj11depGJ7nXcPOoe4Ut6ceM+S1ZJ7fMXWLtr1XW+czaHNqK11pVa60FAR2C4bQBFc47zjtZ6qNZ6aFBYuF3L2JqS+gxgx7pVGMvLamzfuGwhXfsObNKxWnIXrFu4L5uOlNQY0aq1Zl16Md1sd2uOCvLxIMDLg13ZNcu89nBxnWFxZb0ifSkzWUg9pvJdm1ZcdVe5IdEBXhRWVJJeVPMu2LrDJXQP9zvOq9yHZPZfnl7exHfpzpZVS2ps37hsId2igo7zKvsLtF347Twmg9a/udoZ7Bbux7rDxTW25ZSaOFxkpGNwq/fPb5FQPw/WHvNZUgsrKDFZ6BXZuLx1C/et9e9xqKACk0UT4ee8AUn2IHmtKanvQDYuX1RjW3FBHgd2bKFTjz6tVo6ephTWpdf8m8srM5NaaCQxpGYDsVu4LxuPlNQYLGytj0tq1ceubkC0P6tSi7FU6yZs0ZpVqUX0j/Zv1DG6hfuyLr2kxjEqbasaumKb47gtB6XUXGo/GqqitZ7Z2DfRWucrpRYB04Ct1X6VBiQAqUopTyAE6+AHp9u/fRPzPn6b9IP7iOvSjekXXUNSnwEtOmZc564MHD2B52++lLOuuZWAoFD++v4zUg/s4drHX2rUMezx+LhruC/dIvx4emkq5/aNwMfTwPw9eeSXVzI2sWbjwKAUFwyI5LllaVw+uANdQn3ZkF7Ml1uzeXB8R+btzmVpsnUwwPD4QGb2CsfX0zUHL/l7ezI0LpCHF6Zw9dBoOgZ7szq1mG+253D32Mb1R/fxNHCObaWziwdGEW87xvw9eTw7tZODP0H92nNmC/Nymf/pO2xeuQT/wEDGn3YO42acXeupT1Odc/2dvPXI7VxwywN07X8CO9au5OtXnuCe4a3X0FBKcWH/KP674jCXDupAUpi10v1ya3adg2pm9Ajjzt8P4utpYFynYDJLTHy0MZOZPcM5kFfO3F15ZJaYSArz5cw+4TXmVXY1lw6K4rllh/E0wIiOQaQVGnlnXQZD4wJqDSw8nrP6RPDgX4fwMCiGxgVyqKCC/9uYyXn9IvGqY0q81tKe82qxWFgy9xuWzvuWspISBo6ewPSLriYwpGWDOqecfREPXjidL159hnHTzyQvO5Ov//c8E2edS2hkBzuVvn4Bc6baMpiMv5eBcYnBZJWY+HhTFtN7hNV6itkjwpeOwd48+XcKs/tF4uNh4KdduZQYK+kR7ss76zLYnllKiK8np3QPZWTH1ruAb6oz+4Tzy+48nl2Wxjl9rA88vtmWQ6nJwpm9G3fOHBoXyPc7cnlpZToze4Zh0fD9jhzCfD0b3RBvTccdWKiUmlDfC7XWf9d7YKWiAJMt3H7A78BzWut51fa5AehfbdDDmVrrc+s7bmsMeti+diWv3nMdZ1x1Cz0GDmX3prX88O4r3PL8W/QeMrJFx640m/ntyw9ZNv97KspKGTRmEjMvv5GQ8MgGX2vP/pemSs3cXbksSS7EWGltAJ/VJ6LOQQwrUgr53z9HiPTzosRUSZCPB5nFJnpG+mGs1JzVJwJvD8W83Xnklpl56sREl+5b+MH6jKrPHeLrwaWDOjCiiSempcmFzN+TVzUY4py+EU5rjBwd9NBeM1taXMTDF8+k1+ARjD/tHIryc/ju7ZfpO2wUF972UIuOHTBnKpuOlPDD/nIOF1aQGOLD2d39G30X1J7WHS7mp525ZJSY6BLqw9l9I+vsd59bZub2X60DaIuNlfh6Gig2VtI/2p+tmWVcOCCSLqG+rE8vZu6uPJ6YnFiry4QrWZNWxAcbMskvr8THQzGuUzBXDI5u0jGS8yv4dnsOu3PKiPT3ZEaPcEYlOKcx0t7zCvB/cx5hz+Z1nHHlLQSFhrP4py/Zu2U9j//fz/j6B7To2DlHDvPjB6+xdfUyAoJDmHT6+Uw643wMBsff3KleR2cUG/l6Ww7bMksJ9vHg5G6hTO4SUueF/XPLUjmQZ31CatHg7WHtT1xYUcn4TsGMSQziSLGJz7dkM61bKLN6ue7TgvxyM3OWp3EwvwKFdfzGXWPim9R/udRUyQ87clmRUoTBNkj/jF7hteaJbi31DSx05OwcA7AOaPDA2m3ka63140qpx4G1WuufbVP0fAKcAOQC52mt99d33NYI+KOXn8m08y9n5NQZVdtW/PYTf379MQ+//51D37suzpyYXWvN9b/s54bhsfTr8O9V4IcbMlh8sJD3ZnatmnLLojX3/XmImT3DGONms3a4MzuO9nfLzC747D12b17HLc+9WbWtuCCP22aN47mv/yC8Q+3pCxvizoshfLwxk3KzhauHxlRtO1Js5JYFB3hkYgJ9ov7N8U87c9mdU8ZdbjZrhztr73nNTk/j/gum8fLPy/EP+reeePGOK+k3fCwnzb7UYe/tKC05X+zNLee5Zam8MT2pRl169c/7GB4fxNVD/71gzCg2cvtvB3lvZje3nK7SXdWX2QYvDZRS3YFngD5A1W0PrXVSfa/TWm/GGtxjtz9c7fty4JyGytCatNbs2byOB9+uOVvG0Ikn8+ZDt7Z6eZxdmRcZKykor6RvVM07b/5eHgyODaixJLZBKYbHB7Iru0wa0U7U3jK7Z8t6hkw4qca2wJAwegwYyv7tm5vUiHZ23uxhV0455xwzXePRLla9j7mDPqJjYJ0zfIjW097yum/bRnoNHlGjAQ0wZMJJbPtnuds1olt6ztiVXcYJsYG16lJvDwMjE2ou/BMd6E2HAC9SCyuqZpsRztWY++sfAo8ALwGTgMtw8IBEZ1JKERYVzeED+0js0btq++GD+wiLiqnnlfbjShW5n63yzS0z11gVyGzRHCqoPcVUaqGxzpG1olW1q8yGRUWTnryvxjaLxcLh5H2Ed2hcZl0pcy0V7udJWqGRQTH/Phb3NEClBfLKKwn3+/e0n1poJNyJq30JoD3m9eB+tNY1ujZY69imddNxNnucN8L9PFmRUljn71ILKhgQ/W+OK8wWskrMLjG1m7BqTFD9tNZ/Ye36kay1fhSY7thiOdfJsy/jg2fuJy/rCAB5WUf48JkHOGn2JQ5/b1erzL08DExJCuGNNUeqpoNKLzKyMqWQnFIT83bnUmnRaK1ZfqiQdYeLmdDZNe5CG80Wvticxb1/JDNnedpx55VtjmJjJfNtk8ivTSt2taXQ21VmJ59xAX99+ynb1qxAa42xopxv3/wvQSFhdOnd8GBgV8tcS53aPZRvt+ewJ8c6m0e52cInm7LpEODJG2uOVC1Nn15k5MMNmUzvHurE0ta0Jq2IhxYe4sG/DrE61X5LNldaNKtTi/hkUxYL9uRRYnSpqe3aVV67DxiCl48v3739EiZjBVprtqxeyt8/fcXEM853dvEazV7njaFxgWSVmGvVpfnlZr7Znsu+3HLAOtXle+sz6NvBj6gA17jwzSg28t/ladz7RzKfbs7E2MipYhsjOb+Cr7Zm8/W27FozarmSxlzOVCilDMAepdSNWEf7BjbwGrc2/eJrKCkq4K6zpxASHkFBbg5Tz7mIU/9ztcPe05Ur8osGduD9DRlcO3cfIb4eFFVYOLNPOCPiA3l19RG+3pqDh0ER6G3ggfEdXeIquajCzE3zDxDg5cG4TsEczC/nxvkHuHZoNCcmhbbo2HtsqxwOjAkgPsibz7dk8dOuXB4c39FpAx+O0a4yG5/UnWsee5F3Hr8LbbFQXlpM176DuO2Fd+udncOVM9cSvaP8uXRQFM8sTcPLQ1FUUcmgmACeOjGRL7fmcE21HJ/TN4IJnVt3OeTjeervFLZkljElKQSl4OVV6fSOzOfhiQktOm6ZycLjf6dgsmiGxgWyJbOUr7Zm88jEBLq4xrLK7SqvSinufPl93n70Dm44eSi+fgF4eHpy3RMvE5tY9yqUrsae5w4vD8UjExN4dXV6VV0a5O3BIxMTSC8y8uSSVLyP5jg2gJtHNH2MhyMsOVjA6/8cYVBMAINiA1iWXMgfewt47dQuda5M2BTfbMtm3u48JnYOoVJr7v/zEOf0jeC0nq43oLLBgYVKqWHADiAUeALrFDnPV19atDW15pKk5aUl5Bw5TERMXItHDNfHXSrzYmMl+eVmovy9ajQWs0pMmC2amECvFk8pZi+PL07BojUPTUjAwzZTyJLkQt5ee4RPzujW7JHaWmtuWXCQc/pGMM623HKlRfPMsjT6RPlxZu/aS4c72rGDHtprZi0WCxkpB/H1D6j3sbC75K2lKi2a9GIjQd4eNeZBP16OnWnTkRKeXmodXHW021humZnr5+3nzjGxDI1r/kwaX27N5lB+BXeOicNgOz/9tT+fBXvz+e9Jne1R/CaRvP4rL+sI5aWlRCd0bpXZM+zBkeePuupSs0VzpI4cO5PFYuHiH/ZyxeBoJnWxXoRbtOapJamYLZrHJiU2+9iHCip4aOEhXp7WhTBb17OsEhO3/XqQF07uRHRg689136KBhVrrNQC2K+Wbtdb2e8bm4nz9A4hP6t7wjs3kbpV5oLdHrTkuAZd5tFTdvrxy7hgVV9WABhibGMTba4+wJbOMgTHNuyg6Umyi2FhzPm0Pg2JmjzA+3ZzllEb0sdprZg0GA7Gdjj8Wy93y1lIeBlXntIvHy7Ezzd2Vy7hOwTXGXYT7eTKxSzDzduW3qBG9KqWIa4fFVDWgASZ2DuHDDVnklJpqvKcztNe8Aq02zsheHH0Oqasu9TxOjp1pV045lZoaXTcNSnFm7wieXZbWomOvSiliQqfgqgY0WP9dRicEsTq1mJkuNr1fg5d+SqmhSqktwGZgi1Jqk1JqiOOL1ra1twq9tSmg8piHLFpbv1py882grFfcxz6/MWtdo5J2JslsbZI312ZQqs5xBZUWTUtvUBoUtY6tAY1rZFby6h7kHPIvD4Oy1oPHRLZSa1qaKINStepusN6Rd8WHFY0p0gfA9VrrzlrrzsANWEcTC+Gyekb68fXW7BpLm/+1Px8Pg6Jvh+Z3zekQ4EWEvxd/7S+o2maq1PywI5exnVxmJSnJrHArZ/YOZ/mhItKL/h38m1FsZMnBQs5o4Z2nMYnBfL8jp0ZD+te9eSSG+NS42+VEklfhVnpE+OHjYeC3vXlV28wWzddbc+gR0bJxBqMTg1iSXFhjIoC0QiOr04oY5YKrNTbmDFKptV569Aet9TKllNmBZWqz5Eq29dw2Ko6b5u/n6rn7GNkxkOT8CvbmlnPH6MYt7308SiluGRnLo4tTWJFSRHyQN2sPF9Mp1Idp3Vq2ZK0dSWZtJHPuoVeUP6MTgrh5wQFGdgzEoBQrU4oYER9I/+iWjUeZ2TOMbVml3DT/AIPjAkgpqOBwkZFHJza/36adSV5dmJxD6nbzyBjmLD/M0kNFdA71YXVqMQYFr55S7/TmDYoL8ua8fpHc8dtBRnQMwqI1/6QVc/kJHZze9aoujWlE/62Uehv4AutTsNnAYqXUYACt9XoHlq/NkCC2Ll9PA2/PSOL3fQVVjdz7xsUT5NPyO0+JIT68OT2J1anF5JaZuGVkLL0i/VxmUCWSWUAy525uHRXHjB5lfL8jFzOaJyYn0tMOy6t7eRh4aHxHtmeVsSenjJ4RfozoGIi3h8s8G5a8uig5hxzf0Lgg3p/Vlc+35JBZbOTsvuFM6xpqlwGi03uEMTw+kNVpRSgU/xkQ5ZINaGhcI3qg7b+PHLP9BKyBn2zXErUxEkLnMRgMTOsexrTu9r9D7ONpYLyLzIddh3abWcmbe+sW4cfdY+2/BLlSir4d/Onbwb/hnVtfu82rK5NzScMCvT25eohjFsiJCvBiRg/XGkRYl8bMzjGpNQrSFkkIhTO0x8xK1oS7ao95dXVyPhGN1WAjWikVDTwNxGmtT1FK9QFGaa3fd3jp3JCETzhbe8usZE64s/aWV1cn5xPRFI3pzvER1pHCD9h+3g18BUjAjyHhszqYV87nW7LZnlVKiK8n07qFMr1HmEtMJ9VOfEQ7yKzkzT5KTZV8tTWHpYcKqbRoRnQM4oL+kS6x8mg78RHtIK/uwF3OKQsPFPDzzlyySk10DfPlvP6R9Ilyya5KbV5jeoBHaq2/BiwAWmszUOnQUrmZgDlT3SZ8jna4yMjDi1IYEOPP66cmcdOIWJYmF/LJpixnF609afOZlbzZh7atMpZXZubxSQk8P7UT3gbFg38dwlhpcXbx2os2n1d34C7nlF925/Hd9hwuO6EDb0xPYmKXEJ5dmsau7DJnF61dasythhKlVATWAQ4opUYCBfW/pP1wl+C1lp935TKte2jVgIBQP0/uH9+R6+ft56w+ES63Ulob1SYzK1mzv62ZpRRWVPLE5MSqJ0VXDonmkUWHWHGoiIm2JX2FQ7XJvLoDdzunmC2ab7Zl8/jkRBJDrKsYTu4SgqlS8932HO4f39HJJWx/GtOIvh34GeiqlFoORAFnO7RUbsDdwtdakvMruHBAVI1tob6exAR5c7jISI+Ilk9ZJRrU5jIreXOMg/kV9O/gX6ur1YDoAA7kVzDROcVqb9pcXt2BO55TCsrNaKhqQB81INqfb7fnOKdQ7VxjZudYr5SaAPTEupryLq21yeElc2HuGL7WEhPozZ6cMvpVm0qq1FRJRrGRDi46z2Nb05YyK1lzrNggb5YkF9bavie3nBNiWrbIiWictpRXd+Gu55VgHw8qLZrMEhMdAv6tT/fklhMXJPWrMxy3Ea2UGgakaK2PaK3NSqkhwFlAslLqUa11bn0HVkolAB8D0VgfU72jtX7lmH0mAj8BB2ybvtdaP97cD+NI7hq61nZazzAeXZRCbJA3w+ICyS0z8/a6DEZ2DCLUQUvsaq3ZnFHK8kNFWNCM6hjE4NgAuyx+UmaysPBAAbtzyoj092Jq1xBiAr3tUGr7a0lmXTGvkjnHOyEmgI83ZvF/GzM5u08EngbFgj157Mkp4+YRMQ573yPFRv7YV0B2qYkeEX5M6hKMv1fLu3pprdmQXsKK1CIUMCYhmIEx/q60EFIVqWOdw53PK14eBqb3COPFFYe5cUQM8UHebM0s5aMNmdzowLyWmy0sOlDAzuwywv08OalrKLFB9qkHDxcZ+WNfPrllZnpF+jGpSwi+ni6zEFKD6ivp24ARQCk1HngWa2ALgHcacWwzcIfWug8wErjBNnXPsZZqrQfZvlwy3O4cutaWFObLHaPj+HpbDrO/2c0tCw4QH+TNtUMdMyE7wEcbs3hr7RHig73pFOLDBxsyeXttRouPW1hRyV2/H2RzRgn9o/0xWzR3/Z7MpiMldii1Q7Qksy6R16ODdCVzrcPDoHhsUgIZJSYu/XEvF363h82ZpTw5OdEujdq6bM4o4a7fkzFbNP2j/at+Lqxo+UrX76zL4P0NmXQK8SEh2Ie312Xw4YZMO5TaIaSObWVt4bwyu18kg+MCuP/PQ5zz9W7eWpvBFYM7MDg20CHvV2ys5O7fk1mXXkJ/2xPmu/9IZn16cYuPvT69mHv+SAagf7Q/69JLuPv3ZIqN7jOutr5bgx7VroRnY73K/Q74Tim1saEDa63TgXTb90VKqR1APLC9ZUVuPW0hcM4wMCaAF2MCqDBb8DQoPAyOuwuUnF/B3wcLeH16UtWgxROTQrhlwQH25JTRvQV9sH/YkUPvKD9uGB5btW1AtD9vr83gf9O7uOLdrWZn1tl5law5T5ifJ3ePicds0Wit8XLgcthaa95em8EtI2IZGm+t9KckhfLGmiP8sCOXSwZ1aPax9+SUsSatmFdP7VJ1ATA5KYSb5h9gclIInUN97fIZ7Kjd17Gtqa2cYwxKcW7fSM7uE4GxUuPjoRxaF/20M5eu4T7cPCK26n0GxgTw5pojvDkjoNlT11q05q01Gdw5Oo6Btq5jJ3YJ4dXV6fy8K5cL+kc1cATXUN/Z0kMpdbSRfSKwsNrvmvRcXinVGesSpqvr+PUopdQmpdQCpVTfphzXkdpK4JzJx9Pg0AY0WK9kRycE1Zj1w9/Lg7GJwaw73LI7xuvTS5jaNbTGtsGxAZSZLWSUuGSXRbtktrXzKllzDZ4G5dAGNEBmiYkSk4UhcTX7W09NCrFLXscm1uwWEujtweiEINa38NgO0q7r2NbUFs8xBqXw9TQ4/GbO+vQSTuoaWuN9Bkb7Y9HWrhjNdfS1A6L/HT+llGJq11BXzWud6gvqF8DfSqlsoAxYCqCU6kYTpt9RSgUC3wG3aq2PHcGyHuiktS5WSp0K/Ah0r+MYVwNXA0TGxDf2rZusLQatrfP1NFBkrD2fbZGxktDglvXB9vU0UFxR87GSyaKpMFvwdXBjo5lanFl75NV2nAYzK3lrf3w9DRjNFusdNM9/K+ViowVfr5Zlys/TQGYdF7fFxko6BrvkOIZ2V8e2NjnHtJyvp4HCY+pBs8U6XqglfZd9PQ2UmS1Uaqh2KqCoorJt9InWWj8F3IF1NaWxWmtd7TU3NebgSikvrOH+TGv9fR3vUai1LrZ9Px/wUkpF1rHfO1rroVrroUFh4Y156yaTsP0rp9TEskOFbM4oodKiG36BE41JDGJDejE7q000vy+3nFUpRYxNDGrRsSd3CeHLrdmUmqwnEK0132zLoWekn8MGSbZESzNrr7zafl9vZiVv9mOstLAmrZiVKUUu35cwxNeTXlF+fL0th6N/nqWmSr7Yms2kzsEtOvaYxCBWpRaxJ+ffc8Gu7DLWHbY+rXI17a2ObW2ueo7RWrM7p4ylyYWkFlY4uzgNmtwlmK+35VBi/Lce/H5HDl3CfIhswYxbkf5edA714fvt/54LSoyVfLUth8luND99vS0BrfWqOrbtbsyBlfXe//vADq31i8fZJwbI0FprpdRwrCePVp3s0FWD5gxaaz7fks38PXn07eBPVomJCrPmwQkdibPTSFx7C/bx5LZRcTz5dypdwnwwKGsj+obhsUS0cEq9qV1DOJBfztU/76dfBz9SCo34eBp4YJzr3qlpbmZbI6+SNfvbklHCnOWHSQjxxsfDwGur07lqSDSTXLgSunF4LE8tSWVVahEJIT5syyxldEIQJ3cLbdFxI/y9uGlELI8tTiEpzBcNHMir4NaRcYS46BLm7aGOdQZXPdcUVlTyzFLrCqFdwnx5d10GJ8QGcNOIWDwd3PWxuSZ3CWF/XgXXzN1Hvw7+pBUZ8VCKBye0fGGXW0bG8uTfqSw7VERckBdbM0uZ2DmESV1adkHdmhx5ZhkDXARsqTZI4n4gEUBr/RbWCeWvU0qZsT7OOq/a1bhoZf+kFbMipYg3ZyQR7GP90/hldx5zlqfx4smdXXEgHQBD4gJ5f1ZXNmWUoLV1oQi/Fj4aBmufs2uHxnBGr3D25JYT4edJr0g/l/13aCHJq5spM1l4fvlh7hodxwDbwJyUggru/+sQPSL8iHfNLgyE+3ny35M6sTO7jJwyM5cOirLbtJEjOwYxMDqAzRnWPpWDYgLwcaNHw00kmXUz767LoEuYL0+d2AGDUlSYLTy1JJWfd+ZyZp8IZxevTkoprhoSzaxe4ezOsU5x19tO9WCkvxcvTevMjuwycsvMXD44usb81+7AYY1orfUyrBPH17fP68DrjipDQ1z1atVZFh4o4KzeEVUNaIBTuofy485cDhUY6RTqU8+rncvH08DweMc8so0O9CbaReeGthdH51WyZn9r0orpGeFb1YAGSAjxYXKXEJYkF3C+C49uV0rRO8q/4R2bwc/LwIiOrtd9w97coY51Blc915SbLfyTVswHs7pWzWjh42ngwgFRvLY63WUb0Ud1CPBySANXKUUfB50LWoNrPuNyMFcNmbOVmy0E+tS8a2NQikBv6wAAIZrDkLEH6OzsYrQ55WZLjVlpjgr0NlBU4dp9o4VwBFeu202VGgW1Bs0F+XhQLvWr22qzz7mOx5VD5myDYwP5c18B1Z/27cmxPmbpGua6d6GFaI8Gxviz9nAx+eX/LlJSYbaw+GAhQ+Ics/CCEK7K1ev2IB8PEkK8WX6oqMb23/fmM1jy6rbaxZ1oVw+Xq5jWLZTlhwp5ZFEKYzsFk1Vi4re9+VwzNNrh88cKIZomOtCb03qGc9fvBzmlWxg+ngb+2JdPt3DfGnOvCtHWuUsdf9WQaJ78O5VdOWV0CfNhY3oJe3LLefrERGcXTTRTm29Eu0u4XIGPp4EnT0xkSXIh2zJLCfbx5MkTE0kMkbvQQrii2f0i6dfBn6XJhZgsmvP7RzIsPrCtDn4VohZ3quN7RPjxwsmd+WN/PlszSuke4ce1w2Lq7JYl3EObbUS7U7BcibeHgSlJoUxJCnV2UYQQjdC3gz99O8idZ9G+uGsdHxXg5TZLWouGtcln9O4aLiGEEELUT+p44SrazJ1oCZUQQgjRtkldL1xJm2hES6jaj13ZZczfk0d2qYlu4X6c1jOsRUuPCiEcJ7/MzNzdeezMLiPM14NTuodJ1xPRbFLXO1alRbPoQAHLU4qwaBiVEMiUpFCXXU3RFbh1d46AOVMlVO3IypQinlmaSrdwX87tG0ml1tz1ezIZxUZnF00IcYzcMjN3/XGQEmMl5/SNoE8Hf15YcZiFBwqcXTThhqSudyytNS+vSueP/QVM7RrCtG6hLE0u4vnlacgil8fntneiJVDti0VrPtyQyd1j46tWNxoYE4Cfp4Hvtudy/fAYJ5dQCFHdTztzGR4fxFVDogHrEty9Iv14fHEK4xKD8fKQu1uicaS+d7zdOeXszinjtVO74G2b0nZoXCC3/nqALRmlNVZGFf9yuzvRhow9Eqh2KLvUjMmiay0POiYxiK2ZpU4qlRDieLZmljI2seby20lhvvh7eZBWVOGkUgl3I/V969iWWcrw+MCqBjSAl4diVEIQW7Okjj0et70TLdqXAC8D5WYLJcZKAqrNqZlZYiLEV+bYFMLVhPh6kFlione12bwqzBYKK8wE+0jVI+onjefWFeLrwa6cslrbM4tN9Iz0c0KJ3IPb3YkW7VOAtwfD4wN5b30mFWYLANmlJj7dlM20bqHOLZwQopZp3UL5Ykt21ZgFU6WFjzZm0qeDP+F+0ogWdZOxTs4xKiGIndllLD9UiNYarTVr0orZcKSEcZ2CnV08lyVnMuE2rhkazaurj3DFT/uICfTicJGR03uFM14CLoTLGR4fRHqRidt/O0h0oDfZJSa6R/hy68g4ZxdNuChpPDuPv5cHD47vyIsr0/l0cxYKRaXW3DcunmAfedp7PNKIFm7D38uDe8fGk1ViIrfMTMdg7xpdO4QQrmVWr3Cmdg0hpcBIqK8H0YHezi6ScFHSgHa+7hF+/G96Fw7mV6A1dAnzwaBkAHB9pBEt3E5UgBdRATI3tBDuwN/LQ/pUinpJA9p1GJQiKczX2cVwG9InWgghhBBCiCaSRrQbKDVVklJQQbltQJ0QwnVVWjSphRXkl5mdXRQhXJ4r3IXOLjVxuMgoi4qIJnNYdw6lVALwMRANaOAdrfUrx+yjgFeAU4FS4FKt9XpHlcndVFo0H23M5K/9BYT4elJYYWZmz3DO7RuBkn5Kwo4kr/axMqWI99Zn4KEUJaZK+kT5c9OIGJnSTdhdW8issxvQGcVGXl2dzqECI14eCl8PA9cNi6Z/tCwsIhrHkWd2M3CH1nq9UioIWKeU+kNrvb3aPqcA3W1fI4A3bf8VwJdbszmYX8EbM5II9fUkq8TEM0tTCfbx4JTuYc4unmhbJK8ttC+3nLfWHuHesfH0jvKnwmzh081ZzFl+mCcmJzq7eKLtcdvMOrvxDNabVI//ncqJSSE8NikcDwXrDpfw/PLDvHhyZxl3IxrFYd05tNbpR694tdZFwA4g/pjdZgEfa6tVQKhSKtZRZXInFq1ZsCef64fFEOprvdaJCvDiqiHR/LI7z8mlE22N5LXlft2bx6ye4fS2rarp42ng0kEdSCs0cqhAVugT9uWumXWFBjTAloxSfD0NnNk7Ak+DQinF0PhAxiUGsfBAgbOLJ9xEq/SJVkp1Bk4AVh/zq3ggpdrPqdQ+CaCUuloptVYptbawvH30MzRbNGVmC9GBNa+G44O9yZW+lsKBWppX2zHaXWZzy8zEB9ecws3DoIgN8pLMCoeyZx1blJfrsHK6SgMabHkNqj3lYnywDzmSV9FIDm9EK6UCge+AW7XWhc05htb6Ha31UK310GDf9tG30NvDQEKwN+vTS2psX51aLNNFCYexR16hfWa2Z6Qfq1OLa2zLKzNzIK9CpowSDmPvOjYoLNy+BbRxpQY0QI9IXzZllFStgAugtWZ1ahG9IqSOFY3j0NpNKeWFNdyfaa2/r2OXNCCh2s8dbdsEcNHAKF5dnc55/SLpEeHH5owSvt+Ry0MTOjq7aKINkry2zLRuYdz1+0HeXnuEiZ1DyCkz8eWWHGb2DJcVv4RDuEtmXa0BDdAx2IehcYE8vCiFc/pG4O9lYMGefIqNlYztFOTs4gk34bA70bZRwe8DO7TWLx5nt5+Bi5XVSKBAa53uqDK5myFxgdw3riObM0p5bXU6B/MreGJSAj3kKlnYmeS15YJ9PHhuSie8PQy8tfYIC/bkc3bfCGb3i3B20UQb5C6ZdcUG9FHXD4thUpdgvtuew3vrM4kL8uKJyYl4e8jsv6JxHHknegxwEbBFKbXRtu1+IBFAa/0WMB/r1Dt7sU6/c5kDy+OWekX60Wtsnd1OhbAnyasdhPp5ctkJHZxdDNE+uHxmXbkBDdYxC9O6hTGtm8x2JZrHYY1orfUyoN7JjLV1ZvMbHFUGIUTjSF6FcC+unFlXbzwLYS/yzEIIIYQQdiENaNGeSCNaCCGEEC0mDWjR3kgjWgghhBAtIg1o0R5JI1oIIYQQzSYNaNFeSSNaCCGEEM0iDWjRnkkjWgghhBBNJg1o0d5JI1oIIYQQTSINaCGkES2EEEIIIUSTOXLFQiGEEEK0IXIHWoh/yZ1oIYQQQjRIGtBC1CR3ooUQQghxXNJ4FqJucidaCCGEEHWSBrQQxyeNaCGEEELUIg1oIeonjWghhBBC1CANaCEaJo1oIYQQQgBgyNgjDWghGkka0UIIIYQQQjSRNKKFEEIIIYRoIoc1opVSHyilMpVSW4/z+4lKqQKl1Ebb18OOKosQomGSWSHch+RVCOdz5DzRHwGvAx/Xs89SrfUMB5ZBCNF4HyGZFcJdfITkVQinctidaK31EiDXUccXQtiXZFYI9yF5FcL5nN0nepRSapNSaoFSqq+TyyKEaJhkVgj3IXkVwoGU1tpxB1eqMzBPa92vjt8FAxatdbFS6lTgFa119+Mc52rgatuPPYFddewWCWTbpeCuqa1/PpDP2FydtNZR9jhQK2ZW/l+7v7b++UDyWl1b///d1j8ftP3P6KjPd9zMOq0RXce+B4GhWutm/QMopdZqrYc257XuoK1/PpDP6ApaK7Ou/u9gD239M7b1zweu/xmljrWftv75oO1/Rmd8Pqd151BKxSillO374bay5DirPEKI+klmhXAfklchHM9hs3Mopb4AJgKRSqlU4BHAC0Br/RZwNnCdUsoMlAHnaUfeFhdC1EsyK4T7kLwK4XwOa0Rrrc9v4PevY52ex17eseOxXFFb/3wgn9GpWjmzLvvvYEdt/TO29c8HLvwZpY61u7b++aDtf8ZW/3wO7RMthBBCCCFEW+TsKe6EEEIIIYRwO22iEa2U8lBKbVBKzXN2WRxBKXVQKbXFtnTrWmeXxxGUUqFKqW+VUjuVUjuUUqOcXSZ7UUr1rLb07kalVKFS6lZnl8uZ2nJmJa/uTzJbU1vOK7T9zEpeHceRy363pluAHUCwswviQJOaOzWRm3gF+FVrfbZSyhvwd3aB7EVrvQsYBNbKCEgDfnBmmVxAW8+s5NWNSWZraet5hbadWcmrg7j9nWilVEdgOvCes8simkcpFQKMB94H0Fobtdb5Ti2U45wI7NNaJzu7IM4imXVv7Syv0M4zK3l1b5JXx3L7RjTwMnA3YHFyORxJA78rpdbZVpZqa7oAWcCHtkeG7ymlApxdKAc5D/jC2YVwspdp25mVvLYt7T2zL9O28wptO7OSVwdy60a0UmoGkKm1XufssjjYWK31YOAU4Aal1HhnF8jOPIHBwJta6xOAEuBe5xbJ/myP0WYC3zi7LM7STjIreW0j2ntm20leoW1nVvLqQG7diAbGADNty5l+CUxWSn3q3CLZn9Y6zfbfTKz9fIY7t0R2lwqkaq1X237+Fmvo25pTgPVa6wxnF8SJ2nxmJa9tSnvPbJvPK7T5zEpeHcitG9Fa6/u01h211p2x3sJfqLX+j5OLZVdKqQClVNDR74GTgK3OLZV9aa2PAClKqZ62TScC251YJEc5n/b9WLjNZ1by2ua068y29bxC28+s5NWx2srsHG1ZNPCDUgqs/78+11r/6twiOcRNwGe2xzH7gcucXB67sp2cpwLXOLsswqEkr22EZLbdaA+Zlbw66n1lxUIhhBBCCCGaxq27cwghhBBCCOEM0ogWQgghhBCiiaQRLYQQQgghRBNJI1oIIYQQQogmkka0EEIIIYQQTSSNaDenlHpAKbVNKbVZKbVRKTXCzsefqJSa19jtdni/05VSfar9vFgpNdTe7yOEs0hmhXAfkldRH5kn2o0ppUYBM4DBWusKpVQk4O3kYrXU6cA82u5k8KIdk8wK4T4kr6IhcifavcUC2VrrCgCtdbbW+jCAUmqIUupvpdQ6pdRvSqlY2/bFSqlXbFfUW5VSw23bhyulViqlNiilVlRb3ahBthWfPlBK/WN7/Szb9kuVUt8rpX5VSu1RSj1f7TVXKKV2217zrlLqdaXUaKzr3s+xla+rbfdzbPvtVkqNs8c/nBBOIpkVwn1IXkX9tNby5aZfQCCwEdgNvAFMsG33AlYAUbafZwMf2L5fDLxr+348sNX2fTDgaft+CvCd7fuJwLw63rtqO/A08B/b96G28gQAl2JdHSkE8AWSgQQgDjgIhNvKuhR43fb6j4Czq73PYuAF2/enAn86+99dvuSruV+SWfmSL/f5krzKV0Nf0p3DjWmti5VSQ4BxwCTgK6XUvcBaoB/wh7IuZeoBpFd76Re21y9RSgUrpUKBIOD/lFLdAY01eI11EjBTKXWn7WdfINH2/V9a6wIApdR2oBMQCfyttc61bf8G6FHP8b+3/Xcd0LkJ5RLCpUhmhXAfklfREGlEuzmtdSXWK8nFSqktwCVYg7BNaz3qeC+r4+cngEVa6zOUUp1tx2wsBZyltd5VY6N1AEZFtU2VNO9v7ugxmvt6IVyGZFYI9yF5FfWRPtFuTCnV03ZVe9QgrI9zdgFRyjooAqWUl1Kqb7X9Ztu2jwUKbFexIUCa7feXNrEovwE3KdsluVLqhAb2XwNMUEqFKaU8gbOq/a4I6xW7EG2OZFYI9yF5FQ2RRrR7C8T6eGi7Umoz0Ad4VGttBM4GnlNKbcLap2t0tdeVK6U2AG8BV9i2PQ88Y9ve1CvRJ7A+mtqslNpm+/m4tNZpWPt4/QMsx9p3q8D26y+Bu2yDJ7rWfQQh3JZkVgj3IXkV9VK2zuSinVBKLQbu1FqvdXI5Am39zTyBH7AOyvjBmWUSwhVJZoVwH5LX9kXuRAtneVQptRHYChwAfnRqaYQQDZHMCuE+JK+tQO5ECyGEEEII0URyJ1oIIYQQQogmkka0EEIIIYQQTSSNaCGEEEIIIZpIGtFCCCGEEEI0kTSihRBCCCGEaCJpRAshhBBCCNFE/w+rJL4JcdnJQAAAAABJRU5ErkJggg==\n", 479 | "text/plain": [ 480 | "
" 481 | ] 482 | }, 483 | "metadata": { 484 | "needs_background": "light" 485 | }, 486 | "output_type": "display_data" 487 | } 488 | ], 489 | "source": [ 490 | "from sklearn.linear_model import Perceptron, RidgeClassifier\n", 491 | "from sklearn.svm import SVC\n", 492 | "models = [SVC(kernel='linear', C=10000), Perceptron(), RidgeClassifier()]\n", 493 | "\n", 494 | "def fit_and_create_boundary(model):\n", 495 | " model.fit(X, iris_y2)\n", 496 | " Z = model.predict(np.c_[xx.ravel(), yy.ravel()])\n", 497 | " Z = Z.reshape(xx.shape)\n", 498 | " return Z\n", 499 | "\n", 500 | "plt.figure(figsize=(12,3))\n", 501 | "for i, model in enumerate(models):\n", 502 | " plt.subplot('13%d' % (i+1))\n", 503 | " Z = fit_and_create_boundary(model)\n", 504 | " plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired) \n", 505 | "\n", 506 | " # Plot also the training points\n", 507 | " plt.scatter(X[:, 0], X[:, 1], c=iris_y2, edgecolors='k', cmap=plt.cm.Paired)\n", 508 | " if i == 0:\n", 509 | " plt.title('Good Margin')\n", 510 | " else:\n", 511 | " plt.title('Bad Margin')\n", 512 | " plt.xlabel('Sepal length')\n", 513 | " plt.ylabel('Sepal width')\n", 514 | "\n", 515 | "plt.show()" 516 | ] 517 | }, 518 | { 519 | "cell_type": "markdown", 520 | "metadata": { 521 | "slideshow": { 522 | "slide_type": "subslide" 523 | } 524 | }, 525 | "source": [ 526 | "
" 527 | ] 528 | }, 529 | { 530 | "cell_type": "markdown", 531 | "metadata": { 532 | "slideshow": { 533 | "slide_type": "slide" 534 | } 535 | }, 536 | "source": [ 537 | "# The Kernel Trick\n", 538 | "\n", 539 | "Many algorithms in machine learning only involve dot products $\\phi(x)^\\top \\phi(z)$ but not the features $\\phi$ themselves." 540 | ] 541 | }, 542 | { 543 | "cell_type": "markdown", 544 | "metadata": { 545 | "slideshow": { 546 | "slide_type": "fragment" 547 | } 548 | }, 549 | "source": [ 550 | "We can often compute $\\phi(x)^\\top \\phi(z)$ very efficiently for complex $\\phi$ using a kernel function $K(x,z) = \\phi(x)^\\top \\phi(z)$. This is the __kernel trick__." 551 | ] 552 | }, 553 | { 554 | "cell_type": "markdown", 555 | "metadata": { 556 | "slideshow": { 557 | "slide_type": "subslide" 558 | } 559 | }, 560 | "source": [ 561 | "
" 562 | ] 563 | }, 564 | { 565 | "cell_type": "markdown", 566 | "metadata": { 567 | "slideshow": { 568 | "slide_type": "slide" 569 | } 570 | }, 571 | "source": [ 572 | "# Tree-Based Models\n", 573 | "\n", 574 | "Decision trees output target based on a tree of human-interpretable decision rules.\n", 575 | "* __Random forests__ combine large trees using *bagging* to reduce overfitting.\n", 576 | "* __Boosted trees__ combine small trees to reduce underfitting." 577 | ] 578 | }, 579 | { 580 | "cell_type": "markdown", 581 | "metadata": { 582 | "slideshow": { 583 | "slide_type": "subslide" 584 | } 585 | }, 586 | "source": [ 587 | "
" 588 | ] 589 | }, 590 | { 591 | "cell_type": "markdown", 592 | "metadata": { 593 | "slideshow": { 594 | "slide_type": "slide" 595 | } 596 | }, 597 | "source": [ 598 | "# Neural Networks\n", 599 | "\n", 600 | "Neural network models are inspired by the brain.\n", 601 | "* A Perceptron is an artificial model of a neuron.\n", 602 | "* MLP stack multiple layers of artifical neurons.\n", 603 | "* ConvNets tie the weights of neighboring neurons into receptive fields that implement the convolution operation." 604 | ] 605 | }, 606 | { 607 | "cell_type": "markdown", 608 | "metadata": { 609 | "slideshow": { 610 | "slide_type": "subslide" 611 | } 612 | }, 613 | "source": [ 614 | "
" 615 | ] 616 | }, 617 | { 618 | "cell_type": "markdown", 619 | "metadata": { 620 | "slideshow": { 621 | "slide_type": "slide" 622 | } 623 | }, 624 | "source": [ 625 | "# Unsupervised Learning\n", 626 | "\n", 627 | "We have a dataset *without* labels. Our goal is to learn something interesting about the structure of the data:\n", 628 | "* __Clusters__ hidden in the dataset.\n", 629 | "* A __low-dimensional representation__ of the data.\n", 630 | "* Recover the __probability density__ that generated the data." 631 | ] 632 | }, 633 | { 634 | "cell_type": "markdown", 635 | "metadata": { 636 | "slideshow": { 637 | "slide_type": "subslide" 638 | } 639 | }, 640 | "source": [ 641 | "
" 642 | ] 643 | }, 644 | { 645 | "cell_type": "markdown", 646 | "metadata": { 647 | "slideshow": { 648 | "slide_type": "subslide" 649 | } 650 | }, 651 | "source": [ 652 | "
" 653 | ] 654 | }, 655 | { 656 | "cell_type": "markdown", 657 | "metadata": { 658 | "slideshow": { 659 | "slide_type": "subslide" 660 | } 661 | }, 662 | "source": [ 663 | "
" 664 | ] 665 | }, 666 | { 667 | "cell_type": "markdown", 668 | "metadata": { 669 | "slideshow": { 670 | "slide_type": "slide" 671 | } 672 | }, 673 | "source": [ 674 | "# How To Decide Which Algorithm to Use\n", 675 | "\n", 676 | "One factor is how much data you have. In the __small data__ (<10,000) regime, consider:\n", 677 | "* Linear models with hand-crafted features (LASSO, LR, NB, SVMs)\n", 678 | "* Kernel methods often work best (e.g., SVM + RBF kernel)\n", 679 | "* Non-parametric methods (kernels, nearest neighbors) are also powerful" 680 | ] 681 | }, 682 | { 683 | "cell_type": "markdown", 684 | "metadata": { 685 | "slideshow": { 686 | "slide_type": "subslide" 687 | } 688 | }, 689 | "source": [ 690 | "In the __big data__ regime,\n", 691 | "* If using \"high-level\" features, gradient boosted trees are state-of-the-art\n", 692 | "* When using \"low-level\" representations (images, sound signals), neural networks work best\n", 693 | "* Linear models with good features are also good and reliable" 694 | ] 695 | }, 696 | { 697 | "cell_type": "markdown", 698 | "metadata": { 699 | "slideshow": { 700 | "slide_type": "subslide" 701 | } 702 | }, 703 | "source": [ 704 | "Some additional advice:\n", 705 | "* If interpretability matters, use decision trees or LASSO.\n", 706 | "* When uncertainty estimates are important use probabilistic methods.\n", 707 | "* If you know the data generating process, use generative models." 708 | ] 709 | }, 710 | { 711 | "cell_type": "markdown", 712 | "metadata": { 713 | "slideshow": { 714 | "slide_type": "slide" 715 | } 716 | }, 717 | "source": [ 718 | "# What's Next? Ideas for Courses\n", 719 | "\n", 720 | "Consider the following courses to keep learning about ML:\n", 721 | "* Graduate courses in the Spring semester at Cornell (generative models, NLP, etc.)\n", 722 | "* Masters courses: Deep Learning Clinic, ML Engineering, Data Science, etc.\n", 723 | "* Online courses, e.g. Full Stack Deep Learning" 724 | ] 725 | }, 726 | { 727 | "cell_type": "markdown", 728 | "metadata": { 729 | "slideshow": { 730 | "slide_type": "slide" 731 | } 732 | }, 733 | "source": [ 734 | "# What's Next? Ideas for Research\n", 735 | "\n", 736 | "In order to get involved in research, I recommend:\n", 737 | "* Contacting research groups at Cornell for openings\n", 738 | "* Watching online ML tutorials, e.g. NeurIPS\n", 739 | "* Reading and implementing ML papers on your own" 740 | ] 741 | }, 742 | { 743 | "cell_type": "markdown", 744 | "metadata": { 745 | "slideshow": { 746 | "slide_type": "slide" 747 | } 748 | }, 749 | "source": [ 750 | "# What's Next? Ideas for Industry Projects\n", 751 | "\n", 752 | "Finally, a few ideas for how to get more practice applying ML in the real world:\n", 753 | "* Participate in Kaggle competitions and review solutions\n", 754 | "* Build an open-source project that you like and host it on Github" 755 | ] 756 | }, 757 | { 758 | "cell_type": "markdown", 759 | "metadata": { 760 | "slideshow": { 761 | "slide_type": "slide" 762 | } 763 | }, 764 | "source": [ 765 | "# Thank You For Taking Applied Machine Learning 2020!\n", 766 | "\n", 767 | "" 768 | ] 769 | } 770 | ], 771 | "metadata": { 772 | "accelerator": "GPU", 773 | "celltoolbar": "Slideshow", 774 | "colab": { 775 | "collapsed_sections": [], 776 | "name": "neural-ode.ipynb", 777 | "provenance": [] 778 | }, 779 | "kernelspec": { 780 | "display_name": "aml", 781 | "language": "python", 782 | "name": "aml" 783 | }, 784 | "language_info": { 785 | "codemirror_mode": { 786 | "name": "ipython", 787 | "version": 3 788 | }, 789 | "file_extension": ".py", 790 | "mimetype": "text/x-python", 791 | "name": "python", 792 | "nbconvert_exporter": "python", 793 | "pygments_lexer": "ipython3", 794 | "version": "3.6.7" 795 | }, 796 | "rise": { 797 | "controlsTutorial": false, 798 | "height": 900, 799 | "help": false, 800 | "margin": 0, 801 | "maxScale": 2, 802 | "minScale": 0.2, 803 | "progress": true, 804 | "scroll": true, 805 | "theme": "simple", 806 | "width": 1200 807 | } 808 | }, 809 | "nbformat": 4, 810 | "nbformat_minor": 1 811 | } 812 | -------------------------------------------------------------------------------- /notebooks/lecture5b-probabilistic-perspectives.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "colab_type": "text", 7 | "id": "i_f5u2x9nn6I", 8 | "slideshow": { 9 | "slide_type": "slide" 10 | } 11 | }, 12 | "source": [ 13 | "\n", 14 | "\n", 15 | "# Lecture 5b: Probabilistic Perspectives on ML Algorithms\n", 16 | "\n", 17 | "### Applied Machine Learning\n", 18 | "\n", 19 | "__Volodymyr Kuleshov__
Cornell Tech" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "slideshow": { 26 | "slide_type": "slide" 27 | } 28 | }, 29 | "source": [ 30 | "# Part 1: Probabilistic Linear Regression\n", 31 | "\n", 32 | "Previously, we derived *maximum likelihood learning* as a general way of learning machine models.\n", 33 | "\n", 34 | "We will now seehow the algorithms we've seen so far are special cases of this principle." 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": { 40 | "slideshow": { 41 | "slide_type": "slide" 42 | } 43 | }, 44 | "source": [ 45 | "# Review: Probabilistic Models\n", 46 | "\n", 47 | "A probabilistic model is a probability distribution\n", 48 | "$$P(x,y) : \\mathcal{X} \\times \\mathcal{Y} \\to [0,1].$$\n", 49 | "This model can approximate the data distribution $P_\\text{data}(x,y)$." 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "slideshow": { 56 | "slide_type": "fragment" 57 | } 58 | }, 59 | "source": [ 60 | "If we know $P(x,y)$, we can use the conditional $P(y|x)$ for prediction." 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": { 66 | "slideshow": { 67 | "slide_type": "fragment" 68 | } 69 | }, 70 | "source": [ 71 | "Probabilistic models may also have *parameters* $\\theta \\in \\Theta$, which we denote as\n", 72 | "$$P_\\theta(x,y) : \\mathcal{X} \\times \\mathcal{Y} \\to [0,1].$$" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": { 78 | "slideshow": { 79 | "slide_type": "slide" 80 | } 81 | }, 82 | "source": [ 83 | "# Review: Conditional Maximum Likelihood\n", 84 | "\n", 85 | "A general approach of optimizing conditional models of the form $P_\\theta(y|x)$ is by minimizing expected KL divergence with respect to the data distribution:\n", 86 | "$$ \\min_\\theta \\mathbb{E}_{x \\sim \\mathbb{P}_\\text{data}} \\left[ D(P_\\text{data}(y|x) \\mid\\mid P_\\theta(y|x)) \\right]. $$" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": { 92 | "slideshow": { 93 | "slide_type": "fragment" 94 | } 95 | }, 96 | "source": [ 97 | "With a bit of math, we can show that the maximum likelihood objective becomes\n", 98 | "$$ \\max_\\theta \\mathbb{E}_{x, y \\sim \\mathbb{P}_\\text{data}} \\log P_\\theta(y|x). $$\n", 99 | "This is the principle of *conditional maximum likelihood*." 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": { 105 | "slideshow": { 106 | "slide_type": "slide" 107 | } 108 | }, 109 | "source": [ 110 | "# Review: Least Squares\n", 111 | "\n", 112 | "Recall that the linear regression algorithm fits a linear model of the form\n", 113 | "$$ f(x) = \\sum_{j=0}^d \\theta_j \\cdot x_j = \\theta^\\top x. $$" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": { 119 | "slideshow": { 120 | "slide_type": "fragment" 121 | } 122 | }, 123 | "source": [ 124 | "It minimizes the mean squared error (MSE)\n", 125 | "$$J(\\theta)= \\frac{1}{2n} \\sum_{i=1}^n(y^{(i)}-\\theta^\\top x^{(i)})^2$$\n", 126 | "on a dataset $\\{(x^{(1)}, y^{(1)}), (x^{(2)}, y^{(2)}), \\ldots, (x^{(n)}, y^{(n)})\\}$." 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": { 132 | "slideshow": { 133 | "slide_type": "subslide" 134 | } 135 | }, 136 | "source": [ 137 | "Is there a specific reason for us to be optimizing the mean squared error to fit our linear model?\n", 138 | "\n", 139 | "The answer to this can be found by looking at the algorithm from a probabilistic perspective." 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "slideshow": { 146 | "slide_type": "slide" 147 | } 148 | }, 149 | "source": [ 150 | "# Probabilistic Least Squares\n", 151 | "\n", 152 | "Let's derive a probabilistic algorithm by defining a class of probabilistic models and use maximum likelihood as our objective." 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": { 158 | "slideshow": { 159 | "slide_type": "fragment" 160 | } 161 | }, 162 | "source": [ 163 | "1. Let's choose our model family $\\mathcal{M}$ to be the set of Gaussian distributions of the form\n", 164 | "$$ p(y | x; \\theta) = \\frac{1}{\\sqrt{2\\pi}\\sigma} \\exp\\left( -\\frac{(y - \\theta^\\top x)^2}{2 \\sigma^2} \\right).$$\n", 165 | "Each model $\\mathcal{N}(y; \\mu(x), \\sigma)$ is a Gaussian with a standard deviation $\\sigma$ of one and a mean of $\\mu(x) = \\theta^\\top x$ that is parametrized by the parameters $\\theta$." 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": { 171 | "slideshow": { 172 | "slide_type": "subslide" 173 | } 174 | }, 175 | "source": [ 176 | "2. We optimize the model using maximum likelihood. The log-likelihood function at a point $(x,y)$ equals\n", 177 | "\\begin{align*}\n", 178 | "\\log L(\\theta) = \\log p(y | x; \\theta) & = \\log \\frac{1}{\\sqrt{2\\pi}\\sigma} \\exp\\left( -\\frac{(y - \\theta^\\top x)^2}{2 \\sigma^2} \\right) \\\\\n", 179 | "& = -\\frac{(y - \\theta^\\top x)^2}{2 \\sigma^2} + \\text{const.}\n", 180 | "\\end{align*}" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": { 186 | "slideshow": { 187 | "slide_type": "fragment" 188 | } 189 | }, 190 | "source": [ 191 | "Note how this is a mean squared error (MSE) objective! \n", 192 | "\n", 193 | "Thus, minimizing MSE is equivalent to maximizing the log-likelihood of a Normal distribution $\\mathcal{N}(y; \\mu(x), \\sigma)$." 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": { 199 | "slideshow": { 200 | "slide_type": "slide" 201 | } 202 | }, 203 | "source": [ 204 | "# Algorithm: Gaussian Ordinary Least Squares\n", 205 | "\n", 206 | "* __Type__: Supervised learning (regression)\n", 207 | "* __Model family__: Linear models\n", 208 | "* __Objective function__: Mean squared error\n", 209 | "* __Optimizer__: Normal equations\n", 210 | "* __Probabilistic interpretation__: Conditional Gaussian fit using max-likelihood." 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": { 216 | "slideshow": { 217 | "slide_type": "slide" 218 | } 219 | }, 220 | "source": [ 221 | "# Extensions of Gaussian Least Squares\n", 222 | "\n", 223 | "This is an example of how we can interpret a machine learning algorithm in a probabilistic framework.\n", 224 | "\n", 225 | "We will see many algorithms that have these kinds of interpretations. Here are some simple extensions." 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": { 231 | "slideshow": { 232 | "slide_type": "subslide" 233 | } 234 | }, 235 | "source": [ 236 | "We can use a Gaussian model and also parametrize the standard deviation. \n", 237 | "* This is called heteroscedastic regression, and allows us to obtain confidence intevals for our predictions." 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": { 243 | "slideshow": { 244 | "slide_type": "subslide" 245 | } 246 | }, 247 | "source": [ 248 | "We can can also parametrize other distributions, not just the Gaussian. \n", 249 | "* Exponential or Gamma distributions for continuous variables \n", 250 | "* Bernoulli distribution for discrete variables\n", 251 | "\n", 252 | "This yields many new machine learning algorithms." 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": { 258 | "slideshow": { 259 | "slide_type": "slide" 260 | } 261 | }, 262 | "source": [ 263 | "\n", 264 | "# Part 2: Bayesian Algorithms\n", 265 | "\n", 266 | "We can also use what we learned about Bayesian ML do interpret several algrothims that we've seen as special cases of the Bayesian framework." 267 | ] 268 | }, 269 | { 270 | "cell_type": "markdown", 271 | "metadata": { 272 | "slideshow": { 273 | "slide_type": "slide" 274 | } 275 | }, 276 | "source": [ 277 | "# Review: The Bayesian Approach\n", 278 | "\n", 279 | "In Bayesian statistics, $\\theta$ is a *random* variable whose value happens to be unknown." 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": { 285 | "slideshow": { 286 | "slide_type": "subslide" 287 | } 288 | }, 289 | "source": [ 290 | "We formulate two models:\n", 291 | "* A *likelihood* model $P(x, y | \\theta)$ that defines the probability of $x,y$ for any fixed value of $\\theta$.\n", 292 | "* A *prior* $P(\\theta)$ that specifies us existing belief about the distribution of the random variable $\\theta$." 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": { 298 | "slideshow": { 299 | "slide_type": "fragment" 300 | } 301 | }, 302 | "source": [ 303 | "Together, these two models define the *joint* distribution\n", 304 | "$$ P(x, y, \\theta) = P(x, y \\mid \\theta) P(\\theta) $$\n", 305 | "in which both the $x, y$ and the parameters $\\theta$ are random variables." 306 | ] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "metadata": { 311 | "slideshow": { 312 | "slide_type": "slide" 313 | } 314 | }, 315 | "source": [ 316 | "# Review: A Posteriori Learning\n", 317 | "\n", 318 | "Recall that in maximum a posteriori (MAP) learning, we optimize the following objective.\n", 319 | "\\begin{align*}\n", 320 | "\\theta_\\text{MAP} = \\arg\\max_\\theta \\left( \\log \\prod_{i=1}^n P(x^{(i)}, y^{(i)} \\mid \\theta) + \\log P(\\theta) \\right),\n", 321 | "\\end{align*}\n", 322 | "\n", 323 | "Note that we used the same formula as we used for maximum likelihood, except that we have added the prior term $\\log P(\\theta)$." 324 | ] 325 | }, 326 | { 327 | "cell_type": "markdown", 328 | "metadata": { 329 | "slideshow": { 330 | "slide_type": "slide" 331 | } 332 | }, 333 | "source": [ 334 | "# Review: Ridge Regression\n", 335 | "\n", 336 | "Recall that the ridge regression algorithm fits a linear model \n", 337 | "$$ f(x) = \\sum_{j=0}^d \\theta_j \\cdot x_j = \\theta^\\top x. $$" 338 | ] 339 | }, 340 | { 341 | "cell_type": "markdown", 342 | "metadata": { 343 | "slideshow": { 344 | "slide_type": "subslide" 345 | } 346 | }, 347 | "source": [ 348 | "We minimize the L2-regualrized mean squared error (MSE)\n", 349 | "$$J(\\theta)= \\frac{1}{2n} \\sum_{i=1}^n(y^{(i)}-\\theta^\\top x^{(i)})^2 + \\frac{\\lambda}{2}\\sum_{j=1}^d \\theta_j^2$$\n", 350 | "on a dataset $\\{(x^{(1)}, y^{(1)}), (x^{(2)}, y^{(2)}), \\ldots, (x^{(n)}, y^{(n)})\\}$.\n", 351 | "The term $\\frac{1}{2}\\sum_{j=1}^d \\theta_j^2 = \\frac{1}{2}||\\theta||_2^2$ is called the regularizer." 352 | ] 353 | }, 354 | { 355 | "cell_type": "markdown", 356 | "metadata": { 357 | "slideshow": { 358 | "slide_type": "slide" 359 | } 360 | }, 361 | "source": [ 362 | "# Probabilistic Ridge Regession\n", 363 | "\n", 364 | "We can interpet ridge regression as maximum apriori (MAP) estimation as follows." 365 | ] 366 | }, 367 | { 368 | "cell_type": "markdown", 369 | "metadata": { 370 | "slideshow": { 371 | "slide_type": "slide" 372 | } 373 | }, 374 | "source": [ 375 | "1. First, we select our model family $\\mathcal{M}$ to be the set of Gaussian distributions of the form (let's assume $x \\in \\mathbb{R}$ for simplicity).\n", 376 | "$$ p(y | x; \\theta) = \\frac{1}{\\sqrt{2\\pi}\\sigma} \\exp\\left( -\\frac{(y - \\theta^\\top x)^2}{2 \\sigma^2} \\right).$$" 377 | ] 378 | }, 379 | { 380 | "cell_type": "markdown", 381 | "metadata": { 382 | "slideshow": { 383 | "slide_type": "fragment" 384 | } 385 | }, 386 | "source": [ 387 | "2. We assume a Gaussian prior with mean zero and variance $\\tau$ on the parameters $\\theta$:\n", 388 | "$$ p(\\theta) = \\prod_{j=1}^d \\frac{1}{\\sqrt{2\\pi}\\tau} \\exp\\left( -\\frac{\\theta_j^2}{2\\tau^2} \\right).$$" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": { 394 | "slideshow": { 395 | "slide_type": "subslide" 396 | } 397 | }, 398 | "source": [ 399 | "3. We optimize the model using the MAP approach. The objective at a point $(x,y)$ equals\n", 400 | "\\begin{align*}\n", 401 | "\\log L(\\theta) & = \\log p(y | x; \\theta) + \\log p(\\theta) \n", 402 | "\\\\ & = \\log \\frac{1}{\\sqrt{2\\pi}\\sigma} \\exp\\left( -\\frac{(y - \\theta^\\top x)^2}{2 \\sigma^2} \\right) \\\\\n", 403 | "& \\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\;\\; + \\log \\prod_{j=1}^d \\frac{1}{\\sqrt{2\\pi}\\tau} \\exp\\left( -\\frac{\\theta_j^2}{2\\tau^2} \\right) \\\\\n", 404 | "& = -\\frac{(y - \\theta^\\top x)^2}{2 \\sigma^2} - \\frac{1}{2\\tau^2}\\sum_{j=1}^d \\theta_j^2 + \\text{const.}\n", 405 | "\\end{align*}" 406 | ] 407 | }, 408 | { 409 | "cell_type": "markdown", 410 | "metadata": { 411 | "slideshow": { 412 | "slide_type": "fragment" 413 | } 414 | }, 415 | "source": [ 416 | "Thus, we see that ridge regression actually amounts to performing MAP estimation with a Gaussian prior. The strength of the regularizer $\\lambda$ equals $1/\\tau^2$." 417 | ] 418 | }, 419 | { 420 | "cell_type": "markdown", 421 | "metadata": { 422 | "slideshow": { 423 | "slide_type": "slide" 424 | } 425 | }, 426 | "source": [ 427 | "# Algorithm: Probabilistic Ridge Least Squares\n", 428 | "\n", 429 | "* __Type__: Supervised learning (regression)\n", 430 | "* __Model family__: Linear models\n", 431 | "* __Objective function__: L2-regularized mean squared error\n", 432 | "* __Optimizer__: Normal equations\n", 433 | "* __Probabilistic interpretation__: Conditional Gaussian likelihood and Gaussian prior fit using MAP." 434 | ] 435 | }, 436 | { 437 | "cell_type": "markdown", 438 | "metadata": { 439 | "slideshow": { 440 | "slide_type": "slide" 441 | } 442 | }, 443 | "source": [ 444 | "# Bayesian View on ML Algorithms\n", 445 | "\n", 446 | "Very often, we can interpret classical ML algorithms as applications of the probabilistic or Bayesian approaches (although we can derive them in other ways as well!)" 447 | ] 448 | }, 449 | { 450 | "cell_type": "markdown", 451 | "metadata": { 452 | "slideshow": { 453 | "slide_type": "fragment" 454 | } 455 | }, 456 | "source": [ 457 | "* Regularization can often be seen as applying a prior on the weights." 458 | ] 459 | }, 460 | { 461 | "cell_type": "markdown", 462 | "metadata": { 463 | "slideshow": { 464 | "slide_type": "fragment" 465 | } 466 | }, 467 | "source": [ 468 | "* L1 regularization can be seen as applying a *Laplace* prior." 469 | ] 470 | }, 471 | { 472 | "cell_type": "markdown", 473 | "metadata": { 474 | "slideshow": { 475 | "slide_type": "fragment" 476 | } 477 | }, 478 | "source": [ 479 | "* Many other algorithms will have similar interpretations." 480 | ] 481 | }, 482 | { 483 | "cell_type": "markdown", 484 | "metadata": { 485 | "slideshow": { 486 | "slide_type": "skip" 487 | } 488 | }, 489 | "source": [ 490 | "\n", 491 | "# Part 3: Bayesian Ridge Regression\n", 492 | "\n", 493 | "Let's now look at an example of a fully Bayesian machinne learning algorithm.\n", 494 | "\n", 495 | "This section is still under construction and not part of the main lecture." 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "metadata": { 501 | "slideshow": { 502 | "slide_type": "skip" 503 | } 504 | }, 505 | "source": [ 506 | "# Review: The Bayesian Approach\n", 507 | "\n", 508 | "In Bayesian statistics, $\\theta$ is a *random* variable whose value happens to be unknown." 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "metadata": { 514 | "slideshow": { 515 | "slide_type": "skip" 516 | } 517 | }, 518 | "source": [ 519 | "We formulate two models:\n", 520 | "* A *likelihood* model $P(x, y | \\theta)$ that defines the probability of $x,y$ for any fixed value of $\\theta$.\n", 521 | "* A *prior* $P(\\theta)$ that specifies us existing belief about the distribution of the random variable $\\theta$." 522 | ] 523 | }, 524 | { 525 | "cell_type": "markdown", 526 | "metadata": { 527 | "slideshow": { 528 | "slide_type": "skip" 529 | } 530 | }, 531 | "source": [ 532 | "Together, these two models define the *joint* distribution\n", 533 | "$$ P(x, y, \\theta) = P(x, y \\mid \\theta) P(\\theta) $$\n", 534 | "in which both the $x, y$ and the parameters $\\theta$ are random variables." 535 | ] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": { 540 | "slideshow": { 541 | "slide_type": "skip" 542 | } 543 | }, 544 | "source": [ 545 | "# Review: Ridge Regression\n", 546 | "\n", 547 | "Recall that the ridge regression algorithm fits a linear model \n", 548 | "$$ f(x) = \\sum_{j=0}^d \\theta_j \\cdot x_j = \\theta^\\top x. $$" 549 | ] 550 | }, 551 | { 552 | "cell_type": "markdown", 553 | "metadata": { 554 | "slideshow": { 555 | "slide_type": "skip" 556 | } 557 | }, 558 | "source": [ 559 | "We minimize the L2-regualrized mean squared error (MSE)\n", 560 | "$$J(\\theta)= \\frac{1}{2n} \\sum_{i=1}^n(y_i-x_i^\\top\\theta)^2 + \\frac{1}{2}\\sum_{j=1}^d \\theta_j^2$$\n", 561 | "on a dataset $\\{(x^{(1)}, y^{(1)}), (x^{(2)}, y^{(2)}), \\ldots, (x^{(n)}, y^{(n)})\\}$.\n", 562 | "The term $\\frac{1}{2}\\sum_{j=1}^d \\theta_j^2 = \\frac{1}{2}||\\theta||_2^2$ is called the regularizer." 563 | ] 564 | }, 565 | { 566 | "cell_type": "markdown", 567 | "metadata": { 568 | "slideshow": { 569 | "slide_type": "skip" 570 | } 571 | }, 572 | "source": [ 573 | "# Probabilistic Ridge Regession\n", 574 | "\n", 575 | "We can interpet ridge regression as maximum apriori (MAP) estimation as follows." 576 | ] 577 | }, 578 | { 579 | "cell_type": "markdown", 580 | "metadata": { 581 | "slideshow": { 582 | "slide_type": "skip" 583 | } 584 | }, 585 | "source": [ 586 | "# Bayesian Predictions\n", 587 | "\n", 588 | "Suppose we now want to predict the value of $y$ from $x$. Unlike in the frequentist setting, we no longer have a single estimate $\\theta$ of the model params, but instead we have a distribution.\n", 589 | "\n", 590 | "The Bayesian approach to predicting $y$ given an input $x$ and a training dataset $\\mathcal{D}$ consists of taking the prediction of all the possible models\n", 591 | "$$ P(y | x, \\mathcal{D}) = \\int_\\theta P(y \\mid x, \\theta) P(\\theta \\mid \\mathcal{D}) d\\theta. $$\n", 592 | "This is called the *posterior predictive* distribution. Note how each $P(y \\mid x, \\theta)$ is weighted by the probability of $\\theta$ given $\\mathcal{D}$." 593 | ] 594 | } 595 | ], 596 | "metadata": { 597 | "accelerator": "GPU", 598 | "celltoolbar": "Slideshow", 599 | "colab": { 600 | "collapsed_sections": [], 601 | "name": "neural-ode.ipynb", 602 | "provenance": [] 603 | }, 604 | "kernelspec": { 605 | "display_name": "aml", 606 | "language": "python", 607 | "name": "aml" 608 | }, 609 | "language_info": { 610 | "codemirror_mode": { 611 | "name": "ipython", 612 | "version": 3 613 | }, 614 | "file_extension": ".py", 615 | "mimetype": "text/x-python", 616 | "name": "python", 617 | "nbconvert_exporter": "python", 618 | "pygments_lexer": "ipython3", 619 | "version": "3.6.7" 620 | }, 621 | "rise": { 622 | "controlsTutorial": false, 623 | "height": 900, 624 | "help": false, 625 | "margin": 0, 626 | "maxScale": 2, 627 | "minScale": 0.2, 628 | "progress": true, 629 | "scroll": true, 630 | "theme": "simple", 631 | "width": 1200 632 | } 633 | }, 634 | "nbformat": 4, 635 | "nbformat_minor": 1 636 | } 637 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appnope==0.1.0 2 | astroid==2.4.1 3 | attrs==19.3.0 4 | backcall==0.2.0 5 | bleach==3.1.5 6 | cycler==0.10.0 7 | decorator==4.4.2 8 | defusedxml==0.6.0 9 | entrypoints==0.3 10 | importlib-metadata==1.7.0 11 | ipykernel==5.3.4 12 | ipython==7.16.1 13 | ipython-genutils==0.2.0 14 | isort==4.3.21 15 | jedi==0.17.2 16 | Jinja2==2.11.2 17 | jsonschema==3.2.0 18 | jupyter-client==6.1.6 19 | jupyter-contrib-core==0.3.3 20 | jupyter-contrib-nbextensions==0.5.1 21 | jupyter-core==4.6.3 22 | jupyter-highlight-selected-word==0.2.0 23 | jupyter-latex-envs==1.4.6 24 | jupyter-nbextensions-configurator==0.4.1 25 | kiwisolver==1.2.0 26 | latexcodec==2.0.1 27 | lazy-object-proxy==1.4.3 28 | lxml==4.6.3 29 | MarkupSafe==1.1.1 30 | matplotlib==3.2.1 31 | mccabe==0.6.1 32 | mistune==0.8.4 33 | nbconvert==5.6.1 34 | nbformat==5.0.7 35 | notebook==6.0.3 36 | numpy==1.18.1 37 | packaging==20.4 38 | pandocfilters==1.4.2 39 | parso==0.7.1 40 | pexpect==4.8.0 41 | pickleshare==0.7.5 42 | Pillow==7.0.0 43 | prometheus-client==0.8.0 44 | prompt-toolkit==3.0.5 45 | ptyprocess==0.6.0 46 | pybtex==0.23.0 47 | Pygments==2.6.1 48 | pylint==2.5.2 49 | pyparsing==2.4.7 50 | pyrsistent==0.16.0 51 | python-dateutil==2.8.1 52 | PyYAML==5.3 53 | pyzmq==19.0.1 54 | rise==5.6.1 55 | scipy==1.4.1 56 | Send2Trash==1.5.0 57 | six==1.14.0 58 | terminado==0.8.3 59 | testpath==0.4.4 60 | toml==0.10.1 61 | torch==1.4.0 62 | torchdiffeq==0.0.1 63 | torchvision==0.5.0 64 | tornado==6.1.0 65 | tqdm==4.42.1 66 | traitlets==4.3.3 67 | typed-ast==1.4.1 68 | virtualenv-clone==0.5.4 69 | wcwidth==0.2.5 70 | webencodings==0.5.1 71 | wrapt==1.12.1 72 | zipp==3.1.0 73 | --------------------------------------------------------------------------------