├── .gitignore ├── LICENSE ├── README.md ├── code ├── _old-material │ ├── appendix_tensorflow-basics │ │ ├── appendix_g_tensorflow-basics.ipynb │ │ ├── images │ │ │ ├── tensorboard-1.png │ │ │ ├── tensorboard-2.png │ │ │ ├── tensorboard-3.png │ │ │ ├── tensorboard-4.png │ │ │ ├── tensorboard-5.png │ │ │ ├── tensorboard-6.png │ │ │ └── tensors.png │ │ ├── my-model-multiple_ckpts.ckpt-50.data-00000-of-00001 │ │ ├── my-model-multiple_ckpts.ckpt-50.index │ │ ├── my-model-multiple_ckpts.ckpt-50.meta │ │ ├── my-model-multiple_ckpts.ckpt-60.data-00000-of-00001 │ │ ├── my-model-multiple_ckpts.ckpt-60.index │ │ ├── my-model-multiple_ckpts.ckpt-60.meta │ │ ├── my-model-multiple_ckpts.ckpt-70.data-00000-of-00001 │ │ ├── my-model-multiple_ckpts.ckpt-70.index │ │ ├── my-model-multiple_ckpts.ckpt-70.meta │ │ ├── my-model-multiple_ckpts.ckpt-80.data-00000-of-00001 │ │ ├── my-model-multiple_ckpts.ckpt-80.index │ │ ├── my-model-multiple_ckpts.ckpt-80.meta │ │ ├── my-model-multiple_ckpts.ckpt-90.data-00000-of-00001 │ │ ├── my-model-multiple_ckpts.ckpt-90.index │ │ ├── my-model-multiple_ckpts.ckpt-90.meta │ │ ├── my-model.ckpt.data-00000-of-00001 │ │ ├── my-model.ckpt.index │ │ └── my-model.ckpt.meta │ └── ch02_perceptron │ │ ├── ch02_perceptron.ipynb │ │ ├── perceptron.data-00000-of-00001 │ │ ├── perceptron.index │ │ ├── perceptron.meta │ │ ├── perceptron_toydata.txt │ │ └── solutions │ │ ├── 01_weight_zero_learning_rate.py │ │ ├── 02_random_weights_learning_rate.py │ │ └── 03_tensorflow-boundary.py ├── appendix_h_numpy-intro │ ├── appendix_numpy-intro.ipynb │ └── images │ │ └── numpy-intro │ │ ├── array_1.png │ │ ├── array_2.png │ │ ├── broadcasting-1.png │ │ ├── broadcasting-2.png │ │ ├── matmatmul.png │ │ ├── matmul.png │ │ ├── random_1.png │ │ ├── random_2.png │ │ ├── set_theory.png │ │ ├── transpose.png │ │ └── ufunc.png └── tests │ ├── __init__.py │ └── test_notebooks.py ├── images └── ann-and-deeplearning-cover.jpg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Datasets 2 | code/model_zoo/pytorch_ipynb/AFAD-Lite/ 3 | code/model_zoo/pytorch_ipynb/tarball-lite/ 4 | code/model_zoo/pytorch_ipynb/tarball 5 | code/model_zoo/pytorch_ipynb/test_set* 6 | code/model_zoo/pytorch_ipynb/train_set* 7 | code/model_zoo/pytorch_ipynb/AFAD-Full* 8 | celeba_gender_attr_test.txt 9 | celeba_gender_attr_train.txt 10 | iris.h5 11 | test_32x32.mat 12 | train_32x32.mat 13 | code/model_zoo/pytorch_ipynb/svhn_cropped/ 14 | list_attr_celeba.txt 15 | list_eval_partition.txt 16 | img_align_celeba 17 | quickdraw-* 18 | *.csv 19 | *.zip 20 | *.npz 21 | *.npy 22 | *ubyte.gz 23 | *archive.ics.uci.edu* 24 | code/model_zoo/cifar-10 25 | code/model_zoo/pytorch_ipynb/data 26 | 27 | # Binary PyTorch models 28 | *.pt 29 | 30 | # Temporary OS files 31 | .DS_Store 32 | 33 | # TensorFlow Checkpoint files 34 | checkpoint 35 | code/*/*.data-?????-of-????? 36 | code/*/*.index 37 | code/*/*.meta 38 | code/model_zoo/tensorflow_ipynb/*.data-?????-of-????? 39 | code/model_zoo/tensorflow_ipynb/*.index 40 | code/model_zoo/tensorflow_ipynb/*.meta 41 | code/model_zoo/tensorflow_ipynb/cifar-10/* 42 | 43 | # Byte-compiled / optimized / DLL files 44 | __pycache__/ 45 | *.py[cod] 46 | *$py.class 47 | 48 | # C extensions 49 | *.so 50 | 51 | # Distribution / packaging 52 | .Python 53 | env/ 54 | build/ 55 | develop-eggs/ 56 | dist/ 57 | downloads/ 58 | eggs/ 59 | .eggs/ 60 | lib/ 61 | lib64/ 62 | parts/ 63 | sdist/ 64 | var/ 65 | *.egg-info/ 66 | .installed.cfg 67 | *.egg 68 | 69 | # PyInstaller 70 | # Usually these files are written by a python script from a template 71 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 72 | *.manifest 73 | *.spec 74 | 75 | # Installer logs 76 | pip-log.txt 77 | pip-delete-this-directory.txt 78 | 79 | # Unit test / coverage reports 80 | htmlcov/ 81 | .tox/ 82 | .coverage 83 | .coverage.* 84 | .cache 85 | nosetests.xml 86 | coverage.xml 87 | *,cover 88 | .hypothesis/ 89 | 90 | # Translations 91 | *.mo 92 | *.pot 93 | 94 | # Django stuff: 95 | *.log 96 | local_settings.py 97 | 98 | # Flask stuff: 99 | instance/ 100 | .webassets-cache 101 | 102 | # Scrapy stuff: 103 | .scrapy 104 | 105 | # Sphinx documentation 106 | docs/_build/ 107 | 108 | # PyBuilder 109 | target/ 110 | 111 | # IPython Notebook 112 | .ipynb_checkpoints 113 | 114 | # pyenv 115 | .python-version 116 | 117 | # celery beat schedule file 118 | celerybeat-schedule 119 | 120 | # dotenv 121 | .env 122 | 123 | # virtualenv 124 | venv/ 125 | ENV/ 126 | 127 | # Spyder project settings 128 | .spyderproject 129 | 130 | # Rope project settings 131 | .ropeproject 132 | 133 | # Datasets 134 | MNIST* 135 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Code 2 | ===== 3 | 4 | All code in this repository (including the code examples in Jupyter Notebooks) is open source content, released under the MIT software license. In short, the permissive MIT license allows you to do anything with the code with proper attribution and without warranty; please check the MIT LICENSE notice for further details. 5 | 6 | Text and Graphics 7 | ================= 8 | 9 | All non-code content and creative work in this repository, including text and graphics, is under exclusive copyright by the author, Sebastian Raschka. Unless noted otherwise, text content shared in this repository is intended for personal use only. You may use, modify, or share short text passages of this work with proper attribution to the author. However, if you are planning to modify and/or share substantial portions of this book for other writings, such as blog posts, magazine article, or teaching material, contact the author for permission. 10 | 11 | Figures and graphics marked by a Creative Commons Attribution-ShareAlike 4.0 International are free to share under the respective license terms (as listed in the Creative Commons Attribution-ShareAlike 4.0 International section in the LICENSE file) and proper attribution. 12 | 13 | -------------------------------------------------------------------------------- 14 | 15 | ++++++++++++ 16 | MIT License 17 | ++++++++++++ 18 | 19 | Copyright (c) 2016-2018 Sebastian Raschka 20 | 21 | Permission is hereby granted, free of charge, to any person obtaining a copy 22 | of this software and associated documentation files (the "Software"), to deal 23 | in the Software without restriction, including without limitation the rights 24 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 25 | copies of the Software, and to permit persons to whom the Software is 26 | furnished to do so, subject to the following conditions: 27 | 28 | The above copyright notice and this permission notice shall be included in all 29 | copies or substantial portions of the Software. 30 | 31 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 32 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 33 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 34 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 35 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 36 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37 | SOFTWARE. 38 | 39 | ------------------------------------------------------------------------------- 40 | 41 | 42 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 43 | Creative Commons Attribution-ShareAlike 4.0 International 44 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 45 | 46 | Creative Commons Corporation ("Creative Commons") is not a law firm and 47 | does not provide legal services or legal advice. Distribution of 48 | Creative Commons public licenses does not create a lawyer-client or 49 | other relationship. Creative Commons makes its licenses and related 50 | information available on an "as-is" basis. Creative Commons gives no 51 | warranties regarding its licenses, any material licensed under their 52 | terms and conditions, or any related information. Creative Commons 53 | disclaims all liability for damages resulting from their use to the 54 | fullest extent possible. 55 | 56 | Using Creative Commons Public Licenses 57 | 58 | Creative Commons public licenses provide a standard set of terms and 59 | conditions that creators and other rights holders may use to share 60 | original works of authorship and other material subject to copyright 61 | and certain other rights specified in the public license below. The 62 | following considerations are for informational purposes only, are not 63 | exhaustive, and do not form part of our licenses. 64 | 65 | Considerations for licensors: Our public licenses are 66 | intended for use by those authorized to give the public 67 | permission to use material in ways otherwise restricted by 68 | copyright and certain other rights. Our licenses are 69 | irrevocable. Licensors should read and understand the terms 70 | and conditions of the license they choose before applying it. 71 | Licensors should also secure all rights necessary before 72 | applying our licenses so that the public can reuse the 73 | material as expected. Licensors should clearly mark any 74 | material not subject to the license. This includes other CC- 75 | licensed material, or material used under an exception or 76 | limitation to copyright. More considerations for licensors: 77 | wiki.creativecommons.org/Considerations_for_licensors 78 | 79 | Considerations for the public: By using one of our public 80 | licenses, a licensor grants the public permission to use the 81 | licensed material under specified terms and conditions. If 82 | the licensor's permission is not necessary for any reason--for 83 | example, because of any applicable exception or limitation to 84 | copyright--then that use is not regulated by the license. Our 85 | licenses grant only permissions under copyright and certain 86 | other rights that a licensor has authority to grant. Use of 87 | the licensed material may still be restricted for other 88 | reasons, including because others have copyright or other 89 | rights in the material. A licensor may make special requests, 90 | such as asking that all changes be marked or described. 91 | Although not required by our licenses, you are encouraged to 92 | respect those requests where reasonable. More_considerations 93 | for the public: 94 | wiki.creativecommons.org/Considerations_for_licensees 95 | 96 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 97 | 98 | Creative Commons Attribution-ShareAlike 4.0 International Public 99 | License 100 | 101 | By exercising the Licensed Rights (defined below), You accept and agree 102 | to be bound by the terms and conditions of this Creative Commons 103 | Attribution-ShareAlike 4.0 International Public License ("Public 104 | License"). To the extent this Public License may be interpreted as a 105 | contract, You are granted the Licensed Rights in consideration of Your 106 | acceptance of these terms and conditions, and the Licensor grants You 107 | such rights in consideration of benefits the Licensor receives from 108 | making the Licensed Material available under these terms and 109 | conditions. 110 | 111 | 112 | Section 1 -- Definitions. 113 | 114 | a. Adapted Material means material subject to Copyright and Similar 115 | Rights that is derived from or based upon the Licensed Material 116 | and in which the Licensed Material is translated, altered, 117 | arranged, transformed, or otherwise modified in a manner requiring 118 | permission under the Copyright and Similar Rights held by the 119 | Licensor. For purposes of this Public License, where the Licensed 120 | Material is a musical work, performance, or sound recording, 121 | Adapted Material is always produced where the Licensed Material is 122 | synched in timed relation with a moving image. 123 | 124 | b. Adapter's License means the license You apply to Your Copyright 125 | and Similar Rights in Your contributions to Adapted Material in 126 | accordance with the terms and conditions of this Public License. 127 | 128 | c. BY-SA Compatible License means a license listed at 129 | creativecommons.org/compatiblelicenses, approved by Creative 130 | Commons as essentially the equivalent of this Public License. 131 | 132 | d. Copyright and Similar Rights means copyright and/or similar rights 133 | closely related to copyright including, without limitation, 134 | performance, broadcast, sound recording, and Sui Generis Database 135 | Rights, without regard to how the rights are labeled or 136 | categorized. For purposes of this Public License, the rights 137 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 138 | Rights. 139 | 140 | e. Effective Technological Measures means those measures that, in the 141 | absence of proper authority, may not be circumvented under laws 142 | fulfilling obligations under Article 11 of the WIPO Copyright 143 | Treaty adopted on December 20, 1996, and/or similar international 144 | agreements. 145 | 146 | f. Exceptions and Limitations means fair use, fair dealing, and/or 147 | any other exception or limitation to Copyright and Similar Rights 148 | that applies to Your use of the Licensed Material. 149 | 150 | g. License Elements means the license attributes listed in the name 151 | of a Creative Commons Public License. The License Elements of this 152 | Public License are Attribution and ShareAlike. 153 | 154 | h. Licensed Material means the artistic or literary work, database, 155 | or other material to which the Licensor applied this Public 156 | License. 157 | 158 | i. Licensed Rights means the rights granted to You subject to the 159 | terms and conditions of this Public License, which are limited to 160 | all Copyright and Similar Rights that apply to Your use of the 161 | Licensed Material and that the Licensor has authority to license. 162 | 163 | j. Licensor means the individual(s) or entity(ies) granting rights 164 | under this Public License. 165 | 166 | k. Share means to provide material to the public by any means or 167 | process that requires permission under the Licensed Rights, such 168 | as reproduction, public display, public performance, distribution, 169 | dissemination, communication, or importation, and to make material 170 | available to the public including in ways that members of the 171 | public may access the material from a place and at a time 172 | individually chosen by them. 173 | 174 | l. Sui Generis Database Rights means rights other than copyright 175 | resulting from Directive 96/9/EC of the European Parliament and of 176 | the Council of 11 March 1996 on the legal protection of databases, 177 | as amended and/or succeeded, as well as other essentially 178 | equivalent rights anywhere in the world. 179 | 180 | m. You means the individual or entity exercising the Licensed Rights 181 | under this Public License. Your has a corresponding meaning. 182 | 183 | 184 | Section 2 -- Scope. 185 | 186 | a. License grant. 187 | 188 | 1. Subject to the terms and conditions of this Public License, 189 | the Licensor hereby grants You a worldwide, royalty-free, 190 | non-sublicensable, non-exclusive, irrevocable license to 191 | exercise the Licensed Rights in the Licensed Material to: 192 | 193 | a. reproduce and Share the Licensed Material, in whole or 194 | in part; and 195 | 196 | b. produce, reproduce, and Share Adapted Material. 197 | 198 | 2. Exceptions and Limitations. For the avoidance of doubt, where 199 | Exceptions and Limitations apply to Your use, this Public 200 | License does not apply, and You do not need to comply with 201 | its terms and conditions. 202 | 203 | 3. Term. The term of this Public License is specified in Section 204 | 6(a). 205 | 206 | 4. Media and formats; technical modifications allowed. The 207 | Licensor authorizes You to exercise the Licensed Rights in 208 | all media and formats whether now known or hereafter created, 209 | and to make technical modifications necessary to do so. The 210 | Licensor waives and/or agrees not to assert any right or 211 | authority to forbid You from making technical modifications 212 | necessary to exercise the Licensed Rights, including 213 | technical modifications necessary to circumvent Effective 214 | Technological Measures. For purposes of this Public License, 215 | simply making modifications authorized by this Section 2(a) 216 | (4) never produces Adapted Material. 217 | 218 | 5. Downstream recipients. 219 | 220 | a. Offer from the Licensor -- Licensed Material. Every 221 | recipient of the Licensed Material automatically 222 | receives an offer from the Licensor to exercise the 223 | Licensed Rights under the terms and conditions of this 224 | Public License. 225 | 226 | b. Additional offer from the Licensor -- Adapted Material. 227 | Every recipient of Adapted Material from You 228 | automatically receives an offer from the Licensor to 229 | exercise the Licensed Rights in the Adapted Material 230 | under the conditions of the Adapter's License You apply. 231 | 232 | c. No downstream restrictions. You may not offer or impose 233 | any additional or different terms or conditions on, or 234 | apply any Effective Technological Measures to, the 235 | Licensed Material if doing so restricts exercise of the 236 | Licensed Rights by any recipient of the Licensed 237 | Material. 238 | 239 | 6. No endorsement. Nothing in this Public License constitutes or 240 | may be construed as permission to assert or imply that You 241 | are, or that Your use of the Licensed Material is, connected 242 | with, or sponsored, endorsed, or granted official status by, 243 | the Licensor or others designated to receive attribution as 244 | provided in Section 3(a)(1)(A)(i). 245 | 246 | b. Other rights. 247 | 248 | 1. Moral rights, such as the right of integrity, are not 249 | licensed under this Public License, nor are publicity, 250 | privacy, and/or other similar personality rights; however, to 251 | the extent possible, the Licensor waives and/or agrees not to 252 | assert any such rights held by the Licensor to the limited 253 | extent necessary to allow You to exercise the Licensed 254 | Rights, but not otherwise. 255 | 256 | 2. Patent and trademark rights are not licensed under this 257 | Public License. 258 | 259 | 3. To the extent possible, the Licensor waives any right to 260 | collect royalties from You for the exercise of the Licensed 261 | Rights, whether directly or through a collecting society 262 | under any voluntary or waivable statutory or compulsory 263 | licensing scheme. In all other cases the Licensor expressly 264 | reserves any right to collect such royalties. 265 | 266 | 267 | Section 3 -- License Conditions. 268 | 269 | Your exercise of the Licensed Rights is expressly made subject to the 270 | following conditions. 271 | 272 | a. Attribution. 273 | 274 | 1. If You Share the Licensed Material (including in modified 275 | form), You must: 276 | 277 | a. retain the following if it is supplied by the Licensor 278 | with the Licensed Material: 279 | 280 | i. identification of the creator(s) of the Licensed 281 | Material and any others designated to receive 282 | attribution, in any reasonable manner requested by 283 | the Licensor (including by pseudonym if 284 | designated); 285 | 286 | ii. a copyright notice; 287 | 288 | iii. a notice that refers to this Public License; 289 | 290 | iv. a notice that refers to the disclaimer of 291 | warranties; 292 | 293 | v. a URI or hyperlink to the Licensed Material to the 294 | extent reasonably practicable; 295 | 296 | b. indicate if You modified the Licensed Material and 297 | retain an indication of any previous modifications; and 298 | 299 | c. indicate the Licensed Material is licensed under this 300 | Public License, and include the text of, or the URI or 301 | hyperlink to, this Public License. 302 | 303 | 2. You may satisfy the conditions in Section 3(a)(1) in any 304 | reasonable manner based on the medium, means, and context in 305 | which You Share the Licensed Material. For example, it may be 306 | reasonable to satisfy the conditions by providing a URI or 307 | hyperlink to a resource that includes the required 308 | information. 309 | 310 | 3. If requested by the Licensor, You must remove any of the 311 | information required by Section 3(a)(1)(A) to the extent 312 | reasonably practicable. 313 | 314 | b. ShareAlike. 315 | 316 | In addition to the conditions in Section 3(a), if You Share 317 | Adapted Material You produce, the following conditions also apply. 318 | 319 | 1. The Adapter's License You apply must be a Creative Commons 320 | license with the same License Elements, this version or 321 | later, or a BY-SA Compatible License. 322 | 323 | 2. You must include the text of, or the URI or hyperlink to, the 324 | Adapter's License You apply. You may satisfy this condition 325 | in any reasonable manner based on the medium, means, and 326 | context in which You Share Adapted Material. 327 | 328 | 3. You may not offer or impose any additional or different terms 329 | or conditions on, or apply any Effective Technological 330 | Measures to, Adapted Material that restrict exercise of the 331 | rights granted under the Adapter's License You apply. 332 | 333 | 334 | Section 4 -- Sui Generis Database Rights. 335 | 336 | Where the Licensed Rights include Sui Generis Database Rights that 337 | apply to Your use of the Licensed Material: 338 | 339 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 340 | to extract, reuse, reproduce, and Share all or a substantial 341 | portion of the contents of the database; 342 | 343 | b. if You include all or a substantial portion of the database 344 | contents in a database in which You have Sui Generis Database 345 | Rights, then the database in which You have Sui Generis Database 346 | Rights (but not its individual contents) is Adapted Material, 347 | 348 | including for purposes of Section 3(b); and 349 | c. You must comply with the conditions in Section 3(a) if You Share 350 | all or a substantial portion of the contents of the database. 351 | 352 | For the avoidance of doubt, this Section 4 supplements and does not 353 | replace Your obligations under this Public License where the Licensed 354 | Rights include other Copyright and Similar Rights. 355 | 356 | 357 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 358 | 359 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 360 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 361 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 362 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 363 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 364 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 365 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 366 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 367 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 368 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 369 | 370 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 371 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 372 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 373 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 374 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 375 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 376 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 377 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 378 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 379 | 380 | c. The disclaimer of warranties and limitation of liability provided 381 | above shall be interpreted in a manner that, to the extent 382 | possible, most closely approximates an absolute disclaimer and 383 | waiver of all liability. 384 | 385 | 386 | Section 6 -- Term and Termination. 387 | 388 | a. This Public License applies for the term of the Copyright and 389 | Similar Rights licensed here. However, if You fail to comply with 390 | this Public License, then Your rights under this Public License 391 | terminate automatically. 392 | 393 | b. Where Your right to use the Licensed Material has terminated under 394 | Section 6(a), it reinstates: 395 | 396 | 1. automatically as of the date the violation is cured, provided 397 | it is cured within 30 days of Your discovery of the 398 | violation; or 399 | 400 | 2. upon express reinstatement by the Licensor. 401 | 402 | For the avoidance of doubt, this Section 6(b) does not affect any 403 | right the Licensor may have to seek remedies for Your violations 404 | of this Public License. 405 | 406 | c. For the avoidance of doubt, the Licensor may also offer the 407 | Licensed Material under separate terms or conditions or stop 408 | distributing the Licensed Material at any time; however, doing so 409 | will not terminate this Public License. 410 | 411 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 412 | License. 413 | 414 | 415 | Section 7 -- Other Terms and Conditions. 416 | 417 | a. The Licensor shall not be bound by any additional or different 418 | terms or conditions communicated by You unless expressly agreed. 419 | 420 | b. Any arrangements, understandings, or agreements regarding the 421 | Licensed Material not stated herein are separate from and 422 | independent of the terms and conditions of this Public License. 423 | 424 | 425 | Section 8 -- Interpretation. 426 | 427 | a. For the avoidance of doubt, this Public License does not, and 428 | shall not be interpreted to, reduce, limit, restrict, or impose 429 | conditions on any use of the Licensed Material that could lawfully 430 | be made without permission under this Public License. 431 | 432 | b. To the extent possible, if any provision of this Public License is 433 | deemed unenforceable, it shall be automatically reformed to the 434 | minimum extent necessary to make it enforceable. If the provision 435 | cannot be reformed, it shall be severed from this Public License 436 | without affecting the enforceability of the remaining terms and 437 | conditions. 438 | 439 | c. No term or condition of this Public License will be waived and no 440 | failure to comply consented to unless expressly agreed to by the 441 | Licensor. 442 | 443 | d. Nothing in this Public License constitutes or may be interpreted 444 | as a limitation upon, or waiver of, any privileges and immunities 445 | that apply to the Licensor or You, including from the legal 446 | processes of any jurisdiction or authority. 447 | 448 | 449 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 450 | 451 | Creative Commons is not a party to its public 452 | licenses. Notwithstanding, Creative Commons may elect to apply one of 453 | its public licenses to material it publishes and in those instances 454 | will be considered the “Licensor.” The text of the Creative Commons 455 | public licenses is dedicated to the public domain under the CC0 Public 456 | Domain Dedication. Except for the limited purpose of indicating that 457 | material is shared under a Creative Commons public license or as 458 | otherwise permitted by the Creative Commons policies published at 459 | creativecommons.org/policies, Creative Commons does not authorize the 460 | use of the trademark "Creative Commons" or any other trademark or logo 461 | of Creative Commons without its prior written consent including, 462 | without limitation, in connection with any unauthorized modifications 463 | to any of its public licenses or any other arrangements, 464 | understandings, or agreements concerning use of licensed material. For 465 | the avoidance of doubt, this paragraph does not form part of the 466 | public licenses. 467 | 468 | Creative Commons may be contacted at creativecommons.org. 469 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Python 3.6](https://img.shields.io/badge/Python-3.6-blue.svg) 2 | ![License](https://img.shields.io/badge/Code%20License-MIT-blue.svg) 3 | [![Mailing List](https://img.shields.io/badge/-Mailing%20List-lightgrey.svg)](https://groups.google.com/forum/#!forum/ann-and-dl-book) 4 | 5 | # Introduction to Artificial Neural Networks and Deep Learning: A Practical Guide with Applications in Python 6 | 7 | Repository for the book *Introduction to Artificial Neural Networks and Deep Learning: A Practical Guide with Applications in Python*. 8 | 9 | --- 10 | 11 | **Deep learning is not just the talk of the town among tech folks. Deep learning allows us to tackle complex problems, training artificial neural networks to recognize complex patterns for image and speech recognition. In this book, we'll continue where we left off in [*Python Machine Learning*](https://github.com/rasbt/python-machine-learning-book) and implement deep learning algorithms in [PyTorch](https://pytorch.org).** 12 | 13 | --- 14 | 15 | - This repository will contain the instructions, code examples, and solutions for the *Hands-On* and *Exercise* portions of each chapter. 16 | 17 | - PDF and ebook versions of the book will be available from [Leanpub](https://leanpub.com/ann-and-deeplearning). 18 | 19 | [![Deep Learning Book](images/ann-and-deeplearning-cover.jpg)](https://leanpub.com/ann-and-deeplearning) 20 | 21 | 22 | ISBN-10: [TBA] 23 | ISBN-13: [TBA] 24 | Paperback: est. 2018 25 | 26 | --- 27 | 28 | ## Manuscripts / Early Access Drafts 29 | 30 | - 01 - Introduction 31 | 32 | - 02 - The Perceptron 33 | 34 | - 03 - Optimizing Cost Functions with Gradient Descent 35 | 36 | - 04 - Logistic Regression and Softmax Regression 37 | 38 | - 05 - From Softmax Regression to Multilayer Perceptrons 39 | 40 | - 06 - Cross Validation and Performance Metrics 41 | 42 | - 07 - Regularization in Neural Networks 43 | 44 | - 08 - Learning Rates and Weight Initialization 45 | 46 | - 09 - Convolutional Neural Networks 47 | 48 | - 10 - Recurrent Neural Networks 49 | 50 | - 11 - Autoencoders 51 | 52 | - 12 - General Adverserial Neural Networks 53 | 54 | - 13 - Deep Generative Models 55 | 56 | - 14 - Reinforcement Learning 57 | 58 | #### Supporting Material 59 | 60 | - Appendix A: Mathematical Notation [[PDF](https://sebastianraschka.com/pdf/books/dlb/appendix_a_math_notation.pdf)] 61 | 62 | - Appendix B: Algebra Basics [[PDF](https://sebastianraschka.com/pdf/books/dlb/appendix_b_algebra.pdf)] 63 | 64 | - Appendix C: Linear Algebra Essentials 65 | 66 | - Appendix D: Calculus and Differentiation Primer [[PDF](https://sebastianraschka.com/pdf/books/dlb/appendix_d_calculus.pdf)] 67 | 68 | - Appendix E: Probability Theory Overview 69 | 70 | - Appendix F: Notational Conventions Reference 71 | 72 | - Appendix G: Python Setup 73 | 74 | - Appendix H: Introduction to NumPy [[PDF](https://sebastianraschka.com/pdf/books/dlb/appendix_numpy-intro.pdf)] [[Code Notebook](code/appendix_h_numpy-intro/appendix_numpy-intro.ipynb)] 75 | 76 | - Appendix I: PyTorch Basics 77 | 78 | - Appendix I (alt.): TensorFlow Basics [[PDF](https://sebastianraschka.com/pdf/books/dlb/appendix_g_tensorflow.pdf)] [[Code Notebook](code/_old-material/appendix_tensorflow-basics/appendix_tensorflow-basics.ipynb)] 79 | 80 | - Appendix J: Cloud Computing [[PDF](https://sebastianraschka.com/pdf/books/dlb/appendix_cloud-computing.pdf)] 81 | 82 | #### Model Zoo 83 | 84 | - **[Model Zoo: A collection of standalone TensorFlow and PyTorch models in Jupyter Notebooks](https://github.com/rasbt/deeplearning-models)** 85 | 86 | 87 | 88 | --- 89 | 90 | ## About the Book 91 | 92 | Machine learning has become a central part of our life — as consumers, customers, and hopefully as researchers and practitioners! I appreciate all the nice feedback that you sent me about [*Python Machine Learning*](https://github.com/rasbt/python-machine-learning-book), and I am so happy to hear that you found it so useful as a learning guide, helping you with your business applications and research projects. I have received many emails since its release. Also, in these very emails, you were asking me about a possible prequel or sequel. 93 | 94 | Initially, I was inclined to write more about the "math" parts, which can be a real hurdle for almost everyone without (or even with) a math major in college. Initially, I thought that writing a book about "machine learning math" was a cool thing to do. Now, I have ~15 chapters worth of notes about pre-calculus, calculus, linear algebra, statistics, and probability theory. However, I eventually came to a conclusion that there were too many other math books out there, already! Most of them are far better and more comprehensive and accurate than my potential ~500-page introduction to the topics that I had in store. After all, I think that the real motivation for learning and understanding a subject comes from being excited about it in the first place; if you are passionate about machine learning and you stumble upon the chain rule in calculus, you wouldn't have any problems to find a trusted resource via your favorite search engine these days. 95 | 96 | So, instead of writing that "prequel," let me write about something that's built upon the concepts that I introduced in the later chapters of [*Python Machine Learning*](https://github.com/rasbt/python-machine-learning-book) -- algorithms for deep learning. After we coded a multi-layer perceptron (a certain kind of feedforward artificial neural network) from scratch, we took a brief look at some Python libraries for implementing deep learning algorithms, and I introduced convolutional and recurrent neural networks on a conceptual level. 97 | 98 | In this book, I want to continue where I left off and want to implement deep neural networks and algorithms for deep learning algorithms from scratch, using Python, NumPy, and SciPy throughout this educational journey. In addition to the vanilla Python science-stack, we will implement these algorithms in [TensorFlow](https://www.tensorflow.org), highly performant yet very accessible deep learning library for implementing and applying deep learning to real-world problems. 99 | 100 | ## License 101 | 102 | ### Code 103 | 104 | All code in this repository (including the code examples in Jupyter Notebooks) is open source content, released under the [MIT software license](LICENSE). In short, the permissive MIT license allows you to do anything with the code with proper attribution and without warranty; please check the MIT [LICENSE](LICENSE) notice for further details. 105 | 106 | ### Text and Graphics 107 | 108 | All non-code content and creative work in this repository, including text and graphics, is under exclusive copyright by the author, Sebastian Raschka. Unless noted otherwise, text content shared in this repository is intended for personal use only. You may use, modify, or share short text passages of this work with proper attribution to the author. However, if you are planning to modify and/or share substantial portions of this book for other writings, such as blog posts, magazine article, or teaching material, contact the author for permission. 109 | 110 | Figures and graphics marked by a *Creative Commons Attribution-ShareAlike 4.0 International* are free to share under the respective license terms (as listed in the *Creative Commons Attribution-ShareAlike 4.0 International* section in the [LICENSE](LICENSE) file) and proper attribution. 111 | 112 | 113 | ## Acknowledgements 114 | 115 | I would like to give my special thanks to the readers, who caught various typos and errors and offered suggestions for clarifying my writing. 116 | 117 | - Appendix A: Artem Sobolev, Ryan Sun 118 | - Appendix B: Brett Miller, Ryan Sun 119 | - Appendix D: Marcel Blattner, Ignacio Campabadal, Ryan Sun 120 | - Appendix F: Guillermo Moncecchi‏, Ged Ridgway 121 | - Appendix H: Brett Miller 122 | -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/appendix_g_tensorflow-basics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "*Accompanying code examples of the book \"Introduction to Artificial Neural Networks and Deep Learning: A Practical Guide with Applications in Python\" by [Sebastian Raschka](https://sebastianraschka.com). All code examples are released under the [MIT license](https://github.com/rasbt/deep-learning-book/blob/master/LICENSE). If you find this content useful, please consider supporting the work by buying a [copy of the book](https://leanpub.com/ann-and-deeplearning).*\n", 8 | "\n", 9 | "Other code examples and content are available on [GitHub](https://github.com/rasbt/deep-learning-book). The PDF and ebook versions of the book are available through [Leanpub](https://leanpub.com/ann-and-deeplearning)." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "# Appendix G - TensorFlow Basics" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "Sebastian Raschka 2017-07-23 \n", 29 | "\n", 30 | "tensorflow 1.2.0\n", 31 | "numpy 1.13.1\n" 32 | ] 33 | } 34 | ], 35 | "source": [ 36 | "%load_ext watermark\n", 37 | "%watermark -a 'Sebastian Raschka' -d -p tensorflow,numpy" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "### Table of Contents\n", 45 | "\n", 46 | "\n", 47 | "- TensorFlow in a Nutshell\n", 48 | "- Installation\n", 49 | "- Computation Graphs Variables\n", 50 | "- Placeholder Variables\n", 51 | "- Saving and Restoring Models\n", 52 | "- Naming TensorFlow Objects\n", 53 | "- CPU and GPU\n", 54 | "- TensorBoard" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "\n", 62 | "This appendix offers a brief overview of TensorFlow, an open-source library for numerical computation and deep learning. This section is intended for readers who want to gain a basic overview of this library before progressing through the hands-on sections that are concluding the main chapters.\n", 63 | "\n", 64 | "The majority of *hands-on* sections in this book focus on TensorFlow and its Python API, assuming that you have TensorFlow >=1.2 installed if you are planning to execute the code sections shown in this book.\n", 65 | "\n", 66 | "In addition to glancing over this appendix, I recommend the following resources from TensorFlow's official documentation for a more in-depth coverage on using TensorFlow:\n", 67 | "\n", 68 | "- **[Download and setup instructions](https://www.tensorflow.org/get_started/os_setup)**\n", 69 | "- **[Python API documentation](https://www.tensorflow.org/api_docs/python/)**\n", 70 | "- **[Tutorials](https://www.tensorflow.org/tutorials/)**\n", 71 | "- **[TensorBoard, an optional tool for visualizing learning](https://www.tensorflow.org/how_tos/summaries_and_tensorboard/)**" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "## TensorFlow in a Nutshell" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "At its core, TensorFlow is a library for efficient multidimensional array operations with a focus on deep learning. Developed by the Google Brain Team, TensorFlow was open-sourced on November 9th, 2015. And augmented by its convenient Python API layer, TensorFlow has gained much popularity and wide-spread adoption in industry as well as academia.\n", 86 | "\n", 87 | "TensorFlow shares some similarities with NumPy, such as providing data structures and computations based on multidimensional arrays. What makes TensorFlow particularly suitable for deep learning, though, are its primitives for defining functions on tensors, the ability of parallelizing tensor operations, and convenience tools such as automatic differentiation.\n", 88 | "\n", 89 | "While TensorFlow can be run entirely on a CPU or multiple CPUs, one of the core strength of this library is its support of GPUs (Graphical Processing Units) that are very efficient at performing highly parallelized numerical computations. In addition, TensorFlow also supports distributed systems as well as mobile computing platforms, including Android and Apple's iOS.\n", 90 | "\n", 91 | "But what is a *tensor*? In simplifying terms, we can think of tensors as multidimensional arrays of numbers, as a generalization of scalars, vectors, and matrices.\n", 92 | "\n", 93 | "1. Scalar: $\\mathbb{R}$\n", 94 | "2. Vector: $\\mathbb{R}^n$\n", 95 | "3. Matrix: $\\mathbb{R}^n \\times \\mathbb{R}^m$\n", 96 | "4. 3-Tensor: $\\mathbb{R}^n \\times \\mathbb{R}^m \\times \\mathbb{R}^p$\n", 97 | "5. ...\n", 98 | "\n", 99 | "When we describe tensors, we refer to its \"dimensions\" as the *rank* (or *order*) of a tensor, which is not to be confused with the dimensions of a matrix. For instance, an $m \\times n$ matrix, where $m$ is the number of rows and $n$ is the number of columns, would be a special case of a rank-2 tensor. A visual explanation of tensors and their ranks is given is the figure below.\n", 100 | "\n", 101 | "![Tensors](images/tensors.png)\n" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "## Installation" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "Code conventions in this book follow the Python 3.x syntax, and while the code examples should be backward compatible to Python 2.7, I highly recommend the use of Python >=3.5.\n", 116 | "\n", 117 | "Once you have your Python Environment set up ([Appendix - Python Setup]), the most convenient ways for installing TensorFlow are via `pip` or `conda` -- the latter only applies if you have the Anaconda/Miniconda Python distribution installed, which I prefer and recommend.\n", 118 | "\n", 119 | "Since TensorFlow is under active development, I recommend you to consult the official \"[Download and Setup](https://www.tensorflow.org/get_started/os_setup)\" documentation for detailed installation instructions to install TensorFlow on you operating system, macOS, Linux, or Windows.\n" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "## Computation Graphs" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "In contrast to other tools such as NumPy, the numerical computations in TensorFlow can be categorized into two steps: a construction step and an execution step. Consequently, the typical workflow in TensorFlow can be summarized as follows:\n", 134 | "\n", 135 | "- Build a computational graph\n", 136 | "- Start a new *session* to evaluate the graph\n", 137 | " - Initialize variables\n", 138 | " - Execute the operations in the compiled graph\n" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "Note that the computation graph has no numerical values before we initialize and evaluate it. To see how this looks like in practice, let us set up a new graph for computing the column sums of a matrix, which we define as a constant tensor (`reduce_sum` is the TensorFlow equivalent of NumPy's `sum` function).\n" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 1, 151 | "metadata": {}, 152 | "outputs": [ 153 | { 154 | "name": "stdout", 155 | "output_type": "stream", 156 | "text": [ 157 | "tf_x:\n", 158 | " Tensor(\"Const:0\", shape=(3, 2), dtype=float32)\n", 159 | "\n", 160 | "col_sum:\n", 161 | " Tensor(\"Sum:0\", shape=(2,), dtype=float32)\n" 162 | ] 163 | } 164 | ], 165 | "source": [ 166 | "import tensorflow as tf\n", 167 | "\n", 168 | "g = tf.Graph()\n", 169 | "\n", 170 | "with g.as_default() as g:\n", 171 | " tf_x = tf.constant([[1., 2.], \n", 172 | " [3., 4.],\n", 173 | " [5., 6.]], dtype=tf.float32)\n", 174 | " col_sum = tf.reduce_sum(tf_x, axis=0)\n", 175 | "\n", 176 | "print('tf_x:\\n', tf_x)\n", 177 | "print('\\ncol_sum:\\n', col_sum)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "As we can see from the output above, the operations in the graph are represented as `Tensor` objects that require an explicit evaluation before the `tf_x` matrix is populated with numerical values and its column sum gets computed.\n", 185 | "\n", 186 | "Now, we pass the graph that we created earlier to a new, active *session*, where the graph gets compiled and evaluated:" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 2, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "mat:\n", 199 | " [[ 1. 2.]\n", 200 | " [ 3. 4.]\n", 201 | " [ 5. 6.]]\n", 202 | "\n", 203 | "csum:\n", 204 | " [ 9. 12.]\n" 205 | ] 206 | } 207 | ], 208 | "source": [ 209 | "with tf.Session(graph=g) as sess:\n", 210 | " mat, csum = sess.run([tf_x, col_sum])\n", 211 | " \n", 212 | "print('mat:\\n', mat)\n", 213 | "print('\\ncsum:\\n', csum)" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "Note that if we are only interested in the result of a particular operation, we don't need to `run` its dependencies -- TensorFlow will automatically take care of that. For instance, we can directly fetch the numerical values of `col_sum_times_2` in the active session without explicitly passing `col_sum` to `sess.run(...)` as the following example illustrates:" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 3, 226 | "metadata": {}, 227 | "outputs": [ 228 | { 229 | "name": "stdout", 230 | "output_type": "stream", 231 | "text": [ 232 | "csum_2:\n", 233 | " [ 18. 24.]\n" 234 | ] 235 | } 236 | ], 237 | "source": [ 238 | "g = tf.Graph()\n", 239 | "\n", 240 | "with g.as_default() as g:\n", 241 | " tf_x = tf.constant([[1., 2.], \n", 242 | " [3., 4.],\n", 243 | " [5., 6.]], dtype=tf.float32)\n", 244 | " col_sum = tf.reduce_sum(tf_x, axis=0)\n", 245 | " col_sum_times_2 = col_sum * 2\n", 246 | "\n", 247 | "\n", 248 | "with tf.Session(graph=g) as sess:\n", 249 | " csum_2 = sess.run(col_sum_times_2)\n", 250 | " \n", 251 | "print('csum_2:\\n', csum_2)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "## Variables" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "Variables are constructs in TensorFlow that allows us to store and update parameters of our models in the current session during training. To define a \"variable\" tensor, we use TensorFlow's `Variable()` constructor, which looks similar to the use of `constant` that we used to create a matrix previously. However, to execute a computational graph that contains variables, we must initialize all variables in the active session first (using `tf.global_variables_initializer()`), as illustrated in the example below.\n" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 1, 271 | "metadata": {}, 272 | "outputs": [ 273 | { 274 | "name": "stdout", 275 | "output_type": "stream", 276 | "text": [ 277 | "[[ 2. 3.]\n", 278 | " [ 4. 5.]\n", 279 | " [ 6. 7.]]\n" 280 | ] 281 | } 282 | ], 283 | "source": [ 284 | "import tensorflow as tf\n", 285 | "\n", 286 | "g = tf.Graph()\n", 287 | "with g.as_default() as g:\n", 288 | " tf_x = tf.Variable([[1., 2.], \n", 289 | " [3., 4.],\n", 290 | " [5., 6.]], dtype=tf.float32)\n", 291 | " x = tf.constant(1., dtype=tf.float32)\n", 292 | " \n", 293 | " # add a constant to the matrix:\n", 294 | " tf_x = tf_x + x\n", 295 | " \n", 296 | "with tf.Session(graph=g) as sess:\n", 297 | " sess.run(tf.global_variables_initializer())\n", 298 | " result = sess.run(tf_x)\n", 299 | " \n", 300 | "print(result)" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "Now, let us do an experiment and evaluate the same graph twice:" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 2, 313 | "metadata": {}, 314 | "outputs": [ 315 | { 316 | "name": "stdout", 317 | "output_type": "stream", 318 | "text": [ 319 | "[[ 2. 3.]\n", 320 | " [ 4. 5.]\n", 321 | " [ 6. 7.]]\n" 322 | ] 323 | } 324 | ], 325 | "source": [ 326 | "with tf.Session(graph=g) as sess:\n", 327 | " sess.run(tf.global_variables_initializer())\n", 328 | " result = sess.run(tf_x)\n", 329 | " result = sess.run(tf_x)\n", 330 | "\n", 331 | "print(result)" 332 | ] 333 | }, 334 | { 335 | "cell_type": "markdown", 336 | "metadata": {}, 337 | "source": [ 338 | "As we can see, the result of running the computation twice did not affect the numerical values fetched from the graph. To update or to assign new values to a variable, we use TensorFlow's `assign` operation. The function syntax of `assign` is `assign(ref, val, ...)`, where '`ref`' is updated by assigning '`value`' to it:\n" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 3, 344 | "metadata": {}, 345 | "outputs": [ 346 | { 347 | "name": "stdout", 348 | "output_type": "stream", 349 | "text": [ 350 | "[[ 3. 4.]\n", 351 | " [ 5. 6.]\n", 352 | " [ 7. 8.]]\n" 353 | ] 354 | } 355 | ], 356 | "source": [ 357 | "g = tf.Graph()\n", 358 | "\n", 359 | "with g.as_default() as g:\n", 360 | " tf_x = tf.Variable([[1., 2.], \n", 361 | " [3., 4.],\n", 362 | " [5., 6.]], dtype=tf.float32)\n", 363 | " x = tf.constant(1., dtype=tf.float32)\n", 364 | " \n", 365 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 366 | "\n", 367 | "\n", 368 | "with tf.Session(graph=g) as sess:\n", 369 | " sess.run(tf.global_variables_initializer())\n", 370 | " result = sess.run(update_tf_x)\n", 371 | " result = sess.run(update_tf_x)\n", 372 | "\n", 373 | "print(result)" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": {}, 379 | "source": [ 380 | "As we can see, the contents of the variable `tf_x` were successfully updated twice now; in the active session we\n", 381 | "\n", 382 | "- initialized the variable `tf_x`\n", 383 | "- added a constant scalar `1.` to `tf_x` matrix via `assign`\n", 384 | "- added a constant scalar `1.` to the previously updated `tf_x` matrix via `assign`\n", 385 | "\n", 386 | "Although the example above is kept simple for illustrative purposes, variables are an\n", 387 | "important concept in TensorFlow, and we will see throughout the chapters, they are\n", 388 | "not only useful for updating model parameters but also for saving and loading\n", 389 | "variables for reuse." 390 | ] 391 | }, 392 | { 393 | "cell_type": "markdown", 394 | "metadata": {}, 395 | "source": [ 396 | "## Placeholder Variables" 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "\n", 404 | "Another important concept in TensorFlow is the use of placeholder variables,\n", 405 | "which allow us to feed the computational graph with numerical values in an active session at runtime.\n", 406 | "\n", 407 | "In the following example, we will define a computational graph that performs a\n", 408 | "simple matrix multiplication operation. First, we define a placeholder variable\n", 409 | "that can hold 3x2-dimensional matrices. And after initializing the placeholder\n", 410 | "variable in the active session, we will use a dictionary, `feed_dict` we feed\n", 411 | "a NumPy array to the graph, which then evaluates the matrix multiplication operation.\n" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 1, 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "name": "stdout", 421 | "output_type": "stream", 422 | "text": [ 423 | "[[ 25. 39. 53.]\n", 424 | " [ 39. 61. 83.]\n", 425 | " [ 53. 83. 113.]]\n" 426 | ] 427 | } 428 | ], 429 | "source": [ 430 | "import tensorflow as tf\n", 431 | "import numpy as np\n", 432 | "\n", 433 | "g = tf.Graph()\n", 434 | "with g.as_default() as g:\n", 435 | " tf_x = tf.placeholder(dtype=tf.float32,\n", 436 | " shape=(3, 2))\n", 437 | "\n", 438 | " output = tf.matmul(tf_x, tf.transpose(tf_x))\n", 439 | "\n", 440 | "\n", 441 | "with tf.Session(graph=g) as sess:\n", 442 | " sess.run(tf.global_variables_initializer())\n", 443 | " np_ary = np.array([[3., 4.],\n", 444 | " [5., 6.],\n", 445 | " [7., 8.]])\n", 446 | " feed_dict = {tf_x: np_ary}\n", 447 | " print(sess.run(output,\n", 448 | " feed_dict=feed_dict))" 449 | ] 450 | }, 451 | { 452 | "cell_type": "markdown", 453 | "metadata": {}, 454 | "source": [ 455 | "Throughout the main chapters, we will make heavy use of placeholder variables,\n", 456 | "which allow us to pass our datasets to various learning algorithms\n", 457 | "in the computational graphs.\n" 458 | ] 459 | }, 460 | { 461 | "cell_type": "markdown", 462 | "metadata": {}, 463 | "source": [ 464 | "## Saving and Loading Variables" 465 | ] 466 | }, 467 | { 468 | "cell_type": "markdown", 469 | "metadata": {}, 470 | "source": [ 471 | "Training deep neural networks requires a lot of computations and computational resources, and in practice, it would be infeasible to retrain our model each time we start a new TensorFlow session before we can use it to make predictions. In this section, we will go over the basics of saving and re-using the results of our TensorFlow models.\n", 472 | "\n", 473 | "The most convenient way to store the main components of our model is to use TensorFlows `Saver` class (`tf.train.Saver()`). To see how it works, let us reuse the simple example from the [Variables](#variables) section, where we added a constant `1.` to all elements in a 3x2 matrix:\n" 474 | ] 475 | }, 476 | { 477 | "cell_type": "code", 478 | "execution_count": 1, 479 | "metadata": { 480 | "collapsed": true 481 | }, 482 | "outputs": [], 483 | "source": [ 484 | "import tensorflow as tf\n", 485 | "\n", 486 | "g = tf.Graph()\n", 487 | "with g.as_default() as g:\n", 488 | " \n", 489 | " tf_x = tf.Variable([[1., 2.], \n", 490 | " [3., 4.],\n", 491 | " [5., 6.]], dtype=tf.float32)\n", 492 | " x = tf.constant(1., dtype=tf.float32)\n", 493 | " \n", 494 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 495 | " \n", 496 | " # initialize a Saver, which gets all variables\n", 497 | " # within this computation graph context\n", 498 | " saver = tf.train.Saver()" 499 | ] 500 | }, 501 | { 502 | "cell_type": "markdown", 503 | "metadata": {}, 504 | "source": [ 505 | "Now, after we initialized the graph above, let us execute its operations in a new session:" 506 | ] 507 | }, 508 | { 509 | "cell_type": "code", 510 | "execution_count": 2, 511 | "metadata": { 512 | "collapsed": true 513 | }, 514 | "outputs": [], 515 | "source": [ 516 | "with tf.Session(graph=g) as sess:\n", 517 | " sess.run(tf.global_variables_initializer())\n", 518 | " result = sess.run(update_tf_x)\n", 519 | " \n", 520 | " saver.save(sess, save_path='./my-model.ckpt')" 521 | ] 522 | }, 523 | { 524 | "cell_type": "markdown", 525 | "metadata": {}, 526 | "source": [ 527 | "Notice the `saver.save` call above, which saves all variables in the graph to \"checkpoint\" files bearing the prefix `my-model.ckpt` in our local directory (`'./'`). Since we didn't specify which variables we wanted to save when we instantiated a `tf.train.Saver()`, it saved all variables in the graph by default -- here, we only have one variable, `tf_x`. Alternatively, if we are only interested in keeping particular variables, we can specify this by feeding `tf.train.Saver()` a dictionary or list of these variables upon instantiation. For example, if our graph contained more than one variable, but we were only interested in saving `tf_x`, we could instantiate a `saver` object as `tf.train.Saver([tf_x])`.\n", 528 | "\n", 529 | "After we executed the previous code example, we should find the three `my-model.ckpt` files (in binary format) in our local directory:\n", 530 | "\n", 531 | "- `my-model.ckpt.data-00000-of-00001`\n", 532 | "- `my-model.ckpt.index`\n", 533 | "- `my-model.ckpt.meta`\n", 534 | "\n", 535 | "The file `my-model.ckpt.data-00000-of-00001` saves our main variable values, the `.index` file keeps track of the data structures, and the `.meta` file describes the structure of our computational graph that we executed.\n", 536 | "\n", 537 | "Note that in our simple example above, we just saved our variable one single time. However, in real-world applications, we typically train models over multiple iterations or epochs, and it is useful to create intermediate checkpoint files during training so that we can pick up where we left off in case we need to interrupt our session or encounter unforeseen technical difficulties. For instance, by using the `global_step` parameter, we could save our results after each 10th iteration by making the following modification to our code:\n" 538 | ] 539 | }, 540 | { 541 | "cell_type": "code", 542 | "execution_count": 3, 543 | "metadata": { 544 | "collapsed": true 545 | }, 546 | "outputs": [], 547 | "source": [ 548 | "g = tf.Graph()\n", 549 | "with g.as_default() as g:\n", 550 | " \n", 551 | " tf_x = tf.Variable([[1., 2.], \n", 552 | " [3., 4.],\n", 553 | " [5., 6.]], dtype=tf.float32)\n", 554 | " x = tf.constant(1., dtype=tf.float32)\n", 555 | " \n", 556 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 557 | " \n", 558 | " # initialize a Saver, which gets all variables\n", 559 | " # within this computation graph context\n", 560 | " saver = tf.train.Saver()\n", 561 | "\n", 562 | "with tf.Session(graph=g) as sess:\n", 563 | " sess.run(tf.global_variables_initializer())\n", 564 | " \n", 565 | " for epoch in range(100):\n", 566 | " result = sess.run(update_tf_x)\n", 567 | " if not epoch % 10:\n", 568 | " saver.save(sess, \n", 569 | " save_path='./my-model-multiple_ckpts.ckpt', \n", 570 | " global_step=epoch)\n", 571 | " " 572 | ] 573 | }, 574 | { 575 | "cell_type": "markdown", 576 | "metadata": {}, 577 | "source": [ 578 | "After we executed this code we find five `my-model.ckpt` files in our local directory:\n", 579 | "\n", 580 | "- `my-model.ckpt-50 {.data-00000-of-00001, .ckpt.index, .ckpt.meta}`\n", 581 | "- `my-model.ckpt-60 {.data-00000-of-00001, .ckpt.index, .ckpt.meta}`\n", 582 | "- `my-model.ckpt-70 {.data-00000-of-00001, .ckpt.index, .ckpt.meta}`\n", 583 | "- `my-model.ckpt-80 {.data-00000-of-00001, .ckpt.index, .ckpt.meta}`\n", 584 | "- `my-model.ckpt-90 {.data-00000-of-00001, .ckpt.index, .ckpt.meta}`\n", 585 | "\n", 586 | "Although we saved our variables ten times, the `saver` only keeps the five most recent checkpoints by default to save storage space. However, if we want to keep more than five recent checkpoint files, we can provide an optional argument `max_to_keep=n` when we initialize the `saver`, where `n` is an integer specifying the number of the most recent checkpoint files we want to keep.\n", 587 | "\n", 588 | "Now that we learned how to save TensorFlow `Variable`s, let us see how we can restore them. Assuming that we started a fresh computational session, we need to specify the graph first. Then, we can use the `saver`'s `restore` method to restore our variables as shown below:\n" 589 | ] 590 | }, 591 | { 592 | "cell_type": "code", 593 | "execution_count": 4, 594 | "metadata": {}, 595 | "outputs": [ 596 | { 597 | "name": "stdout", 598 | "output_type": "stream", 599 | "text": [ 600 | "INFO:tensorflow:Restoring parameters from ./my-model.ckpt\n", 601 | "[[ 3. 4.]\n", 602 | " [ 5. 6.]\n", 603 | " [ 7. 8.]]\n" 604 | ] 605 | } 606 | ], 607 | "source": [ 608 | "g = tf.Graph()\n", 609 | "with g.as_default() as g:\n", 610 | " \n", 611 | " tf_x = tf.Variable([[1., 2.], \n", 612 | " [3., 4.],\n", 613 | " [5., 6.]], dtype=tf.float32)\n", 614 | " x = tf.constant(1., dtype=tf.float32)\n", 615 | " \n", 616 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 617 | " \n", 618 | " # initialize a Saver, which gets all variables\n", 619 | " # within this computation graph context\n", 620 | " saver = tf.train.Saver()\n", 621 | "\n", 622 | "with tf.Session(graph=g) as sess:\n", 623 | " saver.restore(sess, save_path='./my-model.ckpt')\n", 624 | " result = sess.run(update_tf_x)\n", 625 | " print(result)" 626 | ] 627 | }, 628 | { 629 | "cell_type": "markdown", 630 | "metadata": {}, 631 | "source": [ 632 | "Notice that the returned values of the `tf_x` `Variable` are now increased by a constant of two, compared to the values in the computational graph. The reason is that we ran the graph one time before we saved the variable,\n", 633 | "\n", 634 | "```python\n", 635 | "with tf.Session(graph=g) as sess:\n", 636 | " sess.run(tf.global_variables_initializer())\n", 637 | " result = sess.run(update_tf_x)\n", 638 | "\n", 639 | " # save the model\n", 640 | " saver.save(sess, save_path='./my-model.ckpt')\n", 641 | "```\n", 642 | "\n", 643 | "and we ran it a second time when after we restored the session.\n", 644 | "\n", 645 | "\n", 646 | "Similar to the example above, we can reload one of our checkpoint files by providing the desired checkpoint suffix (here: `-90`, which is the index of our last checkpoint):\n", 647 | "\n" 648 | ] 649 | }, 650 | { 651 | "cell_type": "code", 652 | "execution_count": 5, 653 | "metadata": {}, 654 | "outputs": [ 655 | { 656 | "name": "stdout", 657 | "output_type": "stream", 658 | "text": [ 659 | "INFO:tensorflow:Restoring parameters from ./my-model-multiple_ckpts.ckpt-90\n", 660 | "[[ 93. 94.]\n", 661 | " [ 95. 96.]\n", 662 | " [ 97. 98.]]\n" 663 | ] 664 | } 665 | ], 666 | "source": [ 667 | "with tf.Session(graph=g) as sess:\n", 668 | " saver.restore(sess, save_path='./my-model-multiple_ckpts.ckpt-90')\n", 669 | " result = sess.run(update_tf_x)\n", 670 | " print(result)" 671 | ] 672 | }, 673 | { 674 | "cell_type": "markdown", 675 | "metadata": {}, 676 | "source": [ 677 | "In this section, we merely covered the basics of saving and restoring TensorFlow models. If you want to learn more, please take a look at the official [API documentation](https://www.tensorflow.org/api_docs/python/tf/train/Saver) of TensorFlow's `Saver` class.\n" 678 | ] 679 | }, 680 | { 681 | "cell_type": "markdown", 682 | "metadata": {}, 683 | "source": [ 684 | "## Naming TensorFlow Objects" 685 | ] 686 | }, 687 | { 688 | "cell_type": "markdown", 689 | "metadata": {}, 690 | "source": [ 691 | "When we create new TensorFlow objects like `Variables`, we can provide an optional argument for their `name` parameter -- for example:\n", 692 | "\n", 693 | "```python\n", 694 | "tf_x = tf.Variable([[1., 2.],\n", 695 | " [3., 4.],\n", 696 | " [5., 6.]],\n", 697 | " name='tf_x_0',\n", 698 | " dtype=tf.float32)\n", 699 | "```\n", 700 | "\n", 701 | "Assigning names to `Variable`s explicitly is not a requirement, but I personally recommend making it a habit when building (more) complex models. Let us walk through a scenario to illustrate the importance of naming variables, taking the simple example from the previous section and add new variable `tf_y` to the graph:\n" 702 | ] 703 | }, 704 | { 705 | "cell_type": "code", 706 | "execution_count": 1, 707 | "metadata": { 708 | "collapsed": true 709 | }, 710 | "outputs": [], 711 | "source": [ 712 | "import tensorflow as tf\n", 713 | "\n", 714 | "g = tf.Graph()\n", 715 | "with g.as_default() as g:\n", 716 | " \n", 717 | " tf_x = tf.Variable([[1., 2.], \n", 718 | " [3., 4.],\n", 719 | " [5., 6.]], dtype=tf.float32)\n", 720 | " \n", 721 | " tf_y = tf.Variable([[7., 8.], \n", 722 | " [9., 10.],\n", 723 | " [11., 12.]], dtype=tf.float32)\n", 724 | " \n", 725 | " x = tf.constant(1., dtype=tf.float32)\n", 726 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 727 | " saver = tf.train.Saver()\n", 728 | " \n", 729 | "with tf.Session(graph=g) as sess:\n", 730 | " sess.run(tf.global_variables_initializer())\n", 731 | " result = sess.run(update_tf_x)\n", 732 | " \n", 733 | " saver.save(sess, save_path='./my-model.ckpt')" 734 | ] 735 | }, 736 | { 737 | "cell_type": "markdown", 738 | "metadata": {}, 739 | "source": [ 740 | "The variable `tf_y` does not do anything in the code example above; we added it for illustrative purposes, as we will see in a moment. Now, let us assume we started a new computational session and loaded our saved my-model into the following computational graph:\n" 741 | ] 742 | }, 743 | { 744 | "cell_type": "code", 745 | "execution_count": 2, 746 | "metadata": {}, 747 | "outputs": [ 748 | { 749 | "name": "stdout", 750 | "output_type": "stream", 751 | "text": [ 752 | "INFO:tensorflow:Restoring parameters from ./my-model.ckpt\n", 753 | "[[ 8. 9.]\n", 754 | " [ 10. 11.]\n", 755 | " [ 12. 13.]]\n" 756 | ] 757 | } 758 | ], 759 | "source": [ 760 | "g = tf.Graph()\n", 761 | "with g.as_default() as g:\n", 762 | " \n", 763 | " tf_y = tf.Variable([[7., 8.], \n", 764 | " [9., 10.],\n", 765 | " [11., 12.]], dtype=tf.float32)\n", 766 | " \n", 767 | " tf_x = tf.Variable([[1., 2.], \n", 768 | " [3., 4.],\n", 769 | " [5., 6.]], dtype=tf.float32)\n", 770 | " \n", 771 | " x = tf.constant(1., dtype=tf.float32)\n", 772 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 773 | " saver = tf.train.Saver()\n", 774 | " \n", 775 | "with tf.Session(graph=g) as sess:\n", 776 | " saver.restore(sess, save_path='./my-model.ckpt')\n", 777 | " result = sess.run(update_tf_x)\n", 778 | " print(result)" 779 | ] 780 | }, 781 | { 782 | "cell_type": "markdown", 783 | "metadata": {}, 784 | "source": [ 785 | "Unless you paid close attention on how we initialized the graph above, this result above surely was not the one you expected. What happened? Intuitively, we expected our session to `print` \n", 786 | "\n", 787 | "```python\n", 788 | " [[ 3. 4.]\n", 789 | " [ 5. 6.]\n", 790 | " [ 7. 8.]]\n", 791 | "```" 792 | ] 793 | }, 794 | { 795 | "cell_type": "markdown", 796 | "metadata": {}, 797 | "source": [ 798 | "The explanation behind this unexpected `result` is that we reversed the order of `tf_y` and `tf_x` in the graph above. TensorFlow applies a default naming scheme to all operations in the computational graph, unless we use do it explicitly via the `name` parameter -- or in other words, we confused TensorFlow by reversing the order of two similar objects, `tf_y` and `tf_x`.\n", 799 | "\n", 800 | "To circumvent this problem, we could give our variables specific names -- for example, `'tf_x_0'` and `'tf_y_0'`:\n" 801 | ] 802 | }, 803 | { 804 | "cell_type": "code", 805 | "execution_count": 1, 806 | "metadata": { 807 | "collapsed": true 808 | }, 809 | "outputs": [], 810 | "source": [ 811 | "import tensorflow as tf\n", 812 | "\n", 813 | "g = tf.Graph()\n", 814 | "with g.as_default() as g:\n", 815 | " \n", 816 | " tf_x = tf.Variable([[1., 2.], \n", 817 | " [3., 4.],\n", 818 | " [5., 6.]], \n", 819 | " name='tf_x_0',\n", 820 | " dtype=tf.float32)\n", 821 | " \n", 822 | " tf_y = tf.Variable([[7., 8.], \n", 823 | " [9., 10.,\n", 824 | " ]], \n", 825 | " name='tf_y_0',\n", 826 | " dtype=tf.float32)\n", 827 | " \n", 828 | " x = tf.constant(1., dtype=tf.float32)\n", 829 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 830 | " saver = tf.train.Saver()\n", 831 | "\n", 832 | "with tf.Session(graph=g) as sess:\n", 833 | " sess.run(tf.global_variables_initializer())\n", 834 | " result = sess.run(update_tf_x)\n", 835 | " \n", 836 | " saver.save(sess, save_path='./my-model.ckpt')" 837 | ] 838 | }, 839 | { 840 | "cell_type": "markdown", 841 | "metadata": {}, 842 | "source": [ 843 | "Then, even if we flip the order of these variables in a new computational graph, TensorFlow knows which values to use for each variable when loading our model -- assuming we provide the corresponding variable names:" 844 | ] 845 | }, 846 | { 847 | "cell_type": "code", 848 | "execution_count": 2, 849 | "metadata": {}, 850 | "outputs": [ 851 | { 852 | "name": "stdout", 853 | "output_type": "stream", 854 | "text": [ 855 | "INFO:tensorflow:Restoring parameters from ./my-model.ckpt\n", 856 | "[[ 3. 4.]\n", 857 | " [ 5. 6.]\n", 858 | " [ 7. 8.]]\n" 859 | ] 860 | } 861 | ], 862 | "source": [ 863 | "g = tf.Graph()\n", 864 | "with g.as_default() as g:\n", 865 | " \n", 866 | " tf_y = tf.Variable([[7., 8.], \n", 867 | " [9., 10.,\n", 868 | " ]], \n", 869 | " name='tf_y_0',\n", 870 | " dtype=tf.float32)\n", 871 | " \n", 872 | " tf_x = tf.Variable([[1., 2.], \n", 873 | " [3., 4.],\n", 874 | " [5., 6.]], \n", 875 | " name='tf_x_0',\n", 876 | " dtype=tf.float32)\n", 877 | " \n", 878 | " x = tf.constant(1., dtype=tf.float32)\n", 879 | " update_tf_x = tf.assign(tf_x, tf_x + x)\n", 880 | " saver = tf.train.Saver()\n", 881 | " \n", 882 | "with tf.Session(graph=g) as sess:\n", 883 | " saver.restore(sess, save_path='./my-model.ckpt')\n", 884 | " result = sess.run(update_tf_x)\n", 885 | " print(result)" 886 | ] 887 | }, 888 | { 889 | "cell_type": "markdown", 890 | "metadata": {}, 891 | "source": [ 892 | "## CPU and GPU" 893 | ] 894 | }, 895 | { 896 | "cell_type": "markdown", 897 | "metadata": {}, 898 | "source": [ 899 | "Please note that all code examples in this book, and all TensorFlow operations in general, can be executed on a CPU. If you have a GPU version of TensorFlow installed, TensorFlow will automatically execute those operations that have GPU support on GPUs and use your machine's CPU, otherwise.\n", 900 | "However, if you wish to define your computing device manually, for instance, if you have the GPU version installed but want to use the main CPU for prototyping, we can run an active section on a specific device using the `with` context as follows\n", 901 | "\n", 902 | " with tf.Session() as sess:\n", 903 | " with tf.device(\"/gpu:1\"):\n", 904 | "\n", 905 | "where\n", 906 | "\n", 907 | "- \"/cpu:0\": The CPU of your machine.\n", 908 | "- \"/gpu:0\": The GPU of your machine, if you have one.\n", 909 | "- \"/gpu:1\": The second GPU of your machine, etc.\n", 910 | "- etc.\n", 911 | "\n", 912 | "You can get a list of all available devices on your machine via\n", 913 | "\n", 914 | " from tensorflow.python.client import device_lib\n", 915 | "\n", 916 | " device_lib.list_local_devices()\n", 917 | "\n", 918 | "For more information on using GPUs in TensorFlow, please refer to the GPU documentation at https://www.tensorflow.org/how_tos/using_gpu/.\n", 919 | "\n", 920 | "\n" 921 | ] 922 | }, 923 | { 924 | "cell_type": "markdown", 925 | "metadata": {}, 926 | "source": [ 927 | "```python\n", 928 | "with tf.Session() as sess:\n", 929 | " with tf.device(\"/gpu:1\"):\n", 930 | "\n", 931 | "\n", 932 | "\n", 933 | "\n", 934 | "from tensorflow.python.client import device_lib\n", 935 | "\n", 936 | "device_lib.list_local_devices()\n", 937 | "```" 938 | ] 939 | }, 940 | { 941 | "cell_type": "markdown", 942 | "metadata": {}, 943 | "source": [ 944 | "Another good way to check whether your current TensorFlow session runs on a GPU is to execute\n", 945 | "\n", 946 | "```python\n", 947 | ">>> import tensorflow as tf\n", 948 | ">>> tf.test.gpu_device_name()\n", 949 | "```\n", 950 | "In your current Python session. If a GPU is available to TensorFlow, it will return a non-empty string; for example, `'/gpu:0'`. Otherwise, if now GPU can be found, the function will return an empty string." 951 | ] 952 | }, 953 | { 954 | "cell_type": "markdown", 955 | "metadata": {}, 956 | "source": [ 957 | "## Control Flow" 958 | ] 959 | }, 960 | { 961 | "cell_type": "markdown", 962 | "metadata": {}, 963 | "source": [ 964 | "It is important to discuss TensorFlow's control flow mechanics, the way it handles control statements such as `if/else` and `while`-loops. Control flow in TensorFlow is not a complicated topic, but it can be quite unintuitive at first and a common pitfall for beginners -- especially, in the context of how control flow in Python is handled. \n", 965 | "\n", 966 | "To explain control flow in TensorFlow in a practical manner, let us consider a simple example first. The following graph is meant to add the value `1.0` to a placeholder variable `x` if `addition=True` and subtract `1.0` from `x` otherwise:" 967 | ] 968 | }, 969 | { 970 | "cell_type": "code", 971 | "execution_count": 1, 972 | "metadata": { 973 | "collapsed": true 974 | }, 975 | "outputs": [], 976 | "source": [ 977 | "import tensorflow as tf\n", 978 | "\n", 979 | "addition = True\n", 980 | "\n", 981 | "g = tf.Graph()\n", 982 | "with g.as_default() as g:\n", 983 | " x = tf.placeholder(dtype=tf.float32, shape=None)\n", 984 | " if addition:\n", 985 | " y = x + 1.\n", 986 | " else:\n", 987 | " y = x - 1." 988 | ] 989 | }, 990 | { 991 | "cell_type": "markdown", 992 | "metadata": {}, 993 | "source": [ 994 | "Now, let us create a new session and execute the graph by feeding a `1.0` to the placeholder. If everything works as expected, the session should return the value `2.0` since `addition=True` and `1.0 + 1.0 = 2.0`:" 995 | ] 996 | }, 997 | { 998 | "cell_type": "code", 999 | "execution_count": 2, 1000 | "metadata": {}, 1001 | "outputs": [ 1002 | { 1003 | "name": "stdout", 1004 | "output_type": "stream", 1005 | "text": [ 1006 | "Result:\n", 1007 | " 2.0\n" 1008 | ] 1009 | } 1010 | ], 1011 | "source": [ 1012 | "with tf.Session(graph=g) as sess:\n", 1013 | " result = sess.run(y, feed_dict={x: 1.})\n", 1014 | " \n", 1015 | "print('Result:\\n', result)" 1016 | ] 1017 | }, 1018 | { 1019 | "cell_type": "markdown", 1020 | "metadata": {}, 1021 | "source": [ 1022 | "The previous session call clearly yielded the resulted we expected. Next, let us set `addition=False` to also check the other scenario, that is, subtracting `1.0` from `x`:" 1023 | ] 1024 | }, 1025 | { 1026 | "cell_type": "code", 1027 | "execution_count": 3, 1028 | "metadata": {}, 1029 | "outputs": [ 1030 | { 1031 | "name": "stdout", 1032 | "output_type": "stream", 1033 | "text": [ 1034 | "Result:\n", 1035 | " 2.0\n" 1036 | ] 1037 | } 1038 | ], 1039 | "source": [ 1040 | "addition = False\n", 1041 | "\n", 1042 | "with tf.Session(graph=g) as sess:\n", 1043 | " result = sess.run(y, feed_dict={x: 1.})\n", 1044 | " \n", 1045 | "print('Result:\\n', result)" 1046 | ] 1047 | }, 1048 | { 1049 | "cell_type": "markdown", 1050 | "metadata": {}, 1051 | "source": [ 1052 | "It appears that the session did return the same value that it returned when `addition` was set to `True`. Why did this happen? The explanation for this is that the `if/else` statements in the previous code only apply to the graph construction step. Or in other words, we created a graph by visiting the code contained under the `if` statement, and since TensorFlow graphs are static, we have no way of running the code under the `else` statement -- except for setting `addition=False` and creating a new graph." 1053 | ] 1054 | }, 1055 | { 1056 | "cell_type": "markdown", 1057 | "metadata": {}, 1058 | "source": [ 1059 | "However, we do not have to create a new graph each time we want to include control statements -- TensorFlow implements a variety of helper functions that help with control flow inside a graph. For instance, to accomplish the little exercise of conditionally adding or subtracting a one from the placeholder variable `x`, we could use `tf.cond` as follows:" 1060 | ] 1061 | }, 1062 | { 1063 | "cell_type": "code", 1064 | "execution_count": 4, 1065 | "metadata": { 1066 | "collapsed": true 1067 | }, 1068 | "outputs": [], 1069 | "source": [ 1070 | "addition = True\n", 1071 | "\n", 1072 | "g = tf.Graph()\n", 1073 | "with g.as_default() as g:\n", 1074 | " addition = tf.placeholder(dtype=tf.bool, shape=None)\n", 1075 | " x = tf.placeholder(dtype=tf.float32, shape=None)\n", 1076 | " \n", 1077 | " y = tf.cond(addition, \n", 1078 | " true_fn=lambda: tf.add(x, 1.), \n", 1079 | " false_fn=lambda: tf.subtract(x, 1.))" 1080 | ] 1081 | }, 1082 | { 1083 | "cell_type": "markdown", 1084 | "metadata": {}, 1085 | "source": [ 1086 | "The basic use of `tf.cond` for conditional execution comes with three important arguments: a condition to check (here, if addition is `True` or `False`), a function that gets executed if the condition is `True` (`true_fn`) and a function that gets executed if the condition is `False` (`false_fn`), respectively. \n", 1087 | "\n", 1088 | "Next, let us repeat the little exercise from earlier and see if toggling the `addition` value between `True` and `False` affects the conditional execution that is now part of the graph:" 1089 | ] 1090 | }, 1091 | { 1092 | "cell_type": "code", 1093 | "execution_count": 5, 1094 | "metadata": {}, 1095 | "outputs": [ 1096 | { 1097 | "name": "stdout", 1098 | "output_type": "stream", 1099 | "text": [ 1100 | "Result:\n", 1101 | " 2.0\n" 1102 | ] 1103 | } 1104 | ], 1105 | "source": [ 1106 | "with tf.Session(graph=g) as sess:\n", 1107 | " result = sess.run(y, feed_dict={addition:True,\n", 1108 | " x: 1.})\n", 1109 | " \n", 1110 | "print('Result:\\n', result)" 1111 | ] 1112 | }, 1113 | { 1114 | "cell_type": "code", 1115 | "execution_count": 6, 1116 | "metadata": {}, 1117 | "outputs": [ 1118 | { 1119 | "name": "stdout", 1120 | "output_type": "stream", 1121 | "text": [ 1122 | "Result:\n", 1123 | " 0.0\n" 1124 | ] 1125 | } 1126 | ], 1127 | "source": [ 1128 | "with tf.Session(graph=g) as sess:\n", 1129 | " result = sess.run(y, feed_dict={addition:False,\n", 1130 | " x: 1.})\n", 1131 | " \n", 1132 | "print('Result:\\n', result)" 1133 | ] 1134 | }, 1135 | { 1136 | "cell_type": "markdown", 1137 | "metadata": {}, 1138 | "source": [ 1139 | "Finally, we get the expected results\n", 1140 | "\n", 1141 | "- \"1.0 + 1.0 = 2.0\" if `addition=True`\n", 1142 | "- \"1.0 - 1.0 = 0.0\" if `addition=False`\n", 1143 | "\n", 1144 | "While this section provides you with the most important concept behind control flow in Python versus TensorFlow, there are many control statements (and logical operators) that we have not covered. Since the use of other control statements is analogous to `tf.cond`, I recommend you to visit [TensorFlow's API documentation](https://www.tensorflow.org/api_guides/python/control_flow_ops), which provides an overview of all the different operators for control flow and links to useful examples." 1145 | ] 1146 | }, 1147 | { 1148 | "cell_type": "markdown", 1149 | "metadata": {}, 1150 | "source": [ 1151 | "## TensorBoard" 1152 | ] 1153 | }, 1154 | { 1155 | "cell_type": "markdown", 1156 | "metadata": {}, 1157 | "source": [ 1158 | "TensorBoard is one of the coolest features of TensorFlow, which provides us with a suite of tools to visualize our computational graphs and operations before and during runtime. Especially, when we are implementing large neural networks, our graphs can be quite complicated, and TensorBoard is only useful to visually track the training cost and performance of our network, but it can also be used as an additional tool for debugging our implementation. In this section, we will go over the basic concepts of TensorBoard, but make sure you also check out the [official documentation](https://www.tensorflow.org/how_tos/summaries_and_tensorboard/) for more details. \n", 1159 | "\n", 1160 | "To visualize a computational graph via TensorBoard, let us create a simple graph with two `Variable`s, the tensors `tf_x` and `tf_y` with shape `[2, 3]`. The first operation is to add these two tensors together. Second, we transpose `tf_x` and multiply it with `tf_y`:" 1161 | ] 1162 | }, 1163 | { 1164 | "cell_type": "code", 1165 | "execution_count": 1, 1166 | "metadata": { 1167 | "collapsed": true 1168 | }, 1169 | "outputs": [], 1170 | "source": [ 1171 | "import tensorflow as tf\n", 1172 | "\n", 1173 | "g = tf.Graph()\n", 1174 | "with g.as_default() as g:\n", 1175 | " \n", 1176 | "\n", 1177 | " tf_x = tf.Variable([[1., 2.], \n", 1178 | " [3., 4.],\n", 1179 | " [5., 6.]], \n", 1180 | " name='tf_x_0',\n", 1181 | " dtype=tf.float32)\n", 1182 | " \n", 1183 | " tf_y = tf.Variable([[7., 8.], \n", 1184 | " [9., 10.],\n", 1185 | " [11., 12.]], \n", 1186 | " name='tf_y_0',\n", 1187 | " dtype=tf.float32)\n", 1188 | "\n", 1189 | " output = tf_x + tf_y\n", 1190 | " output = tf.matmul(tf.transpose(tf_x), output)" 1191 | ] 1192 | }, 1193 | { 1194 | "cell_type": "markdown", 1195 | "metadata": {}, 1196 | "source": [ 1197 | "If we want to visualize the graph via TensorBoard, we need to instantiate a new `FileWriter` object in our session, which we provide with a `logdir` and the graph itself. The `FileWriter` object will then write a [protobuf](https://developers.google.com/protocol-buffers/docs/overview) file to the `logdir` path that we can load into TensorBoard:\n" 1198 | ] 1199 | }, 1200 | { 1201 | "cell_type": "code", 1202 | "execution_count": 2, 1203 | "metadata": {}, 1204 | "outputs": [ 1205 | { 1206 | "name": "stdout", 1207 | "output_type": "stream", 1208 | "text": [ 1209 | "[[ 124. 142.]\n", 1210 | " [ 160. 184.]]\n" 1211 | ] 1212 | } 1213 | ], 1214 | "source": [ 1215 | "with tf.Session(graph=g) as sess:\n", 1216 | " sess.run(tf.global_variables_initializer())\n", 1217 | " \n", 1218 | " # create FileWrite object that writes the logs\n", 1219 | " file_writer = tf.summary.FileWriter(logdir='logs/1', graph=g)\n", 1220 | " result = sess.run(output)\n", 1221 | " print(result)" 1222 | ] 1223 | }, 1224 | { 1225 | "cell_type": "markdown", 1226 | "metadata": {}, 1227 | "source": [ 1228 | "If you installed TensorFlow via `pip`, the `tensorboard` command should be available from your command line terminal. So, after running the preceding code examples for defining the grap and running the session, you just need to execute the command `tensorboard --logdir logs/1`. You should then see an output similar to the following:\n", 1229 | "\n", 1230 | " Desktop Sebastian{$$} tensorboard --logdir logs/1\n", 1231 | " Starting TensorBoard b'41' on port 6006\n", 1232 | " (You can navigate to http://xxx.xxx.x.xx:6006)\n", 1233 | "\n", 1234 | "Copy and paste the `http` address from the terminal and open it in your favorite web browser to open the TensorBoard window. Then, click on the `Graph` tab at the top, to visualize the computational graph as shown in the figure below:\n", 1235 | "\n", 1236 | "![TensorBoard](images/tensorboard-1.png)\n", 1237 | "\n", 1238 | "In our TensorBoard window, we can now see a visual summary of our computational graph (as shown in the screenshot above). The dark-shaded nodes labeled as `tf_x_0` and `tf_y_0` are the two variables we initializes, and following the connective lines, we can track the flow of operations. We can see the graph edges that are connecting `tf_x_0` and `tf_y_0` to an `add` node, with is the addition we defined in the graph, followed by the multiplication with the transpose of and the result of `add`.\n", 1239 | "\n", 1240 | "Next, we are introducing the concept of `name_scope`s, which lets us organize different parts in our graph. In the following code example, we are going to take the initial code snippets and add `with tf.name_scope(...)` contexts as follows:\n", 1241 | "\n" 1242 | ] 1243 | }, 1244 | { 1245 | "cell_type": "code", 1246 | "execution_count": 3, 1247 | "metadata": {}, 1248 | "outputs": [ 1249 | { 1250 | "name": "stdout", 1251 | "output_type": "stream", 1252 | "text": [ 1253 | "[[ 124. 142.]\n", 1254 | " [ 160. 184.]]\n" 1255 | ] 1256 | } 1257 | ], 1258 | "source": [ 1259 | "# Graph visualization with name scopes\n", 1260 | "\n", 1261 | "g = tf.Graph()\n", 1262 | "with g.as_default() as g:\n", 1263 | " \n", 1264 | " tf_x = tf.Variable([[1., 2.], \n", 1265 | " [3., 4.],\n", 1266 | " [5., 6.]], \n", 1267 | " name='tf_x_0',\n", 1268 | " dtype=tf.float32)\n", 1269 | " \n", 1270 | " tf_y = tf.Variable([[7., 8.], \n", 1271 | " [9., 10.],\n", 1272 | " [11., 12.]], \n", 1273 | " name='tf_y_0',\n", 1274 | " dtype=tf.float32)\n", 1275 | " \n", 1276 | " # add custom name scope\n", 1277 | " with tf.name_scope('addition'):\n", 1278 | " output = tf_x + tf_y\n", 1279 | " \n", 1280 | " # add custom name scope\n", 1281 | " with tf.name_scope('matrix_multiplication'):\n", 1282 | " output = tf.matmul(tf.transpose(tf_x), output)\n", 1283 | " \n", 1284 | "with tf.Session(graph=g) as sess:\n", 1285 | " sess.run(tf.global_variables_initializer())\n", 1286 | " file_writer = tf.summary.FileWriter(logdir='logs/2', graph=g)\n", 1287 | " result = sess.run(output)\n", 1288 | " print(result)" 1289 | ] 1290 | }, 1291 | { 1292 | "cell_type": "markdown", 1293 | "metadata": {}, 1294 | "source": [ 1295 | "After executing the code example above, quit your previous TensorBoard session by pressing `CTRL+C` in the command line terminal and launch a new TensorBoard session via `tensorboard --logdir logs/2`. After you refreshed your browser window, you should see the following graph:\n", 1296 | "\n", 1297 | "\n", 1298 | "![TensorBoard](images/tensorboard-2.png)\n", 1299 | "\n", 1300 | "Comparing this visualization to our initial one, we can see that our operations have been grouped into our custom name scopes. If we double-click on one of these name scope summary nodes, we can expand it and inspect the individual operations in more details as shown for the `matrix_multiplication` name scope in the screenshot below:\n", 1301 | "\n", 1302 | "![TensorBoard](images/tensorboard-3.png)\n", 1303 | "\n", 1304 | "So far, we have only been looking at the computational graph itself. However, TensorBoard implements many more useful features. In the following example, we will make use of the \"Scalar\" and \"Histogram\" tabs. The \"Scalar\" tab in TensorBoard allows us to track scalar values over time, and the \"Histogram\" tab is useful for displaying the distribution of value in our tensor `Variable`s (for instance, the model parameters during training). For simplicity, let us take our previous code snippet and modify it to demonstrate the capabilities of TensorBoard:\n", 1305 | "\n" 1306 | ] 1307 | }, 1308 | { 1309 | "cell_type": "code", 1310 | "execution_count": 4, 1311 | "metadata": { 1312 | "collapsed": true 1313 | }, 1314 | "outputs": [], 1315 | "source": [ 1316 | "# Graph visualization and variable inspection\n", 1317 | "\n", 1318 | "g = tf.Graph()\n", 1319 | "with g.as_default() as g:\n", 1320 | "\n", 1321 | " some_value = tf.placeholder(dtype=tf.int32, \n", 1322 | " shape=None, \n", 1323 | " name='some_value')\n", 1324 | " \n", 1325 | " tf_x = tf.Variable([[1., 2.], \n", 1326 | " [3., 4.],\n", 1327 | " [5., 6.]], \n", 1328 | " name='tf_x_0',\n", 1329 | " dtype=tf.float32)\n", 1330 | " \n", 1331 | " tf_y = tf.Variable([[7., 8.], \n", 1332 | " [9., 10.],\n", 1333 | " [11., 12.]], \n", 1334 | " name='tf_y_0',\n", 1335 | " dtype=tf.float32)\n", 1336 | " \n", 1337 | " with tf.name_scope('addition'): \n", 1338 | " output = tf_x + tf_y\n", 1339 | " \n", 1340 | " with tf.name_scope('matrix_multiplication'):\n", 1341 | " output = tf.matmul(tf.transpose(tf_x), output)\n", 1342 | " \n", 1343 | " with tf.name_scope('update_tensor_x'): \n", 1344 | " tf_const = tf.constant(2., shape=None, name='some_const')\n", 1345 | " update_tf_x = tf.assign(tf_x, tf_x * tf_const)\n", 1346 | " \n", 1347 | " # create summaries\n", 1348 | " tf.summary.scalar(name='some_value', tensor=some_value) \n", 1349 | " tf.summary.histogram(name='tf_x_values', values=tf_x) \n", 1350 | "\n", 1351 | " # merge all summaries into a single operation\n", 1352 | " merged_summary = tf.summary.merge_all()" 1353 | ] 1354 | }, 1355 | { 1356 | "cell_type": "markdown", 1357 | "metadata": {}, 1358 | "source": [ 1359 | "Notice that we added an additional `placeholder` to the graph which later receives a scalar value from the session. We also added a new operation that updates our `tf_x` tensor by multiplying it with a constant `2.`:\n", 1360 | "\n", 1361 | "```python\n", 1362 | "with tf.name_scope('update_tensor_x'): \n", 1363 | " tf_const = tf.constant(2., shape=None, name='some_const')\n", 1364 | " update_tf_x = tf.assign(tf_x, tf_x * tf_const)\n", 1365 | "```\n", 1366 | "\n", 1367 | "Finally, we added the lines \n", 1368 | "\n", 1369 | "```python\n", 1370 | "# create summaries\n", 1371 | "tf.summary.scalar(name='some_value', tensor=some_value) \n", 1372 | "tf.summary.histogram(name='tf_x_values', values=tf_x) \n", 1373 | "```\n", 1374 | "\n", 1375 | "at the end of our graph. These will create the \"summaries\" of the values we want to display in TensorBoard later. The last line of our graph is \n", 1376 | "\n", 1377 | "```python\n", 1378 | "merged_summary = tf.summary.merge_all()\n", 1379 | "```\n", 1380 | "\n", 1381 | "which summarizes all the `tf.summary` calls to one single operation, so that we only have to fetch one variable from the graph when we execute the session. When we executed the session, we simply fetched this merged summary from `merged_summary` as follows:\n", 1382 | "\n", 1383 | "```python\n", 1384 | "result, summary = sess.run([update_tf_x, merged_summary],\n", 1385 | " feed_dict={some_value: i})\n", 1386 | "```\n", 1387 | "\n", 1388 | "Next, let us add a `for`-loop to our session that runs the graph five times, and feeds the counter of the `range` iterator to the `some_value` `placeholder` variable:\n", 1389 | "\n" 1390 | ] 1391 | }, 1392 | { 1393 | "cell_type": "code", 1394 | "execution_count": 5, 1395 | "metadata": { 1396 | "collapsed": true 1397 | }, 1398 | "outputs": [], 1399 | "source": [ 1400 | "with tf.Session(graph=g) as sess:\n", 1401 | " \n", 1402 | " sess.run(tf.global_variables_initializer())\n", 1403 | " \n", 1404 | " # create FileWrite object that writes the logs\n", 1405 | " file_writer = tf.summary.FileWriter(logdir='logs/3', graph=g)\n", 1406 | " \n", 1407 | " for i in range(5):\n", 1408 | " # fetch the summary from the graph\n", 1409 | " result, summary = sess.run([update_tf_x, merged_summary],\n", 1410 | " feed_dict={some_value: i})\n", 1411 | " # write the summary to the log\n", 1412 | " file_writer.add_summary(summary=summary, global_step=i)\n", 1413 | " file_writer.flush()" 1414 | ] 1415 | }, 1416 | { 1417 | "cell_type": "markdown", 1418 | "metadata": {}, 1419 | "source": [ 1420 | "The two lines at the end of the preceding code snippet,\n", 1421 | "\n", 1422 | "```python\n", 1423 | "file_writer.add_summary(summary=summary, global_step=i)\n", 1424 | "file_writer.flush()\n", 1425 | "```\n", 1426 | "\n", 1427 | "will write the summary data to our log file and the `flush` method updates TensorBoard. Executing `flush` explicitely is usually not necessary in real-world applications, but since the computations in our graph are so simple and \"cheap\" to execute, TensorBoard may not fetch the updates in real time.\n", 1428 | "To visualize the results, quit your previous TensorBoard session (via `CTRL+C`) and execute `tensorboard --logdir logs/3` from the command line. In the TensorBoard window under the tab \"Scalar,\" you should now see an entry called \"some_value_1,\" which refers to our `placeholder` in the graph that we called `some_value`. Since we just fed it the iteration index of our `for`-loop, we expect to see a linear graph with the iteration index on the *x*- and *y*-axis:\n", 1429 | "\n", 1430 | "![TensorBoard](images/tensorboard-4.png)\n", 1431 | "\n", 1432 | "Keep in mind that this is just a simple demonstration of how `tf.summary.scalar` works. For instance, more useful applications include the tracking of the training loss and the predictive performance of a model on training and validation sets throughout the different training rounds or epochs. \n", 1433 | "\n", 1434 | "Next, let us go to the \"Distributions\" tab:\n", 1435 | "\n", 1436 | "\n", 1437 | "![TensorBoard](images/tensorboard-5.png)\n", 1438 | "\n", 1439 | "The \"Distributions\" graph above shows us the distribution of values in `tf_x` for each step in the `for`-loop. Since we doubled the value in the tensor after each `for`-loop iteration, we the distribution graph grows wider over time.\n", 1440 | "\n", 1441 | "Finally, let us head over to the \"Histograms\" tab, which provides us with an individual histogram for each `for`-loop step that we can scroll through. Below, I selected the 3rd `for`-loop step that highlights the histogram of values in `tf_x` during this step:\n", 1442 | "\n", 1443 | "\n", 1444 | "![TensorBoard](images/tensorboard-6.png)\n", 1445 | "\n", 1446 | "\n", 1447 | "Since TensorBoard is such a highly visual tool with graph and data exploration in mind, I highly recommend you to take it for a test drive and explore it interactively. Also, the are several features that we haven't covered in this simple introduction to TensorBoard, so be sure to check out the [official documentation](https://www.tensorflow.org/how_tos/summaries_and_tensorboard/) for more information." 1448 | ] 1449 | } 1450 | ], 1451 | "metadata": { 1452 | "anaconda-cloud": {}, 1453 | "kernelspec": { 1454 | "display_name": "Python 3", 1455 | "language": "python", 1456 | "name": "python3" 1457 | }, 1458 | "language_info": { 1459 | "codemirror_mode": { 1460 | "name": "ipython", 1461 | "version": 3 1462 | }, 1463 | "file_extension": ".py", 1464 | "mimetype": "text/x-python", 1465 | "name": "python", 1466 | "nbconvert_exporter": "python", 1467 | "pygments_lexer": "ipython3", 1468 | "version": "3.6.1" 1469 | } 1470 | }, 1471 | "nbformat": 4, 1472 | "nbformat_minor": 1 1473 | } 1474 | -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensorboard-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensorboard-1.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensorboard-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensorboard-2.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensorboard-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensorboard-3.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensorboard-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensorboard-4.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensorboard-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensorboard-5.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensorboard-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensorboard-6.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/images/tensors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/images/tensors.png -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-50.data-00000-of-00001: -------------------------------------------------------------------------------- 1 | PBTBXB\B`BdB -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-50.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-50.index -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-50.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-50.meta -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-60.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-60.data-00000-of-00001 -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-60.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-60.index -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-60.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-60.meta -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-70.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-70.data-00000-of-00001 -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-70.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-70.index -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-70.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-70.meta -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-80.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-80.data-00000-of-00001 -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-80.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-80.index -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-80.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-80.meta -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-90.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-90.data-00000-of-00001 -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-90.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-90.index -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-90.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model-multiple_ckpts.ckpt-90.meta -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model.ckpt.index -------------------------------------------------------------------------------- /code/_old-material/appendix_tensorflow-basics/my-model.ckpt.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/appendix_tensorflow-basics/my-model.ckpt.meta -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/ch02_perceptron.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "*Accompanying code examples of the book \"Introduction to Artificial Neural Networks and Deep Learning: A Practical Guide with Applications in Python\" by [Sebastian Raschka](https://sebastianraschka.com). All code examples are released under the [MIT license](https://github.com/rasbt/deep-learning-book/blob/master/LICENSE). If you find this content useful, please consider supporting the work by buying a [copy of the book](https://leanpub.com/ann-and-deeplearning).*\n", 8 | "\n", 9 | "Other code examples and content are available on [GitHub](https://github.com/rasbt/deep-learning-book). The PDF and ebook versions of the book are available through [Leanpub](https://leanpub.com/ann-and-deeplearning)." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "# Ch02 - The Perceptron" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Hands-on Section" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "### Table of Contents\n", 31 | "* [Preparing the Dataset](#Preparing-the-Dataset)\n", 32 | "* [Implementing a Perceptron in NumPy](#Implementing-a-Perceptron-in-NumPy)\n", 33 | "* [Implementing a Perceptron in TensorFlow](#Implementing-a-Perceptron-in-TensorFlow)\n" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 1, 39 | "metadata": { 40 | "scrolled": true 41 | }, 42 | "outputs": [ 43 | { 44 | "name": "stdout", 45 | "output_type": "stream", 46 | "text": [ 47 | "Sebastian Raschka 2017-03-31 \n", 48 | "\n", 49 | "tensorflow 1.0.1\n", 50 | "numpy 1.12.1\n", 51 | "matplotlib 2.0.0\n" 52 | ] 53 | } 54 | ], 55 | "source": [ 56 | "%load_ext watermark\n", 57 | "%watermark -a 'Sebastian Raschka' -d -p tensorflow,numpy,matplotlib" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 2, 63 | "metadata": { 64 | "collapsed": true 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "%matplotlib inline" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "import tensorflow as tf\n", 78 | "import numpy as np\n", 79 | "import os\n", 80 | "import matplotlib.pyplot as plt" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "## Preparing the Dataset" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "- Load dataset from tab-seperated text file\n", 95 | "- Dataset contains three columns: feature 1, feature 2, and class labels\n", 96 | "- Dataset contains 100 entries sorted by class labels, 50 examples from each class" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 4, 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "name": "stdout", 106 | "output_type": "stream", 107 | "text": [ 108 | "Class label counts: [50 50]\n" 109 | ] 110 | }, 111 | { 112 | "data": { 113 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHddJREFUeJzt3X9wXfV55/H3gxCViMFObLVgyamdhnUTwMaDQFBPoGta\nDAQwgTDdkJA62YRtdvkRfpgBnOkIQqmpaRPSJEvskLJ2TLP1wrppKHGTuISWDDQSBoPCOiaUHxJk\n45gVOFiOZfnZP+6VkeR7r47uPb/P5zXjGevo6t7nCHOe8/1+n+d7zN0RERE5LOkAREQkHZQQREQE\nUEIQEZEyJQQREQGUEEREpEwJQUREACUEEREpU0IQERFACUFERMoOTzqAqZg1a5bPnTs36TBERDKl\nt7f3l+7eNtnrMpUQ5s6dS09PT9JhiIhkipm9FOR1mjISERFACUFERMqUEEREBMjYGkIlw8PD9Pf3\ns3fv3qRDSY2WlhY6Ojpobm5OOhQRyZDMJ4T+/n6OOuoo5s6di5klHU7i3J1du3bR39/PvHnzkg5H\nRDIk8SkjM2sys61m9p16fn7v3r3MnDlTyaDMzJg5c6ZGTCIyZYknBOAa4LlG3kDJYDz9PkSkHolO\nGZlZB/BB4M+A65KMRSRX7miHfb869PgR0+CWgfjjkUxIeg3hi8CNwFEJxxG67u5upk2bxg033BD6\ne/f29rJ8+XKGhoY477zzuPvuuzUqkPEqJYNKx6eSOJRkci+xKSMzOx/4hbv3TvK6K8ysx8x6du7c\nGVN06faZz3yGtWvXsmPHDnbs2MF3v/vdpEOSrAqaOCZ7bff0UsKQTEtyDWExcKGZvQh8C1hiZt+c\n+CJ3X+Pune7e2dY26VYck9q0dYDFq7Yw76aHWLxqC5u2Nn5ns27dOhYsWMDChQu5/PLLD/n+2rVr\nOeWUU1i4cCGXXHIJe/bsAWDjxo2ccMIJLFy4kDPOOAOAvr4+Tj31VE466SQWLFjAjh07xr3Xa6+9\nxptvvslpp52GmfHxj3+cTZs2NXwOIg2rljAkMxKbMnL3m4GbAczs94Eb3P1jUX7mpq0D3PzgMwwN\njwAwMDjEzQ8+A8BFi+q7u+nr6+P222/nRz/6EbNmzeL1118/5DUXX3wxn/70pwH43Oc+x7333stV\nV13FbbfdxubNm2lvb2dwcBCAe+65h2uuuYaPfvSj7Nu3j5GRkXHvNTAwQEdHx8GvOzo6GBjQcF1E\nGpeGKqPYrN68/WAyGDU0PMLqzdvrfs8tW7Zw6aWXMmvWLADe9a53HfKaZ599lg984AOceOKJbNiw\ngb6+PgAWL17M8uXLWbt27cEL/+mnn84dd9zBnXfeyUsvvURra2vdsYmITEXSi8oAuPsjwCNRf86r\ng0NTOh6W5cuXs2nTJhYuXMh9993HI488ApRGA0888QQPPfQQJ598Mr29vVx22WV0dXXx0EMPcd55\n5/G1r32NJUuWHHyv9vZ2+vv7D37d399Pe7vmbmWCI6ZVXwCOmxajMyMVCSEus2e0MlDh4j97Rv13\n4UuWLOFDH/oQ1113HTNnzuT1118/ZJSwe/dujj32WIaHh9mwYcPBC/jPfvYzurq66Orq4uGHH+aV\nV17hjTfe4D3veQ9XX301L7/8Mtu2bRuXEI499liOPvpoHn/8cbq6uli3bh1XXXVV3fFLTgW90E4l\ncVR77WSmsnAtiSpUQlixdP64NQSA1uYmViydX/d7Hn/88axcuZIzzzyTpqYmFi1axH333TfuNZ//\n/Ofp6uqira2Nrq4udu/eXYpnxQp27NiBu3PWWWexcOFC7rzzTtavX09zczPHHHMMt9xyyyGf+dWv\nfvVg2em5557LueeeW3f8UnBTuUMffW2tO37JNHP3pGMIrLOz0yc+IOe5557jfe97X+D32LR1gNWb\nt/Pq4BCzZ7SyYun8uheU02yqvxeRyHRPr/G9N+KLo8DMrNfdOyd7XaFGCFCqJspjAhAJheb7C61w\nCUFEatB8f3gymFwLVXYqIgmotraQ9zWHDCZXjRBEJFopvRuWQykhiEh6ZHCapW4pPFdNGYlIemRw\nmqVuKTxXJYSIdHd3c9ddd0Xy3itXrmTOnDlMm5bzOViJX1Hn+wXQlFEmXXDBBVx55ZUcd9xxSYci\neZO3aZkkpWn7kICKlRAimrNbt24dd911F2bGggULWL9+/bjvr127ljVr1rBv3z7e+973sn79eo48\n8kg2btzIrbfeSlNTE9OnT+fRRx+lr6+PT3ziE+zbt48DBw7wwAMPHHLhP+200+qOVURiksHkWqyE\nEMGcXdzbX4uEIoULmpI8rSE0SNtfSyalcEETKNYaRgrPtVgjhISEuf21SK4VaXSSwnPVCKFBS5Ys\nYePGjezatQug4pTRxO2vR41uf33bbbfR1tbGK6+8wgsvvHBw++tly5axbdu22M5FRIpNI4QGJbH9\n9Y033sj999/Pnj176Ojo4FOf+hTd3d0xnK2kmtYFpEHF2v66QP/DaPvrAprKNtNZ35I6zf8vNxpb\nBOem7a8rSfofikhaZLBGfpy0LorXiiFobAmeW7ESgkhR3dE+/oYozzdHE0c/aRg1ZEQuEoK7Y2ZJ\nh5EaWZoGlJjEceec1mmcNIwaMiLzVUYtLS3s2rVLF8Eyd2fXrl20tLQkHYoUTZqncSSQzI8QOjo6\n6O/vZ+fOnUmHkhotLS10dHQkHYbErdq6gEhAmU8Izc3NzJs3L+kwRJJ3y0Dt6qGpTumkdQpoNIa0\nJr9GF+wTXPDPfEIQkYCmOqUTxxRQvUmn0vdqJcM4NZosE0y2mV9DEJExUrg/Tk1hJp2snXsKaYQg\nkie17i4nu4OudrceVNK9DUlPY+WAEoKIlDQ6FaQLcuYpIYhkSZoXeiXzlBBEsqSROfewpnTSNCev\nBBkqJQSRomjkAhnVhneNJik1w4VKCUFEkhP1XfxkIwiNMMZRQhDJi0YvbklXCUVhshGERhjjKCGI\n5EWjF7cC3hHLeGpME8kSNV9JhDRCEMmSRhrP8iiP01wJUkIQkexqZJrrjvbw4sgJJQSRIqg0eihC\nJU2tXVH3/UojjAkSSwhmNgdYB/wW4MAad787qXhEMm+qW0LXem2tiqVqP5vGBDPZluBpizdhSY4Q\n9gPXu/uTZnYU0Gtm33P3nyQYk0h2Vbu41bO2UE/FUkFLNfMksSojd3/N3Z8s/3038BygST0RkYSk\nouzUzOYCi4AnKnzvCjPrMbMePSZTRCQ6iS8qm9k04AHgs+7+5sTvu/saYA1AZ2enxxyeSPGktfpG\n20xELtERgpk1U0oGG9z9wSRjEcmtqTazpXUtoNa6Rvf09CayDEmyysiAe4Hn3P2vkopDJPfCvHue\nrMooSWlNZBmS5JTRYuBy4Bkze6p87BZ3/8cEYxKRWjQ1k2uJJQR3/1fAkvp8EREZLxVVRiIikjwl\nBBEZL607qtb7+Wk9nxRKvOxUREJQrSSzksnKNNO6TjAa12TbalT7OZmUEoJIHoS1h1EW6AIfGSUE\nkZBs2jrA6s3beXVwiNkzWlmxdD4XLVJtvGSHEoJICDZtHeDmB59haHgEgIHBIW5+8BkAJQXJDCUE\nkRCs3rz9YDIYNTQ8wurN25UQkqBtLuqihCASglcHh6Z0PHWiuoAmdWGuZ/tuUdmpSBhmz2id0vHQ\nTaWEstJro7qA6sKcKRohiIRgxdL549YQAFqbm1ixdH48AcQ9DaIpmVxSQhAJweg6QWGqjHTnn0tK\nCCIhuWhRez4TQPd03fkXhNYQRGRyWbvz13YVddEIQURKF8ooLvrV3jfqC7NGM3VRQkiAOloldUYv\noN3To3lfyQQlhJipo1VyIak7f4mUEkLM1NEqU5XKEWXW7/xVNluREkLMMt/RKrGKfUSZpjv/WtNX\n3W809t4qm61ICSFms2e0MlDh4h9bR6tkSuwjygLfHYsSQuwS72jNmFROl8QoNyNKTdFkghJCzArX\n0doALcDnaESpKZpMUEJIQG47WkOmBXiNKCVeSgiSWrmZLmmARpQRSdPieYooIUhq5Wa6pEGpHlFm\ndW0g6tgy+ntRQpDU0nRJBkS5NtBoaWmSMrpmUnVzOzObY2bfMrN/MbNbzKx5zPc2xROeFNlFi9r5\n84tPpH1GKwa0z2jlzy8+Mb13y1KdNpvLhFojhG8ADwCPA/8Z+KGZXeDuu4DfjiM4kVRPl0hwKZ4m\nkbfVSght7n5P+e9XmdnHgEfN7ELAow9NREKX0bltiUethNBsZi3uvhfA3b9pZj8HNgPviCU6EQlX\nRue2JR61EsLXgS7gh6MH3P37ZnYp8BdRByYi46WyazvJ8s00j3YyWtZaNSG4+xeqHN8K/GFkEYkE\nkMqLY4RS27Wd5IU3zaOdpBNSnfQITcmc0YvjwOAQztsXx01bs/k/YRC1urZFwqKEIJlTxIujurYl\nDkoIkjlFvDhW686ecte2+gGkhkk7lc3st4A7gNnufq6ZvR843d3vjTw6kQqKuKVFaF3bGZ3blngE\nGSHcR6nUdHb5658Cn40qIJHJrFg6n9bmpnHH8r6lhbq2K9BoJ3RB9jKa5e5/Z2Y3A7j7fjMbmeyH\nRKJS1B1A1bU9gUY7oQuSEN4ys5mUu5PN7DQglF2nzOwc4G6gCfi6u68K430l/3RxFAlfkIRwHfBt\n4HfM7DGgDfhwox9sZk3AVyj1NPQDPzazb7v7Txp9b0mHovUKTIV+N5JGNROCmR0GtABnAvMBA7a7\n+3AIn30q8Ly7v1D+rG8BywAlhBxIbSNVCuh3I2lVc1HZ3Q8AX3H3/e7e5+7PhpQMANqBV8Z83V8+\nJjlQxF6BoPS7kbQKUmX0AzO7xMws8mgqMLMrzKzHzHp27tyZRAhShyL2CgSl342kVZCE8F+AjcCv\nzexNM9ttZm+G8NkDwJwxX3eUj43j7mvcvdPdO9va2kL4WIlDaI1UOaTfjaTVpIvK7n5URJ/9Y+A4\nM5tHKRH8J+CyiD6r0JJYwNTjL6vLzO8mzbuJSiSCdCqfUem4uz/ayAeX+xmupNT01gR8w937GnlP\nOVRSC5hF7RUIIjO/mzTvJiqRMPfaDz8zs38Y82ULpeqgXndfEmVglXR2dnpPT0/cH5tpi1dtqbjN\nQ/uMVh67Kfb/hJFQCWdEuqfX+F4orUgSEzPrdffOyV4XZMrogglvPAf4YgOxSYzyvoCpEk6R8NSz\n22k/8L6wA5Fo5H0BUyWcIuEJsobw15S3raCUQE4CnowyKGnM2CmU6a3NNDcZwyNvTw2mcgGzTnkf\nAYnEKcjWFWMn7fcDf+vuj0UUjzRo4hTK4NAwzYcZ7zyymcE9w7mbYy/iVtixyehzgaV+QRLCDHe/\ne+wBM7tm4jFJh0pTKMMHnCOPOJytf3p2QlFFJzMlnFmk0tLCCbKG8McVji0POQ4JSdGmUPScAJHw\nVB0hmNlHKDWKzTOzb4/51lHA61EHJsGNXTM4zIyRCqXEeZ5C0VbYIuGoNWX0I+A1YBbwl2OO7wa2\nRRmUBDdxzaBSMtAUiogEUTUhuPtLwEvA6fGFI1NVac0AoMmMA+65W0SeKjWtiQQXpOz0NOCvKfUe\nHEFpm4m33P3oiGOTAKqtDRxw599XfTDmaNJFTWsiUxNkUfnLwEeAHUAr8ClKTzrLhE1bB1i8agvz\nbnqIxau2sGlrvion8t541gg1rYlMTaBOZXd/Hmhy9xF3/xvgnGjDCsfoHeLA4BDO23eIeUoKK5bO\np7W5adwxrRmUFK3iSqRRQRLCHjM7AnjKzP7CzK4N+HOJK8Idosouq9PoSWRqgjSmXU4pAVwJXEvp\noTaXRBlUWIpyh6iyy8rUtCYyNUF2O33JzFqBY9391hhiCo22NSi2zDx3QCQlglQZXQDcRanCaJ6Z\nnQTc5u4XRh1co3SHKBo9iQQXZMqom9JDcR4BcPenyo+9TL2s3CGqVl5E0iBIQhh29zfMbOyx2o9Z\nS5G03yGqVr46JUqReAVJCH1mdhnQZGbHAVdT2tZCQlCrEqqIF7/RJDAwOITx9p3HxESpZFGdfjdS\nryAJ4SpgJfBr4H5gM3B7lEEVSRoqodJyAZk4Wpo4DB1bMjxxVHXt/3yKnpde5/aLTowz5NTRiFMa\nUbWfwMzWl//6aXdf6e6nlP98zt33xhRf7iVdK5+m5r1q+zKN9ergUMXXObDh8ZcnjTvvnetF6L2R\n6NRqMDvZzGYDnzSzd5rZu8b+iSvAvEu60zhNF5Ago6LZM1qrvs6hZtxpSn5RScOIU7KrVkK4B/gB\n8LtA74Q/PTV+TqYg6U7jNF1AJhsVNR9m7Nm3v2ZFQ62405T8opL0iFOyrdb2118CvmRm/93dPxNj\nTIWTZCVUmpr3KvWNjC4sz2ht5q19+/l/e4ZrvketuNOU/KKi3htpxKR7EikZ5FvSU1ZjVRotfeGP\nTuLFVR/kHb9xOMMjtaudJ4u7CHfPSY84JduCVBlJjqWtea/aaKnWXbxBoLiLcvec9t4bSS8lBIn8\nAhJGWWu1qa32Ga08dtOSQO+RtuQnkjZKCBKpsOriw7q7n2ryG9so12TGiDvtSiSSU5l4roFkV1iV\nPUnMjY8tUwUY8dIaRh7LVUVAIwSJWLW5/0rTP5OJe268VqNckbcXkfzSCKHgou7crVbBY+XPTrPJ\nylHzVK4qAkoIhRZH5+6KpfOxCscn6ypOg8nKUfNUrioCSgiFFkfn7kWL2qt2Fqf9DrtSj8aoPJar\niighFFhcnbvtGW0IG7uQDdBUfiaImr0kr7SoXGBxbVuR5YYwNXlJkWiEUGBxbVuh7RREskEjhAKL\ns3M373faaXnIkEgjlBAKLisX6jRfcKfSjZ3m8xBJJCGY2WrgAmAf8DPgE+4+mEQsaaCLRG1pfyxk\n0OdiN3Ie+jcicUhqDeF7wAnuvgD4KXBzQnEkrghP8WpU2h9sE7Raq97z0L8RiUsiCcHd/8nd95e/\nfBzoSCKONEj7xS4N0v5gm6DPWaj3PPRvROKShiqjTwIPV/ummV1hZj1m1rNz584Yw4pH2i92jQhr\nW4y0P9gmaLVWveeR538jki6RJQQz+76ZPVvhz7Ixr1kJ7Ac2VHsfd1/j7p3u3tnW1hZVuIlJ+8Wu\nXmFOc6TpqW6VBC2rrfc88vpvRNInskVld/+DWt83s+XA+cBZ7l772Yg5lqWmrU1bB7j1H/oOPtd4\nRmsz3RceX3FxM+hCaxBZeLBNkGqtes8jS/9GJNuSqjI6B7gRONPd9yQRQ1pk4WIHpWSw4n89Pe65\nxoNDw6zY+DRwaJVM2NMc1S64Wau+qafMNyv/RiT7LImbczN7HvgNYFf50OPu/ieT/VxnZ6f39PRE\nGptUtnjVlqrPMBh9gtjYC9Zbv97P4NBwxdcGfeTlZCaWcULpzlld0CLjmVmvu3dO9rpERgju/t4k\nPlfqV+vOfnR9YGx9fXOT0XyYMXzg7RuOsKc5wpyWEpF0VBlJBtRawGwyO+TCPDziTGs5PNL9i1R9\nIxIubV0hgaxYOv+QNQTgkFHAWIN7htn6p2dHFlNcu7WKFIVGCBLIRYvaWf3hhbzzyOaDx2a0NrP6\n0oWJPe8g7eWoIlmjEYIAwap1alXIJFEWqeobkXApIUjDm8cleWHOym6tIlmghCChVOvowiySfVpD\nEFXriAigEYJQjGqdrHU0iyRBIwTJfbVOFM8TCGsnV5E0UUKQwLt1ZlXYzxPQA2skrzRlJEC+F4XD\nXiPRlhmSVxohSO6F/TwBLcJLXmmEILk0dhF5emszzU02btuNRtZIirAIL8WkEYLkzsQ5/sGhYXB4\n55HNoayR5H0RXopLIwTJnUpz/MMHnCOPODyUzfa0ZYbklRKC5E4cc/x5XoSX4lJCkNzRHP+h1Jgn\nQWgNQXJHc/zjqW9CglJCkNzJe6PdVIXdmCf5pSkjySXN8b9NfRMSlEYIIjkXdmOe5JcSgkjOaU1F\ngtKUkUjOqW9CglJCkFCorDHdtKYiQSghSMMafSZzEShhShZoDUEaprLG2tQHIFmhhCANU1ljbUqY\nkhVKCNIwlTXWpoQpWaGEIA1TWWNtSpiSFUoI0jBtFVGbEqZkhaqMCiLqKheVNVanPgDJCiWEAlBZ\naPKUMCULlBAKoFaViy5SMhXqp8g3JYQCUJWLhEEjzfzTonIBqMplvE1bB1i8agvzbnqIxau2qEEs\nIPVT5J8SQgGoyuVt6hqun0aa+aeEUAAqC32b7nLrp5Fm/iW6hmBm1wN3AW3u/sskY8k7VbmU6C63\nfiuWzh+3hgDFHWnmVWIjBDObA5wNvJxUDFI8usutn0aa+ZfkCOELwI3A3ycYgxSM7nIbo5FmviWS\nEMxsGTDg7k+bWRIhSBV5rzNX17BIdZElBDP7PnBMhW+tBG6hNF0U5H2uAK4AePe73x1afHKootSZ\n6y5XpDJz93g/0OxE4AfAnvKhDuBV4FR3/3mtn+3s7PSenp6IIyyuxau2MFBhcbV9RiuP3bQkgYhE\nJAxm1uvunZO9LvYpI3d/BvjN0a/N7EWgU1VGyVMFjkixqQ9BDlIFjkixJZ4Q3H2uRgfpoI5mkWLT\n5nZykCpwRIpNCUHGUQWOSHElPmUkIiLpoBGC5L4ZTUSCUUIouKI0o4nI5DRlVHDaDlpERikhFJya\n0URklBJCwakZTURGKSEUnJrRRGSUFpULTs1oIjJKCUHUjCYigKaMRESkTAlBREQAJQQRESlTQhAR\nEUAJQUREymJ/pnIjzGwn8FLScURkFlDUBwUV9dyLet6gc4/73H/b3dsme1GmEkKemVlPkIdg51FR\nz72o5w0697Seu6aMREQEUEIQEZEyJYT0WJN0AAkq6rkX9bxB555KWkMQERFAIwQRESlTQkghM7ve\nzNzMZiUdS1zMbLWZ/R8z22Zm/9vMZiQdU5TM7Bwz225mz5vZTUnHExczm2Nm/2xmPzGzPjO7JumY\n4mRmTWa21cy+k3QslSghpIyZzQHOBl5OOpaYfQ84wd0XAD8Fbk44nsiYWRPwFeBc4P3AR8zs/clG\nFZv9wPXu/n7gNOC/FejcAa4Bnks6iGqUENLnC8CNQKEWd9z9n9x9f/nLx4GOJOOJ2KnA8+7+grvv\nA74FLEs4pli4+2vu/mT577spXRwLsfe6mXUAHwS+nnQs1SghpIiZLQMG3P3ppGNJ2CeBh5MOIkLt\nwCtjvu6nIBfFscxsLrAIeCLZSGLzRUo3eweSDqQaPSAnZmb2feCYCt9aCdxCabool2qdu7v/ffk1\nKylNK2yIMzaJl5lNAx4APuvubyYdT9TM7HzgF+7ea2a/n3Q81SghxMzd/6DScTM7EZgHPG1mUJoy\nedLMTnX3n8cYYmSqnfsoM1sOnA+c5fmuhx4A5oz5uqN8rBDMrJlSMtjg7g8mHU9MFgMXmtl5QAtw\ntJl9090/lnBc46gPIaXM7EWg090LsQGYmZ0D/BVwprvvTDqeKJnZ4ZQWzs+ilAh+DFzm7n2JBhYD\nK93t/A/gdXf/bNLxJKE8QrjB3c9POpaJtIYgafFl4Cjge2b2lJndk3RAUSkvnl8JbKa0qPp3RUgG\nZYuBy4El5f/OT5XvmiUFNEIQERFAIwQRESlTQhAREUAJQUREypQQREQEUEIQEZEyJQQpFDO72sye\nM7Mpd0Kb2VwzuyyKuMrvf4aZPWlm+83sw1F9jkg1SghSNP8V+EN3/2gdPzsXmHJCKO9uGsTLwHLg\n/ql+hkgYlBCkMMrNbu8BHjaza83sHWb2DTP7t/Ie9cvKr5trZv9Svlt/0sx+r/wWq4APlJuprjWz\n5Wb25THv/53RfWrM7Fdm9pdm9jRwupmdbGY/NLNeM9tsZsdOjM/dX3T3baR48zPJN+1lJIXh7n9S\n3iLjP7r7L83sDmCLu3+y/ECefytvwPcLSqOIvWZ2HPC3QCdwE2O2HCjvvVTNO4An3P368t49PwSW\nuftOM/sj4M8o7eoqkhpKCFJkZ1PacOyG8tctwLuBV4Evm9lJwAjwH+p47xFKG7gBzAdOoLQtB0AT\n8FoDcYtEQglBisyAS9x9+7iDZt3A/wUWUppW3Vvl5/czftq1Zczf97r7yJjP6XP308MIWiQqWkOQ\nItsMXFXegRMzW1Q+Ph14zd0PUNqIbXRReDelDfhGvQicZGaHlR99emqVz9kOtJnZ6eXPaTaz40M9\nE5EQKCFIkX0eaAa2mVlf+WuArwJ/XF4Q/l3grfLxbcCImT1tZtcCjwH/DvwE+BLwZKUPKT8m88PA\nneX3fAr4vYmvM7NTzKwfuBT4Wjkmkdhot1MREQE0QhARkTIlBBERAZQQRESkTAlBREQAJQQRESlT\nQhAREUAJQUREypQQREQEgP8PzP8v2tm5NVsAAAAASUVORK5CYII=\n", 114 | "text/plain": [ 115 | "" 116 | ] 117 | }, 118 | "metadata": {}, 119 | "output_type": "display_data" 120 | } 121 | ], 122 | "source": [ 123 | "data = np.genfromtxt('perceptron_toydata.txt', delimiter='\\t')\n", 124 | "X, y = data[:, :2], data[:, 2]\n", 125 | "y = y.astype(np.int)\n", 126 | "\n", 127 | "print('Class label counts:', np.bincount(y))\n", 128 | "\n", 129 | "plt.scatter(X[y==0, 0], X[y==0, 1], label='class 0', marker='o')\n", 130 | "plt.scatter(X[y==1, 0], X[y==1, 1], label='class 1', marker='s')\n", 131 | "plt.xlabel('feature 1')\n", 132 | "plt.ylabel('feature 2')\n", 133 | "plt.legend()\n", 134 | "plt.show()" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "- Shuffle dataset\n", 142 | "- Split dataset into 70% training and 30% test data\n", 143 | "- Seed random number generator for reproducibility" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 5, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "shuffle_idx = np.arange(y.shape[0])\n", 153 | "shuffle_rng = np.random.RandomState(123)\n", 154 | "shuffle_rng.shuffle(shuffle_idx)\n", 155 | "X, y = X[shuffle_idx], y[shuffle_idx]\n", 156 | "\n", 157 | "X_train, X_test = X[shuffle_idx[:70]], X[shuffle_idx[70:]]\n", 158 | "y_train, y_test = y[shuffle_idx[:70]], y[shuffle_idx[70:]]" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "- Standardize training and test datasets (mean zero, unit variance)" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 6, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "mu, sigma = X_train.mean(axis=0), X_train.std(axis=0)\n", 177 | "X_train = (X_train - mu) / sigma\n", 178 | "X_test = (X_test - mu) / sigma" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "- Check dataset (here: training dataset) after preprocessing steps" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 7, 191 | "metadata": {}, 192 | "outputs": [ 193 | { 194 | "data": { 195 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYVPWd5/H3R9I+9EQjE+mN0A0D7jKMlxGJHdAwmxhy\nQZgYDGpGzeqQGxs3Jk42kkXxmTDGZMiSmdkkJkEcHYesiRM3StjFyGiYxIxZLyAGRUIgTrJ0y0RC\nFsWAK5fv/lGnoemu6q7qrjrnVNXn9Tz1UOfU6XO+Vd2cb/3uigjMzMzKdVzWAZiZWX1x4jAzs4o4\ncZiZWUWcOMzMrCJOHGZmVhEnDjMzq0hmiUPSOEn/JOlZSZslXVvkGEn6sqTtkjZJemMWsZqZ2VGv\nyfDaB4FPRcSTkk4ENkh6MCKe7XXMbGBS8pgOfD3518zMMpJZiSMidkbEk8nzvcAWoL3PYXOBlVHw\nKDBK0piUQzUzs16yLHEcIWkCMBV4rM9L7cCOXttdyb6dA51v9OjRMWHChOoFaGbW4DZs2PDriGgr\n59jME4ekE4DvAH8WES8N4zwLgAUA48ePZ/369VWK0Mys8Un6ZbnHZtqrSlILhaRxV0TcW+SQbmBc\nr+2OZF8/EbEiIjojorOtraykaWZmQ5BlryoBtwNbIuKvSxy2Grgq6V11LvBiRAxYTWVmZrWVZVXV\nDOBK4GlJTyX7bgDGA0TEcuB+YA6wHdgHfCCDOM3MrJfMEkdE/DOgQY4J4GPVuN6BAwfo6urilVde\nqcbpGsLIkSPp6OigpaUl61DMrI5k3jielq6uLk488UQmTJhAoZasuUUEu3fvpquri4kTJ2YdjpnV\nkaaZcuSVV17h5JNPdtJISOLkk092CczMKtY0iQNw0ujDn4eZDUXTVFWZWUo+3w6vvtx///EnwA1F\ne9NbnXHiyNiSJUs44YQTuO6666p+7g0bNjB//nz279/PnDlz+NKXvuRShg1NJcmg2HED7be601RV\nVc3m6quv5rbbbmPbtm1s27aNBx54IOuQrF5VKxksOamQhKyuOXGUsGpjNzOWrmPiojXMWLqOVRuH\nX8ReuXIlZ511FlOmTOHKK6/s9/ptt93Gm970JqZMmcLFF1/Mvn37ALjnnns488wzmTJlCm95y1sA\n2Lx5M9OmTePss8/mrLPOYtu2bceca+fOnbz00kuce+65SOKqq65i1apVw34PZsPmkkfdc1VVEas2\ndnP9vU+z/8AhALr37Of6e58G4KKpQ/u2tHnzZm6++WZ+/OMfM3r0aH7zm9/0O2bevHl85CMfAeDG\nG2/k9ttv5+Mf/zg33XQTa9eupb29nT179gCwfPlyrr32Wt7//vfz6quvcujQoWPO1d3dTUdHx5Ht\njo4Ourtdv2xmw+fEUcSytVuPJI0e+w8cYtnarUNOHOvWrePSSy9l9OjRALz+9a/vd8wzzzzDjTfe\nyJ49e3j55ZeZNWsWADNmzGD+/Pm8733vY968eQCcd955fO5zn6Orq4t58+YxadKkIcVllhtuVK8b\nrqoq4vk9+yvaXy3z58/nlltu4emnn+Yzn/nMkTEWy5cv5+abb2bHjh2cc8457N69myuuuILVq1fT\n2trKnDlzWLdu3THnam9vp6ur68h2V1cX7e2uW7YUHH/C0H7Ojep1w4mjiLGjWivaX46ZM2dyzz33\nsHv3boCiVVV79+5lzJgxHDhwgLvuuuvI/p///OdMnz6dm266iba2Nnbs2MFzzz3Hqaeeyic+8Qnm\nzp3Lpk2bjjnXmDFjeN3rXsejjz5KRLBy5Urmzp075PityZVKBsX239ANS16sbTyWKVdVFbFw1uRj\n2jgAWltGsHDW5CGf84wzzmDx4sW89a1vZcSIEUydOpU777zzmGM++9nPMn36dNra2pg+fTp79+4t\nxLNwIdu2bSMiePvb386UKVP4whe+wDe+8Q1aWlo45ZRTuOGGG/pd82tf+9qR7rizZ89m9uzZQ47f\nmtxQqoqOP6F01ZPVNRXmEWwsnZ2d0Xchpy1btnDaaaeVfY5VG7tZtnYrz+/Zz9hRrSycNXnI7Rt5\nVunnYlYzS04a4DWXYGpN0oaI6CznWJc4SrhoantDJgozs+FyG4eZ5UMl7SiWKZc4zCwf3OW2bmS9\n5vgdkl6Q9EyJ18+X9KKkp5LHn6cdo5mZHSvrEsedwC3AygGO+VFEvDudcMysLniwYKYyLXFExMNA\n/wENZmYD8WDBTNVD4/ibJW2S9D1JZ5Q6SNICSeslrd+1a1ea8Q3LkiVL+OIXv1iTcy9evJhx48Zx\nwgluXDSz6sl74ngSGB8RZwFfAUpO7xoRKyKiMyI629raUgswzy688EIef/zxrMMwswaT68QRES9F\nxMvJ8/uBFkmja37hz7cXBiP1fQxzHYE0p1UHOPfccxkzZsywYjYz6yvrxvEBSToF+FVEhKRpFBLd\n7ppfuAb1p2lPq25mViuZJg5J3wLOB0ZL6gI+A7QARMRy4BLgakkHgf3AZVGnc6R4WnWzKvI8WJnK\nNHFExOWDvH4Lhe66TWH+/PmsWrWKKVOmcOedd/KDH/wAKJQuHnvsMdasWcM555zDhg0buOKKK5g+\nfTpr1qxhzpw53HrrrcycOTPbN2CWFne5zVSu2zgaSdrTqpuZ1Uqu2zgaSRbTqn/605/mm9/8Jvv2\n7aOjo4MPf/jDLFmyJIV3a2Y1l+EgSE+rXkwTjUr1tOrW0PL8f3m4sVV5GnpPqz5cWf9BmVl15HmE\neZ5jG4TbOMzMrCJNVeKICCRlHUZuNGI1peVcnquOrGxNU+IYOXIku3fv9s0yERHs3r2bkSNHZh2K\nNZM6rp6xo5qmxNHR0UFXVxf1NAFirY0cOZKOjo6sw7C8qbRU4FJENjIcBNk0iaOlpYWJEydmHYZZ\n/lVaKshzKSLPI8yHG1uGSblpEoeZNaE8l3jyHNsgmqaNw8zMqsMlDjOrTKk2jXLkuerIyubEYWaV\nGU7bRR1Xz9hRThxmjWg4PZ2qVSpwKaJhOXGYNaLh9HQaTqlgCHMkWf3JtHFc0h2SXpD0TInXJenL\nkrZL2iTpjWnHaGZmx8q6xHEnhYWaVpZ4fTYwKXlMB76e/GtmVj0exFiRTEscEfEw0H9Fo6PmAiuj\n4FFglKQx6URnZkWVaruo5zaNPA9izKGsSxyDaQd29NruSvbtzCYcM/M3cGuYAYCSFkhaL2m956Oy\npteIpQLLjbyXOLqBcb22O5J9/UTECmAFFFYArH1oZjnmUoHVUN5LHKuBq5LeVecCL0aEq6nMLD2f\nb886gtzJtMQh6VvA+cBoSV3AZ4AWgIhYDtwPzAG2A/uAD2QTqVmDa/ZeRaUGPYIbyIvINHFExOWD\nvB7Ax1IKx6x5VfumWW+J6IZuWHJS1lHUjbxXVZlZPfK394bmxGFmZhVx4jAzs4rkvTuumeXVQO0Y\nWRpq+4rXCimbE4eZDe2mmdd2jIHiWnJS6QSSx0b7nHLiMLPq3zTz/O0968TWAJw4zKz6/O29oblx\n3MzMKuLEYWZmFXHiMLOhyesMvFlfvwm4jcOsmVRzKpC8tmP0xJXX7sINwInDrJnktQttLeQ1sTUA\nJw6zlK3a2M2ytVt5fs9+xo5qZeGsyVw01VN3W/1w4jBL0aqN3Vx/79PsP3AIgO49+7n+3qcBnDys\nbjhxmKVo2dqtR5JGj/0HDrFs7dbmThxZTcNeb9O/54QTh1mKnt+zv6L9uVOrG21WbS/N1OZTRZl2\nx5V0gaStkrZLWlTk9fMlvSjpqeTx51nEaVYtY0e1VrS/6obbhdY3WiPDEoekEcBXgXcCXcATklZH\nxLN9Dv1RRLw79QDNamDhrMnHtHEAtLaMYOGsyekEkHb1i6uCGlKWJY5pwPaIeC4iXgXuBuZmGI9Z\nzV00tZ2/nPeHtI9qRUD7qFb+ct4fNkb7xpKTComiN5dQGlKWbRztwI5e213A9CLHvVnSJqAbuC4i\nNhc7maQFwAKA8ePHVzlUs+q5aGp7YySKYpwQmkLepxx5EhgfEWcBXwFWlTowIlZERGdEdLa1taUW\noJlVQVbTl+R12pScy7LE0Q2M67Xdkew7IiJe6vX8fklfkzQ6In6dUoxm1lupdTaGK6v2DrezDEmW\nieMJYJKkiRQSxmXAFb0PkHQK8KuICEnTKJSQdqceaUY8wthyp+dGu+SkbOOwTGWWOCLioKRrgLXA\nCOCOiNgs6aPJ68uBS4CrJR0E9gOXRURkFXOaPMLYKpXLLxp5XgnQhkyNeB/u7OyM9evXZx3GsMxY\nuo7uIoPC2ke18siimRlEZHnW94sGFLr51qzHVp662dYyljy9zxqTtCEiOss51iPHc6ruRxhbqlKf\nyiRPN81advl1d+KinDhyauyo1qIljtRGGFtdaZgvGk30Db+e5b07btNaOGsyrS0jjtmX6gjjOrNq\nYzczlq5j4qI1zFi6jlUbm+smk/lUJtXib/h1wYkjpxp6hHGV9dTvd+/ZT3C0I0EzJQ9/0bA0uaoq\nxxp6hHEVearyoz3tcteryhqSE4fVvYap3x8mf9GoAXcnLsqJw+qeOxLUgVo2etfy5u4G+aKcOKzu\nZT5VuQ2u3EbvoSQB39xTVzJxSBoHLKMwi+33gGURcSB5bVVEXJROiGYDc/1+A3ESqAsDlTjuAL4D\nPAp8CPihpAsjYjfwe2kEZ1Yu1++bpWegxNGWzBcF8HFJ/wF4WNJ7gMabp8TMzMoyUOJokTQyIl4B\niIj/LulfKUxK+NpUojMzGy6PRq+6gQYA/i19VuSLiIeAS4FnahmUmTWYLBdM8mj0qitZ4oiIvymx\nfyPwzppFZFZFuZxqvBn5m31DcXdca1he08SsNjKdq0rSBZK2StouaVGR1yXpy8nrmyS9MYs4rT4N\nNBWJmQ1dZolD0gjgq8Bs4HTgckmn9zlsNjApeSwAvp5qkFbXPBWJWW0MmjgkvUHS7ZK+l2yfLulD\nVbj2NGB7RDwXEa8CdwNz+xwzF1gZBY8CoySNqcK1rQk0zFTjNjxZNsw3qHLaOO4E/g5YnGz/DPgH\n4PZhXrsd2NFru4s+vbhKHNMO7Bzmta0JeCoSA9wwXwPlVFWNjohvA4cBIuIgcGjgH0mfpAWS1kta\nv2vXrqzDsRzwmiZmtVFOieO3kk4mGS0u6VzgxSpcuxsY12u7I9lX6TEARMQKYAVAZ2enR7bXkVp2\nmfVUJGbVV07i+M/AauDfSnoEaAMuqcK1nwAmSZpIIRlcBlzR55jVwDWS7qZQjfViRLiaqoG4y+zA\nPA7F8mjAxCHpOGAk8FZgMiBga88sucMREQclXUNhCpMRwB0RsVnSR5PXlwP3A3OA7cA+4APDva7l\ni1fvK81J1fJqwMQREYclfTUipgKbq33xiLifQnLovW95r+cBfKza17X8cJfZ0pxULa/KaRz/vqSL\nJanm0VjTcZfZ0pxULa/KSRz/EbgH+H+SXpK0V9JLNY7LmsTCWZNpbRlxzD53mS1wUrW8GjRxRMSJ\nEXFcRBwfEa9Ltl+XRnDW+NxltjQnVcurQXtVSXpLsf0R8XD1w7Fm5C6zxXlJXMurcrrjLuz1fCSF\nqUI2ADNrEpFZjdRj11YnVcujQRNHRFzYe1vSOOC/1Swisxpw11az6hnK7LhdwGnVDsSsljzFuln1\nlNPG8RWS6UYoJJqzgSdrGZSlox6rbobKXVvNqqecNo71vZ4fBL4VEY/UKB5LSbNV3Ywd1Up3kSTh\nrq1mlSunqmpURPx98rgrIh6RdG3NI7OaaraqG3dtNauechLHnxbZN7/KcVjKmq3qxuNFzKqnZFWV\npMspzFY7UdLqXi+dCPym1oFZ9fVu0zhO4lD0n32+katu3LXVrDoGauP4MYWV9kYDf9Vr/15gUy2D\nsurr26ZRLGm46sbMylEycUTEL4FfAuelF47VSrE2DYAREocjGr5XlZlVTzndcc8FvkJh7MbxFNbO\n+K3nq6ovpdouDkfwL0v/OOVozKyeldMd9xYKq/PdA3QCVwG/X8ugstDoYxrcHdXMqqWskeMRsR0Y\nERGHIuLvgAuGc1FJr5f0oKRtyb+/W+K4X0h6WtJTktYXO6Yaeur/u/fsJzg6pmHVxqLLm9cld0c1\ns2opJ3Hsk3Q88JSk/yrpk2X+3EAWAd+PiEnA95PtUt4WEWdHROcwr1lSM4xpcHdUM6uWcqqqrqSQ\nKK4BPgmMAy4e5nXnAucnz/8e+AHwX4Z5ziFrljEN7o5qZtVQzuy4v5TUCoyJiL+o0nXfEBE7k+f/\nCryh1OWBhyQdAm6NiBWlTihpAbAAYPz48RUF4/p/M7PyDVrlJOlC4CnggWT77D4DAkv93EOSniny\nmNv7uIgIjk6i2NcfRcTZwGzgY6UWlUrOsyIiOiOis62tbbDwjuH6fzOz8pVTVbWEwuJNPwCIiKck\nTRzshyLiHaVek/QrSWMiYqekMcALJc7Rnfz7gqT7kjiqvvKgV1ozMytfOYnjQES8KKn3vlIlhHKt\npjAH1tLk3+/2PUDSa4HjImJv8vxdwE3DvG5Jrv+vX43eldosb8pJHJslXQGMkDQJ+ASF6UiGYynw\nbUkfojA6/X0AksYCfxsRcyi0e9yXJKzXAN+MiAeGeV0bhjzdoHti6d6zH3H0m0yjTw9fTXn6fVp9\nURSZs+iYA6TfARZT+MYPsBa4OSJeqXFsQ9bZ2Rnr19ds2EdT6jvXFRTagbLo0lsslr7aR7XyyKKZ\nR473DfJYefp9Wj5I2lDusIeSjeOSvpE8/UhELI6INyWPG/OcNKw28jTWpdS8W731dKUeyuDOVRu7\nmbF0HRMXrWHG0nUNNRC0R55+n1Z/BqqqOiepOvqgpJXAsY0cEZ5avYnkaaxLOdc8TmLiojVFp4/v\nuUEW+2bdLCsj5un3afVnoO64yymM6v4DYEOfh+uBmkypMS1ZjHUp55qHIgiKTx8PpW+QzfJNPE+/\nT6s/JRNHRHw5Ik4D7oiIUyNiYq/HqSnGaDmQp7EuxWLpKQ6POLb3X0mlbpDN8k08T79Pqz+DDgCM\niKvTCMTyLU9zXRWL5W/+5Gx+sfSPOTxIZw8Y+AbZLN/E8/T7tPozaK+qeuReVc1rxtJ1RaePKXfB\nKvc2smZVSa+qcsZxmKWiGt1mF86aPKwbv2cRMBucE4flQrV6M1Xjxj+UWQR6D0gckfTkanfSsQbl\nxGG5MFBvpkpvvGlPH9M36fX05GrUrrxmThxWllqPvi7Va6lYe0XeDDQgcajJzyzPhruSnzWBNJbW\nLdVrScn182ywrrqN1pXXzInDBpXGoLiFsyZTbARGJNfPs8G66jZaV14zJw4bVBqD4i6a2l5yrv68\nf2MvNpiuhwfVWSNy4rBBpTUorr1OB9/1HkwHR0eve1CdNSo3jtugSo2NqPY36bSuUwteCMyaSSYl\nDkmXStos6bCkkiMVJV0gaauk7ZIWpRmjHZXW9BSeBsOsPmQy5Yik04DDwK3AdRHRb34QSSOAnwHv\nBLqAJ4DLI+LZwc7vKUcsr7yolOVV7qcciYgtABp4JtNpwPaIeC459m5gLjBo4rDmlecbcyWj4/P8\nPszy3MbRDuzotd0FTM8ollzwzWRgeV+EqdzR8UN9H/77sLTUrI1D0kOSninymFuj6y2QtF7S+l27\ndtXiEplKYxBevcv7Ikzldmseyvvw34elqWaJIyLeERFnFnl8t8xTdAPjem13JPtKXW9FRHRGRGdb\nW9twQs+lvN8U8yDvizCV2615KO/Dfx+WpjyP43gCmCRpoqTjgcuA1RnHlJm83xSHY9XGbmYsXcfE\nRWuYsXTdkL8l530RpnJX3RvK+2jkvw/Ln6y6475XUhdwHrBG0tpk/1hJ9wNExEHgGmAtsAX4dkRs\nziLePMj7TXGoqlnFkvflUMvtbjyU99Gofx+WT14BsE7U08p0qzZ28xf/czP/d98BAEa1trDkPWcU\njbPUin3to1p5ZNHMIV27ERqIK30f9fT3YfmU++64Vrl6WZlu1cZuFv6Pn3Dg0NEvJHv2H2DhPT8B\n+vcKqnYVS6kR3PWWUCodiV4vfx/WGJw46kg9TGuxbO3WY5JGjwOH40hDbe+b20mtLezZf6Df8dWs\nYsl7N91qqYe/D2sMeW4ctzo0UEmh54bduz3jt68epOW4YweCVrtdwj2OzKrLicOqaqCSwgip3w38\nwKHghJGvqen8VO5xZFZdrqqyqlo4a3K/Ng6AluPEgcPFO2Ls2XeAjX/+rprFNHZUa9EGePc4Mhsa\nlzisqi6a2s6yS6bwu7/TcmTfqNYWll06JbP1NvLeTdes3rjEYVU3UCNtFuttuMeRWXU5cVhqsryB\nu8eRWfU4cViqfAM3q39OHGa91NtAQbMsOHGYJZploKDZcLlXlVnCAwXNyuPEYZbwQEGz8jhxmCU8\nNblZeZw4zBIeKGhWHjeOmyU8UNCsPJkkDkmXAkuA04BpEVF01SVJvwD2AoeAg+UuMmI2VB5nYja4\nrEoczwDzgFvLOPZtEfHrGsdjZmZlyiRxRMQWAEmDHWpmZjmT98bxAB6StEHSgoEOlLRA0npJ63ft\n2pVSeGZmzadmJQ5JDwGnFHlpcUR8t8zT/FFEdEv6N8CDkn4aEQ8XOzAiVgArADo7O4sv/GBmZsNW\ns8QREe+owjm6k39fkHQfMA0omjjMzCwdua2qkvRaSSf2PAfeRaFR3czMMpRJ4pD0XkldwHnAGklr\nk/1jJd2fHPYG4J8l/QR4HFgTEQ9kEa+ZmR2VVa+q+4D7iux/HpiTPH8OmJJyaGZmNgiPHLdUeb0L\ns/rnxGGp8XoXZo0ht43j1ni83oVZY3DisNR4vQuzxuDEYanxehdmjcGJw1Lj9S7MGoMbxy01Xu/C\nrDE4cViqvN6FWf1zVZWZmVXEicPMzCrixGFmZhVx4jAzs4q4cdysCM+pZVaaE4dZH55Ty2xgrqoy\n68NzapkNLKuFnJZJ+qmkTZLukzSqxHEXSNoqabukRWnHac3Jc2qZDSyrEseDwJkRcRbwM+D6vgdI\nGgF8FZgNnA5cLun0VKO0puQ5tcwGlkniiIh/jIiDyeajQEeRw6YB2yPiuYh4FbgbmJtWjNa8PKeW\n2cDy0Dj+QeAfiuxvB3b02u4CpqcSkQ2qkXsdeU4ts4HVLHFIegg4pchLiyPiu8kxi4GDwF1VuN4C\nYAHA+PHjh3s6G0Az9DrynFpmpdUscUTEOwZ6XdJ84N3A2yMiihzSDYzrtd2R7Ct1vRXACoDOzs5i\n57MqGajXkW+2Zo0vq15VFwCfBt4TEftKHPYEMEnSREnHA5cBq9OK0UpzryOz5pZVr6pbgBOBByU9\nJWk5gKSxku4HSBrPrwHWAluAb0fE5ozitV7c68isuWXSOB4R/67E/ueBOb227wfuTysuK8/CWZOP\naeMA9zoyayZ56FVldca9jsyamxOHDYl7HZk1LycOK1sjj90ws/I5cVhZmmHshpmVx7PjWlk8Y6yZ\n9XDisLJ47IaZ9XDisLJ47IaZ9XDisLJ4xlgz6+HGcSuLx26YWQ8nDiubx26YGbiqyszMKuTEYWZm\nFXHiMDOzijhxmJlZRZw4zMysIk4cZmZWERVf7ru+SdoF/DLrOAYwGvh11kHkjD+T/vyZ9OfPpL9q\nfSa/FxFt5RzYkIkj7yStj4jOrOPIE38m/fkz6c+fSX9ZfCauqjIzs4o4cZiZWUWcOLKxIusAcsif\nSX/+TPrzZ9Jf6p+J2zjMzKwiLnGYmVlFnDgyIGmZpJ9K2iTpPkmjso4pa5IulbRZ0mFJTd1rRtIF\nkrZK2i5pUdbx5IGkOyS9IOmZrGPJC0njJP2TpGeT/zvXpnVtJ45sPAicGRFnAT8Drs84njx4BpgH\nPJx1IFmSNAL4KjAbOB24XNLp2UaVC3cCF2QdRM4cBD4VEacD5wIfS+tvxYkjAxHxjxFxMNl8FOjI\nMp48iIgtEbE16zhyYBqwPSKei4hXgbuBuRnHlLmIeBj4TdZx5ElE7IyIJ5Pne4EtQCoL5jhxZO+D\nwPeyDsJyox3Y0Wu7i5RuBla/JE0ApgKPpXE9rwBYI5IeAk4p8tLiiPhucsxiCsXNu9KMLSvlfCZm\nVhlJJwDfAf4sIl5K45pOHDUSEe8Y6HVJ84F3A2+PJukTPdhnYgB0A+N6bXck+8z6kdRCIWncFRH3\npnVdV1VlQNIFwKeB90TEvqzjsVx5ApgkaaKk44HLgNUZx2Q5JEnA7cCWiPjrNK/txJGNW4ATgQcl\nPSVpedYBZU3SeyV1AecBayStzTqmLCSdJq4B1lJo7Px2RGzONqrsSfoW8L+ByZK6JH0o65hyYAZw\nJTAzuY88JWlOGhf2yHEzM6uISxxmZlYRJw4zM6uIE4eZmVXEicPMzCrixGFmZhVx4jArQtInJG2R\nVPGofkkTJF1Ri7iS879F0pOSDkq6pFbXMSvFicOsuP8EvDMi3j+En50AVJw4kplxy/F/gPnANyu9\nhlk1OHGY9ZEMyDwV+J6kT0p6bbIexOOSNkqamxw3QdKPkm//T0p6c3KKpcC/TwZkfVLSfEm39Dr/\n/5J0fvL8ZUl/JeknwHmSzpH0Q0kbJK2VNKZvfBHxi4jYBByu8UdhVpTnqjLrIyI+mkwL87aI+LWk\nzwPrIuKDyaJbjycTNr5AoVTyiqRJwLeATmARcF1EvBuOzEtWymuBxyLiU8m8Qz8E5kbELkl/AnyO\nwgzKZrnhxGE2uHcB75F0XbI9EhgPPA/cIuls4BDw+0M49yEKk9QBTAbOpDAVDcAIYOcw4jarCScO\ns8EJuLjvQlOSlgC/AqZQqPZ9pcTPH+TYauGRvZ6/EhGHel1nc0ScV42gzWrFbRxmg1sLfDyZjRRJ\nU5P9JwE7I+Iwhcnmehq391KYxLLHL4CzJR0naRyFVf6K2Qq0STovuU6LpDOq+k7MqsCJw2xwnwVa\ngE2SNifbAF8D/jRp2P4D4LfJ/k3AIUk/kfRJ4BHgX4BngS8DTxa7SLJU7CXAF5JzPgW8ue9xkt6U\nzCR8KXBrEpNZajw7rpmZVcQlDjMzq4gTh5mZVcSJw8zMKuLEYWZmFXHiMDOzijhxmJlZRZw4zMys\nIk4cZmY7ublFAAAACElEQVRWkf8P6LxbxT/kOlAAAAAASUVORK5CYII=\n", 196 | "text/plain": [ 197 | "" 198 | ] 199 | }, 200 | "metadata": {}, 201 | "output_type": "display_data" 202 | } 203 | ], 204 | "source": [ 205 | "plt.scatter(X_train[y_train==0, 0], X_train[y_train==0, 1], label='class 0', marker='o')\n", 206 | "plt.scatter(X_train[y_train==1, 0], X_train[y_train==1, 1], label='class 1', marker='s')\n", 207 | "plt.xlabel('feature 1')\n", 208 | "plt.ylabel('feature 2')\n", 209 | "plt.legend()\n", 210 | "plt.show()" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "## Implementing a Perceptron in NumPy" 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "metadata": {}, 223 | "source": [ 224 | "- Implement function for perceptron training in NumPy" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 8, 230 | "metadata": { 231 | "collapsed": true 232 | }, 233 | "outputs": [], 234 | "source": [ 235 | "def perceptron_train(features, targets, mparams=None,\n", 236 | " zero_weights=True, learning_rate=1., seed=None):\n", 237 | " \"\"\"Perceptron training function for binary class labels\n", 238 | "\n", 239 | " Parameters\n", 240 | " ----------\n", 241 | " features : numpy.ndarray, shape=(n_samples, m_features)\n", 242 | " A 2D NumPy array containing the training examples\n", 243 | "\n", 244 | " targets : numpy.ndarray, shape=(n_samples,)\n", 245 | " A 1D NumPy array containing the true class labels\n", 246 | "\n", 247 | " mparams : dict or None (default: None)\n", 248 | " A dictionary containing the model parameters, for instance\n", 249 | " as returned by this function. If None, a new model parameter\n", 250 | " dictionary is initialized. Note that the values in mparams\n", 251 | " are updated inplace if a mparams dict is provided.\n", 252 | "\n", 253 | " zero_weights : bool (default: True)\n", 254 | " Initializes weights to all zeros, otherwise model weights are\n", 255 | " initialized to small random number from a normal distribution\n", 256 | " with mean zero and standard deviation 0.1.\n", 257 | "\n", 258 | " learning_rate : float (default: 1.0)\n", 259 | " A learning rate for the parameter updates. Note that a learning\n", 260 | " rate has no effect on the direction of the decision boundary\n", 261 | " if if the model weights are initialized to all zeros.\n", 262 | "\n", 263 | " seed : int or None (default: None)\n", 264 | " Seed for the pseudo-random number generator that initializes the\n", 265 | " weights if zero_weights=False\n", 266 | "\n", 267 | " Returns\n", 268 | " -------\n", 269 | " mparams : dict\n", 270 | " The model parameters after training the perceptron for one epoch.\n", 271 | " The mparams dictionary has the form:\n", 272 | " {'weights': np.array([weight_1, weight_2, ... , weight_m]),\n", 273 | " 'bias': np.array([bias])}\n", 274 | "\n", 275 | " \"\"\"\n", 276 | " # initialize model parameters\n", 277 | " if mparams is None:\n", 278 | " mparams = {'bias': np.zeros(1)}\n", 279 | " if zero_weights:\n", 280 | " mparams['weights'] = np.zeros(features.shape[1])\n", 281 | " else:\n", 282 | " rng = np.random.RandomState(seed)\n", 283 | " mparams['weights'] = rng.normal(loc=0.0, scale=0.1,\n", 284 | " size=(features.shape[1]))\n", 285 | "\n", 286 | " # train one epoch\n", 287 | " for training_example, true_label in zip(features, targets):\n", 288 | " linear = np.dot(training_example, mparams['weights']) + mparams['bias']\n", 289 | "\n", 290 | " # if class 1 was predicted but true label is 0\n", 291 | " if linear > 0. and not true_label:\n", 292 | " mparams['weights'] -= learning_rate * training_example\n", 293 | " mparams['bias'] -= learning_rate * 1.\n", 294 | "\n", 295 | " # if class 0 was predicted but true label is 1\n", 296 | " elif linear <= 0. and true_label:\n", 297 | " mparams['weights'] += learning_rate * training_example\n", 298 | " mparams['bias'] += learning_rate * 1.\n", 299 | "\n", 300 | " return mparams" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "- Train the perceptron for 2 epochs" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 9, 313 | "metadata": {}, 314 | "outputs": [], 315 | "source": [ 316 | "model_params = perceptron_train(X_train, y_train, \n", 317 | " mparams=None, zero_weights=True)\n", 318 | "\n", 319 | "for _ in range(2):\n", 320 | " _ = perceptron_train(X_train, y_train, mparams=model_params)" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": {}, 326 | "source": [ 327 | "- Implement a function for perceptron predictions in NumPy" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 10, 333 | "metadata": { 334 | "collapsed": true 335 | }, 336 | "outputs": [], 337 | "source": [ 338 | "def perceptron_predict(features, mparams):\n", 339 | " \"\"\"Perceptron prediction function for binary class labels\n", 340 | "\n", 341 | " Parameters\n", 342 | " ----------\n", 343 | " features : numpy.ndarray, shape=(n_samples, m_features)\n", 344 | " A 2D NumPy array containing the training examples\n", 345 | "\n", 346 | " mparams : dict\n", 347 | " The model parameters aof the perceptron in the form:\n", 348 | " {'weights': np.array([weight_1, weight_2, ... , weight_m]),\n", 349 | " 'bias': np.array([bias])}\n", 350 | "\n", 351 | " Returns\n", 352 | " -------\n", 353 | " predicted_labels : np.ndarray, shape=(n_samples)\n", 354 | " NumPy array containing the predicted class labels.\n", 355 | "\n", 356 | " \"\"\"\n", 357 | " linear = np.dot(features, mparams['weights']) + mparams['bias']\n", 358 | " predicted_labels = np.where(linear.reshape(-1) > 0., 1, 0)\n", 359 | " return predicted_labels" 360 | ] 361 | }, 362 | { 363 | "cell_type": "markdown", 364 | "metadata": {}, 365 | "source": [ 366 | "- Compute training and test error" 367 | ] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": 11, 372 | "metadata": {}, 373 | "outputs": [ 374 | { 375 | "name": "stdout", 376 | "output_type": "stream", 377 | "text": [ 378 | "Number of training errors 0\n", 379 | "Number of test errors 2\n" 380 | ] 381 | } 382 | ], 383 | "source": [ 384 | "train_errors = np.sum(perceptron_predict(X_train, model_params) != y_train)\n", 385 | "test_errors = np.sum(perceptron_predict(X_test, model_params) != y_test)\n", 386 | "\n", 387 | "print('Number of training errors', train_errors)\n", 388 | "print('Number of test errors', test_errors)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "- Visualize the decision boundary" 396 | ] 397 | }, 398 | { 399 | "cell_type": "markdown", 400 | "metadata": {}, 401 | "source": [ 402 | "Perceptron is a linear function with threshold\n", 403 | "\n", 404 | "$$w_{1}x_{1} + w_{2}x_{2} + b \\geq 0.$$\n", 405 | "\n", 406 | "We can rearrange this equation as follows:\n", 407 | "\n", 408 | "$$w_{1}x_{1} + b \\geq 0 - w_{2}x_{2}$$\n", 409 | "\n", 410 | "$$- \\frac{w_{1}x_{1}}{{w_2}} - \\frac{b}{w_2} \\leq x_{2}$$" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 12, 416 | "metadata": {}, 417 | "outputs": [ 418 | { 419 | "data": { 420 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAADFCAYAAAAMsRa3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VNX5wPHvyWRCErJBErYECPuWEELCGtzQVqoCAVll\nETeUzR1/WFpF3EttVXYoiCAqoBBEqlhLrWWVBMIOsgkkUMFgAEmQ7f7+SEIDZJJZ7sy9M/N+nofn\nkUly5yXOmXfOed9zrtI0DSGEEMJbBBgdgBBCCOEISVxCCCG8iiQuIYQQXkUSlxBCCK8iiUsIIYRX\nkcQlhBDCq0jiEkII4VUkcQkhhPAqkriEEEJ4lUAjnjQmJkZLSEgw4qmF0EV2dvZPmqbFGh1HKRlT\nwhfYO64MSVwJCQlkZWUZ8dRC6EIpddjoGMqSMSV8gb3jSpYKhRBCeBVJXEIIIbyKJC4hhBBexZAa\nlxDC/S5evEhubi7nz583OhRTCQ4OJj4+HqvVanQowkk+lbj+tecE//7+JOPvboHVIpNJ4d9yc3MJ\nDw8nISEBpZRT1zj36yV+PneBOtVCCHDyGmaiaRr5+fnk5ubSoEEDo8MRTvKpd/ecowXMW/cDg2Zv\n5OTZX40ORwhDnT9/nujoaKeTFkDhhcucKrzAoZ/OcenyFR2jM4ZSiujoaJmFejmfSlxP/aYp7wxo\nw7a8ArpPXkPO0QKjQxLCUK4kLYDY8CrUrR5K4YXLHDh5jl8vXdYpMuO4+jsRxvOpxAXQs00cn47o\nTKBF0W/GehZvOmp0SEJ4tWqhQTSMqcqlK1c4cOIc5369ZHRIws/5XOICaFUnkhWju9C+QXWe+3Qb\nf8zcwYVL3r/MIYRRqlYJpHFsGJYAOPjTOQoKLzh9rQkTJvDnP/9Zx+j+Jzs7m6SkJBo3bszjjz+O\npmlueR5hLJ9MXADVqgYx74F2PHpzQxZsOMx9szdw4qysawthS+aWPNLfWE2DcStJf2M1mVvyrvl6\nFauFRrFhhFotHDlVyIkz502XGEaMGMHs2bPZt28f+/bt48svvzQ6JOEGPpu4AAItATx/VwsmD0xh\n57EzdJ+8hi1HfjY6LCFMJ3NLHs8v3U5eQREakFdQxPNLt9+QvAItATSIrUpUaBD/PXOe3J+LuFJB\n8po/fz6tW7cmOTmZIUOG3PD12bNn065dO5KTk7n33nspLCwEYMmSJSQmJpKcnMzNN98MwM6dO2nf\nvj1t2rShdevW7Nu375prHT9+nDNnztCxY0eUUgwdOpTMzEwXfzPCjHw6cZXqnlyHpSM7ExQYQP+Z\nG1i06YjRIQlhKpNW7aXo4rWNF0UXLzNp1d4bvjdAKepWC6FmRDA/F17gBxsdhzt37uSVV15h9erV\nbN26lXfeeeeG7+nduzebNm1i69attGjRgjlz5gAwceJEVq1axdatW/nss88AmDFjBk888QQ5OTlk\nZWURHx9/zbXy8vKueSw+Pp68vGsTr/ANfpG4AFrUjmDF6C50aFid//t0O+OXbZe6lxAljhUUOfS4\nUoqaEcHUrRbKORsdh6tXr6Zv377ExMQAUL169Ruus2PHDm666SaSkpJYuHAhO3fuBCA9PZ1hw4Yx\ne/ZsLl8uvm6nTp147bXXePPNNzl8+DAhISFO/3uFd/ObxAUQFRrEvAfaM+LWRizceISBszdw4ozU\nvYSoE1V+ErD1eKlqVYNo4ELH4bBhw5gyZQrbt2/nxRdfvLq/asaMGbzyyiscPXqU1NRU8vPzue++\n+/jss88ICQnhrrvuYvXq1ddcKy4ujtzc3Kt/z83NJS4uzqF4hHfwq8QFYAlQ/F+35ky9ry27jp3h\nnslryD4sdS/h38be2YwQq+Wax0KsFsbe2azSnw2rEkij2DACrus47Nq1K0uWLCE/Px+AU6dO3fCz\nZ8+epXbt2ly8eJGFCxdeffzAgQN06NCBiRMnEhsby9GjRzl48CANGzbk8ccfp2fPnmzbtu2aa9Wu\nXZuIiAg2bNiApmnMnz+fnj17Ovy7EObnd4mr1N2ta7NsVGdCgiwMmLWeDzdK3Uv4r4yUOF7vnURc\nVAgKiIsK4fXeSWSk2DdjCbZaaHxdx2HLli0ZP348t9xyC8nJyTz99NM3/NzLL79Mhw4dSE9Pp3nz\n5lcfHzt2LElJSSQmJtK5c2eSk5NZvHgxiYmJtGnThh07djB06NAbrjdt2jQefvhhGjduTKNGjfjd\n737n9O9EmJcyop01LS1NM8tN704XXuTxj7fw7+9PMrB9XSb0aEWVQEvlPyj8mlIqW9O0NKPjKFXe\nmNq9ezctWrTwaBxXrmjk/lxEQdEFqocGmfaMQyN+N6Jy9o4rv51xlYoMtTJ3WDtG3tqIj747yoBZ\nG/hR6l5COCUgQFG3egg1woM5VdpxeEWaoIS+/D5xQXHd67luzZk2qC17/3uWeyavIeuHG9fjhRCV\nU0pRKzKY+NKOwxPnuOADZxwK85DEVcZdSbXJHJVO1SALA2dv4IMNh013MoAQ3qJ61SAaRIdy6coV\n9p84R6GccSh0IonrOk1rhrN8dBe6NI7hD5k7GPfpdp84EVsII4QFW6/pODztwhmHQpRyOXEppeoq\npf6llNqllNqplHpCj8CMFBliZc797RjTtTGLso7Sf+YG/nta6l5COKO04zDYauHwqUJOnDXfGYfC\nu+gx47oEPKNpWkugIzBKKdVSh+saKiBA8cxvmzFjcCr7fiyue313SOpeQjgj0BJAw5iqRIVY+e/p\n8+QVVHzGoRAVcTlxaZp2XNO0zSX/fRbYDfjMdvVuibXIHJVORHAg983ewPz1P8inReFWvriKAaUd\nh6HMnzKJv771Fj/8dI7LOnccjh8/nrp16xIWFqbrdYW5BOp5MaVUApACbCzna8OB4QD16tXT82nd\nrknNcDJHp/PUxzm8sHwn23NP83JGIsFW2e8l3KJ0FWOzUiocyFZK/UPTtF1ue8bX4uDCLzc+HhQG\nv9fvoFqlFGHBVgKCrJz7tfiMw4ToUIJ02jvZvXt3Ro8eTZMmTXS5njAn3ZozlFJhwKfAk5qmnbn+\n65qmzdI0LU3TtLTY2Fi9ntZjIoKtzB6axuO3N2FJdi79Z663eQCpEK4wZBWjvKRV0eN2snVbk9Cg\nQBrEhPLR/Pdom9aOpNatXb6tCUDHjh2pXbu2SzEL89NlxqWUslKctBZqmrZUj2uaUUCA4unfNCWx\nTgRPL95KjylrmHpfWzo0jDY6NOGjvHkVo/S2JuvWrSMmJuaGswrDgq08PGQA/QcP49IVjffffYM5\nc+YwZsyYq7c1iYuLo6CgAPjfbU0GDRrEhQsXrp4aL/yPHl2FCpgD7NY07S+uh2R+v21VUvcKsTLo\nbxuZt/aQ1L2E7rx9FcOe25rs37ubh/vezb13dObDjz4kK2cbmqbJbU1EhfRYKkwHhgBdlVI5JX/u\n0uG6pta4RhiZo9K5tVkNJqzYxbNLtnH+oh9/AnwtDiZE3vjnNZ/p0/Eof1nFGDZsGFOnTmH3zh08\n9dzvKThzjryCIqZPn+7wbU2E/3B5qVDTtDWA+U7R9ICIYCuzhqQyefV+/vr193z/41lmDEklrpJ7\nGHk1W0V8Wy78UvwzOhb4fZ2vrGJ07dqVXr168fTTTxMdHc2pU6dumHWV3tbk8uVLrFr+CdVja3Hq\n3AX27z/ATe3a0aFDB7744guOHj3K6dOnr97W5MiRI2zbto2uXbsa9K8TRpKTM1wUEKB44o4m/G1o\nGj/8dI7uk9ew/kC+0WG5jzPFehcL/H7I86sYQTbax209bodWrVo5fFuTkCAL8dVCeOWF39OyVRKt\nHLytyXPPPUd8fDyFhYXEx8czYcIEp+N3C1mZ0IXf39ZETwdO/sLw+Vn8kF/I+Lta8EB6AsqEt3Rw\nyYRI/a6lc6u1J8ltTdzr7PmLHMkvRAUoEqJDCQ3SdeeOcb+bisbPhNOei8Ok5LYmBmgUW1z3ur15\nDSZ+votnFm/177pXZWQmJmwID7bSqEYYAcDBk+c4XXTR6JCEiUji0ll4sJUZg1N55jdNWZaTx73T\n15H7c6HRYQnhdYKtFhrVKDnjMP8cJ8/+Kt27ApDE5RYBAYoxtzdhzv1pHMkvpPvkNazb/5PRYQk/\n5O1v9NaSMw4jQ6wcP13EsQLXD+j19t+JkMTlVl2b12T56HRiwqowZO53/O0/B71/0FRUxHehkC/0\nFxwcTH5+vte/5gICFPWqhxIbXoX8c7/yQ34hl68492/SNI38/HyCg4N1jlJ4kr4VT3GDhrFhLBuV\nzrOLt/LKyt1szzvNG71bExLkpeccOtNMoWdDh7BbfHw8ubm5nDx50uhQdHPh10scKLzIYYsiJqwK\nlgDHm5+Cg4OJj493Q3R2CAqzfSakh86L9AWSuDwgrEog0we3Zdo3B/jzV3vZ9+MvzBySSt3qoUaH\n5hm2BiuUn9RkoOrCarXSoEEDo8PQ3b+/P8mohZsJDbIwd1g7EuO86INRRa9rWx/wzNjEZHCSlaVC\nD1FKMeq2xsy9vx1Hfy6kx5Q1rNnnJ3Wv3+cVt/pe/8cWMw5UYRq3NI3l0xGdsVoC6DtjPV/v+tHo\nkPyPmw5ltpckLg+7rXkNVozuQmx4FYbO3cjsb32g7iWEhzWrFc6ykZ1pUjOMRxZk8d7aQ0aHJEp5\nYGO1JC4DJMRUZdnIdLol1uLVv+/m8Y9zKLxwyeiwhPAqNSKC+Xh4R37ToiYvrdjFi8t3cOmyvjem\nFC5y0wxMEpdBqlYJZOp9bXmuWzM+33aM3tPWcfSU7PcSwhGhQYFMH5zKIzc14P31hxm+IJtzv8qH\nQF8nzRkGUkox8tbGtKwdweMfbaH7lDVMHpjCTU3Md4sKnyWdXF7PEqAYf3dL6kVX5cXlO+g7Yz1z\nhqVRO9LLDruuqONQXEMSlwnc2qwGK8Z04dEF2dw/9zue69acR29u6HvnHF7PkW5DZxNJZYnJ4CKz\n0M+QjvWJrxbC6IWbyZi6ljn3e0HHobd+cKpo7HqAJC6TqB9dlaUjOzP2k2288cUetuedZlKf1rof\nLuox9gzI8gam3i3Bkpj8ym3NavDJiM48OG8T/WauZ/LAFG5vUdPosGzz1tfn9WPXw3s1vfRd0TeF\nBgUyZWAKSXGR/OnLPRw4Ubzfq350VaNDc5y3Dkjh9VrUjiBzVDoPvb+JR+Zn8WL3VtzfOcHosBy/\nl5038fAypyQuk1FK8dgtjWhZO4IxH22hx5S1vDswhVuaGlj3qmj2BL47GIXXqhkRzOJHO/H4Rzm8\n+NlOfsg/xx/ubunUSRu68eVx4uFlTUlcJnVz01hWjO7C8AVZDHvvO8be2YwRtzQypu4lsyfhhUKD\nApk5JJVXV+5m7tpDHD1VyDsDUqhaxY/f9ry1pnYdaYc3sXrRoSwd2Zl7WtfhT1/uZdSHm6XVV29u\nuPOvMA9LgOKF7i2Z2LMVq/ecoN/M9fx45rzRYRnHRz6E+vFHD+8QGhTIuwPa0Doukte/2M3+E78w\na0gaCTFeWPeyh95r5ZVdz4s+ZQrnDe2UQN1qoYz+8H8dhy3rRBgdlm3ywalCkri8gFKKR25uSIva\nEYz+aDM9pqzhnYEp3Nashn5PUlkdSy+VXU/vRCKJSZS4rXkNFj/WiYfmZdF3xjqmDGqr7xhyRUVn\nd9rDR5YA7aVL4lJKzQXuAU5ompaoxzXFjbo0iWHF6OL9Xg/O28Szv23GyFt1qnu5YwnB1cFYys8G\npXCfVnUiyRyVzoPzNvHQvE281DORIR3rGx2W63xkCdBeetW45gHddLqWqEDd6qF8OqIzPZLrMGnV\nXkZ8sJlf3F33cmbWpedMzc8GpXCvWpHBLHmsE7c2q8EfM3fwyue7nL4xpTCGLjMuTdO+VUol6HEt\nUbmQIAtv929DUlwkr3+xh15T1zJraBoN3FX3cmSjMOg30xLCTapWCWT20DRe/nwXf1tziMOnCnln\nQBv3bvg3w5FOZohBBx6rcSmlhgPDAerVq+epp/VZSikevqkhLWtHMOrDkrrXgDZ0be6hUwJ8ZAAI\n/2UJUEzo0Yr60aG8/Pku+s/cwJz706gREeyeJzRyWdvHlts9lrg0TZsFzAJIS0uTeblOOjeOuXrO\n4UPvZ/HUHU0ZfVtjAvTcaGnrLsUys3IbqRt7zgPpDahXPZQxH20hY+pa5j7Qjua1TNxx6IzKltu9\nLLFJV6EPiK9WXPd6ful2/vKP79mRd5q3+iUTHmy1/yKOHppZ3vd62Yvf5OYBU4D5BsfhF25vUZPF\nj3biofc30Wf6eqYOaqvvaTXuHhuuroB4WR1ZEpePCLZa+Eu/ZJLiInn177vJKKl7NYq184Vra/A4\ncniml734zUzqxp6XGFfacZjFg/M2MbFnKwZ10Knj0N1jw88+GOrVDv8RcCsQo5TKBV7UNG2OHtcW\n9lNK8WCXBrQoqXtlTFnLX/u34Y6Wbqp76XloqKf2kfkQqRvrr3ZkCEse68SYDzczftkODucXMq5b\nc32X3oXL9OoqHKjHdYQ+OjWKZsWYLjy2IJuH52fx5B1NeLxrE/0Hn54zKZmtOUzqxu4RVtJxOPHz\nXcz69iCH88/xdv8UQoIs7n9yWW63i5xV6KPiooo/OfZuG8fbX+9j+IJszpy/aHRYQniFQEsAL/Vo\nxQv3tOSrXT8yYNZ6Tpz1wBmH7voA52NnckqNy4cFWy281TeZ1nGRvLyypO41JI3GNRx4sVZU9DV6\nNuSlg054h9Kl9/hqITzxcQ69pq5j7rB2NKsVbnRojqtstuZl21skcfk4pRTD0hvQvHYEo0puaf6X\nfsn8tlUt2z9k73KFvY0berz4/az1XurG5vHbVrVY/GgnHnx/E32mr2Pa4Lbc1MTBjkOzJQYvX5KU\nxOUnOjYsrnuN+CCb4Quyefz2Jjx5u426lx7LFX6WaPQmdWNzSYov7jh8aN4mhr23iVcyEhnY3oGG\nGLMlAy+vKUuNywdlbskj/Y3VNBi3kvQ3VpO5pXjQ1IkKYdGjneiTGs+7/9zHI/OzOF3kQt1Lz3Vz\nky5JCFGqtG7cpXEMzy/dzutf7OaKnHFoCJlx+ZjMLXk8v3Q7RRcvA5BXUMTzS7cDkJESR7DVwqQ+\nrUmOj+SlFbtK6l6pNKnpxLq9PZ8i7V2S+H1exe311y9LesmShvAt4cFW5tyfxouf7WTmvw9yJL+Q\nv/ZvQ7BVp45Dsy0pmpQkLh8zadXeq0mrVNHFy0xatZeMlDiguO41pFMCzWpFMHJhNhlT1/JWvzZ0\nS6yg7lWWI+vjjixJOHKYr5csaQjfE2gJ4JWMRBrEVOXVv+/m+KwNzB6aRmx4FdcvLh/G7CKJy8cc\nKyiy+/H2DaoX7/f6YDOPfZDNmK6NefKOplT62bGiZOTISRtCeKnSQ67jq4Xy5KIt9Jq2lveGtbNv\n5cLLGyPMQGpcPqZOVIhDj9eODGHR8I70S4tn8ur9PPz+Jk5bbXRMyXKFENfolliLRcM7cf7iFXpP\nX8fa/T9V/kNmaIzw8n1dMuPyMWPvbHZNjQsgxGph7J3NbP5MsNXCm/e2pnV8FC+t2EnPqNnMejiN\nps7UvYTwM8l1o8gc1ZkH523i/rnf8VqvJPq1q2t0WBXz8pmdX8y4bHXZ+aKMlDhe751EXFQIiuJO\nqNd7J12tb9milGJwx/p89EhHfvn1MhlT1/LF9uOeCVoILxdfLZRPRnSmU6Nonvt0G3/6co90HLqR\nz8+4Kuuy80UZKXEO/dsyt+QxadVejhUUUScqhNG3NWL51mOMWLiZkbc24pnfNsOi9zmHpUsSFXUS\nBoVVfEJH2Xqa1AeEwSKCrcwd1o4Xlu9k2jcHOJxfyFv9kvXrOBRX+XzisqfLzp+Vl9jf/HIvL/ds\nRfNa4Uz75gA7j53h3QEpRIaW3N9Lj+OeLvxS+enyF365cSOzdBkKE7NaAnitVyINYkJ57e97OHa6\niNlD04gJ06HjUFzl80uFjnTZeZoZljBtJfa/fr2P13u35rVeSaw78BM9pq5h73/PFn+DXjMbSTbC\nBymlGH5zI6YPasuuY2foNW0t+0+c/d83eHljhBn4/IyrTlQIeeUkKVtddp5iliXMyhL7fR3q0axW\nOCM+yKbXtLVM6pPM3a1rV7xR8vrEJi3ywg/9Lqk2tSKDeWR+Fr2nrWPGkFQ6N4qRJW0d+PyMa+yd\nzQi5bo25si47T6hoCdOT7GmfT61fjRVjutC8VjijPtzMG1/s4fK43OJlvOv/QHGiKvtHCD+VUq8a\ny0amUzMimKFzvmNJ1lGjQ/IJPp+4nO2yczezLGHam9hrRgTz8fBODOpQjxn/PsCw976joPDCjReU\n5T8hrlG3enHHYceG0Yz9ZBt/XrVXOg5d5PNLheB4l50nmGUJs/T3UrarcOydzcr9fQUFBvBqrySS\n4iJ5YflOuk9Zw6whabSoHeF8ABU1epS35i9nuQkvFBli5b0H2vGHZTuY8q/9HD5VyKQ+raXj0El+\nkbjMyJmNwu7iaGIf0L4eTUvqXr2nrePNPq3pkVzH8Sd2poVd6gPCS1ktAbxxbxIJMVV588s9HCso\nYtaQVKKl49BhkrgMUt5M57bmsUxatZenFuVUOPMxg7b1iuteIz/YzOMfbWFn3mnG3tms4heUPffo\nknPchA9TSjHi1kbUqx7KU4tz6D29+K7KjWJlxcARkrgMVHamY5YuQ0fUCA/mw0c6MvHzncz89iA7\nj51hshZGNeVCncsM57gJ4WZ3ty7uOBxe0nE4c0gqHRtGGx2W19ClOUMp1U0ptVcptV8pNU6Pa/ob\ns3QZOiooMIBXMpJ4894kvjt0iu4XXmPnlfrlfKN8ohSirNT6xR2HMWFBDJmzkU+zc40OyT6vxd3Y\nOTwhsvhxD3F5xqWUsgBTgd8AucAmpdRnmqbtcvXa/sQsXYbO6t+uHs1qRfDYgmzuLXqTNzNa07ON\nOWeKQphFvehQlo5IZ8TCbJ5ZspXDpwp56o4mKKXzEWt6MsGqiB4zrvbAfk3TDmqadgH4GOipw3X9\niqO3IzGjNnWjWDGmC63jonji4xxeXbmLS5evGB2WEOZUMnOJ/FMM83Lvoq/lG9795z6e/OOL/Hrp\ncuU/78f0SFxxQNlddbklj11DKTVcKZWllMo6efKkDk9rbo4e52TWjdKOig2vwsJHOnB/p/rM/s8h\nhs79jlPnytnvJYS/KzNDCVKX+VPgLMYGLmL5pQ4M/ttGGTcV8NgGZE3TZmmalqZpWlpsrI0bFfqI\n0kaLvIIiNP7XaFFR8vLERmlPnY1otQTwUs9EJvVpTdbhn+k+eQ078uzoKAQ5x034DEfHm1IwKnA5\nk63vsjX3NL2nreXQT+c8FK130aOrMA8oe9e0+JLH/JazJ9K7c6O0EV2LfdPq0qxWOI8uyObe6et4\n897WlT+XtLwLH+DKeOtu2UCdBzrwyPzi80FnDk6lg3QcXkOPGdcmoIlSqoFSKggYAHymw3W9lhkb\nLYzqWmwdX1z3Sq4bxZOLcpi4Qupe9pBOXe/m6nhLrV+dZSM7U71qEIPnbGTZFhN1HJpgVcTlGZem\naZeUUqOBVYAFmKtp2k6XI/NiZjnOqSwjk2lMWBUWPtyBV1fuZu7aQ+w6fpqp97WVEwNskE5d76fH\neKsfXZVlI9J59IMsnlq0lcP5hTxxuwk6Dk2wKqJLjUvTtL9rmtZU07RGmqa9qsc1vZkrjRbuqkMZ\n3bVotQQwoUcr3uqbzJYjBXSfvIbtuXbWvfyPdOp6ObvGmx0zl8hQK/Mf7MC9beN5++t9PL14q3Qc\nIidnuIUjB9eW5c46lFnORrw3NZ6mNcN5dEEWfWas4/XeSfRuG+/RGLxAeZ26Ha7/JqXUcGA4QL16\n9TwTmbCLXePNzplLUGAAf+7bmoToUN76x/fklZxxGBUapHfYXkNpmueP109LS9OysrI8/rz2ytyS\n53DS0UP6G6vLXWKMiwph7biuLl/fqH9XefJ/+ZVRH25mw8FTDOucwPi7W2C1eM9ddpRS2Zqmpbnp\n2n2AbpqmPVzy9yFAB03TRtv6GbOPKX/kjvG2PCePsUu2EVcthPeGtSMhpqpO0ZqDveNKZlzXMfLM\nQHfXocx0e5fosCp88FAHXv9iD3PWHGL38TNMHdSWGKl7gXTq+gR3jLeebeKoExXC8PlZ9Jq2lllD\n02iXUF3X5/AG3vMR1wMyt+TxzOKthp0ZaHQdytMCLQH88Z6WvN2/DTlHi+te23ILjA7LDKRTV9jU\nLqE6y0amExUaxKDZG1me43+faWTGVaJ0pnXZxtKpJ7rvzFKH8rSMlDga1wjj0QXZ9JmxnlczEumb\nVrfyH/RRvtypa6blatOy49Y+CTFVWTqiM49+kM0TH+dwJL+Q0V0bG99x6CGmnXF56pSHUuXtuyjL\nE7MeT5yeYVaJcZGsGNOFtPrVGPvJNl5cvoOLfrzfyxc7dZ05UcYv2XmIbbWqQSx4qD29UuJ46x/f\n8+ySbVy45B9jxpQzLiPqTBXNqDw56zFTHcrTqlcNYv6D7Xnzyz3M/s8hdh8/y9RBbYkNl7qXL3D2\nRBlhW5VAC3/pl0z96FDe/nofeQWFzBycRmSo1ejQ3MqUMy4jTnmwNaOyKOU3sx4zCLQEMP7ulrwz\noA3b8orrXjlHpe7lC8x4oowvUErx5B1N+Wv/ZDYfLqDX9LUczvftMw5NmbiMeIHb2jT8Vr9kSVoG\n6Nkmjk9HdCbQoug3Yz2LNx2t/IeEqflb85Gn9UqJZ8FD7Tl17gK9pq0j+/Apo0NyG1MmLiNe4P5c\nXzKrVnUiWTG6C+0bVOe5T7fxh8ztfrOG74t85dY9ZtahYTRLR3QmIjiQgbM3smLrMaNDcgtT1riM\n6q7zVH2ptLMqr6AIi1Jc1jTipMOqXNWqBjHvgXZMWrWXmd8eZM/xs0wb3JYa4cFGhyYc5OyJMn4n\nKMx2V6EdGsaGsXRkOo8uyGLMR1s4cqqQkbc28qmOQ9OenGHGtlk9Yrq+8aQsq0VRNSiQ00UXTfFv\nNtv/g8+2HuO5T7YSGWJl+uBU2tarZlgs7jw5wxlycoZ9zPaadqdfL13muU+2sTznGH1T43m1VxJB\ngaZcZLuY0GpFAAAOa0lEQVTK60/OMFt3nV6djhW13V+8rFFQdPGG65f+nCcHm5EniNjSI7kOjWPD\nePSDLAbM3MDEnq0Y0F7O6BP2MeNr2p2qBFp4u38b6kdX5d1/7iOvoIjpg1J9ouPQ3OnXRPTqdHSk\nwaTo4mUmfLbTkL0vRt2/qzIt60SwYnQXOjSszril2xm/TOpewj5mfU27k1KKp3/TlLf6JrPph1P0\nnr6WI/mFRoflMklcdtKr09HRBpOCoouGDDYzty5HhQYx74H2PHZLIxZuPMLA2Rs4cea80WEJkzPz\na9rd7k2NZ8FDHfjplwv0mraWzUd+Njokl0jispMznY7lnf5RXmeVM9w92MzeumwJUIz7XXOm3JfC\nrmNnuGfyGp9u/xWuM/tr2t06Noxm6cjOVK0SyMBZG1i57bjRITlNEpedHG3ltXW8DXC17b4yIVYL\n1WysR5cOtswtebR56SsSxq0kYdxKUiZ+pcsyore0Lt/Tug7LRnUmJMjCgFkb+HDjEaNDEiblLa9p\nd2oUG8aykZ1JjItk1Iebmf7NAYxo0HOVabsKzciRjiRb99aKCrGS8+JvK/0+i1K81S8ZoNytAa/3\nTgJg7JKtXLxy7f9Dq0UxqY/rG6e9qQPrdOFFxny8hW+/P8nA9nWZ0KMVVQJdn9naIl2FxbzpNQLe\nF6+7nL94mWeXbOXzbccZ0K4uL2ckmuJ+eF7fVejtbC3lFRRdJHNL3tXBYmvP2vWbn8sbbOlvrL4h\naUFxd6Ie57+ZrbOzIpGhVt4b1o63vtrLtG8OsOe/Z5kxOJWaEbLfy13M3qVnK0mZITajBVstvDsg\nhYToqkz5135yfy5i6qC2RIZ4R8ehJC47OTpI60SFlDuTAq5JKvZsyrQ12Cqqc/lDwfl6lgDFc92a\nkxgXybNLtnLP5DVMH9SWND+80Z4nmPnQXLMnVTMICFA8e2cz6keH8vzS7fSZvo65w9pRt3qo0aFV\nyvi5oZdwtJW2onXz65NKRkoca8d15dAbd7N2XFe7B1ZFRWV/KTiX566k2mSOSqdqkIWBszfwwYbD\nXrmOb3Zm7tLzx9Z3Z/VNq8v8B9vz45nz9Jq21isOtZbEZSdHB2lGSlyljRWuGntnM6wBNx7jYrUo\nvyo4l6dpzXCWj+pCeuMY/pC5g3Gfbud8BfdbE44zc5eemZOqGXVuHMPSkcVNTv1nrueL7ebuOHQp\ncSml+iqldiqlriilTFOodgdnBumL3VtV2sXkyg0zM1LimNQ3magy69LVQq30b1eXSav2XnNNT9+Y\n0wwiQ63Mub8dY7o2ZlHWUfrP2sDx0/LGpRczd+mZOamaVeMa4SwbmU7LOhGM/HAzM/9t3o5Dl7oK\nlVItgCvATOBZTdPsamvyxg6o8s4YLK+JwpHndPaajsZpDVCgips2ynsef+i0+nLHcZ5ZvJWQIAvT\nBqXSvoFrdS/pKixm1teOO8aWvzh/8TLPLNnKym3HGdi+HhN7tvJYx6FHugo1Tdtd8mSuXMYjXC3W\nOnuydUVdTO4obpd3zfI6D8uu9/tDEbtbYm0axYYxfEE2983ewAvdWzKkY32veO2amVm79OQkeucF\nWy1MHpBC/eqhTPvmALk/FzJ1UFsigs3TcajLPi6l1DdUMuNSSg0HhgPUq1cv9fDhwy4/ryNs7ZeK\niwph7biubnnOyj6NNhi3kvJ++wo49MbdTj2nrWuWR2G7+9GdvxcjnS66yNOLcvjnnhP0TY3n5YxE\ngp04yURmXMIfLN50lN8v207D2KrMHdaO+Gru7TjUbcallPoaqFXOl8Zrmrbc3oA0TZsFzILiQWbv\nz+nFXcXazC15vLRiJz8XFp/qHhViZUKPVkDlMxlbScOVdfiK2vDL+15/K2JHhliZPTSNt/+5j3f/\nuY+9Pxbv95LahxA36teuLnHVQnjsg2wypq5jzv1pJNeNMjqsypszNE27Q9O0xHL+2J20zMAdxdrM\nLXmM/WTr1aQFxRuMn1yUw5OLciptxy2vuK0oTnLONlCUd01rgMJquXZJrLSI7o9F7ICA4hOzZw5J\n5eDJc/SYsoaNB/ONDksIU0pvHMPSEZ0JtgbQf9Z6vtzxX6ND8p92eHd0QE1atfeahgd7lJ3JZKTE\nXXNuoYKry3zO3r6k7DUVxUt+k/omM6lP8jWPlRapzdwZ5m53tqpF5qjORIRYGfS3jby/7gejQxLC\nlJrULO44bF4rghELs/nbfw4a2nHoUnOGUqoXMBmIBVYqpXI0TbtTl8h05o5irTPLadfPZEqL2+XV\n4Jxt1LBVMK/oMX8tYjeuEU7mqHSeXrSVwguyz0sIW2LDq/Dx8I48vTiHV1bu5tBP53ipRysCDTjj\n0NWuwmXAMp1icTu9O6AcqSdBxTMZI2tNZu0M85SIYCuzhqRihgZDpVRfYALQAmhv7xYTITwh2Gph\nysC2/Kn6Xmb8+wC5Pxcx5b4Uwj3cceg3S4XuMPbOZjfUjmwpuzxXHn+sNZlJQIAyS2v8DqA38K3R\ngQhRnoCSe+G93juJNft/ou+M9R5v5pLE5YKMlDgm9Um2ebQTFM+y3u7fptIzCG3Vmm5rHut3J174\nM03TdmuaJgfqCdMb2L4e8x5oR97PRWRMXcv23NMee245Hd5F1y+zOXuSQHm1ptuax/Jpdp7Pbw4W\nzrlub6TB0Qh/dFOTWD4Z0ZkH522i38z1vDswhd+0rOn255UbSZqYEZumhX1c2YBsz95Iezb1lyVj\nShjpxNnzPPJ+FtvyTvOHu1vyYHqCU0vvciNJH+Bvm4P9haZpdxgdgxB6qhEezMfDO/Hkoi28/Pku\nDuef44V7Wrqt41BqXCYmDRtCCG8REmRh+qBUht/ckPnrD/PI/Cx++fWSW55LEpeJ+fPmYH+llOql\nlMoFOlG8N3KV0TEJYa+AAMXv72rBKxmJfLvvJ0Z8kO2W55GlQhPz983B/sjb9kYKUZ7BHetTt3oo\nEcHuSTGSuEzO3zcHC99n1nt6Cdfc0jTWbdeWxCWEMIyr98kT/klqXEIIw1R0M1UhbJEZl85k2UMI\n+8mWD+EMSVw6kmUPIRzjjpup6kk+iJqTLBXqSJY9hHCMmbd8lH4QzSsoQsP5e+QJ/cmMS0ey7CGE\nYzy55cPR2VNFH0Rl1mUsSVw6MvuyhxBm5IktH84s48sHUfOSpUIdmXnZQwh/5swyvhy5Zl6SuHSU\nkRLH672TiIsKQVH5zSOFEJ7hzOzJ3R9EM7fkyb32nCRLhTpz17KHdDcJ4TxnlvHdWX+TDmTXSOLy\nAvIiF8I1Y+9sds0YAvtmT+76ICqNH66RpUIvIG32QrjGbMv40vjhGpdmXEqpSUB34AJwAHhA07QC\nPQIT/yMvciFcZ6YDq6UD2TWuzrj+ASRqmtYa+B543vWQzM/TRVXpbhLCXFx9D5AOZNe4lLg0TftK\n07TSW1xuAOJdD8ncjNhNLy9yIcxDj/cAsy1dehs9mzMeBBbZ+qJSajgwHKBevXo6Pq1nGVFUlRtK\nCmEeer0HmGnp0ttUmriUUl8Dtcr50nhN05aXfM944BKw0NZ1NE2bBcwCSEtL05yK1gSMqjfJi1wI\nc5Cas/EqTVyapt1R0deVUsOAe4DbNU3z2oRkLymqCuEab9+TKO8BxnOpxqWU6gY8B/TQNK1Qn5DM\nK3NLHud+vXTD41JvEsI+vnDiuidrznK6Rvlc7SqcAoQD/1BK5SilZugQkymVDriCoovXPF4t1CpF\nVSHs5At7Ej3VWOELSd5dXGrO0DStsV6BmF15Aw4gNChQkpYQdvKV+pAnas5yuoZtcnKGnXxlwAlh\nJNmTaD95z7FNEpedZMAJT1BKTVJK7VFKbVNKLVNKRRkdk55kT6L95D3HNklcdpIBJzzEp0+jkY23\n9pP3HNvkdHg7ySZg4Qmapn1V5q8bgD5GxeIusifRPvKeY5skLgfIgBMe5hen0Qjb5D2nfJK4hPAw\nOY1GCNdI4hLCw+Q0GiFcI4lLCBMpcxrNLf5wGo0QzpCuQiHMxW9OoxHCWTLjEsJE/Ok0GiGcpYxY\nQldKnQQOu+nyMcBPbrq2MyQe28wUCzgWT31N02LdGYwjZEwZykzxmCkWcDweu8aVIYnLnZRSWZqm\npRkdRymJxzYzxQLmi8cszPZ7kXhsM1Ms4L54pMYlhBDCq0jiEkII4VV8MXHNMjqA60g8tpkpFjBf\nPGZhtt+LxGObmWIBN8XjczUuIYQQvs0XZ1xCCCF8mCQuIYQQXsUnE5fZbsanlOqrlNqplLqilDKk\nVVUp1U0ptVcptV8pNc6IGMrEMlcpdUIptcPIOEpiqauU+pdSalfJ/6MnjI7JjGRMlRuDjKnyY3H7\nmPLJxIX5bsa3A+gNfGvEkyulLMBU4HdAS2CgUqqlEbGUmAd0M/D5y7oEPKNpWkugIzDK4N+NWcmY\nKkPGVIXcPqZ8MnFpmvaVpmmXSv66AYg3OJ7dmqbtNTCE9sB+TdMOapp2AfgY6GlUMJqmfQucMur5\ny9I07bimaZtL/vsssBuQGyBdR8bUDWRM2eCJMeWTies6DwJfGB2EweKAo2X+nou8Od9AKZUApAAb\njY3E9GRMyZiyi7vGlNcesqvXzfg8GY8wL6VUGPAp8KSmaWeMjscIMqaEntw5prw2cZntZnyVxWOw\nPKBumb/HlzwmAKWUleIBtlDTtKVGx2MUGVMOkTFVAXePKZ9cKixzM74ecjM+ADYBTZRSDZRSQcAA\n4DODYzIFpZQC5gC7NU37i9HxmJWMqRvImLLBE2PKJxMXJrsZn1Kql1IqF+gErFRKrfLk85cU1UcD\nqygulC7WNG2nJ2MoSyn1EbAeaKaUylVKPWRULEA6MAToWvJayVFK3WVgPGYlY6oMGVMVcvuYkiOf\nhBBCeBVfnXEJIYTwUZK4hBBCeBVJXEIIIbyKJC4hhBBeRRKXEEIIryKJSwghhFeRxCWEEMKr/D/V\nKrdRWCwXbQAAAABJRU5ErkJggg==\n", 421 | "text/plain": [ 422 | "" 423 | ] 424 | }, 425 | "metadata": {}, 426 | "output_type": "display_data" 427 | } 428 | ], 429 | "source": [ 430 | "x_min = -2\n", 431 | "y_min = ( -(model_params['weights'][0] * x_min) / model_params['weights'][1]\n", 432 | " -(model_params['bias'] / model_params['weights'][1]) )\n", 433 | "\n", 434 | "x_max = 2\n", 435 | "y_max = ( -(model_params['weights'][0] * x_max) / model_params['weights'][1]\n", 436 | " -(model_params['bias'] / model_params['weights'][1]) )\n", 437 | "\n", 438 | "\n", 439 | "fig, ax = plt.subplots(1, 2, sharex=True, figsize=(7, 3))\n", 440 | "\n", 441 | "ax[0].plot([x_min, x_max], [y_min, y_max])\n", 442 | "ax[1].plot([x_min, x_max], [y_min, y_max])\n", 443 | "\n", 444 | "ax[0].scatter(X_train[y_train==0, 0], X_train[y_train==0, 1], label='class 0', marker='o')\n", 445 | "ax[0].scatter(X_train[y_train==1, 0], X_train[y_train==1, 1], label='class 1', marker='s')\n", 446 | "\n", 447 | "ax[1].scatter(X_test[y_test==0, 0], X_test[y_test==0, 1], label='class 0', marker='o')\n", 448 | "ax[1].scatter(X_test[y_test==1, 0], X_test[y_test==1, 1], label='class 1', marker='s')\n", 449 | "\n", 450 | "ax[1].legend(loc='upper left')\n", 451 | "plt.show()" 452 | ] 453 | }, 454 | { 455 | "cell_type": "markdown", 456 | "metadata": {}, 457 | "source": [ 458 | "- Suggested exercises\n", 459 | "\n", 460 | "\n", 461 | "1. Train a zero-weight perceptron with different learning rates and compare the model parameters and decision boundaries to each other. What do you observe?\n", 462 | "\n", 463 | "2. Repeat the previous exercise with randomly initialized weights." 464 | ] 465 | }, 466 | { 467 | "cell_type": "code", 468 | "execution_count": 13, 469 | "metadata": { 470 | "collapsed": true 471 | }, 472 | "outputs": [], 473 | "source": [ 474 | "# %load solutions/01_weight_zero_learning_rate.py" 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "execution_count": 14, 480 | "metadata": { 481 | "collapsed": true 482 | }, 483 | "outputs": [], 484 | "source": [ 485 | "# %load solutions/02_random_weights_learning_rate.py" 486 | ] 487 | }, 488 | { 489 | "cell_type": "markdown", 490 | "metadata": {}, 491 | "source": [ 492 | "## Implementing a Perceptron in TensorFlow" 493 | ] 494 | }, 495 | { 496 | "cell_type": "markdown", 497 | "metadata": {}, 498 | "source": [ 499 | "- Setting up the perceptron graph" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": 16, 505 | "metadata": {}, 506 | "outputs": [], 507 | "source": [ 508 | "g = tf.Graph()\n", 509 | "\n", 510 | "\n", 511 | "n_features = X_train.shape[1]\n", 512 | "\n", 513 | "with g.as_default() as g:\n", 514 | " \n", 515 | " # initialize model parameters\n", 516 | " features = tf.placeholder(dtype=tf.float32, \n", 517 | " shape=[None, n_features], name='features')\n", 518 | " targets = tf.placeholder(dtype=tf.float32, \n", 519 | " shape=[None, 1], name='targets')\n", 520 | " params = {\n", 521 | " 'weights': tf.Variable(tf.zeros(shape=[n_features, 1], \n", 522 | " dtype=tf.float32), name='weights'),\n", 523 | " 'bias': tf.Variable([[0.]], dtype=tf.float32, name='bias')}\n", 524 | " \n", 525 | " # forward pass\n", 526 | " linear = tf.matmul(features, params['weights']) + params['bias']\n", 527 | " ones = tf.ones(shape=tf.shape(linear)) \n", 528 | " zeros = tf.zeros(shape=tf.shape(linear))\n", 529 | " prediction = tf.where(tf.less(linear, 0.), zeros, ones, name='prediction')\n", 530 | " \n", 531 | " # weight update\n", 532 | " diff = targets - prediction\n", 533 | " weight_update = tf.assign_add(params['weights'], \n", 534 | " tf.reshape(diff * features, (n_features, 1)))\n", 535 | " bias_update = tf.assign_add(params['bias'], diff)\n", 536 | " \n", 537 | " saver = tf.train.Saver()" 538 | ] 539 | }, 540 | { 541 | "cell_type": "markdown", 542 | "metadata": {}, 543 | "source": [ 544 | "- Training the perceptron for 5 training samples for illustration purposes" 545 | ] 546 | }, 547 | { 548 | "cell_type": "code", 549 | "execution_count": 17, 550 | "metadata": {}, 551 | "outputs": [ 552 | { 553 | "name": "stdout", 554 | "output_type": "stream", 555 | "text": [ 556 | "Model parameters:\n", 557 | " {'weights': array([[ 1.47257054],\n", 558 | " [ 0.30436274]], dtype=float32), 'bias': array([[-1.]], dtype=float32)}\n", 559 | "Number of training errors: 3\n" 560 | ] 561 | } 562 | ], 563 | "source": [ 564 | "with tf.Session(graph=g) as sess:\n", 565 | " \n", 566 | " sess.run(tf.global_variables_initializer())\n", 567 | " \n", 568 | " i = 0\n", 569 | " for example, target in zip(X_train, y_train):\n", 570 | " feed_dict = {features: example.reshape(-1, n_features),\n", 571 | " targets: target.reshape(-1, 1)}\n", 572 | " _, _ = sess.run([weight_update, bias_update], feed_dict=feed_dict)\n", 573 | " \n", 574 | " i += 1\n", 575 | " if i >= 4:\n", 576 | " break\n", 577 | " \n", 578 | "\n", 579 | " modelparams = sess.run(params) \n", 580 | " print('Model parameters:\\n', modelparams)\n", 581 | "\n", 582 | " saver.save(sess, save_path='perceptron')\n", 583 | " \n", 584 | " pred = sess.run(prediction, feed_dict={features: X_train})\n", 585 | " errors = np.sum(pred.reshape(-1) != y_train)\n", 586 | " print('Number of training errors:', errors)" 587 | ] 588 | }, 589 | { 590 | "cell_type": "markdown", 591 | "metadata": {}, 592 | "source": [ 593 | "- Continue training of the graph after restoring the session from a local checkpoint (this can be useful if we have to interrupt out computational session)\n", 594 | "- Now train a complete epoch" 595 | ] 596 | }, 597 | { 598 | "cell_type": "code", 599 | "execution_count": 18, 600 | "metadata": {}, 601 | "outputs": [ 602 | { 603 | "name": "stdout", 604 | "output_type": "stream", 605 | "text": [ 606 | "Number of training errors 0\n", 607 | "Number of test errors 0\n" 608 | ] 609 | } 610 | ], 611 | "source": [ 612 | "with tf.Session(graph=g) as sess:\n", 613 | " saver.restore(sess, os.path.abspath('perceptron'))\n", 614 | "\n", 615 | " for epoch in range(1):\n", 616 | " for example, target in zip(X_train, y_train):\n", 617 | " feed_dict = {features: example.reshape(-1, n_features),\n", 618 | " targets: target.reshape(-1, 1)}\n", 619 | " _, _ = sess.run([weight_update, bias_update], feed_dict=feed_dict)\n", 620 | " modelparams = sess.run(params)\n", 621 | "\n", 622 | " saver.save(sess, save_path='perceptron')\n", 623 | " \n", 624 | " pred = sess.run(prediction, feed_dict={features: X_train})\n", 625 | " train_errors = np.sum(pred.reshape(-1) != y_train)\n", 626 | " pred = sess.run(prediction, feed_dict={features: X_train})\n", 627 | " test_errors = np.sum(pred.reshape(-1) != y_train)\n", 628 | "\n", 629 | " print('Number of training errors', train_errors)\n", 630 | " print('Number of test errors', test_errors)" 631 | ] 632 | }, 633 | { 634 | "cell_type": "markdown", 635 | "metadata": {}, 636 | "source": [ 637 | "- Suggested Exercises\n", 638 | "\n", 639 | "\n", 640 | "3) Plot the decision boundary for this TensorFlow perceptron. Why do you think the TensorFlow implementation performs better than our NumPy implementation on the test set?\n", 641 | " - Hint 1: you can re-use the code that we used in the NumPy section\n", 642 | " - Hint 2: since the bias is a 2D array, you need to access the float value via `modelparams['bias'][0]`\n", 643 | " " 644 | ] 645 | }, 646 | { 647 | "cell_type": "code", 648 | "execution_count": 19, 649 | "metadata": {}, 650 | "outputs": [], 651 | "source": [ 652 | "# %load solutions/03_tensorflow-boundary.py" 653 | ] 654 | }, 655 | { 656 | "cell_type": "markdown", 657 | "metadata": {}, 658 | "source": [ 659 | "- Theoretically, we could restart the Jupyter notebook now (we would just have to prepare the dataset again then, though)\n", 660 | "- We are going to restore the session from a meta graph (notice \"`tf.Session()`\")\n", 661 | "- First, we have to load the datasets again" 662 | ] 663 | }, 664 | { 665 | "cell_type": "code", 666 | "execution_count": 20, 667 | "metadata": {}, 668 | "outputs": [ 669 | { 670 | "name": "stdout", 671 | "output_type": "stream", 672 | "text": [ 673 | "Number of training errors 0\n", 674 | "Number of test errors 0\n" 675 | ] 676 | } 677 | ], 678 | "source": [ 679 | "with tf.Session() as sess:\n", 680 | " \n", 681 | " saver = tf.train.import_meta_graph(os.path.abspath('perceptron.meta'))\n", 682 | " saver.restore(sess, os.path.abspath('perceptron'))\n", 683 | " \n", 684 | " pred = sess.run('prediction:0', feed_dict={'features:0': X_train})\n", 685 | " train_errors = np.sum(pred.reshape(-1) != y_train)\n", 686 | " pred = sess.run('prediction:0', feed_dict={'features:0': X_test})\n", 687 | " test_errors = np.sum(pred.reshape(-1) != y_test)\n", 688 | " \n", 689 | " print('Number of training errors', train_errors)\n", 690 | " print('Number of test errors', test_errors)" 691 | ] 692 | } 693 | ], 694 | "metadata": { 695 | "anaconda-cloud": {}, 696 | "kernelspec": { 697 | "display_name": "Python 3", 698 | "language": "python", 699 | "name": "python3" 700 | }, 701 | "language_info": { 702 | "codemirror_mode": { 703 | "name": "ipython", 704 | "version": 3 705 | }, 706 | "file_extension": ".py", 707 | "mimetype": "text/x-python", 708 | "name": "python", 709 | "nbconvert_exporter": "python", 710 | "pygments_lexer": "ipython3", 711 | "version": "3.6.4" 712 | }, 713 | "toc": { 714 | "nav_menu": {}, 715 | "number_sections": true, 716 | "sideBar": true, 717 | "skip_h1_title": false, 718 | "title_cell": "Table of Contents", 719 | "title_sidebar": "Contents", 720 | "toc_cell": false, 721 | "toc_position": {}, 722 | "toc_section_display": true, 723 | "toc_window_display": false 724 | } 725 | }, 726 | "nbformat": 4, 727 | "nbformat_minor": 2 728 | } 729 | -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/perceptron.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/ch02_perceptron/perceptron.data-00000-of-00001 -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/perceptron.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/ch02_perceptron/perceptron.index -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/perceptron.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/_old-material/ch02_perceptron/perceptron.meta -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/perceptron_toydata.txt: -------------------------------------------------------------------------------- 1 | 0.77 -1.14 0 2 | -0.33 1.44 0 3 | 0.91 -3.07 0 4 | -0.37 -1.91 0 5 | -1.84 -1.13 0 6 | -1.50 0.34 0 7 | -0.63 -1.53 0 8 | -1.08 -1.23 0 9 | 0.39 -1.99 0 10 | -1.26 -2.90 0 11 | -5.27 -0.78 0 12 | -0.49 -2.74 0 13 | 1.48 -3.74 0 14 | -1.64 -1.96 0 15 | 0.45 0.36 0 16 | -1.48 -1.17 0 17 | -2.94 -4.47 0 18 | -2.19 -1.48 0 19 | 0.02 -0.02 0 20 | -2.24 -2.12 0 21 | -3.17 -3.69 0 22 | -4.09 1.03 0 23 | -2.41 -2.31 0 24 | -3.45 -0.61 0 25 | -3.96 -2.00 0 26 | -2.95 -1.16 0 27 | -2.42 -3.35 0 28 | -1.74 -1.10 0 29 | -1.61 -1.28 0 30 | -2.59 -2.21 0 31 | -2.64 -2.20 0 32 | -2.84 -4.12 0 33 | -1.45 -2.26 0 34 | -3.98 -1.05 0 35 | -2.97 -1.63 0 36 | -0.68 -1.52 0 37 | -0.10 -3.43 0 38 | -1.14 -2.66 0 39 | -2.92 -2.51 0 40 | -2.14 -1.62 0 41 | -3.33 -0.44 0 42 | -1.05 -3.85 0 43 | 0.38 0.95 0 44 | -0.05 -1.95 0 45 | -3.20 -0.22 0 46 | -2.26 0.01 0 47 | -1.41 -0.33 0 48 | -1.20 -0.71 0 49 | -1.69 0.80 0 50 | -1.52 -1.14 0 51 | 3.88 0.65 1 52 | 0.73 2.97 1 53 | 0.83 3.94 1 54 | 1.59 1.25 1 55 | 3.92 3.48 1 56 | 3.87 2.91 1 57 | 1.14 3.91 1 58 | 1.73 2.80 1 59 | 2.95 1.84 1 60 | 2.61 2.92 1 61 | 2.38 0.90 1 62 | 2.30 3.33 1 63 | 1.31 1.85 1 64 | 1.56 3.85 1 65 | 2.67 2.41 1 66 | 1.23 2.54 1 67 | 1.33 2.03 1 68 | 1.36 2.68 1 69 | 2.58 1.79 1 70 | 2.40 0.91 1 71 | 0.51 2.44 1 72 | 2.17 2.64 1 73 | 4.38 2.94 1 74 | 1.09 3.12 1 75 | 0.68 1.54 1 76 | 1.93 3.71 1 77 | 1.26 1.17 1 78 | 1.90 1.34 1 79 | 3.13 0.92 1 80 | 0.85 1.56 1 81 | 1.50 3.93 1 82 | 2.95 2.09 1 83 | 0.77 2.84 1 84 | 1.00 0.46 1 85 | 3.19 2.32 1 86 | 2.92 2.32 1 87 | 2.86 1.35 1 88 | 0.97 2.68 1 89 | 1.20 1.31 1 90 | 1.54 2.02 1 91 | 1.65 0.63 1 92 | 1.36 -0.22 1 93 | 2.63 0.40 1 94 | 0.90 2.05 1 95 | 1.26 3.54 1 96 | 0.71 2.27 1 97 | 1.96 0.83 1 98 | 2.52 1.83 1 99 | 2.77 2.82 1 100 | 4.16 3.34 1 101 | -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/solutions/01_weight_zero_learning_rate.py: -------------------------------------------------------------------------------- 1 | small_lr_params = perceptron_train(X_train, y_train, learning_rate=0.01, 2 | mparams=None, zero_weights=True) 3 | 4 | for _ in range(2): 5 | _ = perceptron_train(X_train, y_train, mparams=small_lr_params) 6 | 7 | x_min_small = -2 8 | y_min_small = (-(small_lr_params['weights'][0] * x_min) / 9 | small_lr_params['weights'][1] - 10 | (small_lr_params['bias'] / small_lr_params['weights'][1])) 11 | 12 | x_max_small = 2 13 | y_max_small = (-(small_lr_params['weights'][0] * x_max) / 14 | small_lr_params['weights'][1] - 15 | (small_lr_params['bias'] / small_lr_params['weights'][1])) 16 | 17 | 18 | fig, ax = plt.subplots(1, 2, sharex=True, figsize=(7, 3)) 19 | 20 | ax[0].plot([x_min, x_max], [y_min, y_max]) 21 | ax[1].plot([x_min_small, x_max_small], [y_min_small, y_max_small]) 22 | 23 | ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], 24 | label='class 0', marker='o') 25 | ax[0].scatter(X_train[y_trai == 1, 0], X_train[y_train == 1, 1], 26 | label='class 1', marker='s') 27 | 28 | ax[1].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], 29 | label='class 0', marker='o') 30 | ax[1].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], 31 | label='class 1', marker='s') 32 | 33 | ax[1].legend(loc='lower right') 34 | 35 | plt.ylim([-3, 3]) 36 | plt.xlim([-3, 3]) 37 | plt.show() 38 | 39 | 40 | print('Learning=1. rate params:', model_params) 41 | print('Learning=0.01 rate params:', small_lr_params) 42 | 43 | # As we can see, lowering the learning rate changes the model parameters. 44 | # But if we look closely, we can see that in the case of the perceptron 45 | # the learning rate is just a scaling factor of the weight & bias vector 46 | # if we initialize the weights to zero. Therefore, the decision region 47 | # is exactly the same for different learning rates. 48 | -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/solutions/02_random_weights_learning_rate.py: -------------------------------------------------------------------------------- 1 | randw_params_1 = perceptron_train(X_train, y_train, learning_rate=1.0, 2 | mparams=None, zero_weights=False) 3 | 4 | for _ in range(2): 5 | _ = perceptron_train(X_train, y_train, mparams=randw_params_1) 6 | 7 | x_min_1 = -2 8 | y_min_1 = (-(randw_params_1['weights'][0] * x_min) / 9 | randw_params_1['weights'][1] - 10 | (randw_params_1['bias'] / randw_params_1['weights'][1])) 11 | 12 | x_max_1 = 2 13 | y_max_1 = (-(randw_params_1['weights'][0] * x_max) / 14 | randw_params_1['weights'][1] - 15 | (randw_params_1['bias'] / randw_params_1['weights'][1])) 16 | 17 | 18 | randw_params_2 = perceptron_train(X_train, y_train, learning_rate=0.01, 19 | mparams=None, zero_weights=False) 20 | 21 | for _ in range(2): 22 | _ = perceptron_train(X_train, y_train, mparams=randw_params_2) 23 | 24 | x_min_2 = -2 25 | y_min_2 = (-(randw_params_2['weights'][0] * x_min) / 26 | randw_params_2['weights'][1] - 27 | (randw_params_2['bias'] / randw_params_2['weights'][1])) 28 | 29 | x_max_2 = 2 30 | y_max_2 = (-(randw_params_2['weights'][0] * x_max) / 31 | randw_params_2['weights'][1] - 32 | (randw_params_2['bias'] / randw_params_2['weights'][1])) 33 | 34 | 35 | fig, ax = plt.subplots(1, 2, sharex=True, figsize=(7, 3)) 36 | 37 | ax[0].plot([x_min_1, x_max_1], [y_min_1, y_max_1]) 38 | ax[1].plot([x_min_2, x_max_2], [y_min_2, y_max_2]) 39 | 40 | ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], 41 | label='class 0', marker='o') 42 | ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], 43 | label='class 1', marker='s') 44 | 45 | ax[1].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], 46 | label='class 0', marker='o') 47 | ax[1].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], 48 | label='class 1', marker='s') 49 | 50 | ax[1].legend(loc='lower right') 51 | 52 | plt.ylim([-3, 3]) 53 | plt.xlim([-3, 3]) 54 | plt.show() 55 | 56 | # As we can see now, random weight initialization breaks 57 | # the symmetry in the weight updates if we use 58 | # randomly initialized weights 59 | -------------------------------------------------------------------------------- /code/_old-material/ch02_perceptron/solutions/03_tensorflow-boundary.py: -------------------------------------------------------------------------------- 1 | x_min = -2 2 | y_min = (-(modelparams['weights'][0] * x_min) / modelparams['weights'][1] - 3 | (modelparams['bias'][0] / model_params['weights'][1])) 4 | 5 | x_max = 2 6 | y_max = (-(modelparams['weights'][0] * x_max) / modelparams['weights'][1] - 7 | (modelparams['bias'][0] / modelparams['weights'][1])) 8 | 9 | 10 | fig, ax = plt.subplots(1, 2, sharex=True, figsize=(7, 3)) 11 | 12 | ax[0].plot([x_min, x_max], [y_min, y_max]) 13 | ax[1].plot([x_min, x_max], [y_min, y_max]) 14 | 15 | ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], 16 | label='class 0', marker='o') 17 | ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], 18 | label='class 1', marker='s') 19 | 20 | ax[1].scatter(X_test[y_test == 0, 0], X_test[y_test == 0, 1], 21 | label='class 0', marker='o') 22 | ax[1].scatter(X_test[y_test == 1, 0], X_test[y_test == 1, 1], 23 | label='class 1', marker='s') 24 | 25 | ax[1].legend(loc='upper left') 26 | plt.show() 27 | 28 | 29 | # The TensorFlow model performs better on the test set just by random chance. 30 | # Remember, the perceptron algorithm stops learning as soon as it classifies 31 | # the training set perfectly. 32 | # Possible explanations why there is a difference between the NumPy and 33 | # TensorFlow outcomes could thus be numerical precision, or slight differences 34 | # in our implementation. 35 | -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/array_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/array_1.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/array_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/array_2.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/broadcasting-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/broadcasting-1.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/broadcasting-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/broadcasting-2.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/matmatmul.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/matmatmul.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/matmul.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/matmul.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/random_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/random_1.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/random_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/random_2.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/set_theory.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/set_theory.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/transpose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/transpose.png -------------------------------------------------------------------------------- /code/appendix_h_numpy-intro/images/numpy-intro/ufunc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/code/appendix_h_numpy-intro/images/numpy-intro/ufunc.png -------------------------------------------------------------------------------- /code/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Sebastian Raschka 2016-2017 2 | # 3 | # ann is a supporting package for the book 4 | # "Introduction to Artificial Neural Networks and Deep Learning: 5 | # A Practical Guide with Applications in Python" 6 | # 7 | # Author: Sebastian Raschka 8 | # 9 | # License: MIT 10 | -------------------------------------------------------------------------------- /code/tests/test_notebooks.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import subprocess 4 | import tempfile 5 | import watermark 6 | import nbformat 7 | 8 | 9 | def run_ipynb(path): 10 | with tempfile.NamedTemporaryFile(suffix=".ipynb") as fout: 11 | args = ["python", "-m", "nbconvert", "--to", 12 | "notebook", "--execute", "--output", 13 | fout.name, path] 14 | subprocess.check_output(args) 15 | 16 | 17 | def run_ipynb2(path): 18 | args = ["python", "-m", "nbconvert", "--to", 19 | "notebook", "--execute", path] 20 | subprocess.check_output(args) 21 | 22 | 23 | class TestNotebooks(unittest.TestCase): 24 | 25 | def test_02_perceptron(self): 26 | this_dir = os.path.dirname(os.path.abspath(__file__)) 27 | run_ipynb(os.path.join(this_dir, 28 | '../ch02_perceptron/' 29 | 'ch02_perceptron.ipynb')) 30 | 31 | def test_appendix_f_numpy_intro(self): 32 | this_dir = os.path.dirname(os.path.abspath(__file__)) 33 | run_ipynb(os.path.join(this_dir, 34 | '../appendix_f_numpy-intro/' 35 | 'appendix_f_numpy-intro.ipynb')) 36 | 37 | def test_appendix_g_tensorflow_basics(self): 38 | this_dir = os.path.dirname(os.path.abspath(__file__)) 39 | run_ipynb(os.path.join(this_dir, 40 | '../appendix_g_tensorflow-basics/' 41 | 'appendix_g_tensorflow-basics.ipynb')) 42 | 43 | 44 | if __name__ == '__main__': 45 | unittest.main() 46 | -------------------------------------------------------------------------------- /images/ann-and-deeplearning-cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/deep-learning-book/d1c240cc2828db8f1539e308f04458bcf4f5c379/images/ann-and-deeplearning-cover.jpg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import ann 2 | from setuptools import setup, find_packages 3 | 4 | VERSION = ann.__version__ 5 | 6 | setup( 7 | name="ann", 8 | version=VERSION, 9 | packages=find_packages(), 10 | install_requires=['numpy>=1.10.4'], 11 | author="Sebastian Raschka", 12 | author_email="mail@sebastianraschka.com", 13 | description=("Supporting package for the book " 14 | "'Introduction to Artificial Neural Networks " 15 | "and Deep Learning: " 16 | "A Practical Guide with Applications in Python'"), 17 | license="MIT", 18 | keywords=["artificial neural networks", "deep learning", 19 | "machine learning", "artificial intelligence", "data science"], 20 | classifiers=[ 21 | 'License :: OSI Approved :: MIT License', 22 | 'Operating System :: Microsoft :: Windows', 23 | 'Operating System :: POSIX', 24 | 'Operating System :: Unix', 25 | 'Operating System :: MacOS', 26 | 'Programming Language :: Python :: 2.7', 27 | 'Programming Language :: Python :: 3.5', 28 | 'Programming Language :: Python :: 3.6', 29 | 'Topic :: Scientific/Engineering', 30 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 31 | 'Topic :: Scientific/Engineering :: Information Analysis', 32 | 'Topic :: Scientific/Engineering :: Image Recognition', 33 | ], 34 | url="https://github.com/rasbt/deep-learning-book") 35 | --------------------------------------------------------------------------------