├── .gitignore ├── LICENSE ├── README.md ├── environment.yml ├── notebooks ├── 00.00-Preface.ipynb ├── 00.01-Foreword-by-Ariel-Rokem.ipynb ├── 01.00-A-Taste-of-Machine-Learning.ipynb ├── 02.00-Working-with-Data-in-OpenCV.ipynb ├── 02.01-Dealing-with-Data-Using-Python-NumPy.ipynb ├── 02.02-Loading-External-Datasets-in-Python.ipynb ├── 02.03-Visualizing-Data-Using-Matplotlib.ipynb ├── 02.04-Visualizing-Data-from-an-External-Dataset.ipynb ├── 02.05-Dealing-with-Data-Using-the-OpenCV-TrainData-Container-in-C++.ipynb ├── 03.00-First-Steps-in-Supervised-Learning.ipynb ├── 03.01-Measuring-Model-Performance-with-Scoring-Functions.ipynb ├── 03.02-Understanding-the-k-NN-Algorithm.ipynb ├── 03.03-Using-Regression-Models-to-Predict-Continuous-Outcomes.ipynb ├── 03.04-Applying-Lasso-and-Ridge-Regression.ipynb ├── 03.05-Classifying-Iris-Species-Using-Logistic-Regression.ipynb ├── 04.00-Representing-Data-and-Engineering-Features.ipynb ├── 04.01-Preprocessing-Data.ipynb ├── 04.02-Reducing-the-Dimensionality-of-the-Data.ipynb ├── 04.03-Representing-Categorical-Variables.ipynb ├── 04.04-Represening-Text-Features.ipynb ├── 04.05-Representing-Images.ipynb ├── 05.00-Using-Decision-Trees-to-Make-a-Medical-Diagnosis.ipynb ├── 05.01-Building-Our-First-Decision-Tree.ipynb ├── 05.02-Using-Decision-Trees-to-Diagnose-Breast-Cancer.ipynb ├── 05.03-Using-Decision-Trees-for-Regression.ipynb ├── 06.00-Detecting-Pedestrians-with-Support-Vector-Machines.ipynb ├── 06.01-Implementing-Your-First-Support-Vector-Machine.ipynb ├── 06.02-Detecting-Pedestrians-in-the-Wild.ipynb ├── 06.03-Additional-SVM-Exercises.ipynb ├── 07.00-Implementing-a-Spam-Filter-with-Bayesian-Learning.ipynb ├── 07.01-Implementing-Our-First-Bayesian-Classifier.ipynb ├── 07.02-Classifying-Emails-Using-Naive-Bayes.ipynb ├── 08.00-Discovering-Hidden-Structures-with-Unsupervised-Learning.ipynb ├── 08.01-Understanding-k-Means-Clustering.ipynb ├── 08.02-Compressing-Color-Images-Using-k-Means.ipynb ├── 08.03-Classifying-Handwritten-Digits-Using-k-Means.ipynb ├── 08.04-Implementing-Agglomerative-Hierarchical-Clustering.ipynb ├── 09.00-Using-Deep-Learning-to-Classify-Handwritten-Digits.ipynb ├── 09.01-Understanding-perceptrons.ipynb ├── 09.02-Implementing-a-Multi-Layer-Perceptron-in-OpenCV.ipynb ├── 09.03-Getting-Acquainted-with-Deep-Learning.ipynb ├── 09.04-Training-an-MLP-in-OpenCV-to-Classify-Handwritten-Digits.ipynb ├── 09.05-Training-a-Deep-Neural-Net-to-Classify-Handwritten-Digits-Using-Keras.ipynb ├── 10.00-Combining-Different-Algorithms-Into-an-Ensemble.ipynb ├── 10.01-Understanding-Ensemble-Methods.ipynb ├── 10.02-Combining-Decision-Trees-Into-a-Random-Forest.ipynb ├── 10.03-Using-Random-Forests-for-Face-Recognition.ipynb ├── 10.04-Implementing-AdaBoost.ipynb ├── 10.05-Combining-Different-Models-Into-a-Voting-Classifier.ipynb ├── 11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb ├── 11.01-Evaluating-a-Model.ipynb ├── 11.02-Understanding-Cross-Validation-Bootstrapping-and-McNemar's-Test.ipynb ├── 11.03-Tuning-Hyperparameters-with-Grid-Search.ipynb ├── 11.04-Chaining-Algorithms-Together-to-Form-a-Pipeline.ipynb ├── 12.00-Wrapping-Up.ipynb ├── data │ ├── chapter6 │ │ ├── pedestrian_test.jpg │ │ ├── pedestrians128x64.tar.gz │ │ └── pedestrians_neg.tar.gz │ ├── chapter7 │ │ ├── BG.tar.gz │ │ ├── GP.tar.gz │ │ ├── SH.tar.gz │ │ ├── beck-s.tar.gz │ │ ├── farmer-d.tar.gz │ │ ├── kaminski-v.tar.gz │ │ ├── kitchen-l.tar.gz │ │ ├── lokay-m.tar.gz │ │ └── williams-w3.tar.gz │ ├── cover.jpg │ ├── haarcascade_frontalface_default.xml │ └── lena.jpg └── figures │ ├── 02.03-sine.png │ ├── 02.04-digit0.png │ └── 02.04-digits0-9.png ├── requirements.txt └── tools ├── README.md ├── add_book_info.py ├── add_navigation.py ├── fix_kernelspec.py └── generate_contents.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | *.png 91 | *.dot 92 | *.xml 93 | *.ppm 94 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Michael Beyeler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: python3 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python==3.6 6 | - numpy==1.12 7 | - scipy==0.19.1 8 | - scikit-learn==0.18.1 9 | - matplotlib 10 | - opencv==3.1 11 | - jupyter==1.0 12 | - notebook==5.4.1 13 | - pandas==0.22 14 | - theano 15 | - keras==2.1.5 16 | - mkl-service==1.1.2 17 | -------------------------------------------------------------------------------- /notebooks/00.00-Preface.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "\n", 11 | "\n", 12 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 13 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 14 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 15 | "\n", 16 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 17 | "If you find this content useful, please consider supporting the work by\n", 18 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "deletable": true, 25 | "editable": true 26 | }, 27 | "source": [ 28 | "\n", 29 | "| [Contents](../README.md) | [Foreword](00.01-Foreword-by-Ariel-Rokem.ipynb) >" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": { 35 | "collapsed": true, 36 | "deletable": true, 37 | "editable": true 38 | }, 39 | "source": [ 40 | "# Machine Learning for OpenCV\n", 41 | "## A practical introduction to the world of machine learning using OpenCV and Python" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": { 47 | "deletable": true, 48 | "editable": true 49 | }, 50 | "source": [ 51 | "I’m glad you’re here. It’s about time we talked about machine learning.\n", 52 | "\n", 53 | "**Machine learning** is no longer just a buzzword, it is all around us: from protecting your email, to automatically tagging friends in pictures, to predicting what movies you like. As a subfield of **data science**, machine learning enables computers to learn through experience: to make predictions about the future using collected data from the past.\n", 54 | "\n", 55 | "And the amount of data to be analyzed is enormous! Current estimates put the daily amount of produced data at 2.5 exabytes (or roughly 1 billion gigabytes). Can you believe it? This would be enough data to fill up 10 million blu-ray discs, or amount to 90 years of HD video. In order to deal with these vast amounts of data, companies such as Google, Amazon, Microsoft, and Facebook have been heavily investing in the development of data science platforms that allow us to benefit from machine learning wherever we go – scaling from your mobile phone application all the way to supercomputers connected through the cloud.\n", 56 | "\n", 57 | "> **In other words: This is the time to invest in machine learning. And if it is your wish to become a machine learning practitioner, too – then this book is for you!**" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": { 63 | "deletable": true, 64 | "editable": true 65 | }, 66 | "source": [ 67 | "## What this book covers\n", 68 | "\n", 69 | "The first step of this book is to introduce you to the essential concepts of statistical learning, such as [classification and regression](03.00-First-Steps-in-Supervised-Learning.ipynb), with the help of simple and intuitive examples. If you have already studied machine learning theory in detail, this book will show you how to put your knowledge into practice. Oh, and don't worry if you are completely new to the field of machine learning – all you need is the willingness to learn.\n", 70 | "\n", 71 | "Once we covered all the basic concepts, we will start exploring various algorithms such as [decision trees](05.00-Using-Decision-Trees-to-Make-a-Medical-Diagnosis.ipynb), [support vector machines](06.00-Detecting-Pedestrians-with-Support-Vector-Machines.ipynb), and [Bayesian networks](07.00-Implementing-a-Spam-Filter-with-Bayesian-Learning.ipynb), and learn how to combine them with other OpenCV functionality. Along the way, you will learn how to [‘understand the task by understanding the data’](04.00-Representing-Data-and-Engineering-Features.ipynb) and how to build fully functioning [machine learning pipelines](11.03-Chaining-Algorithms-Together-to-Form-a-Pipeline.ipynb).\n", 72 | "\n", 73 | "As the book progresses, so will your machine learning skills, until you are ready to take on today's hottest topic in the field: [deep learning](09.00-Using-Deep-Learning-to-Classify-Handwritten-Digits.ipynb). Combined with the trained skill of knowing [how to select the right tool for the task](11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb), we will make sure you get comfortable with all relevant machine learning fundamentals.\n", 74 | "\n", 75 | "At the end of the book, you will be ready to take on your own machine learning problems, either by building on the existing source code or developing your own algorithm from scratch!" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": { 81 | "deletable": true, 82 | "editable": true 83 | }, 84 | "source": [ 85 | "## What you need for this book\n", 86 | "\n", 87 | "You will need a computer, Python Anaconda, and enthusiasm. Lots of enthusiasm.\n", 88 | "\n", 89 | "*Why Python?,* you may ask. The answer is simple: It has become the de facto language of data science, thanks to its great number of open-source libraries and tools to process and interact with data.\n", 90 | "\n", 91 | "One of these tools is the **Python Anaconda** distribution, which provides all the scientific computing libraries we could possibly ask for, such as **NumPy**, **SciPy**, **Matplotlib**, **scikit-learn**, and **Pandas**. In addition, installing **OpenCV** is essentially a one-liner. No more flipping switches in ccmake or compiling from scratch! We will talk about how to install Python Anaconda in [Chapter 1, A Taste of Machine Learning](01.00-A-Taste-of-Machine-Learning.ipynb).\n", 92 | "\n", 93 | "If you have mostly been using OpenCV in combination with C++, that’s fine. But, at least for the purpose of this book, I would strongly suggest that you switch to Python. C++ is fine when your task is to develop high-performance code or real-time applications. But, when it comes to picking up a new skill, I believe Python to be the fundamentally better choice of language, because you can do more by typing less. Rather than getting annoyed by the syntactic subtleties of C++, or wasting hours trying to convert data from one format into another, Python will help you concentrate on the topic at hand: to become an expert in machine learning.\n", 94 | "\n", 95 | "If you get stuck or have questions about the source code, you are welcome to post in our\n", 96 | "[Discussion group](https://groups.google.com/d/forum/machine-learning-for-opencv).\n", 97 | "Chances are, someone else has already shared a solution to your specific problem." 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": { 103 | "deletable": true, 104 | "editable": true 105 | }, 106 | "source": [ 107 | "## About the author\n", 108 | "\n", 109 | "Michael Beyeler is a [Postdoctoral Fellow in Neuroengineering and Data Science at the\n", 110 | "University of Washington](http://faculty.washington.edu/mbeyeler), where he is working on computational models of bionic vision in\n", 111 | "order to improve the perceptual experience of blind patients implanted with a retinal\n", 112 | "prosthesis (\"bionic eye\"). His work lies at the intersection of neuroscience, computer\n", 113 | "engineering, computer vision, and machine learning. Michael is the author of [OpenCV with\n", 114 | "Python Blueprints (Packt Publishing Ltd., 2015)](https://github.com/mbeyeler/opencv-python-blueprints), \n", 115 | "a practical guide for building advanced\n", 116 | "computer vision projects. He is also an active contributor to several open-source software\n", 117 | "projects, and has professional programming experience in Python, C/C++, CUDA,\n", 118 | "MATLAB, and Android.\n", 119 | "\n", 120 | "Michael received a Ph.D. in Computer Science from the University of California, Irvine as\n", 121 | "well as a M.Sc. in Biomedical Engineering and a B.Sc. in Electrical Engineering from ETH\n", 122 | "Zurich, Switzerland. When he is not \"nerding out\" on brains, he can be found on top of a\n", 123 | "snowy mountain, in front of a live band, or behind the piano." 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": { 129 | "deletable": true, 130 | "editable": true 131 | }, 132 | "source": [ 133 | "\n", 134 | "| [Contents](../README.md) | [Foreword](00.01-Foreword-by-Ariel-Rokem.ipynb) >" 135 | ] 136 | } 137 | ], 138 | "metadata": { 139 | "kernelspec": { 140 | "display_name": "Python 3", 141 | "language": "python", 142 | "name": "python3" 143 | }, 144 | "language_info": { 145 | "codemirror_mode": { 146 | "name": "ipython", 147 | "version": 3 148 | }, 149 | "file_extension": ".py", 150 | "mimetype": "text/x-python", 151 | "name": "python", 152 | "nbconvert_exporter": "python", 153 | "pygments_lexer": "ipython3", 154 | "version": "3.5.3" 155 | } 156 | }, 157 | "nbformat": 4, 158 | "nbformat_minor": 0 159 | } 160 | -------------------------------------------------------------------------------- /notebooks/00.01-Foreword-by-Ariel-Rokem.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Machine Learning for OpenCV](00.00-Preface.ipynb) | [Contents](../README.md) | [A Taste of Machine Learning](01.00-A-Taste-of-Machine-Learning.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Foreword\n", 31 | "\n", 32 | "Over the last few years, our machines have slowly but surely learned how to see for themselves. We now take it for granted that our cameras detect our faces in pictures that we take, and that social media apps can even recognize us and our friends in the photos that we upload from these cameras. Over the next few years we will experience even more radical transformations. Before too long, cars will be driving themselves, our cellphones will be able to read and translate a sign in any language for us, and our x-rays and other medical images will be read and analyzed by powerful algorithms that will be able to accurately suggest a medical diagnosis, and even recommend effective treatments.\n", 33 | "\n", 34 | "These transformations are driven by an explosive combination of increased computing power, masses of image data, and a set of clever ideas taken from math, statistics and computer science. This rapidly growing intersection that is machine learning has taken off, affecting many of our day-to-day interactions with the world, and with each other. One of the most remarkable features of the current machine learning paradigm-shift in computer vision is that it relies to a large part on software tools that are freely available and developed by large groups of volunteers, hobbyists, scientists and engineers in open-source communities. This means that, in principle, the barriers to entry are also lower than ever: anyone who is interested in putting their mind to it can harness machine learning for image processing.\n", 35 | "\n", 36 | "However, just like in a garden with many forking paths, the wealth of tools and ideas, and the rapid development of these ideas underscores the need for a guide who can show you the way, and orient you in the right direction. I have some good news for you: having picked up this book, you are in the good hands of my colleague and collaborator Dr. Michael Beyeler as your guide. With his broad range of expertise, Michael is both a hard-nosed engineer, computer scientist and neuroscientist, as well as a prolific open-source software developer. He has not only taught robots how to see and navigate through complex environments, and computers how to model brain activity, but he also regularly teaches humans how to use programming to solve a variety of different machine learning and image processing problems. This means that you will get to benefit not only from the sure-handed rigor of his expertise and experience, but also that you will get to enjoy his thoughtfulness in teaching the ideas in his book, as well as a good dose of his sense of humor.\n", 37 | "\n", 38 | "The second piece of good news is that this going to be an exhilarating trip. There’s nothing that matches the thrill of understanding that comes from putting together the pieces of the puzzle that go into solving a problem in computer vision and machine learning with code and data. As Richard Feynman put it: “What I cannot create, I do not understand”. So, get ready to get your hands dirty (so to speak) with the code and data in the (open-source!) code examples that accompany this book, and to get creative. Understanding will surely follow." 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "Ariel Rokem\n", 46 | "\n", 47 | "Data Scientist, The University of Washington eScience Institute" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "\n", 55 | "< [Machine Learning for OpenCV](00.00-Preface.ipynb) | [Contents](../README.md) | [A Taste of Machine Learning](01.00-A-Taste-of-Machine-Learning.ipynb) >" 56 | ] 57 | } 58 | ], 59 | "metadata": { 60 | "kernelspec": { 61 | "display_name": "Python 3", 62 | "language": "python", 63 | "name": "python3" 64 | }, 65 | "language_info": { 66 | "codemirror_mode": { 67 | "name": "ipython", 68 | "version": 3 69 | }, 70 | "file_extension": ".py", 71 | "mimetype": "text/x-python", 72 | "name": "python", 73 | "nbconvert_exporter": "python", 74 | "pygments_lexer": "ipython3", 75 | "version": "3.5.3" 76 | } 77 | }, 78 | "nbformat": 4, 79 | "nbformat_minor": 2 80 | } 81 | -------------------------------------------------------------------------------- /notebooks/02.00-Working-with-Data-in-OpenCV.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [A Taste of Machine Learning](01.00-A-Taste-of-Machine-Learning.ipynb) | [Contents](../README.md) | [Dealing with Data Using Python's NumPy Package](02.01-Dealing-with-Data-Using-Python-NumPy.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Working with Data in OpenCV\n", 31 | "\n", 32 | "Now that we have whetted our appetite for machine learning, it is time to delve a little\n", 33 | "deeper into the different parts that make up a typical machine learning system.\n", 34 | "\n", 35 | "Machine learning is all about building mathematical models in order\n", 36 | "to understand data. The learning aspect enters this process when we give a machine\n", 37 | "learning model the capability to adjust its **internal parameters**; we can tweak these\n", 38 | "parameters so that the model explains the data better. In a sense, this can be understood as\n", 39 | "the model learning from the data. Once the model has learned enough—whatever that\n", 40 | "means—we can ask it to explain newly observed data.\n", 41 | "\n", 42 | "Hence machine learning problems are always split into (at least) two\n", 43 | "distinct phases:\n", 44 | "- A **training phase**, during which we aim to train a machine learning model on a set of data that we call the **training dataset**.\n", 45 | "- A **test phase**, during which we evaluate the learned (or finalized) machine learning model on a new set of never-before-seen data that we call the **test dataset**.\n", 46 | "\n", 47 | "The importance of splitting our data into a training set and test set cannot be understated.\n", 48 | "We always evaluate our models on an independent test set because we are interested in\n", 49 | "knowing how well our models generalize to new data. In the end, isn't this what learning is\n", 50 | "all about—be it machine learning or human learning?\n", 51 | "\n", 52 | "Machine learning is also all about the **data**.\n", 53 | "Data can be anything from images and movies to text\n", 54 | "documents and audio files. Therefore, in its raw form, data might be made of pixels, letters,\n", 55 | "words, or even worse: pure bits. It is easy to see that data in such a raw form might not be\n", 56 | "very convenient to work with. Instead, we have to find ways to **preprocess** the data in order\n", 57 | "to bring it into a form that is easy to parse.\n", 58 | "\n", 59 | "In this chapter, we want to learn how data fits in with machine learning, and how to work with data using the tools of our choice: OpenCV and Python.\n", 60 | "\n", 61 | "In specific, we want to address the following questions:\n", 62 | "- What does a typical machine learning workflow look like?\n", 63 | "- What are training data, validation data, and test data - and what are they good for?\n", 64 | "- How do I load, store, and work with such data in OpenCV using Python?\n", 65 | "\n", 66 | "## Outline\n", 67 | "\n", 68 | "- [Dealing with Data Using Python's NumPy Package](02.01-Dealing-with-Data-Using-Python-NumPy.ipynb)\n", 69 | "- [Loading External Datasets in Python](02.02-Loading-External-Datasets-in-Python.ipynb)\n", 70 | "- [Visualizing Data Using Matplotlib](02.03-Visualizing-Data-Using-Matplotlib.ipynb)\n", 71 | "- [Dealing with Data Using OpenCV's TrainData container in C++](02.05-Dealing-with-Data-Using-the-OpenCV-TrainData-Container-in-C%2B%2B.ipynb)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "## Starting a new IPython or Jupyter session\n", 79 | "\n", 80 | "Before we can get started, we need to open an IPython shell or start a Jupyter Notebook:\n", 81 | "\n", 82 | "1. Open a terminal like we did in the previous chapter, and navigate to the `opencv-machine-learning` directory:\n", 83 | "\n", 84 | " ```\n", 85 | " $ cd Desktop/opencv-machine-learning\n", 86 | " ```\n", 87 | "\n", 88 | "2. Activate the conda environment we created in the previous chapter:\n", 89 | "\n", 90 | " ```\n", 91 | " $ source activate Python3 # Mac OS X / Linux\n", 92 | " $ activate Python3 # Windows\n", 93 | " ```\n", 94 | "\n", 95 | "3. Start a new IPython or Jupyter session:\n", 96 | "\n", 97 | " ```\n", 98 | " $ ipython # for an IPython session\n", 99 | " $ jupyter notebook # for a Jupyter session\n", 100 | " ```" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "If you chose to start an IPython session, the program should have greeted you with a\n", 108 | "welcome message such as the following:\n", 109 | " \n", 110 | " $ ipython\n", 111 | " Python 3.5.2 |Continuum Analytics, Inc.| (default, Jul 5 2016, 11:41:13)\n", 112 | " [MSC v.1900 64 bit (AMD64)]\n", 113 | " Type \"copyright\", \"credits\" or \"license\" for more information.\n", 114 | " IPython 3.5.0 -- An enhanced Interactive Python.\n", 115 | " ? -> Introduction and overview of IPython's features.\n", 116 | " %quickref -> Quick reference.\n", 117 | " help -> Python's own help system.\n", 118 | " object? -> Details about 'object', use 'object??' for extra details.\n", 119 | " \n", 120 | " In [1]:" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "The line starting with `In [1]` is where you type in your regular Python commands. In\n", 128 | "addition, you can also use the Tab key while typing the names of variables and functions in\n", 129 | "order to have IPython automatically complete them." 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "If you chose to start a Jupyter session, a new window should have opened in your web\n", 137 | "browser that is pointing to http://localhost:8888. You want to create a new notebook by\n", 138 | "clicking on New in the top-right corner and selecting Notebooks (Python3).\n", 139 | "\n", 140 | "This will open a new window that contains an empty page with the same command line as in an IPython session:\n", 141 | " \n", 142 | " In [ ]:" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": { 149 | "collapsed": true 150 | }, 151 | "outputs": [], 152 | "source": [] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "\n", 159 | "< [A Taste of Machine Learning](01.00-A-Taste-of-Machine-Learning.ipynb) | [Contents](../README.md) | [Dealing with Data Using Python's NumPy Package](02.01-Dealing-with-Data-Using-Python-NumPy.ipynb) >" 160 | ] 161 | } 162 | ], 163 | "metadata": { 164 | "anaconda-cloud": {}, 165 | "kernelspec": { 166 | "display_name": "Python 3", 167 | "language": "python", 168 | "name": "python3" 169 | }, 170 | "language_info": { 171 | "codemirror_mode": { 172 | "name": "ipython", 173 | "version": 3 174 | }, 175 | "file_extension": ".py", 176 | "mimetype": "text/x-python", 177 | "name": "python", 178 | "nbconvert_exporter": "python", 179 | "pygments_lexer": "ipython3", 180 | "version": "3.5.3" 181 | } 182 | }, 183 | "nbformat": 4, 184 | "nbformat_minor": 1 185 | } 186 | -------------------------------------------------------------------------------- /notebooks/02.02-Loading-External-Datasets-in-Python.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Dealing with Data Using Python's NumPy Package](02.01-Dealing-with-Data-Using-Python-NumPy.ipynb) | [Contents](../README.md) | [Visualizing Data Using Matplotlib](02.03-Visualizing-Data-Using-Matplotlib.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Loading External Datasets in Python\n", 31 | "\n", 32 | "Thanks to the SciPy community, there are tons of resources out there for getting your hands on some data.\n", 33 | "\n", 34 | "A particularly useful resource comes in the form of the sklearn.datasets package of scikit-learn. This package comes pre-installed with some small datasets that do not require to download any files from external websites. These datasets include:\n", 35 | "- load_boston: The Boston dataset contains housing prices in different suburbs of Boston, along with a number of interesting features such as per capita crime rate by town, proportion of residential land and non-retail business, etc.\n", 36 | "- load_iris: The Iris dataset contains three different types of iris flowers (setosa, versicolor, and virginica), along with four features describing the width and length of the sepals and petals.\n", 37 | "- load_diabetes: The diabetes dataset lets you classify patients as having diabetes or not, based on features such as patient age, sex, body mass index, average blood pressure, and six blood serum measurements.\n", 38 | "- load_digits: The digits dataset contains 8x8 pixel images of digits 0-9.\n", 39 | "- load_linnerud: The Linnerud dataset contains three physiological and three exercies variables measured on twenty middle-aged men in a fitness club.\n", 40 | "\n", 41 | "Even better, scikit-learn allows you to download datasets directly from external repositories, such as:\n", 42 | "- fetch_olivetti_faces: The Olivetta face dataset contains of ten different images each of 40 distinct subjects.\n", 43 | "- fetch_20newsgroups: The 20 newsgroup dataset contains around 18,000 newsgroups post on 20 topics." 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": {}, 49 | "source": [ 50 | "Even better, it is possible to download datasets directly from the machine learning database at http://mldata.org. \n", 51 | "\n", 52 | "For example, to download the MNIST dataset of handwritten digits, simply type:" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 1, 58 | "metadata": { 59 | "collapsed": true 60 | }, 61 | "outputs": [], 62 | "source": [ 63 | "from sklearn import datasets" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 2, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "mnist = datasets.fetch_mldata('MNIST original')" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "Note that this might take a while, depending on your internet connection.\n", 80 | "\n", 81 | "The MNIST database contains a total of 70,000 examples of handwritten digits (28x28 pixel images, labeled from 0 to 9). Data and labels are delivered in two separate containers, which we can inspect as follows:" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 3, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "data": { 91 | "text/plain": [ 92 | "(70000, 784)" 93 | ] 94 | }, 95 | "execution_count": 3, 96 | "metadata": {}, 97 | "output_type": "execute_result" 98 | } 99 | ], 100 | "source": [ 101 | "mnist.data.shape" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 4, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "data": { 111 | "text/plain": [ 112 | "(70000,)" 113 | ] 114 | }, 115 | "execution_count": 4, 116 | "metadata": {}, 117 | "output_type": "execute_result" 118 | } 119 | ], 120 | "source": [ 121 | "mnist.target.shape" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "Here, we can see that mnist.data contains 70,000 images of 28 x 28 = 784 pixels each.\n", 129 | "Labels are stored in `mnist.target`, where there is only one label per image.\n", 130 | "\n", 131 | "We can further inspect the values of all targets, but we don't just want to print them all.\n", 132 | "Instead, we are interested to see all distinct target values, which is easy to do with NumPy:" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 5, 138 | "metadata": { 139 | "collapsed": true 140 | }, 141 | "outputs": [], 142 | "source": [ 143 | "import numpy as np" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 6, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "data": { 153 | "text/plain": [ 154 | "array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])" 155 | ] 156 | }, 157 | "execution_count": 6, 158 | "metadata": {}, 159 | "output_type": "execute_result" 160 | } 161 | ], 162 | "source": [ 163 | "np.unique(mnist.target)" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "\n", 171 | "< [Dealing with Data Using Python's NumPy Package](02.01-Dealing-with-Data-Using-Python-NumPy.ipynb) | [Contents](../README.md) | [Visualizing Data Using Matplotlib](02.03-Visualizing-Data-Using-Matplotlib.ipynb) >" 172 | ] 173 | } 174 | ], 175 | "metadata": { 176 | "anaconda-cloud": {}, 177 | "kernelspec": { 178 | "display_name": "Python 3", 179 | "language": "python", 180 | "name": "python3" 181 | }, 182 | "language_info": { 183 | "codemirror_mode": { 184 | "name": "ipython", 185 | "version": 3 186 | }, 187 | "file_extension": ".py", 188 | "mimetype": "text/x-python", 189 | "name": "python", 190 | "nbconvert_exporter": "python", 191 | "pygments_lexer": "ipython3", 192 | "version": "3.5.3" 193 | } 194 | }, 195 | "nbformat": 4, 196 | "nbformat_minor": 1 197 | } 198 | -------------------------------------------------------------------------------- /notebooks/02.05-Dealing-with-Data-Using-the-OpenCV-TrainData-Container-in-C++.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "\n", 11 | "\n", 12 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 13 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 14 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 15 | "\n", 16 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 17 | "If you find this content useful, please consider supporting the work by\n", 18 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "deletable": true, 25 | "editable": true 26 | }, 27 | "source": [ 28 | "\n", 29 | "< [Visualizing Data from an External Dataset](02.04-Visualizing-Data-from-an-External-Dataset.ipynb) | [Contents](../README.md) | [First Steps in Supervised Learning](03.00-First-Steps-in-Supervised-Learning.ipynb) >" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": { 35 | "deletable": true, 36 | "editable": true 37 | }, 38 | "source": [ 39 | "# Dealing with Data Using OpenCV's TrainData Container in C++\n", 40 | "\n", 41 | "For the sake of completeness, and for those who insist on using the C++ API of OpenCV, let's do a quick detour on OpenCV's TrainData container that allows us to load numerical data from .csv files.\n", 42 | "\n", 43 | "Among other things, in C++ the `ml` module contains a class called `TrainData`, which\n", 44 | "provides a container to work with data in C++. Its functionality is limited to reading\n", 45 | "(preferably) numerical data from .csv files (containing comma-separated values). Hence, if\n", 46 | "the data that you want to work with comes in a neatly organized `.csv` file, this class will\n", 47 | "save you a lot of time. If your data comes from a different source, I'm afraid your best\n", 48 | "option might be to create a `.csv` file by hand, using a suitable program such as Open Office\n", 49 | "or Microsoft Excel." 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "deletable": true, 56 | "editable": true 57 | }, 58 | "source": [ 59 | "If you have some nice all-float data that lives in a comma-separated file, you could load it like so:\n", 60 | "\n", 61 | " Ptr tDataContainer = TrainData::loadFromCSV(\"file.csv\",\n", 62 | " 0, // number of lines to skip\n", 63 | " 0); // index of 1st and only output var" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": { 69 | "deletable": true, 70 | "editable": true 71 | }, 72 | "source": [ 73 | "Shuffling data is easy:\n", 74 | "\n", 75 | " tDataContainer->shuffleTrainTest();" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": { 81 | "deletable": true, 82 | "editable": true 83 | }, 84 | "source": [ 85 | "And so is splitting data into training and test sets. For example, if your file contains 100 samples, you could assign the first 90 samples to the training set, and leave the remaining 10 to the test set:\n", 86 | "\n", 87 | " tDataContainer->setTrainTestSplit(90);" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": { 93 | "deletable": true, 94 | "editable": true 95 | }, 96 | "source": [ 97 | "You can also load all training/test samples and store them in an OpenCV matrix:\n", 98 | "\n", 99 | " cv::Mat trainData = tDataContainer->getTrainSamples();\n", 100 | " cv::Mat testData = tDataContainer->getTestSamples();" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": { 106 | "deletable": true, 107 | "editable": true 108 | }, 109 | "source": [ 110 | "Most OpenCV classifiers from the ml module then directly take training data as an input. Let's say you have a statistical model saved in the variable model:\n", 111 | "\n", 112 | " model->train(tDataContainer->getTrainSamples());" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": { 118 | "deletable": true, 119 | "editable": true 120 | }, 121 | "source": [ 122 | "You can find all relevant functions described here: http://docs.opencv.org/3.1.0/dc/d32/classcv_1_1ml_1_1TrainData.html.\n", 123 | "\n", 124 | "Other than that, I'm afraid the `TrainData` container and its use cases might be a bit\n", 125 | "antiquated. So for the rest of the book, we will focus on Python instead." 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": { 131 | "deletable": true, 132 | "editable": true 133 | }, 134 | "source": [ 135 | "\n", 136 | "< [Visualizing Data from an External Dataset](02.04-Visualizing-Data-from-an-External-Dataset.ipynb) | [Contents](../README.md) | [First Steps in Supervised Learning](03.00-First-Steps-in-Supervised-Learning.ipynb) >" 137 | ] 138 | } 139 | ], 140 | "metadata": { 141 | "anaconda-cloud": {}, 142 | "kernelspec": { 143 | "display_name": "Python 3", 144 | "language": "python", 145 | "name": "python3" 146 | }, 147 | "language_info": { 148 | "codemirror_mode": { 149 | "name": "ipython", 150 | "version": 3 151 | }, 152 | "file_extension": ".py", 153 | "mimetype": "text/x-python", 154 | "name": "python", 155 | "nbconvert_exporter": "python", 156 | "pygments_lexer": "ipython3", 157 | "version": "3.5.3" 158 | } 159 | }, 160 | "nbformat": 4, 161 | "nbformat_minor": 0 162 | } 163 | -------------------------------------------------------------------------------- /notebooks/03.00-First-Steps-in-Supervised-Learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Dealing with Data Using OpenCV's TrainData Container in C++](02.05-Dealing-with-Data-Using-the-OpenCV-TrainData-Container-in-C++.ipynb) | [Contents](../README.md) | [Measuring-Model-Performance-with-Scoring-Functions](03.01-Measuring-Model-Performance-with-Scoring-Functions.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# First Steps in Supervised Learning\n", 31 | "\n", 32 | "This is the moment you've been waiting for, isn't it?\n", 33 | "\n", 34 | "We have covered all the bases - we have a functioning Python environment, we have\n", 35 | "OpenCV installed, and we know how to handle data in Python. Now it's time to build our\n", 36 | "first machine learning system! And what better way to start off than to focus on one of the\n", 37 | "most common and successful types of machine learning: **supervised learning**?\n", 38 | "\n", 39 | "From the previous chapter, we already know that supervised learning is all about learning\n", 40 | "regularities in some training data by using the labels that come with it so that we can\n", 41 | "predict the labels of some new, never-seen-before test data. In this chapter, we want to dig a\n", 42 | "little deeper, and learn how to turn our theoretical knowledge into something practical.\n", 43 | "\n", 44 | "Along the way, we want to address the following questions:\n", 45 | "- What's the difference between classification and regression, and when do I use which?\n", 46 | "- What is a $k$-nearest neighbor ($k$-NN) classifier, and how do I implement one in OpenCV?\n", 47 | "- How do I use logistic regression for classification, and why is it named so confusingly?\n", 48 | "- How do I build a linear regression model in OpenCV, and how does it differ from Lasso and ridge regression?\n", 49 | "\n", 50 | "## Outline\n", 51 | "\n", 52 | "- [Measuring Model Performance with Scoring Functions](03.01-Measuring-Model-Performance-with-Scoring-Functions.ipynb)\n", 53 | "- [Understanding the k-NN Algorithm](03.02-Understanding-the-k-NN-Algorithm.ipynb)\n", 54 | "- [Using Regression Models to Predict Continuous Outcomes](03.03-Using-Regression-Models-to-Predict-Continuous-Outcomes.ipynb)\n", 55 | "- [Applying Lasso and Ridge Regression](03.04-Applying-Lasso-and-Ridge-Regression.ipynb)\n", 56 | "- [Classifying Iris Species Using Logistic Regression](03.05-Classifying-Iris-Species-Using-Logistic-Regression.ipynb)\n", 57 | "\n", 58 | "Let's jump right in!\n", 59 | "\n", 60 | "> The book provides an overview of common supervised learnig methods, and features a detailed treatment of common machine learning workflows. Below is a summary of these topics. For more information, please refer to the book.\n", 61 | "\n", 62 | "\n", 63 | "## Supervised learning in OpenCV\n", 64 | "\n", 65 | "OpenCV provides a pretty straightforward interface for all its statistical\n", 66 | "learning models, which includes all supervised learning models.\n", 67 | "In OpenCV, every machine learning model derives from the `cv::ml::StatModel` base\n", 68 | "class. This is fancy talk for saying that if we want to be a machine learning model in\n", 69 | "OpenCV, we have to provide all the functionality that `StatModel` tells us to. This includes\n", 70 | "a method to train the model (called `train`) and a method to measure the performance of\n", 71 | "the model (called `calcError`).\n", 72 | "\n", 73 | "Thanks to this organization of the software, setting up a machine learning model in\n", 74 | "OpenCV always follows the same logic:\n", 75 | "- **Initialization**: We call the model by name to create an empty instance of the model.\n", 76 | "- **Set parameters**: If the model needs some parameters, we can set them via setter methods, which can be different for every model. For example, in order for a $k$-NN algorithm to work, we need to specify its open parameter, $k$ (as we will find out later).\n", 77 | "- **Train the model**: Every model must provide a method called `train`, used to fit the model to some data.\n", 78 | "- **Predict new labels**: Every model must provide a method called `predict`, used to predict the labels of new data.\n", 79 | "- **Score the model**: Every model must provide a method called `calcError`, used to measure performance. This calculation might be different for every model." 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "\n", 87 | "< [Dealing with Data Using OpenCV's TrainData Container in C++](02.05-Dealing-with-Data-Using-the-OpenCV-TrainData-Container-in-C++.ipynb) | [Contents](../README.md) | [Measuring-Model-Performance-with-Scoring-Functions](03.01-Measuring-Model-Performance-with-Scoring-Functions.ipynb) >" 88 | ] 89 | } 90 | ], 91 | "metadata": { 92 | "kernelspec": { 93 | "display_name": "Python 3", 94 | "language": "python", 95 | "name": "python3" 96 | }, 97 | "language_info": { 98 | "codemirror_mode": { 99 | "name": "ipython", 100 | "version": 3 101 | }, 102 | "file_extension": ".py", 103 | "mimetype": "text/x-python", 104 | "name": "python", 105 | "nbconvert_exporter": "python", 106 | "pygments_lexer": "ipython3", 107 | "version": "3.5.3" 108 | } 109 | }, 110 | "nbformat": 4, 111 | "nbformat_minor": 1 112 | } 113 | -------------------------------------------------------------------------------- /notebooks/04.00-Representing-Data-and-Engineering-Features.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Classifying Iris Species Using Logistic Regression](03.05-Classifying-Iris-Species-Using-Logistic-Regression.ipynb) | [Contents](../README.md) | [Preprocessing Data](04.01-Preprocessing-Data.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Representing Data and Engineering Features\n", 33 | "\n", 34 | "In the last chapter, we built our very first supervised learning models and applied them to\n", 35 | "some classic datasets, such as the **Iris** and the **Boston** datasets. However, in the real world,\n", 36 | "data rarely comes in a neat `` **feature matrix** that is part of a\n", 37 | "pre-packaged database. Instead, it is our own responsibility to find a way to represent the\n", 38 | "data in a meaningful way. The process of finding the best way to represent our data is\n", 39 | "known as **feature engineering**, and it is one of the main tasks of data scientists and machine\n", 40 | "learning practitioners trying to solve real-world problems.\n", 41 | "\n", 42 | "I know you would rather jump right to the end and build the deepest neural network\n", 43 | "mankind has ever seen. But, trust me, this stuff is important! Representing our data in the\n", 44 | "right way can have a much greater influence on the performance of our supervised model\n", 45 | "than the exact parameters we choose. And we get to invent our own features, too.\n", 46 | "\n", 47 | "In this chapter, we will therefore go over some common feature engineering tasks.\n", 48 | "Specifically, we want to answer the following questions:\n", 49 | "- What are some common preprocessing techniques that everyone uses but nobody talks about?\n", 50 | "- How do we represent categorical variables, such as the names of products, of colors, or of fruits?\n", 51 | "- How would we even go about representing text?\n", 52 | "- What is the best way to encode images, and what do SIFT and SURF stand for?\n", 53 | "\n", 54 | "## Outline\n", 55 | "- [Preprocessing Data](04.01-Preprocessing-Data.ipynb)\n", 56 | "- [Reducing the Dimensionality of the Data](04.02-Reducing-the-Dimensionality-of-the-Data.ipynb)\n", 57 | "- [Representing Categorical Variables](04.03-Representing-Categorical-Variables.ipynb)\n", 58 | "- [Representing Text Features](04.04-Represening-Text-Features.ipynb)\n", 59 | "- [Representing Images](04.05-Representing-Images.ipynb)\n", 60 | "\n", 61 | "> The book features a detailed treatment of feature engineering, data preprocessing, and data transformation. Below is a short summary of these topics. For more information, please refer to the book." 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "## Feature engineering\n", 69 | "\n", 70 | "Feature engineering comes in two stages:\n", 71 | "- **Feature selection**: This is the process of identifying important attributes (or\n", 72 | " features) in the data. Possible features of an image might be the location of edges,\n", 73 | " corners, or ridges. In this chapter, we will look at some of the more advanced\n", 74 | " feature descriptors that OpenCV provides, such as the Scale-Invariant Feature\n", 75 | " Transform (SIFT) and Speeded Up Robust Features (SURF).\n", 76 | "- **Feature extraction**: This is the actual process of transforming the raw data into\n", 77 | " the desired feature space used to feed a machine learning algorithm as illustrated\n", 78 | " in the figure above. An example would be the Harris operator, which allows you\n", 79 | " to extract corners (that is, a selected feature) in an image.\n", 80 | "\n", 81 | "\n", 82 | "## Preprocessing data\n", 83 | "\n", 84 | "The preprocessing pipeline:\n", 85 | "\n", 86 | "- *Data formatting*: The data may not be in a format that is suitable for you to work\n", 87 | " with. For example, the data might be provided in a proprietary file format, which\n", 88 | " your favorite machine learning algorithm does not understand.\n", 89 | "- *Data cleaning*: The data may contain invalid or missing entries, which need to be\n", 90 | " cleaned up or removed.\n", 91 | "- *Data sampling*: The data may be far too numerous for your specific purpose,\n", 92 | " forcing you to sample the data in a smart way.\n", 93 | " \n", 94 | "\n", 95 | "## Transforming data\n", 96 | "\n", 97 | "Once the data has been preprocessed, you are ready for the actual feature engineering: to transform the\n", 98 | "preprocessed data to fit your specific machine learning algorithm. This step usually involves one or more of\n", 99 | "three possible procedures:\n", 100 | "\n", 101 | "- *Scaling*: Machine learning algorithms often require the data to be within a\n", 102 | " common range, such as to have zero mean and unit variance. Scaling is the\n", 103 | " process of bringing all features (which might have different physical units) into a\n", 104 | " common range of values.\n", 105 | "- *Decomposition*: Datasets often have many more features than you could possibly\n", 106 | " process. Feature decomposition is the process of compressing data into a smaller\n", 107 | " number of highly informative data components.\n", 108 | "- *Aggregation*: There may be features that can be aggregated into a single feature\n", 109 | " that would be more meaningful to the problem you are trying to solve. For\n", 110 | " example, a database might contain the date and time each user logged into a webbased\n", 111 | " system. Depending on the task, this data might be better represented by\n", 112 | " simply counting the number of logins per user. " 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "\n", 120 | "< [Classifying Iris Species Using Logistic Regression](03.05-Classifying-Iris-Species-Using-Logistic-Regression.ipynb) | [Contents](../README.md) | [Preprocessing Data](04.01-Preprocessing-Data.ipynb) >" 121 | ] 122 | } 123 | ], 124 | "metadata": { 125 | "kernelspec": { 126 | "display_name": "Python 3", 127 | "language": "python", 128 | "name": "python3" 129 | }, 130 | "language_info": { 131 | "codemirror_mode": { 132 | "name": "ipython", 133 | "version": 3 134 | }, 135 | "file_extension": ".py", 136 | "mimetype": "text/x-python", 137 | "name": "python", 138 | "nbconvert_exporter": "python", 139 | "pygments_lexer": "ipython3", 140 | "version": "3.5.3" 141 | } 142 | }, 143 | "nbformat": 4, 144 | "nbformat_minor": 1 145 | } 146 | -------------------------------------------------------------------------------- /notebooks/04.03-Representing-Categorical-Variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Reducing the Dimensionality of the Data](04.02-Reducing-the-Dimensionality-of-the-Data.ipynb) | [Contents](../README.md) | [Representing Text Features](04.04-Represening-Text-Features.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Representing Categorical Variables\n", 33 | "\n", 34 | "One of the most common data types we might encounter while building a machine learning\n", 35 | "system are **categorical features** (also known as **discrete features**), such as the color of a fruit\n", 36 | "or the name of a company.\n", 37 | "\n", 38 | "The challenge with categorical features is that they don't change\n", 39 | "in a continuous way, which makes it hard to represent them with numbers. For example, a\n", 40 | "banana is either green or yellow, but not both. A product belongs either in the clothing\n", 41 | "department or in the books department, but rarely in both, and so on.\n", 42 | "\n", 43 | "How would you go about representing such features?\n", 44 | "\n", 45 | "Consider the following data containing a list of some of the forefathers of machine learning and artificial intelligence:" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 1, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "data = [\n", 55 | " {'name': 'Alan Turing', 'born': 1912, 'died': 1954},\n", 56 | " {'name': 'Herbert A. Simon', 'born': 1916, 'died': 2001},\n", 57 | " {'name': 'Jacek Karpinski', 'born': 1927, 'died': 2010},\n", 58 | " {'name': 'J.C.R. Licklider', 'born': 1915, 'died': 1990},\n", 59 | " {'name': 'Marvin Minsky', 'born': 1927, 'died': 2016},\n", 60 | "]" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "While the features 'born' and 'died' are already in numeric format, the 'name' feature is\n", 68 | "a bit trickier to encode. We might be intrigued to encode them in the following way:" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 2, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "{'Alan Turing': 1,\n", 78 | " 'Herbert A. Simon': 2,\n", 79 | " 'Jacek Karpinsky': 3,\n", 80 | " 'J.C.R. Licklider': 4,\n", 81 | " 'Marvin Minsky': 5};" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "Although this seems like a good idea, it does not make much sense from a machine learning\n", 89 | "perspective. Why not?\n", 90 | "\n", 91 | "Refer to the book for the answer (p. 97)." 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "A better way is to use a `DictVectorizer`, also known as a *one-hot encoding*. The way it works is by feeding a\n", 99 | "dictionary containing the data to the `fit_transform` function, and the function\n", 100 | "automatically determines which features to encode:" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 3, 106 | "metadata": {}, 107 | "outputs": [ 108 | { 109 | "data": { 110 | "text/plain": [ 111 | "array([[1912, 1954, 1, 0, 0, 0, 0],\n", 112 | " [1916, 2001, 0, 1, 0, 0, 0],\n", 113 | " [1927, 2010, 0, 0, 0, 1, 0],\n", 114 | " [1915, 1990, 0, 0, 1, 0, 0],\n", 115 | " [1927, 2016, 0, 0, 0, 0, 1]], dtype=int64)" 116 | ] 117 | }, 118 | "execution_count": 3, 119 | "metadata": {}, 120 | "output_type": "execute_result" 121 | } 122 | ], 123 | "source": [ 124 | "from sklearn.feature_extraction import DictVectorizer\n", 125 | "vec = DictVectorizer(sparse=False, dtype=int)\n", 126 | "vec.fit_transform(data)" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "What happened here? The two year entries are still intact, but the rest of the rows have been\n", 134 | "replaced by ones and zeros. We can call `get_feature_names` to find out the listed order of\n", 135 | "the features:" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 4, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "data": { 145 | "text/plain": [ 146 | "['born',\n", 147 | " 'died',\n", 148 | " 'name=Alan Turing',\n", 149 | " 'name=Herbert A. Simon',\n", 150 | " 'name=J.C.R. Licklider',\n", 151 | " 'name=Jacek Karpinski',\n", 152 | " 'name=Marvin Minsky']" 153 | ] 154 | }, 155 | "execution_count": 4, 156 | "metadata": {}, 157 | "output_type": "execute_result" 158 | } 159 | ], 160 | "source": [ 161 | "vec.get_feature_names()" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "The first row of our data matrix, which stands for Alan Turing, is now encoded as\n", 169 | "`'born'=1912`, `'died'=1954`, `'Alan Turing'=1`, `'Herbert A. Simon'=0`, `'J.C.R Licklider'=0`, `'Jacek Karpinsik'=0`, and `'Marvin Minsky'=0`." 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "If your category has many possible values, it is better to use a sparse matrix:" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 5, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "data": { 186 | "text/plain": [ 187 | "<5x7 sparse matrix of type ''\n", 188 | "\twith 15 stored elements in Compressed Sparse Row format>" 189 | ] 190 | }, 191 | "execution_count": 5, 192 | "metadata": {}, 193 | "output_type": "execute_result" 194 | } 195 | ], 196 | "source": [ 197 | "vec = DictVectorizer(sparse=True, dtype=int)\n", 198 | "vec.fit_transform(data)" 199 | ] 200 | }, 201 | { 202 | "cell_type": "markdown", 203 | "metadata": {}, 204 | "source": [ 205 | "We will come back to this technique when we talk about neural networks in [Chapter 9](09.00-Using-Deep-Learning-to-Classify-Handwritten-Digits.ipynb),\n", 206 | "*Using Deep Learning to Classify Handwritten Digits*." 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "\n", 214 | "< [Reducing the Dimensionality of the Data](04.02-Reducing-the-Dimensionality-of-the-Data.ipynb) | [Contents](../README.md) | [Representing Text Features](04.04-Represening-Text-Features.ipynb) >" 215 | ] 216 | } 217 | ], 218 | "metadata": { 219 | "kernelspec": { 220 | "display_name": "Python 3", 221 | "language": "python", 222 | "name": "python3" 223 | }, 224 | "language_info": { 225 | "codemirror_mode": { 226 | "name": "ipython", 227 | "version": 3 228 | }, 229 | "file_extension": ".py", 230 | "mimetype": "text/x-python", 231 | "name": "python", 232 | "nbconvert_exporter": "python", 233 | "pygments_lexer": "ipython3", 234 | "version": "3.5.3" 235 | } 236 | }, 237 | "nbformat": 4, 238 | "nbformat_minor": 1 239 | } 240 | -------------------------------------------------------------------------------- /notebooks/04.04-Represening-Text-Features.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Representing Categorical Variables](04.03-Representing-Categorical-Variables.ipynb) | [Contents](../README.md) | [Representing images](04.05-Representing-Images.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Representing Text Features\n", 33 | "\n", 34 | "Similar to categorical features, scikit-learn offers an easy way to encode another common\n", 35 | "feature type, **text features**. When working with text features, it is often convenient to\n", 36 | "encode individual words or phrases as numerical values.\n", 37 | "\n", 38 | "Let's consider a dataset that contains a small corpus of text phrases:" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 1, 44 | "metadata": { 45 | "collapsed": true 46 | }, 47 | "outputs": [], 48 | "source": [ 49 | "sample = [\n", 50 | " 'feature engineering',\n", 51 | " 'feature selection',\n", 52 | " 'feature extraction'\n", 53 | "]" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "One of the simplest methods of encoding such data is by **word count**; for each phrase, we count the occurrences of each word within it. In scikit-learn, this is easily done using\n", 61 | "`CountVectorizer`, which functions akin to `DictVectorizer`:" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 2, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "data": { 71 | "text/plain": [ 72 | "<3x4 sparse matrix of type ''\n", 73 | "\twith 6 stored elements in Compressed Sparse Row format>" 74 | ] 75 | }, 76 | "execution_count": 2, 77 | "metadata": {}, 78 | "output_type": "execute_result" 79 | } 80 | ], 81 | "source": [ 82 | "from sklearn.feature_extraction.text import CountVectorizer\n", 83 | "vec = CountVectorizer()\n", 84 | "X = vec.fit_transform(sample)\n", 85 | "X" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "By default, this will store our feature matrix `X` as a sparse matrix. If we want to manually\n", 93 | "inspect it, we need to convert it to a regular array:" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 3, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "array([[1, 0, 1, 0],\n", 105 | " [0, 0, 1, 1],\n", 106 | " [0, 1, 1, 0]], dtype=int64)" 107 | ] 108 | }, 109 | "execution_count": 3, 110 | "metadata": {}, 111 | "output_type": "execute_result" 112 | } 113 | ], 114 | "source": [ 115 | "X.toarray()" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "...with corresponding feature names:" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 4, 128 | "metadata": {}, 129 | "outputs": [ 130 | { 131 | "data": { 132 | "text/plain": [ 133 | "['engineering', 'extraction', 'feature', 'selection']" 134 | ] 135 | }, 136 | "execution_count": 4, 137 | "metadata": {}, 138 | "output_type": "execute_result" 139 | } 140 | ], 141 | "source": [ 142 | "vec.get_feature_names()" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": {}, 148 | "source": [ 149 | "One possible shortcoming of this approach is that we might put too much weight on words\n", 150 | "that appear very frequently. One approach to fix this is known as term **frequency-inverse\n", 151 | "document frequency (TF-IDF)**. What TF-IDF does might be easier to understand than its\n", 152 | "name, which is basically to weigh the word counts by a measure of how often they appear\n", 153 | "in the entire dataset.\n", 154 | "\n", 155 | "The syntax for TF-IDF is pretty much similar to the previous command:" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 5, 161 | "metadata": {}, 162 | "outputs": [ 163 | { 164 | "data": { 165 | "text/plain": [ 166 | "array([[ 0.861037 , 0. , 0.50854232, 0. ],\n", 167 | " [ 0. , 0. , 0.50854232, 0.861037 ],\n", 168 | " [ 0. , 0.861037 , 0.50854232, 0. ]])" 169 | ] 170 | }, 171 | "execution_count": 5, 172 | "metadata": {}, 173 | "output_type": "execute_result" 174 | } 175 | ], 176 | "source": [ 177 | "from sklearn.feature_extraction.text import TfidfVectorizer\n", 178 | "vec = TfidfVectorizer()\n", 179 | "X = vec.fit_transform(sample)\n", 180 | "X.toarray()" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "We note that the numbers are now smaller than before, with the third column taking the\n", 188 | "biggest hit. This makes sense, as the third column corresponds to the most frequent word\n", 189 | "across all three phrases, `'feature'`:" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 6, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "data": { 199 | "text/plain": [ 200 | "['engineering', 'extraction', 'feature', 'selection']" 201 | ] 202 | }, 203 | "execution_count": 6, 204 | "metadata": {}, 205 | "output_type": "execute_result" 206 | } 207 | ], 208 | "source": [ 209 | "vec.get_feature_names()" 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "Representing text features will become important in [Chapter 7](07.00-Implementing-a-Spam-Filter-with-Bayesian-Learning.ipynb), *Implementing a Spam Filter\n", 217 | "with Bayesian Learning*." 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "metadata": {}, 223 | "source": [ 224 | "\n", 225 | "< [Representing Categorical Variables](04.03-Representing-Categorical-Variables.ipynb) | [Contents](../README.md) | [Representing images](04.05-Representing-Images.ipynb) >" 226 | ] 227 | } 228 | ], 229 | "metadata": { 230 | "kernelspec": { 231 | "display_name": "Python 3", 232 | "language": "python", 233 | "name": "python3" 234 | }, 235 | "language_info": { 236 | "codemirror_mode": { 237 | "name": "ipython", 238 | "version": 3 239 | }, 240 | "file_extension": ".py", 241 | "mimetype": "text/x-python", 242 | "name": "python", 243 | "nbconvert_exporter": "python", 244 | "pygments_lexer": "ipython3", 245 | "version": "3.5.3" 246 | } 247 | }, 248 | "nbformat": 4, 249 | "nbformat_minor": 1 250 | } 251 | -------------------------------------------------------------------------------- /notebooks/05.00-Using-Decision-Trees-to-Make-a-Medical-Diagnosis.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Representing images](04.05-Representing-Images.ipynb) | [Contents](../README.md) | [Building Our First Decision Tree](05.01-Building-Our-First-Decision-Tree.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Using Decision Trees to Make a Medical Diagnosis\n", 31 | "\n", 32 | "Now that we know how to handle data in all shapes and forms, be it numerical, categorical,\n", 33 | "text, or image data, it is time to put our newly gained knowledge to good use.\n", 34 | "\n", 35 | "In this chapter, we will learn how to build a machine learning system that can make a\n", 36 | "medical diagnosis. We aren't all doctors, but we've probably all been to one at some point in\n", 37 | "our lives. Typically, a doctor would gain as much information as possible about a patient's\n", 38 | "history and symptoms in order to make an informed diagnosis. We will mimic a doctor's\n", 39 | "decision-making process with the help of what is known as decision trees.\n", 40 | "\n", 41 | "A **decision tree** is a simple yet powerful supervised learning algorithm that resembles a\n", 42 | "flow chart; we will talk more about this in just a minute. Other than in medicine, decision\n", 43 | "trees are commonly used in fields such as astronomy (for example, for filtering noise from\n", 44 | "Hubble Space Telescope images or to classify star-galaxy clusters), manufacturing and\n", 45 | "production (for example, by Boeing to discover flaws in the manufacturing process), and\n", 46 | "object recognition (for example, for recognizing 3D objects).\n", 47 | "\n", 48 | "Specifically, we want to address the following questions in this chapter:\n", 49 | "- How do we build simple decision trees from data, and use them for either classification or regression?\n", 50 | "- How do we decide which decision to make next?\n", 51 | "- How do we prune a tree, and what is that good for?\n", 52 | "\n", 53 | "\n", 54 | "## Outline\n", 55 | "\n", 56 | "- [Building Your First Decision Tree](05.01-Building-Your-First-Decision-Tree.ipynb)\n", 57 | "- [Using Decision Trees to Diagnose Breast Cancer](05.02-Using-Decision-Trees-to-Diagnose-Breast-Cancer.ipynb)\n", 58 | "- [Using Decision Trees for Regression](05.03-Using-Decision-Trees-for-Regression.ipynb)\n", 59 | "\n", 60 | "\n", 61 | "\n", 62 | "But first, let's talk about what decision trees actually are." 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "## Understanding Decision Trees\n", 70 | "\n", 71 | "Decision trees are simple yet powerful model for supervised learning problems. Like the name suggests, you can think of them as a tree in which information flows along different branches in the tree - starting at the trunk, and going all the way to the individual leaves.\n", 72 | "\n", 73 | "> Here the book offers a detailed treatment of the inner workings of decision trees, along with illustrations and simple examples. For more information on decision trees, please refer to the book.\n", 74 | "\n", 75 | "Let's say we have a dataset consisting of a single e-mail:" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 1, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "data = [\n", 85 | " 'I am Mohammed Abacha, the son of the late Nigerian Head of '\n", 86 | " 'State who died on the 8th of June 1998. Since i have been '\n", 87 | " 'unsuccessful in locating the relatives for over 2 years now '\n", 88 | " 'I seek your consent to present you as the next of kin so '\n", 89 | " 'that the proceeds of this account valued at US$15.5 Million '\n", 90 | " 'Dollars can be paid to you. If you are capable and willing '\n", 91 | " 'to assist, contact me at once via email with following '\n", 92 | " 'details: 1. Your full name, address, and telephone number. '\n", 93 | " '2. Your Bank Name, Address. 3.Your Bank Account Number and '\n", 94 | " 'Beneficiary Name - You must be the signatory.'\n", 95 | "]" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "This data can be vectorized using Scikit-Learn's `CountVectorizer`, which turns the e-mail into its individual words and corresponding word counts:" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 2, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "from sklearn.feature_extraction.text import CountVectorizer\n", 112 | "vec = CountVectorizer()\n", 113 | "X = vec.fit_transform(data)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "Let's look at the first five words in the dictionaries, and their word counts:" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 3, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "data": { 130 | "text/plain": [ 131 | "['15', '1998', '8th', 'abacha', 'account']" 132 | ] 133 | }, 134 | "execution_count": 3, 135 | "metadata": {}, 136 | "output_type": "execute_result" 137 | } 138 | ], 139 | "source": [ 140 | "vec.get_feature_names()[:5]" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 4, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/plain": [ 151 | "array([1, 1, 1, 1, 2], dtype=int64)" 152 | ] 153 | }, 154 | "execution_count": 4, 155 | "metadata": {}, 156 | "output_type": "execute_result" 157 | } 158 | ], 159 | "source": [ 160 | "X.toarray()[0, :5]" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": {}, 166 | "source": [ 167 | "So, how would you have checked if the e-mail is from a Nigerian prince?\n", 168 | "\n", 169 | "One way to do this is to look if the e-mail contained both the words 'nigerian' and 'prince':" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 5, 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "data": { 179 | "text/plain": [ 180 | "True" 181 | ] 182 | }, 183 | "execution_count": 5, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | } 187 | ], 188 | "source": [ 189 | "'nigerian' in vec.get_feature_names()" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 6, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "data": { 199 | "text/plain": [ 200 | "False" 201 | ] 202 | }, 203 | "execution_count": 6, 204 | "metadata": {}, 205 | "output_type": "execute_result" 206 | } 207 | ], 208 | "source": [ 209 | "'prince' in vec.get_feature_names()" 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": { 215 | "collapsed": true 216 | }, 217 | "source": [ 218 | "And what do we find to our surprise? The word 'prince' does not occur in the e-mail.\n", 219 | "\n", 220 | "Does this mean the message is legit?\n", 221 | "\n", 222 | "No, of course not. Instead of 'prince', the e-mail went with the words 'head of state' instead - effectively circumventing our all-too-simple spam detector.\n", 223 | "\n", 224 | "Luckily, the theoretical framework behind decision trees provides us with help with both finding the right decision rules as well as which decisions to tackle next." 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "\n", 232 | "< [Representing images](04.05-Representing-Images.ipynb) | [Contents](../README.md) | [Building Our First Decision Tree](05.01-Building-Our-First-Decision-Tree.ipynb) >" 233 | ] 234 | } 235 | ], 236 | "metadata": { 237 | "anaconda-cloud": {}, 238 | "kernelspec": { 239 | "display_name": "Python 3", 240 | "language": "python", 241 | "name": "python3" 242 | }, 243 | "language_info": { 244 | "codemirror_mode": { 245 | "name": "ipython", 246 | "version": 3 247 | }, 248 | "file_extension": ".py", 249 | "mimetype": "text/x-python", 250 | "name": "python", 251 | "nbconvert_exporter": "python", 252 | "pygments_lexer": "ipython3", 253 | "version": "3.5.3" 254 | } 255 | }, 256 | "nbformat": 4, 257 | "nbformat_minor": 1 258 | } 259 | -------------------------------------------------------------------------------- /notebooks/06.00-Detecting-Pedestrians-with-Support-Vector-Machines.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "\n", 11 | "\n", 12 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 13 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 14 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 15 | "\n", 16 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 17 | "If you find this content useful, please consider supporting the work by\n", 18 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "deletable": true, 25 | "editable": true 26 | }, 27 | "source": [ 28 | "\n", 29 | "< [Using Decision Trees for Regression](05.03-Using-Decision-Trees-for-Regression.ipynb) | [Contents](../README.md) | [Implementing Our First Support Vector Machine](06.01-Implementing-Your-First-Support-Vector-Machine.ipynb) >" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": { 35 | "deletable": true, 36 | "editable": true 37 | }, 38 | "source": [ 39 | "# Detecting Pedestrians with Support Vector Machines\n", 40 | "\n", 41 | "In the previous chapter, we talked about how to use decision trees for classification and\n", 42 | "regression. In this chapter, we want to direct our attention to another well established\n", 43 | "supervised learner in the machine learning world: **support vector machines (SVMs)**.\n", 44 | "\n", 45 | "Soon after their introduction in the early 1990, SVMs quickly became popular in the\n", 46 | "machine learning community, largely because of their success in early **handwritten digit\n", 47 | "classification**. They remain relevant to this day, especially in application domains, such as\n", 48 | "computer vision.\n", 49 | "\n", 50 | "The goal of this chapter is to apply SVMs to a popular problem in computer vision:\n", 51 | "**pedestrian detection**. In contrast to a recognition task (where we name the category of an\n", 52 | "object), the goal of a detection task is to say whether a particular object (or in our case, a\n", 53 | "pedestrian) is present in an image or not.\n", 54 | "\n", 55 | "You might already know that OpenCV can do this in two to three lines of code. But, we\n", 56 | "won't learn anything if we do it that way. So instead, we'll build the whole pipeline from\n", 57 | "scratch! We will obtain a real-world dataset, perform feature extraction using the **histogram\n", 58 | "of oriented gradients (HOG)**, and apply an SVM to it.\n", 59 | "\n", 60 | "Along the way, we want to address the following questions:\n", 61 | "- How do we implement SVMs in OpenCV with Python, and how do they work?\n", 62 | "- What's the deal with decision boundaries, and why are SVMs also called maximum-margin classifiers?\n", 63 | "- What is the kernel trick, and how do we put it into practice?\n", 64 | "\n", 65 | "## Outline\n", 66 | "- [Implementing Your First Support Vector Machine](06.01-Implementing-Your-First-Support-Vector-Machine.ipynb)\n", 67 | "- [Detecting Pedestrians in the Wild](06.02-Detecting-Pedestrians-in-the-Wild.ipynb)\n", 68 | "- [Additional SVM Exercises](06.03-Additional-SVM-Exercises.ipynb)\n", 69 | "\n", 70 | "Excited? Then let's go!" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": { 76 | "deletable": true, 77 | "editable": true 78 | }, 79 | "source": [ 80 | "\n", 81 | "< [Using Decision Trees for Regression](05.03-Using-Decision-Trees-for-Regression.ipynb) | [Contents](../README.md) | [Implementing Our First Support Vector Machine](06.01-Implementing-Your-First-Support-Vector-Machine.ipynb) >" 82 | ] 83 | } 84 | ], 85 | "metadata": { 86 | "kernelspec": { 87 | "display_name": "Python 3", 88 | "language": "python", 89 | "name": "python3" 90 | }, 91 | "language_info": { 92 | "codemirror_mode": { 93 | "name": "ipython", 94 | "version": 3 95 | }, 96 | "file_extension": ".py", 97 | "mimetype": "text/x-python", 98 | "name": "python", 99 | "nbconvert_exporter": "python", 100 | "pygments_lexer": "ipython3", 101 | "version": "3.5.3" 102 | } 103 | }, 104 | "nbformat": 4, 105 | "nbformat_minor": 1 106 | } 107 | -------------------------------------------------------------------------------- /notebooks/07.00-Implementing-a-Spam-Filter-with-Bayesian-Learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Detecting Pedestrians with Support Vector Machines](06.03-Additional-SVM-Exercises.ipynb) | [Contents](../README.md) | [Implementing Our First Bayesian Classifier](07.01-Implementing-Our-First-Bayesian-Classifier.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Implementing a Spam Filter with Bayesian Learning\n", 33 | "\n", 34 | "Before we get to grips with advanced topics, such as cluster analysis, deep learning, and\n", 35 | "ensemble models, let's turn our attention to a much simpler model that we have overlooked\n", 36 | "so far: the **naive Bayes classifier**.\n", 37 | "\n", 38 | "Naive Bayes classifiers have their roots in **Bayesian inference**, named after famed\n", 39 | "statistician and philosopher Thomas Bayes (1701-1761). **Bayes' theorem** famously describes\n", 40 | "the **probability** of an event based on **prior knowledge** of conditions that might lead to the\n", 41 | "event. We can use Bayes' theorem to build a statistical model that can not only classify data\n", 42 | "but also provide us with an estimate of how likely it is that our classification is correct. In\n", 43 | "our case, we can use Bayesian inference to dismiss an email as spam with high confidence,\n", 44 | "and to determine the probability of a woman having breast cancer, given a positive\n", 45 | "screening test.\n", 46 | "\n", 47 | "We have now gained enough experience with the mechanics of implementing machine\n", 48 | "learning methods, and so we should no longer be afraid to try and understand the theory\n", 49 | "behind them. Don't worry, we won't write a book on it (ahem), but we need some\n", 50 | "understanding of the theory to appreciate a model's inner workings. After that, I am sure\n", 51 | "you will find that Bayesian classifiers are easy to implement, computationally efficient, and\n", 52 | "tend to perform quite well on relatively small datasets.\n", 53 | "\n", 54 | "Along the way, we also want to address the following questions:\n", 55 | "- What makes a naive Bayes classifier so darn naive in the first place?\n", 56 | "- What is the difference between discriminative and generative models?\n", 57 | "- How do I implement Bayesian classifiers in OpenCV and how do I use them to detect fraudulent email messages?\n", 58 | "\n", 59 | "## Outline\n", 60 | "- [Implementing Our First Bayesian Classifier](07.01-Implementing-Our-First-Bayesian-Classifier.ipynb)\n", 61 | "- [Classifying E-Mails Using Naive Bayes](07.02-Classifying-Emails-Using-Naive-Bayes.ipynb)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "\n", 69 | "< [Detecting Pedestrians with Support Vector Machines](06.03-Additional-SVM-Exercises.ipynb) | [Contents](../README.md) | [Implementing Our First Bayesian Classifier](07.01-Implementing-Our-First-Bayesian-Classifier.ipynb) >" 70 | ] 71 | } 72 | ], 73 | "metadata": { 74 | "kernelspec": { 75 | "display_name": "Python 3", 76 | "language": "python", 77 | "name": "python3" 78 | }, 79 | "language_info": { 80 | "codemirror_mode": { 81 | "name": "ipython", 82 | "version": 3 83 | }, 84 | "file_extension": ".py", 85 | "mimetype": "text/x-python", 86 | "name": "python", 87 | "nbconvert_exporter": "python", 88 | "pygments_lexer": "ipython3", 89 | "version": "3.5.3" 90 | } 91 | }, 92 | "nbformat": 4, 93 | "nbformat_minor": 1 94 | } 95 | -------------------------------------------------------------------------------- /notebooks/08.00-Discovering-Hidden-Structures-with-Unsupervised-Learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Classifying Emails Using the Naive Bayes Classifier](07.02-Classifying-Emails-Using-Naive-Bayes.ipynb) | [Contents](../README.md) | [Understanding k-means clustering](08.01-Understanding-k-Means-Clustering.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Discovering Hidden Structures with Unsupervised Learning\n", 33 | "\n", 34 | "So far, we have focused our attention exclusively on supervised learning problems, where\n", 35 | "every data point in the dataset had a known label or target value. However, what do we do\n", 36 | "when there is no known output, or no teacher to supervise the learning algorithm?\n", 37 | "\n", 38 | "This is what **unsupervised learning** is all about. In unsupervised learning, the learning is\n", 39 | "shown only in the input data and is asked to extract knowledge from this data without\n", 40 | "further instruction. We have already talked about one of the many forms that unsupervised\n", 41 | "learning comes in—**dimensionality reduction**. Another popular domain is cluster analysis,\n", 42 | "which aims to partition data into distinct groups of similar items.\n", 43 | "\n", 44 | "In this chapter, we want to understand how different clustering algorithms can be used to\n", 45 | "extract hidden structures in simple, unlabeled datasets. These hidden structures have many\n", 46 | "benefits, whether they are used in feature extraction, image processing, or even as a\n", 47 | "preprocessing step for supervised learning tasks. As a concrete example, we will learn how\n", 48 | "to apply clustering to images in order to reduce their color spaces to 16 bits.\n", 49 | "\n", 50 | "More specifically, we want to answer the following questions:\n", 51 | "- What are $k$-means clustering and expectation-maximization, and how do I implement these things in OpenCV?\n", 52 | "- How can I arrange clustering algorithms in hierarchical trees, and what are the benefits that come from that?\n", 53 | "- How can I use unsupervised learning for preprocessing, image processing, and classification?\n", 54 | "\n", 55 | "\n", 56 | "## Outline\n", 57 | "\n", 58 | "- [Understanding k-Means Clustering](08.01-Understanding-k-Means-Clustering.ipynb)\n", 59 | "- [Compressing Color Images Using k-Means](08.02-Compressing-Color-Images-Using-k-Means.ipynb)\n", 60 | "- [Classifying Handwritten Digits Using k-Means](08.03-Classifying-Handwritten-Digits-Using-k-Means.ipynb)\n", 61 | "- [Implementing Agglomerative Hierarchical Clustering](08.04-Implementing-Agglomerative-Hierarchical-Clustering.ipynb)\n", 62 | "\n", 63 | "Let's get started!\n", 64 | "\n", 65 | "> The book features a detailed treatment of different unsupervised learning algorithms, including a walk-through of k-means and expectation maximization. For more information on the topic, please refer to the book." 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "\n", 73 | "< [Classifying Emails Using the Naive Bayes Classifier](07.02-Classifying-Emails-Using-Naive-Bayes.ipynb) | [Contents](../README.md) | [Understanding k-means clustering](08.01-Understanding-k-Means-Clustering.ipynb) >" 74 | ] 75 | } 76 | ], 77 | "metadata": { 78 | "kernelspec": { 79 | "display_name": "Python 3", 80 | "language": "python", 81 | "name": "python3" 82 | }, 83 | "language_info": { 84 | "codemirror_mode": { 85 | "name": "ipython", 86 | "version": 3 87 | }, 88 | "file_extension": ".py", 89 | "mimetype": "text/x-python", 90 | "name": "python", 91 | "nbconvert_exporter": "python", 92 | "pygments_lexer": "ipython3", 93 | "version": "3.5.3" 94 | } 95 | }, 96 | "nbformat": 4, 97 | "nbformat_minor": 1 98 | } 99 | -------------------------------------------------------------------------------- /notebooks/08.04-Implementing-Agglomerative-Hierarchical-Clustering.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "\n", 11 | "\n", 12 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 13 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 14 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 15 | "\n", 16 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 17 | "If you find this content useful, please consider supporting the work by\n", 18 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "deletable": true, 25 | "editable": true 26 | }, 27 | "source": [ 28 | "\n", 29 | "< [Classifying handwritten digits using k-means](08.03-Classifying-Handwritten-Digits-Using-k-Means.ipynb) | [Contents](../README.md) | [9. Using Deep Learning to Classify Handwritten Digits](09.00-Using-Deep-Learning-to-Classify-Handwritten-Digits.ipynb) >" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": {}, 35 | "source": [ 36 | "# Implementing Agglomerative Hierarchical Clustering\n", 37 | "\n", 38 | "Although OpenCV does not provide an implementation of agglomerative hierarchical\n", 39 | "clustering, it is a popular algorithm that should, by all means, belong to our machine\n", 40 | "learning repertoire.\n", 41 | "\n", 42 | "We start out by generating 10 random data points, just like in the previous figure:" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 1, 48 | "metadata": { 49 | "collapsed": true, 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "from sklearn.datasets import make_blobs\n", 56 | "X, y = make_blobs(n_samples=10, random_state=100)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "Using the familiar statistical modeling API, we import the `AgglomerativeClustering`\n", 64 | "algorithm and specify the desired number of clusters:" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 2, 70 | "metadata": { 71 | "collapsed": false, 72 | "deletable": true, 73 | "editable": true 74 | }, 75 | "outputs": [], 76 | "source": [ 77 | "from sklearn import cluster\n", 78 | "agg = cluster.AgglomerativeClustering(n_clusters=3)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "Fitting the model to the data works, as usual, via the `fit_predict` method:" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": 3, 91 | "metadata": { 92 | "collapsed": false, 93 | "deletable": true, 94 | "editable": true 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "labels = agg.fit_predict(X)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "We can generate a scatter plot where every data point is colored according to the predicted\n", 106 | "label:" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 4, 112 | "metadata": { 113 | "collapsed": false, 114 | "deletable": true, 115 | "editable": true 116 | }, 117 | "outputs": [ 118 | { 119 | "data": { 120 | "text/plain": [ 121 | "" 122 | ] 123 | }, 124 | "execution_count": 4, 125 | "metadata": {}, 126 | "output_type": "execute_result" 127 | }, 128 | { 129 | "data": { 130 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAF0CAYAAAD2C+d2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHolJREFUeJzt3XtwnPV56PFndbNkW5Ys344vGBvLhFgFA7F9EggOBk5p\nOSY1KdHh0iZk4nYCpmGStgPUSaBjXCCBUqg7zgQopAO0MQkwkJJCwuWQQDgYMMXI4eI0xoChNpIv\nsmRZlrTnD4qCq/Vdv9219PnMZJLdfaV95smM/NW+q30z2Ww2GwAAJFNS6AEAAAY6wQUAkJjgAgBI\nTHABACQmuAAAEhNcAACJlfXHN1m+fHm8+OKLUVNTEzfccENERGzfvj3+7u/+LjZt2hRjx46Nr33t\nazF06ND+eDoAgMNKv7zCNW/evFi8ePFu9z3wwANx7LHHxs033xwNDQ1x//3379f3ampq6o+RBhx7\nyc1e+rKT3OwlN3vJzV76spPc9ncv/RJcxxxzTAwbNmy3+55//vn4zGc+ExERp556aqxcuXK/vpf/\nQ3Ozl9zspS87yc1ecrOX3OylLzvJLa/BlcvWrVujtrY2IiJqa2tj27ZtqZ4KAKCoedM8AEBi/fKm\n+Vxqa2tjy5Ytvf9dU1OT87impqbdXo5rbGxMNdJhzV5ys5e+7CQ3e8nNXnKzl77sJLfGxsZYsWJF\n7+2GhoZoaGjoc1y/BVc2m42PXgf7E5/4RDz55JOxYMGCePLJJ2PWrFk5vy7XYBs2bOivsQaM6urq\naG1tLfQYRcde+rKT3OwlN3vJzV76spPcJkyYsF8x2i/BdfPNN8eaNWuitbU1Lr744mhsbIwFCxbE\nTTfdFE888USMHj06vv71r/fHUwEAHHYy2Y++LFUkvMLVl98scrOXvuwkN3vJzV5ys5e+7CS3CRMm\n7Ndx3jQPAJCY4AIASExwAQAkJrgAABITXAAAiQkuAIDEBBcAQGKCCwAgMcEFAJCY4AIASExwAQAk\nJrgAABITXAAAiQkuAIDEBBcAQGKCCwAgMcEFAJCY4AIASExwAQAkJrgAABITXAAAiQkuAIDEBBcA\nQGKCCwAgMcEFAJBYWaEHACC/Oss7o7ukO0qzJZHNDi/0ODAoCC6AQaJ9SFusH/JmPD90ZWzPbI9h\n2WExu2NOHDFkcgzbKbwgJcEFMAi0VbbFwyMeiuay5t/el2mLJ4c+ETUVtXH2tj+I4R2iC1LxHi6A\nAS6bycYrVat3i62P2lq2JV4c+kJkM9k8TwaDh+ACGODaKtri5cp/3+sxrw5ZE9uHtOZpIhh8BBfA\nANdavi26M117PaYn0xM7SnbkaSIYfAQXwAC2tWpLvF+2ab+OLc2WJp4GBi/BBTBAZTPZ+FXlr6K5\ntDnGdI3Z67G1XSOjumtEniaDwUdwAQxQHRUdsbry5Xij4vVo6Dw2Yk/vic9GnNL+majYVZHX+WAw\nEVwAA1RPpie6M13RnemOporV8ekdc6Oyp3K3Y4b0DInT2/9XjG//HwWaEgYHn8MFMEBVdJdHTVdN\nbC3bGpvKNkV7SXsc23lcVGQroid6oiRKYlfsilGdo6Kkx/u3ICXBBTBAle+qiDkdn4yfDn8kIiLa\nStpiZeVzux1T2zUyjt1+XCHGg0HFKUWAAWxix6SY2nlUzscqshXx+zvOinLv3YLkvMIFMIBVdlbG\nZ7bNi+lDjo7nqp6NLaVboizK47iOmfGxjmNiYunE2B7bCz0mDHiCC2CAq+ysjKM6p8URHZNjV2ln\nZHpKompXVUQ2IlOdKfR4MCgILoBBonxXeZTvKi/0GDAoeQ8XAEBiggsAIDHBBQCQmPdwAVCUdlbs\njO6S7ijtKY0hnUMKPQ4cEsEFQFFpG9IWb1auixeqVkZbSVsM7Rkan9gxO6bsnBrDOoYVejw4KIIL\ngKLRVrk9fjziwdhctrn3vvaS9vj5sP8bLw/59zg7/iCGdwwv4IRwcLyHC4DikIl4uerfd4utj9pa\ntiVeGvpiZDPZPA8Gh05wAVAUtg9pjVcqV+/1mDVDmmL7kNY8TQT9R3ABUBQ6SjqiO9O912N6Mj3R\nUdKRp4mg/wguAIpCabZ0v44r2c/joJgILgCKwvBd1TG2a+xej6nrGhUjuqrzNBH0H8EFQFEo7yqP\nk9tPidjTe+KzEae0z43yXRV5nQv6g+ACoGiMaRsb/7v17Kjsqdrt/sqeyjhr+/wY2z6uQJPBofE5\nXAAUjZJsSRyxfXL8n87zY3N5S3SWdEZFT0WM7KqLqp1V+/4GUKQEFwBFp6qzKqo6JxZ6DOg3TikC\nACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQE\nFwBAYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJlaV+gkWLFsXQoUMjk8lEaWlpXHvttamf\nEgCgqCQPrkwmE1dddVUMHz489VMBABSl5KcUs9lsZLPZ1E8DAFC08vIK19KlSyOTycTpp58eZ5xx\nRuqnBAAoKsmD65prrona2trYtm1bLFmyJCZNmhTHHHNM6qcFACgamWwez/fde++9UVVVFfPnz++9\nr6mpKZqamnpvNzY2Rmtra75GOmxUVFREZ2dnoccoOvbSl53kZi+52Utu9tKXneRWXV0dK1as6L3d\n0NAQDQ0NfY5LGlw7d+6MbDYblZWV0dHREUuXLo1zzz03Zs6cudev27BhQ6qRDlvV1dVCNAd76ctO\ncrOX3OwlN3vpy05ymzBhwn4dl/SU4tatW+M73/lOZDKZ6O7ujlNOOWWfsQUAMNAkDa6xY8fGd77z\nnZRPAQBQ9HzSPABAYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAx\nwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQmuAAA\nEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBAYoIL\nACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQm\nuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBA\nYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHAB\nACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkFhZ6id46aWX4s4774xsNhvz5s2LBQsWpH5K\nAICikvQVrp6enrj99ttj8eLFceONN8bTTz8d77zzTsqnBAAoOkmDa+3atTF+/PgYM2ZMlJWVxckn\nnxwrV65M+ZQAAEUnaXC1tLTEqFGjem/X1dVFS0tLyqcEACg6yd/D9d9lMpndbjc1NUVTU1Pv7cbG\nxqiurs73WEWvoqLCXnKwl77sJDd7yc1ecrOXvuxkz1asWNH7vxsaGqKhoaHPMUmDq66uLt5///3e\n2y0tLTFy5Mjdjsk1WGtra8qxDkvV1dX2koO99GUnudlLbvaSm730ZSe5VVdXR2Nj4z6PS3pKsb6+\nPt57773YtGlTdHV1xdNPPx2zZs1K+ZQAAEUn6StcJSUl8eUvfzmuueaayGazcdppp8WkSZNSPiUA\nQNFJ/h6u448/Pm6++ebUTwMAULR80jwAQGKCCwAgMcEFAJCY4AIASExwAQAkJrgAABITXAAAiQku\nAIDEBBcAQGKCCwAgMcEFAJCY4AIASExwAQAkJrgAABITXAAAiQkuAIDEBBcAQGKCCwAgMcEFAJCY\n4AIASExwAQAkJrgAABITXAAAiQkuAIDEBBcAQGKCCwAgMcEFAJCY4AIASExwAQAkJrgAABITXAAA\niQkuAIDEBBcAQGKCCwAgMcEFAJCY4AIASExwAQAkJrgAABITXAAAiQkuAIDEBBcAQGKCCwAgMcEF\nAJCY4AIASExwAQAkJrgAABITXAAAiQkuAIDEBBcAQGKCCwAgMcEFAJCY4AIASExwAQAkJrgAABIT\nXAAAiQkuAKBXJpOJkhJ50N/KCj0AAFB4HRs3RvMrr8TrP/xhdLa2xhGnnhqT5s6N6qOOikxpaaHH\nO+wJLgA4BD2dndH6m9/EtjffjIiIEUceGdVTp0ZJRUWBJ9t/29eti59cdFFseeON3vvWP/54ZEpL\n43e/97044owzCjjdwCC4AOAg7diwIZ67/vp4/Uc/ishmP7gzk4mj//APY87ll0fVhAmFHXA/7Nq2\nLR5btGi32PpQtrs7Hv2TP4nP/eu/xoiTTy7AdAOHk7QAcBB2NjfH41/9arz+wx/+NrYiIrLZeP2H\nP4zHL7ssdjY3F27A/bT1jTdi40sv7fHxbE9PvLZiRXTt2pXHqQYewQUAB2Hzq6/Ghl/+co+Pb3jm\nmdjy2mu9t3e8+26894tfxFuPPhqbnn8+dm3Zko8x92nL2rX7PObXDz0UO95/Pw/TDFxOKQLAAcpk\nMrH2/vv3edwb998fo084IdY/8kj8fPHi2PmRyKqtr4/Tbr45Rh1/fMpR+0cmU+gJDnte4QKAA5TJ\nZKJt48Z9Htfd2Rlv/exn8bNFi3aLrYgPXll68NxzY8uvfpVqzP1SW1+/z2PqP/vZGDpmTB6mGbgE\nFwAcoGw2G+PnzNnncdPOPjt+sXjxHh/v2rEjVt92W2S7uvpzvANSM316jDvxxD0+nikpiY99/vNR\nWuak2KEQXABwgLLZbEw+7bR9HpcpKYkd+3jj/Gv33hvt777bX6MdsPIRI+L0Zcti5Mc+1uexkrKy\nOPO226J2xowCTDawyFUAOAgj6uvjlGuvjZ9feWXOx0+59trd/3pxD7Ld3dHT2dnf4x2QYUceGfN/\n8INoaWqKtQ88EJ3bt8ekuXNjwkknRfXUqT74tB8ILgA4CCUVFTG9sTFG1tfHi3//9/H2U09FRMSk\nuXPjxD/7sxh94omx9SN/pbgnlXV1UT58eOpx9z3HmDEx4dRTY+K8eZHJZKKnp6fQIw0oggsADlJp\nZWWMO+mk+N0TTuj9zK0ho0ZFaVVVRESMmDYtxp5wQmxctWqP3+MTl10WlePG5WXe/ZHNZiO7H6/M\ncWC8hwsADlFpVVUMnTQphk6a1BtbERFlw4fHZ7797aiors75dWNPOCGmnnVWvsakgAQXACRUO2NG\nnPPggzHzK1+J0srKiIioGjMmTrn22vjdW289LC7/w6FzShEAEhtx9NEx+xvfiGMXLozunTujfPjw\nGDJ6dKHHIo8EFwDkQSaTiarx4ws9BgXilCIAQGKCCwAgsWSnFO+999547LHHoqamJiIizj///Dj+\ncLhAJwBAP0v6Hq758+fH/PnzUz4FAEDRS3pK0QenAQAkfoXrkUceiaeeeiqmTZsWX/jCF2Lo0KEp\nnw4AoCgdUnAtWbIktm7d2ns7m81GJpOJ8847L84888w499xzI5PJxL/8y7/E97///bj44osPeWAA\ngMNNJpuH836bNm2K66+/Pm644YY+jzU1NUVTU1Pv7cbGxmhtbU090mGnoqIiOgt8NfliZC992Ulu\n9pKbveRmL33ZSW7V1dWxYsWK3tsNDQ3R0NDQ57hkwbVly5aora2NiIgf//jH8etf/zouu+yy/fra\nDRs2pBjpsFZdXS1Ec7CXvuwkN3vJzV5ys5e+7CS3Cft5aaZk7+G66667Yt26dZHJZGLMmDHxp3/6\np6meCgCgqCULrksvvTTVtwaAwaO7O7a/9VZ0bN4cZUOGxPDJk6Ns+PBCT8UBci1FAChSbW++Gatv\nuy3W3HVXdP/X+6fGnXhifOqb34zRJ54YmTL/jB8uXNoHAIpQ2/r18a8XXBCr//Efe2MrIuI/X3wx\nHvjc5+LdX/yigNNxoAQXABShdf/2b7F13brcD2az8eSf/3ns3LQprzNx8AQXABSZnZs2xYu33LLX\nY9reey+2vPFGnibiUAkuACgyXTt2RMfmzfs8bue2bXmYhv4guACgyJRWVsaQmpp9HldRXZ2HaegP\nggsAikzVuHFx/KJFez9mzJionT49TxNxqAQXABSZbDYb084+O4ZPmpT7gEwm5t14Y1SOHZvfwTho\ngouisXlzZ6xcuSkefvjNeOyxd2L9+rZIf6VPgOI0bPLk+OwPfhDHnHdeZEpLe+8fNWNGfHbFihg/\nd24Bp+NA+cQ0isLLL7fEokU/i//4j62991VVlcUVV/zP+Pznp0dNTXkBpwMojGFTpsSnr78+jr/k\nkujYvDlKhwyJ6iOPjPIRIwo9GgdIcFFwr722Nc4998Foa9u12/07dnTFVVc9HRERX/7yjMhkCjEd\nQGFlysqietq08Pb4w5tTihRUNpuNFSte6xNbH3Xddf8v1q9vy+NUANC/BBcF9Z//uTPuvPOVvR6z\nY0dX/PrXW/I0EQD0P8FFQXV19URHR/c+j9u5c9/HAECxElwUVHV1eRx11L4/3G/MmKo8TAMAaQgu\nCqqmpjy+/vVZez2mvr426utr8zQRAPQ/wUXBnXLKxDjzzCk5Hxs+vDyWLTsjamt9LAQAhy/BRcGN\nHj0kvv3tufEP/3BGTJv2wenFqqqyWLTohHjooc/FsceOLPCEAHBofA4XRWH06CGxYMHUmDdvUmzf\nvivKykpi7NhKn70FwIAguCgqNTXlPlUegAHHKUUAgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQm\nuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBA\nYoILACAxwQUAkJjgAgBITHABACQmuAAAEhNcAACJCS4AgMQEFwBAYoILACAxwQUAkFhZoQcgors7\nG+vXb49XX90czc0dMW5cVXzsY3VxxBHDIpMp9HQAwKESXAXW1tYVP/rR2rjqqmeis7O79/6hQ8vi\nhhtOjd///SOjosILkQBwOPMveYE98cQ7ceWVP98ttiIi2tu74pJLfhbPPbexQJMBAP1FcBXQpk07\n4+qrn97rMddc88tobe3K00QAQAqCK896eiJaWjpj8+bO2LixPd59t22vx69e/X68/fb2PE0HAKTg\nPVx5smtXNn71q81x772vx8MP/0eUlGTis5+tj2uuOSVuu+3lWLdu6x6/tqurJ4+TAgD9TXDlQVdX\nTzzyyFvxla88Gtnsb+//7ndfirKykrj66pNj+fJV8c47fV/Jqq6uiLq6yjxOCwD0N6cU82Dt2ta4\n+OKf7hZbH+rq6oklS56JL33p2Jxfu2jR8TFp0rDEEwIAKQmuxDKZTDz++Pro6clRW/9l587uaG/f\nFdXVFbvdf9JJE+Lznz86srlKDQA4bDilmFhPTzYee2z9Po/7zW+2xrx5R8RTT70dRx45IhYtOiFm\nzx4XY8c6nQgAhzvBlVhJSSaGDdv3mmtqhsTixf8z2tt3RVVVWQwbVpqH6QCAfHBKMbFsNhvnnffx\nfR73e783JYYOLYnRo4eILQAYYARXHpxwwpiYNGn4Hh9vaBgVH/94XR4nAgDySXDlwfjxVXHPPfPj\n6KP7RtWsWePi1lvPjNGjhxRgMgAgH7yHK0+mTauO++47O954Y0u8/vrmiMjExz9eF/X1NVFTU17o\n8QCAhARXHo0cWRFz5oyNOXPGFnoUACCPnFIEAEhMcAEAJCa4AAASE1wAAIkJLgCAxAQXAEBiggsA\nIDHBBQCQmOACAEhMcAEAJCa4AAASO6RrKT777LNx7733xttvvx3XXnttHHXUUb2P3X///fHEE09E\naWlpXHTRRTFz5sxDHhYA4HB0SK9wTZ48Of7iL/4iZsyYsdv9b7/9dvzyl7+Mm266Ka688sq47bbb\nIpvNHtKgAACHq0MKrgkTJsT48eP73P/888/HSSedFKWlpTF27NgYP358rF279lCeCgDgsJXkPVwt\nLS0xevTo3tt1dXXR0tKS4qkAAIrePt/DtWTJkti6dWvv7Ww2G5lMJs4777yYNWtWzq/Jdfowk8kc\nwpgAAIevfQbXN7/5zQP+pqNGjYr333+/93Zzc3OMHDky57FNTU3R1NTUe7uxsTEmTJhwwM85GFRX\nVxd6hKJkL33ZSW72kpu95GYvfdlJbitWrOj93w0NDdHQ0NDnmCSnFGfNmhXPPPNMdHV1xcaNG+O9\n996L+vr6nMc2NDREY2Nj738+OjS/ZS+52UtfdpKbveRmL7nZS192ktuKFSt265hcsRVxiB8L8dxz\nz8Udd9wR27Zti+uuuy6mTJkSf/VXfxWTJk2KT33qU/G1r30tysrKYuHChU4pAgCD1iEF15w5c2LO\nnDk5HzvnnHPinHPOOZRvDwAwIJReffXVVxd6iP9u7NixhR6hKNlLbvbSl53kZi+52Utu9tKXneS2\nP3vJZH0iKQBAUq6lCACQmOACAEjskN4035/2dCHsl19+Oe65557o7u6OsrKyuPDCC+N3fud3Cjxt\n/rhA+N6tW7cubr311ti1a1eUlpbGwoULY9q0aYUeqyj85Cc/iUceeSRKS0vjxBNPjAsvvLDQIxWN\nBx98MO6+++64/fbbY/jw4YUep+DuuuuueOGFF6KsrCzGjRsXl1xySQwdOrTQYxXESy+9FHfeeWdk\ns9mYN29eLFiwoNAjFVxzc3MsW7YstmzZEiUlJXH66afHWWedVeixikZPT09ceeWVUVdXF5dffvke\njyua4PrwQtjf+973drt/xIgRccUVV0RtbW289dZbsXTp0vjud79boCnzb097+egFwpubm2PJkiVx\nyy23DLqP37j77rujsbExZs6cGatWrYq77rorrrrqqkKPVXBNTU3xwgsvxI033hilpaWxbdu2Qo9U\nNJqbm2P16tW7XX5ssDvuuOPiggsuiJKSkrj77rvjgQceiAsuuKDQY+VdT09P3H777fGtb30rRo4c\nGVdeeWXMnj07Jk6cWOjRCqq0tDS++MUvxpQpU6KjoyMuv/zymDlz5qDfy4cefvjhmDhxYuzYsWOv\nxxXNKcU9XQh7ypQpUVtbGxERRxxxROzatSu6urryPV7BuED43mUymWhvb4+IiLa2tj1e0WCwefTR\nR2PBggVRWloaER/84sIHvv/978cf//EfF3qMonLcccdFSckH/xxMnz49mpubCzxRYaxduzbGjx8f\nY8aMibKysjj55JNj5cqVhR6r4Gpra2PKlCkREVFZWRkTJ050feT/0tzcHKtWrYrTTz99n8cWzStc\n++PZZ5+NqVOnRlnZYTV2Ei0tLXH00Uf33h6sFwj/4he/GEuXLo1/+qd/iogPrv1JxLvvvhtr1qyJ\nf/7nf46Kior4oz/6I6da44NfVEaNGhWTJ08u9ChF64knnoiTTz650GMUREtLS4waNar3dl1d3aD8\nRXZvNm7cGG+++WZMnz690KMUhQ9/gfvwF/+9yWu5HMyFsD/01ltvxT333BPf+MY3Uo+Zdy4Qvnd7\n28/q1avjoosuijlz5sSzzz4by5cvP6jrfx6O9raX7u7uaG9vj6VLl8batWvjpptuimXLlhVw2vzZ\n217uv//+3X6GDKZPxdmfnzP33XdflJaWxqc//elCjVl0BurP1YPR0dERf/u3fxsXXXRRVFZWFnqc\ngnvxxRejpqYmpkyZEk1NTfv8eZLX4DrYfwibm5vjhhtuiEsvvXRAfuha6guEH+72tp9ly5bFl770\npYiI+OQnPxnLly/P11gFt7e9/PSnP+29CkR9fX1kMplobW0dFBee3dNe1q9fHxs3boy//Mu/jGw2\nGy0tLXHFFVfE3/zN30RNTU2ep8y/ff2cefLJJ2PVqlXxrW99K08TFZ+6urrdfq62tLQM2J+rB6q7\nuztuvPHGmDt3bsyePbvQ4xSFV199NZ5//vlYtWpVdHZ2xo4dO2LZsmVx6aWX5jy+6M/Ntbe3x3XX\nXRcXXnjhbqfQBrtZs2bFLbfcEvPnz4+Wlpa9XiB8IKurq4s1a9bEjBkzYvXq1TFhwoRCj1QUZs+e\nHa+88krMmDEjNmzYEN3d3YMitvZm8uTJceutt/beXrRoUVx//fX+SjE++Mu8Bx98MP76r/86ysvL\nCz1OwdTX18d7770XmzZtipEjR8bTTz8dl112WaHHKgrLly+PSZMm+evEj7jgggt6/7hkzZo18dBD\nD+0xtiKK6JPmP3oh7GHDhvVeCPu+++6LBx54IMaPH9/7EvjixYsHzZuA97SXiA8+FuLxxx+PsrKy\nQfuxEK+99lrccccd0dPTE+Xl5bFw4cKYOnVqoccquK6urli+fHmsW7cuysvL4wtf+ELMmDGj0GMV\nlUsvvTSuu+46wRURX/3qV6Orq6s3yqdPnx4LFy4s8FSF8dJLL8Udd9wR2Ww2TjvtNB8LER+8knPV\nVVfF5MmTI5PJRCaTifPPPz+OP/74Qo9WND4Mrr19LETRBBcAwEBVNB8LAQAwUAkuAIDEBBcAQGKC\nCwAgMcEFAJCY4AIASExwAQAkJrgAABL7/243JgHeVd8jAAAAAElFTkSuQmCC\n", 131 | "text/plain": [ 132 | "" 133 | ] 134 | }, 135 | "metadata": {}, 136 | "output_type": "display_data" 137 | } 138 | ], 139 | "source": [ 140 | "import matplotlib.pyplot as plt\n", 141 | "%matplotlib inline\n", 142 | "plt.style.use('ggplot')\n", 143 | "plt.figure(figsize=(10, 6))\n", 144 | "plt.scatter(X[:, 0], X[:, 1], c=labels, s=100)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "That's it! This marks the end of another wonderful adventure." 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": { 157 | "deletable": true, 158 | "editable": true 159 | }, 160 | "source": [ 161 | "\n", 162 | "< [Classifying handwritten digits using k-means](08.03-Classifying-Handwritten-Digits-Using-k-Means.ipynb) | [Contents](../README.md) | [9. Using Deep Learning to Classify Handwritten Digits](09.00-Using-Deep-Learning-to-Classify-Handwritten-Digits.ipynb) >" 163 | ] 164 | } 165 | ], 166 | "metadata": { 167 | "kernelspec": { 168 | "display_name": "Python 3", 169 | "language": "python", 170 | "name": "python3" 171 | }, 172 | "language_info": { 173 | "codemirror_mode": { 174 | "name": "ipython", 175 | "version": 3 176 | }, 177 | "file_extension": ".py", 178 | "mimetype": "text/x-python", 179 | "name": "python", 180 | "nbconvert_exporter": "python", 181 | "pygments_lexer": "ipython3", 182 | "version": "3.5.3" 183 | } 184 | }, 185 | "nbformat": 4, 186 | "nbformat_minor": 2 187 | } 188 | -------------------------------------------------------------------------------- /notebooks/09.00-Using-Deep-Learning-to-Classify-Handwritten-Digits.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Implementing Agglomerative Hierarchical Clustering](08.04-Implementing-Agglomerative-Hierarchical-Clustering.ipynb) | [Contents](../README.md) | [Understanding Perceptrons](09.01-Understanding-perceptrons.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Using Deep Learning to Classify Handwritten Digits\n", 33 | "\n", 34 | "In this chapter, we want to wrap our heads around some simple versions of artificial neural\n", 35 | "nets, such as the McCulloch-Pitts neuron, the [perceptron](09.01-Understanding-Perceptrons.ipynb),\n", 36 | "and the [multi-layer perceptron](09.02-Implementing-a-Multi-Layer-Perceptron-in-OpenCV.ipynb).\n", 37 | "Once we have familiarized ourselves with the basics, we will be ready to implement a more\n", 38 | "[sophisticated deep neural net](09.03-Getting-Acquainted-with-Deep-Learning.ipynb)\n", 39 | "in order to classify handwritten digits from the popular\n", 40 | "[MNIST database](09.04-Classifying-Handwritten-Digits.ipynb)\n", 41 | "(short for Mixed National Institute of Standards and Technology\n", 42 | "database). For this, we will be making use of [Keras](09.05-Training-a-Deep-Neural-Net-Using-Keras.ipynb), \n", 43 | "a high-level neural network library,\n", 44 | "which is also frequently used by researchers and tech companies.\n", 45 | "\n", 46 | "Along the way, we want to get answers to the following questions:\n", 47 | "- How do I implement perceptrons and multilayer perceptrons in OpenCV?\n", 48 | "- What is the difference between stochastic and batch gradient descent, and how does it fit in with backpropagation?\n", 49 | "- How do I know what size my neural net should be?\n", 50 | "- How can I use Keras to build sophisticated deep neural networks?\n", 51 | "\n", 52 | "\n", 53 | "## Outline\n", 54 | "\n", 55 | "- [Understanding Perceptrons](09.01-Understanding-Perceptrons.ipynb)\n", 56 | "- [Implementing a Multi-Layer Perceptron in OpenCV](09.02-Implementing-a-Multi-Layer-Perceptron-in-OpenCV.ipynb)\n", 57 | "- [Getting Acquainted with Deep Learning](09.03-Getting-Acquainted-with-Deep-Learning.ipynb)\n", 58 | "- [Training an MLP in OpenCV to Classify Handwritten Digits](09.04-Training-an-MLP-in-OpenCV-to-Classify-Handwritten-Digits)\n", 59 | "- [Training a Deep Neural Net to Classify Handwritten Digits Using Keras](09.05-Training-a-Deep-Neural-Net-to-Classify-Handwritten-Digits-Using-Keras)\n", 60 | "\n", 61 | "> The book offers a detailed treatment of the McCulloch-Pitts neuron, the perceptron, multi-layer perceptrons, and backrpopagation. For more information on the same, please refer to the book.\n", 62 | "\n", 63 | "\n", 64 | "Excited? Then let's go!" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "\n", 72 | "< [Implementing Agglomerative Hierarchical Clustering](08.04-Implementing-Agglomerative-Hierarchical-Clustering.ipynb) | [Contents](../README.md) | [Understanding Perceptrons](09.01-Understanding-perceptrons.ipynb) >" 73 | ] 74 | } 75 | ], 76 | "metadata": { 77 | "kernelspec": { 78 | "display_name": "Python 3", 79 | "language": "python", 80 | "name": "python3" 81 | }, 82 | "language_info": { 83 | "codemirror_mode": { 84 | "name": "ipython", 85 | "version": 3 86 | }, 87 | "file_extension": ".py", 88 | "mimetype": "text/x-python", 89 | "name": "python", 90 | "nbconvert_exporter": "python", 91 | "pygments_lexer": "ipython3", 92 | "version": "3.5.3" 93 | } 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 1 97 | } 98 | -------------------------------------------------------------------------------- /notebooks/09.03-Getting-Acquainted-with-Deep-Learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Implementing a Multi-Layer Perceptron (MLP) in OpenCV](09.02-Implementing-a-Multi-Layer-Perceptron-in-OpenCV.ipynb) | [Contents](../README.md) | [Training an MLP in OpenCV to Classify Handwritten Digits](09.04-Training-an-MLP-in-OpenCV-to-Classify-Handwritten-Digits.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "https://github.com/fchollet/keras/blob/master/examples/mnist_cnn.py" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "# Getting Acquainted with Deep Learning\n", 40 | "\n", 41 | "Back when deep learning didn't have a fancy name yet, it was called artificial neural\n", 42 | "networks. So you already know a great deal about it! This was a respected field in itself, but\n", 43 | "after the days of Rosenblatt's perceptron, many researchers and machine learning\n", 44 | "practitioners slowly began to lose interest in the field since no one had a good solution for\n", 45 | "training a neural network with multiple layers.\n", 46 | "\n", 47 | "With the current popularity of deep learning in both industry and academia, we are\n", 48 | "fortunate enough to have a whole range of open-source deep learning frameworks at our\n", 49 | "disposal:\n", 50 | "- **Google Brain's [TensorFlow](http://www.tensorflow.org)**: This is a machine learning library that describes computations as dataflow graphs. To date, this is one of the most commonly used deep learning libraries. Hence, it is also evolving quickly, so you might have to check back often for software updates. TensorFlow provides a whole range of user interfaces, including Python, C++, and Java interface.\n", 51 | "- **Microsoft Research's [Cognitive Toolkit (CNTK)](https://www.microsoft.com/en-us/research/product/cognitive-toolkit)**: This is a deep learning framework that describes neural networks as a series of computational steps via a directed graph.\n", 52 | "- UC Berkeley's [Caffe](http://caffe.berkeleyvision.org): This is a pure deep learning framework written in C++, with an additional Python interface.\n", 53 | "- **University of Montreal's [Theano](http://deeplearning.net/software/theano)**: This is a numerical computation library compiled to run efficiently on CPU and GPU architectures. Theano is more than a machine learning library; it can express any computation using a specialized computer algebra system. Hence, it is best suited for people who wish to write their machine learning algorithms from scratch.\n", 54 | "- **[Torch](http://www.torch.ch)**: This is a scientific computing framework based on the Lua programming language. Like Theano, Torch is more than a machine learning library, but it is heavily used for deep learning by companies such as Facebook, IBM, and Yandex.\n", 55 | "\n", 56 | "Finally, there is also [Keras](http://keras.io), which we will be using in the following sections. In contrast to\n", 57 | "the preceding frameworks, Keras understands itself as an interface rather than an end-toend\n", 58 | "deep learning framework. It allows you to specify deep neural nets using an easy-tounderstand\n", 59 | "API, which can then be run on backends, such as TensorFlow, CNTK, or\n", 60 | "Theano." 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "## Getting acquainted with Keras\n", 68 | "\n", 69 | "The core data structure of Keras is a model, which is similar to OpenCV's classifier object,\n", 70 | "except it focuses on neural networks only. The simplest type of model is the `Sequential`\n", 71 | "model, which arranges the different layers of the neural net in a linear stack, just like we did\n", 72 | "for the MLP in OpenCV:" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 1, 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "name": "stderr", 82 | "output_type": "stream", 83 | "text": [ 84 | "Using Theano backend.\n" 85 | ] 86 | } 87 | ], 88 | "source": [ 89 | "from keras.models import Sequential\n", 90 | "model = Sequential()" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "Then different layers can be added to the model one by one. In Keras, layers do not just\n", 98 | "contain neurons, they also perform a function. Some core layer types include the following:\n", 99 | "\n", 100 | "- `Dense`: This is a densely connected layer. This is exactly what we used when we designed our MLP: a layer of neurons that is connected to every neuron in the previous layer.\n", 101 | "- `Activation`: This applies an activation function to an output. Keras provides a whole range of activation functions, including OpenCV's identify function (`linear`), the hyperbolic tangent (`tanh`), a sigmoidal squashing function (`sigmoid`), a softmax function (`softmax`), and many more.\n", 102 | "- `Reshape`: This reshapes an output to a certain shape.\n", 103 | "\n", 104 | "There are other layers that calculate arithmetic or geometric operations on their inputs:\n", 105 | "- **Convolutional layers**: These layers allow you to specify a kernel with which the input layer is convolved. This allows you to perform operations such as a Sobel filter or apply a Gaussian kernel in 1D, 2D, or even 3D.\n", 106 | "- **Pooling layers**: These layers perform a max pooling operation on their input, where the output neuron's activity is given by the maximally active input neuron.\n", 107 | "\n", 108 | "Some other layers that are popular in deep learning are as follows:\n", 109 | "- `Dropout`: This layer randomly sets a fraction of input units to zero at each update. This is a way to inject noise into the training process, making it more robust.\n", 110 | "- `Embedding`: This layer encodes categorical data, similar to some functions from scikit-learn's `preprocessing` module.\n", 111 | "- `GaussianNoise`: This layer applies additive zero-centered Gaussian noise. This is another way of injecting noise into the training process, making it more robust." 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "A perceptron similar to the preceding one could thus be implemented using a\n", 119 | "`Dense` layer that has two inputs and one output. Staying true to our earlier\n", 120 | "example, we will initialize the weights to zero and use the hyperbolic tangent as\n", 121 | "an activation function:" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 2, 127 | "metadata": { 128 | "collapsed": true 129 | }, 130 | "outputs": [], 131 | "source": [ 132 | "from keras.layers import Dense\n", 133 | "model.add(Dense(1, activation='linear', input_dim=2, kernel_initializer='zeros'))" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "Finally, we want to specify the training method. Keras provides a number of optimizers,\n", 141 | "including the following:\n", 142 | "- **stochastic gradient descent** (`'sgd'`): This is what we have discussed before\n", 143 | "- **root mean square propagation** (`'RMSprop'`): This is a method in which the\n", 144 | "learning rate is adapted for each of the parameters\n", 145 | "- **adaptive moment estimation** (`'Adam'`): This is an update to root mean square propagation and many more\n", 146 | "\n", 147 | "In addition, Keras also provides a number of different loss functions:\n", 148 | "- **mean squared error** (`'mean_squared_error'`): This is what was discussed before\n", 149 | "- **hinge loss** (`'hinge'`): This is a maximum-margin classifier often used with SVM, as discussed in [Chapter 6](06.00-Detecting-Pedestrians-with-Support-Vector-Machines.ipynb), *Detecting Pedestrians with Support Vector Machines*, and many more" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "You can see that there's a whole plethora of parameters to be specified and methods to\n", 157 | "choose from. To stay true to our aforementioned perceptron implementation, we will\n", 158 | "choose stochastic gradient descent as an optimizer, the mean squared error as a cost\n", 159 | "function, and accuracy as a scoring function:" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 3, 165 | "metadata": { 166 | "collapsed": true 167 | }, 168 | "outputs": [], 169 | "source": [ 170 | "model.compile(optimizer='sgd',\n", 171 | " loss='mean_squared_error',\n", 172 | " metrics=['accuracy'])" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "In order to compare the performance of the Keras implementation to our home-brewed\n", 180 | "version, we will apply the classifier to the same dataset:" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 4, 186 | "metadata": { 187 | "collapsed": true 188 | }, 189 | "outputs": [], 190 | "source": [ 191 | "from sklearn.datasets.samples_generator import make_blobs\n", 192 | "X, y = make_blobs(n_samples=100, centers=2, cluster_std=2.2, random_state=42)" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "metadata": {}, 198 | "source": [ 199 | "Finally, a Keras model is fit to the data with a very familiar syntax. Here, we can also choose\n", 200 | "how many iterations to train for (`epochs`), how many samples to present before we\n", 201 | "calculate the error gradient (`batch_size`), whether to shuffle the dataset (`shuffle`), and\n", 202 | "whether to output progress updates (`verbose`):" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 5, 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "data": { 212 | "text/plain": [ 213 | "" 214 | ] 215 | }, 216 | "execution_count": 5, 217 | "metadata": {}, 218 | "output_type": "execute_result" 219 | } 220 | ], 221 | "source": [ 222 | "model.fit(X, y, epochs=400, batch_size=100, shuffle=False, verbose=0)" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "metadata": {}, 228 | "source": [ 229 | "After the training completes, we can evaluate the classifier as follows:" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 6, 235 | "metadata": {}, 236 | "outputs": [ 237 | { 238 | "name": "stdout", 239 | "output_type": "stream", 240 | "text": [ 241 | "\r", 242 | " 32/100 [========>.....................] - ETA: 0s" 243 | ] 244 | }, 245 | { 246 | "data": { 247 | "text/plain": [ 248 | "[0.040941802412271501, 1.0]" 249 | ] 250 | }, 251 | "execution_count": 6, 252 | "metadata": {}, 253 | "output_type": "execute_result" 254 | } 255 | ], 256 | "source": [ 257 | "model.evaluate(X, y)" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "metadata": {}, 263 | "source": [ 264 | "Here, the first reported value is the mean squared error, whereas the second value denotes\n", 265 | "accuracy. This means that the final mean squared error was 0.04, and we had 100%\n", 266 | "accuracy. Way better than our own implementation!" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 7, 272 | "metadata": { 273 | "collapsed": true 274 | }, 275 | "outputs": [], 276 | "source": [ 277 | "import numpy as np\n", 278 | "np.random.seed(1337) # for reproducibility" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "With these tools in hand, we are now ready to approach a real-world dataset!" 286 | ] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": {}, 291 | "source": [ 292 | "\n", 293 | "< [Implementing a Multi-Layer Perceptron (MLP) in OpenCV](09.02-Implementing-a-Multi-Layer-Perceptron-in-OpenCV.ipynb) | [Contents](../README.md) | [Training an MLP in OpenCV to Classify Handwritten Digits](09.04-Training-an-MLP-in-OpenCV-to-Classify-Handwritten-Digits.ipynb) >" 294 | ] 295 | } 296 | ], 297 | "metadata": { 298 | "kernelspec": { 299 | "display_name": "Python 3", 300 | "language": "python", 301 | "name": "python3" 302 | }, 303 | "language_info": { 304 | "codemirror_mode": { 305 | "name": "ipython", 306 | "version": 3 307 | }, 308 | "file_extension": ".py", 309 | "mimetype": "text/x-python", 310 | "name": "python", 311 | "nbconvert_exporter": "python", 312 | "pygments_lexer": "ipython3", 313 | "version": "3.5.3" 314 | } 315 | }, 316 | "nbformat": 4, 317 | "nbformat_minor": 1 318 | } 319 | -------------------------------------------------------------------------------- /notebooks/09.05-Training-a-Deep-Neural-Net-to-Classify-Handwritten-Digits-Using-Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Training an MLP in OpenCV to Classify Handwritten Digits](09.04-Training-an-MLP-in-OpenCV-to-Classify-Handwritten-Digits.ipynb) | [Contents](../README.md) | [Combining Different Algorithms Into an Ensemble](10.00-Combining-Different-Algorithms-Into-an-Ensemble.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "https://github.com/fchollet/keras/blob/master/examples/mnist_cnn.py" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "# Training a Deep Neural Net to Classify Handwritten Digits Using Keras\n", 40 | "\n", 41 | "Although we achieved a formidable score with the MLP above, our result does not hold up\n", 42 | "to state-of-the-art results. Currently the best result has close to 99.8% accuracy—better than\n", 43 | "human performance! This is why nowadays, the task of classifying handwritten digits is\n", 44 | "largely regarded as solved.\n", 45 | "\n", 46 | "To get closer to the state-of-the-art results, we need to use state-of-the-art techniques. Thus,\n", 47 | "we return to Keras." 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "## Preprocessing the MNIST dataset\n", 55 | "\n", 56 | "To make sure we get the same result every time we run the experiment, we will pick a\n", 57 | "random seed for NumPy's random number generator. This way, shuffling the training\n", 58 | "samples from the MNIST dataset will always result in the same order:" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 1, 64 | "metadata": { 65 | "collapsed": true 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "import numpy as np\n", 70 | "np.random.seed(1337) # for reproducibility" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "Keras provides a loading function similar to train_test_split from scikit-learn's\n", 78 | "`model_selection` module. Its syntax might look strangely familiar to you:" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 2, 84 | "metadata": {}, 85 | "outputs": [ 86 | { 87 | "name": "stderr", 88 | "output_type": "stream", 89 | "text": [ 90 | "Using Theano backend.\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "from keras.datasets import mnist\n", 96 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "The neural nets in Keras act on the feature matrix slightly differently than the standard\n", 104 | "OpenCV and scikit-learn estimators. Whereas the rows of a feature matrix in Keras still\n", 105 | "correspond to the number of samples (`X_train.shape[0]` in the code below), we can\n", 106 | "preserve the two-dimensional nature of the input images by adding more dimensions to the\n", 107 | "feature matrix:" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 3, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "img_rows, img_cols = 28, 28\n", 117 | "X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n", 118 | "X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n", 119 | "input_shape = (img_rows, img_cols, 1)\n", 120 | " \n", 121 | "X_train = X_train.astype('float32') / 255.0\n", 122 | "X_test = X_test.astype('float32') / 255.0" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "Here we have reshaped the feature matrix into a four-dimensional matrix with dimensions\n", 130 | "`n_features x 28 x 28 x 1`.\n", 131 | "We also made sure we operate on 32-bit floating\n", 132 | "point numbers between [0, 1], rather than unsigned integers in [0, 255]." 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "Then, we can one-hot encode the training labels like we did before. This will make sure each\n", 140 | "category of target labels can be assigned to a neuron in the output layer. We could do this\n", 141 | "with scikit-learn's `preprocessing`, but in this case it is easier to use Keras' own utility\n", 142 | "function:" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 4, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "from keras.utils import np_utils\n", 152 | "n_classes = 10\n", 153 | "Y_train = np_utils.to_categorical(y_train, n_classes)\n", 154 | "Y_test = np_utils.to_categorical(y_test, n_classes)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "## Creating a convolutional neural network\n", 162 | "\n", 163 | "Once we have preprocessed the data, it is time to define the actual model. Here, we will\n", 164 | "once again rely on the `Sequential` model to define a feedforward neural network:" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 5, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "from keras.models import Sequential\n", 174 | "model = Sequential()" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "However, this time, we will be smarter about the individual layers. We will design our\n", 182 | "neural network around a **convolutional layer**, where the kernel is a 3 x 3 pixel two-dimensional\n", 183 | "convolution.\n", 184 | "\n", 185 | "A two-dimensional convolutional layer operates akin to image filtering in\n", 186 | "OpenCV, where each image in the input data is convolved with a small\n", 187 | "two-dimensional kernel. In Keras, we can specify the kernel size and the\n", 188 | "stride:" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 6, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "from keras.layers import Conv2D\n", 198 | "n_filters = 32\n", 199 | "kernel_size = (3, 3)\n", 200 | "model.add(Conv2D(n_filters, (kernel_size[0], kernel_size[1]),\n", 201 | " padding='valid',\n", 202 | " input_shape=input_shape))" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "After that, we will use a linear rectified unit as an activation function:" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 7, 215 | "metadata": { 216 | "collapsed": true 217 | }, 218 | "outputs": [], 219 | "source": [ 220 | "from keras.layers import Activation\n", 221 | "model.add(Activation('relu'))" 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "metadata": {}, 227 | "source": [ 228 | "In a deep convolutional neural net, we can have as many layers as we want. A popular\n", 229 | "version of this structure applied to MNIST involves performing the convolution and\n", 230 | "rectification twice:" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": 8, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "model.add(Conv2D(n_filters, (kernel_size[0], kernel_size[1])))\n", 240 | "model.add(Activation('relu'))" 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": {}, 246 | "source": [ 247 | "Finally, we will pool the activations and add a `Dropout` layer:" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 9, 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [ 256 | "from keras.layers import MaxPooling2D, Dropout\n", 257 | "pool_size = (2, 2)\n", 258 | "model.add(MaxPooling2D(pool_size=pool_size))\n", 259 | "model.add(Dropout(0.25))" 260 | ] 261 | }, 262 | { 263 | "cell_type": "markdown", 264 | "metadata": {}, 265 | "source": [ 266 | "Then we will flatten the model and finally pass it through a `softmax` function to arrive at\n", 267 | "the output layer:" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 10, 273 | "metadata": {}, 274 | "outputs": [], 275 | "source": [ 276 | "from keras.layers import Flatten, Dense\n", 277 | "model.add(Flatten())\n", 278 | "model.add(Dense(128))\n", 279 | "model.add(Activation('relu'))\n", 280 | "model.add(Dropout(0.5))\n", 281 | "model.add(Dense(n_classes))\n", 282 | "model.add(Activation('softmax'))" 283 | ] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "metadata": {}, 288 | "source": [ 289 | "Here, we will use the cross-entropy loss and the Adadelta algorithm:" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": 11, 295 | "metadata": { 296 | "collapsed": true 297 | }, 298 | "outputs": [], 299 | "source": [ 300 | "model.compile(loss='categorical_crossentropy',\n", 301 | " optimizer='adadelta',\n", 302 | " metrics=['accuracy'])" 303 | ] 304 | }, 305 | { 306 | "cell_type": "markdown", 307 | "metadata": {}, 308 | "source": [ 309 | "## Fitting the model\n", 310 | "\n", 311 | "We fit the model like we do with all other classifiers:\n", 312 | "\n", 313 | "> Caution! This might take several hours depending on your machine." 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 12, 319 | "metadata": {}, 320 | "outputs": [ 321 | { 322 | "name": "stdout", 323 | "output_type": "stream", 324 | "text": [ 325 | "Train on 60000 samples, validate on 10000 samples\n", 326 | "Epoch 1/12\n", 327 | "60000/60000 [==============================] - 88s - loss: 0.3501 - acc: 0.8929 - val_loss: 0.0887 - val_acc: 0.9714\n", 328 | "Epoch 2/12\n", 329 | "60000/60000 [==============================] - 87s - loss: 0.1263 - acc: 0.9618 - val_loss: 0.0596 - val_acc: 0.9803\n", 330 | "Epoch 3/12\n", 331 | "60000/60000 [==============================] - 87s - loss: 0.0999 - acc: 0.9705 - val_loss: 0.0475 - val_acc: 0.9844\n", 332 | "Epoch 4/12\n", 333 | "60000/60000 [==============================] - 88s - loss: 0.0839 - acc: 0.9752 - val_loss: 0.0416 - val_acc: 0.9861\n", 334 | "Epoch 5/12\n", 335 | "60000/60000 [==============================] - 87s - loss: 0.0742 - acc: 0.9780 - val_loss: 0.0384 - val_acc: 0.9876\n", 336 | "Epoch 6/12\n", 337 | "60000/60000 [==============================] - 87s - loss: 0.0661 - acc: 0.9802 - val_loss: 0.0366 - val_acc: 0.9876\n", 338 | "Epoch 7/12\n", 339 | "60000/60000 [==============================] - 87s - loss: 0.0603 - acc: 0.9814 - val_loss: 0.0378 - val_acc: 0.9869\n", 340 | "Epoch 8/12\n", 341 | "60000/60000 [==============================] - 87s - loss: 0.0580 - acc: 0.9825 - val_loss: 0.0351 - val_acc: 0.9880\n", 342 | "Epoch 9/12\n", 343 | "60000/60000 [==============================] - 87s - loss: 0.0535 - acc: 0.9830 - val_loss: 0.0325 - val_acc: 0.9881\n", 344 | "Epoch 10/12\n", 345 | "60000/60000 [==============================] - 87s - loss: 0.0518 - acc: 0.9845 - val_loss: 0.0305 - val_acc: 0.9893\n", 346 | "Epoch 11/12\n", 347 | "60000/60000 [==============================] - 87s - loss: 0.0490 - acc: 0.9854 - val_loss: 0.0308 - val_acc: 0.9899\n", 348 | "Epoch 12/12\n", 349 | "60000/60000 [==============================] - 87s - loss: 0.0459 - acc: 0.9862 - val_loss: 0.0301 - val_acc: 0.9898\n" 350 | ] 351 | }, 352 | { 353 | "data": { 354 | "text/plain": [ 355 | "" 356 | ] 357 | }, 358 | "execution_count": 12, 359 | "metadata": {}, 360 | "output_type": "execute_result" 361 | } 362 | ], 363 | "source": [ 364 | "model.fit(X_train, Y_train, batch_size=128, epochs=12,\n", 365 | " verbose=1, validation_data=(X_test, Y_test))" 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": {}, 371 | "source": [ 372 | "After training completes, we can evaluate the classifier:" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 13, 378 | "metadata": {}, 379 | "outputs": [ 380 | { 381 | "name": "stdout", 382 | "output_type": "stream", 383 | "text": [ 384 | " 9920/10000 [============================>.] - ETA: 0s" 385 | ] 386 | }, 387 | { 388 | "data": { 389 | "text/plain": [ 390 | "[0.030051206669808015, 0.98980000000000001]" 391 | ] 392 | }, 393 | "execution_count": 13, 394 | "metadata": {}, 395 | "output_type": "execute_result" 396 | } 397 | ], 398 | "source": [ 399 | "model.evaluate(X_test, Y_test, verbose=1)" 400 | ] 401 | }, 402 | { 403 | "cell_type": "markdown", 404 | "metadata": {}, 405 | "source": [ 406 | "And we achieve 99.25% accuracy! Worlds apart from the MLP classifier we implemented\n", 407 | "before. And this is just one way to do things. As you can see, neural networks provide a\n", 408 | "plethora of tuning parameters, and it is not at all clear which ones will lead to the best\n", 409 | "performance." 410 | ] 411 | }, 412 | { 413 | "cell_type": "markdown", 414 | "metadata": {}, 415 | "source": [ 416 | "\n", 417 | "< [Training an MLP in OpenCV to Classify Handwritten Digits](09.04-Training-an-MLP-in-OpenCV-to-Classify-Handwritten-Digits.ipynb) | [Contents](../README.md) | [Combining Different Algorithms Into an Ensemble](10.00-Combining-Different-Algorithms-Into-an-Ensemble.ipynb) >" 418 | ] 419 | } 420 | ], 421 | "metadata": { 422 | "kernelspec": { 423 | "display_name": "Python 3", 424 | "language": "python", 425 | "name": "python3" 426 | }, 427 | "language_info": { 428 | "codemirror_mode": { 429 | "name": "ipython", 430 | "version": 3 431 | }, 432 | "file_extension": ".py", 433 | "mimetype": "text/x-python", 434 | "name": "python", 435 | "nbconvert_exporter": "python", 436 | "pygments_lexer": "ipython3", 437 | "version": "3.5.3" 438 | } 439 | }, 440 | "nbformat": 4, 441 | "nbformat_minor": 1 442 | } 443 | -------------------------------------------------------------------------------- /notebooks/10.00-Combining-Different-Algorithms-Into-an-Ensemble.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Training a Deep Neural Net to Classify Handwritten Digits Using Keras](09.05-Training-a-Deep-Neural-Net-to-Classify-Handwritten-Digits-Using-Keras.ipynb) | [Contents](../README.md) | [Understanding Ensemble Methods](10.01-Understanding-Ensemble-Methods.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Combining Different Algorithms Into an Ensemble\n", 33 | "\n", 34 | "So far, we have looked at a number of interesting machine learning algorithms, from classic\n", 35 | "methods such as linear regression to more advanced techniques such as deep neural\n", 36 | "networks. At various points, we pointed out that every algorithm has its own strengths and\n", 37 | "weaknesses—and we took note of how to spot and overcome these weaknesses.\n", 38 | "\n", 39 | "However, wouldn't it be great if we could simply stack together a bunch of average\n", 40 | "classifiers to form a much stronger **ensemble** of classifiers?\n", 41 | "\n", 42 | "In this chapter, we will do just that. Ensemble methods are techniques that bind multiple\n", 43 | "different models together in order to solve a shared problem. Their use has become a\n", 44 | "common practice in competitive machine learning—making use of an ensemble typically\n", 45 | "improves an individual classifier's performance by a small percentage.\n", 46 | "\n", 47 | "Among these techniques are the so-called **bagging methods**, where the vote of multiple\n", 48 | "classifiers is averaged to make a final decision, and **boosting methods**, where one classifier\n", 49 | "is trying to rectify the errors made by another. One of these methods is known as **random\n", 50 | "forest**, which is a combination of multiple decision trees. In addition, you might already be\n", 51 | "familiar with **Adaptive Boosting** (also known as **AdaBoost**), a powerful boosting technique\n", 52 | "and popular feature of OpenCV.\n", 53 | "\n", 54 | "As we progress through the chapter, we want to find answers to the following questions:\n", 55 | "- How can we combine multiple models to form an ensemble classifier?\n", 56 | "- What are random forests and what do they have to do with decision trees?\n", 57 | "- What's the different between bagging and boosting?\n", 58 | "\n", 59 | "## Outline\n", 60 | "\n", 61 | "- [Understanding Ensemble Methods](10.01-Understanding-Ensemble-Methods.ipynb)\n", 62 | "- [Combining Decision Trees Into a Random Forest](10.02-Combining-Decision-Trees-Into-a-Random-Forest.ipynb)\n", 63 | "- [Using Random Forests for Face Recognition](10.03-Using-Random-Forests-for-Face-Recognition.ipynb)\n", 64 | "- [Implementing AdaBoost](10.04-Implementing-AdaBoost.ipynb)\n", 65 | "- [Combining Different Models Into a Voting Classifier](10.05-Combining-Different-Models-Into-a-Voting-Classifier.ipynb)\n", 66 | "\n", 67 | "Let's jump right in!" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "\n", 75 | "< [Training a Deep Neural Net to Classify Handwritten Digits Using Keras](09.05-Training-a-Deep-Neural-Net-to-Classify-Handwritten-Digits-Using-Keras.ipynb) | [Contents](../README.md) | [Understanding Ensemble Methods](10.01-Understanding-Ensemble-Methods.ipynb) >" 76 | ] 77 | } 78 | ], 79 | "metadata": { 80 | "kernelspec": { 81 | "display_name": "Python 3", 82 | "language": "python", 83 | "name": "python3" 84 | }, 85 | "language_info": { 86 | "codemirror_mode": { 87 | "name": "ipython", 88 | "version": 3 89 | }, 90 | "file_extension": ".py", 91 | "mimetype": "text/x-python", 92 | "name": "python", 93 | "nbconvert_exporter": "python", 94 | "pygments_lexer": "ipython3", 95 | "version": "3.5.3" 96 | } 97 | }, 98 | "nbformat": 4, 99 | "nbformat_minor": 1 100 | } 101 | -------------------------------------------------------------------------------- /notebooks/10.05-Combining-Different-Models-Into-a-Voting-Classifier.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Implementing AdaBoost](10.04-Implementing-AdaBoost.ipynb) | [Contents](../README.md) | [Selecting the Right Model with Hyper-Parameter Tuning](11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "source": [ 32 | "# Combining Different Models Into a Voting Classifier\n", 33 | "\n", 34 | "So far, we saw how to combine different instances of the same classifier or regressor into an\n", 35 | "ensemble. In this chapter, we are going to take this idea a step further and combine\n", 36 | "conceptually different classifiers into what is known as a **voting classifier**.\n", 37 | "\n", 38 | "The idea behind voting classifiers is that the individual learners in the ensemble don't\n", 39 | "necessarily need to be of the same type. After all, no matter how the individual classifiers\n", 40 | "arrived at their prediction, in the end, we are going to apply a decision rule that integrates\n", 41 | "all the votes of the individual classifiers. This is also known as a **voting scheme**.\n", 42 | "\n", 43 | "Two different voting schemes are common among voting classifiers:\n", 44 | "- In **hard voting** (also known as **majority voting**), every individual classifier votes for a class, and the majority wins. In statistical terms, the predicted target label of the ensemble is the mode of the distribution of individually predicted labels.\n", 45 | "- In **soft voting**, every individual classifier provides a probability value that a specific data point belongs to a particular target class. The predictions are weighted by the classifier's importance and summed up. Then the target label with the greatest sum of weighted probabilities wins the vote.\n", 46 | "\n", 47 | "You can find an example of how these voting schemes work in practice in the book." 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "## Implementing a Voting Classifier\n", 55 | "\n", 56 | "Let's look at a simple example of a voting classifier that combines three different algorithms:\n", 57 | "- A logistic regression classifier from [Chapter 3](03.00-First-Steps-in-Supervised-Learning.ipynb), *First Steps in Supervised Learning*\n", 58 | "- A Gaussian naive Bayes classifier from [Chapter 7](07.00-Implementing-a-Spam-Filter-with-Bayesian-Learning.ipynb), *Implementing a Spam Filter with Bayesian Learning*\n", 59 | "- A random forest classifier from this chapter" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "We can combine these three algorithms into a voting classifier and apply it to the breast\n", 67 | "cancer dataset with the following steps.\n", 68 | "\n", 69 | "Load the dataset, and split it into training and test sets:" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 1, 75 | "metadata": { 76 | "collapsed": true 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "from sklearn.datasets import load_breast_cancer\n", 81 | "iris = load_breast_cancer()\n", 82 | "X = iris.data\n", 83 | "y = iris.target" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 2, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "from sklearn.model_selection import train_test_split\n", 93 | "X_train, X_test, y_train, y_test = train_test_split(\n", 94 | " X, y, random_state=13\n", 95 | ")" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "Instantiate the individual classifiers:" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 3, 108 | "metadata": { 109 | "collapsed": true 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "from sklearn.linear_model import LogisticRegression\n", 114 | "model1 = LogisticRegression(random_state=13)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 4, 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "outputs": [], 124 | "source": [ 125 | "from sklearn.naive_bayes import GaussianNB\n", 126 | "model2 = GaussianNB()" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 5, 132 | "metadata": { 133 | "collapsed": true 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "from sklearn.ensemble import RandomForestClassifier\n", 138 | "model3 = RandomForestClassifier(random_state=13)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "Assign the individual classifiers to the voting ensemble. Here, we need to pass a\n", 146 | "list of tuples (`estimators`), where every tuple consists of the name of the\n", 147 | "classifier (a string of our choosing) and the model object. The voting scheme can\n", 148 | "be either `voting='hard'` or `voting='soft'`:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 6, 154 | "metadata": { 155 | "collapsed": true 156 | }, 157 | "outputs": [], 158 | "source": [ 159 | "from sklearn.ensemble import VotingClassifier\n", 160 | "vote = VotingClassifier(estimators=[('lr', model1),\n", 161 | " ('gnb', model2),\n", 162 | " ('rfc', model3)],\n", 163 | " voting='hard')" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "Fit the ensemble to the training data and score it on the test data:" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 7, 176 | "metadata": {}, 177 | "outputs": [ 178 | { 179 | "data": { 180 | "text/plain": [ 181 | "0.95104895104895104" 182 | ] 183 | }, 184 | "execution_count": 7, 185 | "metadata": {}, 186 | "output_type": "execute_result" 187 | } 188 | ], 189 | "source": [ 190 | "vote.fit(X_train, y_train)\n", 191 | "vote.score(X_test, y_test)" 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "metadata": {}, 197 | "source": [ 198 | "In order to convince us that 95.1% is a great accuracy score, we can compare the ensemble's\n", 199 | "performance to the theoretical performance of each individual classifier. We do this by\n", 200 | "fitting the individual classifiers to the data. Then we will see that the logistic regression\n", 201 | "model achieves 94.4% accuracy on its own:" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": 8, 207 | "metadata": {}, 208 | "outputs": [ 209 | { 210 | "data": { 211 | "text/plain": [ 212 | "0.94405594405594406" 213 | ] 214 | }, 215 | "execution_count": 8, 216 | "metadata": {}, 217 | "output_type": "execute_result" 218 | } 219 | ], 220 | "source": [ 221 | "model1.fit(X_train, y_train)\n", 222 | "model1.score(X_test, y_test)" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "metadata": {}, 228 | "source": [ 229 | "Similarly, the naive Bayes classifier achieves 93.0% accuracy:" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 9, 235 | "metadata": {}, 236 | "outputs": [ 237 | { 238 | "data": { 239 | "text/plain": [ 240 | "0.93006993006993011" 241 | ] 242 | }, 243 | "execution_count": 9, 244 | "metadata": {}, 245 | "output_type": "execute_result" 246 | } 247 | ], 248 | "source": [ 249 | "model2.fit(X_train, y_train)\n", 250 | "model2.score(X_test, y_test)" 251 | ] 252 | }, 253 | { 254 | "cell_type": "markdown", 255 | "metadata": {}, 256 | "source": [ 257 | "Last but not least, the random forest classifier also achieved 94.4% accuracy:" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": 10, 263 | "metadata": {}, 264 | "outputs": [ 265 | { 266 | "data": { 267 | "text/plain": [ 268 | "0.94405594405594406" 269 | ] 270 | }, 271 | "execution_count": 10, 272 | "metadata": {}, 273 | "output_type": "execute_result" 274 | } 275 | ], 276 | "source": [ 277 | "model3.fit(X_train, y_train)\n", 278 | "model3.score(X_test, y_test)" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "All in all, we were just able to gain a good percent in performance by combining three\n", 286 | "unrelated classifiers into an ensemble. Each of these classifiers might have made different\n", 287 | "mistakes on the training set, but that's OK because on average, we need just two out of three\n", 288 | "classifiers to be correct." 289 | ] 290 | }, 291 | { 292 | "cell_type": "markdown", 293 | "metadata": {}, 294 | "source": [ 295 | "\n", 296 | "< [Implementing AdaBoost](10.04-Implementing-AdaBoost.ipynb) | [Contents](../README.md) | [Selecting the Right Model with Hyper-Parameter Tuning](11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb) >" 297 | ] 298 | } 299 | ], 300 | "metadata": { 301 | "kernelspec": { 302 | "display_name": "Python 3", 303 | "language": "python", 304 | "name": "python3" 305 | }, 306 | "language_info": { 307 | "codemirror_mode": { 308 | "name": "ipython", 309 | "version": 3 310 | }, 311 | "file_extension": ".py", 312 | "mimetype": "text/x-python", 313 | "name": "python", 314 | "nbconvert_exporter": "python", 315 | "pygments_lexer": "ipython3", 316 | "version": "3.5.3" 317 | } 318 | }, 319 | "nbformat": 4, 320 | "nbformat_minor": 2 321 | } 322 | -------------------------------------------------------------------------------- /notebooks/11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Combining Different Models Into a Voting Classifier](10.05-Combining-Different-Models-Into-a-Voting-Classifier.ipynb) | [Contents](../README.md) | [Evaluating a Model](11.01-Evaluating-a-Model.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Selecting the Right Model with Hyper-Parameter Tuning\n", 31 | "\n", 32 | "In this chapter, we will dive deeper into **model evaluation** and **hyperparameter\n", 33 | "tuning**. Assume that we have two different models that might apply to our task. How can\n", 34 | "we know which one is better? Answering this question often involves repeatedly fitting\n", 35 | "different versions of our model to different subsets of the data, such as in **cross-validation**\n", 36 | "and **bootstrapping**. In combination with different scoring functions, we can obtain reliable\n", 37 | "estimates of the generalization performance of our models.\n", 38 | "\n", 39 | "But what if two different models give similar results? Can we be sure that the two models\n", 40 | "are equivalent, or is it possible that one of them just got lucky? How can we know whether\n", 41 | "one of them is significantly better than the other? Answering these questions will lead us to\n", 42 | "discussing some useful statistical tests such as **Students t-test** and **McNemar's test**.\n", 43 | "\n", 44 | "As we will get familiar with these techniques, we will also want to answer the following\n", 45 | "questions:\n", 46 | "- What's the best strategy to tweak the hyperparameters of a model?\n", 47 | "- How can we compare the performance of different models in a fair way?\n", 48 | "- How do we select the right machine learning tool for the task at hand?\n", 49 | "\n", 50 | "## Outline\n", 51 | "\n", 52 | "- [Evaluating a Model](11.01-Evaluating-a-Model.ipynb)\n", 53 | "- [Understanding Cross-Validation, Bootstrapping, and McNemar's Test](11.02-Understanding-Cross-Validation-Bootstrapping-and-McNemar's-Test.ipynb)\n", 54 | "- [Tuning Hyperparameters with Grid Search](11.03-Tuning-Hyperparameters-with-Grid-Search.ipynb)\n", 55 | "- [Chaining Algorithms Together to Form a Pipeline](11.04-Chaining-Algorithms-Together-to-Form-a-Pipeline.ipynb)\n", 56 | " " 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "\n", 64 | "< [Combining Different Models Into a Voting Classifier](10.05-Combining-Different-Models-Into-a-Voting-Classifier.ipynb) | [Contents](../README.md) | [Evaluating a Model](11.01-Evaluating-a-Model.ipynb) >" 65 | ] 66 | } 67 | ], 68 | "metadata": { 69 | "kernelspec": { 70 | "display_name": "Python 3", 71 | "language": "python", 72 | "name": "python3" 73 | }, 74 | "language_info": { 75 | "codemirror_mode": { 76 | "name": "ipython", 77 | "version": 3 78 | }, 79 | "file_extension": ".py", 80 | "mimetype": "text/x-python", 81 | "name": "python", 82 | "nbconvert_exporter": "python", 83 | "pygments_lexer": "ipython3", 84 | "version": "3.5.3" 85 | } 86 | }, 87 | "nbformat": 4, 88 | "nbformat_minor": 1 89 | } 90 | -------------------------------------------------------------------------------- /notebooks/11.01-Evaluating-a-Model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Selecting the Right Model with Hyper-Parameter Tuning](11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb) | [Contents](../README.md) | [Understanding Cross-Validation](11.02-Understanding-Cross-Validation-Bootstrapping-and-McNemar's-Test.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Evaluating a Model\n", 31 | "\n", 32 | "Model evaluation strategies come in many different forms and shapes. In the following\n", 33 | "sections, we will, therefore, highlight three of the most commonly used techniques to\n", 34 | "compare models against each other:\n", 35 | "- $k$-fold cross-validation\n", 36 | "- bootstrapping\n", 37 | "- McNemar's test\n", 38 | "\n", 39 | "In principle, model evaluation is simple: after training a model on some data, we can\n", 40 | "estimate its effectiveness by comparing model predictions to some ground truth values. We\n", 41 | "learned early on that we should split the data into a training and a test set, and we tried to\n", 42 | "follow this instruction whenever possible. But why exactly did we do that again?" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "## Evaluating a model the wrong way\n", 50 | "\n", 51 | "The reason we never evaluate a model on the training set is that, in principle, any dataset\n", 52 | "can be learned if we throw a strong enough model at it.\n", 53 | "\n", 54 | "A quick demonstration of this can be given with help of the Iris dataset, which we talked\n", 55 | "about extensively in [Chapter 3](03.00-First-Steps-in-Supervised-Learning.ipynb), *First Steps in Supervised Learning*. There the goal was to\n", 56 | "classify species of Iris flowers based on their physical dimensions. We can load the Iris\n", 57 | "dataset using scikit-learn:" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 1, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "from sklearn.datasets import load_iris\n", 67 | "iris = load_iris()" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "An innocent approach to this problem would be to store all data points in matrix `X` and all\n", 75 | "class labels in the vector `y`:" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 2, 81 | "metadata": { 82 | "collapsed": true 83 | }, 84 | "outputs": [], 85 | "source": [ 86 | "import numpy as np\n", 87 | "X = iris.data.astype(np.float32)\n", 88 | "y = iris.target" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "Next, we choose a model and its hyperparameters. For example, let's use the $k$-NN\n", 96 | "algorithm from [Chapter 3](03.00-First-Steps-in-Supervised-Learning.ipynb), *First Steps in Supervised Learning*, which provides only a single\n", 97 | "hyperparameter: the number of neighbors, $k$. With $k=1$, we get a very simple model that\n", 98 | "classifies the label of an unknown point as belonging to the same class as its closest\n", 99 | "neighbor.\n", 100 | "\n", 101 | "In OpenCV, $k$-NN instantiates as follows:" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 3, 107 | "metadata": { 108 | "collapsed": true 109 | }, 110 | "outputs": [], 111 | "source": [ 112 | "import cv2\n", 113 | "knn = cv2.ml.KNearest_create()\n", 114 | "knn.setDefaultK(1)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "Then we train the model and use it to predict labels for the data that we already know:" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 4, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "knn.train(X, cv2.ml.ROW_SAMPLE, y)\n", 131 | "_, y_hat = knn.predict(X)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": {}, 137 | "source": [ 138 | "Finally, we compute the fraction of correctly labeled points:" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 5, 144 | "metadata": {}, 145 | "outputs": [ 146 | { 147 | "data": { 148 | "text/plain": [ 149 | "1.0" 150 | ] 151 | }, 152 | "execution_count": 5, 153 | "metadata": {}, 154 | "output_type": "execute_result" 155 | } 156 | ], 157 | "source": [ 158 | "from sklearn.metrics import accuracy_score\n", 159 | "accuracy_score(y, y_hat)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "We see an accuracy score of 1.0, which indicates that 100% of points were correctly labeled\n", 167 | "by our model!\n", 168 | "\n", 169 | "But is this truly measuring the expected accuracy? Have we really come up with a model\n", 170 | "that we expect to be correct 100% of the time?\n", 171 | "\n", 172 | "As you may have gathered, the answer is no. This example shows that even a simple\n", 173 | "algorithm is capable of memorizing a real-world dataset. Imagine how easy this task would\n", 174 | "have been for a deep neural network! Usually, the more parameters a model has, the more\n", 175 | "powerful it is. We will come back to this shortly." 176 | ] 177 | }, 178 | { 179 | "cell_type": "markdown", 180 | "metadata": {}, 181 | "source": [ 182 | "## Evaluating a model the right way\n", 183 | "\n", 184 | "A better sense of a model's performance can be found using what's known as a test set, but\n", 185 | "you already knew this. When presented with data held out from the training procedure, we\n", 186 | "can check whether a model has learned some dependencies in the data that hold across the\n", 187 | "board or whether it just memorized the training set.\n", 188 | "\n", 189 | "We can split the data into training and test sets using the familiar `train_test_split` from\n", 190 | "scikit-learn's `model_selection` module:" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 6, 196 | "metadata": { 197 | "collapsed": true 198 | }, 199 | "outputs": [], 200 | "source": [ 201 | "from sklearn.model_selection import train_test_split" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "But how do we choose the right train-test ratio? Is there even such a thing as a right ratio?\n", 209 | "Or is this considered another hyperparameter of the model?\n", 210 | "\n", 211 | "There are two competing concerns here:\n", 212 | "- If our training set is too small, our model might not be able to extract the relevant data dependencies. As a result, our model performance might differ significantly from run to run, that is, if we repeat our experiment multiple times with different random number seeds. As an extreme example, consider a training set with a single data point from the Iris dataset. In this case, there would be no way for the model to even learn that there are multiple species in the dataset!\n", 213 | "- If our test set is too small, our performance metric might differ significantly from run to run. As a result, we would have to rerun our experiment multiple times to get an idea of how well our model does on average. As an extreme example, consider a test set with a single data point. Since there are three different classes in the Iris dataset, we might get either 0, 33%, 66%, or 100% correct.\n", 214 | "\n", 215 | "A good starting point is usually a 80-20 training-test split. However, it all depends on the\n", 216 | "amount of data available. For relatively small datasets, a 50-50 split might be more suitable:" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 7, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=37,\n", 226 | " train_size=0.8)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "metadata": {}, 232 | "source": [ 233 | "Then we retrain the preceding model on the training set:" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": 8, 239 | "metadata": {}, 240 | "outputs": [], 241 | "source": [ 242 | "knn = cv2.ml.KNearest_create()\n", 243 | "knn.setDefaultK(1)\n", 244 | "knn.train(X_train, cv2.ml.ROW_SAMPLE, y_train);" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "When we test the model on the test set, we suddenly get a different result:" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": 9, 257 | "metadata": {}, 258 | "outputs": [ 259 | { 260 | "data": { 261 | "text/plain": [ 262 | "0.96666666666666667" 263 | ] 264 | }, 265 | "execution_count": 9, 266 | "metadata": {}, 267 | "output_type": "execute_result" 268 | } 269 | ], 270 | "source": [ 271 | "_, y_test_hat = knn.predict(X_test)\n", 272 | "accuracy_score(y_test, y_test_hat)" 273 | ] 274 | }, 275 | { 276 | "cell_type": "markdown", 277 | "metadata": {}, 278 | "source": [ 279 | "We see a more reasonable result here, although 97% accuracy is still a formidable result. But\n", 280 | "is this the best possible result—and how can we know for sure?\n", 281 | "\n", 282 | "To answer this question is, we have to dig a little deeper." 283 | ] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "metadata": {}, 288 | "source": [ 289 | "\n", 290 | "< [Selecting the Right Model with Hyper-Parameter Tuning](11.00-Selecting-the-Right-Model-with-Hyper-Parameter-Tuning.ipynb) | [Contents](../README.md) | [Understanding Cross-Validation](11.02-Understanding-Cross-Validation-Bootstrapping-and-McNemar's-Test.ipynb) >" 291 | ] 292 | } 293 | ], 294 | "metadata": { 295 | "kernelspec": { 296 | "display_name": "Python 3", 297 | "language": "python", 298 | "name": "python3" 299 | }, 300 | "language_info": { 301 | "codemirror_mode": { 302 | "name": "ipython", 303 | "version": 3 304 | }, 305 | "file_extension": ".py", 306 | "mimetype": "text/x-python", 307 | "name": "python", 308 | "nbconvert_exporter": "python", 309 | "pygments_lexer": "ipython3", 310 | "version": "3.5.3" 311 | } 312 | }, 313 | "nbformat": 4, 314 | "nbformat_minor": 1 315 | } 316 | -------------------------------------------------------------------------------- /notebooks/11.04-Chaining-Algorithms-Together-to-Form-a-Pipeline.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Tuning Hyperparameters with Grid Search](11.03-Tuning-Hyperparameters-with-Grid-Search.ipynb) | [Contents](../README.md) | [Wrapping Up](12.00-Wrapping-Up.ipynb) >" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Chaining Algorithms Together to Form a Pipeline\n", 31 | "\n", 32 | "Most machine learning problems we have discussed so far consist of at least a\n", 33 | "preprocessing step and a classification step. The more complicated the problem, the longer\n", 34 | "this **processing chain** might get. One convenient way to glue multiple processing steps\n", 35 | "together and even use them in grid search is by using the `Pipeline` class from scikit-learn." 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "## Implementing pipelines in scikit-learn\n", 43 | "\n", 44 | "The Pipeline class itself has a `fit`, a `predict`, and a `score` method, which behave just\n", 45 | "like any other estimator in scikit-learn. The most common used case of the `Pipeline` class\n", 46 | "is to chain different preprocessing steps together with a supervised model like a classifier.\n", 47 | "\n", 48 | "Let's return to the breast cancer dataset from [Chapter 5](05.00-Using-Decision-Trees-to-Make-a-Medical-Diagnosis.ipynb), *Using Decision Trees to Make a\n", 49 | "Medical Diagnosis*. Using scikit-learn, we import the dataset and split it into training and test\n", 50 | "sets:" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 1, 56 | "metadata": { 57 | "collapsed": true 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "from sklearn.datasets import load_breast_cancer\n", 62 | "import numpy as np\n", 63 | "cancer = load_breast_cancer()\n", 64 | "X = cancer.data.astype(np.float32)\n", 65 | "y = cancer.target" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 2, 71 | "metadata": { 72 | "collapsed": true 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "from sklearn.model_selection import train_test_split\n", 77 | "X_train, X_test, y_train, y_test = train_test_split(\n", 78 | " X, y, random_state=37\n", 79 | ")" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "Instead of the $k$-NN algorithm, we could fit a support vector machine (SVM) to the data:" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 3, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "text/plain": [ 97 | "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n", 98 | " decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',\n", 99 | " max_iter=-1, probability=False, random_state=None, shrinking=True,\n", 100 | " tol=0.001, verbose=False)" 101 | ] 102 | }, 103 | "execution_count": 3, 104 | "metadata": {}, 105 | "output_type": "execute_result" 106 | } 107 | ], 108 | "source": [ 109 | "from sklearn.svm import SVC\n", 110 | "svm = SVC()\n", 111 | "svm.fit(X_train, y_train)" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "Without straining our brains too hard, this algorithm achieves an accuracy score of 65%:" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 4, 124 | "metadata": {}, 125 | "outputs": [ 126 | { 127 | "data": { 128 | "text/plain": [ 129 | "0.65034965034965031" 130 | ] 131 | }, 132 | "execution_count": 4, 133 | "metadata": {}, 134 | "output_type": "execute_result" 135 | } 136 | ], 137 | "source": [ 138 | "svm.score(X_test, y_test)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "Now if we wanted to run the algorithm again using some preprocessing step (for example,\n", 146 | "by scaling the data first with `MinMaxScaler`),we would do the preprocessing step by hand\n", 147 | "and then feed the preprocessed data into the classifiers `fit` method.\n", 148 | "\n", 149 | "An alternative is to use a pipeline object. Here, we want to specify a list of processing steps,\n", 150 | "where each step is a tuple containing a name (any string of our choosing) and an instance of\n", 151 | "an estimator:" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 5, 157 | "metadata": { 158 | "collapsed": true 159 | }, 160 | "outputs": [], 161 | "source": [ 162 | "from sklearn.pipeline import Pipeline\n", 163 | "from sklearn.preprocessing import MinMaxScaler\n", 164 | "pipe = Pipeline([(\"scaler\", MinMaxScaler()), (\"svm\", SVC())])" 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "metadata": {}, 170 | "source": [ 171 | "Here, we created two steps: the first, called `\"scaler\"`, is an instance of `MinMaxScaler`, and\n", 172 | "the second, called `\"svm\"`, is an instance of `SVC`. Now we can fit the pipeline like any other\n", 173 | "scikit-learn estimator:" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 6, 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/plain": [ 184 | "Pipeline(steps=[('scaler', MinMaxScaler(copy=True, feature_range=(0, 1))), ('svm', SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n", 185 | " decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',\n", 186 | " max_iter=-1, probability=False, random_state=None, shrinking=True,\n", 187 | " tol=0.001, verbose=False))])" 188 | ] 189 | }, 190 | "execution_count": 6, 191 | "metadata": {}, 192 | "output_type": "execute_result" 193 | } 194 | ], 195 | "source": [ 196 | "pipe.fit(X_train, y_train)" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "Here, the `fit` method first calls `fit` on the first step (the scaler), then it transforms the\n", 204 | "training data using the scaler, and finally it fits the SVM with the scaled data.\n", 205 | "\n", 206 | "And voila! When we score the classifier on the test data, we see a drastic improvement in\n", 207 | "performance:" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 7, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "data": { 217 | "text/plain": [ 218 | "0.95104895104895104" 219 | ] 220 | }, 221 | "execution_count": 7, 222 | "metadata": {}, 223 | "output_type": "execute_result" 224 | } 225 | ], 226 | "source": [ 227 | "pipe.score(X_test, y_test)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "Calling the score method on the pipeline first transforms the test data using the scaler and\n", 235 | "then calls the score method on the SVM using the scaled test data. And scikit-learn did all\n", 236 | "this with only four lines of code!\n", 237 | "\n", 238 | "The main benefit of using the pipeline, however, is that we can now use this single\n", 239 | "estimator in `cross_val_score` or `GridSearchCV`." 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "metadata": {}, 245 | "source": [ 246 | "## Using pipelines in grid searches\n", 247 | "\n", 248 | "Using a pipeline in a grid search works the same way as using any other estimator.\n", 249 | "\n", 250 | "We define a parameter grid to search over and construct a `GridSearchCV` from the pipeline\n", 251 | "and the parameter grid. When specifying the parameter grid, there is, however, a slight\n", 252 | "change. We need to specify for each parameter which step of the pipeline it belongs to. Both\n", 253 | "parameters that we want to adjust, `C` and `gamma`, are parameters of SVC, the second step. In\n", 254 | "the preceding section, we gave this step the name `\"svm\"`. The syntax to define a parameter\n", 255 | "grid for a pipeline is to specify for each parameter the step name, followed by `__` (a double\n", 256 | "underscore), followed by the parameter name.\n", 257 | "\n", 258 | "Hence, we would construct the parameter grid as follows:" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": 8, 264 | "metadata": { 265 | "collapsed": true 266 | }, 267 | "outputs": [], 268 | "source": [ 269 | "param_grid = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100],\n", 270 | " 'svm__gamma': [0.001, 0.01, 0.1, 1, 10, 100]}" 271 | ] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "metadata": {}, 276 | "source": [ 277 | "With this parameter grid, we can use `GridSearchCV` as usual:" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": 9, 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "data": { 287 | "text/plain": [ 288 | "GridSearchCV(cv=10, error_score='raise',\n", 289 | " estimator=Pipeline(steps=[('scaler', MinMaxScaler(copy=True, feature_range=(0, 1))), ('svm', SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n", 290 | " decision_function_shape=None, degree=3, gamma='auto', kernel='rbf',\n", 291 | " max_iter=-1, probability=False, random_state=None, shrinking=True,\n", 292 | " tol=0.001, verbose=False))]),\n", 293 | " fit_params={}, iid=True, n_jobs=1,\n", 294 | " param_grid={'svm__C': [0.001, 0.01, 0.1, 1, 10, 100], 'svm__gamma': [0.001, 0.01, 0.1, 1, 10, 100]},\n", 295 | " pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n", 296 | " scoring=None, verbose=0)" 297 | ] 298 | }, 299 | "execution_count": 9, 300 | "metadata": {}, 301 | "output_type": "execute_result" 302 | } 303 | ], 304 | "source": [ 305 | "from sklearn.model_selection import GridSearchCV\n", 306 | "grid = GridSearchCV(pipe, param_grid=param_grid, cv=10)\n", 307 | "grid.fit(X_train, y_train)" 308 | ] 309 | }, 310 | { 311 | "cell_type": "markdown", 312 | "metadata": {}, 313 | "source": [ 314 | "The best score in the grid is stored in `best_score_`:" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 10, 320 | "metadata": {}, 321 | "outputs": [ 322 | { 323 | "data": { 324 | "text/plain": [ 325 | "0.97652582159624413" 326 | ] 327 | }, 328 | "execution_count": 10, 329 | "metadata": {}, 330 | "output_type": "execute_result" 331 | } 332 | ], 333 | "source": [ 334 | "grid.best_score_" 335 | ] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "metadata": {}, 340 | "source": [ 341 | "Similarly, the best parameters are stored in `best_params_`:" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 11, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "data": { 351 | "text/plain": [ 352 | "{'svm__C': 1, 'svm__gamma': 1}" 353 | ] 354 | }, 355 | "execution_count": 11, 356 | "metadata": {}, 357 | "output_type": "execute_result" 358 | } 359 | ], 360 | "source": [ 361 | "grid.best_params_" 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": {}, 367 | "source": [ 368 | "But recall that the cross-validation score might be overly optimistic. In order to know the\n", 369 | "true performance of the classifier, we need to score it on the test set:" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "execution_count": 12, 375 | "metadata": {}, 376 | "outputs": [ 377 | { 378 | "data": { 379 | "text/plain": [ 380 | "0.965034965034965" 381 | ] 382 | }, 383 | "execution_count": 12, 384 | "metadata": {}, 385 | "output_type": "execute_result" 386 | } 387 | ], 388 | "source": [ 389 | "grid.score(X_test, y_test)" 390 | ] 391 | }, 392 | { 393 | "cell_type": "markdown", 394 | "metadata": {}, 395 | "source": [ 396 | "In contrast to the grid search we did before, now for each split in the cross-validation,\n", 397 | "`MinMaxScaler` is refit with only the training splits, and no information is leaked from the\n", 398 | "test split into the parameter search.\n", 399 | "\n", 400 | "This makes it easy to build a pipeline to chain together a whole variety of steps!\n", 401 | "\n", 402 | "How would you mix and match different estimators in a single pipeline? Turn to page 330 to find the answer." 403 | ] 404 | }, 405 | { 406 | "cell_type": "markdown", 407 | "metadata": {}, 408 | "source": [ 409 | "\n", 410 | "< [Tuning Hyperparameters with Grid Search](11.03-Tuning-Hyperparameters-with-Grid-Search.ipynb) | [Contents](../README.md) | [Wrapping Up](12.00-Wrapping-Up.ipynb) >" 411 | ] 412 | } 413 | ], 414 | "metadata": { 415 | "kernelspec": { 416 | "display_name": "Python 3", 417 | "language": "python", 418 | "name": "python3" 419 | }, 420 | "language_info": { 421 | "codemirror_mode": { 422 | "name": "ipython", 423 | "version": 3 424 | }, 425 | "file_extension": ".py", 426 | "mimetype": "text/x-python", 427 | "name": "python", 428 | "nbconvert_exporter": "python", 429 | "pygments_lexer": "ipython3", 430 | "version": "3.5.3" 431 | } 432 | }, 433 | "nbformat": 4, 434 | "nbformat_minor": 1 435 | } 436 | -------------------------------------------------------------------------------- /notebooks/12.00-Wrapping-Up.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "*This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler.\n", 10 | "The code is released under the [MIT license](https://opensource.org/licenses/MIT),\n", 11 | "and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).*\n", 12 | "\n", 13 | "*Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations.\n", 14 | "If you find this content useful, please consider supporting the work by\n", 15 | "[buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "< [Chaining Algorithms Together to Form a Pipeline](11.04-Chaining-Algorithms-Together-to-Form-a-Pipeline.ipynb) | [Contents](../README.md) |" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "# Wrapping Up\n", 31 | "\n", 32 | "Congratulations! You have just made a big step toward becoming a machine learning\n", 33 | "practitioner. Not only are you familiar with a wide variety of fundamental machine\n", 34 | "learning algorithms, you also know how to apply them to both supervised and\n", 35 | "unsupervised learning problems.\n", 36 | "\n", 37 | "Before we part ways, I want to give you some final words of advice, point you toward some\n", 38 | "additional resources, and give you some suggestions on how you can further improve your\n", 39 | "machine learning and data science skills." 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "## Approaching a machine learning problem\n", 47 | "\n", 48 | "When you see a new machine learning problem in the wild, you might be tempted to jump\n", 49 | "ahead and throw your favorite algorithm at the problem—perhaps the one you understood\n", 50 | "best or had the most fun implementing. But knowing beforehand which algorithm will\n", 51 | "perform best on your specific problem is not often possible.\n", 52 | "\n", 53 | "Instead, you need to take a step back and look at the big picture.\n", 54 | "Here the book provides an easy-to-follow outline on how to approach machine learning problems in the wild (p.331ff.)." 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "## Writing your own OpenCV based classifier in C++\n", 62 | "\n", 63 | "Since OpenCV is one of those Python libraries that does not contain a single line of Python\n", 64 | "code under the hood (I'm kidding, but it's close), you will have to implement your custom\n", 65 | "estimator in C++." 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "The first step is to define a \n", 73 | "file `MyClass.cpp`:\n", 74 | "\n", 75 | " #include \n", 76 | " #include \n", 77 | " #include \n", 78 | "\n", 79 | " class MyClass : public cv::ml::StatModel\n", 80 | " {\n", 81 | " public:\n", 82 | " MyClass()\n", 83 | " {\n", 84 | " print(\"MyClass constructor\\n\");\n", 85 | " }\n", 86 | "\n", 87 | " ~MyClass() {}\n", 88 | "\n", 89 | " int getVarCount() const\n", 90 | " {\n", 91 | " // returns the number of variables in the training samples\n", 92 | " return 0;\n", 93 | " }\n", 94 | "\n", 95 | " bool empty() const\n", 96 | " {\n", 97 | " return true;\n", 98 | " }\n", 99 | "\n", 100 | " bool isTrained() const\n", 101 | " {\n", 102 | " // returns true if the model is trained\n", 103 | " return false;\n", 104 | " }\n", 105 | "\n", 106 | " bool isClassifier() const\n", 107 | " {\n", 108 | " // returns true if the model is a classifier\n", 109 | " return true;\n", 110 | " }\n", 111 | "\n", 112 | " bool train(const cv::Ptr& trainData, int flags=0) const\n", 113 | " {\n", 114 | " // trains the model\n", 115 | " // trainData: training data that can be loaded from file using\n", 116 | " // TrainData::loadFromCSV or created with TrainData::create.\n", 117 | " // flags: optional flags, depending on the model. Some of the models\n", 118 | " // can be updated with the new training samples, not completely\n", 119 | " // overwritten (such as NormalBayesClassifier or ANN_MLP).\n", 120 | " return false;\n", 121 | " }\n", 122 | "\n", 123 | " bool train(cv::InputArray samples, int layout, cv::InputArray responses)\n", 124 | " {\n", 125 | " // trains the model\n", 126 | " // samples: training samples\n", 127 | " // layout: see ml::SampleTypes\n", 128 | " // responses: vector of responses associated with the training samples\n", 129 | " return false;\n", 130 | " }\n", 131 | "\n", 132 | " float calcError(const cv::Ptr& data, bool test, cv::OutputArray resp)\n", 133 | " {\n", 134 | " // calculates the error on the training or test set\n", 135 | " // data: the training data\n", 136 | " // test: if true, the error is computed over the test subset of the data, otherwise\n", 137 | " // it's computed over the training subset of the data.\n", 138 | " return 0.0f;\n", 139 | " }\n", 140 | "\n", 141 | " float predict(cv::InputArray samples, cv::OutputArray results=cv::noArray(), int flags=0) const\n", 142 | " {\n", 143 | " // predicts responses for the provided samples\n", 144 | " // samples: the input samples, floating-point matrix\n", 145 | " // results: the optional matrix of results\n", 146 | " // flags: the optional flags, model-dependent. see cv::ml::StatModel::Flags\n", 147 | " return 0.0f;\n", 148 | " }\n", 149 | " };\n", 150 | "\n", 151 | " int main()\n", 152 | " {\n", 153 | " MyClass myclass;\n", 154 | " return 0;\n", 155 | " }" 156 | ] 157 | }, 158 | { 159 | "cell_type": "markdown", 160 | "metadata": {}, 161 | "source": [ 162 | "Then create a file `CMakeLists.txt`:\n", 163 | " \n", 164 | " cmake_minimum_required(VERSION 2.8)\n", 165 | " project(MyClass)\n", 166 | " find_package(OpenCV REQUIRED)\n", 167 | " add_executable(MyClass MyClass.cpp)\n", 168 | " target_link_libraries(MyClass ${OpenCV_LIBS})" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "Then you can compile the file from the command line via `cmake` and `make`:\n", 176 | " \n", 177 | " $ cmake .\n", 178 | " $ make" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "Then run the file:\n", 186 | " \n", 187 | " $ ./MyClass\n", 188 | " \n", 189 | "This should not generate any error, and print to console:\n", 190 | " \n", 191 | " MyClass constructor" 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "metadata": {}, 197 | "source": [ 198 | "## Writing your own Scikit-Learn based classifier in Python:\n", 199 | "\n", 200 | "Alternatively, you can write your own classifier using the scikit-learn library.\n", 201 | "\n", 202 | "You can do this by importing `BaseEstimator` and `ClassifierMixin`. The latter will\n", 203 | "provide a corresponding `score` method, which works for all classifiers. Optionally, you can\n", 204 | "overwrite the score method to provide your own.\n", 205 | "\n", 206 | "The following mixins are available:\n", 207 | "- `ClassifierMixin` if you are writing a classifier (will provide a basic `score` method)\n", 208 | "- `RegressorMixin` if you are writing a regressor (will provide a basic `score` method)\n", 209 | "- `ClusterMixin` if you are writing a clustering algorithm (will provide a basic `fit_predict` method)\n", 210 | "- `TransformerMixin` if you are writing a transformer (will provide a basic `fit_predict` method)" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 1, 216 | "metadata": { 217 | "collapsed": true 218 | }, 219 | "outputs": [], 220 | "source": [ 221 | "import numpy as np\n", 222 | "from sklearn.base import BaseEstimator, ClassifierMixin" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 2, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "class MyClassifier(BaseEstimator, ClassifierMixin):\n", 232 | " \"\"\"An example classifier\"\"\"\n", 233 | "\n", 234 | " def __init__(self, param1=1, param2=2):\n", 235 | " \"\"\"Called when initializing the classifier\n", 236 | " \n", 237 | " The constructor is used to define some optional\n", 238 | " parameters of the classifier. Store them as class\n", 239 | " attributes for future access.\n", 240 | " \n", 241 | " Parameters\n", 242 | " ----------\n", 243 | " param1 : int, optional, default: 1\n", 244 | " The first parameter\n", 245 | " param2 : int, optional, default: 2\n", 246 | " The second parameter\n", 247 | " \"\"\"\n", 248 | " self.param1 = param1\n", 249 | " self.param2 = param2\n", 250 | " \n", 251 | " def fit(self, X, y=None):\n", 252 | " \"\"\"Fits the classifier to data\n", 253 | " \n", 254 | " This should fit the classifier to the training data.\n", 255 | " All the \"work\" should be done here.\n", 256 | " \n", 257 | " Parameters\n", 258 | " ----------\n", 259 | " X : array-like\n", 260 | " The training data, where the first dimension is\n", 261 | " the number of training samples, and the second\n", 262 | " dimension is the number of features.\n", 263 | " y : array-like, optional, default: None\n", 264 | " Vector of class labels\n", 265 | " \n", 266 | " Returns\n", 267 | " -------\n", 268 | " The fit method returns the classifier object it\n", 269 | " belongs to.\n", 270 | " \"\"\"\n", 271 | " return self\n", 272 | " \n", 273 | " def predict(self, X):\n", 274 | " \"\"\"Predicts target labels\n", 275 | " \n", 276 | " This should predict the target labels of some data `X`.\n", 277 | " \n", 278 | " Parameters\n", 279 | " ----------\n", 280 | " X : array-like\n", 281 | " Data samples for which to predict the target labels.\n", 282 | " \n", 283 | " Returns\n", 284 | " -------\n", 285 | " y_pred : array-like\n", 286 | " Target labels for every data sample in `X`\n", 287 | " \"\"\"\n", 288 | " return np.zeros(X.shape[0])" 289 | ] 290 | }, 291 | { 292 | "cell_type": "markdown", 293 | "metadata": {}, 294 | "source": [ 295 | "The classifier can be instantiated as follows:" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": 3, 301 | "metadata": { 302 | "collapsed": true 303 | }, 304 | "outputs": [], 305 | "source": [ 306 | "myclass = MyClassifier()" 307 | ] 308 | }, 309 | { 310 | "cell_type": "markdown", 311 | "metadata": {}, 312 | "source": [ 313 | "You can then fit the model to some arbitrary data:" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 4, 319 | "metadata": {}, 320 | "outputs": [ 321 | { 322 | "data": { 323 | "text/plain": [ 324 | "MyClassifier(param1=1, param2=2)" 325 | ] 326 | }, 327 | "execution_count": 4, 328 | "metadata": {}, 329 | "output_type": "execute_result" 330 | } 331 | ], 332 | "source": [ 333 | "X = np.random.rand(10, 3)\n", 334 | "myclass.fit(X)" 335 | ] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "metadata": {}, 340 | "source": [ 341 | "And then you can proceed to predicting the target responses:" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 5, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "data": { 351 | "text/plain": [ 352 | "array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])" 353 | ] 354 | }, 355 | "execution_count": 5, 356 | "metadata": {}, 357 | "output_type": "execute_result" 358 | } 359 | ], 360 | "source": [ 361 | "myclass.predict(X)" 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": {}, 367 | "source": [ 368 | "## Where to go from here?\n", 369 | "\n", 370 | "The goal of this book was to introduce you to the world of machine learning and prepare\n", 371 | "you to become a machine learning practitioner. Now that you know everything about the\n", 372 | "fundamental algorithms, you might want to investigate some topics in more depth.\n", 373 | "\n", 374 | "Although it is not necessary to understand all the details of all the algorithms we\n", 375 | "implemented in this book, knowing some of the theory behind them might just make you a\n", 376 | "better data scientist.\n", 377 | "\n", 378 | "Turn to the book to find a list of suggested reading materials, books, and machine learning software!" 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": {}, 384 | "source": [ 385 | "## Summary\n", 386 | "\n", 387 | "In this book, we covered a lot of theory and practice.\n", 388 | "\n", 389 | "We discussed a wide variety of fundamental machine learning algorithms, be it supervised\n", 390 | "or unsupervised, illustrated best practices as well as ways to avoid common pitfalls, and we\n", 391 | "touched upon a variety of commands and packages for data analysis, machine learning, and\n", 392 | "visualization.\n", 393 | "\n", 394 | "If you made it this far, you have already made a big step toward machine learning mastery.\n", 395 | "From here on out, I am confident you will do just fine on your own.\n", 396 | "All that's left to say is farewell! I hope you enjoyed the ride; I certainly did." 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "\n", 404 | "< [Chaining Algorithms Together to Form a Pipeline](11.04-Chaining-Algorithms-Together-to-Form-a-Pipeline.ipynb) | [Contents](../README.md) |" 405 | ] 406 | } 407 | ], 408 | "metadata": { 409 | "kernelspec": { 410 | "display_name": "Python 3", 411 | "language": "python", 412 | "name": "python3" 413 | }, 414 | "language_info": { 415 | "codemirror_mode": { 416 | "name": "ipython", 417 | "version": 3 418 | }, 419 | "file_extension": ".py", 420 | "mimetype": "text/x-python", 421 | "name": "python", 422 | "nbconvert_exporter": "python", 423 | "pygments_lexer": "ipython3", 424 | "version": "3.5.3" 425 | } 426 | }, 427 | "nbformat": 4, 428 | "nbformat_minor": 1 429 | } 430 | -------------------------------------------------------------------------------- /notebooks/data/chapter6/pedestrian_test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter6/pedestrian_test.jpg -------------------------------------------------------------------------------- /notebooks/data/chapter6/pedestrians128x64.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter6/pedestrians128x64.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter6/pedestrians_neg.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter6/pedestrians_neg.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/BG.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/BG.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/GP.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/GP.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/SH.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/SH.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/beck-s.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/beck-s.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/farmer-d.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/farmer-d.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/kaminski-v.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/kaminski-v.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/kitchen-l.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/kitchen-l.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/lokay-m.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/lokay-m.tar.gz -------------------------------------------------------------------------------- /notebooks/data/chapter7/williams-w3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/chapter7/williams-w3.tar.gz -------------------------------------------------------------------------------- /notebooks/data/cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/cover.jpg -------------------------------------------------------------------------------- /notebooks/data/lena.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/data/lena.jpg -------------------------------------------------------------------------------- /notebooks/figures/02.03-sine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/figures/02.03-sine.png -------------------------------------------------------------------------------- /notebooks/figures/02.04-digit0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/figures/02.04-digit0.png -------------------------------------------------------------------------------- /notebooks/figures/02.04-digits0-9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbeyeler/opencv-machine-learning/954727c0ef52e281c1388efe9a0bc60a607db935/notebooks/figures/02.04-digits0-9.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.12 2 | scipy==0.19.1 3 | scikit-learn==0.18.1 4 | matplotlib 5 | opencv==3.1 6 | jupyter==1.0 7 | notebook>=5.7.2 8 | pandas==0.22 9 | theano 10 | keras==2.1.5 11 | mkl-service==1.1.2 12 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | # Tools 2 | 3 | These are tools for managing the notebooks in this repository, adapted from 4 | Jake Vanderplas' [Data Science Handbook](https://github.com/jakevdp/PythonDataScienceHandbook). 5 | 6 | - ``generate_contents.py``: This will generate a markdown table of contents for use in the README.md. 7 | 8 | - ``add_navigation.py``: This script adds navigation links at the top and bottom of each notebook. 9 | 10 | - ``add_book_info.py``: This script adds book information to the top of each notebook. -------------------------------------------------------------------------------- /tools/add_book_info.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import nbformat 4 | from nbformat.v4.nbbase import new_markdown_cell 5 | 6 | from generate_contents import iter_notebooks, NOTEBOOK_DIR 7 | 8 | 9 | BOOK_COMMENT = "" 10 | 11 | 12 | BOOK_INFO = BOOK_COMMENT + """ 13 | 14 | *This notebook contains an excerpt from the book [Machine Learning for OpenCV](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv) by Michael Beyeler. 15 | The code is released under the [MIT license](https://opensource.org/licenses/MIT), 16 | and is available on [GitHub](https://github.com/mbeyeler/opencv-machine-learning).* 17 | 18 | *Note that this excerpt contains only the raw code - the book is rich with additional explanations and illustrations. 19 | If you find this content useful, please consider supporting the work by 20 | [buying the book](https://www.packtpub.com/big-data-and-business-intelligence/machine-learning-opencv)!*""" 21 | 22 | 23 | def add_book_info(): 24 | for nb_name in iter_notebooks(): 25 | nb_file = os.path.join(NOTEBOOK_DIR, nb_name) 26 | nb = nbformat.read(nb_file, as_version=4) 27 | 28 | is_comment = lambda cell: cell.source.startswith(BOOK_COMMENT) 29 | 30 | if is_comment(nb.cells[0]): 31 | print('- amending comment for {0}'.format(nb_name)) 32 | nb.cells[0].source = BOOK_INFO 33 | else: 34 | print('- inserting comment for {0}'.format(nb_name)) 35 | nb.cells.insert(0, new_markdown_cell(BOOK_INFO)) 36 | nbformat.write(nb, nb_file) 37 | 38 | 39 | if __name__ == '__main__': 40 | add_book_info() 41 | -------------------------------------------------------------------------------- /tools/add_navigation.py: -------------------------------------------------------------------------------- 1 | import os 2 | import itertools 3 | 4 | from ipykernel import kernelspec as ks 5 | import nbformat 6 | from nbformat.v4.nbbase import new_markdown_cell 7 | 8 | from generate_contents import NOTEBOOK_DIR, REG, iter_notebooks, get_notebook_title 9 | 10 | 11 | def prev_this_next(it): 12 | a, b, c = itertools.tee(it, 3) 13 | next(c) 14 | return zip(itertools.chain([None], a), b, itertools.chain(c, [None])) 15 | 16 | 17 | PREV_TEMPLATE = "< [{title}]({url}) " 18 | CONTENTS = "| [Contents](../README.md) |" 19 | NEXT_TEMPLATE = " [{title}]({url}) >" 20 | NAV_COMMENT = "\n" 21 | 22 | 23 | def iter_navbars(): 24 | for prev_nb, nb, next_nb in prev_this_next(iter_notebooks()): 25 | navbar = NAV_COMMENT 26 | if prev_nb: 27 | navbar += PREV_TEMPLATE.format(title=get_notebook_title(prev_nb), 28 | url=prev_nb) 29 | navbar += CONTENTS 30 | if next_nb: 31 | navbar += NEXT_TEMPLATE.format(title=get_notebook_title(next_nb), 32 | url=next_nb) 33 | yield os.path.join(NOTEBOOK_DIR, nb), navbar 34 | 35 | 36 | def write_navbars(): 37 | for nb_name, navbar in iter_navbars(): 38 | nb = nbformat.read(nb_name, as_version=4) 39 | nb_file = os.path.basename(nb_name) 40 | is_comment = lambda cell: cell.source.startswith(NAV_COMMENT) 41 | 42 | if is_comment(nb.cells[1]): 43 | print("- amending navbar for {0}".format(nb_file)) 44 | nb.cells[1].source = navbar 45 | else: 46 | print("- inserting navbar for {0}".format(nb_file)) 47 | nb.cells.insert(1, new_markdown_cell(source=navbar)) 48 | 49 | if is_comment(nb.cells[-1]): 50 | nb.cells[-1].source = navbar 51 | else: 52 | nb.cells.append(new_markdown_cell(source=navbar)) 53 | nbformat.write(nb, nb_name) 54 | 55 | 56 | if __name__ == '__main__': 57 | write_navbars() 58 | -------------------------------------------------------------------------------- /tools/fix_kernelspec.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import nbformat 4 | 5 | from generate_contents import iter_notebooks, NOTEBOOK_DIR 6 | 7 | 8 | def fix_kernelspec(): 9 | for nb_name in iter_notebooks(): 10 | nb_file = os.path.join(NOTEBOOK_DIR, nb_name) 11 | nb = nbformat.read(nb_file, as_version=4) 12 | 13 | print("- Updating kernelspec for {0}".format(nb_name)) 14 | nb['metadata']['kernelspec']['display_name'] = 'Python 3' 15 | nb['metadata']['kernelspec']['name'] = 'python3' 16 | 17 | nbformat.write(nb, nb_file) 18 | 19 | 20 | if __name__ == '__main__': 21 | fix_kernelspec() 22 | -------------------------------------------------------------------------------- /tools/generate_contents.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import itertools 4 | import nbformat 5 | 6 | NOTEBOOK_DIR = os.path.join(os.path.dirname(__file__), '..', 'notebooks') 7 | 8 | CHAPTERS = {"00": "Preface", 9 | "01": "A Taste of Machine Learning", 10 | "02": "Working with Data in OpenCV", 11 | "03": "First Steps in Supervised Learning", 12 | "04": "Representing Data and Engineering Features", 13 | "05": "Using Decision Trees to Make a Medical Diagnosis", 14 | "06": "Detecting Pedestrians with Support Vector Machines", 15 | "07": "Implementing a Spam Filter with Bayesian Learning", 16 | "08": "Discovering Hidden Structures with Unsupervised Learning", 17 | "09": "Using Deep Learning to Classify Handwritten Digits", 18 | "10": "Combining Different Algorithms Into an Ensemble", 19 | "11": "Selecting the Right Model with Hyper-Parameter Tuning", 20 | "12": "Conclusion"} 21 | 22 | REG = re.compile(r'(\d\d)\.(\d\d)-(.*)\.ipynb') 23 | 24 | 25 | def iter_notebooks(): 26 | return sorted(nb for nb in os.listdir(NOTEBOOK_DIR) if REG.match(nb)) 27 | 28 | 29 | def get_notebook_title(nb_file): 30 | nb = nbformat.read(os.path.join(NOTEBOOK_DIR, nb_file), as_version=4) 31 | for cell in nb.cells: 32 | if cell.source.startswith('#'): 33 | return cell.source[1:].splitlines()[0].strip() 34 | 35 | 36 | def gen_contents(directory=None): 37 | for nb in iter_notebooks(): 38 | if directory: 39 | nb_url = os.path.join(directory, nb) 40 | else: 41 | nb_url = nb 42 | chapter, section, title = REG.match(nb).groups() 43 | title = get_notebook_title(nb) 44 | if section == '00': 45 | if chapter in ['00', '06']: 46 | yield '\n### [{0}]({1})'.format(title, nb_url) 47 | else: 48 | yield '\n### [{0}. {1}]({2})'.format(int(chapter), 49 | title, nb_url) 50 | else: 51 | yield "- [{0}]({1})".format(title, nb_url) 52 | 53 | 54 | def print_contents(directory=None): 55 | print('\n'.join(gen_contents(directory))) 56 | 57 | 58 | if __name__ == '__main__': 59 | print_contents() 60 | print('\n', 70 * '#', '\n') 61 | print_contents('http://nbviewer.jupyter.org/github/mbeyeler/opencv-machine-learning/blob/master/notebooks/') 62 | --------------------------------------------------------------------------------