├── .gitignore ├── LICENSE ├── README.rst ├── docs ├── Makefile ├── conf.py ├── examples.plankton.rst ├── index.rst ├── make.bat ├── pydnn.aws_util.rst ├── pydnn.data.rst ├── pydnn.img_util.rst ├── pydnn.neuralnet.rst ├── pydnn.preprocess.rst ├── pydnn.tools.rst └── readme.rst ├── examples ├── __init__.py └── plankton │ ├── __init__.py │ ├── experiment.py │ ├── plankton.conf │ ├── plankton.py │ └── weight_averages.ods ├── pydnn ├── __init__.py ├── aws_util.conf ├── aws_util.py ├── data.py ├── img_util.py ├── neuralnet.py ├── preprocess.py └── tools.py ├── setup.py └── tests ├── examples └── test_plankton.py └── test_nn.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # pycharm ide 6 | .idea 7 | 8 | # temp files 9 | .~*# 10 | scratch 11 | 12 | # generate sphinx documentation 13 | docs/_build/ 14 | 15 | # distribution / packaging 16 | *.egg-info/ 17 | build/ 18 | dist/ 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Isaac Kriegman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ****************************************************************************** 2 | pydnn: High performance GPU neural network library for deep learning in Python 3 | ****************************************************************************** 4 | 5 | pydnn is a deep neural network library written in Python using `Theano `_ (symbolic math and optimizing compiler package). It was written for `Kaggle's National Data Science Bowl `_ competition in March 2015, where it produced an entry finishing in the `top 6% `_. Continued development is planned, including support for even more of the most important deep learning techniques (RNNs...) 6 | 7 | .. contents:: 8 | 9 | ============ 10 | Design Goals 11 | ============ 12 | 13 | * **Simplicity** 14 | Wherever possible simplify code to make it a clear expression of underlying deep learning algorithms. Minimize cognitive overhead, so that it is easy for someone who has completed the `deeplearning.net tutorials `_ to pickup this library as a next step and easily start learning about, using, and coding more advanced techniques. 15 | 16 | * **Completeness** 17 | Include all the important and popular techniques for effective deep learning and **not** techniques with more marginal or ambiguous benefit. 18 | 19 | * **Ease of use** 20 | Make preparing a dataset, building a model and training a deep network only a few lines of code; enable users to work with NumPy rather than Theano. 21 | 22 | * **Performance** 23 | Should be roughly on par with other Theano neural net libraries so that pydnn is a viable choice for computationally intensive deep learning. 24 | 25 | ======== 26 | Features 27 | ======== 28 | 29 | * High performance GPU training (courtesy of Theano) 30 | * Quick start tools to instantly get started training on `inexpensive `_ Amazon EC2 GPU instances. 31 | * Implementations of important new techniques recently reported in the literature: 32 | * `Batch Normalization `_ 33 | * `Parametric ReLU `_ activation function, 34 | * `Adam `_ optimization 35 | * `AdaDelta `_ optimization 36 | * etc. 37 | * Implementations of standard deep learning techniques: 38 | * Stochastic Gradient Descent with Momentum 39 | * Dropout 40 | * convolutions with max-pooling using overlapping windows 41 | * ReLU/Tanh/sigmoid activation functions 42 | * etc. 43 | 44 | ============= 45 | Documentation 46 | ============= 47 | 48 | http://pydnn.readthedocs.org/en/latest/index.html 49 | 50 | ============ 51 | Installation 52 | ============ 53 | 54 | pip install pydnn 55 | 56 | ========= 57 | Home Page 58 | ========= 59 | 60 | https://github.com/zackriegman/pydnn 61 | 62 | ===== 63 | Usage 64 | ===== 65 | 66 | First download and unzip raw image data from somewhere (e.g. Kaggle). Then:: 67 | 68 | import pydnn 69 | import numpy as np 70 | rng = np.random.RandomState(e.rng_seed) 71 | 72 | # build data, split into training/validation sets, preprocess 73 | train_dir = 'home\ubuntu\train' 74 | data = pydnn.data.DirectoryLabeledImageSet(train_dir).build() 75 | data = pydnn.preprocess.split_training_data(data, 64, 80, 15, 5) 76 | resizer = pydnn.preprocess.StretchResizer() 77 | pre = pydnn.preprocess.Rotator360(data, (64, 64), resizer, rng) 78 | 79 | # build the neural network 80 | net = pydnn.nn.NN(pre, 'images', 121, 64, rng, pydnn.nn.relu) 81 | net.add_convolution(72, (7, 7), (2, 2)) 82 | net.add_dropout() 83 | net.add_convolution(128, (5, 5), (2, 2)) 84 | net.add_dropout() 85 | net.add_convolution(128, (3, 3), (2, 2)) 86 | net.add_dropout() 87 | net.add_hidden(3072) 88 | net.add_dropout() 89 | net.add_hidden(3072) 90 | net.add_dropout() 91 | net.add_logistic() 92 | 93 | # train the network 94 | lr = pydnn.nn.Adam(learning_rate=pydnn.nn.LearningRateDecay( 95 | learning_rate=0.006, 96 | decay=.1)) 97 | net.train(lr) 98 | 99 | From raw data to trained network (including specifying 100 | network architecture) in 25 lines of code. 101 | 102 | 103 | ================ 104 | Short Term Goals 105 | ================ 106 | 107 | * Implement popular RNN techniques. 108 | * Integrate with Amazon EC2 clustering software (such as `StarCluster `_). 109 | * Integrate with hyper-parameter optimization frameworks (such as `Spearmint `_ and `hyperopt `_). 110 | 111 | ======= 112 | Authors 113 | ======= 114 | 115 | Isaac Kriegman -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pydnn.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pydnn.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pydnn" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pydnn" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pydnn documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Mar 24 09:05:39 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import shlex 18 | 19 | # http://docs.readthedocs.org/en/latest/faq.html 20 | # from mock import Mock as MagicMock 21 | # class Mock(MagicMock): 22 | # @classmethod 23 | # def __getattr__(cls, name): 24 | # return Mock() 25 | # MOCK_MODULES = ['numpy', 26 | # 'scipy', 27 | # 'theano', 28 | # # 'yaml', 29 | # 'pandas'] 30 | # sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) 31 | 32 | # http://blog.rtwilson.com/how-to-make-your-sphinx-documentation-compile-with-readthedocs-when-youre-using-numpy-and-scipy/ 33 | import mock 34 | for mod_name in ['numpy', 35 | 'distutils', 36 | 'scipy', 37 | 'scipy.misc', 38 | 'scipy.ndimage', 39 | 'theano', 40 | 'theano.tensor.signal', 41 | 'theano.tensor.nnet', 42 | 'theano.tensor', 43 | 'theano.ifelse', 44 | 'theano.printing', 45 | 'theano.gof', 46 | 'theano.gof.graph', 47 | 'gof.graph', 48 | 'boto.ec2', 49 | 'boto.ec2.blockdevicemapping', 50 | # 'yaml', 51 | 'pandas']: 52 | sys.modules[mod_name] = mock.Mock() 53 | 54 | # If extensions (or modules to document with autodoc) are in another directory, 55 | # add these directories to sys.path here. If the directory is relative to the 56 | # documentation root, use os.path.abspath to make it absolute, like shown here. 57 | #sys.path.insert(0, os.path.abspath('.')) 58 | 59 | # -- General configuration ------------------------------------------------ 60 | 61 | # If your documentation needs a minimal Sphinx version, state it here. 62 | #needs_sphinx = '1.0' 63 | 64 | # Add any Sphinx extension module names here, as strings. They can be 65 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 66 | # ones. 67 | extensions = [ 68 | 'sphinx.ext.autodoc', 69 | 'sphinx.ext.todo' 70 | # 'sphinx.ext.viewcode', 71 | # 'sphinx.ext.pngmath' 72 | ] 73 | 74 | # Add any paths that contain templates here, relative to this directory. 75 | templates_path = ['_templates'] 76 | 77 | # The suffix(es) of source filenames. 78 | # You can specify multiple suffix as a list of string: 79 | # source_suffix = ['.rst', '.md'] 80 | source_suffix = '.rst' 81 | 82 | # The encoding of source files. 83 | #source_encoding = 'utf-8-sig' 84 | 85 | # The master toctree document. 86 | master_doc = 'index' 87 | 88 | # General information about the project. 89 | project = u'pydnn' 90 | copyright = u'2015, Isaac Kriegman' 91 | author = u'Isaac Kriegman' 92 | 93 | # The version info for the project you're documenting, acts as replacement for 94 | # |version| and |release|, also used in various other places throughout the 95 | # built documents. 96 | # 97 | # The short X.Y version. 98 | version = '0.0' 99 | # The full version, including alpha/beta/rc tags. 100 | release = '0.0' 101 | 102 | # The language for content autogenerated by Sphinx. Refer to documentation 103 | # for a list of supported languages. 104 | # 105 | # This is also used if you do content translation via gettext catalogs. 106 | # Usually you set "language" from the command line for these cases. 107 | language = 'en' 108 | 109 | # There are two options for replacing |today|: either, you set today to some 110 | # non-false value, then it is used: 111 | #today = '' 112 | # Else, today_fmt is used as the format for a strftime call. 113 | #today_fmt = '%B %d, %Y' 114 | 115 | # List of patterns, relative to source directory, that match files and 116 | # directories to ignore when looking for source files. 117 | exclude_patterns = ['_build'] 118 | 119 | # The reST default role (used for this markup: `text`) to use for all 120 | # documents. 121 | #default_role = None 122 | 123 | # If true, '()' will be appended to :func: etc. cross-reference text. 124 | #add_function_parentheses = True 125 | 126 | # If true, the current module name will be prepended to all description 127 | # unit titles (such as .. function::). 128 | #add_module_names = True 129 | 130 | # If true, sectionauthor and moduleauthor directives will be shown in the 131 | # output. They are ignored by default. 132 | #show_authors = False 133 | 134 | # The name of the Pygments (syntax highlighting) style to use. 135 | pygments_style = 'sphinx' 136 | 137 | # A list of ignored prefixes for module index sorting. 138 | #modindex_common_prefix = [] 139 | 140 | # If true, keep warnings as "system message" paragraphs in the built documents. 141 | #keep_warnings = False 142 | 143 | # If true, `todo` and `todoList` produce output, else they produce nothing. 144 | todo_include_todos = True 145 | 146 | 147 | # -- Options for HTML output ---------------------------------------------- 148 | 149 | # The theme to use for HTML and HTML Help pages. See the documentation for 150 | # a list of builtin themes. 151 | # html_theme = 'alabaster' 152 | 153 | # Theme options are theme-specific and customize the look and feel of a theme 154 | # further. For a list of options available for each theme, see the 155 | # documentation. 156 | #html_theme_options = {} 157 | 158 | # Add any paths that contain custom themes here, relative to this directory. 159 | #html_theme_path = [] 160 | 161 | # The name for this set of Sphinx documents. If None, it defaults to 162 | # " v documentation". 163 | #html_title = None 164 | 165 | # A shorter title for the navigation bar. Default is the same as html_title. 166 | #html_short_title = None 167 | 168 | # The name of an image file (relative to this directory) to place at the top 169 | # of the sidebar. 170 | #html_logo = None 171 | 172 | # The name of an image file (within the static path) to use as favicon of the 173 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 174 | # pixels large. 175 | #html_favicon = None 176 | 177 | # Add any paths that contain custom static files (such as style sheets) here, 178 | # relative to this directory. They are copied after the builtin static files, 179 | # so a file named "default.css" will overwrite the builtin "default.css". 180 | html_static_path = ['_static'] 181 | 182 | # Add any extra paths that contain custom files (such as robots.txt or 183 | # .htaccess) here, relative to this directory. These files are copied 184 | # directly to the root of the documentation. 185 | #html_extra_path = [] 186 | 187 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 188 | # using the given strftime format. 189 | #html_last_updated_fmt = '%b %d, %Y' 190 | 191 | # If true, SmartyPants will be used to convert quotes and dashes to 192 | # typographically correct entities. 193 | #html_use_smartypants = True 194 | 195 | # Custom sidebar templates, maps document names to template names. 196 | #html_sidebars = {} 197 | 198 | # Additional templates that should be rendered to pages, maps page names to 199 | # template names. 200 | #html_additional_pages = {} 201 | 202 | # If false, no module index is generated. 203 | #html_domain_indices = True 204 | 205 | # If false, no index is generated. 206 | #html_use_index = True 207 | 208 | # If true, the index is split into individual pages for each letter. 209 | #html_split_index = False 210 | 211 | # If true, links to the reST sources are added to the pages. 212 | #html_show_sourcelink = True 213 | 214 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 215 | #html_show_sphinx = True 216 | 217 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 218 | #html_show_copyright = True 219 | 220 | # If true, an OpenSearch description file will be output, and all pages will 221 | # contain a tag referring to it. The value of this option must be the 222 | # base URL from which the finished HTML is served. 223 | #html_use_opensearch = '' 224 | 225 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 226 | #html_file_suffix = None 227 | 228 | # Language to be used for generating the HTML full-text search index. 229 | # Sphinx supports the following languages: 230 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 231 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 232 | #html_search_language = 'en' 233 | 234 | # A dictionary with options for the search language support, empty by default. 235 | # Now only 'ja' uses this config value 236 | #html_search_options = {'type': 'default'} 237 | 238 | # The name of a javascript file (relative to the configuration directory) that 239 | # implements a search results scorer. If empty, the default will be used. 240 | #html_search_scorer = 'scorer.js' 241 | 242 | # Output file base name for HTML help builder. 243 | htmlhelp_basename = 'pydnndoc' 244 | 245 | # -- Options for LaTeX output --------------------------------------------- 246 | 247 | latex_elements = { 248 | # The paper size ('letterpaper' or 'a4paper'). 249 | #'papersize': 'letterpaper', 250 | 251 | # The font size ('10pt', '11pt' or '12pt'). 252 | #'pointsize': '10pt', 253 | 254 | # Additional stuff for the LaTeX preamble. 255 | #'preamble': '', 256 | 257 | # Latex figure (float) alignment 258 | #'figure_align': 'htbp', 259 | } 260 | 261 | # Grouping the document tree into LaTeX files. List of tuples 262 | # (source start file, target name, title, 263 | # author, documentclass [howto, manual, or own class]). 264 | latex_documents = [ 265 | (master_doc, 'pydnn.tex', u'pydnn Documentation', 266 | u'Isaac Kriegman', 'manual'), 267 | ] 268 | 269 | # The name of an image file (relative to this directory) to place at the top of 270 | # the title page. 271 | #latex_logo = None 272 | 273 | # For "manual" documents, if this is true, then toplevel headings are parts, 274 | # not chapters. 275 | #latex_use_parts = False 276 | 277 | # If true, show page references after internal links. 278 | #latex_show_pagerefs = False 279 | 280 | # If true, show URL addresses after external links. 281 | #latex_show_urls = False 282 | 283 | # Documents to append as an appendix to all manuals. 284 | #latex_appendices = [] 285 | 286 | # If false, no module index is generated. 287 | #latex_domain_indices = True 288 | 289 | 290 | # -- Options for manual page output --------------------------------------- 291 | 292 | # One entry per manual page. List of tuples 293 | # (source start file, name, description, authors, manual section). 294 | man_pages = [ 295 | (master_doc, 'pydnn', u'pydnn Documentation', 296 | [author], 1) 297 | ] 298 | 299 | # If true, show URL addresses after external links. 300 | #man_show_urls = False 301 | 302 | 303 | # -- Options for Texinfo output ------------------------------------------- 304 | 305 | # Grouping the document tree into Texinfo files. List of tuples 306 | # (source start file, target name, title, author, 307 | # dir menu entry, description, category) 308 | texinfo_documents = [ 309 | (master_doc, 'pydnn', u'pydnn Documentation', 310 | author, 'pydnn', 'One line description of project.', 311 | 'Miscellaneous'), 312 | ] 313 | 314 | # Documents to append as an appendix to all manuals. 315 | #texinfo_appendices = [] 316 | 317 | # If false, no module index is generated. 318 | #texinfo_domain_indices = True 319 | 320 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 321 | #texinfo_show_urls = 'footnote' 322 | 323 | # If true, do not generate a @detailmenu in the "Top" node's menu. 324 | #texinfo_no_detailmenu = False 325 | 326 | 327 | # -- Options for Epub output ---------------------------------------------- 328 | 329 | # Bibliographic Dublin Core info. 330 | epub_title = project 331 | epub_author = author 332 | epub_publisher = author 333 | epub_copyright = copyright 334 | 335 | # The basename for the epub file. It defaults to the project name. 336 | #epub_basename = project 337 | 338 | # The HTML theme for the epub output. Since the default themes are not optimized 339 | # for small screen space, using the same theme for HTML and epub output is 340 | # usually not wise. This defaults to 'epub', a theme designed to save visual 341 | # space. 342 | #epub_theme = 'epub' 343 | 344 | # The language of the text. It defaults to the language option 345 | # or 'en' if the language is not set. 346 | #epub_language = '' 347 | 348 | # The scheme of the identifier. Typical schemes are ISBN or URL. 349 | #epub_scheme = '' 350 | 351 | # The unique identifier of the text. This can be a ISBN number 352 | # or the project homepage. 353 | #epub_identifier = '' 354 | 355 | # A unique identification for the text. 356 | #epub_uid = '' 357 | 358 | # A tuple containing the cover image and cover page html template filenames. 359 | #epub_cover = () 360 | 361 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 362 | #epub_guide = () 363 | 364 | # HTML files that should be inserted before the pages created by sphinx. 365 | # The format is a list of tuples containing the path and title. 366 | #epub_pre_files = [] 367 | 368 | # HTML files shat should be inserted after the pages created by sphinx. 369 | # The format is a list of tuples containing the path and title. 370 | #epub_post_files = [] 371 | 372 | # A list of files that should not be packed into the epub file. 373 | epub_exclude_files = ['search.html'] 374 | 375 | # The depth of the table of contents in toc.ncx. 376 | #epub_tocdepth = 3 377 | 378 | # Allow duplicate toc entries. 379 | #epub_tocdup = True 380 | 381 | # Choose between 'default' and 'includehidden'. 382 | #epub_tocscope = 'default' 383 | 384 | # Fix unsupported image types using the Pillow. 385 | #epub_fix_images = False 386 | 387 | # Scale large images. 388 | #epub_max_image_width = 0 389 | 390 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 391 | #epub_show_urls = 'inline' 392 | 393 | # If false, no index is generated. 394 | #epub_use_index = True 395 | 396 | 397 | # def skip(app, what, name, obj, skip, options): 398 | # if name == "__init__": 399 | # return False 400 | # return skip 401 | # 402 | # def setup(app): 403 | # app.connect("autodoc-skip-member", skip) -------------------------------------------------------------------------------- /docs/examples.plankton.rst: -------------------------------------------------------------------------------- 1 | examples.plankton package 2 | ========================= 3 | 4 | .. automodule:: examples.plankton.plankton 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pydnn documentation master file, created by 2 | sphinx-quickstart on Tue Mar 24 09:05:39 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pydnn's documentation! 7 | ================================= 8 | 9 | 10 | .. toctree:: 11 | :maxdepth: 4 12 | 13 | readme 14 | pydnn.neuralnet 15 | pydnn.preprocess 16 | pydnn.data 17 | pydnn.aws_util 18 | pydnn.img_util 19 | pydnn.tools 20 | examples.plankton 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pydnn.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pydnn.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /docs/pydnn.aws_util.rst: -------------------------------------------------------------------------------- 1 | pydnn.aws_util module 2 | ===================== 3 | 4 | .. automodule:: pydnn.aws_util 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/pydnn.data.rst: -------------------------------------------------------------------------------- 1 | pydnn.data module 2 | ================= 3 | 4 | .. automodule:: pydnn.data 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/pydnn.img_util.rst: -------------------------------------------------------------------------------- 1 | pydnn.img_util module 2 | ===================== 3 | 4 | .. automodule:: pydnn.img_util 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/pydnn.neuralnet.rst: -------------------------------------------------------------------------------- 1 | pydnn.neuralnet module 2 | ====================== 3 | 4 | .. py:module:: pydnn.neuralnet 5 | 6 | Overview 7 | -------- 8 | 9 | :class:`NN` is the workhorse of pydnn. Using an instance of :class:`NN` the user defines the network, trains the network and uses the network for inference. :class:`NN` takes care of the bookkeeping and wires the layers together, calculating any intermediate configuration necessary for doing so without user input. See the :ref:`section on NN ` for more details. 10 | 11 | Learning rules define how the network updates weights based on the gradients calculated during training. Learning rules are passed to :class:`NN` objects when calling :func:`NN.train` to train the network. :class:`Momentum` and :class:`Adam` are good default choices. See :ref:`Learning_Rules` for more details. 12 | 13 | All the learning rules defined in this package depend in part on a global learning rate that effects how all parameters are updated on training passes. It is frequently beneficial to anneal the learning rate over the course of training and different approaches to annealing can result in substantially different convergence losses and times. Different approaches to annealing can be achieved by using one of the various learning rate annealing objects which are passed to :class:`LearningRule` objects during instantiation. :class:`LearningRateDecay` is a good default choice. See :ref:`Learning_Rates` for more details. 14 | 15 | A variety of activation functions, or nonlinearities, can be applied to layers. :func:`relu` is the most common, however :class:`PReLULayer` has recently been reported to achieve state of the art results. See :ref:`Activations` for more details. 16 | 17 | Finally there are a few utilities for saving and reloading trained networks and for estimating the size and training time for networks before training. See :ref:`Utilities` for more details. 18 | 19 | .. contents:: 20 | 21 | .. _NN_section: 22 | 23 | The main class: NN 24 | ------------------ 25 | 26 | .. autoclass:: NN 27 | :members: 28 | :member-order: bysource 29 | 30 | .. _Learning_Rules: 31 | 32 | Learning Rules (Optimization Methods) 33 | ------------------------------------- 34 | 35 | .. autoclass:: LearningRule 36 | .. autoclass:: StochasticGradientDescent 37 | :members: 38 | :inherited-members: 39 | .. autoclass:: Momentum 40 | :members: 41 | :inherited-members: 42 | .. autoclass:: Adam 43 | :members: 44 | :inherited-members: 45 | .. autoclass:: AdaDelta 46 | :members: 47 | :inherited-members: 48 | 49 | .. _Learning_Rates: 50 | 51 | Learning Rate Annealing 52 | ----------------------- 53 | 54 | .. autoclass:: LearningRateAdjuster 55 | .. autoclass:: LearningRateDecay 56 | :members: 57 | :inherited-members: 58 | .. autoclass:: LearningRateSchedule 59 | :members: 60 | :inherited-members: 61 | .. autoclass:: WackyLearningRateAnnealer 62 | :members: 63 | :inherited-members: 64 | 65 | .. _Activations: 66 | 67 | Activation Functions (Nonlinearities) 68 | ------------------------------------- 69 | 70 | .. autofunction:: relu 71 | .. autofunction:: prelu 72 | .. autofunction:: sigmoid 73 | .. autofunction:: tanh 74 | 75 | .. _Utilities: 76 | 77 | Utility Functions 78 | ----------------- 79 | 80 | .. autofunction:: save 81 | .. autofunction:: load 82 | .. autofunction:: net_size 83 | -------------------------------------------------------------------------------- /docs/pydnn.preprocess.rst: -------------------------------------------------------------------------------- 1 | pydnn.preprocess module 2 | ======================= 3 | 4 | .. py:module:: pydnn.preprocess 5 | 6 | Overview 7 | -------- 8 | 9 | Most of the code in this module is currently pretty specific to processing images like those in Kaggle's plankton competition. Those images were unique in that they (1) were presented with a uniform background, (2) they varied in size in a way that provided meaningful information about the subject, and (3) they were mostly randomly oriented. These features have to do with real world constraints on the way that marine biologist collect the images, and are obviously quite different from popular datasets like ImageNet, MNIST, etc. As I (or others) use pydnn in a greater variety of machine learning contexts a variety of preprocessing approaches can be maintained here. 10 | 11 | .. contents:: 12 | 13 | Training Set 14 | ------------ 15 | 16 | .. autofunction:: split_training_data 17 | 18 | .. _Preprocessors: 19 | 20 | Preprocessors 21 | ------------- 22 | 23 | Preprocessors take care of online augmentation, shuffling, zero centering and normalizing, resizing, and other related transformations of the traning data. Because the plankton images could be in any orientation, to achieve good performance it was important to augment the data with many rotations of the training set so the network could learn to recognize images in different orientations. Initially I experimented with 90 degree rotations and a flip, however I found that unconstrained degree rotations (:class:`Rotator360` and :class:`Rotator360PlusGeometry`) performed better. Another approach that I experimented with was rotating and flipping all images into a canonicalized orientation based on their shape and size (:class:`Canonicalizer`), which significantly improves early training progress, but shortly thereafter falls behind a 360 degree rotation approach. 24 | 25 | Another thing that these preprocessors do is add additional data channels. For instance, since, in the case of the plankton dataset, the size of the images carries important information (because image size was related to the size of the organism) it was useful to add a data channel with the original image size (:class:`Rotator360`), because that information is lost when uniformly resizing images to be fed into the neural network. Another approach, instead of the original image size, was to create a channel with the size of the largest contiguous image shape, and it's rotation in comparison to it's canonicalized rotation (:class:`Rotator360PlusGeometry`). 26 | 27 | .. autoclass:: Rotator360 28 | .. autoclass:: Rotator360PlusGeometry 29 | .. autoclass:: Canonicalizer 30 | 31 | Resizing 32 | -------- 33 | 34 | Users do not use the resizers directly but pass them to a :ref:`preprocessor ` to control how the preprocessor resizes images. 35 | 36 | .. autofunction:: Resizer 37 | .. autofunction:: StretchResizer 38 | .. autofunction:: ContiguousBoxPreserveAspectRatioResizer 39 | .. autofunction:: ContiguousBoxStretchResizer 40 | .. autofunction:: ThresholdBoxPreserveAspectRatioResizer 41 | .. autofunction:: ThresholdBoxStretchResizer 42 | .. autofunction:: PreserveAspectRatioResizer 43 | .. autofunction:: StochasticStretchResizer 44 | 45 | -------------------------------------------------------------------------------- /docs/pydnn.tools.rst: -------------------------------------------------------------------------------- 1 | pydnn.tools module 2 | ================== 3 | 4 | .. automodule:: pydnn.tools 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | 2 | .. include:: ../README.rst -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'isaac' 2 | -------------------------------------------------------------------------------- /examples/plankton/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'isaac' 2 | -------------------------------------------------------------------------------- /examples/plankton/plankton.conf: -------------------------------------------------------------------------------- 1 | plankton: 2 | input_train: # directory containing kaggle training data 3 | input_test: # directory containing kaggle testing data 4 | input_post: # directory to find input for post processing (averaging submissions and generating confusion matrices; likely the same as output directory) 5 | output: # directory to save output from training 6 | dtype: float32 7 | -------------------------------------------------------------------------------- /examples/plankton/plankton.py: -------------------------------------------------------------------------------- 1 | __author__ = 'isaac' 2 | 3 | from pydnn import neuralnet as nn 4 | from pydnn import preprocess as pp 5 | from pydnn import tools 6 | from pydnn import data 7 | from pydnn import img_util 8 | 9 | import numpy as np 10 | import pandas as pd 11 | from scipy.misc import imread 12 | 13 | import os 14 | from os.path import join 15 | import time 16 | 17 | config = tools.load_config('PLANKTON_CONFIG', __file__, 'plankton.conf')['plankton'] 18 | train_set = data.DirectoryLabeledImageSet(config['input_train'], config['dtype']) 19 | test_set = data.UnlabeledImageSet(config['input_test']) 20 | 21 | 22 | def write_submission_csv_file(file_name, probs, image_file_names): 23 | import gzip 24 | df = pd.DataFrame(data=probs, index=image_file_names, columns=train_set.get_labels()) 25 | df.index.name = 'image' 26 | with gzip.open(file_name, 'w') as outFile: 27 | df.to_csv(outFile) 28 | 29 | 30 | def generate_submission_file(net, name, num=None): 31 | if num is None: 32 | num = 130400 33 | 34 | if num < 130400: 35 | batch_size = num 36 | num_batches = 1 37 | else: 38 | batch_size = 16300 39 | num_batches = 8 40 | 41 | probabilities = [] 42 | files = [] 43 | dotter = tools.Dot() 44 | print('generating probabilities...') 45 | for i in range(num_batches): 46 | fns, images, = test_set.build(i * batch_size, 47 | (i + 1) * batch_size) 48 | _, probs = net.predict({'images': images}) 49 | probabilities.append(probs) 50 | files += fns 51 | dotter.dot(str(i) + ' ') 52 | dotter.stop() 53 | probabilities = np.row_stack(probabilities) 54 | print('writing csv file...') 55 | write_submission_csv_file(name, probabilities, files) 56 | 57 | 58 | def load_net_and_generate_submission_file(net_name, submission_name): 59 | print('loading net') 60 | net = nn.load(net_name) 61 | generate_submission_file(net, submission_name) 62 | # n = 'e0??' 63 | # load_net_and_generate_submission_file(n + '_best_net.pkl', n + '_sub_best.csv.gz') 64 | # load_net_and_generate_submission_file(n + '_final_net.pkl', n + '_sub_final.csv.gz') 65 | 66 | 67 | def write_confusion_matrices_to_csv_files(experiment, num_images, matrices): 68 | set_names = ['train', 'valid', 'test'] 69 | labels = train_set.get_labels() 70 | files, given_labels = zip(*train_set.get_files(num_images)) 71 | for (matrix, mistakes), set_name in zip(matrices, set_names): 72 | df = pd.DataFrame(matrix, index=labels, columns=labels) 73 | df.to_csv(join(config['output'], experiment + '_conf_mtrx_' + set_name + '.csv')) 74 | 75 | file_indices, right_indices, wrong_indices = zip(*mistakes) 76 | file_names = [files[index] for index in file_indices] 77 | right_labels = [given_labels[index] for index in file_indices] 78 | wrong_labels = [labels[index] for index in wrong_indices] 79 | df = pd.DataFrame({'wrong': wrong_labels, 'right': right_labels}, 80 | index=file_names) 81 | df.to_csv(join(config['output'], experiment + '_mistakes_' + set_name + '.csv')) 82 | 83 | 84 | def make_confusion_matrix_from_saved_network(e): 85 | print('making confusion matrices...') 86 | data = train_set.build(e.num_images) 87 | data = pp.split_training_data(data, e.batch_size, e.train_pct, e.valid_pct, e.test_pct) 88 | net = nn.load(join(config['input_post'], e.name + '_best_net.pkl')) 89 | net.preprocessor.set_data(data) 90 | write_confusion_matrices_to_csv_files(e.name, net.get_confusion_matrices()) 91 | print('...done making confusion matrices') 92 | 93 | 94 | def analyze_confusion_matrix(matrix_file): 95 | n = 121 96 | rng = np.random.RandomState(123) 97 | x = pd.read_csv(matrix_file, index_col=0) 98 | data = np.index_exp[:n, :n] 99 | 100 | 101 | x['total'] = x.iloc[data].sum(axis=1) 102 | total_predictions = x['total'].sum() 103 | values = x.iloc[data].values # values can sometimes return a copy 104 | np.fill_diagonal(values, 0) # so must save, zero and reassign 105 | x.iloc[data] = values # (I've discovered after some confusion) 106 | x['bad'] = x.iloc[data].sum(axis=1) 107 | total_bad = x['bad'].sum() 108 | x['pct_bad'] = x['bad'] / x['total'] 109 | 110 | top_by_num = x.sort('total', ascending=False)[0:10].index.values 111 | worst_by_num = x.sort('bad', ascending=False)[0:10].index.values 112 | worst_by_num_ct = x.sort('bad', ascending=False)[0:10].values 113 | worst_by_pct = x.sort('pct_bad', ascending=False)[0:10].index.values 114 | worst_by_pct_ct = x.sort('pct_bad', ascending=False)[0:10].values 115 | 116 | print("total predictions: {}".format(total_predictions)) 117 | print("total bad predictions: {}".format(total_bad)) 118 | 119 | print("most common classes (regardless of error rate): " + str(top_by_num)) 120 | 121 | def most_confused_with(c): 122 | # get the row, and only the class values (not the generated columns) 123 | row = x.loc[c] 124 | row = row.iloc[:n] 125 | row.sort(ascending=False) 126 | 127 | last_non_zero = 10 128 | # print(row.iloc[:10].values) 129 | for index, z in enumerate(row.iloc[:last_non_zero].values): 130 | if z <= 0: 131 | last_non_zero = index 132 | break 133 | 134 | # return the top classes for the row 135 | return zip(row.iloc[:last_non_zero].index.values, row.iloc[:last_non_zero].values) 136 | 137 | def print_worst(classes): 138 | for c in classes: 139 | c_total = x.loc[c, 'total'] 140 | c_bad = x.loc[c, 'bad'] 141 | c_contribution_to_error = float(c_bad) / total_bad 142 | c_fair_share_of_error = c_total / total_predictions 143 | print('\nclass {}:'.format(c)) 144 | print('total predictions: {}'.format(c_total)) 145 | print('total bad predictions: {}'.format(c_bad)) 146 | print('fair share of error: {:.3%}'.format(c_fair_share_of_error)) 147 | print('contribution to error: {:.3%} ({:.3f} time fair share)'.format( 148 | c_contribution_to_error, c_contribution_to_error / c_fair_share_of_error)) 149 | print('most often confused with' + str(most_confused_with(c))) 150 | 151 | import matplotlib.pyplot as plt 152 | import matplotlib.cm as cm 153 | import matplotlib.image as mpimg 154 | 155 | def show_worst(worst): 156 | def add_row(directory, count, index): 157 | first = True 158 | for i in range(5): 159 | sub = fig.add_subplot(11, 5, index) 160 | fn = rng.choice(tools.get_files(join(config['input_train'], directory), 161 | cache=True)) 162 | image = mpimg.imread(fn) 163 | plt.imshow(image, interpolation='none', cmap=cm.Greys_r) 164 | if first: 165 | title = '{}: {} ({}x{})'.format( 166 | directory, count, image.shape[0], image.shape[1]) 167 | first = False 168 | else: 169 | title = '({}x{})'.format(image.shape[0], image.shape[1]) 170 | sub.set_title(title, size=10) 171 | 172 | sub.axis('off') 173 | index += 1 174 | return index 175 | 176 | for c in worst: 177 | fig = plt.figure() 178 | pos = add_row(c, x.loc[c, 'bad'], 1) 179 | for i, num in most_confused_with(c): 180 | pos = add_row(i, num, pos) 181 | plt.show() 182 | 183 | print("---------- worst classes by number -----------") 184 | print_worst(worst_by_num) 185 | show_worst(worst_by_num) 186 | print("---------- worst classes by percent ----------") 187 | print_worst(worst_by_pct) 188 | show_worst(worst_by_num) 189 | 190 | print('might also be useful to look at the post transformed images to gain' 191 | 'insight into why the net is not able to recognize them well') 192 | 193 | print('also remember to look at whether confusion is symmetrical (i.e. if A ' 194 | 'is frequently confused for B, is B also frequently confused for A?)') 195 | 196 | print('at some point might be worth looking at the specific image that were' 197 | 'incorrectly classified, but to begin with Im just looking for the most' 198 | 'important trends (classes with the most confusion) and individual images' 199 | 'shouldnt tell me too much') 200 | 201 | 202 | def run_experiment(e): 203 | print('############## {} ################'.format(e.name)) 204 | print('start time: ' + tools.now()) 205 | rng = np.random.RandomState(e.rng_seed) 206 | 207 | data = train_set.build(e.num_images) 208 | data = pp.split_training_data(data, e.batch_size, e.train_pct, e.valid_pct, e.test_pct) 209 | 210 | if e.resizer == pp.StochasticStretchResizer: 211 | resizer = e.resizer(rng, e.stochastic_stretch_range) 212 | elif e.resizer in [pp.ThresholdBoxPreserveAspectRatioResizer, 213 | pp.ThresholdBoxStretchResizer]: 214 | resizer = e.resizer(e.box_threshold) 215 | elif e.resizer in [pp.ContiguousBoxPreserveAspectRatioResizer, 216 | pp.ContiguousBoxStretchResizer]: 217 | resizer = e.resizer(e.contiguous_box_threshold) 218 | else: 219 | resizer = e.resizer() 220 | 221 | preprocessor = e.preprocessor(data, e.image_shape, resizer, rng, config['dtype']) 222 | net = nn.NN(preprocessor=preprocessor, 223 | channel='images', 224 | num_classes=121, 225 | batch_size=e.batch_size, 226 | rng=rng, 227 | activation=e.activation, 228 | name=e.name, 229 | output_dir=config['output']) 230 | e.build_net(net) 231 | 232 | try: 233 | net.train( 234 | updater=e.learning_rule, 235 | epochs=e.epochs, 236 | final_epochs=e.final_epochs, 237 | l1_reg=e.l1_reg, 238 | l2_reg=e.l2_reg) 239 | finally: 240 | print('Experiment "{}" ended'.format(e.name)) 241 | 242 | print('generating probabilities based on final network...') 243 | generate_submission_file(net, 244 | join(config['output'], e.name + '_submission_final.csv.gz'), 245 | e.num_submission_images) 246 | 247 | net = nn.load(join(config['output'], e.name + '_best_net.pkl')) 248 | 249 | print('generating probabilities based on best network...') 250 | generate_submission_file(net, 251 | join(config['output'], e.name + '_submission_best.csv.gz'), 252 | e.num_submission_images) 253 | 254 | print('generating and writing confusion matrix based on best network...') 255 | net.preprocessor.set_data(data) 256 | write_confusion_matrices_to_csv_files(e.name, e.num_images, 257 | net.get_confusion_matrices()) 258 | 259 | print('end time: ' + tools.now()) 260 | return net 261 | 262 | 263 | def average_submissions(in_files, weights=None): 264 | import gzip 265 | subs = [] 266 | for f in in_files: 267 | print('loading ' + f) 268 | with gzip.open(join(config['input_post'], f), 'r') as inFile: 269 | subs.append(np.loadtxt( 270 | fname=inFile, 271 | dtype=config['dtype'], 272 | delimiter=',', 273 | skiprows=1, 274 | usecols=range(1, 122))) 275 | # avg = np.mean(subs, axis=0) 276 | avg = np.average(subs, axis=0, weights=weights) 277 | out_file = (join(config['input_post'], 'avg_probs_' + 278 | time.strftime("%Y-%m-%d--%H-%M-%S") + '.csv.gz')) 279 | print('saving...') 280 | write_submission_csv_file(out_file, avg, test_set.get_files()) 281 | print('done') 282 | 283 | 284 | def show_mistakes(mistakes_file): 285 | mistakes = pd.read_csv(mistakes_file, index_col=0) 286 | for index, row in mistakes.iterrows(): 287 | images = [imread(join(config['input_train'], row['right'], 288 | os.path.basename(index)))] 289 | 290 | right_images = np.random.choice( 291 | tools.get_files(join(config['input_train'], row['right'])), 292 | 9, replace=False) 293 | wrong_images = np.random.choice( 294 | tools.get_files(join(config['input_train'], row['wrong'])), 295 | 10, replace=False) 296 | 297 | images.extend([imread(fn) for fn in right_images]) 298 | images.extend([imread(fn) for fn in wrong_images]) 299 | print(os.path.basename(index), row['right'], row['wrong']) 300 | img_util.show_images_as_tiles(images, size=(128, 128), canvas_dims=(4, 5)) 301 | #show_mistakes(SUBMISSION_DIR + '/results/e075_mistakes_valid.csv') -------------------------------------------------------------------------------- /examples/plankton/weight_averages.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zackriegman/pydnn/e5e70f5213b10fae3da0d2c765667004eea864f3/examples/plankton/weight_averages.ods -------------------------------------------------------------------------------- /pydnn/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'isaac' 2 | -------------------------------------------------------------------------------- /pydnn/aws_util.conf: -------------------------------------------------------------------------------- 1 | ec2: 2 | default_name: cudnn 3 | region: us-east-1 4 | pem_name: 5 | max_price: 0.2 6 | 7 | email: 8 | from: 9 | to: 10 | username: 11 | password: 12 | smtp: smtp.gmail.com:587 13 | -------------------------------------------------------------------------------- /pydnn/aws_util.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | __author__ = 'isaac' 4 | 5 | ''' 6 | See "~/.boto" for aws_access_key_id and aws_secret_access_key 7 | #g2.2xlarge has NVIDIA Kepler GK104GL [GRID K520] GPU 8 | # CUDA Computer Capability: 3.0 9 | 10 | See http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html 11 | for list of valid filters for get_all_images() 12 | ''' 13 | 14 | import boto.ec2 15 | import boto.ec2.blockdevicemapping 16 | from datetime import datetime 17 | from datetime import timedelta 18 | from time import sleep 19 | import argparse 20 | import subprocess 21 | import shlex 22 | import tools 23 | 24 | config = tools.load_config('AWS_UTIL_CONFIG', __file__, 'aws_util.conf') 25 | 26 | 27 | def get_image_id_by_name(conn, name): 28 | image = get_image_by_name(conn, name) 29 | if image is not None: 30 | return image.id 31 | else: 32 | raise Exception("image '{}' not found".format(name)) 33 | 34 | 35 | def get_recent_gpu_price(conn): 36 | def get_price_for_zone(zone): 37 | recent_prices = conn.get_spot_price_history( 38 | start_time=(datetime.utcnow()-timedelta(seconds=1)).isoformat(), 39 | end_time=datetime.utcnow().isoformat(), 40 | instance_type='g2.2xlarge', 41 | product_description='Linux/UNIX', 42 | availability_zone=zone 43 | ) 44 | return recent_prices[0].price 45 | 46 | zones = ['us-east-1' + l for l in ('a', 'b', 'd', 'e')] 47 | prices = [get_price_for_zone(z) for z in zones] 48 | print(zip(zones, prices)) 49 | return max(prices), min(prices), zones[prices.index(min(prices))] 50 | 51 | 52 | def get_running_instances(conn, name=None): 53 | filters = {'instance-state-name': 'running'} 54 | if name: 55 | filters["tag:name"] = name 56 | return conn.get_only_instances(filters=filters) 57 | 58 | 59 | def get_unique_instance(conn, name): 60 | instances = get_running_instances(conn, name) 61 | if len(instances) > 1: 62 | raise Exception('more than one instance fits criteria') 63 | elif len(instances) == 0: 64 | raise Exception('no instances fit criteria') 65 | else: 66 | return instances[0] 67 | 68 | 69 | def start_spot_instance(conn, name, image_id=None): 70 | if len(get_running_instances(conn, name)) != 0: 71 | raise Exception("there is already an instance with that name") 72 | 73 | highest_price, lowest_price, lowest_cost_zone = get_recent_gpu_price(conn) 74 | print('highest price: {}'.format(highest_price)) 75 | # bid_price = highest_price + 0.03 76 | if config['ec2']['max_price'] < lowest_price * 1.5: 77 | raise Exception("bid price is less than 1.5 times current prices... too risky") 78 | 79 | print('bidding {} max price to instantiate image "{}" in zone "{}"'.format( 80 | config['ec2']['max_price'], image_id, lowest_cost_zone)) 81 | 82 | sda1 = boto.ec2.blockdevicemapping.EBSBlockDeviceType() 83 | sda1.size = 10 84 | bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() 85 | bdm['/dev/sda1'] = sda1 86 | 87 | spot_request = conn.request_spot_instances( 88 | price=str(config['ec2']['max_price']), 89 | # image_id='ami-9eaa1cf6', # plain ubuntu 90 | # image_id='ami-2ca87b44', # GPU and theano preinstalled 91 | image_id=image_id, 92 | count=1, 93 | type='one-time', 94 | valid_from=(datetime.utcnow()+timedelta(seconds=5)).isoformat(), 95 | valid_until=(datetime.utcnow()+timedelta(minutes=15)).isoformat(), 96 | key_name=config['ec2']['pem_name'], 97 | security_groups=['all'], 98 | instance_type='g2.2xlarge', 99 | # instance_type='t2.micro', 100 | # block_device_map=bdm, 101 | ebs_optimized=True, 102 | placement=lowest_cost_zone 103 | )[0] 104 | 105 | print('waiting for spot instance request fulfillment') 106 | instance_id = None 107 | dotter = tools.Dot() 108 | while instance_id is None: 109 | dotter.dot() 110 | sleep(10) 111 | request = conn.get_all_spot_instance_requests(spot_request.id)[0] 112 | instance_id = request.instance_id 113 | if instance_id is None and request.state != 'open': 114 | raise Exception("request {}, status: {}".format( 115 | request.state, request.status)) 116 | print("\nspot instance fulfilled with id:" + str(instance_id)) 117 | 118 | sleep(10) # give AWS a chance to setup the IP and everything 119 | instance = conn.get_only_instances(instance_ids=[instance_id])[0] 120 | instance.add_tag("name", name) 121 | # login_exec = "ssh -i {} -o StrictHostKeyChecking=no ubuntu@{}".\ 122 | # format('"' + PEM_FILE + '"', instance.ip_address) 123 | login_exec = get_login_exec(instance.ip_address) 124 | sftp_exec = get_sftp_exec(instance.ip_address) 125 | connect_message = ('connect to instance with "' + login_exec + '" or "' + 126 | sftp_exec + '"') 127 | tools.send_email( 128 | from_addr=config['email']['from'], 129 | to_addr=config['email']['to'], 130 | username=config['email']['username'], 131 | password=config['email']['password'], 132 | smtp=config['email']['smtp'], 133 | subject='Subject: EC2 spot instance "{}" is ready'.format(name), 134 | body=connect_message 135 | ) 136 | print(connect_message) 137 | return instance_id, login_exec, sftp_exec 138 | 139 | 140 | def get_login_exec(instance_ip): 141 | return "ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=100 ubuntu@{}".\ 142 | format(instance_ip) 143 | 144 | 145 | def get_sftp_exec(instance_ip): 146 | return "nautilus ssh://ubuntu@{}/home/ubuntu".format(instance_ip) 147 | 148 | 149 | def get_image_by_name(conn, name): 150 | images = conn.get_all_images(owners=['self'], filters={'name': name}) 151 | if len(images) > 1: 152 | raise Exception("more than one image with name {}".format(name)) 153 | if len(images) == 0: 154 | return None 155 | else: 156 | return images[0] 157 | 158 | 159 | def save_spot_instance(conn, name, image_name, terminate=True): 160 | instance = get_unique_instance(conn, name) 161 | 162 | image = get_image_by_name(conn, image_name) 163 | if image is not None: 164 | print('deleting old persist image') 165 | image.deregister() 166 | 167 | print('creating new persist image...') 168 | dotter = tools.Dot() 169 | image_id = instance.create_image( 170 | name=image_name, 171 | description='image used to persist instance', 172 | no_reboot=False 173 | ) 174 | 175 | sleep(5) # give AWS a change to realize update its image index 176 | 177 | image = conn.get_all_images(image_ids=[image_id])[0] 178 | while image.state == 'pending': 179 | dotter.dot() 180 | sleep(5) 181 | image.update() 182 | if image.state == 'available': 183 | print('successfully created image') 184 | else: 185 | raise Exception( 186 | "failed to create image with message '{}'".format(image.state)) 187 | 188 | print('creating backup of new persist image...') 189 | conn.copy_image( 190 | source_region=config['ec2']['region'], 191 | source_image_id=image_id, 192 | name="_" + image_name+'_'+datetime.now().strftime('%Y%m%d%H%M%S%f'), 193 | description='backup' 194 | ) 195 | 196 | if terminate: 197 | stop_spot_instance_without_saving(conn, instance.id) 198 | 199 | 200 | def launch_instance(conn, name, image_id): 201 | instance_id, login_exec, sftp_exec = start_spot_instance(conn, name, image_id) 202 | sleep(10) # give the computer a chance to boot up and start SSH server 203 | subprocess.call(shlex.split(login_exec)) 204 | 205 | 206 | def ssh(conn, name): 207 | login_exec = get_login_exec(get_unique_instance(conn, name).ip_address) 208 | print(login_exec) 209 | subprocess.call(shlex.split(login_exec)) 210 | 211 | 212 | def sftp(conn, name): 213 | sftp_exec = get_sftp_exec(get_unique_instance(conn, name).ip_address) 214 | print(sftp_exec) 215 | subprocess.Popen(shlex.split(sftp_exec)) 216 | 217 | 218 | def stop_all_spot_instances_without_saving(conn): 219 | instances = get_running_instances(conn) 220 | for instance in instances: 221 | if instance.state == 'running' and instance.spot_instance_request_id: 222 | print("stopping spot instance {}".format(instance.id)) 223 | stop_spot_instance_without_saving(conn, instance.id) 224 | list_instances(conn) 225 | 226 | 227 | def print_instances(instances): 228 | print("-- Instances --") 229 | for instance in instances: 230 | if 'name' in instance.tags: 231 | name = instance.tags['name'] 232 | else: 233 | name = '' 234 | 235 | print("name: {}, id: {}, state: {}, ip: {}, type: {}, launch time: {}".format( 236 | name, instance.id, instance.state, instance.ip_address, 237 | instance.instance_type, 238 | instance.launch_time)) 239 | 240 | 241 | def stop_spot_instance_without_saving(conn, instance_id): 242 | print("terminating instance...") 243 | conn.terminate_instances([instance_id]) 244 | 245 | 246 | def list_instances(conn): 247 | instances = get_running_instances(conn) 248 | print_instances(instances) 249 | 250 | 251 | def list_amis(conn): 252 | print('-- AMIs --') 253 | for image in conn.get_all_images(owners=['self']): 254 | if image.name[0] != '_': 255 | print('name: {}, id: {}'.format(image.name, image.id)) 256 | 257 | 258 | def create_security_group(conn): 259 | print(conn.get_all_security_groups()) 260 | security_group = conn.create_security_group( 261 | 'all', 262 | 'wide open security group' 263 | ) 264 | security_group.authorize('tcp', 0, 65535, '0.0.0.0/0') 265 | print(conn.get_all_security_groups()) 266 | 267 | 268 | def handle_command_line(): 269 | conn = boto.ec2.connect_to_region(config['ec2']['region']) 270 | if conn is None: 271 | raise Exception('connection failed') 272 | 273 | parser = argparse.ArgumentParser( 274 | description="start and stop persistent GPU spot instances on Amazon EC2") 275 | 276 | name_parser = argparse.ArgumentParser(add_help=False) 277 | name_parser.add_argument('instance', 278 | nargs='?', 279 | type=str, 280 | help='name of instance') 281 | 282 | ami_parser = argparse.ArgumentParser(add_help=False) 283 | ami_parser.add_argument('AMI', 284 | nargs='?', 285 | type=str, 286 | help='name of AMI') 287 | 288 | ami_id_parser = argparse.ArgumentParser(add_help=False) 289 | ami_id_parser.add_argument('-a', '--AMI_ID', type=str, help='ID of AMI') 290 | 291 | default_msg = ' (using default instance name and AMI name if not provided)' 292 | default_name_msg = ' (using default instance name if not provided)' 293 | 294 | subparsers = parser.add_subparsers(dest='sub_command') 295 | 296 | def add_cmd(name, desc, func, parents=None): 297 | if parents is None: 298 | option = subparsers.add_parser(name, 299 | help=desc, 300 | description=desc) 301 | else: 302 | option = subparsers.add_parser(name, 303 | help=desc, 304 | description=desc, 305 | parents=parents) 306 | option.set_defaults(func=func) 307 | return option 308 | 309 | def default_start_ami_id(arg): 310 | if arg.AMI_ID is not None: 311 | if arg.AMI is not None: 312 | raise Exception('only specify one of AMI name or AMI id') 313 | name = conn.get_all_images(image_ids=[arg.AMI_ID])[0].name 314 | return arg.AMI_ID, name 315 | else: 316 | if arg.AMI is not None: 317 | return get_image_id_by_name(conn, arg.AMI), arg.AMI 318 | else: 319 | amis = {image.name: image for image in 320 | conn.get_all_images(owners=['self']) 321 | if image.name[0] != '_'} 322 | if len(amis) == 1: 323 | return amis.values()[0].id, amis.values()[0] 324 | elif config['ec2']['default_name'] in amis: 325 | return (amis[config['ec2']['default_name']].id, 326 | config['ec2']['default_name']) 327 | elif len(amis) == 0: 328 | raise Exception('no AMIs available to load') 329 | else: 330 | raise Exception('please specify AMI to load ' + str(amis)) 331 | 332 | def default_start_instance(arg, ami_name): 333 | print("default_start_instance", arg, ami_name) 334 | if arg.instance: 335 | return arg.instance 336 | else: 337 | instances = [instance.tags['name'] for instance in 338 | get_running_instances(conn)] 339 | print("default_start_instance:instances", instances) 340 | if ami_name not in instances: 341 | return ami_name 342 | else: 343 | suffix = 0 344 | while (ami_name + '_' + str(suffix)) in instances: 345 | suffix += 1 346 | return ami_name + '_' + str(suffix) 347 | 348 | def default_save_ami_name(arg, instance_name): 349 | if arg.AMI is not None: 350 | return arg.AMI 351 | else: 352 | return instance_name 353 | 354 | def default_running_instance(arg): 355 | if arg.instance: 356 | return arg.instance 357 | else: 358 | instances = [instance.tags['name'] for instance in 359 | get_running_instances(conn)] 360 | if len(instances) == 1: 361 | return instances[0] 362 | elif config['ec2']['default_name'] in instances: 363 | return config['ec2']['default_name'] 364 | elif len(instances) == 0: 365 | raise Exception('no instances are running') 366 | else: 367 | raise Exception('please specify an instance ' + str(instances)) 368 | 369 | ################ 370 | # sub commands # 371 | ################ 372 | 373 | def cmd_start(arg): 374 | ami_id, ami_name = default_start_ami_id(arg) 375 | instance_name = default_start_instance(arg, ami_name) 376 | print('starting "{}" as "{}"'.format(ami_name, instance_name)) 377 | start_spot_instance(conn, name=instance_name, image_id=ami_id) 378 | add_cmd('start', 'start a spot instance' + default_msg, 379 | cmd_start, [ami_parser, name_parser, ami_id_parser]) 380 | 381 | def cmd_launch(arg): 382 | ami_id, ami_name = default_start_ami_id(arg) 383 | instance_name = default_start_instance(arg, ami_name) 384 | print('starting "{}" as "{}"'.format(ami_name, instance_name)) 385 | launch_instance(conn, name=instance_name, image_id=ami_id) 386 | add_cmd('launch', 'start a spot instance and log in via SSH' + default_msg, 387 | cmd_launch, [ami_parser, name_parser, ami_id_parser]) 388 | 389 | def cmd_ssh(arg): 390 | name = default_running_instance(arg) 391 | print('logging into "{}"'.format(name)) 392 | ssh(conn, name=name) 393 | add_cmd('ssh', 'log into spot instance' + default_name_msg, 394 | cmd_ssh, [name_parser]) 395 | 396 | def cmd_sftp(arg): 397 | name = default_running_instance(arg) 398 | print('opening sftp connection to "{}"'.format(name)) 399 | sftp(conn, name=name) 400 | add_cmd('sftp', 'open sftp connection in file browser' + default_name_msg, 401 | cmd_sftp, [name_parser]) 402 | 403 | def cmd_save(arg): 404 | instance_name = default_running_instance(arg) 405 | ami_name = default_save_ami_name(arg, instance_name) 406 | print('saving "{}" as "{}"'.format(instance_name, ami_name)) 407 | save_spot_instance(conn, name=instance_name, image_name=ami_name, 408 | terminate=False) 409 | add_cmd('save', 'save spot instance and reboot' + default_msg, 410 | cmd_save, [name_parser, ami_parser]) 411 | 412 | def cmd_persist(arg): 413 | instance_name = default_running_instance(arg) 414 | ami_name = default_save_ami_name(arg, instance_name) 415 | print('persisting "{}" as "{}"'.format(instance_name, ami_name)) 416 | save_spot_instance(conn, name=instance_name, image_name=ami_name) 417 | add_cmd('persist', 'save and terminate spot instance' + default_msg, 418 | cmd_persist, [name_parser, ami_parser]) 419 | 420 | def cmd_kill(arg): 421 | if arg.all: 422 | print('killing all instances') 423 | stop_all_spot_instances_without_saving(conn) 424 | elif arg.name: 425 | inst = get_unique_instance(conn, args.name) 426 | print('killing "{}"'.format(args.name)) 427 | stop_spot_instance_without_saving(conn, inst.id) 428 | else: 429 | print('must specify instance to kill') 430 | kill_parser = add_cmd('kill', 'kill spot instance(s)', cmd_kill) 431 | kill_group = kill_parser.add_mutually_exclusive_group() 432 | kill_group.add_argument('name', nargs='?', type=str, 433 | help='name of instance to kill') 434 | kill_group.add_argument('--all', action='store_true') 435 | 436 | def cmd_list(arg): 437 | if arg.instances: 438 | list_instances(conn) 439 | elif arg.amis: 440 | list_amis(conn) 441 | else: 442 | list_instances(conn) 443 | list_amis(conn) 444 | list_parser = add_cmd('list', 445 | 'list all active instances and AMIs associated with account ' 446 | '(except AMIs starting with "_")', cmd_list) 447 | list_group = list_parser.add_mutually_exclusive_group() 448 | list_group.add_argument('-i', '--instances', action='store_true') 449 | list_group.add_argument('-a', '--amis', action='store_true') 450 | 451 | args = parser.parse_args() 452 | args.func(args) 453 | 454 | if __name__ == "__main__": 455 | handle_command_line() -------------------------------------------------------------------------------- /pydnn/data.py: -------------------------------------------------------------------------------- 1 | import tools 2 | 3 | from os.path import join 4 | import numpy as np 5 | from scipy.misc import imread 6 | import theano 7 | 8 | 9 | class DirectoryLabeledImageSet(object): 10 | """ 11 | Builds training data from a directory where each subdirectory 12 | is the name of a class, and contains all the examples of images that class. 13 | 14 | :param string base_dir: the directory containing the class directories 15 | :param string dtype: the data type to use for the ndarray containing the labels 16 | """ 17 | def __init__(self, base_dir, dtype=theano.config.floatX): 18 | """ 19 | Build data set from images in directories by class names. 20 | """ 21 | print('DirectoryLabeledImageSet: {}'.format(base_dir)) 22 | self.base_dir = base_dir 23 | self.dtype = dtype 24 | 25 | def get_labels(self): 26 | return sorted(tools.get_sub_dirs(self.base_dir, rel=True)) 27 | 28 | def get_files(self, num_files=None): 29 | files = [(fn, label) for label in self.get_labels() for fn in 30 | sorted(tools.get_files(join(self.base_dir, label), rel=True))] 31 | if num_files is None: 32 | return files 33 | else: 34 | import random 35 | random.seed(12345) 36 | return random.sample(files, num_files) 37 | 38 | def get_random_file(self, rng): 39 | sub_dir = rng.choice(tools.get_sub_dirs(self.base_dir, cache=True)) 40 | return rng.choice(tools.get_files(sub_dir, cache=True)) 41 | 42 | def build(self, num_images=None): 43 | files = self.get_files(num_images) 44 | num_images = len(files) 45 | images = [None] * num_images 46 | classes = np.zeros(shape=num_images, dtype=self.dtype) 47 | file_indices = np.zeros(shape=num_images, dtype='int') 48 | labels = self.get_labels() 49 | 50 | print('loading {} of 30336 images...'.format(num_images)) 51 | dotter = tools.Dot(skip=num_images / 20) 52 | for i, (fn, label) in enumerate(files): 53 | images[i] = imread(join(self.base_dir, label, fn)) 54 | classes[i] = labels.index(label) 55 | file_indices[i] = i 56 | dotter.dot(str(i) + ' ') 57 | dotter.stop() 58 | return images, classes, file_indices 59 | 60 | 61 | class UnlabeledImageSet(object): 62 | """ 63 | Builds an inference set from a directory containing unlabeled images. 64 | 65 | :param string base_dir: the directory containing the images 66 | """ 67 | def __init__(self, base_dir): 68 | self.base_dir = base_dir 69 | self.files = None 70 | 71 | def get_files(self): 72 | if self.files is None: 73 | self.files = sorted(tools.get_files(self.base_dir, rel=True)) 74 | return self.files 75 | 76 | def build(self, start, stop): 77 | files = self.get_files()[start: stop] 78 | return files, [imread(join(self.base_dir, f)) for f in files] -------------------------------------------------------------------------------- /pydnn/img_util.py: -------------------------------------------------------------------------------- 1 | import tools 2 | import preprocess as pp 3 | 4 | import numpy as np 5 | 6 | 7 | def dimensions_to_fit_images(length): 8 | sqrt = int(np.sqrt(length)) 9 | rows = sqrt + 1 if sqrt * sqrt < length else sqrt 10 | cols = sqrt + 1 if sqrt * rows < length else sqrt 11 | return rows, cols 12 | 13 | 14 | def show_image_tiles(images, canvas_dims=None): 15 | import scipy.misc 16 | 17 | canvas_dims = (dimensions_to_fit_images(images.shape[0]) if 18 | canvas_dims is None else canvas_dims) 19 | image = tools.tile_2d_images(images, canvas_dims) 20 | scipy.misc.imshow(image) 21 | 22 | 23 | def show_images_as_tiles(images, size, canvas_dims=None): 24 | images_array = np.zeros(shape=(len(images), size[0], size[1]), 25 | dtype='uint8') 26 | for array_slice, image in zip(images_array, images): 27 | # print(image.shape, image) 28 | resized = pp._resize_preserving_aspect_ratio(image, size) 29 | # print(array_slice.shape, image.shape, resized.shape, size) 30 | # print(array_slice) 31 | array_slice[:] = resized 32 | show_image_tiles(images_array, canvas_dims) 33 | 34 | 35 | def show_images(images, titles=None, canvas_dims=None): 36 | import matplotlib.pyplot as plt 37 | import matplotlib.cm as cm 38 | import scipy 39 | 40 | titles = [''] * len(images) if titles is None else titles 41 | rows, cols = (dimensions_to_fit_images(images.shape[0]) if 42 | canvas_dims is None else canvas_dims) 43 | 44 | fig = plt.figure() 45 | for n, (title, image) in enumerate(zip(images, titles)): 46 | scipy.misc.imshow(image) 47 | sub = fig.add_subplot(rows, cols, n + 1) 48 | plt.imshow(image, interpolation='none', cmap=cm.Greys_r) 49 | sub.set_title('{} ({}x{})'.format(title, image.shape[0], image.shape[1]), 50 | size=10) 51 | sub.axis('off') 52 | plt.show() 53 | 54 | 55 | def test_show_images(): 56 | rng = np.random.RandomState() 57 | import scipy 58 | from os.path import basename 59 | images = [] 60 | for i in range(30): 61 | f = train_set.get_random_file(rng) 62 | image = scipy.misc.imread(f) 63 | title = '{} {} {}x{}'.format(i, basename(f), image.shape[0], image.shape[1]) 64 | images.append((title, image)) 65 | show_images(images, (pp._resize_preserving_aspect_ratio(image, 128, 128))) 66 | # test_show_images() -------------------------------------------------------------------------------- /pydnn/preprocess.py: -------------------------------------------------------------------------------- 1 | __author__ = 'isaac' 2 | 3 | import scipy 4 | import scipy.ndimage 5 | import numpy as np 6 | import tools 7 | import theano 8 | 9 | 10 | def _flatten_3d_to_2d(data): 11 | s = data.shape 12 | return data.reshape((s[0], s[1] * s[2])) 13 | 14 | 15 | def split_training_data(data, batch_size, train_per, valid_per, test_per): 16 | """ 17 | Split training data into training set, validation set and test sets. If split 18 | results in incomplete batches this function will allocate observations from 19 | incomplete batches in validation and test sets to the training set to attempt 20 | to make a complete batch. This function also reports on the split and whether 21 | there were observations that did not fit evenly into any batch. (Currently 22 | :class:`NN ` assumes a validation and test set, so if allocating 100% of data to 23 | training set, :func:`split_training_data` will duplicate the first batch of 24 | the training set for the validation and test sets so :class:`NN ` does not fail. 25 | In that case, obviously the validation and test loss will be meaningless.) 26 | 27 | :param tuple data: all the data, including x and y values 28 | :param int batch_size: number of observations that will be in each batch 29 | :param float train_per: percentage of data to put in training set 30 | :param float valid_per: percentage of data to put in validation set 31 | :param float test_per: percentage of data to put in test set 32 | :return: a list containing the training, validation and test sets, each 33 | of which is a list containing each variable (in the case where x data is a single 34 | image that will be a list of images and a list of y classes, but there can also 35 | be more than one x variable). 36 | """ 37 | assert train_per + valid_per + test_per == 100 38 | num_obs = len(data[0]) 39 | 40 | # calculate number of observations in set, rounding down to batch size 41 | def calc_num(per): 42 | num = int(num_obs * per / 100.0) 43 | num -= num % batch_size 44 | return num 45 | 46 | num_train = calc_num(train_per) 47 | num_valid = calc_num(valid_per) 48 | num_test = calc_num(test_per) 49 | 50 | # if there are enough residual observation to make a batch, add them to train 51 | num_train += (num_obs - num_train - num_valid - num_test) / batch_size * batch_size 52 | 53 | # print out some sanity check numbers 54 | num_discard = num_obs - num_train - num_valid - num_test 55 | print("train: {} obs, {} batches; valid: {} obs, {} batches; test: {} obs, {} batches". 56 | format(num_train, num_train / batch_size, num_valid, num_valid / batch_size, 57 | num_test, num_test / batch_size)) 58 | print("of {} observations, {} are used and {} are discarded because they do not fit evenly into a batch of size {}". 59 | format(num_obs, num_obs - num_discard, num_discard, batch_size)) 60 | 61 | # I use a separate random seed here to make sure that this shuffle happens 62 | # the same way each time; and won't get inadvertently change if I change the order 63 | # of code execution at some point. That way, if I don't change the percentage 64 | # breakdown between train, validation and test sets I should always be getting 65 | # the same sets. 66 | _shuffle_lists(np.random.RandomState(24680), data) 67 | 68 | # builds sets, e.g. set[test|valid|train][images|shapes|classes] 69 | if train_per == 100: # have to add some dummy data to valid/test to avoid exceptions 70 | print("TRAINING ON ALL DATA, VALIDATION AND TESTING SETS ARE ONLY DUMMIES") 71 | sets = [[var[start:stop] for var in data] 72 | for (start, stop) 73 | in ((0, num_train), 74 | (0, batch_size), 75 | (0, batch_size))] 76 | else: 77 | sets = [[var[start:stop] for var in data] 78 | for (start, stop) 79 | in ((0, num_train), 80 | (num_train, num_train + num_valid), 81 | (num_train + num_valid, num_train + num_valid + num_test))] 82 | 83 | # if show_samples: 84 | # import scipy 85 | # from MyTools import tile_2d_images 86 | # 87 | # for s in sets: 88 | # image = tile_2d_images(s[0], (batch_size / 20 + 1, 20)) 89 | # scipy.misc.imshow(image) 90 | 91 | return sets 92 | 93 | 94 | def _expand_white(image, size): 95 | """ 96 | Given an image and a target size, creates a new image of the target size and 97 | places image in the middle, thus expanding the image with white. 98 | 99 | :param ndarray image: image being expanded 100 | :param tuple size: size of the new image 101 | :return: white expanded image 102 | """ 103 | white = np.ndarray(size, dtype=image.dtype) 104 | white.fill(255) 105 | # white.fill(0) 106 | v_off = int((size[0] - image.shape[0]) / 2) 107 | h_off = int((size[1] - image.shape[1]) / 2) 108 | white[v_off:v_off + image.shape[0], 109 | h_off:h_off + image.shape[1]] = image 110 | return white 111 | 112 | 113 | class Resizer(object): 114 | """ 115 | Base class for :class:`StretchResizer`, :class:`ContiguousBoxPreserveAspectRatioResizer`, 116 | :class:`ContiguousBoxStretchResizer`, :class:`ThresholdBoxPreserveAspectRatioResizer`, 117 | :class:`ThresholdBoxStretchResizer`, :class:`PreserveAspectRatioResizer`, 118 | and :class:`StochasticStretchResizer` 119 | 120 | Some resizers may want to resize training images differently from 121 | validation or testing images so this class gives them the option of doing so. 122 | """ 123 | def training(self, image, size): 124 | return self._resize(image, size) 125 | 126 | def prediction(self, image, size): 127 | return self._resize(image, size) 128 | 129 | def _resize(self, image, size): 130 | raise NotImplemented() 131 | 132 | 133 | class StretchResizer(Resizer): 134 | """ 135 | Stretches the images to a uniform shape ignoring aspect ratio 136 | """ 137 | def _resize(self, image, size): 138 | return scipy.misc.imresize(image, size) 139 | 140 | 141 | def _crop_with_threshold(image, threshold): 142 | def top(): 143 | for i in range(image.shape[0]): 144 | for j in range(image.shape[1]): 145 | if image[i, j] < threshold: 146 | return i 147 | 148 | def bottom(): 149 | for i in range(1, image.shape[0]): 150 | for j in range(image.shape[1]): 151 | if image[-i, j] < threshold: 152 | return image.shape[0] - i 153 | 154 | def left(): 155 | for i in range(image.shape[1]): 156 | for j in range(image.shape[0]): 157 | if image[j, i] < threshold: 158 | return i 159 | 160 | def right(): 161 | for i in range(1, image.shape[1]): 162 | for j in range(image.shape[0]): 163 | if image[j, -i] < threshold: 164 | return image.shape[1] - i 165 | 166 | lft, rght, tp, bttm = left(), right(), top(), bottom() 167 | if (any(num is None for num in (lft, rght, tp, bttm)) or 168 | lft >= rght or tp >= bttm): 169 | return image 170 | else: 171 | return image[tp: bttm, lft: rght] 172 | 173 | 174 | def _crop_biggest_contiguous(image, threshold): 175 | # contiguous region selection based on: 176 | # http://stackoverflow.com/questions/4087919/how-can-i-improve-my-paw-detection 177 | thresh = 255 - image > threshold 178 | coded, num = scipy.ndimage.label(thresh) 179 | slices = [image[slc] for slc in scipy.ndimage.find_objects(coded)] 180 | biggest = slices[np.argmax([slc.size for slc in slices])] 181 | return biggest 182 | 183 | 184 | class ContiguousBoxPreserveAspectRatioResizer(Resizer): 185 | """ 186 | First crops the images around the largest contiguous region, then stretches 187 | them to a uniform size preserving aspect ratio. 188 | """ 189 | def __init__(self, threshold): 190 | self.threshold = threshold 191 | 192 | def _resize(self, image, size): 193 | image = _crop_biggest_contiguous(image, self.threshold) 194 | return _resize_preserving_aspect_ratio(image, size) 195 | 196 | class ContiguousBoxStretchResizer(Resizer): 197 | """ 198 | First crops the images around the largest contiguous region, then stretches 199 | them to a uniform ignoring aspect ratio. 200 | """ 201 | def __init__(self, threshold): 202 | self.threshold = threshold 203 | 204 | def _resize(self, image, size): 205 | image = _crop_biggest_contiguous(image, self.threshold) 206 | return scipy.misc.imresize(image, size) 207 | 208 | 209 | class ThresholdBoxPreserveAspectRatioResizer(Resizer): 210 | """ 211 | First crops the images, throwing away outside space without pixels 212 | that exceed a given threshold, then stretches 213 | them to a uniform size preserving aspect ratio. 214 | """ 215 | def __init__(self, threshold): 216 | self.threshold = threshold 217 | 218 | def _resize(self, image, size): 219 | image = _crop_with_threshold(image, self.threshold) 220 | return _resize_preserving_aspect_ratio(image, size) 221 | 222 | 223 | class ThresholdBoxStretchResizer(Resizer): 224 | """ 225 | First crops the images, throwing away outside space without pixels 226 | that exceed a given threshold, then stretches 227 | them to a uniform ignoring aspect ratio. 228 | """ 229 | def __init__(self, threshold): 230 | self.threshold = threshold 231 | 232 | def _resize(self, image, size): 233 | image = _crop_with_threshold(image, self.threshold) 234 | return scipy.misc.imresize(image, size) 235 | 236 | 237 | def _resize_preserving_aspect_ratio(image, size): 238 | if float(image.shape[0]) / image.shape[1] > float(size[0]) / size[1]: 239 | new_size = (size[0], int((float(size[0]) / image.shape[0]) * image.shape[1])) 240 | else: 241 | new_size = (int((float(size[1]) / image.shape[1]) * image.shape[0]), size[1]) 242 | image = scipy.misc.imresize(image, new_size) 243 | image = _expand_white(image, size) 244 | 245 | return image 246 | 247 | 248 | class PreserveAspectRatioResizer(Resizer): 249 | """ 250 | Stretches images to a uniform size preserving aspect ratio. 251 | """ 252 | def _resize(self, image, size): 253 | return _resize_preserving_aspect_ratio(image, size) 254 | 255 | 256 | class StochasticStretchResizer(Resizer): 257 | """ 258 | Stretches images to a uniform size ignoring aspect ratio. 259 | 260 | Randomly varies how much training images are stretched. 261 | """ 262 | def __init__(self, rng, rand_range): 263 | self.rng = rng 264 | self.rand_range = rand_range 265 | 266 | def _crop_range(self, rand, fixed): 267 | if rand > fixed: 268 | off = (rand - fixed) / 2 269 | return off, off + fixed 270 | else: 271 | return 0, fixed 272 | 273 | def training(self, image, size): 274 | # randomly fluctuate each dimension by normal dist with variance 0.05 275 | # using the normal distribution can cause exceptions due to extreme values 276 | # rand = numpy.cast['int'](size * (.05 * self.rng.randn(2) + 1)) 277 | 278 | # randomly fluctuate each dimension by uniform distribution 279 | rand = np.cast['int'](size * (self.rng.uniform(self.rand_range[0], 280 | self.rand_range[0], size=2))) 281 | resized = scipy.misc.imresize(image, rand) 282 | 283 | # if dimension is bigger than desired size, crop to desired size 284 | vert = self._crop_range(rand[0], size[0]) 285 | horz = self._crop_range(rand[1], size[1]) 286 | cropped = resized[vert[0]:vert[1], horz[0]:horz[1]] 287 | 288 | # if dimension is smaller than desired size, fill with white 289 | return _expand_white(cropped, size) 290 | 291 | def prediction(self, image, size): 292 | # print( 'image', image, 'size', size) 293 | return scipy.misc.imresize(image, size) 294 | 295 | 296 | def _make_shape_array(images, dtype): 297 | shapes = np.zeros(shape=(len(images), 2), dtype=dtype) 298 | for image, shape in zip(images, shapes): 299 | shape[:] = image.shape 300 | return shapes 301 | 302 | 303 | class Rotator360(object): 304 | """ 305 | Rotates training set images randomly. 306 | 307 | (Also zero centers by pixel, normalizes, shuffles, resizes, etc.) 308 | 309 | :param data: x and y data 310 | :param image_shape: the target image shape 311 | :param resizer: the resizer to use when uniformly resizing images 312 | :param rng: random number generator 313 | :param string dtype: the datatype to output 314 | """ 315 | def __init__(self, data, image_shape, resizer, rng, dtype=theano.config.floatX): 316 | 317 | # start_time = time.time() 318 | self.dtype = dtype 319 | self.rng = rng 320 | 321 | self.train_image = None 322 | self.train_shape = None 323 | self.train_class = None 324 | self.valid_image = None 325 | self.valid_shape = None 326 | self.valid_class = None 327 | self.test_image = None 328 | self.test_shape = None 329 | self.test_class = None 330 | self.data = None 331 | self.set_data(data) 332 | 333 | self.image_shape = image_shape 334 | self.resizer = resizer 335 | 336 | self.image_mean = self._approximate_image_mean(self.train_image) 337 | self.shape_mean = np.mean(self.train_shape, axis=0) 338 | self.shape_std = np.std(self.train_shape, axis=0) 339 | self.shape_range = (np.max(self.train_shape, axis=0) - 340 | np.min(self.train_shape, axis=0)) 341 | print('shape', 'mean', self.shape_mean, 'std', self.shape_std, 342 | 'range', self.shape_range) 343 | 344 | def set_data(self, data): 345 | dtype = self.dtype 346 | self.train_image = data[0][0] 347 | self.train_shape = _make_shape_array(data[0][0], dtype) 348 | self.train_class = data[0][1].astype(dtype) 349 | self.valid_image = data[1][0] 350 | self.valid_shape = _make_shape_array(data[1][0], dtype) 351 | self.valid_class = data[1][1].astype(dtype) 352 | self.test_image = data[2][0] 353 | self.test_shape = _make_shape_array(data[2][0], dtype) 354 | self.test_class = data[2][1].astype(dtype) 355 | 356 | self.data = data 357 | 358 | # I want an even distribution of random rotations, but if there is any 359 | # regularity in the orientations (e.g. because of gravity) I want to break 360 | # that up, so each image gets a fixed rotation, and all the images are adjusted 361 | # by the master rotation after each epoch, so that after 360 epochs, the net 362 | # has seen each in each of 360 degree rotations. 363 | self.image_rots = self.rng.choice( 364 | np.arange(0, 360, 1), 365 | size=len(self.train_image) + len(self.valid_image) + len(self.test_image), 366 | replace=True) 367 | self.master_rots = np.concatenate([ 368 | self.rng.choice( 369 | np.arange(offset, 360 + offset, 1), 370 | size=360, 371 | replace=False) for offset in (0, 0.5, 0.25, 0.75)]) 372 | self.rot_index = 0 373 | 374 | def _approximate_image_mean(self, training_images): 375 | images = self._resize_consolidate(training_images, self.resizer.training) 376 | images = self._transform_images(images) 377 | return np.mean(images, axis=0) 378 | 379 | def _resize_consolidate(self, images, resizer): 380 | # convert a list of images of varying sizes into a single 381 | # array of images of uniform size 382 | images_array = np.zeros(shape=(len(images), self.image_shape[0], 383 | self.image_shape[1]), 384 | dtype='uint8') 385 | for array_slice, image in zip(images_array, images): 386 | 387 | array_slice[:] = resizer(image, self.image_shape) 388 | 389 | return images_array 390 | 391 | def _zero_center_normalize(self, images, shapes): 392 | # normalize images and shapes 393 | images = (images - self.image_mean) / 255.0 394 | 395 | shapes = (shapes - self.shape_mean) / self.shape_std 396 | # shapes = (shapes - self.shape_mean) / self.shape_range 397 | # shapes = (shapes - self.shape_mean) / self.shape_mean 398 | cast = np.cast[self.dtype] 399 | return cast(_flatten_3d_to_2d(images)), shapes 400 | 401 | def _transform_images(self, images): 402 | # rotations seem to change the background color to grey haphazardly 403 | # when done on floating point type images so assert that this is 404 | # 256 valued int. 405 | assert images.dtype == 'uint8' 406 | images = images.copy() 407 | rotations = self.image_rots + self.master_rots[self.rot_index] 408 | self.rot_index = self.rot_index + 1 if self.rot_index < self.master_rots.size - 1 else 0 409 | 410 | for index, image in enumerate(images): 411 | rot = rotations[index] if rotations[index] < 360 else rotations[index] - 360 412 | image[:] = scipy.ndimage.interpolation.rotate( 413 | input=image, 414 | angle=rot, 415 | reshape=False, 416 | mode='constant', 417 | cval=255) 418 | if self.rng.choice([True, False]): 419 | image[:] = np.fliplr(image) 420 | return images 421 | 422 | def _prep_for_training(self, images, shapes, classes): 423 | # transformation must happen before normalization because rotations, 424 | # shearing, etc, introduce white values at the corners 425 | # which must also be zero centered and normalized 426 | 427 | images = self._resize_consolidate(images, self.resizer.training) 428 | images = self._transform_images(images) 429 | images, shapes = self._zero_center_normalize(images, shapes) 430 | 431 | # must copy the classes because they'll be shuffled inplace 432 | # (images and shapes lists are already copied in above functions) 433 | classes = classes.copy() 434 | _shuffle_lists(self.rng, [images, shapes, classes]) 435 | 436 | return self._wrap_dict(images, shapes, classes) 437 | 438 | 439 | def _prep_for_prediction(self, images, shapes): 440 | images = self._resize_consolidate(images, self.resizer.prediction) 441 | return self._zero_center_normalize(images, shapes) 442 | 443 | def _wrap_dict(self, images, shapes, classes=None): 444 | if classes is None: 445 | return {'images': images, 'shapes': shapes} 446 | else: 447 | return {'images': images, 'shapes': shapes}, classes 448 | 449 | def shape_for(self, channel): 450 | img_shape = self.image_shape 451 | shapes = {'images': (1, img_shape[0], img_shape[1]), 'shapes': (2,)} 452 | return shapes[channel] 453 | 454 | @tools.time_once 455 | def get_train(self): 456 | return self._prep_for_training(self.train_image, self.train_shape, 457 | self.train_class) 458 | 459 | def get_all_train(self): 460 | x = self.train_image + self.valid_image + self.test_image 461 | shapes = np.vstack((self.train_shape, self.valid_shape, self.test_shape)) 462 | y = np.append(self.train_class, np.append(self.valid_class, self.test_class)) 463 | return self._prep_for_training(x, shapes, y) 464 | 465 | def get_valid(self): 466 | images, shapes = self._prep_for_prediction(self.valid_image, self.valid_shape) 467 | # print('get_valid', images, shapes) 468 | return self._wrap_dict(images, shapes, self.valid_class) 469 | 470 | def get_test(self): 471 | images, shapes = self._prep_for_prediction(self.test_image, self.test_shape) 472 | # print('get_test', images, shapes) 473 | return self._wrap_dict(images, shapes, self.test_class) 474 | 475 | def get_raw_x(self): 476 | return [{'images': s[0]} for s in self.data] 477 | 478 | def get_raw_y(self): 479 | return [s[1] for s in self.data] 480 | 481 | def get_files(self): 482 | return [s[2] for s in self.data] 483 | 484 | def preprocess(self, data): 485 | image_data = data['images'] 486 | shape_data = _make_shape_array(image_data, self.dtype) 487 | images, shapes = self._prep_for_prediction(image_data, shape_data) 488 | return self._wrap_dict(images, shapes) 489 | 490 | def __getstate__(self): 491 | return {'image_mean': self.image_mean, 492 | 'shape_mean': self.shape_mean, 493 | 'shape_std': self.shape_std, 494 | 'shape_range': self.shape_range, 495 | 'image_shape': self.image_shape, 496 | 'resizer': self.resizer, 497 | 'dtype': self.dtype, 498 | 'rng': self.rng, 499 | 'image_rots': self.image_rots, 500 | 'master_rots': self.master_rots, 501 | 'rot_index': self.rot_index 502 | } 503 | 504 | 505 | class Rotator360PlusGeometry(object): 506 | """ 507 | Rotates training set images randomly, but also generates 508 | additional geometric data about the size and orientation of 509 | the organism in the image. 510 | 511 | (Also zero centers by pixel, normalizes, shuffles, resizes, etc.) 512 | 513 | :param data: x and y data 514 | :param image_shape: the target image shape 515 | :param resizer: the resizer to use when uniformly resizing images 516 | :param rng: random number generator 517 | :param string dtype: the datatype to output 518 | """ 519 | def __init__(self, data, image_shape, resizer, rng, dtype): 520 | 521 | # start_time = time.time() 522 | self.dtype = dtype 523 | self.rng = rng 524 | 525 | self.train_image = None 526 | self.train_geom = None 527 | self.train_class = None 528 | self.valid_image = None 529 | self.valid_geom = None 530 | self.valid_class = None 531 | self.test_image = None 532 | self.test_geom = None 533 | self.test_class = None 534 | self.data = None 535 | self.set_data(data) 536 | 537 | self.image_shape = image_shape 538 | self.resizer = resizer 539 | 540 | self.image_mean = self._approximate_image_mean(self.train_image) 541 | self.geom_mean = np.mean(self.train_geom, axis=0) 542 | self.geom_std = np.std(self.train_geom, axis=0) 543 | self.geom_range = (np.max(self.train_geom, axis=0) - 544 | np.min(self.train_geom, axis=0)) 545 | print('geom', 'mean', self.geom_mean, 'std', self.geom_std, 546 | 'range', self.geom_range) 547 | 548 | def make_geometric_data(self, images): 549 | geoms = np.zeros(shape=(len(images), 2 + 2 + 1 + 2), dtype=self.dtype) 550 | print('calculating geometric data...') 551 | dotter = tools.Dot(skip=len(images)/40) 552 | for image, geom in zip(images, geoms): 553 | img, rot, ud, lr = _canonicalize_image(image, include_info_in_result=True) 554 | geom[0:2] = image.shape 555 | geom[2:4] = _crop_biggest_contiguous(img, 2).shape 556 | geom[4] = rot 557 | geom[5] = ud 558 | geom[6] = lr 559 | dotter.dot() 560 | dotter.stop() 561 | return geoms 562 | 563 | def set_data(self, data): 564 | dtype = self.dtype 565 | self.train_image = data[0][0] 566 | self.train_geom = self.make_geometric_data(data[0][0]) 567 | self.train_class = data[0][1].astype(dtype) 568 | self.valid_image = data[1][0] 569 | self.valid_geom = self.make_geometric_data(data[1][0]) 570 | self.valid_class = data[1][1].astype(dtype) 571 | self.test_image = data[2][0] 572 | self.test_geom = self.make_geometric_data(data[2][0]) 573 | self.test_class = data[2][1].astype(dtype) 574 | 575 | self.data = data 576 | 577 | # I want an even distribution of random rotations, but if there is any 578 | # regularity in the orientations (e.g. because of gravity) I want to break 579 | # that up, so each image gets a fixed rotation, and all the images are adjusted 580 | # by the master rotation after each epoch, so that after 360 epochs, the net 581 | # has seen each in each of 360 degree rotations. 582 | self.image_rots = self.rng.choice( 583 | np.arange(0, 360, 1), 584 | size=len(self.train_image) + len(self.valid_image) + len(self.test_image), 585 | replace=True) 586 | self.master_rots = np.concatenate([ 587 | self.rng.choice( 588 | np.arange(offset, 360 + offset, 1), 589 | size=360, 590 | replace=False) for offset in (0, 0.5, 0.25, 0.75)]) 591 | self.rot_index = 0 592 | 593 | def _approximate_image_mean(self, training_images): 594 | images = self._resize_consolidate(training_images, self.resizer.training) 595 | images = self._transform_images(images) 596 | return np.mean(images, axis=0) 597 | 598 | def _resize_consolidate(self, images, resizer): 599 | # convert a list of images of varying sizes into a single 600 | # array of images of uniform size 601 | images_array = np.zeros(shape=(len(images), self.image_shape[0], 602 | self.image_shape[1]), 603 | dtype='uint8') 604 | for array_slice, image in zip(images_array, images): 605 | array_slice[:] = resizer(image, self.image_shape) 606 | 607 | return images_array 608 | 609 | def _zero_center_normalize(self, images, geoms): 610 | # normalize images and shapes 611 | images = (images - self.image_mean) / 255.0 612 | 613 | geoms = (geoms - self.geom_mean) / self.geom_std 614 | # shapes = (shapes - self.shape_mean) / self.shape_range 615 | # shapes = (shapes - self.shape_mean) / self.shape_mean 616 | cast = np.cast[self.dtype] 617 | return cast(_flatten_3d_to_2d(images)), geoms 618 | 619 | def _transform_images(self, images): 620 | # rotations seem to change the background color to grey haphazardly 621 | # when done on floating point type images so assert that this is 622 | # 256 valued int. 623 | assert images.dtype == 'uint8' 624 | images = images.copy() 625 | rotations = self.image_rots + self.master_rots[self.rot_index] 626 | self.rot_index = self.rot_index + 1 if self.rot_index < self.master_rots.size - 1 else 0 627 | 628 | for index, image in enumerate(images): 629 | rot = rotations[index] if rotations[index] < 360 else rotations[index] - 360 630 | image[:] = scipy.ndimage.interpolation.rotate( 631 | input=image, 632 | angle=rot, 633 | reshape=False, 634 | mode='constant', 635 | cval=255) 636 | if self.rng.choice([True, False]): 637 | image[:] = np.fliplr(image) 638 | return images 639 | 640 | def _prep_for_training(self, images, geoms, classes): 641 | # transformation must happen before normalization because rotations, 642 | # shearing, etc, introduce white values at the corners 643 | # which must also be zero centered and normalized 644 | 645 | images = self._resize_consolidate(images, self.resizer.training) 646 | images = self._transform_images(images) 647 | images, geoms = self._zero_center_normalize(images, geoms) 648 | 649 | # must copy the classes because they'll be shuffled inplace 650 | # (images and shapes lists are already copied in above functions) 651 | classes = classes.copy() 652 | _shuffle_lists(self.rng, [images, geoms, classes]) 653 | 654 | return self._wrap_dict(images, geoms, classes) 655 | 656 | def _prep_for_prediction(self, images, geoms): 657 | images = self._resize_consolidate(images, self.resizer.prediction) 658 | return self._zero_center_normalize(images, geoms) 659 | 660 | def _wrap_dict(self, images, geoms, classes=None): 661 | if classes is None: 662 | return {'images': images, 'geometry': geoms} 663 | else: 664 | return {'images': images, 'geometry': geoms}, classes 665 | 666 | def shape_for(self, channel): 667 | img_shape = self.image_shape 668 | shapes = {'images': (1, img_shape[0], img_shape[1]), 'geometry': (7,)} 669 | return shapes[channel] 670 | 671 | @tools.time_once 672 | def get_train(self): 673 | return self._prep_for_training(self.train_image, self.train_geom, 674 | self.train_class) 675 | 676 | def get_all_train(self): 677 | x = self.train_image + self.valid_image + self.test_image 678 | geoms = np.vstack((self.train_geom, self.valid_geom, self.test_geom)) 679 | y = np.append(self.train_class, np.append(self.valid_class, self.test_class)) 680 | return self._prep_for_training(x, geoms, y) 681 | 682 | def get_valid(self): 683 | images, geom = self._prep_for_prediction(self.valid_image, self.valid_geom) 684 | # print('get_valid', images, shapes) 685 | return self._wrap_dict(images, geom, self.valid_class) 686 | 687 | def get_test(self): 688 | images, geom = self._prep_for_prediction(self.test_image, self.test_geom) 689 | # print('get_test', images, shapes) 690 | return self._wrap_dict(images, geom, self.test_class) 691 | 692 | def get_raw_x(self): 693 | return [{'images': s[0]} for s in self.data] 694 | 695 | def get_raw_y(self): 696 | return [s[1] for s in self.data] 697 | 698 | def get_files(self): 699 | return [s[2] for s in self.data] 700 | 701 | def preprocess(self, data): 702 | image_data = data['images'] 703 | geom_data = self.make_geometric_data(image_data) 704 | images, geoms = self._prep_for_prediction(image_data, geom_data) 705 | return self._wrap_dict(images, geoms) 706 | 707 | def __getstate__(self): 708 | return {'image_mean': self.image_mean, 709 | 'geom_mean': self.geom_mean, 710 | 'geom_std': self.geom_std, 711 | 'geom_range': self.geom_range, 712 | 'image_shape': self.image_shape, 713 | 'resizer': self.resizer, 714 | 'dtype': self.dtype, 715 | 'rng': self.rng, 716 | 'image_rots': self.image_rots, 717 | 'master_rots': self.master_rots, 718 | 'rot_index': self.rot_index 719 | } 720 | 721 | 722 | class Canonicalizer(object): 723 | """ 724 | Rotates and flips all images into a canonicalized form. Using 725 | a statistical measure of object height rotates each image to 726 | minimize height. (Can also either (1) flip images so aggregate 727 | pixel intensity is highest in one corner, or (2) generate random 728 | flips of training images.) 729 | 730 | (Also zero centers by pixel, normalizes, shuffles, resizes, etc.) 731 | 732 | :param data: x and y data 733 | :param image_shape: the target image shape 734 | :param resizer: the resizer to use when uniformly resizing images 735 | :param rng: random number generator 736 | :param string dtype: the datatype to output 737 | """ 738 | def __init__(self, data, image_shape, resizer, rng, dtype): 739 | self.dtype = dtype 740 | self.rng = rng 741 | self.image_shape = image_shape 742 | self.resizer = resizer 743 | 744 | self.train_image = None 745 | self.train_geom = None 746 | self.train_class = None 747 | self.valid_image = None 748 | self.valid_geom = None 749 | self.valid_class = None 750 | self.test_image = None 751 | self.test_geom = None 752 | self.test_class = None 753 | 754 | self.image_mean = None 755 | self.geom_mean = None 756 | self.geom_std = None 757 | self.geom_range = None 758 | 759 | self.data = None 760 | self.set_data(data) 761 | 762 | def set_data(self, data): 763 | dtype = self.dtype 764 | 765 | images, geoms = self._canonicalize_images(data[0][0], 766 | self.resizer.training) 767 | self.geom_mean = np.mean(geoms, axis=0) 768 | self.geom_std = np.std(geoms, axis=0) 769 | self.geom_range = (np.max(geoms, axis=0) - 770 | np.min(geoms, axis=0)) 771 | print('shape', 'mean', self.geom_mean, 'std', self.geom_std, 772 | 'range', self.geom_range) 773 | self.train_image = images 774 | self.image_mean = np.mean([image for image in self.train_image] + 775 | [np.fliplr(image) for image in self.train_image] + 776 | [np.flipud(image) for image in self.train_image] + 777 | [np.fliplr(np.flipud(image)) for image in self.train_image], 778 | axis=0) 779 | self.train_geom = self._zero_center_normalize_geoms(geoms) 780 | self.train_class = data[0][1].astype(dtype) 781 | 782 | images, geoms = self._canonicalize_images(data[1][0], 783 | self.resizer.training) 784 | self.valid_image = images 785 | self.valid_geom = self._zero_center_normalize_geoms(geoms) 786 | self.valid_class = data[1][1].astype(dtype) 787 | 788 | images, geoms = self._canonicalize_images(data[2][0], 789 | self.resizer.training) 790 | self.test_image = images 791 | self.test_geom = self._zero_center_normalize_geoms(geoms) 792 | self.test_class = data[2][1].astype(dtype) 793 | 794 | # all the above should result in copies so the data below 795 | # is completely untransformed 796 | self.data = data 797 | 798 | def _zero_center_normalize_images(self, images): 799 | images = (images - self.image_mean) / 255.0 800 | cast = np.cast[self.dtype] 801 | return cast(_flatten_3d_to_2d(images)) 802 | 803 | def _zero_center_normalize_geoms(self, geoms): 804 | geoms = (geoms - self.geom_mean) / self.geom_std 805 | return geoms 806 | 807 | def _canonicalize_images(self, images, resizer): 808 | images_array = np.zeros(shape=(len(images), self.image_shape[0], 809 | self.image_shape[1]), 810 | dtype='uint8') 811 | geoms = np.zeros(shape=(len(images), 2 + 2 + 1 + 2), dtype=self.dtype) 812 | 813 | print('canonicalizing {} images'.format(len(images))) 814 | dotter = tools.Dot(skip=len(images) / 20) 815 | for i, (img_arr, geom, image) in enumerate(zip(images_array, geoms, images)): 816 | img, rot, ud, lr = _canonicalize_image(image, 817 | include_info_in_result=True) 818 | img_arr[:] = resizer(img, self.image_shape) 819 | geom[0:2] = image.shape 820 | geom[2:4] = _crop_biggest_contiguous(img, 2).shape 821 | geom[4] = rot 822 | geom[5] = ud 823 | geom[6] = lr 824 | dotter.dot(str(i) + ' ') 825 | dotter.stop() 826 | return images_array, geoms 827 | 828 | def _wrap_dict(self, images, geoms, classes=None): 829 | if classes is None: 830 | return {'images': images, 'geometry': geoms} 831 | else: 832 | return {'images': images, 'geometry': geoms}, classes 833 | 834 | def shape_for(self, channel): 835 | img_shape = self.image_shape 836 | shapes = {'images': (1, img_shape[0], img_shape[1]), 'geometry': (7,)} 837 | return shapes[channel] 838 | 839 | @tools.time_once 840 | def _prep_for_training(self, images, geoms, classes): 841 | images, geoms, classes = images.copy(), geoms.copy(), classes.copy() 842 | 843 | for image in images: 844 | if self.rng.choice([True, False]): 845 | image[:] = np.fliplr(image) 846 | if self.rng.choice([True, False]): 847 | image[:] = np.flipud(image) 848 | 849 | _shuffle_lists(self.rng, [images, 850 | geoms, 851 | classes]) 852 | # import plankton 853 | # plankton.show_image_tiles(images) 854 | return self._wrap_dict(self._zero_center_normalize_images(images), geoms, classes) 855 | 856 | @tools.time_once 857 | def get_train(self): 858 | return self._prep_for_training(self.train_image, self.train_geom, 859 | self.train_class) 860 | 861 | def get_all_train(self): 862 | x = np.vstack((self.train_image, self.valid_image, self.test_image)) 863 | shapes = np.vstack((self.train_geom, self.valid_geom, self.test_geom)) 864 | y = np.append(self.train_class, np.append(self.valid_class, self.test_class)) 865 | return self._prep_for_training(x, shapes, y) 866 | 867 | def get_valid(self): 868 | return self._wrap_dict(self._zero_center_normalize_images(self.valid_image), 869 | self.valid_geom, self.valid_class) 870 | 871 | def get_test(self): 872 | return self._wrap_dict(self._zero_center_normalize_images(self.test_image), 873 | self.test_geom, self.test_class) 874 | 875 | def preprocess(self, data): 876 | image_data = data['images'] 877 | images, geoms = self._canonicalize_images(image_data, self.resizer.prediction) 878 | return self._wrap_dict(self._zero_center_normalize_images(images), 879 | self._zero_center_normalize_geoms(geoms)) 880 | 881 | def get_raw_x(self): 882 | return [{'images': s[0]} for s in self.data] 883 | 884 | def get_raw_y(self): 885 | return [s[1] for s in self.data] 886 | 887 | def get_files(self): 888 | return [s[2] for s in self.data] 889 | 890 | def __getstate__(self): 891 | return {'image_mean': self.image_mean, 892 | 'geom_mean': self.geom_mean, 893 | 'geom_std': self.geom_std, 894 | 'geom_range': self.geom_range, 895 | 'image_shape': self.image_shape, 896 | 'resizer': self.resizer, 897 | 'dtype': self.dtype, 898 | 'rng': self.rng} 899 | 900 | 901 | def _canonicalize_image(img, include_info_in_result=False, do_flips=False): 902 | def stat_height(image): 903 | image = 255 - image 904 | row_sums = np.sum(image, axis=1) 905 | all_sum = np.sum(row_sums) 906 | 907 | def find_middle(): 908 | run_tot = 0 909 | for j, row in enumerate(row_sums): 910 | run_tot += row 911 | if run_tot > all_sum / 2: 912 | return j 913 | mid = find_middle() 914 | height = 0 915 | for i, sum in enumerate(row_sums): 916 | height += (abs(i - mid) ** 2) * sum 917 | return height 918 | 919 | def rotation(image, deg): 920 | return scipy.ndimage.interpolation.rotate( 921 | input=image, 922 | angle=deg, 923 | reshape=False, 924 | mode='constant', 925 | cval=255) 926 | 927 | def minimize(func, image, deg=(0.0, 180.0)): 928 | diff = deg[1] - deg[0] 929 | if diff < 1: 930 | return deg[0] + diff / 2 931 | 932 | step = diff / 3.0 933 | rots = deg[0] + step, deg[0] + 2.0 * step 934 | imgs = [rotation(image, rot) for rot in rots] 935 | scores = [func(img) for img in imgs] 936 | # scipy.misc.imshow(numpy.column_stack(imgs)) 937 | range = (deg[0], deg[0] + diff / 2.0) if scores[0] < scores[1] \ 938 | else (deg[0] + diff / 2.0, deg[1]) 939 | return minimize(func, image, range) 940 | 941 | def flips(image): 942 | h_height = image.shape[0] / 2 943 | h_width = image.shape[1] / 2 944 | 945 | return (np.sum(image[:h_height]) < np.sum(image[h_height:]), 946 | np.sum(image[:, :h_width]) > np.sum(image[:, h_width:])) 947 | 948 | # rotations seem to change the background color to grey haphazardly 949 | # when done on floating point type images so assert that this is 950 | # 256 valued int. 951 | assert img.dtype == 'uint8' 952 | 953 | dim = max(img.shape) 954 | img = _expand_white(img, (dim, dim)) 955 | rot = minimize(stat_height, img) 956 | img = rotation(img, rot) 957 | 958 | # don't do flips here if also doing them randomly above for training... 959 | # they make the pixel mean invalid for the validation/test/prediction sets 960 | if do_flips: 961 | ud, lr = flips(img) 962 | img = np.flipud(img) if ud else img 963 | img = np.fliplr(img) if lr else img 964 | if include_info_in_result: 965 | ud, lr = flips(img) 966 | return img, rot, ud, lr 967 | else: 968 | return img 969 | 970 | 971 | def _shuffle_lists(rng, lists): 972 | """ 973 | shuffles, inplace, multiple lists/arrays together so that 974 | corresponding items in each list are still corresponding 975 | after shuffle 976 | """ 977 | length = len(lists[0]) 978 | for l in lists: 979 | assert len(l) == length 980 | 981 | for i in xrange(length): 982 | j = rng.randint(length - 1) 983 | for l in lists: # must make copy in case l is an array: 984 | tmp = l[i].copy() # l[i], l[j] = l[j], l[i] will not work 985 | l[i] = l[j] # because l[i] is written over 986 | l[j] = tmp # before l[j] can receive its value 987 | 988 | 989 | -------------------------------------------------------------------------------- /pydnn/tools.py: -------------------------------------------------------------------------------- 1 | __author__ = 'isaac' 2 | 3 | import smtplib 4 | import sys 5 | from email.mime.text import MIMEText 6 | import os 7 | import numpy 8 | from StringIO import StringIO 9 | import datetime 10 | import time 11 | import yaml 12 | 13 | 14 | def load_config(environ_variable, module_file, default_config): 15 | default_config = os.path.join(os.path.dirname(module_file), default_config) 16 | config = os.getenv(environ_variable, default_config) 17 | print('using config: ' + config) 18 | return yaml.load(file(config)) 19 | 20 | 21 | def send_email( 22 | from_addr, 23 | to_addr, 24 | username, 25 | password, 26 | smtp, 27 | subject='', 28 | body='',): 29 | msg = MIMEText(body) 30 | msg['Subject'] = subject 31 | msg['From'] = from_addr 32 | msg['To'] = to_addr 33 | 34 | server = smtplib.SMTP(smtp) 35 | server.starttls() 36 | server.login(username, password) 37 | server.sendmail(from_addr, to_addr, msg.as_string()) 38 | server.quit() 39 | 40 | 41 | class Dot(object): 42 | def __init__(self, skip=0): 43 | self.number = 0 44 | self.skip_count = int(skip) 45 | self.num_skip = int(skip) 46 | 47 | def stop(self): 48 | sys.stdout.write(' done\n') 49 | sys.stdout.flush() 50 | 51 | def dot(self, string=None): 52 | if self.skip_count == self.num_skip: 53 | self.skip_count = 0 54 | string = "." if string is None else string 55 | self.number += 1 56 | 57 | if self.number > 40: 58 | sys.stdout.write("\n") 59 | self.number = 0 60 | 61 | sys.stdout.write(string) 62 | sys.stdout.flush() 63 | else: 64 | self.skip_count += 1 65 | 66 | 67 | def raise_exception(x): 68 | raise Exception(x) 69 | 70 | 71 | def get_sub_dirs(directory, rel=False, cache=False): 72 | path, subs, files = next(os.walk(directory, onerror=raise_exception)) 73 | key = directory + str(rel) 74 | if cache and key in get_sub_dirs.cache: 75 | return get_sub_dirs.cache[key] 76 | if not rel: 77 | subs = [path + os.sep + sub for sub in subs] 78 | if cache: 79 | get_sub_dirs.cache[key] = subs 80 | return subs 81 | get_sub_dirs.cache = {} 82 | 83 | 84 | def get_files(directory, rel=False, cache=False): 85 | path, subs, files = next(os.walk(directory, onerror=raise_exception)) 86 | key = directory + str(rel) 87 | if cache and key in get_files.cache: 88 | return get_files.cache[key] 89 | if not rel: 90 | files = [path + os.sep + f for f in files] 91 | if cache: 92 | get_files.cache[key] = files 93 | return files 94 | get_files.cache = {} 95 | 96 | 97 | def scale_to_unit_interval(ndar, eps=1e-8): 98 | """ Scales all values in the ndarray ndar to be between 0 and 1 """ 99 | ndar = ndar.copy() 100 | ndar -= ndar.min() 101 | ndar *= 1.0 / (ndar.max() + eps) 102 | return ndar 103 | 104 | 105 | def image_tile(X, img_shape, tile_shape, tile_spacing=(0, 0), 106 | scale_rows_to_unit_interval=True, 107 | output_pixel_vals=True): 108 | """ 109 | Transform an array with one flattened image per row, into an array in 110 | which images are reshaped and layed out like tiles on a floor. 111 | 112 | This function is useful for visualizing datasets whose rows are images, 113 | and also columns of matrices for transforming those rows 114 | (such as the first layer of a neural net). 115 | 116 | :type X: a 2-D ndarray or a tuple of 4 channels, elements of which can 117 | be 2-D ndarrays or None; 118 | :param X: a 2-D array in which every row is a flattened image. 119 | 120 | :type img_shape: tuple; (height, width) 121 | :param img_shape: the original shape of each image 122 | 123 | :type tile_shape: tuple; (rows, cols) 124 | :param tile_shape: the number of images to tile (rows, cols) 125 | 126 | :param output_pixel_vals: if output should be pixel values (i.e. int8 127 | values) or floats 128 | 129 | :param scale_rows_to_unit_interval: if the values need to be scaled before 130 | being plotted to [0,1] or not 131 | 132 | 133 | :returns: array suitable for viewing as an image. 134 | (See:`Image.fromarray`.) 135 | :rtype: a 2-d array with same dtype as X. 136 | 137 | """ 138 | 139 | assert len(img_shape) == 2 140 | assert len(tile_shape) == 2 141 | assert len(tile_spacing) == 2 142 | 143 | # The expression below can be re-written in a more C style as 144 | # follows : 145 | # 146 | # out_shape = [0,0] 147 | # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - 148 | # tile_spacing[0] 149 | # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - 150 | # tile_spacing[1] 151 | out_shape = [ 152 | (ishp + tsp) * tshp - tsp 153 | for ishp, tshp, tsp in zip(img_shape, tile_shape, tile_spacing) 154 | ] 155 | 156 | if isinstance(X, tuple): 157 | assert len(X) == 4 158 | # Create an output numpy ndarray to store the image 159 | if output_pixel_vals: 160 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 161 | dtype='uint8') 162 | else: 163 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 164 | dtype=X.dtype) 165 | 166 | #colors default to 0, alpha defaults to 1 (opaque) 167 | if output_pixel_vals: 168 | channel_defaults = [0, 0, 0, 255] 169 | else: 170 | channel_defaults = [0., 0., 0., 1.] 171 | 172 | for i in range(4): 173 | if X[i] is None: 174 | # if channel is None, fill it with zeros of the correct 175 | # dtype 176 | dt = out_array.dtype 177 | if output_pixel_vals: 178 | dt = 'uint8' 179 | out_array[:, :, i] = numpy.zeros( 180 | out_shape, 181 | dtype=dt 182 | ) + channel_defaults[i] 183 | else: 184 | # use a recurrent call to compute the channel and store it 185 | # in the output 186 | out_array[:, :, i] = image_tile( 187 | X[i], img_shape, tile_shape, tile_spacing, 188 | scale_rows_to_unit_interval, output_pixel_vals) 189 | return out_array 190 | 191 | else: 192 | # if we are dealing with only one channel 193 | H, W = img_shape 194 | Hs, Ws = tile_spacing 195 | 196 | # generate a matrix to store the output 197 | dt = X.dtype 198 | if output_pixel_vals: 199 | dt = 'uint8' 200 | out_array = numpy.zeros(out_shape, dtype=dt) 201 | 202 | for tile_row in range(tile_shape[0]): 203 | for tile_col in range(tile_shape[1]): 204 | if tile_row * tile_shape[1] + tile_col < X.shape[0]: 205 | this_x = X[tile_row * tile_shape[1] + tile_col] 206 | if scale_rows_to_unit_interval: 207 | # if we should scale values to be between 0 and 1 208 | # do this by calling the `scale_to_unit_interval` 209 | # function 210 | this_img = scale_to_unit_interval( 211 | this_x.reshape(img_shape)) 212 | else: 213 | this_img = this_x.reshape(img_shape) 214 | # add the slice to the corresponding position in the 215 | # output array 216 | c = 1 217 | if output_pixel_vals: 218 | c = 255 219 | out_array[ 220 | tile_row * (H + Hs): tile_row * (H + Hs) + H, 221 | tile_col * (W + Ws): tile_col * (W + Ws) + W 222 | ] = this_img * c 223 | return out_array 224 | 225 | 226 | def tile_2d_images(images, canvas_shape): 227 | height = images.shape[1] 228 | width = images.shape[2] 229 | canvas = numpy.zeros( 230 | ((height+1)*canvas_shape[0], (width+1)*canvas_shape[1]), 231 | dtype=images.dtype) 232 | 233 | index = 0 234 | for i in range(canvas_shape[0]): 235 | for j in range(canvas_shape[1]): 236 | if index >= images.shape[0]: 237 | return canvas 238 | v_off = i*(height+1) 239 | h_off = j*(width+1) 240 | canvas[v_off:v_off + height, h_off:h_off+width] = images[index] 241 | index += 1 242 | 243 | return canvas 244 | 245 | 246 | def default(variable, dfault): 247 | return dfault if variable is None else variable 248 | 249 | 250 | def num_abbrev(num, abbrev, sep): 251 | import math 252 | if num < 1: 253 | return num 254 | num = float(num) 255 | millidx = max(0, min(len(abbrev) - 1, 256 | int(math.floor(math.log10(abs(num))/3)))) 257 | return '{:.0f}{}{}'.format(num / 10 ** (3 * millidx), sep, abbrev[millidx]) 258 | 259 | 260 | def human(num): 261 | return num_abbrev(num, 262 | ['', 'Thousand', 'Million', 'Billion', 'Trillion', 263 | 'Quadrillion', 'Quintillion', 'TOO BIG'], ' ') 264 | 265 | 266 | def hum(num): 267 | return num_abbrev(num, 268 | ['', 'thou', 'mill', 'bill', 'trill', 269 | 'quad', 'quin', 'TOO BIG'], ' ') 270 | 271 | 272 | def H(num): 273 | return num_abbrev(num, 274 | ['', 't', 'M', 'B', 'T', 275 | 'Q', 'N', 'TOO BIG'], '') 276 | 277 | 278 | def h(num): 279 | return num_abbrev(num, 280 | [' ', 't', 'm', 'b', 'tr', 281 | 'qd', 'qn', '##'], '') 282 | 283 | 284 | def now(): 285 | return ((datetime.datetime.utcnow() - datetime.timedelta(hours=5)) 286 | .strftime('%Y-%m-%d %I:%M:%S %p')) + ' (EST)' 287 | 288 | 289 | def save_output(filename, func, *args, **kw): 290 | class SplitIO(StringIO): 291 | def __init__(self, term): 292 | StringIO.__init__(self) 293 | self.term = term 294 | 295 | def write(self, output): 296 | StringIO.write(self, output) 297 | self.term.write(output) 298 | 299 | def flush(self): 300 | StringIO.flush(self) 301 | self.term.flush() 302 | 303 | term_out, term_error = sys.stdout, sys.stderr 304 | try: 305 | sys.stdout, sys.stderr = SplitIO(term_out), SplitIO(term_error) 306 | func(*args, **kw) 307 | finally: 308 | out, err = sys.stdout, sys.stderr 309 | sys.stdout, sys.stderr = term_out, term_error 310 | out, err = out.getvalue(), err.getvalue() 311 | with open(filename, 'w') as f: 312 | f.write(out + "\n---------------------------\n" + err) 313 | 314 | 315 | def time_once(method): 316 | 317 | def timed(*args, **kw): 318 | if timed.first: 319 | ts = time.time() 320 | 321 | result = method(*args, **kw) 322 | 323 | if timed.first: 324 | te = time.time() 325 | 326 | # print '%r (%r, %r) %2.2f sec' % \ 327 | # (method.__name__, args, kw, te-ts) 328 | 329 | print('func: {}; time: {}'.format(method.__name__, te - ts)) 330 | 331 | timed.first = False 332 | return result 333 | 334 | timed.first = True 335 | 336 | return timed -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import find_packages 3 | from setuptools import setup 4 | 5 | root = os.path.abspath(os.path.dirname(__file__)) 6 | 7 | setup( 8 | name='pydnn', 9 | version='0.0.dev', 10 | description='deep neural network library in Python', 11 | long_description=open(os.path.join(root, 'README.rst')).read(), 12 | classifiers=[ 13 | 'Development Status :: 2 - Pre-Alpha', 14 | 'Programming Language :: Python :: 2.7', 15 | 'Programming Language :: Python :: 3.4', 16 | 'License :: OSI Approved :: MIT License', 17 | 'Environment :: Console', 18 | 'Intended Audience :: Science/Research', 19 | 'Intended Audience :: Developers', 20 | 'Intended Audience :: Education', 21 | 'Intended Audience :: End Users/Desktop', 22 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 23 | 'Topic :: Scientific/Engineering :: Image Recognition', 24 | 'Topic :: Scientific/Engineering :: Information Analysis', 25 | 'Topic :: Software Development :: Libraries'], 26 | keywords='neural network deep learning AI machine learning', 27 | author='Isaac Kriegman', 28 | author_email='zackriegman@gmail.com', 29 | url='https://github.com/zackriegman/pydnn', 30 | license='MIT', 31 | packages=find_packages(), 32 | package_data={ 33 | 'pydnn': ['aws_util.conf'], 34 | 'examples.plankton': ['plankton.conf']}, 35 | zip_safe=False, 36 | install_requires=[ 37 | 'numpy', 38 | 'scipy', 39 | 'Theano>=0.7.0rc1.dev', 40 | 'pyyaml', 41 | 'boto', 42 | 'pandas'], 43 | extras_require={'docs': ['Sphinx']}, 44 | ) -------------------------------------------------------------------------------- /tests/examples/test_plankton.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from examples.plankton import plankton 3 | 4 | from pydnn import neuralnet as nn 5 | from pydnn import preprocess as pp 6 | from os.path import join 7 | 8 | 9 | class QuickTest(object): 10 | def __init__(p): 11 | p.name = 'quick_test' 12 | p.batch_size = 4 13 | # careful changing image shape... can cause theano exceptions 14 | # with convolution networks 15 | p.image_shape = (64, 64) 16 | p.num_images = p.batch_size * 40 17 | p.num_submission_images = 10 18 | # p.num_submission_images = None 19 | p.epochs = 30 20 | p.final_epochs = 2 21 | p.train_pct = 80 22 | p.valid_pct = 15 23 | p.test_pct = 5 24 | p.rng_seed = 13579 25 | 26 | p.annealer = nn.WackyLearningRateAnnealer( 27 | learning_rate=0.08, 28 | min_learning_rate=0.008, 29 | patience=1, 30 | train_improvement_threshold=0.9, 31 | valid_improvement_threshold=0.95, 32 | reset_on_decay=None) 33 | p.learning_rule = nn.Momentum( 34 | initial_momentum=0.5, 35 | max_momentum=0.5, 36 | learning_rate=p.annealer) 37 | 38 | p.activation = nn.relu 39 | p.l1_reg = 0.000 40 | p.l2_reg = 0.000 41 | p.dropout_conv = 0.3 42 | p.dropout_hidd = 0.5 43 | 44 | p.batch_normalize = False 45 | 46 | # p.resizer = nn.PreserveAspectRatioResizer 47 | p.resizer = pp.StochasticStretchResizer 48 | # p.resizer = pp.StretchResizer 49 | p.stochastic_stretch_range = (0.5, 1.5) 50 | p.preprocessor = pp.Rotator360 51 | 52 | def build_net(p, n): 53 | n.add_convolution(8, (5, 5), (10, 10), (4, 4), batch_normalize=p.batch_normalize) 54 | n.add_dropout(p.dropout_conv) 55 | n.merge_data_channel('shapes') 56 | n.add_hidden(256, batch_normalize=p.batch_normalize) 57 | n.add_dropout(p.dropout_hidd) 58 | n.add_logistic() 59 | 60 | 61 | class QuickTest1(QuickTest): 62 | def __init__(p): 63 | QuickTest.__init__(p) 64 | p.resizer = pp.PreserveAspectRatioResizer 65 | p.name = 'quick_test_1' 66 | p.epochs = 1 67 | p.final_epochs = 0 68 | p.learning_rule = nn.StochasticGradientDescent(p.annealer) 69 | 70 | def build_net(p, n): 71 | """ test net without augmentation or drop out """ 72 | n.add_convolution(4, (5, 5), (10, 10)) 73 | n.add_mlp(32) 74 | 75 | 76 | class QuickTest2(QuickTest1): 77 | def __init__(p): 78 | QuickTest1.__init__(p) 79 | p.batch_size = 4 80 | p.num_images = p.batch_size * 20 81 | # p.num_images = None 82 | p.preprocessor = pp.Canonicalizer 83 | p.resizer = pp.ThresholdBoxStretchResizer 84 | p.name = 'quick_test_2' 85 | p.epochs = 10 86 | p.final_epochs = 2 87 | p.rng_seed = 128928 88 | p.box_threshold = 253 89 | 90 | 91 | class QuickTest3(QuickTest2): 92 | def __init__(p): 93 | QuickTest2.__init__(p) 94 | p.resizer = pp.ContiguousBoxStretchResizer 95 | p.name = 'quick_test_3' 96 | p.epochs = 10 97 | p.final_epochs = 2 98 | p.contiguous_box_threshold = 2 99 | 100 | 101 | class QuickTest4(QuickTest3): 102 | def __init__(p): 103 | QuickTest3.__init__(p) 104 | p.name = 'quick_test_4' 105 | 106 | def build_net(p, n): 107 | """ test net without augmentation or drop out """ 108 | n.add_dropout(0.5) 109 | n.add_convolution(4, (5, 5), (10, 10)) 110 | n.add_mlp(32) 111 | 112 | 113 | class QuickTest5(QuickTest): 114 | def __init__(p): 115 | QuickTest.__init__(p) 116 | p.name = 'quick_test_5' 117 | p.resizer = pp.StretchResizer 118 | p.num_images = None 119 | p.epochs = 10 120 | p.final_epochs = 1 121 | p.train_pct = 100 122 | p.valid_pct = 0 123 | p.test_pct = 0 124 | 125 | p.activation = nn.tanh 126 | 127 | def build_net(p, n): 128 | """ test net without augmentation or drop out """ 129 | n.add_mlp(4) 130 | 131 | 132 | class QuickTest6(QuickTest): 133 | def __init__(p): 134 | QuickTest.__init__(p) 135 | p.name = 'quick_test_6' 136 | p.resizer = pp.StretchResizer 137 | p.epochs = 10 138 | p.final_epochs = 1 139 | p.annealer = nn.LearningRateDecay( 140 | learning_rate=0.08, 141 | decay=.03) 142 | p.learning_rule = nn.AdaDelta( 143 | rho=0.95, 144 | epsilon=1e-6, 145 | learning_rate=p.annealer) 146 | 147 | p.activation = nn.sigmoid 148 | 149 | def build_net(p, n): 150 | """ test net without augmentation or drop out """ 151 | n.add_mlp(4) 152 | 153 | 154 | class QuickTest7(QuickTest6): 155 | def __init__(p): 156 | super(QuickTest7, p).__init__() 157 | p.name = 'quick_test_7' 158 | p.epochs = 1 159 | p.final_epochs = 1 160 | p.image_shape = (2, 2) 161 | p.annealer = nn.LearningRateDecay( 162 | learning_rate=0.08, 163 | decay=.03, 164 | min_learning_rate=0.000000008) 165 | p.learning_rule = nn.StochasticGradientDescent(p.annealer) 166 | 167 | def build_net(p, n): 168 | n.add_mlp(1) 169 | 170 | 171 | class QuickTest8(QuickTest7): 172 | def __init__(p): 173 | super(QuickTest8, p).__init__() 174 | p.name = 'quick_test_8' 175 | p.epochs = 100 176 | p.final_epochs = 2 177 | p.image_shape = (2, 2) 178 | p.batch_size = 2 179 | p.num_images = p.batch_size * 20 180 | 181 | p.annealer = nn.WackyLearningRateAnnealer( 182 | learning_rate=0.08, 183 | min_learning_rate=0.000000008, 184 | patience=2, 185 | train_improvement_threshold=0.9, 186 | valid_improvement_threshold=0.95, 187 | reset_on_decay=None) 188 | p.learning_rule = nn.Momentum( 189 | initial_momentum=0.5, 190 | max_momentum=0.6, 191 | learning_rate=p.annealer) 192 | 193 | p.activation = nn.prelu 194 | 195 | 196 | class QuickTest9(QuickTest): 197 | def __init__(p): 198 | super(QuickTest9, p).__init__() 199 | p.name = 'quick_test_9' 200 | p.image_shape = (32, 32) 201 | p.num_images = p.batch_size * 20 202 | p.num_submission_images = 10 203 | p.epochs = 2 204 | p.final_epochs = 0 205 | 206 | p.batch_normalize = True 207 | 208 | p.resizer = pp.StochasticStretchResizer 209 | # p.resizer = pp.StretchResizer 210 | p.stochastic_stretch_range = (0.5, 1.5) 211 | p.preprocessor = pp.Rotator360PlusGeometry 212 | 213 | def build_net(p, n): 214 | n.add_conv_pool(8, (5, 5), (10, 10), (4, 4), use_bias=False) 215 | n.merge_data_channel('geometry') 216 | n.add_batch_normalization() 217 | n.add_nonlinearity(nn.relu) 218 | n.add_dropout(p.dropout_conv) 219 | n.add_fully_connected(8, weight_init=nn.relu, use_bias=False) 220 | n.merge_data_channel('geometry') 221 | n.add_batch_normalization() 222 | n.add_nonlinearity(nn.relu) 223 | n.add_dropout(p.dropout_hidd) 224 | n.add_logistic() 225 | 226 | 227 | class QuickTest10(QuickTest): 228 | def __init__(p): 229 | super(QuickTest10, p).__init__() 230 | p.name = 'quick_test_10' 231 | p.learning_rule = nn.Adam( 232 | b1=0.9, 233 | b2=0.999, 234 | e=1e-8, 235 | lmbda=1 - 1e-8, 236 | learning_rate=0.001) 237 | 238 | 239 | class QuickTest11(QuickTest8): 240 | def __init__(p): 241 | super(QuickTest11, p).__init__() 242 | p.annealer = nn.WackyLearningRateAnnealer( 243 | learning_rate=0.08, 244 | min_learning_rate=0.000000008, 245 | patience=2, 246 | train_improvement_threshold=0.9, 247 | valid_improvement_threshold=0.95, 248 | reset_on_decay='validation') 249 | p.learning_rule = nn.Momentum( 250 | initial_momentum=0.5, 251 | max_momentum=0.6, 252 | learning_rate=p.annealer) 253 | 254 | 255 | class QuickTest12(QuickTest7): 256 | def __init__(p): 257 | super(QuickTest7, p).__init__() 258 | p.name = 'quick_test_12' 259 | p.epochs = 8 260 | p.final_epochs = 2 261 | 262 | p.learning_rule = nn.StochasticGradientDescent( 263 | learning_rate=nn.LearningRateSchedule( 264 | schedule=((0, .4), 265 | (2, .2), 266 | (3, .1), 267 | (7, .05)))) 268 | 269 | 270 | class QuickTest13(QuickTest): 271 | def __init__(p): 272 | super(QuickTest13, p).__init__() 273 | p.name = 'quick_test_13' 274 | 275 | def build_net(p, n): 276 | n.add_convolution(8, (5, 5), (5, 5), (4, 4), batch_normalize=p.batch_normalize) 277 | n.add_dropout(p.dropout_conv) 278 | pathways = n.split_pathways(3) 279 | for pw in pathways + [n]: 280 | pw.add_convolution(8, (5, 5), (10, 10), (4, 4), batch_normalize=p.batch_normalize) 281 | pw.merge_data_channel('shapes') 282 | pw.add_hidden(8, batch_normalize=p.batch_normalize) 283 | n.merge_pathways(pathways) 284 | n.add_dropout(p.dropout_hidd) 285 | n.add_hidden(8, batch_normalize=p.batch_normalize) 286 | n.add_dropout(p.dropout_hidd) 287 | n.add_logistic() 288 | 289 | 290 | class QuickTest14(QuickTest): 291 | def __init__(p): 292 | super(QuickTest14, p).__init__() 293 | p.name = 'quick_test_14' 294 | 295 | def build_net(p, n): 296 | n.add_convolution(8, (5, 5), (5, 5), (4, 4), batch_normalize=p.batch_normalize) 297 | n.add_dropout(p.dropout_conv) 298 | pathway = n.split_pathways() 299 | for pw in [pathway, n]: 300 | pw.add_convolution(8, (5, 5), (10, 10), (4, 4), batch_normalize=p.batch_normalize) 301 | pw.merge_data_channel('shapes') 302 | pw.add_hidden(8, batch_normalize=p.batch_normalize) 303 | n.merge_pathways(pathway) 304 | n.add_dropout(p.dropout_hidd) 305 | n.add_hidden(8, batch_normalize=p.batch_normalize) 306 | n.add_dropout(p.dropout_hidd) 307 | n.add_logistic() 308 | 309 | 310 | class TestGeneral(unittest.TestCase): 311 | def test_prediction_with_saved_reloaded_net(self): 312 | plankton.run_experiment(QuickTest()) 313 | print('testing prediction with saved/reloaded net...') 314 | net = nn.load(join(plankton.config['output'], 'quick_test_best_net.pkl')) 315 | print('loading submission data') 316 | files, images = plankton.test_set.build(0, 10) 317 | print('generating probabilities') 318 | preds, probs_all = net.predict({'images': images}) 319 | print(preds, probs_all) 320 | 321 | def test_continue_training_with_save_reloaded_net(self): 322 | plankton.run_experiment(QuickTest1()) 323 | print('testing continue training with saved/reloaded net...') 324 | 325 | e = QuickTest1() 326 | data = plankton.train_set.build(32 * 40) 327 | data = pp.split_training_data(data, 32, 80, 10, 10) 328 | 329 | net = nn.load(join(plankton.config['output'], 'quick_test_1_best_net.pkl')) 330 | 331 | net.preprocessor.set_data(data) 332 | 333 | net.train( 334 | updater=e.learning_rule, 335 | epochs=e.epochs, 336 | final_epochs=e.final_epochs, 337 | l1_reg=e.l1_reg, 338 | l2_reg=e.l2_reg) 339 | 340 | def test_various_experiment_setting_combinations(self): 341 | import theano 342 | print('theano version' + theano.__version__) 343 | 344 | plankton.run_experiment(QuickTest2()) 345 | plankton.run_experiment(QuickTest3()) 346 | plankton.run_experiment(QuickTest4()) 347 | plankton.run_experiment(QuickTest5()) 348 | plankton.run_experiment(QuickTest6()) 349 | plankton.run_experiment(QuickTest7()) 350 | plankton.run_experiment(QuickTest8()) 351 | plankton.run_experiment(QuickTest9()) 352 | plankton.run_experiment(QuickTest10()) 353 | plankton.run_experiment(QuickTest11()) 354 | plankton.run_experiment(QuickTest12()) 355 | plankton.run_experiment(QuickTest13()) 356 | plankton.run_experiment(QuickTest14()) -------------------------------------------------------------------------------- /tests/test_nn.py: -------------------------------------------------------------------------------- 1 | from pydnn import neuralnet as nn 2 | import unittest 3 | 4 | 5 | class TestNonlinearities(unittest.TestCase): 6 | def test_ReLU(self): 7 | self.assertEqual( 8 | [nn.relu(x) for x in [-10**8, -1, -0.5, 0, 0.5, 1, 10**8]], 9 | [0, 0, 0, 0, 0.5, 1, 10**8]) --------------------------------------------------------------------------------