├── .coveragerc
├── .gitignore
├── .travis.yml
├── CHANGES.txt
├── LICENSE.txt
├── README.rst
├── docs
├── LayerDefExample.svg
├── Makefile
├── cache.rst
├── conf.py
├── dbn.rst
├── decaf.rst
├── index.rst
├── inischema.rst
├── lasagne.rst
├── metrics.rst
└── notebooks
│ └── CNN_tutorial.ipynb
├── nolearn
├── __init__.py
├── _compat.py
├── cache.py
├── caffe.py
├── dbn.py
├── decaf.py
├── grid_search.py
├── inischema.py
├── lasagne
│ ├── __init__.py
│ ├── base.py
│ ├── handlers.py
│ ├── tests
│ │ ├── conftest.py
│ │ ├── test_base.py
│ │ ├── test_handlers.py
│ │ └── test_visualize.py
│ ├── util.py
│ └── visualize.py
├── metrics.py
├── overfeat.py
├── tests
│ ├── __init__.py
│ ├── conftest.py
│ ├── test_cache.py
│ ├── test_grid_search.py
│ ├── test_inischema.py
│ └── test_metrics.py
└── util.py
├── requirements.txt
├── setup.cfg
└── setup.py
/.coveragerc:
--------------------------------------------------------------------------------
1 | [run]
2 | omit = *test*py
3 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .coverage
2 | nolearn.egg-info
3 | cache/
4 |
5 | bin/
6 | dist/
7 | docs/_build/
8 | include/
9 | lib/
10 | man/
11 | local
12 | pip-selfcheck.json
13 |
14 | *pyc
15 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: python
2 | sudo: false
3 | branches:
4 | only:
5 | - master
6 | python:
7 | - "2.7"
8 | - "3.5"
9 | addons:
10 | apt:
11 | packages:
12 | - libblas-dev
13 | - liblapack-dev
14 | - gfortran
15 | - graphviz
16 | before_install:
17 | - pip install -U pip setuptools wheel
18 | install:
19 | - travis_wait travis_retry pip install -r requirements.txt
20 | - travis_retry pip install -e .[all]
21 | - mkdir -p ~/scikit_learn_data/mldata
22 | - wget -P ~/scikit_learn_data/mldata https://github.com/amplab/datascience-sp14/raw/master/lab7/mldata/mnist-original.mat
23 | script: py.test
24 | cache:
25 | - apt
26 | - directories:
27 | - $HOME/.cache/pip
28 | - $HOME/.theano
29 |
--------------------------------------------------------------------------------
/CHANGES.txt:
--------------------------------------------------------------------------------
1 | Change History
2 | ==============
3 |
4 | 0.6.1 - 2019-11-05
5 | ------------------
6 |
7 | See Github for a list of changes between this release and the last:
8 | https://github.com/dnouri/nolearn/pulls?q=is%3Apr+is%3Aclosed
9 |
10 | 0.6.0 - 2016-08-27
11 | ------------------
12 |
13 | Thanks to @BenjaminBossan, @cancan101, @DanChianucci who greatly
14 | contributed to this release.
15 |
16 | - lasagne: Many improvements to the nolearn.lasagne interface. Some
17 | of the more important changes:
18 |
19 | - Add basic support for multiple outputs
20 | https://github.com/dnouri/nolearn/pull/278
21 |
22 | - Extra scores can now be computed as part of Theano computation
23 | graph
24 | https://github.com/dnouri/nolearn/pull/261
25 |
26 | - Fix excessive memory usage in batch iterator when using shuffle
27 | https://github.com/dnouri/nolearn/pull/238
28 |
29 | - Add visualization code for saliency maps
30 | https://github.com/dnouri/nolearn/pull/223
31 |
32 | - Add method for convenient access of network's intermediate layer
33 | output
34 | https://github.com/dnouri/nolearn/pull/196
35 |
36 | - Allow gradients to be scaled per layer
37 | https://github.com/dnouri/nolearn/pull/195
38 |
39 | - Add shuffling to BatchIterator
40 | https://github.com/dnouri/nolearn/pull/193
41 |
42 | - Add l1 and l2 regularization to default objective
43 | https://github.com/dnouri/nolearn/pull/169
44 |
45 | - Add RememberBestWeights handler: restores best weights after
46 | training
47 | https://github.com/dnouri/nolearn/pull/155
48 |
49 | - Passing Lasagne layer instances to 'layers' parameter of NeuralNet
50 | is now possible
51 | https://github.com/dnouri/nolearn/pull/146
52 |
53 | - Add feature visualization functions plot_loss, plot_weights,
54 | plot_activity, and plot_occlusion. The latter shows for image
55 | samples, which part of the images are crucial for the prediction
56 | https://github.com/dnouri/nolearn/pull/74
57 |
58 | - Add SaveWeights handler that saves weights to disk every n epochs
59 | https://github.com/dnouri/nolearn/pull/91
60 |
61 | - In verbose mode, print out more detailed layer information before
62 | starting with training
63 | https://github.com/dnouri/nolearn/pull/85
64 |
65 | - List of NeuralNet's 'layers' parameter may now be formatted to
66 | contain '(layer_factory, layer_kwargs)' tuples
67 | https://github.com/dnouri/nolearn/pull/73
68 |
69 | - dbn: Added back module dbn because there's a few online articles
70 | referencing it. Works with Python 2 only.
71 |
72 | - Removed deprecated modules. Also deprecate grid_search module.
73 |
74 | 0.5 - 2015-01-22
75 | ----------------
76 |
77 | - Deprecated modules console, dataset, dbn, and model.
78 |
79 | - lasagne: Added scikit-learn compatible wrapper around the `Lasagne`
80 | neural network library for building simple feed-forward networks.
81 |
82 | 0.5b1 - 2014-08-09
83 | ------------------
84 |
85 | - overfeat: Add OverFeat-based feature extractor.
86 |
87 | - caffe: Add feature extractor based on ImageNet-pretrained nets found
88 | in caffe.
89 |
90 | 0.4 - 2014-01-15
91 | ----------------
92 |
93 | - cache: Use joblib's `numpy_pickle` instead of `cPickle` to persist.
94 |
95 | 0.3.1 - 2013-11-18
96 | ------------------
97 |
98 | - convnet: Add `center_only` and `classify_direct` options.
99 |
100 | 0.3 - 2013-11-02
101 | ----------------
102 |
103 | - convnet: Add scikit-learn estimator based on Jia and Donahue's
104 | `DeCAF`.
105 |
106 | - dbn: Change default args of `use_re_lu=True` and `nesterov=True`.
107 |
108 | 0.2 - 2013-03-03
109 | ----------------
110 |
111 | - dbn: Add parameters `learn_rate_decays` and `learn_rate_minimums`,
112 | which allow for decreasing the learning after each epoch of
113 | fine-tuning.
114 |
115 | - dbn: Allow `-1` as the value of the input and output layers of the
116 | neural network. The shapes of `X` and `y` will then be used to
117 | determine those.
118 |
119 | - dbn: Add support for processing sparse input data matrices.
120 |
121 | - dbn: Improve miserable speed of `DBN.predict_proba`.
122 |
123 | 0.2b1 - 2012-12-30
124 | ------------------
125 |
126 | - Added a scikit-learn estimator based on George Dahl's `gdbn` in
127 | `nolearn.dbn`.
128 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2012-2015 Daniel Nouri
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining
4 | a copy of this software and associated documentation files (the
5 | "Software"), to deal in the Software without restriction, including
6 | without limitation the rights to use, copy, modify, merge, publish,
7 | distribute, sublicense, and/or sell copies of the Software, and to
8 | permit persons to whom the Software is furnished to do so, subject to
9 | the following conditions:
10 |
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 |
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | *nolearn* contains a number of wrappers and abstractions around
2 | existing neural network libraries, most notably `Lasagne
3 | `_, along with a few machine learning
4 | utility modules. All code is written to be compatible with
5 | `scikit-learn `_.
6 |
7 | .. note::
8 |
9 | nolearn is currently unmaintained. However, if you follow the
10 | installation instructions, you should still be able to get it to
11 | work (namely with library versions that are outdated at this point).
12 |
13 | If you're looking for an alternative to *nolearn.lasagne*, a library
14 | that integrates neural networks with scikit-learn, then take a look
15 | at `skorch `_, which wraps
16 | PyTorch for scikit-learn.
17 |
18 |
19 | .. image:: https://travis-ci.org/dnouri/nolearn.svg?branch=master
20 | :target: https://travis-ci.org/dnouri/nolearn
21 |
22 | Installation
23 | ============
24 |
25 | We recommend using `venv
26 | `_ (when using Python 3)
27 | or `virtualenv
28 | `_
29 | (Python 2) to install nolearn.
30 |
31 | nolearn comes with a list of known good versions of dependencies that
32 | we test with in ``requirements.txt``. To install the latest version
33 | of nolearn from Git along with these known good dependencies, run
34 | these two commands::
35 |
36 | pip install -r https://raw.githubusercontent.com/dnouri/nolearn/master/requirements.txt
37 | pip install git+https://github.com/dnouri/nolearn.git
38 |
39 | Documentation
40 | =============
41 |
42 | If you're looking for how to use *nolearn.lasagne*, then there's two
43 | introductory tutorials that you can choose from:
44 |
45 | - `Using convolutional neural nets to detect facial keypoints tutorial
46 | `_
47 | with `code `__
48 |
49 | - `Training convolutional neural networks with nolearn
50 | `_
51 |
52 | For specifics around classes and functions out of the *lasagne*
53 | package, such as layers, updates, and nonlinearities, you'll want to
54 | look at the `Lasagne project's documentation
55 | `_.
56 |
57 | *nolearn.lasagne* comes with a `number of tests
58 | `__
59 | that demonstrate some of the more advanced features, such as networks
60 | with merge layers, and networks with multiple inputs.
61 |
62 | `nolearn's own documentation `__
63 | is somewhat out of date at this point. But there's more resources
64 | online.
65 |
66 | Finally, there's a few presentations and examples from around the web.
67 | Note that some of these might need a specific version of nolearn and
68 | Lasange to run:
69 |
70 | - Oliver Dürr's `Convolutional Neural Nets II Hands On
71 | `_ with
72 | `code `__
73 |
74 | - Roelof Pieters' presentation `Python for Image Understanding
75 | `_
76 | comes with nolearn.lasagne code examples
77 |
78 | - Benjamin Bossan's `Otto Group Product Classification Challenge
79 | using nolearn/lasagne
80 | `_
81 |
82 | - Kaggle's `instructions on how to set up an AWS GPU instance to run
83 | nolearn.lasagne
84 | `_
85 | and the facial keypoint detection tutorial
86 |
87 | - `An example convolutional autoencoder
88 | `_
89 |
90 | - Winners of the saliency prediction task in the 2015 `LSUN Challenge
91 | `_ have published their
92 | `lasagne/nolearn-based code
93 | `__.
94 |
95 | - The winners of the 2nd place in the `Kaggle Diabetic Retinopathy Detection
96 | challenge `_ have
97 | published their `lasagne/nolearn-based code
98 | `__.
99 |
100 | - The winner of the 2nd place in the `Kaggle Right Whale Recognition
101 | challenge `_ has
102 | published his `lasagne/nolearn-based code
103 | `__.
104 |
105 | Support
106 | =======
107 |
108 | If you're seeing a bug with nolearn, please submit a bug report to the
109 | `nolearn issue tracker `_.
110 | Make sure to include information such as:
111 |
112 | - how to reproduce the error: show us how to trigger the bug using a
113 | minimal example
114 |
115 | - what versions you are using: include the Git revision and/or version
116 | of nolearn (and possibly Lasagne) that you're using
117 |
118 | Please also make sure to search the issue tracker to see if your issue
119 | has been encountered before or fixed.
120 |
121 | If you believe that you're seeing an issue with Lasagne, which is a
122 | different software project, please use the `Lasagne issue tracker
123 | `_ instead.
124 |
125 | There's currently no user mailing list for nolearn. However, if you
126 | have a question related to Lasagne, you might want to try the `Lasagne
127 | users list `_, or use
128 | Stack Overflow. Please refrain from contacting the authors for
129 | non-commercial support requests directly; public forums are the right
130 | place for these.
131 |
132 | Citation
133 | ========
134 |
135 | Citations are welcome:
136 |
137 | Daniel Nouri. 2014. *nolearn: scikit-learn compatible neural
138 | network library* https://github.com/dnouri/nolearn
139 |
140 | License
141 | =======
142 |
143 | See the `LICENSE.txt `_ file for license rights and
144 | limitations (MIT).
145 |
--------------------------------------------------------------------------------
/docs/LayerDefExample.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
447 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line.
5 | SPHINXOPTS =
6 | SPHINXBUILD = sphinx-build
7 | PAPER =
8 | BUILDDIR = _build
9 |
10 | # Internal variables.
11 | PAPEROPT_a4 = -D latex_paper_size=a4
12 | PAPEROPT_letter = -D latex_paper_size=letter
13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
14 | # the i18n builder cannot share the environment and doctrees with the others
15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
16 |
17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
18 |
19 | help:
20 | @echo "Please use \`make ' where is one of"
21 | @echo " html to make standalone HTML files"
22 | @echo " dirhtml to make HTML files named index.html in directories"
23 | @echo " singlehtml to make a single large HTML file"
24 | @echo " pickle to make pickle files"
25 | @echo " json to make JSON files"
26 | @echo " htmlhelp to make HTML files and a HTML help project"
27 | @echo " qthelp to make HTML files and a qthelp project"
28 | @echo " devhelp to make HTML files and a Devhelp project"
29 | @echo " epub to make an epub"
30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
31 | @echo " latexpdf to make LaTeX files and run them through pdflatex"
32 | @echo " text to make text files"
33 | @echo " man to make manual pages"
34 | @echo " texinfo to make Texinfo files"
35 | @echo " info to make Texinfo files and run them through makeinfo"
36 | @echo " gettext to make PO message catalogs"
37 | @echo " changes to make an overview of all changed/added/deprecated items"
38 | @echo " linkcheck to check all external links for integrity"
39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)"
40 |
41 | clean:
42 | -rm -rf $(BUILDDIR)/*
43 |
44 | html:
45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
46 | @echo
47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
48 |
49 | dirhtml:
50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
51 | @echo
52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
53 |
54 | singlehtml:
55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
56 | @echo
57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
58 |
59 | pickle:
60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
61 | @echo
62 | @echo "Build finished; now you can process the pickle files."
63 |
64 | json:
65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
66 | @echo
67 | @echo "Build finished; now you can process the JSON files."
68 |
69 | htmlhelp:
70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
71 | @echo
72 | @echo "Build finished; now you can run HTML Help Workshop with the" \
73 | ".hhp project file in $(BUILDDIR)/htmlhelp."
74 |
75 | qthelp:
76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
77 | @echo
78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \
79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/nolearn.qhcp"
81 | @echo "To view the help file:"
82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/nolearn.qhc"
83 |
84 | devhelp:
85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
86 | @echo
87 | @echo "Build finished."
88 | @echo "To view the help file:"
89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/nolearn"
90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/nolearn"
91 | @echo "# devhelp"
92 |
93 | epub:
94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
95 | @echo
96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
97 |
98 | latex:
99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
100 | @echo
101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \
103 | "(use \`make latexpdf' here to do that automatically)."
104 |
105 | latexpdf:
106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
107 | @echo "Running LaTeX files through pdflatex..."
108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf
109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
110 |
111 | text:
112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
113 | @echo
114 | @echo "Build finished. The text files are in $(BUILDDIR)/text."
115 |
116 | man:
117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
118 | @echo
119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
120 |
121 | texinfo:
122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
123 | @echo
124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
125 | @echo "Run \`make' in that directory to run these through makeinfo" \
126 | "(use \`make info' here to do that automatically)."
127 |
128 | info:
129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
130 | @echo "Running Texinfo files through makeinfo..."
131 | make -C $(BUILDDIR)/texinfo info
132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
133 |
134 | gettext:
135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
136 | @echo
137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
138 |
139 | changes:
140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
141 | @echo
142 | @echo "The overview file is in $(BUILDDIR)/changes."
143 |
144 | linkcheck:
145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
146 | @echo
147 | @echo "Link check complete; look for any errors in the above output " \
148 | "or in $(BUILDDIR)/linkcheck/output.txt."
149 |
150 | doctest:
151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
152 | @echo "Testing of doctests in the sources finished, look at the " \
153 | "results in $(BUILDDIR)/doctest/output.txt."
154 |
--------------------------------------------------------------------------------
/docs/cache.rst:
--------------------------------------------------------------------------------
1 | :mod:`nolearn.cache`
2 | --------------------
3 |
4 | .. automodule:: nolearn.cache
5 |
6 | .. autofunction:: cached
7 |
--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import sys, os
4 |
5 | # If extensions (or modules to document with autodoc) are in another directory,
6 | # add these directories to sys.path here. If the directory is relative to the
7 | # documentation root, use os.path.abspath to make it absolute, like shown here.
8 | #sys.path.insert(0, os.path.abspath('.'))
9 |
10 | # -- General configuration -----------------------------------------------------
11 | needs_sphinx = '1.3'
12 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon']
13 |
14 | source_suffix = '.rst'
15 | master_doc = 'index'
16 |
17 | # General information about the project.
18 | project = u'nolearn'
19 | copyright = u'2012-2016, Daniel Nouri'
20 |
21 | version = '0.6' # The short X.Y version.
22 | release = '0.6.1.dev0' # The full version, including alpha/beta/rc tags.
23 |
24 | exclude_patterns = ['_build']
25 |
26 | pygments_style = 'sphinx'
27 |
28 | # A list of ignored prefixes for module index sorting.
29 | #modindex_common_prefix = []
30 |
31 |
32 | # -- Options for HTML output ---------------------------------------------------
33 | # The theme to use for HTML and HTML Help pages. See the documentation for
34 | # a list of builtin themes.
35 | force_rtd = os.environ.get('FORCE_RTD_THEME', None) == 'True'
36 | if force_rtd: # only import and set the theme if we're building docs locally
37 | import sphinx_rtd_theme
38 | html_theme = 'sphinx_rtd_theme'
39 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
40 | else:
41 | html_theme = 'default'
42 |
43 |
44 |
45 |
46 | htmlhelp_basename = 'nolearndoc'
47 |
48 |
49 | # Grouping the document tree into LaTeX files. List of tuples
50 | # (source start file, target name, title, author, documentclass [howto/manual]).
51 | latex_documents = [
52 | ('index', 'nolearn.tex', u'nolearn Documentation',
53 | u'Daniel Nouri', 'manual'),
54 | ]
55 |
56 |
57 |
58 |
59 | # -- Options for manual page output --------------------------------------------
60 | # One entry per manual page. List of tuples
61 | # (source start file, name, description, authors, manual section).
62 | man_pages = [
63 | ('index', 'nolearn', u'nolearn Documentation',
64 | [u'Daniel Nouri'], 1)
65 | ]
66 |
67 |
68 |
69 | # -- Options for Texinfo output ------------------------------------------------
70 | # Grouping the document tree into Texinfo files. List of tuples
71 | # (source start file, target name, title, author,
72 | # dir menu entry, description, category)
73 | texinfo_documents = [
74 | ('index', 'nolearn', u'nolearn Documentation',
75 | u'Daniel Nouri', 'nolearn', 'One line description of project.',
76 | 'Miscellaneous'),
77 | ]
78 |
79 |
80 |
81 |
82 |
83 | #-- Options for Module Mocking output ------------------------------------------------
84 | def blank_fn(cls, args, *kwargs):
85 | pass
86 |
87 | class _MockModule(object):
88 | def __init__(self, *args, **kwargs):
89 | pass
90 |
91 | def __call__(self, *args, **kwargs):
92 | return _MockModule()
93 |
94 | @classmethod
95 | def __getattr__(cls, name):
96 | if name in ('__file__', '__path__'):
97 | return os.devnull
98 | elif name[0] == name[0].upper():
99 | # Not very good, we assume Uppercase names are classes...
100 | mocktype = type(name, (), {})
101 | mocktype.__module__ = __name__
102 | mocktype.__init__ = blank_fn
103 | return mocktype
104 | else:
105 | return _MockModule()
106 |
107 |
108 | #autodoc_mock_imports =
109 | MOCK_MODULES = ['numpy',
110 | 'lasagne',
111 | 'lasagne.layers',
112 | 'lasagne.objectives',
113 | 'lasagne.updates',
114 | 'lasagne.regularization',
115 | 'lasagne.utils',
116 | 'sklearn',
117 | 'sklearn.BaseEstimator',
118 | 'sklearn.base',
119 | 'sklearn.metrics',
120 | 'sklearn.cross_validation',
121 | 'sklearn.preprocessing',
122 | 'sklearn.grid_search',
123 | 'caffe.imagenet',
124 | 'skimage.io',
125 | 'skimage.transform',
126 | 'joblib',
127 | 'matplotlib',
128 | 'matplotlib.pyplot',
129 | 'theano',
130 | 'theano.tensor',
131 | 'tabulate']
132 |
133 | sys.modules.update((mod_name, _MockModule()) for mod_name in MOCK_MODULES)
134 |
--------------------------------------------------------------------------------
/docs/dbn.rst:
--------------------------------------------------------------------------------
1 | :mod:`nolearn.dbn`
2 | ------------------
3 |
4 | .. warning::
5 |
6 | The nolearn.dbn module is no longer supported. Take a look at
7 | *nolearn.lasagne* for a more modern neural net toolkit.
8 |
9 | API
10 | ~~~
11 |
12 | .. automodule:: nolearn.dbn
13 |
14 | .. autoclass:: DBN
15 | :special-members:
16 | :members:
17 |
18 |
19 | Example: MNIST
20 | ~~~~~~~~~~~~~~
21 |
22 | Let's train 2-layer neural network to do digit recognition on the
23 | `MNIST dataset `_.
24 |
25 | We first load the MNIST dataset, and split it up into a training and a
26 | test set:
27 |
28 | .. code-block:: python
29 |
30 | from sklearn.cross_validation import train_test_split
31 | from sklearn.datasets import fetch_mldata
32 |
33 | mnist = fetch_mldata('MNIST original')
34 | X_train, X_test, y_train, y_test = train_test_split(
35 | mnist.data / 255.0, mnist.target)
36 |
37 | We then configure a neural network with 300 hidden units, a learning
38 | rate of ``0.3`` and a learning rate decay of ``0.9``, which is the
39 | number that the learning rate will be multiplied with after each
40 | epoch.
41 |
42 | .. code-block:: python
43 |
44 | from nolearn.dbn import DBN
45 |
46 | clf = DBN(
47 | [X_train.shape[1], 300, 10],
48 | learn_rates=0.3,
49 | learn_rate_decays=0.9,
50 | epochs=10,
51 | verbose=1,
52 | )
53 |
54 | Let us now train our network for 10 epochs. This will take around
55 | five minutes on a CPU:
56 |
57 | .. code-block:: python
58 |
59 | clf.fit(X_train, y_train)
60 |
61 | After training, we can use our trained neural network to predict the
62 | examples in the test set. We'll observe that our model has an
63 | accuracy of around **97.5%**.
64 |
65 | .. code-block:: python
66 |
67 | from sklearn.metrics import classification_report
68 | from sklearn.metrics import zero_one_score
69 |
70 | y_pred = clf.predict(X_test)
71 | print "Accuracy:", zero_one_score(y_test, y_pred)
72 | print "Classification report:"
73 | print classification_report(y_test, y_pred)
74 |
75 |
76 | Example: Iris
77 | ~~~~~~~~~~~~~
78 |
79 | In this example, we'll train a neural network for classification on
80 | the `Iris flower data set
81 | `_. Due to the
82 | small number of examples, an SVM will typically perform better, but
83 | let us still see if our neural network is up to the task:
84 |
85 | .. code-block:: python
86 |
87 | from sklearn.cross_validation import cross_val_score
88 | from sklearn.datasets import load_iris
89 | from sklearn.preprocessing import scale
90 |
91 | iris = load_iris()
92 | clf = DBN(
93 | [4, 4, 3],
94 | learn_rates=0.3,
95 | epochs=50,
96 | )
97 |
98 | scores = cross_val_score(clf, scale(iris.data), iris.target, cv=10)
99 | print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() / 2)
100 |
101 | This will print something like::
102 |
103 | Accuracy: 0.97 (+/- 0.03)
104 |
105 |
106 | Example: CIFAR-10
107 | ~~~~~~~~~~~~~~~~~
108 |
109 | In this example, we'll train a neural network to do image
110 | classification using a subset of the `CIFAR-10 dataset
111 | `_.
112 |
113 | We assume that you have the Python version of the CIFAR-10 dataset
114 | downloaded and available in your working directory. We'll use only
115 | the first three batches of the dataset; the first two for training,
116 | the third one for testing.
117 |
118 | Let us load the dataset:
119 |
120 | .. code-block:: python
121 |
122 | import cPickle
123 | import numpy as np
124 |
125 | def load(name):
126 | with open(name, 'rb') as f:
127 | return cPickle.load(f)
128 |
129 | dataset1 = load('data_batch_1')
130 | dataset2 = load('data_batch_2')
131 | dataset3 = load('data_batch_3')
132 |
133 | data_train = np.vstack([dataset1['data'], dataset2['data']])
134 | labels_train = np.hstack([dataset1['labels'], dataset2['labels']])
135 |
136 | data_train = data_train.astype('float') / 255.
137 | labels_train = labels_train
138 | data_test = dataset3['data'].astype('float') / 255.
139 | labels_test = np.array(dataset3['labels'])
140 |
141 | We can now train our network. We'll configure the network so that it
142 | has 1024 units in the hidden layer, i.e. ``[3072, 1024, 10]``. We'll
143 | train our network for 50 epochs, which will take a while if you're not
144 | using `CUDAMat `_.
145 |
146 | .. code-block:: python
147 |
148 | n_feat = data_train.shape[1]
149 | n_targets = labels_train.max() + 1
150 |
151 | net = DBN(
152 | [n_feat, n_feat / 3, n_targets],
153 | epochs=50,
154 | learn_rates=0.03,
155 | verbose=1,
156 | )
157 | net.fit(data_train, labels_train)
158 |
159 | Finally, we'll look at our network's performance:
160 |
161 | .. code-block:: python
162 |
163 | from sklearn.metrics import classification_report
164 | from sklearn.metrics import confusion_matrix
165 |
166 | expected = labels_test
167 | predicted = net.predict(data_test)
168 |
169 | print "Classification report for classifier %s:\n%s\n" % (
170 | net, classification_report(expected, predicted))
171 | print "Confusion matrix:\n%s" % confusion_matrix(expected, predicted)
172 |
173 | You should see an f1-score of **0.49** and a confusion matrix that
174 | looks something like this::
175 |
176 | air aut bir cat dee dog fro hor shi tru
177 | [[459 48 66 39 91 21 5 39 182 44] airplane
178 | [ 28 584 12 31 23 22 8 29 117 188] automobile
179 | [ 49 13 279 101 244 124 31 71 37 16] bird
180 | [ 20 16 54 363 106 255 38 70 36 39] cat
181 | [ 33 10 79 81 596 66 15 75 26 9] deer
182 | [ 16 23 57 232 103 448 17 82 26 25] dog
183 | [ 10 18 70 179 212 106 304 32 21 26] frog
184 | [ 20 8 40 80 125 98 10 575 21 38] horse
185 | [ 54 49 10 29 43 25 4 9 707 31] ship
186 | [ 28 129 9 48 33 36 10 57 118 561]] truck
187 |
188 | We should be able to improve on this score by using the full dataset
189 | and by training longer.
190 |
--------------------------------------------------------------------------------
/docs/decaf.rst:
--------------------------------------------------------------------------------
1 | :mod:`nolearn.decaf`
2 | --------------------
3 |
4 | API
5 | ~~~
6 |
7 | .. automodule:: nolearn.decaf
8 |
9 | .. autoclass:: ConvNetFeatures
10 | :special-members:
11 | :members:
12 |
13 | Installing DeCAF and downloading parameter files
14 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
15 |
16 | You'll need to manually `install DeCAF
17 | `_ for
18 | :class:`ConvNetFeatures` to work.
19 |
20 | You will also need to download a tarball that contains `pretrained
21 | parameter files
22 | `_ from
23 | Yangqing Jia's homepage.
24 |
25 | Refer to the location of the two files contained in the tarball when
26 | you instantiate :class:`ConvNetFeatures` like so:
27 |
28 | .. code-block:: python
29 |
30 | convnet = ConvNetFeatures(
31 | pretrained_params='/path/to/imagenet.decafnet.epoch90',
32 | pretrained_meta='/path/to/imagenet.decafnet.meta',
33 | )
34 |
35 | For more information on how DeCAF works, please refer to [1]_.
36 |
37 | Example: Dogs vs. Cats
38 | ~~~~~~~~~~~~~~~~~~~~~~
39 |
40 | What follows is a simple example that uses :class:`ConvNetFeatures`
41 | and scikit-learn to classify images from the `Kaggle Dogs vs. Cats
42 | challenge `_. Before you
43 | start, you must download the images from the Kaggle competition page.
44 | The ``train/`` folder will be referred to further down as
45 | ``TRAIN_DATA_DIR``.
46 |
47 | We'll first define a few imports and the paths to the files that we
48 | just downloaded:
49 |
50 | .. code-block:: python
51 |
52 | import os
53 |
54 | from nolearn.decaf import ConvNetFeatures
55 | from sklearn.linear_model import LogisticRegression
56 | from sklearn.metrics import accuracy_score
57 | from sklearn.pipeline import Pipeline
58 | from sklearn.utils import shuffle
59 |
60 | DECAF_IMAGENET_DIR = '/path/to/imagenet-files/'
61 | TRAIN_DATA_DIR = '/path/to/dogs-vs-cats-training-images/'
62 |
63 | A ``get_dataset`` function will return a list of all image filenames
64 | and labels, shuffled for our convenience:
65 |
66 | .. code-block:: python
67 |
68 | def get_dataset():
69 | cat_dir = TRAIN_DATA_DIR + 'cat/'
70 | cat_filenames = [cat_dir + fn for fn in os.listdir(cat_dir)]
71 | dog_dir = TRAIN_DATA_DIR + 'dog/'
72 | dog_filenames = [dog_dir + fn for fn in os.listdir(dog_dir)]
73 |
74 | labels = [0] * len(cat_filenames) + [1] * len(dog_filenames)
75 | filenames = cat_filenames + dog_filenames
76 | return shuffle(filenames, labels, random_state=0)
77 |
78 | We can now define our ``sklearn.pipeline.Pipeline``, which merely
79 | consists of :class:`ConvNetFeatures` and a
80 | ``sklearn.linear_model.LogisticRegression`` classifier.
81 |
82 | .. code-block:: python
83 |
84 | def main():
85 | convnet = ConvNetFeatures(
86 | pretrained_params=DECAF_IMAGENET_DIR + 'imagenet.decafnet.epoch90',
87 | pretrained_meta=DECAF_IMAGENET_DIR + 'imagenet.decafnet.meta',
88 | )
89 | clf = LogisticRegression()
90 | pl = Pipeline([
91 | ('convnet', convnet),
92 | ('clf', clf),
93 | ])
94 |
95 | X, y = get_dataset()
96 | X_train, y_train = X[:100], y[:100]
97 | X_test, y_test = X[100:300], y[100:300]
98 |
99 | print "Fitting..."
100 | pl.fit(X_train, y_train)
101 | print "Predicting..."
102 | y_pred = pl.predict(X_test)
103 | print "Accuracy: %.3f" % accuracy_score(y_test, y_pred)
104 |
105 | main()
106 |
107 | Note that we use only 100 images to train our classifier (and 200 for
108 | testing). Regardless, and thanks to the magic of pre-trained
109 | convolutional nets, we're able to reach an accuracy of around 94%,
110 | which is an improvement of 11% over the classifier described in [2]_.
111 |
112 |
113 | .. [1] Jeff Donahue, Yangqing Jia, Oriol Vinyals, Judy Hoffman, Ning
114 | Zhang, Eric Tzeng, and Trevor Darrell. `Decaf: A deep
115 | convolutional activation feature for generic visual
116 | recognition. `_ arXiv preprint
117 | arXiv:1310.1531, 2013.
118 |
119 | .. [2] P. Golle. `Machine learning attacks against the asirra
120 | captcha. `_
121 | In ACM CCS 2008, 2008.
122 |
--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
1 | Welcome to nolearn's documentation!
2 | ===================================
3 |
4 | This package contains a number of utility modules that are helpful
5 | with machine learning tasks. Most of the modules work together with
6 | `scikit-learn `_, others are more generally
7 | useful.
8 |
9 | nolearn's `source is hosted on Github
10 | `_. `Releases can be downloaded on
11 | PyPI `_.
12 |
13 | |build status|_
14 |
15 | .. |build status| image:: https://secure.travis-ci.org/dnouri/nolearn.png?branch=master
16 | .. _build status: http://travis-ci.org/dnouri/nolearn
17 |
18 |
19 | Installation
20 | ============
21 |
22 | We recommend using `virtualenv
23 | `_
24 | to install nolearn.
25 |
26 | To install the latest version of nolearn from Git using `pip
27 | `_, run the following commands::
28 |
29 | pip install -r https://raw.githubusercontent.com/dnouri/nolearn/master/requirements.txt
30 | pip install git+https://github.com/dnouri/nolearn.git@master#egg=nolearn==0.7.git
31 |
32 | To instead install the release from PyPI (which is somewhat old at
33 | this point), do::
34 |
35 | pip install nolearn
36 |
37 | Modules
38 | =======
39 |
40 | .. toctree::
41 | :maxdepth: 2
42 |
43 | cache
44 | dbn
45 | decaf
46 | inischema
47 | lasagne
48 | metrics
49 |
50 |
51 | Indices and tables
52 | ==================
53 |
54 | * :ref:`genindex`
55 | * :ref:`modindex`
56 | * :ref:`search`
57 |
--------------------------------------------------------------------------------
/docs/inischema.rst:
--------------------------------------------------------------------------------
1 | :mod:`nolearn.inischema`
2 | ------------------------
3 |
4 | .. automodule:: nolearn.inischema
5 |
6 | .. autofunction:: parse_config
7 |
--------------------------------------------------------------------------------
/docs/lasagne.rst:
--------------------------------------------------------------------------------
1 | :mod:`nolearn.lasagne`
2 | ----------------------
3 |
4 | Two introductory tutorials exist for *nolearn.lasagne*:
5 |
6 | - `Using convolutional neural nets to detect facial keypoints tutorial
7 | `_
8 | with `code `_
9 |
10 | - `Training convolutional neural networks with nolearn
11 | `_
12 |
13 | For specifics around classes and functions out of the *lasagne*
14 | package, such as layers, updates, and nonlinearities, you'll want to
15 | look at the `Lasagne project's documentation
16 | `_.
17 |
18 | *nolearn.lasagne* comes with a `number of tests
19 | `_
20 | that demonstrate some of the more advanced features, such as networks
21 | with merge layers, and networks with multiple inputs.
22 |
23 | Finally, there's a few presentations and examples from around the web.
24 | Note that some of these might need a specific version of nolearn and
25 | Lasange to run:
26 |
27 | - Oliver Dürr's `Convolutional Neural Nets II Hands On
28 | `_ with
29 | `code `_
30 |
31 | - Roelof Pieters' presentation `Python for Image Understanding
32 | `_
33 | comes with nolearn.lasagne code examples
34 |
35 | - Benjamin Bossan's `Otto Group Product Classification Challenge
36 | using nolearn/lasagne
37 | `_
38 |
39 | - Kaggle's `instructions on how to set up an AWS GPU instance to run
40 | nolearn.lasagne
41 | `_
42 | and the facial keypoint detection tutorial
43 |
44 | - `An example convolutional autoencoder
45 | `_
46 |
47 | - Winners of the saliency prediction task in the 2015 `LSUN Challenge
48 | `_ have published their
49 | `lasagne/nolearn-based code
50 | `_.
51 |
52 | - The winners of the 2nd place in the `Kaggle Diabetic Retinopathy Detection
53 | challenge `_ have
54 | published their `lasagne/nolearn-based code
55 | `_.
56 |
57 | - The winner of the 2nd place in the `Kaggle Right Whale Recognition
58 | challenge `_ has
59 | published his `lasagne/nolearn-based code
60 | `_.
61 |
62 | .. _layer-def:
63 |
64 | Defining The Layers Of A Neural Network
65 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
66 |
67 | There are two supported methods of providing a Layer Definition to the
68 | :class:`.NeuralNet` constructor. The first involves generating the
69 | stack of :mod:`lasagne.layer` instances directly, while the second
70 | uses a declarative list of definitions whereas NeuralNet will do the
71 | actual instantiation of layers.
72 |
73 | The sample network below is given as an example
74 |
75 | .. image:: LayerDefExample.svg
76 |
77 | Passing a Layer Instance Directly
78 | =================================
79 |
80 | This method of defining the layers is more flexible. Very similarly
81 | to how you would define layers in pure Lasagne, we first define and
82 | set up our stack of layer instances, and then pass the output layer(s)
83 | to :meth:`NeuralNet.__init__`. This method is more versatile than the
84 | one described next, and supports all types of :mod:`lasagne.layers`.
85 |
86 | Here's a toy example:
87 |
88 | .. code-block:: python
89 |
90 | from nolearn.lasagne import NeuralNet
91 | from lasagne import layers
92 |
93 | l_input = layers.InputLayer(
94 | shape=(None, 3, 32, 32),
95 | )
96 | l_conv = layers.Conv2DLayer(
97 | l_input,
98 | num_filters=16,
99 | filter_size=7,
100 | )
101 | l_pool = layers.MaxPool2dLayer(
102 | l_conv,
103 | pool_size=2,
104 | )
105 | l_out = layers.DenseLayer(
106 | l_pool,
107 | num_units=300,
108 | )
109 | net = NeuralNet(layers=l_out)
110 |
111 |
112 | Declarative Layer Definition
113 | ============================
114 |
115 | In some situations it's preferable to use the declarative style of
116 | defining layers. It's not as flexible but it's sometimes easier to
117 | write out and manipulate when using features like scikit-learn's model
118 | selection.
119 |
120 | The following example is equivalent to the previous:
121 |
122 | .. code-block:: python
123 |
124 | from nolearn.lasagne import NeuralNet
125 | from lasagne import layers
126 |
127 | net = NeuralNet(
128 | layers=[
129 | (layers.InputLayer, {'shape': (None, 3, 32, 32)}),
130 | (layers.Conv2DLayer, {'num_filters': 16, 'filter_size': 7}),
131 | (layers.MaxPool2DLayer, {'pool_size': 2, 'name': 'pool'}),
132 | (layers.DenseLayer, {'num_units': 300'}),
133 | ],
134 | )
135 |
136 | To give a concrete example of when this is useful when doing model
137 | selection, consider this example that uses
138 | :class:`sklearn.grid_search.GridSearchCV` to find the optimal value
139 | for the pool size of our max pooling layer:
140 |
141 | .. code-block:: python
142 |
143 | from sklearn.grid_search import GridSearchCV
144 |
145 | gs = GridSearchCV(estimator=net, param_grid={'pool__pool_size': [2, 3, 4]})
146 | gs.fit(...)
147 |
148 | Note that we can set the max pooling layer's ``pool_size`` parameter
149 | using a double underscore syntax, the part before the double
150 | underscores refer to the layer's name in the layer definition above.
151 |
152 | API
153 | ~~~
154 |
155 | .. automodule:: nolearn.lasagne
156 |
157 | .. autoclass:: NeuralNet(self, layers, **kwargs)
158 | :members:
159 |
160 | .. automethod:: __init__(self, layers, **kwargs)
161 |
162 | .. autoclass:: BatchIterator
163 | :members:
164 |
165 | .. autoclass:: TrainSplit
166 | :members:
167 |
168 |
--------------------------------------------------------------------------------
/docs/metrics.rst:
--------------------------------------------------------------------------------
1 | :mod:`nolearn.metrics`
2 | ----------------------
3 |
4 | .. automodule:: nolearn.metrics
5 |
6 | .. autofunction:: multiclass_logloss
7 | .. autofunction:: learning_curve
8 | .. autofunction:: learning_curve_logloss
9 |
--------------------------------------------------------------------------------
/nolearn/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dnouri/nolearn/8f4473190a06caf80878821d29178b82329d8bee/nolearn/__init__.py
--------------------------------------------------------------------------------
/nolearn/_compat.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | PY2 = sys.version_info[0] == 2
4 |
5 | if PY2:
6 | from ConfigParser import ConfigParser
7 | from StringIO import StringIO
8 | import cPickle as pickle
9 | import __builtin__ as builtins
10 |
11 | basestring = basestring
12 |
13 | def chain_exception(exc1, exc2):
14 | exec("raise exc1, None, sys.exc_info()[2]")
15 |
16 | else:
17 | from configparser import ConfigParser
18 | from io import StringIO
19 | import pickle as pickle
20 | import builtins
21 |
22 | basestring = str
23 |
24 | def chain_exception(exc1, exc2):
25 | exec("raise exc1 from exc2")
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/nolearn/cache.py:
--------------------------------------------------------------------------------
1 | """This module contains a decorator :func:`cached` that can be used to
2 | cache the results of any Python functions to disk.
3 |
4 | This is useful when you have functions that take a long time to
5 | compute their value, and you want to cache the results of those
6 | functions between runs.
7 |
8 | Python's :mod:`pickle` is used to serialize data. All cache files go
9 | into the `cache/` directory inside your working directory.
10 |
11 | `@cached` uses a cache key function to find out if it has the value
12 | for some given function arguments cached on disk. The way it
13 | calculates that cache key by default is to simply use the string
14 | representation of all arguments passed into the function. Thus, the
15 | default cache key function looks like this:
16 |
17 | .. code-block:: python
18 |
19 | def default_cache_key(*args, **kwargs):
20 | return str(args) + str(sorted(kwargs.items()))
21 |
22 | Here is an example use of the :func:`cached` decorator:
23 |
24 | .. code-block:: python
25 |
26 | import math
27 | @cached()
28 | def fac(x):
29 | print 'called!'
30 | return math.factorial(x)
31 |
32 | fac(20)
33 | called!
34 | 2432902008176640000
35 | fac(20)
36 | 2432902008176640000
37 |
38 | Often you will want to use a more intelligent cache key, one that
39 | takes more things into account. Here's an example cache key function
40 | for a cache decorator used with a `transform` method of a scikit-learn
41 | :class:`~sklearn.base.BaseEstimator`:
42 |
43 | .. doctest::
44 |
45 | >>> def transform_cache_key(self, X):
46 | ... return ','.join([
47 | ... str(X[:20]),
48 | ... str(X[-20:]),
49 | ... str(X.shape),
50 | ... str(sorted(self.get_params().items())),
51 | ... ])
52 |
53 | This function puts the first and the last twenty rows of the matrix
54 | `X` into the cache key. On top of that, it adds the shape of the
55 | matrix `X.shape` along with the items in `self.get_params`, which with
56 | a scikit-learn :class:`~sklearn.base.BaseEstimator` class is the
57 | dictionary of model parameters. This makes sure that even though the
58 | input matrix is the same, it will still calculate the value again if
59 | the value of `self.get_params()` is different.
60 |
61 | Your estimator class can then use the decorator like so:
62 |
63 | .. code-block:: python
64 |
65 | class MyEstimator(BaseEstimator):
66 | @cached(transform_cache_key)
67 | def transform(self, X):
68 | # ...
69 | """
70 |
71 | from functools import wraps
72 | import hashlib
73 | import logging
74 | import random
75 | import os
76 | import string
77 | import traceback
78 |
79 | from joblib import numpy_pickle
80 |
81 |
82 | CACHE_PATH = 'cache/'
83 | if not os.path.exists(CACHE_PATH): # pragma: no cover
84 | os.mkdir(CACHE_PATH)
85 |
86 | logger = logging.getLogger(__name__)
87 |
88 |
89 | def default_cache_key(*args, **kwargs):
90 | return str(args) + str(sorted(kwargs.items()))
91 |
92 |
93 | class DontCache(Exception):
94 | pass
95 |
96 |
97 | def cached(cache_key=default_cache_key, cache_path=None):
98 | def cached(func):
99 | @wraps(func)
100 | def wrapper(*args, **kwargs):
101 | # Calculation of the cache key is delegated to a function
102 | # that's passed in via the decorator call
103 | # (`default_cache_key` by default).
104 | try:
105 | key = str(cache_key(*args, **kwargs)).encode('ascii')
106 | except DontCache:
107 | return func(*args, **kwargs)
108 |
109 | hashed_key = hashlib.sha1(key).hexdigest()[:8]
110 |
111 | # We construct the filename using the cache key. If the
112 | # file exists, unpickle and return the value.
113 | filename = os.path.join(
114 | cache_path or CACHE_PATH,
115 | '{}.{}-cache-{}'.format(
116 | func.__module__, func.__name__, hashed_key))
117 |
118 | if os.path.exists(filename):
119 | filesize = os.path.getsize(filename)
120 | size = "%0.1f MB" % (filesize / (1024 * 1024.0))
121 | logger.debug(" * cache hit: {} ({})".format(filename, size))
122 | return numpy_pickle.load(filename)
123 | else:
124 | logger.debug(" * cache miss: {}".format(filename))
125 | value = func(*args, **kwargs)
126 | tmp_filename = '{}-{}.tmp'.format(
127 | filename,
128 | ''.join(random.sample(string.ascii_letters, 4)),
129 | )
130 | try:
131 | numpy_pickle.dump(value, tmp_filename, compress=9)
132 | os.rename(tmp_filename, filename)
133 | except Exception:
134 | logger.exception(
135 | "Saving pickle {} resulted in Exception".format(
136 | filename))
137 | return value
138 |
139 | wrapper.uncached = func
140 | return wrapper
141 | return cached
142 |
--------------------------------------------------------------------------------
/nolearn/caffe.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 |
3 | from caffe.imagenet import wrapper
4 | from joblib import Parallel
5 | from joblib import delayed
6 | from nolearn import cache
7 | import numpy as np
8 | from sklearn.base import BaseEstimator
9 | from skimage.io import imread
10 | from skimage.transform import resize
11 |
12 | from .util import ChunkedTransform
13 |
14 |
15 | def _forward_cache_key(self, X):
16 | if len(X) == 1:
17 | raise cache.DontCache
18 | return ','.join([
19 | str(X),
20 | self.model_def,
21 | self.pretrained_model,
22 | self.oversample,
23 | ])
24 |
25 |
26 | def _transform_cache_key(self, X):
27 | if len(X) == 1 or not isinstance(X[0], str):
28 | raise cache.DontCache
29 | return ','.join([
30 | str(X),
31 | str(sorted(self.get_params().items())),
32 | ])
33 |
34 |
35 | def _prepare_image(cls, image, oversample='center_only'):
36 | if isinstance(image, str):
37 | image = imread(image)
38 | if image.ndim == 2:
39 | image = np.tile(image[:, :, np.newaxis], (1, 1, 3))
40 |
41 | if oversample in ('center_only', 'corners'):
42 | # Resize and convert to BGR
43 | image = (resize(
44 | image, (cls.IMAGE_DIM, cls.IMAGE_DIM)) * 255)[:, :, ::-1]
45 | # subtract main
46 | image -= cls.IMAGENET_MEAN
47 | return wrapper.oversample(
48 | image, center_only=oversample == 'center_only')
49 | else:
50 | raise ValueError("oversample must be one of 'center_only', 'corners'")
51 |
52 |
53 | _cached_nets = {}
54 |
55 |
56 | class CaffeImageNet(ChunkedTransform, BaseEstimator):
57 | IMAGE_DIM = wrapper.IMAGE_DIM
58 | CROPPED_DIM = wrapper.CROPPED_DIM
59 | IMAGENET_MEAN = wrapper.IMAGENET_MEAN
60 |
61 | def __init__(
62 | self,
63 | model_def='examples/imagenet_deploy.prototxt',
64 | pretrained_model='caffe_reference_imagenet_model',
65 | gpu=True,
66 | oversample='center_only',
67 | num_output=1000,
68 | merge='max',
69 | batch_size=200,
70 | verbose=0,
71 | ):
72 | self.model_def = model_def
73 | self.pretrained_model = pretrained_model
74 | self.gpu = gpu
75 | self.oversample = oversample
76 | self.num_output = num_output
77 | self.merge = merge
78 | self.batch_size = batch_size
79 | self.verbose = verbose
80 |
81 | @classmethod
82 | def Net(cls):
83 | # soft dependency
84 | try:
85 | from caffe import CaffeNet
86 | except ImportError:
87 | from caffe import Net as CaffeNet
88 | return CaffeNet
89 |
90 | @classmethod
91 | def _create_net(cls, model_def, pretrained_model):
92 | key = (cls.__name__, model_def, pretrained_model)
93 | net = _cached_nets.get(key)
94 | if net is None:
95 | net = cls.Net()(model_def, pretrained_model)
96 | _cached_nets[key] = net
97 | return net
98 |
99 | def fit(self, X=None, y=None):
100 | self.net_ = self._create_net(self.model_def, self.pretrained_model)
101 | self.net_.set_phase_test()
102 | if self.gpu:
103 | self.net_.set_mode_gpu()
104 | return self
105 |
106 | @cache.cached(_forward_cache_key)
107 | def _forward(self, images):
108 | if isinstance(images[0], str):
109 | images = Parallel(n_jobs=-1)(delayed(_prepare_image)(
110 | self.__class__,
111 | image,
112 | oversample=self.oversample,
113 | ) for image in images)
114 |
115 | output_blobs = [
116 | np.empty((image.shape[0], self.num_output, 1, 1), dtype=np.float32)
117 | for image in images
118 | ]
119 |
120 | for i in range(len(images)):
121 | # XXX We would expect that we can send in a list of input
122 | # blobs and output blobs. However that produces an
123 | # assertion error.
124 | self.net_.Forward([images[i]], [output_blobs[i]])
125 |
126 | return np.vstack([a[np.newaxis, ...] for a in output_blobs])
127 |
128 | @cache.cached(_transform_cache_key)
129 | def transform(self, X):
130 | return super(CaffeImageNet, self).transform(X)
131 |
132 | def _compute_features(self, images):
133 | output_blobs = self._forward(images)
134 |
135 | features = []
136 | for blob in output_blobs:
137 | blob = blob.reshape((blob.shape[0], blob.shape[1]))
138 | if self.merge == 'max':
139 | blob = blob.max(0)
140 | else:
141 | blob = self.merge(blob)
142 | features.append(blob)
143 |
144 | return np.vstack(features)
145 |
146 | prepare_image = classmethod(_prepare_image)
147 |
148 | def __getstate__(self):
149 | d = self.__dict__.copy()
150 | d.pop('net_', None)
151 | return d
152 |
153 | def __setstate__(self, state):
154 | self.__dict__.update(state)
155 | self.fit()
156 |
--------------------------------------------------------------------------------
/nolearn/dbn.py:
--------------------------------------------------------------------------------
1 | from datetime import timedelta
2 | from time import time
3 | import warnings
4 |
5 | from gdbn.dbn import buildDBN
6 | from gdbn import activationFunctions
7 | import numpy as np
8 | from sklearn.base import BaseEstimator
9 | from sklearn.preprocessing import LabelEncoder
10 | from sklearn.preprocessing import OneHotEncoder
11 |
12 | warnings.warn("""\
13 | The nolearn.dbn module is no longer supported. Take a look at
14 | *nolearn.lasagne* for a more modern neural net toolkit.
15 | """)
16 |
17 |
18 | class DBN(BaseEstimator): # pragma: no cover
19 | """A scikit-learn estimator based on George Dahl's DBN
20 | implementation `gdbn`.
21 | """
22 | def __init__(
23 | self,
24 | layer_sizes=None,
25 | scales=0.05,
26 | fan_outs=None,
27 | output_act_funct=None,
28 | real_valued_vis=True,
29 | use_re_lu=True,
30 | uniforms=False,
31 |
32 | learn_rates=0.1,
33 | learn_rate_decays=1.0,
34 | learn_rate_minimums=0.0,
35 | momentum=0.9,
36 | l2_costs=0.0001,
37 | dropouts=0,
38 | nesterov=True,
39 | nest_compare=True,
40 | rms_lims=None,
41 |
42 | learn_rates_pretrain=None,
43 | momentum_pretrain=None,
44 | l2_costs_pretrain=None,
45 | nest_compare_pretrain=None,
46 |
47 | epochs=10,
48 | epochs_pretrain=0,
49 | loss_funct=None,
50 | minibatch_size=64,
51 | minibatches_per_epoch=None,
52 |
53 | pretrain_callback=None,
54 | fine_tune_callback=None,
55 |
56 | random_state=None,
57 |
58 | verbose=0,
59 | ):
60 | """
61 | Many parameters such as `learn_rates`, `dropouts` etc. will
62 | also accept a single value, in which case that value will be
63 | used for all layers. To control the value per layer, pass a
64 | list of values instead; see examples below.
65 |
66 | Parameters ending with `_pretrain` may be provided to override
67 | the given parameter for pretraining. Consider an example
68 | where you want the pre-training to use a lower learning rate
69 | than the fine tuning (the backprop), then you'd maybe pass
70 | something like::
71 |
72 | DBN([783, 300, 10], learn_rates=0.1, learn_rates_pretrain=0.005)
73 |
74 | If you don't pass the `learn_rates_pretrain` parameter, the
75 | value of `learn_rates` will be used for both pre-training and
76 | fine tuning. (Which seems to not work very well.)
77 |
78 | :param layer_sizes: A list of integers of the form
79 | ``[n_vis_units, n_hid_units1,
80 | n_hid_units2, ..., n_out_units]``.
81 |
82 | An example: ``[784, 300, 10]``
83 |
84 | The number of units in the input layer and
85 | the output layer will be set automatically
86 | if you set them to -1. Thus, the above
87 | example is equivalent to ``[-1, 300, -1]``
88 | if you pass an ``X`` with 784 features,
89 | and a ``y`` with 10 classes.
90 |
91 | :param scales: Scale of the randomly initialized weights. A
92 | list of floating point values. When you find
93 | good values for the scale of the weights you
94 | can speed up training a lot, and also improve
95 | performance. Defaults to `0.05`.
96 |
97 | :param fan_outs: Number of nonzero incoming connections to a
98 | hidden unit. Defaults to `None`, which means
99 | that all connections have non-zero weights.
100 |
101 | :param output_act_funct: Output activation function. Instance
102 | of type
103 | :class:`~gdbn.activationFunctions.Sigmoid`,
104 | :class:`~.gdbn.activationFunctions.Linear`,
105 | :class:`~.gdbn.activationFunctions.Softmax`
106 | from the
107 | :mod:`gdbn.activationFunctions`
108 | module. Defaults to
109 | :class:`~.gdbn.activationFunctions.Softmax`.
110 |
111 | :param real_valued_vis: Set `True` (the default) if visible
112 | units are real-valued.
113 |
114 | :param use_re_lu: Set `True` to use rectified linear units.
115 | Defaults to `False`.
116 |
117 | :param uniforms: Not documented at this time.
118 |
119 | :param learn_rates: A list of learning rates, one entry per
120 | weight layer.
121 |
122 | An example: ``[0.1, 0.1]``
123 |
124 | :param learn_rate_decays: The number with which the
125 | `learn_rate` is multiplied after
126 | each epoch of fine-tuning.
127 |
128 | :param learn_rate_minimums: The minimum `learn_rates`; after
129 | the learn rate reaches the minimum
130 | learn rate, the
131 | `learn_rate_decays` no longer has
132 | any effect.
133 |
134 | :param momentum: Momentum
135 |
136 | :param l2_costs: L2 costs per weight layer.
137 |
138 | :param dropouts: Dropouts per weight layer.
139 |
140 | :param nesterov: Not documented at this time.
141 |
142 | :param nest_compare: Not documented at this time.
143 |
144 | :param rms_lims: Not documented at this time.
145 |
146 | :param learn_rates_pretrain: A list of learning rates similar
147 | to `learn_rates_pretrain`, but
148 | used for pretraining. Defaults
149 | to value of `learn_rates` parameter.
150 |
151 | :param momentum_pretrain: Momentum for pre-training. Defaults
152 | to value of `momentum` parameter.
153 |
154 | :param l2_costs_pretrain: L2 costs per weight layer, for
155 | pre-training. Defaults to the value
156 | of `l2_costs` parameter.
157 |
158 | :param nest_compare_pretrain: Not documented at this time.
159 |
160 | :param epochs: Number of epochs to train (with backprop).
161 |
162 | :param epochs_pretrain: Number of epochs to pre-train (with CDN).
163 |
164 | :param loss_funct: A function that calculates the loss. Used
165 | for displaying learning progress and for
166 | :meth:`score`.
167 |
168 | :param minibatch_size: Size of a minibatch.
169 |
170 | :param minibatches_per_epoch: Number of minibatches per epoch.
171 | The default is to use as many as
172 | fit into our training set.
173 |
174 | :param pretrain_callback: An optional function that takes as
175 | arguments the :class:`DBN` instance,
176 | the epoch and the layer index as its
177 | argument, and is called for each
178 | epoch of pretraining.
179 |
180 | :param fine_tune_callback: An optional function that takes as
181 | arguments the :class:`DBN` instance
182 | and the epoch, and is called for
183 | each epoch of fine tuning.
184 |
185 | :param random_state: An optional int used as the seed by the
186 | random number generator.
187 |
188 | :param verbose: Debugging output.
189 | """
190 |
191 | if layer_sizes is None:
192 | layer_sizes = [-1, -1]
193 |
194 | if output_act_funct is None:
195 | output_act_funct = activationFunctions.Softmax()
196 | elif isinstance(output_act_funct, str):
197 | output_act_funct = getattr(activationFunctions, output_act_funct)()
198 |
199 | if random_state is not None:
200 | raise ValueError("random_sate must be an int")
201 |
202 | self.layer_sizes = layer_sizes
203 | self.scales = scales
204 | self.fan_outs = fan_outs
205 | self.output_act_funct = output_act_funct
206 | self.real_valued_vis = real_valued_vis
207 | self.use_re_lu = use_re_lu
208 | self.uniforms = uniforms
209 |
210 | self.learn_rates = learn_rates
211 | self.learn_rate_decays = learn_rate_decays
212 | self.learn_rate_minimums = learn_rate_minimums
213 | self.momentum = momentum
214 | self.l2_costs = l2_costs
215 | self.dropouts = dropouts
216 | self.nesterov = nesterov
217 | self.nest_compare = nest_compare
218 | self.rms_lims = rms_lims
219 |
220 | self.learn_rates_pretrain = learn_rates_pretrain
221 | self.momentum_pretrain = momentum_pretrain
222 | self.l2_costs_pretrain = l2_costs_pretrain
223 | self.nest_compare_pretrain = nest_compare_pretrain
224 |
225 | self.epochs = epochs
226 | self.epochs_pretrain = epochs_pretrain
227 | self.loss_funct = loss_funct
228 | self.use_dropout = True if dropouts else False
229 | self.minibatch_size = minibatch_size
230 | self.minibatches_per_epoch = minibatches_per_epoch
231 |
232 | self.pretrain_callback = pretrain_callback
233 | self.fine_tune_callback = fine_tune_callback
234 | self.random_state = random_state
235 | self.verbose = verbose
236 |
237 | def _fill_missing_layer_sizes(self, X, y):
238 | layer_sizes = self.layer_sizes
239 | if layer_sizes[0] == -1: # n_feat
240 | layer_sizes[0] = X.shape[1]
241 | if layer_sizes[-1] == -1 and y is not None: # n_classes
242 | layer_sizes[-1] = y.shape[1]
243 |
244 | def _vp(self, value):
245 | num_weights = len(self.layer_sizes) - 1
246 | if not hasattr(value, '__iter__'):
247 | value = [value] * num_weights
248 | return list(value)
249 |
250 | def _build_net(self, X, y=None):
251 | v = self._vp
252 |
253 | self._fill_missing_layer_sizes(X, y)
254 | if self.verbose: # pragma: no cover
255 | print("[DBN] layers {}".format(self.layer_sizes))
256 |
257 | if self.random_state is not None:
258 | np.random.seed(self.random_state)
259 |
260 | net = buildDBN(
261 | self.layer_sizes,
262 | v(self.scales),
263 | v(self.fan_outs),
264 | self.output_act_funct,
265 | self.real_valued_vis,
266 | self.use_re_lu,
267 | v(self.uniforms),
268 | )
269 |
270 | return net
271 |
272 | def _configure_net_pretrain(self, net):
273 | v = self._vp
274 |
275 | self._configure_net_finetune(net)
276 |
277 | learn_rates = self.learn_rates_pretrain
278 | momentum = self.momentum_pretrain
279 | l2_costs = self.l2_costs_pretrain
280 | nest_compare = self.nest_compare_pretrain
281 |
282 | if learn_rates is None:
283 | learn_rates = self.learn_rates
284 | if momentum is None:
285 | momentum = self.momentum
286 | if l2_costs is None:
287 | l2_costs = self.l2_costs
288 | if nest_compare is None:
289 | nest_compare = self.nest_compare
290 |
291 | net.learnRates = v(learn_rates)
292 | net.momentum = momentum
293 | net.L2Costs = v(l2_costs)
294 | net.nestCompare = nest_compare
295 |
296 | return net
297 |
298 | def _configure_net_finetune(self, net):
299 | v = self._vp
300 |
301 | net.learnRates = v(self.learn_rates)
302 | net.momentum = self.momentum
303 | net.L2Costs = v(self.l2_costs)
304 | net.dropouts = v(self.dropouts)
305 | net.nesterov = self.nesterov
306 | net.nestCompare = self.nest_compare
307 | net.rmsLims = v(self.rms_lims)
308 |
309 | return net
310 |
311 | def _minibatches(self, X, y=None):
312 | while True:
313 | idx = np.random.randint(X.shape[0], size=(self.minibatch_size,))
314 |
315 | X_batch = X[idx]
316 | if hasattr(X_batch, 'todense'):
317 | X_batch = X_batch.todense()
318 |
319 | if y is not None:
320 | yield (X_batch, y[idx])
321 | else:
322 | yield X_batch
323 |
324 | def _onehot(self, y):
325 | return np.array(
326 | OneHotEncoder().fit_transform(y.reshape(-1, 1)).todense())
327 |
328 | def _num_mistakes(self, targets, outputs):
329 | if hasattr(targets, 'as_numpy_array'): # pragma: no cover
330 | targets = targets.as_numpy_array()
331 | if hasattr(outputs, 'as_numpy_array'):
332 | outputs = outputs.as_numpy_array()
333 | return np.sum(outputs.argmax(1) != targets.argmax(1))
334 |
335 | def _learn_rate_adjust(self):
336 | if self.learn_rate_decays == 1.0:
337 | return
338 |
339 | learn_rate_decays = self._vp(self.learn_rate_decays)
340 | learn_rate_minimums = self._vp(self.learn_rate_minimums)
341 |
342 | for index, decay in enumerate(learn_rate_decays):
343 | new_learn_rate = self.net_.learnRates[index] * decay
344 | if new_learn_rate >= learn_rate_minimums[index]:
345 | self.net_.learnRates[index] = new_learn_rate
346 |
347 | if self.verbose >= 2:
348 | print("Learn rates: {}".format(self.net_.learnRates))
349 |
350 | def fit(self, X, y):
351 | if self.verbose:
352 | print("[DBN] fitting X.shape=%s" % (X.shape,))
353 | self._enc = LabelEncoder()
354 | y = self._enc.fit_transform(y)
355 | y = self._onehot(y)
356 |
357 | self.net_ = self._build_net(X, y)
358 |
359 | minibatches_per_epoch = self.minibatches_per_epoch
360 | if minibatches_per_epoch is None:
361 | minibatches_per_epoch = X.shape[0] / self.minibatch_size
362 |
363 | loss_funct = self.loss_funct
364 | if loss_funct is None:
365 | loss_funct = self._num_mistakes
366 |
367 | errors_pretrain = self.errors_pretrain_ = []
368 | losses_fine_tune = self.losses_fine_tune_ = []
369 | errors_fine_tune = self.errors_fine_tune_ = []
370 |
371 | if self.epochs_pretrain:
372 | self.epochs_pretrain = self._vp(self.epochs_pretrain)
373 | self._configure_net_pretrain(self.net_)
374 | for layer_index in range(len(self.layer_sizes) - 1):
375 | errors_pretrain.append([])
376 | if self.verbose: # pragma: no cover
377 | print("[DBN] Pre-train layer {}...".format(layer_index + 1))
378 | time0 = time()
379 | for epoch, err in enumerate(
380 | self.net_.preTrainIth(
381 | layer_index,
382 | self._minibatches(X),
383 | self.epochs_pretrain[layer_index],
384 | minibatches_per_epoch,
385 | )):
386 | errors_pretrain[-1].append(err)
387 | if self.verbose: # pragma: no cover
388 | print(" Epoch {}: err {}".format(epoch + 1, err))
389 | elapsed = str(timedelta(seconds=time() - time0))
390 | print(" ({})".format(elapsed.split('.')[0]))
391 | time0 = time()
392 | if self.pretrain_callback is not None:
393 | self.pretrain_callback(
394 | self, epoch + 1, layer_index)
395 |
396 | self._configure_net_finetune(self.net_)
397 | if self.verbose: # pragma: no cover
398 | print("[DBN] Fine-tune...")
399 | time0 = time()
400 | for epoch, (loss, err) in enumerate(
401 | self.net_.fineTune(
402 | self._minibatches(X, y),
403 | self.epochs,
404 | minibatches_per_epoch,
405 | loss_funct,
406 | self.verbose,
407 | self.use_dropout,
408 | )):
409 | losses_fine_tune.append(loss)
410 | errors_fine_tune.append(err)
411 | self._learn_rate_adjust()
412 | if self.verbose: # pragma: no cover
413 | print("Epoch {}:".format(epoch + 1))
414 | print(" loss {}".format(loss))
415 | print(" err {}".format(err))
416 | elapsed = str(timedelta(seconds=time() - time0))
417 | print(" ({})".format(elapsed.split('.')[0]))
418 | time0 = time()
419 | if self.fine_tune_callback is not None:
420 | self.fine_tune_callback(self, epoch + 1)
421 |
422 | def predict(self, X):
423 | y_ind = np.argmax(self.predict_proba(X), axis=1)
424 | return self._enc.inverse_transform(y_ind)
425 |
426 | def predict_proba(self, X):
427 | if hasattr(X, 'todense'):
428 | return self._predict_proba_sparse(X)
429 | res = np.zeros((X.shape[0], self.layer_sizes[-1]))
430 | for i, el in enumerate(self.net_.predictions(X, asNumpy=True)):
431 | res[i] = el
432 | return res
433 |
434 | def _predict_proba_sparse(self, X):
435 | batch_size = self.minibatch_size
436 | res = []
437 | for i in range(0, X.shape[0], batch_size):
438 | X_batch = X[i:min(i + batch_size, X.shape[0])].todense()
439 | res.extend(self.net_.predictions(X_batch))
440 | return np.array(res).reshape(X.shape[0], -1)
441 |
442 | def score(self, X, y):
443 | loss_funct = self.loss_funct
444 | if loss_funct is None:
445 | loss_funct = self._num_mistakes
446 |
447 | outputs = self.predict_proba(X)
448 | targets = self._onehot(self._enc.transform(y))
449 | mistakes = loss_funct(outputs, targets)
450 | return - float(mistakes) / len(y) + 1
451 |
452 | @property
453 | def classes_(self):
454 | return self._enc.classes_
455 |
--------------------------------------------------------------------------------
/nolearn/decaf.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 |
4 | from nolearn import cache
5 | import numpy as np
6 | from sklearn.base import BaseEstimator
7 |
8 |
9 | def _transform_cache_key(self, X):
10 | if len(X) == 1:
11 | raise cache.DontCache
12 | return ','.join([
13 | str(X),
14 | str(len(X)),
15 | str(sorted(self.get_params().items())),
16 | ])
17 |
18 |
19 | class ConvNetFeatures(BaseEstimator):
20 | """Extract features from images using a pretrained ConvNet.
21 |
22 | Based on Yangqing Jia and Jeff Donahue's `DeCAF
23 | `_.
24 | Please make sure you read and accept DeCAF's license before you
25 | use this class.
26 |
27 | If ``classify_direct=False``, expects its input X to be a list of
28 | image filenames or arrays as produced by
29 | `np.array(Image.open(filename))`.
30 | """
31 | verbose = 0
32 |
33 | def __init__(
34 | self,
35 | feature_layer='fc7_cudanet_out',
36 | pretrained_params='imagenet.decafnet.epoch90',
37 | pretrained_meta='imagenet.decafnet.meta',
38 | center_only=True,
39 | classify_direct=False,
40 | verbose=0,
41 | ):
42 | """
43 | :param feature_layer: The ConvNet layer that's used for
44 | feature extraction. Defaults to
45 | `fc7_cudanet_out`. A description of all
46 | available layers for the
47 | ImageNet-1k-pretrained ConvNet is found
48 | in the DeCAF wiki. They are:
49 |
50 | - `pool5_cudanet_out`
51 | - `fc6_cudanet_out`
52 | - `fc6_neuron_cudanet_out`
53 | - `fc7_cudanet_out`
54 | - `fc7_neuron_cudanet_out`
55 | - `probs_cudanet_out`
56 |
57 | :param pretrained_params: This must point to the file with the
58 | pretrained parameters. Defaults to
59 | `imagenet.decafnet.epoch90`. For
60 | the ImageNet-1k-pretrained ConvNet
61 | this file can be obtained from here:
62 | http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/
63 |
64 | :param pretrained_meta: Similar to `pretrained_params`, this
65 | must file to the file with the
66 | pretrained parameters' metadata.
67 | Defaults to `imagenet.decafnet.meta`.
68 |
69 | :param center_only: Use the center patch of the image only
70 | when extracting features. If `False`, use
71 | four corners, the image center and flipped
72 | variants and average a total of 10 feature
73 | vectors, which will usually yield better
74 | results. Defaults to `True`.
75 |
76 | :param classify_direct: When `True`, assume that input X is an
77 | array of shape (num x 256 x 256 x 3)
78 | as returned by `prepare_image`.
79 | """
80 | self.feature_layer = feature_layer
81 | self.pretrained_params = pretrained_params
82 | self.pretrained_meta = pretrained_meta
83 | self.center_only = center_only
84 | self.classify_direct = classify_direct
85 | self.net_ = None
86 |
87 | if (not os.path.exists(pretrained_params) or
88 | not os.path.exists(pretrained_meta)):
89 | raise ValueError(
90 | "Pre-trained ConvNet parameters not found. You may"
91 | "need to download the files from "
92 | "http://www.eecs.berkeley.edu/~jiayq/decaf_pretrained/ and "
93 | "pass the path to the two files as `pretrained_params` and "
94 | "`pretrained_meta` to the `{}` estimator.".format(
95 | self.__class__.__name__))
96 |
97 | def fit(self, X=None, y=None):
98 | from decaf.scripts.imagenet import DecafNet # soft dep
99 |
100 | if self.net_ is None:
101 | self.net_ = DecafNet(
102 | self.pretrained_params,
103 | self.pretrained_meta,
104 | )
105 | return self
106 |
107 | @cache.cached(_transform_cache_key)
108 | def transform(self, X):
109 | features = []
110 | for img in X:
111 | if self.classify_direct:
112 | images = self.net_.oversample(
113 | img, center_only=self.center_only)
114 | self.net_.classify_direct(images)
115 | else:
116 | if isinstance(img, str):
117 | import Image # soft dep
118 | img = np.array(Image.open(img))
119 | self.net_.classify(img, center_only=self.center_only)
120 | feat = None
121 | for layer in self.feature_layer.split(','):
122 | val = self.net_.feature(layer)
123 | if feat is None:
124 | feat = val
125 | else:
126 | feat = np.hstack([feat, val])
127 | if not self.center_only:
128 | feat = feat.flatten()
129 | features.append(feat)
130 | if self.verbose:
131 | sys.stdout.write(
132 | "\r[ConvNet] %d%%" % (100. * len(features) / len(X)))
133 | sys.stdout.flush()
134 | if self.verbose:
135 | sys.stdout.write('\n')
136 | return np.vstack(features)
137 |
138 | def prepare_image(self, image):
139 | """Returns image of shape `(256, 256, 3)`, as expected by
140 | `transform` when `classify_direct = True`.
141 | """
142 | from decaf.util import transform # soft dep
143 | _JEFFNET_FLIP = True
144 |
145 | # first, extract the 256x256 center.
146 | image = transform.scale_and_extract(transform.as_rgb(image), 256)
147 | # convert to [0,255] float32
148 | image = image.astype(np.float32) * 255.
149 | if _JEFFNET_FLIP:
150 | # Flip the image if necessary, maintaining the c_contiguous order
151 | image = image[::-1, :].copy()
152 | # subtract the mean
153 | image -= self.net_._data_mean
154 | return image
155 |
--------------------------------------------------------------------------------
/nolearn/grid_search.py:
--------------------------------------------------------------------------------
1 | """:func:`grid_search` is a wrapper around
2 | :class:`sklearn.grid_search.GridSearchCV`.
3 |
4 | :func:`grid_search` adds a printed report to the standard
5 | :class:`GridSearchCV` functionality, so you know about the best score
6 | and parameters.
7 |
8 | Usage example:
9 |
10 | .. doctest::
11 |
12 | >>> import numpy as np
13 | >>> class Dataset:
14 | ... def __init__(self, data, target):
15 | ... self.data, self.target = data, target
16 | ...
17 | >>> from sklearn.linear_model import LogisticRegression
18 | >>> data = np.array([[1, 2, 3], [3, 3, 3]] * 20)
19 | >>> target = np.array([0, 1] * 20)
20 | >>> dataset = Dataset(data, target)
21 |
22 | >>> model = LogisticRegression()
23 | >>> parameters = dict(C=[1.0, 3.0])
24 | >>> grid_search(dataset, model, parameters) # doctest: +ELLIPSIS
25 | parameters:
26 | {'C': [1.0, 3.0]}
27 | ...
28 | Best score: 1.0000
29 | Best grid parameters:
30 | C=1.0,
31 | ...
32 | """
33 |
34 | from __future__ import print_function
35 |
36 | from pprint import pprint
37 | import warnings
38 |
39 | from sklearn.base import BaseEstimator
40 | from sklearn.grid_search import GridSearchCV
41 |
42 | warnings.warn("""\
43 | The nolearn.grid_search module will be removed in nolearn 0.6. If you want to
44 | continue to use this module, please consider copying the code into
45 | your own project.
46 | """)
47 |
48 |
49 | def print_report(grid_search, parameters):
50 | print()
51 | print("== " * 20)
52 | print("All parameters:")
53 | best_parameters = grid_search.best_estimator_.get_params()
54 | for param_name, value in sorted(best_parameters.items()):
55 | if not isinstance(value, BaseEstimator):
56 | print(" %s=%r," % (param_name, value))
57 |
58 | print()
59 | print("== " * 20)
60 | print("Best score: %0.4f" % grid_search.best_score_)
61 | print("Best grid parameters:")
62 | for param_name in sorted(parameters.keys()):
63 | print(" %s=%r," % (param_name, best_parameters[param_name]))
64 | print("== " * 20)
65 |
66 | return grid_search
67 |
68 |
69 | def grid_search(dataset, clf, parameters, cv=None, verbose=4, n_jobs=1,
70 | **kwargs):
71 | # See http://scikit-learn.org/stable/modules/grid_search.html
72 |
73 | grid_search = GridSearchCV(
74 | clf,
75 | parameters,
76 | cv=cv,
77 | verbose=verbose,
78 | n_jobs=n_jobs,
79 | **kwargs
80 | )
81 |
82 | if verbose:
83 | print("parameters:")
84 | pprint(parameters)
85 |
86 | grid_search.fit(dataset.data, dataset.target)
87 |
88 | if verbose:
89 | print_report(grid_search, parameters)
90 |
91 | return grid_search
92 |
--------------------------------------------------------------------------------
/nolearn/inischema.py:
--------------------------------------------------------------------------------
1 | """:mod:`inischema` allows the definition of schemas for `.ini`
2 | configuration files.
3 |
4 | Consider this sample schema:
5 |
6 | .. doctest::
7 |
8 | >>> schema = '''
9 | ... [first]
10 | ... value1 = int
11 | ... value2 = string
12 | ... value3 = float
13 | ... value4 = listofstrings
14 | ... value5 = listofints
15 | ...
16 | ... [second]
17 | ... value1 = string
18 | ... value2 = int
19 | ... '''
20 |
21 | This schema defines the sections, names and types of values expected
22 | in a schema file.
23 |
24 | Using a concrete configuration, we can then use the schema to extract
25 | values:
26 |
27 | .. doctest::
28 |
29 | >>> config = '''
30 | ... [first]
31 | ... value1 = 2
32 | ... value2 = three
33 | ... value3 = 4.4
34 | ... value4 = five six seven
35 | ... value5 = 8 9
36 | ...
37 | ... [second]
38 | ... value1 = ten
39 | ... value2 = 100
40 | ... value3 = what?
41 | ... '''
42 |
43 | >>> result = parse_config(schema, config)
44 | >>> from pprint import pprint
45 | >>> pprint(result)
46 | {'first': {'value1': 2,
47 | 'value2': 'three',
48 | 'value3': 4.4,
49 | 'value4': ['five', 'six', 'seven'],
50 | 'value5': [8, 9]},
51 | 'second': {'value1': 'ten', 'value2': 100, 'value3': 'what?'}}
52 |
53 | Values in the config file that are not in the schema are assumed to be
54 | strings.
55 |
56 | This module is used in :mod:`nolearn.console` to allow for convenient
57 | passing of values from `.ini` files as function arguments for command
58 | line scripts.
59 | """
60 |
61 | from ._compat import ConfigParser
62 | from ._compat import StringIO
63 |
64 |
65 | def string(value):
66 | return value.strip()
67 |
68 |
69 | def listofstrings(value):
70 | return [string(v) for v in value.split()]
71 |
72 |
73 | def listofints(value):
74 | return [int(v) for v in value.split()]
75 |
76 |
77 | converters = {
78 | 'int': int,
79 | 'string': string,
80 | 'float': float,
81 | 'listofstrings': listofstrings,
82 | 'listofints': listofints,
83 | }
84 |
85 |
86 | def parse_config(schema, config):
87 | schemaparser = ConfigParser()
88 | schemaparser.readfp(StringIO(schema))
89 | cfgparser = ConfigParser()
90 | cfgparser.readfp(StringIO(config))
91 |
92 | result = {}
93 | for section in cfgparser.sections():
94 | result_section = {}
95 | schema = {}
96 | if section in schemaparser.sections():
97 | schema = dict(schemaparser.items(section))
98 | for key, value in cfgparser.items(section):
99 | converter = converters[schema.get(key, 'string')]
100 | result_section[key] = converter(value)
101 | result[section] = result_section
102 | return result
103 |
--------------------------------------------------------------------------------
/nolearn/lasagne/__init__.py:
--------------------------------------------------------------------------------
1 | from .handlers import (
2 | PrintLayerInfo,
3 | PrintLog,
4 | RememberBestWeights,
5 | SaveWeights,
6 | WeightLog,
7 | )
8 | from .base import (
9 | BatchIterator,
10 | grad_scale,
11 | objective,
12 | NeuralNet,
13 | TrainSplit,
14 | )
15 |
--------------------------------------------------------------------------------
/nolearn/lasagne/handlers.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 | from csv import DictWriter
3 | from datetime import datetime
4 | from functools import reduce
5 | import operator
6 | import sys
7 |
8 | import numpy
9 | from tabulate import tabulate
10 |
11 | from .._compat import pickle
12 | from .util import ansi
13 | from .util import get_conv_infos
14 | from .util import is_conv2d
15 |
16 |
17 | class PrintLog:
18 | def __init__(self):
19 | self.first_iteration = True
20 |
21 | def __call__(self, nn, train_history):
22 | print(self.table(nn, train_history))
23 | sys.stdout.flush()
24 |
25 | def table(self, nn, train_history):
26 | info = train_history[-1]
27 |
28 | info_tabulate = OrderedDict([
29 | ('epoch', info['epoch']),
30 | ('trn loss', "{}{:.5f}{}".format(
31 | ansi.CYAN if info['train_loss_best'] else "",
32 | info['train_loss'],
33 | ansi.ENDC if info['train_loss_best'] else "",
34 | )),
35 | ('val loss', "{}{:.5f}{}".format(
36 | ansi.GREEN if info['valid_loss_best'] else "",
37 | info['valid_loss'],
38 | ansi.ENDC if info['valid_loss_best'] else "",
39 | )),
40 | ('trn/val', info['train_loss'] / info['valid_loss']),
41 | ])
42 |
43 | if not nn.regression:
44 | info_tabulate['valid acc'] = info['valid_accuracy']
45 |
46 | for name, func in nn.scores_train:
47 | info_tabulate[name] = info[name]
48 |
49 | for name, func in nn.scores_valid:
50 | info_tabulate[name] = info[name]
51 |
52 | if nn.custom_scores:
53 | for custom_score in nn.custom_scores:
54 | info_tabulate[custom_score[0]] = info[custom_score[0]]
55 |
56 | info_tabulate['dur'] = "{:.2f}s".format(info['dur'])
57 |
58 | tabulated = tabulate(
59 | [info_tabulate], headers="keys", floatfmt='.5f')
60 |
61 | out = ""
62 | if self.first_iteration:
63 | out = "\n".join(tabulated.split('\n', 2)[:2])
64 | out += "\n"
65 | self.first_iteration = False
66 |
67 | out += tabulated.rsplit('\n', 1)[-1]
68 | return out
69 |
70 |
71 | class SaveWeights:
72 | def __init__(self, path, every_n_epochs=1, only_best=False,
73 | pickle=False, verbose=0):
74 | self.path = path
75 | self.every_n_epochs = every_n_epochs
76 | self.only_best = only_best
77 | self.pickle = pickle
78 | self.verbose = verbose
79 |
80 | def __call__(self, nn, train_history):
81 | if self.only_best:
82 | this_loss = train_history[-1]['valid_loss']
83 | best_loss = min([h['valid_loss'] for h in train_history])
84 | if this_loss > best_loss:
85 | return
86 |
87 | if train_history[-1]['epoch'] % self.every_n_epochs != 0:
88 | return
89 |
90 | format_args = {
91 | 'loss': train_history[-1]['valid_loss'],
92 | 'timestamp': datetime.now().strftime('%Y-%m-%d-%H-%M-%S'),
93 | 'epoch': '{:04d}'.format(train_history[-1]['epoch']),
94 | }
95 | path = self.path.format(**format_args)
96 |
97 | if self.verbose:
98 | print("Writing {}".format(path))
99 |
100 | if self.pickle:
101 | with open(path, 'wb') as f:
102 | pickle.dump(nn, f, -1)
103 | else:
104 | nn.save_params_to(path)
105 |
106 |
107 | class RestoreBestWeights:
108 | def __init__(self, remember=None):
109 | self.remember = remember
110 |
111 | def __call__(self, nn, train_history):
112 | if self.remember is None:
113 | [self.remember] = [handler for handler in nn.on_epoch_finished
114 | if isinstance(handler, RememberBestWeights)]
115 | nn.load_params_from(self.remember.best_weights)
116 | if self.remember.verbose:
117 | print("Loaded best weights from epoch {} where {} was {}".format(
118 | self.remember.best_weights_epoch,
119 | self.remember.score or self.remember.loss,
120 | self.remember.best_weights_loss * (
121 | -1 if self.remember.score else 1),
122 | ))
123 |
124 |
125 | _RestoreBestWeights = RestoreBestWeights
126 |
127 |
128 | class RememberBestWeights:
129 | def __init__(self, loss='valid_loss', score=None, verbose=1):
130 | self.loss = loss
131 | self.score = score
132 | self.verbose = verbose
133 | self.best_weights = None
134 | self.best_weights_loss = sys.maxsize
135 | self.best_weights_epoch = None
136 |
137 | def __call__(self, nn, train_history):
138 | key = self.score if self.score is not None else self.loss
139 |
140 | curr_loss = train_history[-1][key]
141 | if self.score:
142 | curr_loss *= -1
143 |
144 | if curr_loss < self.best_weights_loss:
145 | self.best_weights = nn.get_all_params_values()
146 | self.best_weights_loss = curr_loss
147 | self.best_weights_epoch = train_history[-1]['epoch']
148 |
149 |
150 | class PrintLayerInfo:
151 | def __init__(self):
152 | pass
153 |
154 | def __call__(self, nn, train_history=None):
155 | if train_history:
156 | return
157 |
158 | message = self._get_greeting(nn)
159 | print(message)
160 | print("## Layer information")
161 | print("")
162 |
163 | layers_contain_conv2d = is_conv2d(nn.layers_.values())
164 | if not layers_contain_conv2d or (nn.verbose < 2):
165 | layer_info = self._get_layer_info_plain(nn)
166 | legend = None
167 | else:
168 | layer_info, legend = self._get_layer_info_conv(nn)
169 | print(layer_info)
170 | if legend is not None:
171 | print(legend)
172 | print("")
173 | sys.stdout.flush()
174 |
175 | @staticmethod
176 | def _get_greeting(nn):
177 | shapes = [param.get_value().shape for param in
178 | nn.get_all_params(trainable=True) if param]
179 | nparams = reduce(operator.add, [reduce(operator.mul, shape) for
180 | shape in shapes])
181 | message = ("# Neural Network with {} learnable parameters"
182 | "\n".format(nparams))
183 | return message
184 |
185 | @staticmethod
186 | def _get_layer_info_plain(nn):
187 | nums = list(range(len(nn.layers_)))
188 | names = [layer.name for layer in nn.layers_.values()]
189 | output_shapes = ['x'.join(map(str, layer.output_shape[1:]))
190 | for layer in nn.layers_.values()]
191 |
192 | table = OrderedDict([
193 | ('#', nums),
194 | ('name', names),
195 | ('size', output_shapes),
196 | ])
197 | layer_infos = tabulate(table, 'keys')
198 | return layer_infos
199 |
200 | @staticmethod
201 | def _get_layer_info_conv(nn):
202 | if nn.verbose > 2:
203 | detailed = True
204 | else:
205 | detailed = False
206 |
207 | layer_infos = get_conv_infos(nn, detailed=detailed)
208 |
209 | mag = "{}{}{}".format(ansi.MAGENTA, "magenta", ansi.ENDC)
210 | cya = "{}{}{}".format(ansi.CYAN, "cyan", ansi.ENDC)
211 | red = "{}{}{}".format(ansi.RED, "red", ansi.ENDC)
212 | legend = (
213 | "\nExplanation"
214 | "\n X, Y: image dimensions"
215 | "\n cap.: learning capacity"
216 | "\n cov.: coverage of image"
217 | "\n {}: capacity too low (<1/6)"
218 | "\n {}: image coverage too high (>100%)"
219 | "\n {}: capacity too low and coverage too high\n"
220 | "".format(mag, cya, red)
221 | )
222 |
223 | return layer_infos, legend
224 |
225 |
226 | class WeightLog:
227 | """Keep a log of your network's weights and weight changes.
228 |
229 | Pass instances of :class:`WeightLog` as an `on_batch_finished`
230 | handler into your network.
231 | """
232 | def __init__(self, save_to=None, write_every=8):
233 | """
234 | :param save_to: If given, `save_to` must be a path into which
235 | I will write weight statistics in CSV format.
236 | """
237 | self.last_weights = None
238 | self.history = []
239 | self.save_to = save_to
240 | self.write_every = write_every
241 | self._dictwriter = None
242 | self._save_to_file = None
243 |
244 | def __call__(self, nn, train_history):
245 | weights = nn.get_all_params_values()
246 |
247 | if self.save_to and self._dictwriter is None:
248 | fieldnames = []
249 | for key in weights.keys():
250 | for i, p in enumerate(weights[key]):
251 | fieldnames.extend([
252 | '{}_{} wdiff'.format(key, i),
253 | '{}_{} wabsmean'.format(key, i),
254 | '{}_{} wmean'.format(key, i),
255 | ])
256 |
257 | newfile = self.last_weights is None
258 | if newfile:
259 | self._save_to_file = open(self.save_to, 'w')
260 | else:
261 | self._save_to_file = open(self.save_to, 'a')
262 | self._dictwriter = DictWriter(self._save_to_file, fieldnames)
263 | if newfile:
264 | self._dictwriter.writeheader()
265 |
266 | entry = {}
267 | lw = self.last_weights if self.last_weights is not None else weights
268 | for key in weights.keys():
269 | for i, (p1, p2) in enumerate(zip(lw[key], weights[key])):
270 | wdiff = numpy.abs(p1 - p2).mean()
271 | wabsmean = numpy.abs(p2).mean()
272 | wmean = p2.mean()
273 | entry.update({
274 | '{}_{} wdiff'.format(key, i): wdiff,
275 | '{}_{} wabsmean'.format(key, i): wabsmean,
276 | '{}_{} wmean'.format(key, i): wmean,
277 | })
278 | self.history.append(entry)
279 |
280 | if self.save_to:
281 | if len(self.history) % self.write_every == 0:
282 | self._dictwriter.writerows(self.history[-self.write_every:])
283 | self._save_to_file.flush()
284 |
285 | self.last_weights = weights
286 |
287 | def __getstate__(self):
288 | state = dict(self.__dict__)
289 | state['_save_to_file'] = None
290 | state['_dictwriter'] = None
291 | return state
292 |
--------------------------------------------------------------------------------
/nolearn/lasagne/tests/conftest.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pytest
3 | from sklearn.datasets import load_boston
4 | from sklearn.datasets import fetch_mldata
5 | from sklearn.preprocessing import StandardScaler
6 | from sklearn.utils import shuffle
7 |
8 | from lasagne.layers import Conv2DLayer
9 | from lasagne.layers import DenseLayer
10 | from lasagne.layers import InputLayer
11 | from lasagne.layers import MaxPool2DLayer
12 | from lasagne.layers import NonlinearityLayer
13 | from lasagne.nonlinearities import softmax
14 | from lasagne.updates import nesterov_momentum
15 |
16 |
17 | @pytest.fixture(scope='session')
18 | def NeuralNet():
19 | from nolearn.lasagne import NeuralNet
20 | return NeuralNet
21 |
22 |
23 | @pytest.fixture
24 | def nn(NeuralNet):
25 | return NeuralNet([('input', object())], input_shape=(10, 10))
26 |
27 |
28 | @pytest.fixture(scope='session')
29 | def mnist():
30 | dataset = fetch_mldata('mnist-original')
31 | X, y = dataset.data, dataset.target
32 | X = X.astype(np.float32) / 255.0
33 | y = y.astype(np.int32)
34 | return shuffle(X, y, random_state=42)
35 |
36 |
37 | @pytest.fixture(scope='session')
38 | def boston():
39 | dataset = load_boston()
40 | X, y = dataset.data, dataset.target
41 | # X, y = make_regression(n_samples=100000, n_features=13)
42 | X = StandardScaler().fit_transform(X).astype(np.float32)
43 | y = y.reshape(-1, 1).astype(np.float32)
44 | return shuffle(X, y, random_state=42)
45 |
46 |
47 | class _OnEpochFinished:
48 | def __call__(self, nn, train_history):
49 | self.train_history = train_history
50 | if len(train_history) > 1:
51 | raise StopIteration()
52 |
53 |
54 | @pytest.fixture(scope='session')
55 | def X_train(mnist):
56 | X, y = mnist
57 | return X[:10000].reshape(-1, 1, 28, 28)
58 |
59 |
60 | @pytest.fixture(scope='session')
61 | def y_train(mnist):
62 | X, y = mnist
63 | return y[:10000]
64 |
65 |
66 | @pytest.fixture(scope='session')
67 | def X_test(mnist):
68 | X, y = mnist
69 | return X[60000:].reshape(-1, 1, 28, 28)
70 |
71 |
72 | @pytest.fixture(scope='session')
73 | def y_pred(net_fitted, X_test):
74 | return net_fitted.predict(X_test)
75 |
76 |
77 | @pytest.fixture(scope='session')
78 | def net(NeuralNet):
79 | l = InputLayer(shape=(None, 1, 28, 28))
80 | l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8)
81 | l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2))
82 | l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8)
83 | l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2))
84 | l = DenseLayer(l, name='hidden1', num_units=128)
85 | l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10)
86 |
87 | return NeuralNet(
88 | layers=l,
89 |
90 | update=nesterov_momentum,
91 | update_learning_rate=0.01,
92 | update_momentum=0.9,
93 |
94 | max_epochs=5,
95 | on_epoch_finished=[_OnEpochFinished()],
96 | verbose=99,
97 | )
98 |
99 |
100 | @pytest.fixture(scope='session')
101 | def net_fitted(net, X_train, y_train):
102 | return net.fit(X_train, y_train)
103 |
104 |
105 | @pytest.fixture(scope='session')
106 | def net_color_non_square(NeuralNet):
107 | l = InputLayer(shape=(None, 3, 20, 28))
108 | l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=1)
109 | l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2))
110 | l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8)
111 | l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2))
112 | l = DenseLayer(l, name='hidden1', num_units=128)
113 | l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10)
114 |
115 | net = NeuralNet(
116 | layers=l,
117 |
118 | update=nesterov_momentum,
119 | update_learning_rate=0.01,
120 | update_momentum=0.9,
121 |
122 | max_epochs=1,
123 | )
124 | net.initialize()
125 | return net
126 |
127 |
128 | @pytest.fixture(scope='session')
129 | def net_with_nonlinearity_layer(NeuralNet):
130 | l = InputLayer(shape=(None, 1, 28, 28))
131 | l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8)
132 | l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2))
133 | l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8)
134 | l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2))
135 | l = DenseLayer(l, name='hidden1', num_units=128)
136 | l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10)
137 | l = NonlinearityLayer(l)
138 |
139 | net = NeuralNet(
140 | layers=l,
141 |
142 | update=nesterov_momentum,
143 | update_learning_rate=0.01,
144 | update_momentum=0.9,
145 |
146 | max_epochs=5,
147 | on_epoch_finished=[_OnEpochFinished()],
148 | verbose=99,
149 | )
150 | net.initialize()
151 | return net
152 |
153 |
154 | @pytest.fixture
155 | def net_no_conv(NeuralNet):
156 | l = InputLayer(shape=(None, 100))
157 | l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10)
158 |
159 | return NeuralNet(
160 | layers=l,
161 |
162 | update=nesterov_momentum,
163 | update_learning_rate=0.01,
164 | update_momentum=0.9,
165 |
166 | max_epochs=1,
167 | verbose=99,
168 | )
169 |
--------------------------------------------------------------------------------
/nolearn/lasagne/tests/test_base.py:
--------------------------------------------------------------------------------
1 | import pickle
2 | import sys
3 |
4 | from lasagne.layers import get_output
5 | from lasagne.layers import BatchNormLayer
6 | from lasagne.layers import ConcatLayer
7 | from lasagne.layers import Conv2DLayer
8 | from lasagne.layers import DenseLayer
9 | from lasagne.layers import InputLayer
10 | from lasagne.layers import RecurrentLayer
11 | from lasagne.layers import Layer
12 | from lasagne.nonlinearities import identity
13 | from lasagne.nonlinearities import softmax
14 | from lasagne.nonlinearities import sigmoid
15 | from lasagne.objectives import categorical_crossentropy
16 | from lasagne.objectives import aggregate
17 | from lasagne.updates import nesterov_momentum
18 | from mock import Mock
19 | from mock import patch
20 | import numpy as np
21 | import pytest
22 | from sklearn.base import clone
23 | from sklearn.datasets import make_classification
24 | from sklearn.datasets import make_regression
25 | from sklearn.grid_search import GridSearchCV
26 | from sklearn.metrics import accuracy_score
27 | from sklearn.metrics import mean_absolute_error
28 | from sklearn.metrics import r2_score
29 | import theano
30 | import theano.tensor as T
31 |
32 | floatX = theano.config.floatX
33 |
34 |
35 | class TestLayers:
36 | @pytest.fixture
37 | def layers(self):
38 | from nolearn.lasagne.base import Layers
39 | return Layers([('one', 1), ('two', 2), ('three', 3)])
40 |
41 | def test_getitem_with_key(self, layers):
42 | assert layers['one'] == 1
43 |
44 | def test_getitem_with_index(self, layers):
45 | assert layers[0] == 1
46 |
47 | def test_getitem_with_slice(self, layers):
48 | from nolearn.lasagne.base import Layers
49 | sliced = layers[:2]
50 | assert isinstance(sliced, Layers)
51 | assert sliced.keys() == ['one', 'two']
52 | assert sliced.values() == [1, 2]
53 |
54 | def test_keys_returns_list(self, layers):
55 | assert layers.keys() == ['one', 'two', 'three']
56 |
57 | def test_values_returns_list(self, layers):
58 | assert layers.values() == [1, 2, 3]
59 |
60 |
61 | class TestFunctionalToy:
62 | def classif(self, NeuralNet, X, y):
63 | l = InputLayer(shape=(None, X.shape[1]))
64 | l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
65 | net = NeuralNet(l, update_learning_rate=0.01)
66 | return net.fit(X, y)
67 |
68 | def classif_no_valid(self, NeuralNet, X, y):
69 | from nolearn.lasagne import TrainSplit
70 | l = InputLayer(shape=(None, X.shape[1]))
71 | l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
72 | net = NeuralNet(
73 | l, update_learning_rate=0.01, train_split=TrainSplit(0))
74 | return net.fit(X, y)
75 |
76 | def regr(self, NeuralNet, X, y):
77 | l = InputLayer(shape=(None, X.shape[1]))
78 | l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None)
79 | net = NeuralNet(l, regression=True, update_learning_rate=0.01)
80 | return net.fit(X, y)
81 |
82 | def test_classif_two_classes(self, NeuralNet):
83 | X, y = make_classification()
84 | X = X.astype(floatX)
85 | y = y.astype(np.int32)
86 | self.classif(NeuralNet, X, y)
87 |
88 | def test_classif_ten_classes(self, NeuralNet):
89 | X, y = make_classification(n_classes=10, n_informative=10)
90 | X = X.astype(floatX)
91 | y = y.astype(np.int32)
92 | self.classif(NeuralNet, X, y)
93 |
94 | def test_classif_no_valid_two_classes(self, NeuralNet):
95 | X, y = make_classification()
96 | X = X.astype(floatX)
97 | y = y.astype(np.int32)
98 | self.classif_no_valid(NeuralNet, X, y)
99 |
100 | def test_regr_one_target(self, NeuralNet):
101 | X, y = make_regression()
102 | X = X.astype(floatX)
103 | y = y.reshape(-1, 1).astype(np.float32)
104 | self.regr(NeuralNet, X, y)
105 |
106 | def test_regr_ten_targets(self, NeuralNet):
107 | X, y = make_regression(n_targets=10)
108 | X = X.astype(floatX)
109 | y = y.astype(floatX)
110 | self.regr(NeuralNet, X, y)
111 |
112 |
113 | class TestFunctionalMNIST:
114 | def test_accuracy(self, net_fitted, mnist, X_test, y_pred):
115 | X, y = mnist
116 | y_test = y[60000:]
117 | acc = accuracy_score(y_pred, y_test)
118 | assert acc > 0.85
119 | assert net_fitted.score(X_test, y_test) == acc
120 |
121 | def test_train_history(self, net_fitted):
122 | history = net_fitted.train_history_
123 | assert len(history) == 2 # due to early stopping
124 | assert history[1]['valid_accuracy'] > 0.85
125 | assert history[1]['valid_accuracy'] > history[0]['valid_accuracy']
126 | assert set(history[0].keys()) == set([
127 | 'dur', 'epoch', 'train_loss', 'train_loss_best',
128 | 'valid_loss', 'valid_loss_best', 'valid_accuracy',
129 | ])
130 |
131 | def test_early_stopping(self, net_fitted):
132 | early_stopping = net_fitted.on_epoch_finished[0]
133 | assert early_stopping.train_history == net_fitted.train_history_
134 |
135 | def test_pickle(self, net_fitted, X_test, y_pred):
136 | recursionlimit = sys.getrecursionlimit()
137 | sys.setrecursionlimit(10000)
138 | pickled = pickle.dumps(net_fitted, -1)
139 | net_loaded = pickle.loads(pickled)
140 | assert np.array_equal(net_loaded.predict(X_test), y_pred)
141 | sys.setrecursionlimit(recursionlimit)
142 |
143 | def test_load_params_from_net(self, net, net_fitted, X_test, y_pred):
144 | net_loaded = clone(net)
145 | net_loaded.load_params_from(net_fitted)
146 | assert np.array_equal(net_loaded.predict(X_test), y_pred)
147 |
148 | def test_load_params_from_params_values(self, net, net_fitted,
149 | X_test, y_pred):
150 | net_loaded = clone(net)
151 | net_loaded.load_params_from(net_fitted.get_all_params_values())
152 | assert np.array_equal(net_loaded.predict(X_test), y_pred)
153 |
154 | def test_save_params_to_path(self, net_fitted, X_test, y_pred):
155 | path = '/tmp/test_lasagne_functional_mnist.params'
156 | net_fitted.save_params_to(path)
157 | net_loaded = clone(net_fitted)
158 | net_loaded.load_params_from(path)
159 | assert np.array_equal(net_loaded.predict(X_test), y_pred)
160 |
161 | def test_load_params_from_message(self, net, net_fitted, capsys):
162 | net2 = clone(net)
163 | net2.verbose = 1
164 | net2.load_params_from(net_fitted)
165 |
166 | out = capsys.readouterr()[0]
167 | message = """\
168 | Loaded parameters to layer 'conv1' (shape 8x1x5x5).
169 | Loaded parameters to layer 'conv1' (shape 8).
170 | Loaded parameters to layer 'conv2' (shape 8x8x5x5).
171 | Loaded parameters to layer 'conv2' (shape 8).
172 | Loaded parameters to layer 'hidden1' (shape 128x128).
173 | Loaded parameters to layer 'hidden1' (shape 128).
174 | Loaded parameters to layer 'output' (shape 128x10).
175 | Loaded parameters to layer 'output' (shape 10).
176 | """
177 | assert out == message
178 |
179 | def test_partial_fit(self, net, X_train, y_train):
180 | net2 = clone(net)
181 | assert net2.partial_fit(X_train, y_train) is net2
182 | net2.partial_fit(X_train, y_train)
183 | history = net2.train_history_
184 | assert len(history) == 2
185 | assert history[1]['valid_accuracy'] > 0.85
186 |
187 |
188 | def test_lasagne_functional_grid_search(mnist, monkeypatch):
189 | # Make sure that we can satisfy the grid search interface.
190 | from nolearn.lasagne import NeuralNet
191 |
192 | nn = NeuralNet(
193 | layers=[],
194 | )
195 |
196 | param_grid = {
197 | 'more_params': [{'hidden_num_units': 100}, {'hidden_num_units': 200}],
198 | 'update_momentum': [0.9, 0.98],
199 | }
200 | X, y = mnist
201 |
202 | vars_hist = []
203 |
204 | def fit(self, X, y):
205 | vars_hist.append(vars(self).copy())
206 | return self
207 |
208 | with patch.object(NeuralNet, 'fit', autospec=True) as mock_fit:
209 | mock_fit.side_effect = fit
210 | with patch('nolearn.lasagne.NeuralNet.score') as score:
211 | score.return_value = 0.3
212 | gs = GridSearchCV(nn, param_grid, cv=2, refit=False, verbose=4)
213 | gs.fit(X, y)
214 |
215 | assert [entry['update_momentum'] for entry in vars_hist] == [
216 | 0.9, 0.9, 0.98, 0.98] * 2
217 | assert [entry['more_params'] for entry in vars_hist] == (
218 | [{'hidden_num_units': 100}] * 4 +
219 | [{'hidden_num_units': 200}] * 4
220 | )
221 |
222 |
223 | def test_clone():
224 | from nolearn.lasagne import NeuralNet
225 | from nolearn.lasagne import BatchIterator
226 | from nolearn.lasagne import objective
227 |
228 | params = dict(
229 | layers=[
230 | ('input', InputLayer),
231 | ('hidden', DenseLayer),
232 | ('output', DenseLayer),
233 | ],
234 | input_shape=(100, 784),
235 | output_num_units=10,
236 | output_nonlinearity=softmax,
237 |
238 | more_params={
239 | 'hidden_num_units': 100,
240 | },
241 | update=nesterov_momentum,
242 | update_learning_rate=0.01,
243 | update_momentum=0.9,
244 |
245 | regression=False,
246 | objective=objective,
247 | objective_loss_function=categorical_crossentropy,
248 | batch_iterator_train=BatchIterator(batch_size=100),
249 | y_tensor_type=T.ivector,
250 | use_label_encoder=False,
251 | on_epoch_finished=None,
252 | on_training_finished=None,
253 | max_epochs=100,
254 | eval_size=0.1, # BBB
255 | check_input=True,
256 | verbose=0,
257 | )
258 | nn = NeuralNet(**params)
259 |
260 | nn2 = clone(nn)
261 | params1 = nn.get_params()
262 | params2 = nn2.get_params()
263 |
264 | for ignore in (
265 | 'batch_iterator_train',
266 | 'batch_iterator_test',
267 | 'output_nonlinearity',
268 | 'loss',
269 | 'objective',
270 | 'train_split',
271 | 'eval_size',
272 | 'X_tensor_type',
273 | 'on_epoch_finished',
274 | 'on_batch_finished',
275 | 'on_training_started',
276 | 'on_training_finished',
277 | 'custom_scores',
278 | 'scores_train',
279 | 'scores_valid',
280 | ):
281 | for par in (params, params1, params2):
282 | par.pop(ignore, None)
283 |
284 | assert params == params1 == params2
285 |
286 |
287 | def test_lasagne_functional_regression(boston):
288 | from nolearn.lasagne import NeuralNet
289 |
290 | X, y = boston
291 |
292 | layer1 = InputLayer(shape=(128, 13))
293 | layer2 = DenseLayer(layer1, num_units=100)
294 | output = DenseLayer(layer2, num_units=1, nonlinearity=identity)
295 |
296 | nn = NeuralNet(
297 | layers=output,
298 | update_learning_rate=0.01,
299 | update_momentum=0.1,
300 | regression=True,
301 | max_epochs=50,
302 | )
303 |
304 | nn.fit(X[:300], y[:300])
305 | assert mean_absolute_error(nn.predict(X[300:]), y[300:]) < 3.0
306 | assert r2_score(nn.predict(X[300:]), y[300:]) == nn.score(X[300:], y[300:])
307 |
308 |
309 | class TestDefaultObjective:
310 | @pytest.fixture
311 | def get_output(self, monkeypatch):
312 | from nolearn.lasagne import base
313 | get_output_mock = Mock()
314 | monkeypatch.setattr(base, 'get_output', get_output_mock)
315 | return get_output_mock
316 |
317 | @pytest.fixture
318 | def objective(self):
319 | from nolearn.lasagne.base import objective
320 | return objective
321 |
322 | def test_with_defaults(self, objective, get_output):
323 | loss_function, target = Mock(), Mock()
324 | loss_function.return_value = np.array([1, 2, 3])
325 | result = objective(
326 | [1, 2, 3], loss_function=loss_function, target=target)
327 | assert result == 2.0
328 | get_output.assert_called_with(3, deterministic=False)
329 | loss_function.assert_called_with(get_output.return_value, target)
330 |
331 | def test_with_get_output_kw(self, objective, get_output):
332 | loss_function, target = Mock(), Mock()
333 | loss_function.return_value = np.array([1, 2, 3])
334 | objective(
335 | [1, 2, 3], loss_function=loss_function, target=target,
336 | get_output_kw={'i_was': 'here'},
337 | )
338 | get_output.assert_called_with(3, deterministic=False, i_was='here')
339 |
340 |
341 | class TestTrainSplit:
342 | @pytest.fixture
343 | def TrainSplit(self):
344 | from nolearn.lasagne import TrainSplit
345 | return TrainSplit
346 |
347 | def test_reproducable(self, TrainSplit, nn):
348 | X, y = np.random.random((100, 10)), np.repeat([0, 1, 2, 3], 25)
349 | X_train1, X_valid1, y_train1, y_valid1 = TrainSplit(0.2)(
350 | X, y, nn)
351 | X_train2, X_valid2, y_train2, y_valid2 = TrainSplit(0.2)(
352 | X, y, nn)
353 | assert np.all(X_train1 == X_train2)
354 | assert np.all(y_valid1 == y_valid2)
355 |
356 | def test_eval_size_zero(self, TrainSplit, nn):
357 | X, y = np.random.random((100, 10)), np.repeat([0, 1, 2, 3], 25)
358 | X_train, X_valid, y_train, y_valid = TrainSplit(0.0)(
359 | X, y, nn)
360 | assert len(X_train) == len(X)
361 | assert len(y_train) == len(y)
362 | assert len(X_valid) == 0
363 | assert len(y_valid) == 0
364 |
365 | def test_eval_size_half(self, TrainSplit, nn):
366 | X, y = np.random.random((100, 10)), np.repeat([0, 1, 2, 3], 25)
367 | X_train, X_valid, y_train, y_valid = TrainSplit(0.51)(
368 | X, y, nn)
369 | assert len(X_train) + len(X_valid) == 100
370 | assert len(y_train) + len(y_valid) == 100
371 | assert len(X_train) > 45
372 |
373 | def test_regression(self, TrainSplit, nn):
374 | X = np.random.random((100, 10))
375 | y = np.random.random((100))
376 | nn.regression = True
377 | X_train, X_valid, y_train, y_valid = TrainSplit(0.2)(
378 | X, y, nn)
379 | assert len(X_train) == len(y_train) == 80
380 | assert len(X_valid) == len(y_valid) == 20
381 |
382 | def test_stratified(self, TrainSplit, nn):
383 | X = np.random.random((100, 10))
384 | y = np.hstack([np.repeat([0, 0, 0], 25), np.repeat([1], 25)])
385 | X_train, X_valid, y_train, y_valid = TrainSplit(0.2)(
386 | X, y, nn)
387 | assert y_train.sum() == 0.8 * 25
388 | assert y_valid.sum() == 0.2 * 25
389 |
390 | def test_not_stratified(self, TrainSplit, nn):
391 | X = np.random.random((100, 10))
392 | y = np.hstack([np.repeat([0, 0, 0], 25), np.repeat([1], 25)])
393 | X_train, X_valid, y_train, y_valid = TrainSplit(0.2, stratify=False)(
394 | X, y, nn)
395 | assert y_train.sum() == 25
396 | assert y_valid.sum() == 0
397 |
398 | def test_X_is_dict(self, TrainSplit, nn):
399 | X = {
400 | '1': np.random.random((100, 10)),
401 | '2': np.random.random((100, 10)),
402 | }
403 | y = np.repeat([0, 1, 2, 3], 25)
404 |
405 | X_train, X_valid, y_train, y_valid = TrainSplit(0.2)(
406 | X, y, nn)
407 | assert len(X_train['1']) == len(X_train['2']) == len(y_train) == 80
408 | assert len(X_valid['1']) == len(X_valid['2']) == len(y_valid) == 20
409 |
410 | def test_X_is_dict_eval_size_0(self, TrainSplit, nn):
411 | X = {
412 | '1': np.random.random((100, 10)),
413 | '2': np.random.random((100, 10)),
414 | }
415 | y = np.repeat([0, 1, 2, 3], 25)
416 |
417 | X_train, X_valid, y_train, y_valid = TrainSplit(0)(
418 | X, y, nn)
419 | assert len(X_train['1']) == len(X_train['2']) == len(y_train) == 100
420 | assert len(X_valid['1']) == len(X_valid['2']) == len(y_valid) == 0
421 |
422 |
423 | class TestTrainTestSplitBackwardCompatibility:
424 | @pytest.fixture
425 | def LegacyNet(self, NeuralNet):
426 | class LegacyNet(NeuralNet):
427 | def train_test_split(self, X, y, eval_size):
428 | self.__call_args__ = (X, y, eval_size)
429 | split = int(X.shape[0] * eval_size)
430 | return X[:split], X[split:], y[:split], y[split:]
431 | return LegacyNet
432 |
433 | def test_legacy_eval_size(self, NeuralNet):
434 | net = NeuralNet([], eval_size=0.3, max_epochs=0)
435 | assert net.train_split.eval_size == 0.3
436 |
437 | def test_legacy_method_default_eval_size(self, LegacyNet):
438 | net = LegacyNet([], max_epochs=0)
439 | X, y = np.ones((10, 3)), np.zeros(10)
440 | net.train_loop(X, y)
441 | assert net.__call_args__ == (X, y, 0.2)
442 |
443 | def test_legacy_method_given_eval_size(self, LegacyNet):
444 | net = LegacyNet([], eval_size=0.3, max_epochs=0)
445 | X, y = np.ones((10, 3)), np.zeros(10)
446 | net.train_loop(X, y)
447 | assert net.__call_args__ == (X, y, 0.3)
448 |
449 |
450 | class TestBatchIterator:
451 | @pytest.fixture
452 | def BatchIterator(self):
453 | from nolearn.lasagne import BatchIterator
454 | return BatchIterator
455 |
456 | @pytest.fixture
457 | def X(self):
458 | return np.arange(200).reshape((10, 20)).T.astype('float')
459 |
460 | @pytest.fixture
461 | def X_dict(self):
462 | return {
463 | 'one': np.arange(200).reshape((10, 20)).T.astype('float'),
464 | 'two': np.arange(200).reshape((20, 10)).astype('float'),
465 | }
466 |
467 | @pytest.fixture
468 | def y(self):
469 | return np.arange(20)
470 |
471 | @pytest.mark.parametrize("shuffle", [True, False])
472 | def test_simple_x_and_y(self, BatchIterator, X, y, shuffle):
473 | bi = BatchIterator(2, shuffle=shuffle)(X, y)
474 | batches = list(bi)
475 | assert len(batches) == 10
476 |
477 | X0, y0 = batches[0]
478 | assert X0.shape == (2, 10)
479 | assert y0.shape == (2,)
480 |
481 | Xt = np.vstack(b[0] for b in batches)
482 | yt = np.hstack(b[1] for b in batches)
483 | assert Xt.shape == X.shape
484 | assert yt.shape == y.shape
485 | np.testing.assert_equal(Xt[:, 0], yt)
486 |
487 | if shuffle is False:
488 | np.testing.assert_equal(X[:2], X0)
489 | np.testing.assert_equal(y[:2], y0)
490 |
491 | @pytest.mark.parametrize("shuffle", [True, False])
492 | def test_simple_x_no_y(self, BatchIterator, X, shuffle):
493 | bi = BatchIterator(2, shuffle=shuffle)(X)
494 | batches = list(bi)
495 | assert len(batches) == 10
496 |
497 | X0, y0 = batches[0]
498 | assert X0.shape == (2, 10)
499 | assert y0 is None
500 |
501 | if shuffle is False:
502 | np.testing.assert_equal(X[:2], X0)
503 |
504 | @pytest.mark.parametrize("shuffle", [True, False])
505 | def test_X_is_dict(self, BatchIterator, X_dict, shuffle):
506 | bi = BatchIterator(2, shuffle=shuffle)(X_dict)
507 | batches = list(bi)
508 | assert len(batches) == 10
509 |
510 | X0, y0 = batches[0]
511 | assert X0['one'].shape == (2, 10)
512 | assert X0['two'].shape == (2, 10)
513 | assert y0 is None
514 |
515 | Xt1 = np.vstack(b[0]['one'] for b in batches)
516 | Xt2 = np.vstack(b[0]['two'] for b in batches)
517 | assert Xt1.shape == X_dict['one'].shape
518 | assert Xt2.shape == X_dict['two'].shape
519 | np.testing.assert_equal(Xt1[:, 0], Xt2[:, 0] / 10)
520 |
521 | if shuffle is False:
522 | np.testing.assert_equal(X_dict['one'][:2], X0['one'])
523 | np.testing.assert_equal(X_dict['two'][:2], X0['two'])
524 |
525 | def test_shuffle_no_copy(self, BatchIterator, X, y):
526 | bi = BatchIterator(2, shuffle=True)(X, y)
527 | X0, y0 = list(bi)[0]
528 | assert X0.base is X # make sure X0 is a view
529 |
530 |
531 | class TestCheckForUnusedKwargs:
532 | def test_okay(self, NeuralNet):
533 | net = NeuralNet(
534 | layers=[('input', Mock), ('mylayer', Mock)],
535 | input_shape=(10, 10),
536 | mylayer_hey='hey',
537 | update_foo=1,
538 | update_bar=2,
539 | )
540 | net._create_iter_funcs = lambda *args: (1, 2, 3)
541 | net.initialize()
542 |
543 | def test_unused(self, NeuralNet):
544 | net = NeuralNet(
545 | layers=[('input', Mock), ('mylayer', Mock)],
546 | input_shape=(10, 10),
547 | mylayer_hey='hey',
548 | yourlayer_ho='ho',
549 | update_foo=1,
550 | update_bar=2,
551 | )
552 | net._create_iter_funcs = lambda *args: (1, 2, 3)
553 |
554 | with pytest.raises(ValueError) as err:
555 | net.initialize()
556 | assert str(err.value) == 'Unused kwarg: yourlayer_ho'
557 |
558 |
559 | class TestInitializeLayers:
560 | def test_initialization_with_layer_instance(self, NeuralNet):
561 | layer1 = InputLayer(shape=(128, 13)) # name will be assigned
562 | layer2 = DenseLayer(layer1, name='output', num_units=2) # has name
563 | nn = NeuralNet(layers=layer2)
564 | out = nn.initialize_layers()
565 | assert nn.layers_['output'] == layer2 == out[0]
566 | assert nn.layers_['input0'] == layer1
567 |
568 | def test_initialization_with_layer_instance_bad_params(self, NeuralNet):
569 | layer = DenseLayer(InputLayer(shape=(128, 13)), num_units=2)
570 | nn = NeuralNet(layers=layer, dense1_num_units=3)
571 | with pytest.raises(ValueError):
572 | nn.initialize_layers()
573 |
574 | def test_initialization_with_tuples(self, NeuralNet):
575 | input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
576 | hidden1, hidden2, output = [
577 | Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(3)]
578 | nn = NeuralNet(
579 | layers=[
580 | (input, {'shape': (10, 10), 'name': 'input'}),
581 | (hidden1, {'some': 'param', 'another': 'param'}),
582 | (hidden2, {}),
583 | (output, {'name': 'output'}),
584 | ],
585 | input_shape=(10, 10),
586 | mock1_some='iwin',
587 | )
588 | out = nn.initialize_layers(nn.layers)
589 |
590 | input.assert_called_with(
591 | name='input', shape=(10, 10))
592 | assert nn.layers_['input'] is input.return_value
593 |
594 | hidden1.assert_called_with(
595 | incoming=input.return_value, name='mock1',
596 | some='iwin', another='param')
597 | assert nn.layers_['mock1'] is hidden1.return_value
598 |
599 | hidden2.assert_called_with(
600 | incoming=hidden1.return_value, name='mock2')
601 | assert nn.layers_['mock2'] is hidden2.return_value
602 |
603 | output.assert_called_with(
604 | incoming=hidden2.return_value, name='output')
605 |
606 | assert out[0] is nn.layers_['output']
607 |
608 | def test_initializtion_with_tuples_resolve_layers(self, NeuralNet):
609 | nn = NeuralNet(
610 | layers=[
611 | ('lasagne.layers.InputLayer', {'shape': (None, 10)}),
612 | ('lasagne.layers.DenseLayer', {'num_units': 33}),
613 | ],
614 | )
615 | out, = nn.initialize_layers(nn.layers)
616 | assert out.num_units == 33
617 |
618 | def test_initialization_legacy(self, NeuralNet):
619 | input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
620 | hidden1, hidden2, output = [
621 | Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(3)]
622 | nn = NeuralNet(
623 | layers=[
624 | ('input', input),
625 | ('hidden1', hidden1),
626 | ('hidden2', hidden2),
627 | ('output', output),
628 | ],
629 | input_shape=(10, 10),
630 | hidden1_some='param',
631 | )
632 | out = nn.initialize_layers(nn.layers)
633 |
634 | input.assert_called_with(
635 | name='input', shape=(10, 10))
636 | assert nn.layers_['input'] is input.return_value
637 |
638 | hidden1.assert_called_with(
639 | incoming=input.return_value, name='hidden1', some='param')
640 | assert nn.layers_['hidden1'] is hidden1.return_value
641 |
642 | hidden2.assert_called_with(
643 | incoming=hidden1.return_value, name='hidden2')
644 | assert nn.layers_['hidden2'] is hidden2.return_value
645 |
646 | output.assert_called_with(
647 | incoming=hidden2.return_value, name='output')
648 |
649 | assert out[0] is nn.layers_['output']
650 |
651 | def test_initializtion_legacy_resolve_layers(self, NeuralNet):
652 | nn = NeuralNet(
653 | layers=[
654 | ('input', 'lasagne.layers.InputLayer'),
655 | ('output', 'lasagne.layers.DenseLayer'),
656 | ],
657 | input_shape=(None, 10),
658 | output_num_units=33,
659 | )
660 | out, = nn.initialize_layers(nn.layers)
661 | assert out.num_units == 33
662 |
663 | def test_initialization_legacy_with_unicode_names(self, NeuralNet):
664 | # Test whether legacy initialization is triggered; if not,
665 | # raises error.
666 | input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
667 | hidden1, hidden2, output = [
668 | Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(3)]
669 | nn = NeuralNet(
670 | layers=[
671 | (u'input', input),
672 | (u'hidden1', hidden1),
673 | (u'hidden2', hidden2),
674 | (u'output', output),
675 | ],
676 | input_shape=(10, 10),
677 | hidden1_some='param',
678 | )
679 | nn.initialize_layers()
680 |
681 | def test_diamond(self, NeuralNet):
682 | input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
683 | hidden1, hidden2, concat, output = [
684 | Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(4)]
685 | nn = NeuralNet(
686 | layers=[
687 | ('input', input),
688 | ('hidden1', hidden1),
689 | ('hidden2', hidden2),
690 | ('concat', concat),
691 | ('output', output),
692 | ],
693 | input_shape=(10, 10),
694 | hidden2_incoming='input',
695 | concat_incomings=['hidden1', 'hidden2'],
696 | )
697 | nn.initialize_layers(nn.layers)
698 |
699 | input.assert_called_with(name='input', shape=(10, 10))
700 | hidden1.assert_called_with(incoming=input.return_value, name='hidden1')
701 | hidden2.assert_called_with(incoming=input.return_value, name='hidden2')
702 | concat.assert_called_with(
703 | incomings=[hidden1.return_value, hidden2.return_value],
704 | name='concat'
705 | )
706 | output.assert_called_with(incoming=concat.return_value, name='output')
707 |
708 | def test_initialization_with_mask_input(self, NeuralNet):
709 | nn = NeuralNet(
710 | layers=[
711 | (InputLayer, {'shape': (None, 20, 32), 'name': 'l_in'}),
712 | (InputLayer, {'shape': (None, 20), 'name': 'l_mask'}),
713 | (RecurrentLayer, {'incoming': 'l_in',
714 | 'mask_input': 'l_mask',
715 | 'num_units': 2,
716 | 'name': 'l_rec'}),
717 | ])
718 | nn.initialize_layers()
719 | assert nn.layers_['l_rec'].mask_incoming_index == 1
720 |
721 | def test_legacy_initialization_with_mask_input(self, NeuralNet):
722 | nn = NeuralNet(
723 | layers=[
724 | ('l_in', InputLayer),
725 | ('l_mask', InputLayer),
726 | ('l_rec', RecurrentLayer),
727 | ],
728 | l_in_shape=(None, 20, 32),
729 | l_in_name='l_in',
730 | l_mask_shape=(None, 20),
731 | l_mask_name='l_mask',
732 | l_rec_incoming='l_in',
733 | l_rec_mask_input='l_mask',
734 | l_rec_num_units=2,
735 | l_rec_name='l_rec',
736 | )
737 | nn.initialize_layers()
738 | assert nn.layers_['l_rec'].mask_incoming_index == 1
739 |
740 |
741 | class TestCheckGoodInput:
742 | @pytest.fixture
743 | def check_good_input(self, nn):
744 | return nn._check_good_input
745 |
746 | @pytest.fixture
747 | def X(self):
748 | return np.arange(100).reshape(10, 10).astype(floatX)
749 |
750 | @pytest.fixture
751 | def y(self):
752 | return np.arange(10).astype(np.int32)
753 |
754 | @pytest.fixture
755 | def y_regr(self):
756 | return np.arange(10).reshape(-1, 1).astype(floatX)
757 |
758 | def test_X_OK(self, check_good_input, X):
759 | assert check_good_input(X) == (X, None)
760 |
761 | def test_X_and_y_OK(self, check_good_input, X, y):
762 | assert check_good_input(X, y) == (X, y)
763 |
764 | def test_X_and_y_OK_regression(self, nn, check_good_input, X, y_regr):
765 | nn.regression = True
766 | assert check_good_input(X, y_regr) == (X, y_regr)
767 |
768 | def test_X_and_y_length_mismatch(self, check_good_input, X, y):
769 | with pytest.raises(ValueError):
770 | check_good_input(
771 | X[:9],
772 | y
773 | )
774 |
775 | def test_X_dict_and_y_length_mismatch(self, check_good_input, X, y):
776 | with pytest.raises(ValueError):
777 | check_good_input(
778 | {'one': X, 'two': X},
779 | y[:9],
780 | )
781 |
782 | def test_X_dict_length_mismatch(self, check_good_input, X):
783 | with pytest.raises(ValueError):
784 | check_good_input({
785 | 'one': X,
786 | 'two': X[:9],
787 | })
788 |
789 | def test_y_regression_1dim(self, nn, check_good_input, X, y_regr):
790 | y = y_regr.reshape(-1)
791 | nn.regression = True
792 | X1, y1 = check_good_input(X, y)
793 | assert (X1 == X).all()
794 | assert (y1 == y.reshape(-1, 1)).all()
795 |
796 | def test_y_regression_2dim(self, nn, check_good_input, X, y_regr):
797 | y = y_regr
798 | nn.regression = True
799 | X1, y1 = check_good_input(X, y)
800 | assert (X1 == X).all()
801 | assert (y1 == y).all()
802 |
803 |
804 | class TestGetOutput:
805 | def test_layer_object(self, net_fitted, X_train):
806 | layer = net_fitted.layers_['conv2']
807 | output = net_fitted.get_output(layer, X_train[:3])
808 | assert output.shape == (3, 8, 8, 8)
809 |
810 | def test_layer_name(self, net_fitted, X_train):
811 | output = net_fitted.get_output('conv2', X_train[:3])
812 | assert output.shape == (3, 8, 8, 8)
813 |
814 | def test_get_output_last_layer(self, net_fitted, X_train):
815 | result = net_fitted.get_output(net_fitted.layers_[-1], X_train[:129])
816 | expected = net_fitted.predict_proba(X_train[:129])
817 | np.testing.assert_equal(result, expected)
818 |
819 | def test_no_conv(self, net_no_conv):
820 | net_no_conv.initialize()
821 | X = np.random.random((10, 100)).astype(floatX)
822 | result = net_no_conv.get_output('output', X)
823 | expected = net_no_conv.predict_proba(X)
824 | np.testing.assert_equal(result, expected)
825 |
826 |
827 | class TestMultiInputFunctional:
828 | @pytest.fixture(scope='session')
829 | def net(self, NeuralNet):
830 | return NeuralNet(
831 | layers=[
832 | (InputLayer,
833 | {'name': 'input1', 'shape': (None, 392)}),
834 | (DenseLayer,
835 | {'name': 'hidden1', 'num_units': 98}),
836 | (InputLayer,
837 | {'name': 'input2', 'shape': (None, 392)}),
838 | (DenseLayer,
839 | {'name': 'hidden2', 'num_units': 98}),
840 | (ConcatLayer,
841 | {'incomings': ['hidden1', 'hidden2']}),
842 | (DenseLayer,
843 | {'name': 'hidden3', 'num_units': 98}),
844 | (DenseLayer,
845 | {'name': 'output', 'num_units': 10, 'nonlinearity': softmax}),
846 | ],
847 |
848 | update=nesterov_momentum,
849 | update_learning_rate=0.01,
850 | update_momentum=0.9,
851 |
852 | max_epochs=2,
853 | verbose=4,
854 | )
855 |
856 | @pytest.fixture(scope='session')
857 | def net_fitted(self, net, mnist):
858 | X, y = mnist
859 | X_train, y_train = X[:10000], y[:10000]
860 | X_train1, X_train2 = X_train[:, :392], X_train[:, 392:]
861 | return net.fit({'input1': X_train1, 'input2': X_train2}, y_train)
862 |
863 | @pytest.fixture(scope='session')
864 | def y_pred(self, net_fitted, mnist):
865 | X, y = mnist
866 | X_test = X[60000:]
867 | X_test1, X_test2 = X_test[:, :392], X_test[:, 392:]
868 | return net_fitted.predict({'input1': X_test1, 'input2': X_test2})
869 |
870 | def test_accuracy(self, net_fitted, mnist, y_pred):
871 | X, y = mnist
872 | y_test = y[60000:]
873 | assert accuracy_score(y_pred, y_test) > 0.85
874 |
875 |
876 | class TestGradScale:
877 | @pytest.fixture
878 | def grad_scale(self):
879 | from nolearn.lasagne import grad_scale
880 | return grad_scale
881 |
882 | @pytest.mark.parametrize("layer", [
883 | BatchNormLayer(InputLayer((None, 16))),
884 | Conv2DLayer(InputLayer((None, 1, 28, 28)), 2, 3),
885 | DenseLayer(InputLayer((None, 16)), 16),
886 | ])
887 | def test_it(self, grad_scale, layer):
888 | layer2 = grad_scale(layer, 0.33)
889 | assert layer2 is layer
890 | for param in layer.get_params(trainable=True):
891 | np.testing.assert_almost_equal(param.tag.grad_scale, 0.33)
892 | for param in layer.get_params(trainable=False):
893 | assert hasattr(param.tag, 'grad_scale') is False
894 |
895 |
896 | class TestMultiOutput:
897 | @pytest.fixture(scope='class')
898 | def mo_net(self, NeuralNet):
899 | def objective(layers_, target, **kwargs):
900 | out_a_layer = layers_['output_a']
901 | out_b_layer = layers_['output_b']
902 |
903 | # Get the outputs
904 | out_a, out_b = get_output([out_a_layer, out_b_layer])
905 |
906 | # Get the targets
907 | gt_a = T.cast(target[:, 0], 'int32')
908 | gt_b = target[:, 1].reshape((-1, 1))
909 |
910 | # Calculate the multi task loss
911 | cls_loss = aggregate(categorical_crossentropy(out_a, gt_a))
912 | reg_loss = aggregate(categorical_crossentropy(out_b, gt_b))
913 | loss = cls_loss + reg_loss
914 | return loss
915 |
916 | # test that both branches of the multi output network are included,
917 | # and also that a single layer isn't included multiple times.
918 | l = InputLayer(shape=(None, 1, 28, 28), name="input")
919 | l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8)
920 | l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8)
921 |
922 | la = DenseLayer(l, name='hidden_a', num_units=128)
923 | la = DenseLayer(la, name='output_a', nonlinearity=softmax,
924 | num_units=10)
925 |
926 | lb = DenseLayer(l, name='hidden_b', num_units=128)
927 | lb = DenseLayer(lb, name='output_b', nonlinearity=sigmoid, num_units=1)
928 |
929 | net = NeuralNet(layers=[la, lb],
930 | update_learning_rate=0.5,
931 | y_tensor_type=None,
932 | regression=True,
933 | objective=objective)
934 | net.initialize()
935 | return net
936 |
937 | def test_layers_included(self, mo_net):
938 | expected_names = sorted(["input", "conv1", "conv2",
939 | "hidden_a", "output_a",
940 | "hidden_b", "output_b"])
941 | network_names = sorted(list(mo_net.layers_.keys()))
942 |
943 | assert (expected_names == network_names)
944 |
945 | def test_predict(self, mo_net):
946 | dummy_data = np.zeros((2, 1, 28, 28), np.float32)
947 | p_cls, p_reg = mo_net.predict(dummy_data)
948 | assert(p_cls.shape == (2, 10))
949 | assert(p_reg.shape == (2, 1))
950 |
--------------------------------------------------------------------------------
/nolearn/lasagne/tests/test_handlers.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 | import pickle
3 |
4 | from lasagne.layers import ConcatLayer
5 | from lasagne.layers import Conv2DLayer
6 | from lasagne.layers import DenseLayer
7 | from lasagne.layers import MaxPool2DLayer
8 | from lasagne.layers import InputLayer
9 | from lasagne.nonlinearities import softmax
10 | from lasagne.updates import nesterov_momentum
11 | from mock import patch
12 | from mock import Mock
13 | import numpy
14 | import pytest
15 |
16 | from nolearn._compat import builtins
17 |
18 |
19 | def test_print_log(mnist):
20 | from nolearn.lasagne import PrintLog
21 |
22 | nn = Mock(
23 | regression=False,
24 | custom_scores=[('my1', 0.99)],
25 | scores_train=[('my2', 0.98)],
26 | scores_valid=[('my3', 0.33)],
27 | )
28 |
29 | train_history = [{
30 | 'epoch': 1,
31 | 'train_loss': 0.8,
32 | 'valid_loss': 0.7,
33 | 'train_loss_best': False,
34 | 'valid_loss_best': False,
35 | 'valid_accuracy': 0.9,
36 | 'my1': 0.99,
37 | 'my2': 0.98,
38 | 'my3': 0.33,
39 | 'dur': 1.0,
40 | }]
41 | output = PrintLog().table(nn, train_history)
42 | assert output.split() == [
43 | 'epoch',
44 | 'trn',
45 | 'loss',
46 | 'val',
47 | 'loss',
48 | 'trn/val',
49 | 'valid',
50 | 'acc',
51 | 'my2',
52 | 'my3',
53 | 'my1',
54 | 'dur',
55 | '-------',
56 | '----------',
57 | '----------',
58 | '---------',
59 | '-----------',
60 | '-------',
61 | '-------',
62 | '-------',
63 | '-----',
64 | '1',
65 | '0.80000',
66 | '0.70000',
67 | '1.14286',
68 | '0.90000',
69 | '0.98000',
70 | '0.33000',
71 | '0.99000',
72 | '1.00s',
73 | ]
74 |
75 |
76 | class TestSaveWeights():
77 | @pytest.fixture
78 | def SaveWeights(self):
79 | from nolearn.lasagne import SaveWeights
80 | return SaveWeights
81 |
82 | def test_every_n_epochs_true(self, SaveWeights):
83 | train_history = [{'epoch': 9, 'valid_loss': 1.1}]
84 | nn = Mock()
85 | handler = SaveWeights('mypath', every_n_epochs=3)
86 | handler(nn, train_history)
87 | assert nn.save_params_to.call_count == 1
88 | nn.save_params_to.assert_called_with('mypath')
89 |
90 | def test_every_n_epochs_false(self, SaveWeights):
91 | train_history = [{'epoch': 9, 'valid_loss': 1.1}]
92 | nn = Mock()
93 | handler = SaveWeights('mypath', every_n_epochs=4)
94 | handler(nn, train_history)
95 | assert nn.save_params_to.call_count == 0
96 |
97 | def test_only_best_true_single_entry(self, SaveWeights):
98 | train_history = [{'epoch': 9, 'valid_loss': 1.1}]
99 | nn = Mock()
100 | handler = SaveWeights('mypath', only_best=True)
101 | handler(nn, train_history)
102 | assert nn.save_params_to.call_count == 1
103 |
104 | def test_only_best_true_two_entries(self, SaveWeights):
105 | train_history = [
106 | {'epoch': 9, 'valid_loss': 1.2},
107 | {'epoch': 10, 'valid_loss': 1.1},
108 | ]
109 | nn = Mock()
110 | handler = SaveWeights('mypath', only_best=True)
111 | handler(nn, train_history)
112 | assert nn.save_params_to.call_count == 1
113 |
114 | def test_only_best_false_two_entries(self, SaveWeights):
115 | train_history = [
116 | {'epoch': 9, 'valid_loss': 1.2},
117 | {'epoch': 10, 'valid_loss': 1.3},
118 | ]
119 | nn = Mock()
120 | handler = SaveWeights('mypath', only_best=True)
121 | handler(nn, train_history)
122 | assert nn.save_params_to.call_count == 0
123 |
124 | def test_with_path_interpolation(self, SaveWeights):
125 | train_history = [{'epoch': 9, 'valid_loss': 1.1}]
126 | nn = Mock()
127 | handler = SaveWeights('mypath-{epoch}-{timestamp}-{loss}.pkl')
128 | handler(nn, train_history)
129 | path = nn.save_params_to.call_args[0][0]
130 | assert path.startswith('mypath-0009-2')
131 | assert path.endswith('-1.1.pkl')
132 |
133 | def test_pickle(self, SaveWeights):
134 | train_history = [{'epoch': 9, 'valid_loss': 1.1}]
135 | nn = Mock()
136 | with patch('nolearn.lasagne.handlers.pickle') as pickle:
137 | with patch.object(builtins, 'open') as mock_open:
138 | handler = SaveWeights('mypath', every_n_epochs=3, pickle=True)
139 | handler(nn, train_history)
140 |
141 | mock_open.assert_called_with('mypath', 'wb')
142 | pickle.dump.assert_called_with(nn, mock_open().__enter__(), -1)
143 |
144 |
145 | class TestRememberBestWeights:
146 | @pytest.fixture
147 | def RememberBestWeights(self):
148 | from nolearn.lasagne.handlers import RememberBestWeights
149 | return RememberBestWeights
150 |
151 | @pytest.fixture
152 | def RestoreBestWeights(self):
153 | from nolearn.lasagne.handlers import RestoreBestWeights
154 | return RestoreBestWeights
155 |
156 | @pytest.mark.parametrize('loss_name', ['valid_loss', 'my_loss'])
157 | def test_simple(self, RememberBestWeights, loss_name):
158 | nn1, nn2, nn3 = Mock(), Mock(), Mock()
159 | rbw = RememberBestWeights(loss=loss_name)
160 | train_history = []
161 |
162 | train_history.append({'epoch': 1, loss_name: 1.0})
163 | rbw(nn1, train_history)
164 | assert rbw.best_weights is nn1.get_all_params_values()
165 |
166 | train_history.append({'epoch': 2, loss_name: 1.1})
167 | rbw(nn2, train_history)
168 | assert rbw.best_weights is nn1.get_all_params_values()
169 |
170 | train_history.append({'epoch': 3, loss_name: 0.9})
171 | rbw(nn3, train_history)
172 | assert rbw.best_weights is nn3.get_all_params_values()
173 |
174 | def test_custom_score(self, RememberBestWeights):
175 | nn1, nn2, nn3 = Mock(), Mock(), Mock()
176 | rbw = RememberBestWeights(score='myscr')
177 | train_history = []
178 |
179 | train_history.append({'epoch': 1, 'myscr': 1.0})
180 | rbw(nn1, train_history)
181 | assert rbw.best_weights is nn1.get_all_params_values()
182 |
183 | train_history.append({'epoch': 2, 'myscr': 1.1})
184 | rbw(nn2, train_history)
185 | assert rbw.best_weights is nn2.get_all_params_values()
186 |
187 | train_history.append({'epoch': 3, 'myscr': 0.9})
188 | rbw(nn3, train_history)
189 | assert rbw.best_weights is nn2.get_all_params_values()
190 |
191 | def test_restore(self, RememberBestWeights, RestoreBestWeights):
192 | nn = Mock()
193 | remember_best_weights = RememberBestWeights()
194 | restore_best_weights = RestoreBestWeights(
195 | remember=remember_best_weights)
196 | train_history = []
197 | train_history.append({'epoch': 1, 'valid_loss': 1.0})
198 | remember_best_weights(nn, train_history)
199 | restore_best_weights(nn, train_history)
200 | nn.load_params_from.assert_called_with(nn.get_all_params_values())
201 | nn.load_params_from.assert_called_with(
202 | remember_best_weights.best_weights)
203 |
204 |
205 | class TestPrintLayerInfo():
206 | @pytest.fixture(scope='session')
207 | def X_train(self, mnist):
208 | X, y = mnist
209 | return X[:100].reshape(-1, 1, 28, 28)
210 |
211 | @pytest.fixture(scope='session')
212 | def y_train(self, mnist):
213 | X, y = mnist
214 | return y[:100]
215 |
216 | @pytest.fixture(scope='session')
217 | def nn(self, NeuralNet, X_train, y_train):
218 | nn = NeuralNet(
219 | layers=[
220 | ('input', InputLayer),
221 | ('dense0', DenseLayer),
222 | ('dense1', DenseLayer),
223 | ('output', DenseLayer),
224 | ],
225 | input_shape=(None, 1, 28, 28),
226 | output_num_units=10,
227 | output_nonlinearity=softmax,
228 |
229 | more_params=dict(
230 | dense0_num_units=16,
231 | dense1_num_units=16,
232 | ),
233 |
234 | update=nesterov_momentum,
235 | update_learning_rate=0.01,
236 | update_momentum=0.9,
237 |
238 | max_epochs=3,
239 | )
240 | nn.initialize()
241 |
242 | return nn
243 |
244 | @pytest.fixture(scope='session')
245 | def cnn(self, NeuralNet, X_train, y_train):
246 | nn = NeuralNet(
247 | layers=[
248 | ('input', InputLayer),
249 | ('conv1', Conv2DLayer),
250 | ('conv2', Conv2DLayer),
251 | ('pool2', MaxPool2DLayer),
252 | ('conv3', Conv2DLayer),
253 | ('output', DenseLayer),
254 | ],
255 | input_shape=(None, 1, 28, 28),
256 | output_num_units=10,
257 | output_nonlinearity=softmax,
258 |
259 | more_params=dict(
260 | conv1_filter_size=5, conv1_num_filters=16,
261 | conv2_filter_size=3, conv2_num_filters=16,
262 | pool2_pool_size=8, pool2_ignore_border=False,
263 | conv3_filter_size=3, conv3_num_filters=16,
264 | hidden1_num_units=16,
265 | ),
266 |
267 | update=nesterov_momentum,
268 | update_learning_rate=0.01,
269 | update_momentum=0.9,
270 |
271 | max_epochs=3,
272 | )
273 |
274 | nn.initialize()
275 | return nn
276 |
277 | @pytest.fixture
278 | def is_conv2d(self):
279 | from nolearn.lasagne.util import is_conv2d
280 | return is_conv2d
281 |
282 | @pytest.fixture
283 | def is_maxpool2d(self):
284 | from nolearn.lasagne.util import is_maxpool2d
285 | return is_maxpool2d
286 |
287 | @pytest.fixture
288 | def print_info(self):
289 | from nolearn.lasagne.handlers import PrintLayerInfo
290 | return PrintLayerInfo()
291 |
292 | def test_is_conv2d_net_false(self, nn, is_conv2d):
293 | assert is_conv2d(nn.layers_.values()) is False
294 |
295 | def test_is_conv2d_net_true(self, cnn, is_conv2d):
296 | assert is_conv2d(cnn.layers_.values()) is True
297 |
298 | def test_is_conv2d_layer(self, nn, cnn, is_conv2d):
299 | assert is_conv2d(nn.layers_['input']) is False
300 | assert is_conv2d(cnn.layers_['pool2']) is False
301 | assert is_conv2d(cnn.layers_['conv1']) is True
302 |
303 | def test_is_maxpool2d_net_false(self, nn, is_maxpool2d):
304 | assert is_maxpool2d(nn.layers_.values()) is False
305 |
306 | def test_is_maxpool2d_net_true(self, cnn, is_maxpool2d):
307 | assert is_maxpool2d(cnn.layers_.values()) is True
308 |
309 | def test_is_maxpool2d_layer(self, nn, cnn, is_maxpool2d):
310 | assert is_maxpool2d(nn.layers_['input']) is False
311 | assert is_maxpool2d(cnn.layers_['pool2']) is True
312 | assert is_maxpool2d(cnn.layers_['conv1']) is False
313 |
314 | def test_print_layer_info_greeting(self, nn, print_info):
315 | # number of learnable parameters is weights + biases:
316 | # 28 * 28 * 16 + 16 + 16 * 16 + 16 + 16 * 10 + 10 = 13002
317 | expected = '# Neural Network with 13002 learnable parameters\n'
318 | message = print_info._get_greeting(nn)
319 | assert message == expected
320 |
321 | def test_print_layer_info_plain_nn(self, nn, print_info):
322 | expected = """\
323 | # name size
324 | --- ------ -------
325 | 0 input 1x28x28
326 | 1 dense0 16
327 | 2 dense1 16
328 | 3 output 10"""
329 | message = print_info._get_layer_info_plain(nn)
330 | assert message == expected
331 |
332 | def test_print_layer_info_plain_cnn(self, cnn, print_info):
333 | expected = """\
334 | # name size
335 | --- ------ --------
336 | 0 input 1x28x28
337 | 1 conv1 16x24x24
338 | 2 conv2 16x22x22
339 | 3 pool2 16x3x3
340 | 4 conv3 16x1x1
341 | 5 output 10"""
342 | message = print_info._get_layer_info_plain(cnn)
343 | assert message == expected
344 |
345 | def test_print_layer_info_conv_cnn(self, cnn, print_info):
346 | expected = """\
347 | name size total cap.Y cap.X cov.Y cov.X
348 | ------ -------- ------- ------- ------- ------- -------
349 | input 1x28x28 784 100.00 100.00 100.00 100.00
350 | conv1 16x24x24 9216 100.00 100.00 17.86 17.86
351 | conv2 16x22x22 7744 42.86 42.86 25.00 25.00
352 | pool2 16x3x3 144 42.86 42.86 25.00 25.00
353 | conv3 16x1x1 16 104.35 104.35 82.14 82.14
354 | output 10 10 100.00 100.00 100.00 100.00"""
355 | message, legend = print_info._get_layer_info_conv(cnn)
356 | assert message == expected
357 |
358 | expected = """
359 | Explanation
360 | X, Y: image dimensions
361 | cap.: learning capacity
362 | cov.: coverage of image
363 | \x1b[35mmagenta\x1b[0m: capacity too low (<1/6)
364 | \x1b[36mcyan\x1b[0m: image coverage too high (>100%)
365 | \x1b[31mred\x1b[0m: capacity too low and coverage too high
366 | """
367 | assert legend == expected
368 |
369 | def test_print_layer_info_with_empty_shape(self, print_info, NeuralNet):
370 | # construct a net with both conv layer (to trigger
371 | # get_conv_infos) and a layer with shape (None,).
372 | l_img = InputLayer(shape=(None, 1, 28, 28))
373 | l_conv = Conv2DLayer(l_img, num_filters=3, filter_size=3)
374 | l0 = DenseLayer(l_conv, num_units=10)
375 | l_inp = InputLayer(shape=(None,)) # e.g. vector input
376 | l1 = DenseLayer(l_inp, num_units=10)
377 | l_merge = ConcatLayer([l0, l1])
378 |
379 | nn = NeuralNet(l_merge, update_learning_rate=0.1, verbose=2)
380 | nn.initialize()
381 | # used to raise TypeError
382 | print_info(nn)
383 |
384 |
385 | class TestWeightLog:
386 | @pytest.fixture
387 | def WeightLog(self):
388 | from nolearn.lasagne import WeightLog
389 | return WeightLog
390 |
391 | @pytest.fixture
392 | def nn(self):
393 | nn = Mock()
394 | nn.get_all_params_values.side_effect = [
395 | OrderedDict([
396 | ('layer1', numpy.array([[-1, -2]])),
397 | ('layer2', numpy.array([[3, 4]])),
398 | ]),
399 | OrderedDict([
400 | ('layer1', numpy.array([[-2, -3]])),
401 | ('layer2', numpy.array([[5, 7]])),
402 | ]),
403 | ]
404 | return nn
405 |
406 | def test_history(self, WeightLog, nn):
407 | wl = WeightLog()
408 | wl(nn, None)
409 | wl(nn, None)
410 |
411 | assert wl.history[0] == {
412 | 'layer1_0 wdiff': 0,
413 | 'layer1_0 wabsmean': 1.5,
414 | 'layer1_0 wmean': -1.5,
415 |
416 | 'layer2_0 wdiff': 0,
417 | 'layer2_0 wabsmean': 3.5,
418 | 'layer2_0 wmean': 3.5,
419 | }
420 |
421 | assert wl.history[1]['layer1_0 wdiff'] == 1.0
422 | assert wl.history[1]['layer2_0 wdiff'] == 2.5
423 |
424 | def test_save_to(self, WeightLog, nn, tmpdir):
425 | save_to = tmpdir.join("hello.csv")
426 | wl = WeightLog(save_to=save_to.strpath, write_every=1)
427 | wl(nn, None)
428 | wl(nn, None)
429 |
430 | assert save_to.readlines() == [
431 | 'layer1_0 wdiff,layer1_0 wabsmean,layer1_0 wmean,'
432 | 'layer2_0 wdiff,layer2_0 wabsmean,layer2_0 wmean\n',
433 | '0.0,1.5,-1.5,0.0,3.5,3.5\n',
434 | '1.0,2.5,-2.5,2.5,6.0,6.0\n',
435 | ]
436 |
437 | def test_pickle(self, WeightLog, nn, tmpdir):
438 | save_to = tmpdir.join("hello.csv")
439 | pkl = tmpdir.join("hello.pkl")
440 | wl = WeightLog(save_to=save_to.strpath, write_every=1)
441 | wl(nn, None)
442 |
443 | with open(pkl.strpath, 'wb') as f:
444 | pickle.dump(wl, f)
445 |
446 | with open(pkl.strpath, 'rb') as f:
447 | wl = pickle.load(f)
448 |
449 | wl(nn, None)
450 | assert save_to.readlines() == [
451 | 'layer1_0 wdiff,layer1_0 wabsmean,layer1_0 wmean,'
452 | 'layer2_0 wdiff,layer2_0 wabsmean,layer2_0 wmean\n',
453 | '0.0,1.5,-1.5,0.0,3.5,3.5\n',
454 | '1.0,2.5,-2.5,2.5,6.0,6.0\n',
455 | ]
456 |
--------------------------------------------------------------------------------
/nolearn/lasagne/tests/test_visualize.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | import numpy as np
3 | import pytest
4 |
5 |
6 | class TestCNNVisualizeFunctions:
7 | @pytest.fixture
8 | def X_non_square(self, X_train):
9 | X = np.hstack(
10 | (X_train[:, :20 * 28], X_train[:, :20 * 28], X_train[:, :20 * 28]))
11 | X = X.reshape(-1, 3, 20, 28)
12 | return X
13 |
14 | def test_plot_loss(self, net_fitted):
15 | from nolearn.lasagne.visualize import plot_loss
16 | plot_loss(net_fitted)
17 | plt.clf()
18 | plt.cla()
19 |
20 | def plot_conv_weights(self, net, **kwargs):
21 | from nolearn.lasagne.visualize import plot_conv_weights
22 | plot_conv_weights(net.layers_['conv1'], **kwargs)
23 | plt.clf()
24 | plt.cla()
25 |
26 | @pytest.mark.parametrize('kwargs', [{}, {'figsize': (3, 4)}])
27 | def test_plot_conv_weights(self, net_fitted, net_color_non_square, kwargs):
28 | # XXX workaround: fixtures cannot be used (yet) in conjunction
29 | # with parametrize
30 | self.plot_conv_weights(net_fitted, **kwargs)
31 | self.plot_conv_weights(net_color_non_square, **kwargs)
32 |
33 | def plot_conv_activity(self, net, X, **kwargs):
34 | from nolearn.lasagne.visualize import plot_conv_activity
35 | plot_conv_activity(net.layers_['conv1'], X, **kwargs)
36 | plt.clf()
37 | plt.cla()
38 |
39 | @pytest.mark.parametrize('kwargs', [{}, {'figsize': (3, 4)}])
40 | def test_plot_conv_activity(
41 | self, net_fitted, net_color_non_square, X_train, X_non_square,
42 | kwargs):
43 | # XXX see above
44 | self.plot_conv_activity(net_fitted, X_train[:1], **kwargs)
45 | self.plot_conv_activity(net_fitted, X_train[10:11])
46 |
47 | self.plot_conv_activity(
48 | net_color_non_square, X_non_square[:1], **kwargs)
49 | self.plot_conv_activity(
50 | net_color_non_square, X_non_square[10:11], **kwargs)
51 |
52 | def plot_occlusion(self, net, X, y, **kwargs):
53 | from nolearn.lasagne.visualize import plot_occlusion
54 | plot_occlusion(net, X, y, **kwargs)
55 | plt.clf()
56 | plt.cla()
57 |
58 | @pytest.mark.parametrize(
59 | 'kwargs', [{}, {'square_length': 3, 'figsize': (3, 4)}])
60 | def test_plot_occlusion(
61 | self, net_fitted, net_color_non_square,
62 | net_with_nonlinearity_layer, X_train, X_non_square, kwargs):
63 | # XXX see above
64 | self.plot_occlusion(net_fitted, X_train[3:4], [0], **kwargs)
65 | self.plot_occlusion(net_fitted, X_train[2:5], [1, 2, 3], **kwargs)
66 |
67 | self.plot_occlusion(
68 | net_with_nonlinearity_layer, X_train[3:4], [0], **kwargs)
69 | self.plot_occlusion(
70 | net_with_nonlinearity_layer, X_train[2:5], [1, 2, 3], **kwargs)
71 |
72 | self.plot_occlusion(
73 | net_color_non_square, X_non_square[3:4], [0], **kwargs)
74 | self.plot_occlusion(
75 | net_color_non_square, X_non_square[2:5], [1, 2, 3], **kwargs)
76 |
77 | def test_draw_to_file_net(self, net_fitted, tmpdir):
78 | from nolearn.lasagne.visualize import draw_to_file
79 | fn = str(tmpdir.join('network.pdf'))
80 | draw_to_file(
81 | net_fitted, fn, output_shape=False)
82 |
83 | def test_draw_to_notebook_net(self, net_fitted):
84 | from nolearn.lasagne.visualize import draw_to_notebook
85 | draw_to_notebook(net_fitted, output_shape=False)
86 |
87 | def test_draw_to_file_layers(self, net_fitted, tmpdir):
88 | from nolearn.lasagne.visualize import draw_to_file
89 | fn = str(tmpdir.join('network.pdf'))
90 | draw_to_file(
91 | net_fitted.get_all_layers(), fn, output_shape=False)
92 |
93 | def test_draw_to_notebook_layers(self, net_fitted):
94 | from nolearn.lasagne.visualize import draw_to_notebook
95 | draw_to_notebook(net_fitted.get_all_layers(), output_shape=False)
96 |
--------------------------------------------------------------------------------
/nolearn/lasagne/util.py:
--------------------------------------------------------------------------------
1 | from functools import reduce
2 | from operator import mul
3 |
4 | from lasagne.layers import Layer
5 | from lasagne.layers import Conv2DLayer
6 | from lasagne.layers import MaxPool2DLayer
7 | import numpy as np
8 | from tabulate import tabulate
9 |
10 | convlayers = [Conv2DLayer]
11 | maxpoollayers = [MaxPool2DLayer]
12 | try:
13 | from lasagne.layers.cuda_convnet import Conv2DCCLayer
14 | from lasagne.layers.cuda_convnet import MaxPool2DCCLayer
15 | convlayers.append(Conv2DCCLayer)
16 | maxpoollayers.append(MaxPool2DCCLayer)
17 | except ImportError:
18 | pass
19 | try:
20 | from lasagne.layers.dnn import Conv2DDNNLayer
21 | from lasagne.layers.dnn import MaxPool2DDNNLayer
22 | convlayers.append(Conv2DDNNLayer)
23 | maxpoollayers.append(MaxPool2DDNNLayer)
24 | except ImportError:
25 | pass
26 |
27 | try:
28 | from lasagne.layers.corrmm import Conv2DMMLayer
29 | convlayers.append(Conv2DMMLayer)
30 | except ImportError:
31 | pass
32 |
33 |
34 | class ansi:
35 | BLUE = '\033[94m'
36 | CYAN = '\033[36m'
37 | GREEN = '\033[32m'
38 | MAGENTA = '\033[35m'
39 | RED = '\033[31m'
40 | ENDC = '\033[0m'
41 |
42 |
43 | def is_conv2d(layers):
44 | if isinstance(layers, Layer):
45 | return isinstance(layers, tuple(convlayers))
46 | return any([isinstance(layer, tuple(convlayers))
47 | for layer in layers])
48 |
49 |
50 | def is_maxpool2d(layers):
51 | if isinstance(layers, Layer):
52 | return isinstance(layers, tuple(maxpoollayers))
53 | return any([isinstance(layer, tuple(maxpoollayers))
54 | for layer in layers])
55 |
56 |
57 | def get_real_filter(layers, img_size):
58 | """Get the real filter sizes of each layer involved in
59 | convoluation. See Xudong Cao:
60 | https://www.kaggle.com/c/datasciencebowl/forums/t/13166/happy-lantern-festival-report-and-code
61 | This does not yet take into consideration feature pooling,
62 | padding, striding and similar gimmicks.
63 | """
64 | real_filter = np.zeros((len(layers), 2))
65 | conv_mode = True
66 | first_conv_layer = True
67 | expon = np.ones((1, 2))
68 |
69 | for i, layer in enumerate(layers[1:]):
70 | j = i + 1
71 | if not conv_mode:
72 | real_filter[j] = img_size
73 | continue
74 |
75 | if is_conv2d(layer):
76 | if not first_conv_layer:
77 | new_filter = np.array(layer.filter_size) * expon
78 | real_filter[j] = new_filter
79 | else:
80 | new_filter = np.array(layer.filter_size) * expon
81 | real_filter[j] = new_filter
82 | first_conv_layer = False
83 | elif is_maxpool2d(layer):
84 | real_filter[j] = real_filter[i]
85 | expon *= np.array(layer.pool_size)
86 | else:
87 | conv_mode = False
88 | real_filter[j] = img_size
89 |
90 | real_filter[0] = img_size
91 | return real_filter
92 |
93 |
94 | def get_receptive_field(layers, img_size):
95 | """Get the real filter sizes of each layer involved in
96 | convoluation. See Xudong Cao:
97 | https://www.kaggle.com/c/datasciencebowl/forums/t/13166/happy-lantern-festival-report-and-code
98 | This does not yet take into consideration feature pooling,
99 | padding, striding and similar gimmicks.
100 | """
101 | receptive_field = np.zeros((len(layers), 2))
102 | conv_mode = True
103 | first_conv_layer = True
104 | expon = np.ones((1, 2))
105 |
106 | for i, layer in enumerate(layers[1:]):
107 | j = i + 1
108 | if not conv_mode:
109 | receptive_field[j] = img_size
110 | continue
111 |
112 | if is_conv2d(layer):
113 | if not first_conv_layer:
114 | last_field = receptive_field[i]
115 | new_field = (last_field + expon *
116 | (np.array(layer.filter_size) - 1))
117 | receptive_field[j] = new_field
118 | else:
119 | receptive_field[j] = layer.filter_size
120 | first_conv_layer = False
121 | elif is_maxpool2d(layer):
122 | receptive_field[j] = receptive_field[i]
123 | expon *= np.array(layer.pool_size)
124 | else:
125 | conv_mode = False
126 | receptive_field[j] = img_size
127 |
128 | receptive_field[0] = img_size
129 | return receptive_field
130 |
131 |
132 | def get_conv_infos(net, min_capacity=100. / 6, detailed=False):
133 | CYA = ansi.CYAN
134 | END = ansi.ENDC
135 | MAG = ansi.MAGENTA
136 | RED = ansi.RED
137 |
138 | layers = net.layers_.values()
139 | # assume that first layer is input layer
140 | img_size = layers[0].output_shape[2:]
141 |
142 | header = ['name', 'size', 'total', 'cap.Y', 'cap.X',
143 | 'cov.Y', 'cov.X']
144 | if detailed:
145 | header += ['filter Y', 'filter X', 'field Y', 'field X']
146 |
147 | shapes = [layer.output_shape[1:] for layer in layers]
148 | totals = [str(reduce(mul, shape)) if shape else '0' for shape in shapes]
149 | shapes = ['x'.join(map(str, shape)) for shape in shapes]
150 | shapes = np.array(shapes).reshape(-1, 1)
151 | totals = np.array(totals).reshape(-1, 1)
152 |
153 | real_filters = get_real_filter(layers, img_size)
154 | receptive_fields = get_receptive_field(layers, img_size)
155 | capacity = 100. * real_filters / receptive_fields
156 | capacity[np.logical_not(np.isfinite(capacity))] = 1
157 | img_coverage = 100. * receptive_fields / img_size
158 | layer_names = [layer.name if layer.name
159 | else str(layer).rsplit('.')[-1].split(' ')[0]
160 | for layer in layers]
161 |
162 | colored_names = []
163 | for name, (covy, covx), (capy, capx) in zip(
164 | layer_names, img_coverage, capacity):
165 | if (
166 | ((covy > 100) or (covx > 100)) and
167 | ((capy < min_capacity) or (capx < min_capacity))
168 | ):
169 | name = "{}{}{}".format(RED, name, END)
170 | elif (covy > 100) or (covx > 100):
171 | name = "{}{}{}".format(CYA, name, END)
172 | elif (capy < min_capacity) or (capx < min_capacity):
173 | name = "{}{}{}".format(MAG, name, END)
174 | colored_names.append(name)
175 | colored_names = np.array(colored_names).reshape(-1, 1)
176 |
177 | table = np.hstack((colored_names, shapes, totals, capacity, img_coverage))
178 | if detailed:
179 | table = np.hstack((table, real_filters.astype(int),
180 | receptive_fields.astype(int)))
181 |
182 | return tabulate(table, header, floatfmt='.2f')
183 |
--------------------------------------------------------------------------------
/nolearn/lasagne/visualize.py:
--------------------------------------------------------------------------------
1 | from itertools import product
2 |
3 | from lasagne.layers import get_output
4 | from lasagne.layers import get_output_shape
5 | from lasagne.objectives import binary_crossentropy
6 | import matplotlib.pyplot as plt
7 | import numpy as np
8 | import theano
9 | import theano.tensor as T
10 | import io
11 | import lasagne
12 |
13 |
14 |
15 | def plot_loss(net):
16 | train_loss = [row['train_loss'] for row in net.train_history_]
17 | valid_loss = [row['valid_loss'] for row in net.train_history_]
18 | plt.plot(train_loss, label='train loss')
19 | plt.plot(valid_loss, label='valid loss')
20 | plt.xlabel('epoch')
21 | plt.ylabel('loss')
22 | plt.legend(loc='best')
23 | return plt
24 |
25 |
26 | def plot_conv_weights(layer, figsize=(6, 6)):
27 | """Plot the weights of a specific layer.
28 |
29 | Only really makes sense with convolutional layers.
30 |
31 | Parameters
32 | ----------
33 | layer : lasagne.layers.Layer
34 |
35 | """
36 | W = layer.W.get_value()
37 | shape = W.shape
38 | nrows = np.ceil(np.sqrt(shape[0])).astype(int)
39 | ncols = nrows
40 |
41 | for feature_map in range(shape[1]):
42 | figs, axes = plt.subplots(nrows, ncols, figsize=figsize, squeeze=False)
43 |
44 | for ax in axes.flatten():
45 | ax.set_xticks([])
46 | ax.set_yticks([])
47 | ax.axis('off')
48 |
49 | for i, (r, c) in enumerate(product(range(nrows), range(ncols))):
50 | if i >= shape[0]:
51 | break
52 | axes[r, c].imshow(W[i, feature_map], cmap='gray',
53 | interpolation='none')
54 | return plt
55 |
56 |
57 | def plot_conv_activity(layer, x, figsize=(6, 8)):
58 | """Plot the acitivities of a specific layer.
59 |
60 | Only really makes sense with layers that work 2D data (2D
61 | convolutional layers, 2D pooling layers ...).
62 |
63 | Parameters
64 | ----------
65 | layer : lasagne.layers.Layer
66 |
67 | x : numpy.ndarray
68 | Only takes one sample at a time, i.e. x.shape[0] == 1.
69 |
70 | """
71 | if x.shape[0] != 1:
72 | raise ValueError("Only one sample can be plotted at a time.")
73 |
74 | # compile theano function
75 | xs = T.tensor4('xs').astype(theano.config.floatX)
76 | get_activity = theano.function([xs], get_output(layer, xs))
77 |
78 | activity = get_activity(x)
79 | shape = activity.shape
80 | nrows = np.ceil(np.sqrt(shape[1])).astype(int)
81 | ncols = nrows
82 |
83 | figs, axes = plt.subplots(nrows + 1, ncols, figsize=figsize, squeeze=False)
84 | axes[0, ncols // 2].imshow(1 - x[0][0], cmap='gray',
85 | interpolation='none')
86 | axes[0, ncols // 2].set_title('original')
87 |
88 | for ax in axes.flatten():
89 | ax.set_xticks([])
90 | ax.set_yticks([])
91 | ax.axis('off')
92 |
93 | for i, (r, c) in enumerate(product(range(nrows), range(ncols))):
94 | if i >= shape[1]:
95 | break
96 | ndim = activity[0][i].ndim
97 | if ndim != 2:
98 | raise ValueError("Wrong number of dimensions, image data should "
99 | "have 2, instead got {}".format(ndim))
100 | axes[r + 1, c].imshow(-activity[0][i], cmap='gray',
101 | interpolation='none')
102 | return plt
103 |
104 |
105 | def occlusion_heatmap(net, x, target, square_length=7):
106 | """An occlusion test that checks an image for its critical parts.
107 |
108 | In this function, a square part of the image is occluded (i.e. set
109 | to 0) and then the net is tested for its propensity to predict the
110 | correct label. One should expect that this propensity shrinks of
111 | critical parts of the image are occluded. If not, this indicates
112 | overfitting.
113 |
114 | Depending on the depth of the net and the size of the image, this
115 | function may take awhile to finish, since one prediction for each
116 | pixel of the image is made.
117 |
118 | Currently, all color channels are occluded at the same time. Also,
119 | this does not really work if images are randomly distorted by the
120 | batch iterator.
121 |
122 | See paper: Zeiler, Fergus 2013
123 |
124 | Parameters
125 | ----------
126 | net : NeuralNet instance
127 | The neural net to test.
128 |
129 | x : np.array
130 | The input data, should be of shape (1, c, x, y). Only makes
131 | sense with image data.
132 |
133 | target : int
134 | The true value of the image. If the net makes several
135 | predictions, say 10 classes, this indicates which one to look
136 | at.
137 |
138 | square_length : int (default=7)
139 | The length of the side of the square that occludes the image.
140 | Must be an odd number.
141 |
142 | Results
143 | -------
144 | heat_array : np.array (with same size as image)
145 | An 2D np.array that at each point (i, j) contains the predicted
146 | probability of the correct class if the image is occluded by a
147 | square with center (i, j).
148 |
149 | """
150 | if (x.ndim != 4) or x.shape[0] != 1:
151 | raise ValueError("This function requires the input data to be of "
152 | "shape (1, c, x, y), instead got {}".format(x.shape))
153 | if square_length % 2 == 0:
154 | raise ValueError("Square length has to be an odd number, instead "
155 | "got {}.".format(square_length))
156 |
157 | num_classes = get_output_shape(net.layers_[-1])[1]
158 | img = x[0].copy()
159 | bs, col, s0, s1 = x.shape
160 |
161 | heat_array = np.zeros((s0, s1))
162 | pad = square_length // 2 + 1
163 | x_occluded = np.zeros((s1, col, s0, s1), dtype=img.dtype)
164 | probs = np.zeros((s0, s1, num_classes))
165 |
166 | # generate occluded images
167 | for i in range(s0):
168 | # batch s1 occluded images for faster prediction
169 | for j in range(s1):
170 | x_pad = np.pad(img, ((0, 0), (pad, pad), (pad, pad)), 'constant')
171 | x_pad[:, i:i + square_length, j:j + square_length] = 0.
172 | x_occluded[j] = x_pad[:, pad:-pad, pad:-pad]
173 | y_proba = net.predict_proba(x_occluded)
174 | probs[i] = y_proba.reshape(s1, num_classes)
175 |
176 | # from predicted probabilities, pick only those of target class
177 | for i in range(s0):
178 | for j in range(s1):
179 | heat_array[i, j] = probs[i, j, target]
180 | return heat_array
181 |
182 |
183 | def _plot_heat_map(net, X, figsize, get_heat_image):
184 | if (X.ndim != 4):
185 | raise ValueError("This function requires the input data to be of "
186 | "shape (b, c, x, y), instead got {}".format(X.shape))
187 |
188 | num_images = X.shape[0]
189 | if figsize[1] is None:
190 | figsize = (figsize[0], num_images * figsize[0] / 3)
191 | figs, axes = plt.subplots(num_images, 3, figsize=figsize)
192 |
193 | for ax in axes.flatten():
194 | ax.set_xticks([])
195 | ax.set_yticks([])
196 | ax.axis('off')
197 |
198 | for n in range(num_images):
199 | heat_img = get_heat_image(net, X[n:n + 1, :, :, :], n)
200 |
201 | ax = axes if num_images == 1 else axes[n]
202 | img = X[n, :, :, :].mean(0)
203 | ax[0].imshow(-img, interpolation='nearest', cmap='gray')
204 | ax[0].set_title('image')
205 | ax[1].imshow(-heat_img, interpolation='nearest', cmap='Reds')
206 | ax[1].set_title('critical parts')
207 | ax[2].imshow(-img, interpolation='nearest', cmap='gray')
208 | ax[2].imshow(-heat_img, interpolation='nearest', cmap='Reds',
209 | alpha=0.6)
210 | ax[2].set_title('super-imposed')
211 | return plt
212 |
213 |
214 | def plot_occlusion(net, X, target, square_length=7, figsize=(9, None)):
215 | """Plot which parts of an image are particularly import for the
216 | net to classify the image correctly.
217 |
218 | See paper: Zeiler, Fergus 2013
219 |
220 | Parameters
221 | ----------
222 | net : NeuralNet instance
223 | The neural net to test.
224 |
225 | X : numpy.array
226 | The input data, should be of shape (b, c, 0, 1). Only makes
227 | sense with image data.
228 |
229 | target : list or numpy.array of ints
230 | The true values of the image. If the net makes several
231 | predictions, say 10 classes, this indicates which one to look
232 | at. If more than one sample is passed to X, each of them needs
233 | its own target.
234 |
235 | square_length : int (default=7)
236 | The length of the side of the square that occludes the image.
237 | Must be an odd number.
238 |
239 | figsize : tuple (int, int)
240 | Size of the figure.
241 |
242 | Plots
243 | -----
244 | Figure with 3 subplots: the original image, the occlusion heatmap,
245 | and both images super-imposed.
246 |
247 | """
248 | return _plot_heat_map(
249 | net, X, figsize, lambda net, X, n: occlusion_heatmap(
250 | net, X, target[n], square_length))
251 |
252 |
253 | def saliency_map(input, output, pred, X):
254 | score = -binary_crossentropy(output[:, pred], np.array([1])).sum()
255 | return np.abs(T.grad(score, input).eval({input: X}))
256 |
257 |
258 | def saliency_map_net(net, X):
259 | input = net.layers_[0].input_var
260 | output = get_output(net.layers_[-1])
261 | pred = output.eval({input: X}).argmax(axis=1)
262 | return saliency_map(input, output, pred, X)[0].transpose(1, 2, 0).squeeze()
263 |
264 |
265 | def plot_saliency(net, X, figsize=(9, None)):
266 | return _plot_heat_map(
267 | net, X, figsize, lambda net, X, n: -saliency_map_net(net, X))
268 |
269 |
270 | def get_hex_color(layer_type):
271 | """
272 | Determines the hex color for a layer.
273 | :parameters:
274 | - layer_type : string
275 | Class name of the layer
276 | :returns:
277 | - color : string containing a hex color for filling block.
278 | """
279 | COLORS = ['#4A88B3', '#98C1DE', '#6CA2C8', '#3173A2', '#17649B',
280 | '#FFBB60', '#FFDAA9', '#FFC981', '#FCAC41', '#F29416',
281 | '#C54AAA', '#E698D4', '#D56CBE', '#B72F99', '#B0108D',
282 | '#75DF54', '#B3F1A0', '#91E875', '#5DD637', '#3FCD12']
283 |
284 | hashed = int(hash(layer_type)) % 5
285 |
286 | if "conv" in layer_type.lower():
287 | return COLORS[:5][hashed]
288 | if layer_type in lasagne.layers.pool.__all__:
289 | return COLORS[5:10][hashed]
290 | if layer_type in lasagne.layers.recurrent.__all__:
291 | return COLORS[10:15][hashed]
292 | else:
293 | return COLORS[15:20][hashed]
294 |
295 |
296 | def make_pydot_graph(layers, output_shape=True, verbose=False):
297 | """
298 | :parameters:
299 | - layers : list
300 | List of the layers, as obtained from lasagne.layers.get_all_layers
301 | - output_shape: (default `True`)
302 | If `True`, the output shape of each layer will be displayed.
303 | - verbose: (default `False`)
304 | If `True`, layer attributes like filter shape, stride, etc.
305 | will be displayed.
306 | :returns:
307 | - pydot_graph : PyDot object containing the graph
308 | """
309 | import pydotplus as pydot
310 | pydot_graph = pydot.Dot('Network', graph_type='digraph')
311 | pydot_nodes = {}
312 | pydot_edges = []
313 | for i, layer in enumerate(layers):
314 | layer_name = getattr(layer, 'name', None)
315 | if layer_name is None:
316 | layer_name = layer.__class__.__name__
317 | layer_type = '{0}'.format(layer_name)
318 | key = repr(layer)
319 | label = layer_type
320 | color = get_hex_color(layer_type)
321 | if verbose:
322 | for attr in ['num_filters', 'num_units', 'ds',
323 | 'filter_shape', 'stride', 'strides', 'p']:
324 | if hasattr(layer, attr):
325 | label += '\n{0}: {1}'.format(attr, getattr(layer, attr))
326 | if hasattr(layer, 'nonlinearity'):
327 | try:
328 | nonlinearity = layer.nonlinearity.__name__
329 | except AttributeError:
330 | nonlinearity = layer.nonlinearity.__class__.__name__
331 | label += '\nnonlinearity: {0}'.format(nonlinearity)
332 |
333 | if output_shape:
334 | label += '\nOutput shape: {0}'.format(layer.output_shape)
335 |
336 | pydot_nodes[key] = pydot.Node(
337 | key, label=label, shape='record', fillcolor=color, style='filled')
338 |
339 | if hasattr(layer, 'input_layers'):
340 | for input_layer in layer.input_layers:
341 | pydot_edges.append([repr(input_layer), key])
342 |
343 | if hasattr(layer, 'input_layer'):
344 | pydot_edges.append([repr(layer.input_layer), key])
345 |
346 | for node in pydot_nodes.values():
347 | pydot_graph.add_node(node)
348 |
349 | for edges in pydot_edges:
350 | pydot_graph.add_edge(
351 | pydot.Edge(pydot_nodes[edges[0]], pydot_nodes[edges[1]]))
352 | return pydot_graph
353 |
354 |
355 | def draw_to_file(layers, filename, **kwargs):
356 | """
357 | Draws a network diagram to a file
358 | :parameters:
359 | - layers : list or NeuralNet instance
360 | List of layers or the neural net to draw.
361 | - filename : string
362 | The filename to save output to
363 | - **kwargs: see docstring of make_pydot_graph for other options
364 | """
365 | layers = (layers.get_all_layers() if hasattr(layers, 'get_all_layers')
366 | else layers)
367 | dot = make_pydot_graph(layers, **kwargs)
368 | ext = filename[filename.rfind('.') + 1:]
369 | with io.open(filename, 'wb') as fid:
370 | fid.write(dot.create(format=ext))
371 |
372 |
373 | def draw_to_notebook(layers, **kwargs):
374 | """
375 | Draws a network diagram in an IPython notebook
376 | :parameters:
377 | - layers : list or NeuralNet instance
378 | List of layers or the neural net to draw.
379 | - **kwargs : see the docstring of make_pydot_graph for other options
380 | """
381 | from IPython.display import Image
382 | layers = (layers.get_all_layers() if hasattr(layers, 'get_all_layers')
383 | else layers)
384 | dot = make_pydot_graph(layers, **kwargs)
385 | return Image(dot.create_png())
386 |
--------------------------------------------------------------------------------
/nolearn/metrics.py:
--------------------------------------------------------------------------------
1 | from __future__ import print_function
2 |
3 | import numpy as np
4 | from sklearn.base import clone
5 | from sklearn.metrics import f1_score
6 |
7 |
8 | def multiclass_logloss(actual, predicted, eps=1e-15):
9 | """Multi class version of Logarithmic Loss metric.
10 |
11 | :param actual: Array containing the actual target classes
12 | :param predicted: Matrix with class predictions, one probability per class
13 | """
14 | # Convert 'actual' to a binary array if it's not already:
15 | if len(actual.shape) == 1:
16 | actual2 = np.zeros((actual.shape[0], predicted.shape[1]))
17 | for i, val in enumerate(actual):
18 | actual2[i, val] = 1
19 | actual = actual2
20 |
21 | clip = np.clip(predicted, eps, 1 - eps)
22 | rows = actual.shape[0]
23 | vsota = np.sum(actual * np.log(clip))
24 | return -1.0 / rows * vsota
25 |
26 |
27 | class LearningCurve(object):
28 | score_func = staticmethod(f1_score)
29 |
30 | def __init__(self, score_func=None):
31 | if score_func is None:
32 | score_func = self.score_func
33 | self.score_func = score_func
34 |
35 | def predict(self, clf, X):
36 | return clf.predict(X)
37 |
38 | def __call__(self, dataset, classifier, steps=10,
39 | verbose=0, random_state=42):
40 | """Create a learning curve that uses more training cases with
41 | each step.
42 |
43 | :param dataset: Dataset to work with
44 | :type dataset: :class:`~nolearn.dataset.Dataset`
45 | :param classifier: Classifier for fitting and making predictions.
46 | :type classifier: :class:`~sklearn.base.BaseEstimator`
47 | :param steps: Number of steps in the learning curve.
48 | :type steps: int
49 |
50 | :result: 3-tuple with lists `scores_train`, `scores_test`, `sizes`
51 |
52 | Drawing the resulting learning curve can be done like this:
53 |
54 | .. code-block:: python
55 |
56 | dataset = Dataset()
57 | clf = LogisticRegression()
58 | scores_train, scores_test, sizes = learning_curve(dataset, clf)
59 | pl.plot(sizes, scores_train, 'b', label='training set')
60 | pl.plot(sizes, scores_test, 'r', label='test set')
61 | pl.legend(loc='lower right')
62 | pl.show()
63 | """
64 | X_train, X_test, y_train, y_test = dataset.train_test_split()
65 |
66 | scores_train = []
67 | scores_test = []
68 | sizes = []
69 |
70 | if verbose:
71 | print(" n train test")
72 |
73 | for frac in np.linspace(0.1, 1.0, num=steps):
74 | frac_size = int(X_train.shape[0] * frac)
75 | sizes.append(frac_size)
76 | X_train1 = X_train[:frac_size]
77 | y_train1 = y_train[:frac_size]
78 |
79 | clf = clone(classifier)
80 | clf.fit(X_train1, y_train1)
81 |
82 | predict_train = self.predict(clf, X_train1)
83 | predict_test = self.predict(clf, X_test)
84 |
85 | score_train = self.score_func(y_train1, predict_train)
86 | score_test = self.score_func(y_test, predict_test)
87 |
88 | scores_train.append(score_train)
89 | scores_test.append(score_test)
90 |
91 | if verbose:
92 | print(" %8d %0.4f %0.4f" % (
93 | frac_size, score_train, score_test))
94 |
95 | return scores_train, scores_test, sizes
96 |
97 |
98 | class LearningCurveProbas(LearningCurve):
99 | score_func = staticmethod(multiclass_logloss)
100 |
101 | def predict(self, clf, X):
102 | return clf.predict_proba(X)
103 |
104 | learning_curve = LearningCurve().__call__
105 | #: Same as :func:`learning_curve` but uses :func:`multiclass_logloss`
106 | #: as the loss funtion.
107 | learning_curve_logloss = LearningCurveProbas().__call__
108 |
--------------------------------------------------------------------------------
/nolearn/overfeat.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 |
3 | import subprocess
4 |
5 | import Image
6 | import ImageOps
7 | import numpy as np
8 |
9 | from nolearn import cache
10 | from sklearn.base import BaseEstimator
11 |
12 | from .util import ChunkedTransform
13 |
14 |
15 | def _overfeat_cache_key(self, images):
16 | if len(images) == 1:
17 | raise cache.DontCache
18 | if isinstance(images[0], Image.Image):
19 | images = [im.filename for im in images]
20 | return ','.join([
21 | str(images),
22 | str(self.feature_layer),
23 | str(self.network_size),
24 | str(self.pretrained_params),
25 | ])
26 |
27 |
28 | class OverFeatShell(ChunkedTransform, BaseEstimator):
29 | """Extract features from images using a pretrained ConvNet.
30 |
31 | Uses the executable from the OverFeat library by Sermanet et al.
32 | Please make sure you read and accept OverFeat's license before you
33 | use this software.
34 | """
35 |
36 | def __init__(
37 | self,
38 | feature_layer=21,
39 | overfeat_bin='overfeat', # or 'overfeat_cuda'
40 | pretrained_params=None,
41 | network_size=0,
42 | merge='maxmean',
43 | batch_size=200,
44 | verbose=0,
45 | ):
46 | """
47 | :param feature_layer: The ConvNet layer that's used for
48 | feature extraction. Defaults to layer
49 | `21`.
50 |
51 | :param overfeat_bin: The path to the `overfeat` binary.
52 |
53 | :param pretrained_params: The path to the pretrained
54 | parameters file. These files come
55 | with the overfeat distribution and
56 | can be found in `overfeat/data`.
57 |
58 | :param network_size: Use the small (0) or large network (1).
59 |
60 | :param merge: How spatial features are merged. May be one of
61 | 'maxmean', 'meanmax' or a callable.
62 | """
63 | self.feature_layer = feature_layer
64 | self.overfeat_bin = overfeat_bin
65 | self.pretrained_params = pretrained_params
66 | self.network_size = network_size
67 | self.merge = merge
68 | self.batch_size = batch_size
69 | self.verbose = verbose
70 |
71 | def fit(self, X=None, y=None):
72 | return self
73 |
74 | @cache.cached(_overfeat_cache_key)
75 | def _call_overfeat(self, fnames):
76 | cmd = [
77 | self.overfeat_bin,
78 | '-L', str(self.feature_layer),
79 | ]
80 | if self.network_size:
81 | cmd += ['-l']
82 | if self.pretrained_params:
83 | cmd += ['-d', self.pretrained_params]
84 | cmd += ["'{0}'".format(fn) for fn in fnames]
85 |
86 | def _call(cmd):
87 | out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
88 | if out == '':
89 | raise RuntimeError("Call failed; try lower 'batch_size'")
90 | elif ("unable" in out or
91 | "Invalid" in out or
92 | "error" in out or
93 | "Assertion" in out):
94 | raise RuntimeError("\n%s ... %s\n\n%s" % (
95 | out[:250], out[-250:], list(fnames)))
96 | return out
97 |
98 | try:
99 | output = _call(cmd)
100 | except RuntimeError:
101 | try:
102 | output = _call(cmd)
103 | except RuntimeError:
104 | raise
105 |
106 | return output.splitlines()
107 |
108 | def _compute_features(self, fnames):
109 | data = self._call_overfeat(fnames)
110 |
111 | features = []
112 | for i in range(len(data) / 2):
113 | n_feat, n_rows, n_cols = data[i * 2].split()
114 | n_feat, n_rows, n_cols = int(n_feat), int(n_rows), int(n_cols)
115 | feat = np.fromstring(data[i * 2 + 1], dtype=np.float32, sep=' ')
116 | feat = feat.reshape(n_feat, n_rows, n_cols)
117 | if self.merge == 'maxmean':
118 | feat = feat.max(2).mean(1)
119 | elif self.merge == 'meanmax':
120 | feat = feat.mean(2).max(1)
121 | else:
122 | feat = self.merge(feat)
123 | features.append(feat)
124 |
125 | return np.vstack(features)
126 |
127 |
128 | OverFeat = OverFeatShell # BBB
129 |
130 |
131 | class OverFeatPy(ChunkedTransform, BaseEstimator):
132 | """Extract features from images using a pretrained ConvNet.
133 |
134 | Uses the Python API from the OverFeat library by Sermanet et al.
135 | Please make sure you read and accept OverFeat's license before you
136 | use this software.
137 | """
138 |
139 | kernel_size = 231
140 |
141 | def __init__(
142 | self,
143 | feature_layer=21,
144 | pretrained_params='net_weight_0',
145 | network_size=None,
146 | merge='maxmean',
147 | batch_size=200,
148 | verbose=0,
149 | ):
150 | """
151 | :param feature_layer: The ConvNet layer that's used for
152 | feature extraction. Defaults to layer
153 | `21`. Please refer to `this post
154 | `_
155 | to find out which layers are available
156 | for the two different networks.
157 |
158 | :param pretrained_params: The path to the pretrained
159 | parameters file. These files come
160 | with the overfeat distribution and
161 | can be found in `overfeat/data`.
162 |
163 | :param merge: How spatial features are merged. May be one of
164 | 'maxmean', 'meanmax' or a callable.
165 | """
166 | if network_size is None:
167 | network_size = int(pretrained_params[-1])
168 | self.feature_layer = feature_layer
169 | self.pretrained_params = pretrained_params
170 | self.network_size = network_size
171 | self.merge = merge
172 | self.batch_size = batch_size
173 | self.verbose = verbose
174 |
175 | def fit(self, X=None, y=None):
176 | import overfeat # soft dep
177 | overfeat.init(self.pretrained_params, self.network_size)
178 | return self
179 |
180 | @classmethod
181 | def prepare_image(cls, image):
182 | if isinstance(image, str):
183 | image = Image.open(image)
184 |
185 | if isinstance(image, Image.Image):
186 | if (image.size[0] < cls.kernel_size or
187 | image.size[1] < cls.kernel_size):
188 | image = ImageOps.fit(image, (cls.kernel_size, cls.kernel_size))
189 | image = np.array(image)
190 |
191 | image = image.swapaxes(1, 2).swapaxes(0, 1).astype(np.float32)
192 | return image
193 |
194 | @cache.cached(_overfeat_cache_key)
195 | def _compute_features(self, images):
196 | import overfeat # soft dep
197 |
198 | features = []
199 | for image in images:
200 | image = self.prepare_image(image)
201 | overfeat.fprop(image)
202 | feat = overfeat.get_output(self.feature_layer)
203 | if self.merge == 'maxmean':
204 | feat = feat.max(2).mean(1)
205 | elif self.merge == 'meanmax':
206 | feat = feat.mean(2).max(1)
207 | else:
208 | feat = self.merge(feat)
209 | features.append(feat)
210 | return np.vstack(features)
211 |
212 | def __getstate__(self):
213 | return self.__dict__.copy()
214 |
215 | def __setstate__(self, state):
216 | self.__dict__.update(state)
217 | self.fit()
218 |
--------------------------------------------------------------------------------
/nolearn/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dnouri/nolearn/8f4473190a06caf80878821d29178b82329d8bee/nolearn/tests/__init__.py
--------------------------------------------------------------------------------
/nolearn/tests/conftest.py:
--------------------------------------------------------------------------------
1 | def pytest_configure(config):
2 | # Make matplotlib happy when running without an X display:
3 | import matplotlib
4 | matplotlib.use('Agg')
5 |
--------------------------------------------------------------------------------
/nolearn/tests/test_cache.py:
--------------------------------------------------------------------------------
1 | from mock import patch
2 |
3 |
4 | def test_cached(tmpdir):
5 | from ..cache import cached
6 |
7 | called = []
8 |
9 | @cached(cache_path=str(tmpdir))
10 | def add(one, two):
11 | called.append([one, two])
12 | return one + two
13 |
14 | assert add(2, 3) == 5
15 | assert len(called) == 1
16 | assert add(2, 3) == 5
17 | assert len(called) == 1
18 | assert add(2, 4) == 6
19 | assert len(called) == 2
20 |
21 |
22 | def test_cache_with_cache_key(tmpdir):
23 | from ..cache import cached
24 |
25 | called = []
26 |
27 | def cache_key(one, two):
28 | return one
29 |
30 | @cached(cache_key=cache_key, cache_path=str(tmpdir))
31 | def add(one, two):
32 | called.append([one, two])
33 | return one + two
34 |
35 | assert add(2, 3) == 5
36 | assert len(called) == 1
37 | assert add(2, 3) == 5
38 | assert len(called) == 1
39 | assert add(2, 4) == 5
40 | assert len(called) == 1
41 |
42 |
43 | def test_cache_systemerror(tmpdir):
44 | from ..cache import cached
45 |
46 | @cached(cache_path=str(tmpdir))
47 | def add(one, two):
48 | return one + two
49 |
50 | with patch('nolearn.cache.numpy_pickle.dump') as dump:
51 | dump.side_effect = SystemError()
52 | assert add(2, 3) == 5
53 |
--------------------------------------------------------------------------------
/nolearn/tests/test_grid_search.py:
--------------------------------------------------------------------------------
1 | from sklearn.linear_model import LogisticRegression
2 | from sklearn.grid_search import GridSearchCV
3 | from mock import Mock
4 | import numpy as np
5 |
6 |
7 | def test_grid_search():
8 | from ..grid_search import grid_search
9 |
10 | dataset = Mock()
11 | dataset.data = np.array([[1, 2, 3], [3, 3, 3]] * 20)
12 | dataset.target = np.array([0, 1] * 20)
13 | pipeline = LogisticRegression()
14 | parameters = dict(C=[1.0, 3.0])
15 |
16 | result = grid_search(dataset, pipeline, parameters)
17 | assert isinstance(result, GridSearchCV)
18 | assert hasattr(result, 'best_estimator_')
19 | assert hasattr(result, 'best_score_')
20 |
--------------------------------------------------------------------------------
/nolearn/tests/test_inischema.py:
--------------------------------------------------------------------------------
1 | SAMPLE_SCHEMA = """
2 | [first]
3 | value1 = int
4 | value2 = string
5 | value3 = float
6 | value4 = listofstrings
7 | value5 = listofints
8 |
9 | [second]
10 | value1 = string
11 | value2 = int
12 | """
13 |
14 | SAMPLE_CONFIGURATION = """
15 | [first]
16 | value1 = 3
17 | value2 = Three
18 | value3 = 3.0
19 | value4 = Three Drei
20 | value5 = 3 3
21 |
22 | [second]
23 | value1 =
24 | a few line breaks
25 | are no problem
26 | neither is a missing value2
27 | """
28 |
29 |
30 | def test_parse_config():
31 | from ..inischema import parse_config
32 | result = parse_config(SAMPLE_SCHEMA, SAMPLE_CONFIGURATION)
33 | assert result['first']['value1'] == 3
34 | assert result['first']['value2'] == u'Three'
35 | assert result['first']['value3'] == 3.0
36 | assert result['first']['value4'] == [u'Three', u'Drei']
37 | assert result['first']['value5'] == [3, 3]
38 | assert result['second']['value1'] == (
39 | u'a few line breaks\nare no problem\nneither is a missing value2')
40 | assert 'value2' not in result['second']
41 |
--------------------------------------------------------------------------------
/nolearn/tests/test_metrics.py:
--------------------------------------------------------------------------------
1 | from mock import Mock
2 | import numpy as np
3 | from sklearn.cross_validation import train_test_split
4 | from sklearn.linear_model import LogisticRegression
5 |
6 |
7 | def test_multiclass_logloss():
8 | from ..metrics import multiclass_logloss
9 |
10 | act = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]])
11 | pred = np.array([[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]])
12 |
13 | result = multiclass_logloss(act, pred)
14 | assert result == 0.69049112401021973
15 |
16 |
17 | def test_multiclass_logloss_actual_conversion():
18 | from ..metrics import multiclass_logloss
19 |
20 | act = np.array([1, 0, 2])
21 | pred = np.array([[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]])
22 |
23 | result = multiclass_logloss(act, pred)
24 | assert result == 0.69049112401021973
25 |
26 |
27 | def _learning_curve(learning_curve):
28 | X = np.array([[0, 1], [1, 0], [1, 1]] * 100, dtype=float)
29 | X[:, 1] += (np.random.random((300)) - 0.5)
30 | y = np.array([0, 0, 1] * 100)
31 |
32 | dataset = Mock()
33 | dataset.train_test_split.return_value = train_test_split(X, y)
34 | dataset.data = X
35 | dataset.target = y
36 |
37 | return learning_curve(dataset, LogisticRegression(), steps=5, verbose=1)
38 |
39 |
40 | def test_learning_curve():
41 | from ..metrics import learning_curve
42 |
43 | scores_train, scores_test, sizes = _learning_curve(learning_curve)
44 | assert len(scores_train) == 5
45 | assert len(scores_test) == 5
46 | assert sizes[0] == 22
47 | assert sizes[-1] == 225.0
48 |
49 |
50 | def test_learning_curve_logloss():
51 | from ..metrics import learning_curve_logloss
52 |
53 | scores_train, scores_test, sizes = _learning_curve(learning_curve_logloss)
54 | assert len(scores_train) == 5
55 | assert len(scores_test) == 5
56 | assert sizes[0] == 22
57 | assert sizes[-1] == 225.0
58 |
--------------------------------------------------------------------------------
/nolearn/util.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | import numpy as np
4 |
5 |
6 | def chunks(l, n):
7 | """ Yield successive n-sized chunks from l.
8 | """
9 | for i in xrange(0, len(l), n):
10 | yield l[i:i + n]
11 |
12 |
13 | class ChunkedTransform(object):
14 | verbose = 0
15 |
16 | def transform(self, X):
17 | features = None
18 | for chunk in chunks(X, self.batch_size):
19 | if features is not None:
20 | features = np.vstack(
21 | [features, self._compute_features(chunk)])
22 | else:
23 | features = self._compute_features(chunk)
24 | if self.verbose:
25 | sys.stdout.write(
26 | "\r[%s] %d%%" % (
27 | self.__class__.__name__,
28 | 100. * len(features) / len(X),
29 | ))
30 | sys.stdout.flush()
31 | if self.verbose:
32 | sys.stdout.write('\n')
33 | return features
34 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy==1.10.4
2 | scipy==0.16.1
3 | Theano==0.8
4 | -e git+https://github.com/Lasagne/Lasagne.git@8f4f9b2#egg=Lasagne==0.2.git
5 | joblib==0.9.3
6 | scikit-learn==0.17
7 | tabulate==0.7.5
8 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [tool:pytest]
2 | addopts =
3 | --doctest-modules
4 | --cov=nolearn --cov-report=term-missing
5 | --flakes
6 | --pep8
7 | -v
8 | nolearn/tests/ nolearn/lasagne/tests/
9 | python_files = test*py
10 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | import codecs
3 |
4 | from setuptools import setup, find_packages
5 |
6 | version = '0.6.2.dev0'
7 |
8 | here = os.path.abspath(os.path.dirname(__file__))
9 | try:
10 | README = codecs.open(os.path.join(here, 'README.rst'),
11 | encoding='utf-8').read()
12 | CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()
13 | except IOError:
14 | README = CHANGES = ''
15 |
16 |
17 | install_requires = [
18 | 'joblib',
19 | 'scikit-learn',
20 | 'tabulate',
21 | 'Lasagne',
22 | 'Theano',
23 | ]
24 |
25 | visualization_require = [
26 | 'matplotlib<3.0.999',
27 | 'pydotplus',
28 | 'ipython<5.999'
29 | ]
30 |
31 | tests_require = [
32 | 'mock',
33 | 'pytest',
34 | 'pytest-cov',
35 | 'pytest-flakes',
36 | 'pytest-pep8',
37 | ]
38 |
39 | docs_require = [
40 | 'Sphinx<1.999',
41 | ]
42 |
43 | gdbn_require = [
44 | "gdbn"
45 | ]
46 |
47 | all_require = (visualization_require + tests_require + docs_require + gdbn_require)
48 |
49 | setup(name='nolearn',
50 | version=version,
51 | description="scikit-learn compatible neural network library",
52 | long_description='\n\n'.join([README, CHANGES]),
53 | classifiers=[
54 | "Development Status :: 4 - Beta",
55 | "Programming Language :: Python :: 2.7",
56 | "Programming Language :: Python :: 3.4",
57 | ],
58 | keywords='',
59 | author='Daniel Nouri',
60 | author_email='daniel.nouri@gmail.com',
61 | url='https://github.com/dnouri/nolearn',
62 | license='MIT',
63 | packages=find_packages(),
64 | include_package_data=True,
65 | zip_safe=False,
66 | install_requires=install_requires,
67 | extras_require={
68 | 'visualization': visualization_require,
69 | 'testing': tests_require,
70 | 'docs': docs_require,
71 | 'gdbn': gdbn_require,
72 | 'all': all_require,
73 | },
74 | )
75 |
--------------------------------------------------------------------------------