├── .gitignore ├── .travis.yml ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── brainstorm ├── __about__.py ├── __init__.py ├── all.py ├── data_iterators.py ├── describable.py ├── handlers │ ├── __init__.py │ ├── _cpuop.pyx │ ├── base_handler.py │ ├── debug_handler.py │ ├── numpy_handler.py │ └── pycuda_handler.py ├── hooks.py ├── initializers.py ├── layers │ ├── __init__.py │ ├── base_layer.py │ ├── batch_normalization_layer.py │ ├── binomial_cross_entropy_layer.py │ ├── clockwork_layer.py │ ├── clockwork_lstm_layer.py │ ├── convolution_layer_2d.py │ ├── deltas_scaling_layer.py │ ├── dropout_layer.py │ ├── elementwise_layer.py │ ├── fully_connected_layer.py │ ├── highway_layer.py │ ├── input_layer.py │ ├── l1_decay.py │ ├── l2_decay.py │ ├── loss_layer.py │ ├── lstm_layer.py │ ├── mask_layer.py │ ├── merge_layer.py │ ├── noop_layer.py │ ├── pooling_layer_2d.py │ ├── recurrent_layer.py │ ├── sigmoid_ce_layer.py │ ├── softmax_ce_layer.py │ ├── squared_difference_layer.py │ └── squared_error_layer.py ├── optional.py ├── randomness.py ├── scorers.py ├── structure │ ├── __init__.py │ ├── architecture.py │ ├── buffer_structure.py │ ├── buffer_views.py │ ├── buffers.py │ ├── construction.py │ ├── layout.py │ ├── network.py │ └── view_references.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── helpers.py │ ├── test_array.py │ ├── test_data_iterators.py │ ├── test_describable.py │ ├── test_finite_differences │ │ ├── __init__.py │ │ └── test_deltas.py │ ├── test_handler_operations.py │ ├── test_handlers_against_reference.py │ ├── test_initializers.py │ ├── test_integration │ │ ├── __init__.py │ │ ├── test_initialization.py │ │ └── test_xor.py │ ├── test_layers.py │ ├── test_network.py │ ├── test_randomness.py │ ├── test_schedules.py │ ├── test_scorers.py │ ├── test_structure │ │ ├── __init__.py │ │ ├── test_architecture.py │ │ ├── test_buffer_structure.py │ │ ├── test_buffer_view.py │ │ ├── test_construction_layer.py │ │ ├── test_generate_architecture.py │ │ ├── test_layout.py │ │ └── test_view_references.py │ ├── test_uniquely_named.py │ ├── test_utils.py │ ├── test_weight_modifiers.py │ └── tools.py ├── tools.py ├── training │ ├── __init__.py │ ├── schedules.py │ ├── steppers.py │ ├── trainer.py │ └── utils.py ├── utils.py └── value_modifiers.py ├── data ├── create_cifar10.py ├── create_cifar100.py ├── create_hutter.py └── create_mnist.py ├── doc_requirements.txt ├── docs ├── Makefile ├── apidoc.rst ├── conf.py ├── contributing.rst ├── credits.rst ├── data_format.rst ├── examples.rst ├── history.rst ├── hooks.rst ├── index.rst ├── installation.rst ├── internals.rst ├── layers.rst ├── make.bat ├── network.rst ├── quickstart.rst ├── trainer.rst └── walkthrough.rst ├── examples ├── README.md ├── cifar10_cnn.py ├── custom_layer.py ├── hutter_lstm.py └── mnist_pi.py ├── pycuda_requirements.txt ├── requirements.txt ├── setup.cfg ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io 2 | 3 | *.hdf5 4 | *.gz 5 | *.npy 6 | 7 | ### Python ### 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | 12 | 13 | # Distribution / packaging 14 | .Python 15 | env/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | 48 | # Sphinx documentation 49 | docs/_build/ 50 | 51 | # PyBuilder 52 | target/ 53 | 54 | 55 | ### IPythonNotebook ### 56 | # Temporary data 57 | .ipynb_checkpoints/ 58 | 59 | 60 | ### PyCharm ### 61 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 62 | 63 | /*.iml 64 | 65 | ## Directory-based project format: 66 | .idea/ 67 | 68 | ## File-based project format: 69 | *.ipr 70 | *.iws 71 | 72 | ## Plugin-specific files: 73 | 74 | # IntelliJ 75 | out/ 76 | 77 | # mpeltonen/sbt-idea plugin 78 | .idea_modules/ 79 | 80 | # JIRA plugin 81 | atlassian-ide-plugin.xml 82 | 83 | # Crashlytics plugin (for Android Studio and IntelliJ) 84 | com_crashlytics_export_strings.xml 85 | 86 | 87 | ### Linux ### 88 | *~ 89 | 90 | # KDE directory preferences 91 | .directory 92 | 93 | 94 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | # Setup anaconda 6 | before_install: 7 | - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh 8 | - chmod +x miniconda.sh 9 | - ./miniconda.sh -b 10 | - export PATH=/home/travis/miniconda/bin:$PATH 11 | - conda update --yes conda 12 | # Install packages 13 | install: 14 | - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy 15 | - pip install wheel six mock pytest pytest-pep8 16 | 17 | # Run test 18 | script: 19 | - py.test $PYTEST_ARGS brainstorm 20 | 21 | matrix: 22 | include: 23 | - python: "2.7" 24 | env: PYTEST_ARGS= 25 | - python: "3.3" 26 | env: PYTEST_ARGS= 27 | - python: "3.4" 28 | env: PYTEST_ARGS= 29 | - python: "2.7" 30 | env: PYTEST_ARGS="--pep8 -m pep8" 31 | 32 | branches: 33 | only: 34 | - master 35 | - /^release.*$/ -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/IDSIA/brainstorm/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whoever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whoever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | brainstorm could always use more documentation, whether as part of the 40 | official brainstorm docs, in docstrings, or even on the web in blog posts, 41 | articles, and such. 42 | 43 | Submit Feedback 44 | ~~~~~~~~~~~~~~~ 45 | 46 | The best way to send feedback is to file an issue at https://github.com/IDSIA/brainstorm/issues. 47 | 48 | If you are proposing a feature: 49 | 50 | * Explain in detail how it would work. 51 | * Keep the scope as narrow as possible, to make it easier to implement. 52 | * Remember that this is a volunteer-driven project, and that contributions 53 | are welcome :) 54 | 55 | Get Started! 56 | ------------ 57 | 58 | Ready to contribute? Here's how to set up `brainstorm` for 59 | local development. 60 | 61 | 1. Fork_ the `brainstorm` repo on GitHub. 62 | 2. Clone your fork locally:: 63 | 64 | $ git clone git@github.com:your_name_here/brainstorm.git 65 | 66 | 3. Create a branch for local development:: 67 | 68 | $ git checkout -b name-of-your-bugfix-or-feature 69 | 70 | Now you can make your changes locally. 71 | 72 | 4. When you're done making changes, check that your changes pass style and unit 73 | tests, including testing other Python versions with tox:: 74 | 75 | $ tox 76 | 77 | To get tox, just pip install it. 78 | 79 | 5. Commit your changes and push your branch to GitHub:: 80 | 81 | $ git add . 82 | $ git commit -m "Your detailed description of your changes." 83 | $ git push origin name-of-your-bugfix-or-feature 84 | 85 | 6. Submit a pull request through the GitHub website. 86 | 87 | .. _Fork: https://github.com/IDSIA/brainstorm/fork 88 | 89 | Pull Request Guidelines 90 | ----------------------- 91 | 92 | Before you submit a pull request, check that it meets these guidelines: 93 | 94 | 1. The pull request should include tests. 95 | 2. If the pull request adds functionality, the docs should be updated. Put 96 | your new functionality into a function with a docstring, and add the 97 | feature to the list in README.rst. 98 | 3. The pull request should work for Python 2.6, 2.7, and 3.3, and for PyPy. 99 | Check https://travis-ci.org/IDSIA/brainstorm 100 | under pull requests for active pull requests or run the ``tox`` command and 101 | make sure that the tests pass for all supported Python versions. 102 | 103 | 104 | Tips 105 | ---- 106 | 107 | To run a subset of tests:: 108 | 109 | $ py.test test/test_brainstorm.py 110 | 111 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | 0.5 (2015-12-01) 7 | ++++++++++++++++ 8 | Changed Behaviour 9 | """"""""""""""""" 10 | * examples now run on CPU by default 11 | * added ``brainstorm.tools.shuffle_data`` and ``brainstorm.tools.split`` to help with data preparation 12 | * ``SigmoidCE`` and ``SquaredDifference`` layers now outputs a loss for each dimension instead of summing over features. 13 | * ``SquaredDifference`` layer does no longer scale by one half. 14 | * Added a ``SquaredLoss`` layer that computes half the squared difference and 15 | has an interface that is compatible with the ``SigmoidCE`` and ``SigmoidCE`` layers. 16 | * Output `probabilities` renamed to `predictions` in ``SigmoidCE`` and ``SigmoidCE`` layers. 17 | 18 | New Features 19 | """""""""""" 20 | * added a `use_conv` option to ``brainstorm.tools.create_net_from_spec`` 21 | * added `criterion` option to ``brainstorm.hooks.EarlyStopper`` hook 22 | * added ``brainstorm.tools.get_network_info`` function that returns information 23 | about the network as a string 24 | * added ``brainstorm.tools.extract`` function that applies a network to some 25 | data and saves a set of requested buffers. 26 | * ``brainstorm.layers.mask`` layer now supports masking individual features 27 | * added ``brainstorm.hooks.StopAfterThresholdReached`` hook 28 | 29 | Improvements 30 | """""""""""" 31 | * EarlyStopper now works for any timescale and interval 32 | * Recurrent, Lstm, Clockwork, and ClockworkLstm layers now accept inputs of 33 | arbitrary shape by implicitly flattening them. 34 | * several fixes to make building the docs easier 35 | * some performance improvements of NumpyHandler operations ``binarize_t`` and ``index_m_by_v`` 36 | * sped up tests 37 | * several improvements to installation scripts 38 | 39 | Bugfixes 40 | """""""" 41 | * fixed `sqrt` operation for ``PyCudaHandler``. This should fix problems with BatchNormalization on GPU. 42 | * fixed a bug for task_type='regression' in ``brainstorm.tools.get_in_out_layers`` 43 | and ``brainstorm.tools.create_net_from_spec`` 44 | * removed defunct name argument from input layer 45 | * fixed a crash when applying ``brainstorm.hooks.SaveBestNetwork`` to `rolling_training` loss 46 | * various minor fixes of the ``brainstorm.hooks.BokehVisualizer`` 47 | * fixed a problem with ``sum_t`` operation in ``brainstorm.handlers.PyCudaHandler`` 48 | * fixed a blocksize problem in convolutional and pooling operations in ``brainstorm.handlers.PyCudaHandler`` 49 | 50 | 51 | 0.5b0 (2015-10-25) 52 | ++++++++++++++++++ 53 | * First release on PyPI. 54 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTING.rst 2 | include HISTORY.rst 3 | include LICENSE 4 | include README.md 5 | recursive-include brainstorm *.c *.h *.pyx 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help clean clean-pyc clean-build clean-test clean-docs lint test \ 2 | test-all coverage docs release dist 3 | 4 | help: 5 | @echo "clean - remove all build, doc, test, coverage and Python artifacts" 6 | @echo "clean-build - remove build artifacts" 7 | @echo "clean-pyc - remove Python file artifacts" 8 | @echo "clean-test - remove test and coverage artifacts" 9 | @echo "clean-doc - remove documentation artifacts" 10 | @echo "lint - check style with flake8" 11 | @echo "test - run tests quickly with the default Python" 12 | @echo "test-all - run tests on every Python version with tox" 13 | @echo "coverage - check code coverage quickly with the default Python" 14 | @echo "docs - generate Sphinx HTML documentation, including API docs" 15 | @echo "release - package and upload a release" 16 | @echo "dist - package" 17 | 18 | clean: clean-build clean-pyc clean-test clean-docs 19 | 20 | clean-build: 21 | rm -fr build/ 22 | rm -fr dist/ 23 | rm -fr *.egg-info 24 | 25 | clean-pyc: 26 | find . -name '*.pyc' -exec rm -f {} + 27 | find . -name '*.pyo' -exec rm -f {} + 28 | find . -name '*~' -exec rm -f {} + 29 | find . -name '__pycache__' -exec rm -fr {} + 30 | 31 | clean-test: 32 | rm -fr .tox/ 33 | rm -f .coverage 34 | rm -fr htmlcov/ 35 | rm -fr .cache/ 36 | 37 | clean-docs: 38 | $(MAKE) -C docs clean 39 | 40 | lint: 41 | flake8 brainstorm 42 | 43 | test: clean-pyc clean-test 44 | python setup.py test 45 | 46 | test-all: clean-pyc 47 | tox 48 | 49 | coverage: clean-pyc 50 | coverage run --source brainstorm setup.py test 51 | coverage report -m 52 | coverage html 53 | xdg-open htmlcov/index.html 54 | 55 | docs: 56 | $(MAKE) -C docs clean 57 | python setup.py build_ext --inplace 58 | $(MAKE) -C docs html 59 | xdg-open docs/_build/html/index.html 60 | 61 | release: clean 62 | python setup.py build_ext --inplace 63 | python setup.py sdist upload 64 | python setup.py bdist_wheel upload 65 | 66 | dist: clean 67 | python setup.py build_ext --inplace 68 | python setup.py sdist 69 | python setup.py bdist_wheel 70 | ls -l dist 71 | -------------------------------------------------------------------------------- /brainstorm/__about__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | """ 4 | This module contains meta-information about the brainstorm package. 5 | 6 | It is kept simple and separate from the main module, because this information 7 | is also read by the setup.py. And during installation the brainstorm module 8 | might not be importable yet. 9 | """ 10 | from __future__ import division, print_function, unicode_literals 11 | 12 | __all__ = ("__version__", "__author__", "__url__", "__license__") 13 | 14 | __version__ = "0.5" 15 | __author__ = "The Swiss AI Lab IDSIA" 16 | __url__ = "https://github.com/IDSIA/brainstorm" 17 | __license__ = "MIT" 18 | -------------------------------------------------------------------------------- /brainstorm/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from brainstorm.describable import get_description, create_from_description 5 | from brainstorm.randomness import global_rnd 6 | from brainstorm.structure import Network, generate_architecture 7 | from brainstorm.training import Trainer 8 | 9 | from brainstorm import initializers 10 | from brainstorm import data_iterators 11 | from brainstorm import value_modifiers 12 | from brainstorm import tools 13 | from brainstorm import hooks 14 | from brainstorm import layers 15 | from brainstorm import handlers 16 | from brainstorm import training 17 | from brainstorm import scorers 18 | from brainstorm.__about__ import __version__ 19 | 20 | 21 | __all__ = ['get_description', 'create_from_description', 'global_rnd', 22 | 'Network', 'generate_architecture', 'Trainer', 23 | 'initializers', 'data_iterators', 'value_modifiers', 'tools', 24 | 'hooks', 'layers', 'handlers', 'training', 'scorers', 25 | '__version__'] 26 | -------------------------------------------------------------------------------- /brainstorm/all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | """ 4 | Convenience namespace containing all relevant brainstorm objects and functions. 5 | """ 6 | from __future__ import division, print_function, unicode_literals 7 | 8 | from brainstorm.data_iterators import * 9 | from brainstorm.describable import create_from_description, get_description 10 | from brainstorm.handlers import * 11 | from brainstorm.hooks import * 12 | from brainstorm.initializers import * 13 | from brainstorm.layers import * 14 | from brainstorm.randomness import global_rnd 15 | from brainstorm.structure import Network, generate_architecture 16 | from brainstorm.scorers import * 17 | from brainstorm.tools import * 18 | from brainstorm.training import * 19 | from brainstorm.value_modifiers import * 20 | -------------------------------------------------------------------------------- /brainstorm/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function 4 | from brainstorm.handlers.numpy_handler import NumpyHandler 5 | from brainstorm.handlers.debug_handler import DebugHandler 6 | from brainstorm.optional import has_pycuda, pycuda_mock 7 | import numpy as np 8 | 9 | if has_pycuda: 10 | from brainstorm.handlers.pycuda_handler import PyCudaHandler 11 | else: 12 | PyCudaHandler = pycuda_mock 13 | 14 | default_handler = NumpyHandler(np.float32) 15 | 16 | __all__ = ['NumpyHandler', 'PyCudaHandler', 'default_handler'] 17 | -------------------------------------------------------------------------------- /brainstorm/layers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function 4 | 5 | from brainstorm.layers.base_layer import Layer 6 | from brainstorm.layers.batch_normalization_layer import BatchNorm 7 | from brainstorm.layers.binomial_cross_entropy_layer import BinomialCrossEntropy 8 | from brainstorm.layers.clockwork_lstm_layer import ClockworkLstm 9 | from brainstorm.layers.clockwork_layer import Clockwork 10 | from brainstorm.layers.convolution_layer_2d import Convolution2D 11 | from brainstorm.layers.deltas_scaling_layer import DeltasScaling 12 | from brainstorm.layers.dropout_layer import Dropout 13 | from brainstorm.layers.elementwise_layer import Elementwise 14 | from brainstorm.layers.fully_connected_layer import FullyConnected 15 | from brainstorm.layers.highway_layer import Highway 16 | from brainstorm.layers.input_layer import Input 17 | from brainstorm.layers.l1_decay import L1Decay 18 | from brainstorm.layers.l2_decay import L2Decay 19 | from brainstorm.layers.loss_layer import Loss 20 | from brainstorm.layers.lstm_layer import Lstm 21 | from brainstorm.layers.merge_layer import Merge 22 | from brainstorm.layers.mask_layer import Mask 23 | from brainstorm.layers.noop_layer import NoOp 24 | from brainstorm.layers.pooling_layer_2d import Pooling2D 25 | from brainstorm.layers.recurrent_layer import Recurrent 26 | from brainstorm.layers.sigmoid_ce_layer import SigmoidCE 27 | from brainstorm.layers.softmax_ce_layer import SoftmaxCE 28 | from brainstorm.layers.squared_difference_layer import SquaredDifference 29 | from brainstorm.layers.squared_error_layer import SquaredError 30 | -------------------------------------------------------------------------------- /brainstorm/layers/binomial_cross_entropy_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import (LayerValidationError, flatten_time, 12 | flatten_time_and_features) 13 | 14 | 15 | def BinomialCrossEntropy(name=None): 16 | """Create a Binomial Cross Entropy Layer. 17 | 18 | Calculate the Binomial Cross Entropy between outputs and **binary** targets 19 | 20 | Cross entropy is by definition asymmetric, therefore the inputs are named 21 | 'default' for the network outputs and 'targets' for the binary targets. 22 | This layer only calculates the deltas for the default inputs. 23 | Also note that this implementation only works for **binary** targets and 24 | outputs in the range 0 to 1. 25 | For outputs outside that range or non-binary targets the result is 26 | undefined. 27 | """ 28 | return ConstructionWrapper.create(BinomialCrossEntropyLayerImpl, 29 | name=name) 30 | 31 | 32 | class BinomialCrossEntropyLayerImpl(Layer): 33 | 34 | expected_inputs = {'default': StructureTemplate('T', 'B', '...'), 35 | 'targets': StructureTemplate('T', 'B', '...')} 36 | 37 | expected_kwargs = {} 38 | 39 | computes_no_input_deltas_for = ['targets'] 40 | 41 | def setup(self, kwargs, in_shapes): 42 | if in_shapes['default'] != in_shapes['targets']: 43 | raise LayerValidationError("{}: default and targets must have the " 44 | "same shapes but got {} and {}" 45 | .format(self.name, 46 | in_shapes['default'], 47 | in_shapes['targets'])) 48 | outputs = OrderedDict() 49 | outputs['default'] = BufferStructure('T', 'B', 1) 50 | 51 | feature_shape = in_shapes['default'].feature_shape 52 | internals = OrderedDict() 53 | internals['cee'] = BufferStructure('T', 'B', *feature_shape) 54 | internals['ceed'] = BufferStructure('T', 'B', *feature_shape, 55 | is_backward_only=True) 56 | 57 | return outputs, OrderedDict(), internals 58 | 59 | def forward_pass(self, buffers, training_pass=True): 60 | # prepare 61 | _h = self.handler 62 | y = buffers.inputs.default 63 | t = buffers.inputs.targets 64 | cee = buffers.internals.cee 65 | cee_sum = buffers.outputs.default 66 | 67 | # the binomial cross entropy error is given by 68 | # - t * ln(y) - (1-t) * ln(1-y) 69 | tmp = _h.ones(cee.shape) 70 | _h.subtract_tt(tmp, y, cee) # cee = 1-y 71 | _h.subtract_tt(tmp, t, tmp) # tmp = 1-t 72 | _h.clip_t(cee, 1e-6, 1.0, cee) 73 | _h.log_t(cee, cee) # cee = ln(1-y) 74 | _h.mult_tt(tmp, cee, tmp) # tmp = (1-t) * ln(1-y) 75 | 76 | _h.clip_t(y, 1e-6, 1.0, cee) 77 | _h.log_t(cee, cee) # cee = ln(y) 78 | _h.mult_tt(t, cee, cee) # cee = t * ln(y) 79 | 80 | _h.add_tt(tmp, cee, cee) # cee = (1-t) * ln(1-y) + t * ln(y) 81 | 82 | # reshape for summation 83 | cee = flatten_time_and_features(cee) 84 | cee_sum = flatten_time(cee_sum) 85 | _h.sum_t(cee, axis=1, out=cee_sum) 86 | _h.mult_st(-1, cee_sum, cee_sum) # * -1 87 | 88 | def backward_pass(self, buffers): 89 | # prepare 90 | _h = self.handler 91 | ceed_sum = buffers.output_deltas.default 92 | ceed = buffers.internals.ceed 93 | tmp = _h.allocate(ceed.shape) 94 | 95 | y = buffers.inputs.default 96 | t = buffers.inputs.targets 97 | 98 | yd = buffers.input_deltas.default 99 | 100 | # the derivative of the binomial cross entropy error is given by 101 | # (y - t) / (y - y²) 102 | 103 | _h.mult_tt(y, y, ceed) # ceed = y² 104 | _h.subtract_tt(y, ceed, ceed) # ceed = y - y² 105 | _h.clip_t(ceed, 1e-6, 1.0, ceed) # clip 106 | 107 | _h.subtract_tt(y, t, tmp) # tmp = y - t 108 | 109 | _h.divide_tt(tmp, ceed, ceed) # ceed = (y - t) / (y - y²) 110 | 111 | # ceed_sum has only one feature dimension due to summation, 112 | # so we broadcast to all feature dimensions 113 | _h.broadcast_t(ceed_sum, 2, tmp) 114 | _h.mult_tt(ceed, tmp, ceed) 115 | 116 | _h.add_tt(ceed, yd, yd) 117 | -------------------------------------------------------------------------------- /brainstorm/layers/clockwork_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from collections import OrderedDict 5 | from brainstorm.structure.construction import ConstructionWrapper 6 | from brainstorm.utils import LayerValidationError, flatten_time, \ 7 | flatten_time_and_features 8 | from brainstorm.layers.base_layer import Layer 9 | from brainstorm.structure.buffer_structure import BufferStructure, \ 10 | StructureTemplate 11 | 12 | 13 | def Clockwork(size, activation='tanh', name=None): 14 | return ConstructionWrapper.create(ClockworkLayerImpl, 15 | size=size, 16 | name=name, 17 | activation=activation) 18 | 19 | 20 | class ClockworkLayerImpl(Layer): 21 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 22 | expected_kwargs = {'size', 'activation'} 23 | 24 | computes_no_gradients_for = ['timing'] 25 | 26 | def setup(self, kwargs, in_shapes): 27 | self.activation = kwargs.get('activation', 'tanh') 28 | self.size = kwargs.get('size', in_shapes['default'].feature_size) 29 | 30 | if not isinstance(self.size, int): 31 | raise LayerValidationError('size must be int but was {}'. 32 | format(self.size)) 33 | 34 | in_size = self.in_shapes['default'].feature_size 35 | 36 | outputs = OrderedDict() 37 | outputs['default'] = BufferStructure('T', 'B', self.size, 38 | context_size=1) 39 | 40 | parameters = OrderedDict() 41 | parameters['W'] = BufferStructure(self.size, in_size) 42 | parameters['R'] = BufferStructure(self.size, self.size) 43 | parameters['bias'] = BufferStructure(self.size) 44 | parameters['timing'] = BufferStructure(self.size) 45 | 46 | internals = OrderedDict() 47 | internals['Ha'] = BufferStructure('T', 'B', self.size, context_size=1) 48 | internals['dHa'] = BufferStructure('T', 'B', self.size, context_size=1, 49 | is_backward_only=True) 50 | internals['dHb'] = BufferStructure('T', 'B', self.size, context_size=1, 51 | is_backward_only=True) 52 | 53 | return outputs, parameters, internals 54 | 55 | def forward_pass(self, buffers, training_pass=True): 56 | # prepare 57 | _h = self.handler 58 | W, R, bias, timing = buffers.parameters 59 | inputs = buffers.inputs.default 60 | outputs = buffers.outputs.default 61 | Ha = buffers.internals.Ha 62 | 63 | flat_inputs = flatten_time_and_features(inputs) 64 | flat_H = flatten_time(Ha[:-1]) 65 | 66 | _h.dot_mm(flat_inputs, W, flat_H, transb=True) 67 | _h.add_mv(flat_H, bias.reshape((1, self.size)), flat_H) 68 | 69 | tmp = _h.zeros(timing.shape) 70 | cond = _h.zeros(outputs[0].shape) 71 | for t in range(inputs.shape[0]): 72 | _h.dot_add_mm(outputs[t - 1], R, Ha[t], transb=True) 73 | _h.act_func[self.activation](Ha[t], outputs[t]) 74 | # Undo updates 75 | if t > 0: 76 | _h.fill(tmp, t) 77 | _h.modulo_tt(tmp, timing, tmp) 78 | _h.broadcast_t(tmp.reshape((1, tmp.shape[0])), 0, cond) 79 | _h.copy_to_if(outputs[t - 1], outputs[t], cond) 80 | 81 | def backward_pass(self, buffers): 82 | # prepare 83 | _h = self.handler 84 | W, R, bias, timing = buffers.parameters 85 | dW, dR, dbias, dtiming = buffers.gradients 86 | inputs = buffers.inputs.default 87 | outputs = buffers.outputs.default 88 | dinputs = buffers.input_deltas.default 89 | doutputs = buffers.output_deltas.default 90 | Ha, dHa, dHb = buffers.internals 91 | 92 | tmp = _h.zeros(timing.shape) 93 | cond = _h.zeros(outputs[0].shape) 94 | 95 | _h.copy_to(doutputs, dHb) 96 | T = inputs.shape[0] - 1 97 | _h.act_func_deriv[self.activation](Ha[T], outputs[T], dHb[T], dHa[T]) 98 | for t in range(T - 1, -1, -1): 99 | _h.fill(tmp, t + 1) 100 | _h.modulo_tt(tmp, timing, tmp) 101 | _h.broadcast_t(tmp.reshape((1, tmp.shape[0])), 0, cond) 102 | _h.add_into_if(dHb[t + 1], dHb[t], cond) 103 | _h.fill_if(dHa[t+1], 0.0, cond) 104 | _h.dot_add_mm(dHa[t + 1], R, dHb[t]) 105 | _h.act_func_deriv[self.activation](Ha[t], outputs[t], dHb[t], 106 | dHa[t]) 107 | 108 | flat_inputs = flatten_time_and_features(inputs) 109 | flat_dinputs = flatten_time_and_features(dinputs) 110 | flat_dHa = flatten_time(dHa[:-1]) 111 | 112 | # Calculate in_deltas and gradients 113 | _h.dot_add_mm(flat_dHa, W, flat_dinputs) 114 | _h.dot_add_mm(flat_dHa, flat_inputs, dW, transa=True) 115 | dbias_tmp = _h.allocate(dbias.shape) 116 | _h.sum_t(flat_dHa, axis=0, out=dbias_tmp) 117 | _h.add_tt(dbias, dbias_tmp, dbias) 118 | 119 | flat_outputs = flatten_time(outputs[:-2]) 120 | flat_dHa = flatten_time(dHa[1:-1]) 121 | _h.dot_add_mm(flat_dHa, flat_outputs, dR, transa=True) 122 | _h.dot_add_mm(dHa[0], outputs[-1], dR, transa=True) 123 | -------------------------------------------------------------------------------- /brainstorm/layers/convolution_layer_2d.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | from brainstorm.layers.base_layer import Layer 7 | from brainstorm.structure.buffer_structure import (BufferStructure, 8 | StructureTemplate) 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | from brainstorm.utils import flatten_time 11 | 12 | 13 | def Convolution2D(num_filters, kernel_size, stride=(1, 1), padding=0, 14 | activation='rel', name=None): 15 | """Create a 2D Convolution layer.""" 16 | return ConstructionWrapper.create(Convolution2DLayerImpl, 17 | num_filters=num_filters, 18 | kernel_size=kernel_size, 19 | stride=stride, 20 | padding=padding, 21 | activation=activation, 22 | name=name) 23 | 24 | 25 | class Convolution2DLayerImpl(Layer): 26 | 27 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 28 | expected_kwargs = {'num_filters', 'kernel_size', 'stride', 'padding', 29 | 'activation'} 30 | 31 | def setup(self, kwargs, in_shapes): 32 | self.activation = kwargs.get('activation', 'tanh') 33 | assert 'num_filters' in kwargs, "num_filters must be specified " \ 34 | " for ConvolutionLayer" 35 | assert 'kernel_size' in kwargs, "kernel_size must be specified " \ 36 | "for ConvolutionLayer" 37 | num_filters = kwargs['num_filters'] 38 | kernel_size = kwargs['kernel_size'] 39 | stride = kwargs.get('stride', (1, 1)) 40 | padding = kwargs.get('padding', 0) 41 | assert type(padding) is int and padding >= 0, \ 42 | "Invalid padding: {}".format(padding) 43 | assert type(kernel_size) in [list, tuple] and \ 44 | len(kernel_size) == 2, "Kernel size must be list or tuple of " \ 45 | "length 2: {}".format(kernel_size) 46 | assert type(stride) in [list, tuple] and len(stride) == 2, \ 47 | "Stride must be list or tuple of length 2: {}".format(stride) 48 | in_shape = self.in_shapes['default'].feature_shape 49 | assert stride[0] >= 0 and stride[1] >= 0, \ 50 | "Invalid stride: {}".format(stride) 51 | assert isinstance(in_shape, tuple) and len(in_shape) == 3, \ 52 | "ConvolutionLayer2D must have 3 dimensional input but input " \ 53 | "shape was {}".format(in_shape) 54 | self.num_filters = num_filters 55 | self.kernel_size = tuple(kernel_size) 56 | self.stride = tuple(stride) 57 | self.padding = padding 58 | kernel_x, kernel_y = self.kernel_size 59 | num_input_maps = in_shape[2] 60 | 61 | output_height = ((in_shape[0] + 2 * padding - kernel_x) // 62 | stride[0]) + 1 63 | output_width = ((in_shape[1] + 2 * padding - kernel_y) // 64 | stride[1]) + 1 65 | out_shape = (output_height, output_width, num_filters) 66 | 67 | outputs = OrderedDict() 68 | outputs['default'] = BufferStructure('T', 'B', *out_shape) 69 | 70 | parameters = OrderedDict() 71 | parameters['W'] = BufferStructure(num_filters, kernel_x, kernel_y, 72 | num_input_maps) 73 | parameters['bias'] = BufferStructure(num_filters) 74 | 75 | internals = OrderedDict() 76 | return outputs, parameters, internals 77 | 78 | def forward_pass(self, buffers, training_pass=True): 79 | # prepare 80 | _h = self.handler 81 | W, bias = buffers.parameters 82 | inputs = buffers.inputs.default 83 | outputs = buffers.outputs.default 84 | 85 | # reshape 86 | flat_inputs = flatten_time(inputs) 87 | flat_outputs = flatten_time(outputs) 88 | 89 | # calculate outputs 90 | _h.conv2d_forward_batch(flat_inputs, W, bias, flat_outputs, 91 | self.padding, self.stride) 92 | _h.inplace_act_func[self.activation](outputs) 93 | 94 | def backward_pass(self, buffers): 95 | # prepare 96 | _h = self.handler 97 | W, bias = buffers.parameters 98 | dW, dbias = buffers.gradients 99 | inputs = buffers.inputs.default 100 | outputs = buffers.outputs.default 101 | in_deltas = buffers.input_deltas.default 102 | out_deltas = buffers.output_deltas.default 103 | 104 | # reshape 105 | flat_inputs = flatten_time(inputs) 106 | flat_in_deltas = flatten_time(in_deltas) 107 | flat_out_deltas = flatten_time(out_deltas) 108 | 109 | # calculate in_deltas and gradients 110 | _h.inplace_act_func_deriv[self.activation](outputs, out_deltas) 111 | _h.conv2d_backward_batch(flat_inputs, W, self.padding, self.stride, 112 | flat_in_deltas, flat_out_deltas, dW, dbias) 113 | -------------------------------------------------------------------------------- /brainstorm/layers/deltas_scaling_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import StructureTemplate 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | from brainstorm.utils import LayerValidationError 11 | 12 | 13 | def DeltasScaling(factor, name=None): 14 | """Create an DeltasScaling layer. 15 | 16 | This layer does nothing on the forward pass, but scales the deltas flowing 17 | back during the backward pass by a given factor. 18 | 19 | This can be used to invert the deltas and set up an adversarial branch of 20 | the network. 21 | """ 22 | return ConstructionWrapper.create(DeltasScalingLayerImpl, name=name, 23 | factor=factor) 24 | 25 | 26 | class DeltasScalingLayerImpl(Layer): 27 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 28 | expected_kwargs = {'factor'} 29 | 30 | def setup(self, kwargs, in_shapes): 31 | if 'factor' not in kwargs: 32 | raise LayerValidationError('Missing required "factor" argument') 33 | self.factor = kwargs['factor'] 34 | out_shapes = in_shapes 35 | return out_shapes, OrderedDict(), OrderedDict() 36 | 37 | def forward_pass(self, buffers, training_pass=True): 38 | self.handler.copy_to(buffers.inputs.default, buffers.outputs.default) 39 | 40 | def backward_pass(self, buffers): 41 | self.handler.mult_add_st(self.factor, 42 | buffers.output_deltas.default, 43 | buffers.input_deltas.default) 44 | -------------------------------------------------------------------------------- /brainstorm/layers/dropout_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import StructureTemplate 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | 11 | 12 | def Dropout(drop_prob=0.5, name=None): 13 | """Create a Dropout layer. 14 | 15 | drop_prob is the probability of a unit being dropped, i.e. 0 16 | """ 17 | return ConstructionWrapper.create(DropoutLayerImpl, drop_prob=drop_prob, 18 | name=name) 19 | 20 | 21 | class DropoutLayerImpl(Layer): 22 | 23 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 24 | expected_kwargs = {'drop_prob'} 25 | 26 | def setup(self, kwargs, in_shapes): 27 | self.drop_prob = kwargs.get('drop_prob', 0.5) 28 | 29 | outputs = OrderedDict() 30 | outputs['default'] = in_shapes['default'] 31 | 32 | internals = OrderedDict() 33 | internals['mask'] = self.in_shapes['default'] 34 | return outputs, OrderedDict(), internals 35 | 36 | def forward_pass(self, buffers, training_pass=True): 37 | _h = self.handler 38 | 39 | if training_pass: 40 | _h.generate_probability_mask(buffers.internals.mask, 41 | 1 - self.drop_prob) 42 | _h.mult_tt(buffers.inputs.default, buffers.internals.mask, 43 | out=buffers.outputs.default) 44 | _h.mult_st(1 / (1 - self.drop_prob), buffers.outputs.default, 45 | out=buffers.outputs.default) 46 | else: 47 | _h.copy_to(buffers.inputs.default, buffers.outputs.default) 48 | 49 | def backward_pass(self, buffers): 50 | self.handler.mult_add_tt(buffers.output_deltas.default, 51 | buffers.internals.mask, 52 | buffers.input_deltas.default) 53 | -------------------------------------------------------------------------------- /brainstorm/layers/elementwise_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import StructureTemplate 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | 11 | 12 | def Elementwise(activation='rel', name=None): 13 | """Create an Elementwise layer. 14 | 15 | This layer just applies a unit-wise function to its inputs. 16 | """ 17 | return ConstructionWrapper.create(ElementwiseLayerImpl, name=name, 18 | activation=activation) 19 | 20 | 21 | class ElementwiseLayerImpl(Layer): 22 | 23 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 24 | expected_kwargs = {'activation'} 25 | 26 | def setup(self, kwargs, in_shapes): 27 | self.activation = kwargs.get('activation', 'rel') 28 | return in_shapes, OrderedDict(), OrderedDict() 29 | 30 | def forward_pass(self, buffers, training_pass=True): 31 | self.handler.act_func[self.activation](buffers.inputs.default, 32 | buffers.outputs.default) 33 | 34 | def backward_pass(self, buffers): 35 | tmp = self.handler.allocate(buffers.input_deltas.default.shape) 36 | self.handler.act_func_deriv[self.activation]( 37 | buffers.inputs.default, buffers.outputs.default, 38 | buffers.output_deltas.default, tmp) 39 | self.handler.add_tt(buffers.input_deltas.default, tmp, 40 | buffers.input_deltas.default) 41 | -------------------------------------------------------------------------------- /brainstorm/layers/fully_connected_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import (LayerValidationError, flatten_time, 12 | flatten_time_and_features) 13 | 14 | 15 | def FullyConnected(size=None, activation='rel', name=None): 16 | """Create a Fully Connected (inner product) layer.""" 17 | if size is None: 18 | return ConstructionWrapper.create(FullyConnectedLayerImpl, name=name, 19 | activation=activation) 20 | else: 21 | return ConstructionWrapper.create(FullyConnectedLayerImpl, size=size, 22 | name=name, activation=activation) 23 | 24 | 25 | class FullyConnectedLayerImpl(Layer): 26 | 27 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 28 | expected_kwargs = {'size', 'activation'} 29 | 30 | def setup(self, kwargs, in_shapes): 31 | self.activation = kwargs.get('activation', 'rel') 32 | self.size = kwargs.get('size', self.in_shapes['default'].feature_shape) 33 | self.size = (self.size,) if isinstance(self.size, int) else self.size 34 | 35 | if not isinstance(self.size, (tuple, list)) or \ 36 | not all(isinstance(item, int) for item in self.size): 37 | raise LayerValidationError('size must be int or tuple[int] but ' 38 | 'was {}'.format(self.size)) 39 | in_size = in_shapes['default'].feature_size 40 | 41 | outputs = OrderedDict() 42 | outputs['default'] = BufferStructure('T', 'B', *self.size) 43 | out_size = outputs['default'].feature_size 44 | 45 | parameters = OrderedDict() 46 | parameters['W'] = BufferStructure(out_size, in_size) 47 | parameters['bias'] = BufferStructure(out_size) 48 | 49 | internals = OrderedDict() 50 | return outputs, parameters, internals 51 | 52 | def forward_pass(self, buffers, training_pass=True): 53 | # prepare 54 | _h = self.handler 55 | W, bias = buffers.parameters 56 | inputs = flatten_time_and_features(buffers.inputs.default) 57 | outputs = flatten_time_and_features(buffers.outputs.default) 58 | 59 | # calculate outputs 60 | _h.dot_mm(inputs, W, outputs, transb=True) 61 | _h.add_mv(outputs, bias.reshape((1, bias.shape[0])), outputs) 62 | _h.inplace_act_func[self.activation](outputs) 63 | 64 | def backward_pass(self, buffers): 65 | # prepare 66 | _h = self.handler 67 | W, bias = buffers.parameters 68 | dW, dbias = buffers.gradients 69 | inputs = flatten_time_and_features(buffers.inputs.default) 70 | outputs = flatten_time_and_features(buffers.outputs.default) 71 | in_deltas = flatten_time_and_features(buffers.input_deltas.default) 72 | out_deltas = flatten_time_and_features(buffers.output_deltas.default) 73 | 74 | # calculate in_deltas and gradients 75 | _h.inplace_act_func_deriv[self.activation](outputs, out_deltas) 76 | _h.dot_add_mm(out_deltas, W, out=in_deltas) 77 | _h.dot_mm(out_deltas, inputs, out=dW, transa=True) 78 | _h.sum_t(out_deltas, axis=0, out=dbias) 79 | -------------------------------------------------------------------------------- /brainstorm/layers/highway_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import LayerValidationError 12 | 13 | 14 | def Highway(name=None): 15 | """Create a Highway layer.""" 16 | return ConstructionWrapper.create(HighwayLayerImpl, name=name) 17 | 18 | 19 | class HighwayLayerImpl(Layer): 20 | expected_inputs = {'H': StructureTemplate('T', 'B', '...'), 21 | 'T': StructureTemplate('T', 'B', '...'), 22 | 'x': StructureTemplate('T', 'B', '...')} 23 | 24 | def setup(self, kwargs, in_shapes): 25 | # 'H', 'T' and 'x' must have the same shape 26 | if in_shapes['H'] != in_shapes['T']: 27 | raise LayerValidationError( 28 | "{}: H and T must have the same shape but got {} and {}" 29 | .format(self.name, in_shapes['H'], in_shapes['T'])) 30 | if in_shapes['H'] != in_shapes['x']: 31 | raise LayerValidationError( 32 | "{}: H and x must have the same shape but got {} and {}" 33 | .format(self.name, in_shapes['H'], in_shapes['x'])) 34 | 35 | outputs = OrderedDict() 36 | outputs['default'] = BufferStructure( 37 | 'T', 'B', *self.in_shapes['x'].feature_shape) 38 | return outputs, OrderedDict(), OrderedDict() 39 | 40 | def forward_pass(self, buffers, training_pass=True): 41 | # prepare 42 | _h = self.handler 43 | x = buffers.inputs.x 44 | H = buffers.inputs.H 45 | T = buffers.inputs.T 46 | y = buffers.outputs.default 47 | 48 | tmp = _h.zeros(x.shape) 49 | _h.subtract_tt(H, x, out=tmp) 50 | _h.mult_tt(T, tmp, out=tmp) 51 | _h.add_tt(tmp, x, out=y) 52 | 53 | def backward_pass(self, buffers): 54 | # prepare 55 | _h = self.handler 56 | x = buffers.inputs.x 57 | H = buffers.inputs.H 58 | T = buffers.inputs.T 59 | dx = buffers.input_deltas.x 60 | dH = buffers.input_deltas.H 61 | dT = buffers.input_deltas.T 62 | dy = buffers.output_deltas.default 63 | 64 | tmp = _h.ones(dx.shape) 65 | _h.subtract_tt(tmp, T, out=tmp) 66 | _h.mult_add_tt(tmp, dy, out=dx) 67 | 68 | _h.mult_add_tt(T, dy, out=dH) 69 | 70 | _h.subtract_tt(H, x, out=tmp) 71 | _h.mult_add_tt(tmp, dy, out=dT) 72 | -------------------------------------------------------------------------------- /brainstorm/layers/input_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import BufferStructure 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | from brainstorm.utils import LayerValidationError 11 | 12 | 13 | def Input(out_shapes): 14 | """Create an Input layer. 15 | Special input layer type, that provides access to external data. 16 | 17 | The 'out_shapes' keyword argument is required and specifies the names and 18 | shapes of all external inputs. 19 | """ 20 | return ConstructionWrapper.create(InputLayerImpl, out_shapes=out_shapes) 21 | 22 | 23 | class InputLayerImpl(Layer): 24 | 25 | expected_inputs = {} 26 | expected_kwargs = {'out_shapes'} 27 | 28 | def setup(self, kwargs, in_shapes): 29 | if 'out_shapes' not in kwargs: 30 | raise LayerValidationError("InputLayer requires 'out_shapes'") 31 | if in_shapes: 32 | raise LayerValidationError( 33 | 'InputLayer cannot have any incoming connections!' 34 | '(But had these: {})'.format(in_shapes)) 35 | 36 | outputs = OrderedDict() 37 | for n, s in self.kwargs['out_shapes'].items(): 38 | outputs[n] = BufferStructure(*s) 39 | return outputs, OrderedDict(), OrderedDict() 40 | 41 | def _validate_connections(self): 42 | super(InputLayerImpl, self)._validate_connections() 43 | 44 | if self.incoming: 45 | raise LayerValidationError( 46 | 'InputLayer cannot have any incoming connections!' 47 | '(But had these: {})'.format(self.incoming)) 48 | -------------------------------------------------------------------------------- /brainstorm/layers/l1_decay.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.handlers.base_handler import Handler 8 | from brainstorm.layers.base_layer import Layer 9 | from brainstorm.structure.buffer_structure import (BufferStructure, 10 | StructureTemplate) 11 | from brainstorm.structure.construction import ConstructionWrapper 12 | from brainstorm.utils import flatten_time, flatten_time_and_features 13 | 14 | 15 | def L1Decay(name=None): 16 | """Add L1 regularization to the activations of a layer.""" 17 | return ConstructionWrapper.create(L1DecayLayerImpl, name=name) 18 | 19 | 20 | class L1DecayLayerImpl(Layer): 21 | 22 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 23 | expected_kwargs = {} 24 | 25 | def setup(self, kwargs, in_shapes): 26 | outputs = OrderedDict() 27 | outputs['loss'] = BufferStructure('T', 'B', 1) 28 | 29 | parameters = OrderedDict() 30 | internals = OrderedDict() 31 | internals['tmp'] = in_shapes['default'] 32 | 33 | return outputs, parameters, internals 34 | 35 | def forward_pass(self, buffers, training_pass=True): 36 | # prepare 37 | _h = self.handler 38 | assert isinstance(_h, Handler) 39 | inputs = buffers.inputs.default 40 | tmp = buffers.internals.tmp 41 | outputs = buffers.outputs.loss 42 | 43 | # reshape 44 | flat_inputs = flatten_time_and_features(inputs) 45 | flat_tmp = flatten_time_and_features(tmp) 46 | flat_outputs = flatten_time(outputs) 47 | 48 | # compute 49 | _h.abs_t(flat_inputs, flat_tmp) 50 | _h.sum_t(flat_tmp, 1, flat_outputs) 51 | 52 | def backward_pass(self, buffers): 53 | _h = self.handler 54 | assert isinstance(_h, Handler) 55 | inputs = buffers.inputs.default 56 | tmp = buffers.internals.tmp 57 | output_deltas = buffers.output_deltas.loss 58 | input_deltas = buffers.input_deltas.default 59 | 60 | # reshape 61 | flat_inputs = flatten_time_and_features(inputs) 62 | flat_tmp = flatten_time_and_features(tmp) 63 | flat_output_deltas = flatten_time(output_deltas) 64 | flat_input_deltas = flatten_time_and_features(input_deltas) 65 | 66 | # compute 67 | _h.sign_t(flat_inputs, flat_tmp) 68 | _h.mult_mv(flat_tmp, flat_output_deltas, flat_tmp) 69 | _h.add_tt(flat_tmp, flat_input_deltas, flat_input_deltas) 70 | -------------------------------------------------------------------------------- /brainstorm/layers/l2_decay.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.handlers.base_handler import Handler 8 | from brainstorm.layers.base_layer import Layer 9 | from brainstorm.structure.buffer_structure import (BufferStructure, 10 | StructureTemplate) 11 | from brainstorm.structure.construction import ConstructionWrapper 12 | from brainstorm.utils import flatten_time, flatten_time_and_features 13 | 14 | 15 | def L2Decay(name=None): 16 | """Add L2 regularization to the activations of a layer.""" 17 | return ConstructionWrapper.create(L2DecayLayerImpl, name=name) 18 | 19 | 20 | class L2DecayLayerImpl(Layer): 21 | 22 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 23 | expected_kwargs = {} 24 | 25 | def setup(self, kwargs, in_shapes): 26 | outputs = OrderedDict() 27 | outputs['loss'] = BufferStructure('T', 'B', 1) 28 | 29 | parameters = OrderedDict() 30 | internals = OrderedDict() 31 | internals['tmp'] = in_shapes['default'] 32 | internals['dsq_activations'] = BufferStructure( 33 | *in_shapes['default'].shape, is_backward_only=True) 34 | 35 | return outputs, parameters, internals 36 | 37 | def forward_pass(self, buffers, training_pass=True): 38 | # prepare 39 | _h = self.handler 40 | assert isinstance(_h, Handler) 41 | inputs = buffers.inputs.default 42 | tmp = buffers.internals.tmp 43 | outputs = buffers.outputs.loss 44 | 45 | # reshape 46 | flat_inputs = flatten_time_and_features(inputs) 47 | flat_tmp = flatten_time_and_features(tmp) 48 | flat_outputs = flatten_time(outputs) 49 | 50 | # compute 51 | _h.mult_tt(flat_inputs, flat_inputs, flat_tmp) 52 | _h.mult_st(0.5, flat_tmp, flat_tmp) 53 | _h.sum_t(flat_tmp, 1, flat_outputs) 54 | 55 | def backward_pass(self, buffers): 56 | _h = self.handler 57 | assert isinstance(_h, Handler) 58 | inputs = buffers.inputs.default 59 | tmp = buffers.internals.tmp 60 | output_deltas = buffers.output_deltas.loss 61 | input_deltas = buffers.input_deltas.default 62 | 63 | # reshape 64 | flat_inputs = flatten_time_and_features(inputs) 65 | flat_tmp = flatten_time_and_features(tmp) 66 | flat_output_deltas = flatten_time(output_deltas) 67 | flat_input_deltas = flatten_time_and_features(input_deltas) 68 | 69 | # compute 70 | _h.mult_mv(flat_inputs, flat_output_deltas, flat_tmp) 71 | _h.add_tt(flat_tmp, flat_input_deltas, flat_input_deltas) 72 | -------------------------------------------------------------------------------- /brainstorm/layers/loss_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | 12 | 13 | def Loss(importance=1.0, name=None): 14 | """Create a Loss layer.""" 15 | return ConstructionWrapper.create(LossLayerImpl, importance=importance, 16 | name=name) 17 | 18 | 19 | class LossLayerImpl(Layer): 20 | 21 | expected_inputs = {'default': StructureTemplate('...')} 22 | expected_kwargs = {'importance'} 23 | 24 | def setup(self, kwargs, in_shapes): 25 | assert self.name != 'total_loss' 26 | 27 | self.importance = kwargs.get('importance', 1.0) 28 | self.batch_index = None 29 | if in_shapes['default'].scales_with_time: 30 | self.batch_index = 1 31 | elif in_shapes['default'].scales_with_batch_size: 32 | self.batch_index = 0 33 | 34 | outputs = OrderedDict() 35 | outputs['loss'] = BufferStructure(1) 36 | return outputs, OrderedDict(), OrderedDict() 37 | 38 | def forward_pass(self, buffers, training_pass=True): 39 | if self.batch_index is None: 40 | batch_size = 1.0 41 | else: 42 | batch_size = buffers.inputs.default.shape[self.batch_index] 43 | 44 | self.handler.sum_t(buffers.inputs.default, 45 | None, 46 | buffers.outputs.loss.reshape(tuple())) 47 | self.handler.mult_st(self.importance / batch_size, 48 | buffers.outputs.loss, 49 | buffers.outputs.loss) 50 | 51 | def backward_pass(self, buffers): 52 | if self.batch_index is None: 53 | batch_size = 1.0 54 | else: 55 | batch_size = buffers.inputs.default.shape[self.batch_index] 56 | self.handler.add_st(self.importance / batch_size, 57 | buffers.input_deltas.default, 58 | buffers.input_deltas.default) 59 | -------------------------------------------------------------------------------- /brainstorm/layers/mask_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import StructureTemplate 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | from brainstorm.utils import LayerValidationError, product 11 | 12 | 13 | def Mask(name=None): 14 | """Create a Mask layer.""" 15 | return ConstructionWrapper.create(MaskLayerImpl, name=name) 16 | 17 | 18 | class MaskLayerImpl(Layer): 19 | 20 | expected_inputs = {'default': StructureTemplate('T', 'B', '...'), 21 | 'mask': StructureTemplate('T', 'B', '...')} 22 | 23 | computes_no_input_deltas_for = ['mask'] 24 | 25 | def setup(self, kwargs, in_shapes): 26 | in_shape = in_shapes['default'].feature_shape 27 | expected_shape = in_shape[:-1] + (1,) 28 | 29 | if in_shapes['mask'].feature_shape == (1,): 30 | self.flatten_dim = 2 31 | elif in_shapes['mask'].feature_shape in [expected_shape, in_shape]: 32 | self.flatten_dim = len(in_shape) + 1 33 | else: 34 | raise LayerValidationError( 35 | "Shape of the mask did not match shape of the default inputs. " 36 | "Should be either ('T', 'B', 1) or {} or {}, but was {}" 37 | .format(('T', 'B') + expected_shape, 38 | in_shapes['default'].shape, 39 | in_shapes['mask'])) 40 | outputs = OrderedDict() 41 | outputs['default'] = in_shapes['default'] 42 | return outputs, OrderedDict(), OrderedDict() 43 | 44 | def flatten_buffer(self, buffer): 45 | pre = buffer.shape[:self.flatten_dim] 46 | post = buffer.shape[self.flatten_dim:] 47 | return buffer.reshape((int(product(pre)), int(product(post)))) 48 | 49 | def forward_pass(self, buffers, training_pass=True): 50 | _h = self.handler 51 | 52 | flat_inp = self.flatten_buffer(buffers.inputs.default) 53 | flat_mask = self.flatten_buffer(buffers.inputs.mask) 54 | flat_out = self.flatten_buffer(buffers.outputs.default) 55 | 56 | _h.mult_mv(flat_inp, flat_mask, out=flat_out) 57 | 58 | def backward_pass(self, buffers): 59 | _h = self.handler 60 | 61 | flat_out_deltas = self.flatten_buffer(buffers.output_deltas.default) 62 | tmp = self.handler.allocate(flat_out_deltas.shape) 63 | flat_mask = self.flatten_buffer(buffers.inputs.mask) 64 | flat_in_deltas = self.flatten_buffer(buffers.input_deltas.default) 65 | 66 | _h.mult_mv(flat_out_deltas, flat_mask, tmp) 67 | _h.add_tt(tmp, flat_in_deltas, flat_in_deltas) 68 | -------------------------------------------------------------------------------- /brainstorm/layers/merge_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import LayerValidationError, flatten_all_but_last 12 | 13 | 14 | def Merge(name=None): 15 | """Create a layer that merges two inputs into one along the last dim""" 16 | return ConstructionWrapper.create(MergeLayerImpl, name=name) 17 | 18 | 19 | class MergeLayerImpl(Layer): 20 | expected_inputs = {'inputs_1': StructureTemplate('...'), 21 | 'inputs_2': StructureTemplate('...')} 22 | expected_kwargs = {} 23 | 24 | def setup(self, kwargs, in_shapes): 25 | # 'inputs_1' and 'inputs_2' must have same shape except for last dim 26 | shape_prefix1 = in_shapes['inputs_1'].shape[:-1] 27 | shape_prefix2 = in_shapes['inputs_2'].shape[:-1] 28 | if shape_prefix1 != shape_prefix2: 29 | raise LayerValidationError( 30 | "{}: The shapes of inputs_1 and inputs_2 may only differ in " 31 | "the last dimension but got {} and {}".format( 32 | self.name, 33 | in_shapes['inputs_1'].shape, 34 | in_shapes['inputs_2'].shape)) 35 | 36 | combined_size = (in_shapes['inputs_1'].shape[-1] + 37 | in_shapes['inputs_2'].shape[-1]) 38 | out_shape = shape_prefix1 + (combined_size,) 39 | outputs = OrderedDict() 40 | outputs['default'] = BufferStructure(*out_shape) 41 | 42 | parameters = OrderedDict() 43 | internals = OrderedDict() 44 | return outputs, parameters, internals 45 | 46 | def forward_pass(self, buffers, training_pass=True): 47 | # prepare 48 | self.handler.merge_tt(buffers.inputs.inputs_1, 49 | buffers.inputs.inputs_2, 50 | buffers.outputs.default) 51 | 52 | def backward_pass(self, buffers): 53 | # prepare 54 | _h = self.handler 55 | self.handler.split_add_tt(buffers.output_deltas.default, 56 | buffers.input_deltas.inputs_1, 57 | buffers.input_deltas.inputs_2) 58 | -------------------------------------------------------------------------------- /brainstorm/layers/noop_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import StructureTemplate 9 | from brainstorm.structure.construction import ConstructionWrapper 10 | 11 | 12 | def NoOp(name=None): 13 | """Create a NoOp layer. 14 | 15 | This layer just copies its input into its output. 16 | """ 17 | return ConstructionWrapper.create(NoOpLayerImpl, name=name) 18 | 19 | 20 | class NoOpLayerImpl(Layer): 21 | 22 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 23 | expected_kwargs = {} 24 | 25 | def setup(self, kwargs, in_shapes): 26 | return self.in_shapes, OrderedDict(), OrderedDict() 27 | 28 | def forward_pass(self, buffers, training_pass=True): 29 | self.handler.copy_to(buffers.inputs.default, buffers.outputs.default) 30 | 31 | def backward_pass(self, buffers): 32 | self.handler.add_tt(buffers.output_deltas.default, 33 | buffers.input_deltas.default, 34 | out=buffers.input_deltas.default) 35 | -------------------------------------------------------------------------------- /brainstorm/layers/pooling_layer_2d.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import flatten_time 12 | 13 | 14 | def Pooling2D(kernel_size, type='max', stride=(1, 1), padding=0, name=None): 15 | """Create a 2D Pooling layer.""" 16 | return ConstructionWrapper.create(Pooling2DLayerImpl, 17 | kernel_size=kernel_size, 18 | type=type, stride=stride, 19 | padding=padding, name=name) 20 | 21 | 22 | class Pooling2DLayerImpl(Layer): 23 | 24 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 25 | expected_kwargs = {'kernel_size', 'type', 'stride', 26 | 'padding', 'activation_function'} 27 | 28 | def setup(self, kwargs, in_shapes): 29 | assert 'kernel_size' in kwargs, "kernel_size must be specified for " \ 30 | "Pooling2D" 31 | assert 'type' in kwargs, "type must be specified for Pooling2D" 32 | kernel_size = kwargs['kernel_size'] 33 | ptype = kwargs['type'] 34 | padding = kwargs.get('padding', 0) 35 | stride = kwargs.get('stride', (1, 1)) 36 | in_shape = self.in_shapes['default'].feature_shape 37 | assert ptype in ('max', 'avg') 38 | assert type(padding) is int and padding >= 0, \ 39 | "Invalid padding: {}".format(padding) 40 | assert type(kernel_size) in [list, tuple] and \ 41 | len(kernel_size) == 2, "Kernel size must be list or " \ 42 | "tuple of length 2: {}".format( 43 | kernel_size) 44 | assert type(stride) in [list, tuple] and len(stride) == 2, \ 45 | "Stride must be list or tuple of length 2: {}".format(stride) 46 | assert stride[0] >= 0 and stride[1] >= 0, \ 47 | "Invalid stride: {}".format(stride) 48 | assert isinstance(in_shape, tuple) and len(in_shape) == 3, \ 49 | "PoolingLayer2D must have 3 dimensional input but input " \ 50 | "shape was %s" % in_shape 51 | 52 | self.kernel_size = tuple(kernel_size) 53 | self.type = ptype 54 | self.padding = padding 55 | self.stride = tuple(stride) 56 | output_height = ((in_shape[0] + 2 * padding - kernel_size[0]) // 57 | stride[0]) + 1 58 | output_width = ((in_shape[1] + 2 * padding - kernel_size[1]) // 59 | stride[1]) + 1 60 | assert output_height > 0 and output_width > 0, \ 61 | "Evaluated output height and width must be positive but were " \ 62 | "({}, {})".format(output_height, output_width) 63 | output_shape = (output_height, output_width, in_shape[2]) 64 | 65 | outputs = OrderedDict() 66 | outputs['default'] = BufferStructure('T', 'B', *output_shape) 67 | 68 | internals = OrderedDict() 69 | if self.type == 'max': 70 | argmax_shape = outputs['default'].feature_shape 71 | internals['argmax'] = BufferStructure('T', 'B', *argmax_shape) 72 | return outputs, OrderedDict(), internals 73 | 74 | def forward_pass(self, buffers, training_pass=True): 75 | # prepare 76 | _h = self.handler 77 | inputs = buffers.inputs.default 78 | outputs = buffers.outputs.default 79 | 80 | # reshape 81 | flat_inputs = flatten_time(inputs) 82 | flat_outputs = flatten_time(outputs) 83 | 84 | # calculate outputs 85 | if self.type == 'max': 86 | argmax = buffers.internals.argmax 87 | flat_argmax = flatten_time(argmax) 88 | _h.maxpool2d_forward_batch(flat_inputs, self.kernel_size, 89 | flat_outputs, self.padding, self.stride, 90 | flat_argmax) 91 | elif self.type == 'avg': 92 | _h.avgpool2d_forward_batch(flat_inputs, self.kernel_size, 93 | flat_outputs, self.padding, self.stride) 94 | 95 | def backward_pass(self, buffers): 96 | 97 | # prepare 98 | _h = self.handler 99 | inputs = buffers.inputs.default 100 | outputs = buffers.outputs.default 101 | in_deltas = buffers.input_deltas.default 102 | out_deltas = buffers.output_deltas.default 103 | 104 | # reshape 105 | flat_inputs = flatten_time(inputs) 106 | flat_in_deltas = flatten_time(in_deltas) 107 | flat_out_deltas = flatten_time(out_deltas) 108 | flat_outputs = flatten_time(outputs) 109 | 110 | if self.type == 'max': 111 | argmax = buffers.internals.argmax 112 | flat_argmax = flatten_time(argmax) 113 | _h.maxpool2d_backward_batch(flat_inputs, self.kernel_size, 114 | flat_outputs, self.padding, 115 | self.stride, flat_argmax, 116 | flat_in_deltas, flat_out_deltas) 117 | elif self.type == 'avg': 118 | _h.avgpool2d_backward_batch(flat_inputs, self.kernel_size, 119 | flat_outputs, self.padding, 120 | self.stride, 121 | flat_in_deltas, flat_out_deltas) 122 | -------------------------------------------------------------------------------- /brainstorm/layers/recurrent_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import LayerValidationError, flatten_time, \ 12 | flatten_time_and_features 13 | 14 | 15 | def Recurrent(size, activation='tanh', name=None): 16 | """Create a Simple Recurrent layer.""" 17 | return ConstructionWrapper.create(RecurrentLayerImpl, size=size, 18 | name=name, activation=activation) 19 | 20 | 21 | class RecurrentLayerImpl(Layer): 22 | 23 | expected_inputs = {'default': StructureTemplate('T', 'B', '...')} 24 | expected_kwargs = {'size', 'activation'} 25 | 26 | def setup(self, kwargs, in_shapes): 27 | self.activation = kwargs.get('activation', 'tanh') 28 | self.size = kwargs.get('size', self.in_shapes['default'].feature_size) 29 | if not isinstance(self.size, int): 30 | raise LayerValidationError('size must be int but was {}'. 31 | format(self.size)) 32 | 33 | in_size = self.in_shapes['default'].feature_size 34 | 35 | outputs = OrderedDict() 36 | outputs['default'] = BufferStructure('T', 'B', self.size, 37 | context_size=1) 38 | parameters = OrderedDict() 39 | parameters['W'] = BufferStructure(self.size, in_size) 40 | parameters['R'] = BufferStructure(self.size, self.size) 41 | parameters['bias'] = BufferStructure(self.size) 42 | 43 | internals = OrderedDict() 44 | internals['Ha'] = BufferStructure('T', 'B', self.size, context_size=1) 45 | internals['dHa'] = BufferStructure('T', 'B', self.size, context_size=1, 46 | is_backward_only=True) 47 | internals['dHb'] = BufferStructure('T', 'B', self.size, context_size=1, 48 | is_backward_only=True) 49 | return outputs, parameters, internals 50 | 51 | def forward_pass(self, buffers, training_pass=True): 52 | # prepare 53 | _h = self.handler 54 | W, R, bias = buffers.parameters 55 | inputs = buffers.inputs.default 56 | outputs = buffers.outputs.default 57 | Ha = buffers.internals.Ha 58 | 59 | flat_inputs = flatten_time_and_features(inputs) 60 | flat_H = flatten_time(Ha[:-1]) 61 | 62 | _h.dot_mm(flat_inputs, W, flat_H, transb=True) 63 | _h.add_mv(flat_H, bias.reshape((1, self.size)), flat_H) 64 | 65 | for t in range(inputs.shape[0]): 66 | _h.dot_add_mm(outputs[t - 1], R, Ha[t], transb=True) 67 | _h.act_func[self.activation](Ha[t], outputs[t]) 68 | 69 | def backward_pass(self, buffers): 70 | # prepare 71 | _h = self.handler 72 | W, R, bias = buffers.parameters 73 | dW, dR, dbias = buffers.gradients 74 | inputs = buffers.inputs.default 75 | outputs = buffers.outputs.default 76 | dinputs = buffers.input_deltas.default 77 | doutputs = buffers.output_deltas.default 78 | Ha, dHa, dHb = buffers.internals 79 | 80 | _h.copy_to(doutputs, dHb) 81 | T = inputs.shape[0] - 1 82 | _h.act_func_deriv[self.activation](Ha[T], outputs[T], dHb[T], dHa[T]) 83 | for t in range(T - 1, -1, -1): 84 | _h.dot_add_mm(dHa[t + 1], R, dHb[t]) 85 | _h.act_func_deriv[self.activation](Ha[t], outputs[t], 86 | dHb[t], dHa[t]) 87 | 88 | flat_inputs = flatten_time_and_features(inputs) 89 | flat_dinputs = flatten_time_and_features(dinputs) 90 | flat_dHa = flatten_time(dHa[:-1]) 91 | 92 | # calculate in_deltas and gradients 93 | _h.dot_add_mm(flat_dHa, W, flat_dinputs) 94 | _h.dot_add_mm(flat_dHa, flat_inputs, dW, transa=True) 95 | dbias_tmp = _h.allocate(dbias.shape) 96 | _h.sum_t(flat_dHa, axis=0, out=dbias_tmp) 97 | _h.add_tt(dbias, dbias_tmp, dbias) 98 | 99 | flat_outputs = flatten_time(outputs[:-2]) 100 | flat_dHa = flatten_time(dHa[1:-1]) 101 | _h.dot_add_mm(flat_dHa, flat_outputs, dR, transa=True) 102 | _h.dot_add_mm(dHa[0], outputs[-1], dR, transa=True) 103 | -------------------------------------------------------------------------------- /brainstorm/layers/sigmoid_ce_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.handlers.base_handler import Handler 8 | from brainstorm.layers.base_layer import Layer 9 | from brainstorm.structure.buffer_structure import (BufferStructure, 10 | StructureTemplate) 11 | from brainstorm.structure.construction import ConstructionWrapper 12 | from brainstorm.utils import (LayerValidationError, flatten_time_and_features, 13 | flatten_time) 14 | 15 | 16 | def SigmoidCE(name=None): 17 | """Create a sigmoid layer with integrated Binomial Cross Entropy loss. 18 | 19 | Applies the sigmoid activation function on 'default' input and puts the 20 | results (per-label probabilities) in 'predictions'. 21 | 22 | It also takes as 'targets' a binary vector and computes the binomial 23 | cross-entropy loss. The resulting losses are stored in the 'loss' output. 24 | 25 | WARNING: 26 | This layer does not compute derivatives wrt the 'targets' input. 27 | It also does not use the deltas coming in from the 'predictions'. 28 | """ 29 | return ConstructionWrapper.create(SigmoidCELayerImpl, name=name) 30 | 31 | 32 | class SigmoidCELayerImpl(Layer): 33 | 34 | expected_inputs = {'default': StructureTemplate('T', 'B', '...'), 35 | 'targets': StructureTemplate('T', 'B', '...')} 36 | 37 | computes_no_input_deltas_for = ['targets'] 38 | takes_no_output_deltas_from = ['predictions'] 39 | 40 | def setup(self, kwargs, in_shapes): 41 | in_shape = in_shapes['default'].feature_shape 42 | tar_shape = in_shapes['targets'].feature_shape 43 | 44 | if tar_shape != in_shape: 45 | raise LayerValidationError('input and targets must have the same ' 46 | 'shapes. But got {} != {}' 47 | .format(in_shape, tar_shape)) 48 | 49 | outputs = OrderedDict() 50 | outputs['predictions'] = BufferStructure('T', 'B', *in_shape) 51 | outputs['loss'] = BufferStructure('T', 'B', *in_shape) 52 | 53 | internals = OrderedDict() 54 | internals['dcee'] = BufferStructure('T', 'B', *in_shape, 55 | is_backward_only=True) 56 | return outputs, OrderedDict(), internals 57 | 58 | def forward_pass(self, buffers, training_pass=True): 59 | _h = self.handler 60 | assert isinstance(_h, Handler) 61 | 62 | inputs = flatten_time_and_features(buffers.inputs.default) 63 | targets = flatten_time_and_features(buffers.inputs.targets) 64 | loss = flatten_time_and_features(buffers.outputs.loss) 65 | prob = flatten_time_and_features(buffers.outputs.predictions) 66 | 67 | # Apply sigmoid 68 | _h.sigmoid(inputs, prob) 69 | 70 | # the binomial cross entropy error is given by 71 | # - (t * ln(y) + (1-t) * ln(1-y)) 72 | tmp = _h.ones(prob.shape) 73 | _h.subtract_tt(tmp, prob, loss) # loss = 1-y 74 | _h.subtract_tt(tmp, targets, tmp) # tmp = 1-t 75 | _h.clip_t(loss, 1e-6, 1.0, loss) 76 | _h.log_t(loss, loss) # loss = ln(1-y) 77 | _h.mult_tt(tmp, loss, tmp) # tmp = (1-t) * ln(1-y) 78 | 79 | _h.clip_t(prob, 1e-6, 1.0, loss) 80 | _h.log_t(loss, loss) # loss = ln(y) 81 | _h.mult_tt(targets, loss, loss) # loss = t * ln(y) 82 | 83 | _h.add_tt(tmp, loss, loss) # loss = (1-t) * ln(1-y) + t * ln(y) 84 | 85 | _h.mult_st(-1, loss, loss) # * -1 86 | 87 | def backward_pass(self, buffers): 88 | # prepare 89 | _h = self.handler 90 | assert isinstance(_h, Handler) 91 | 92 | dinputs = flatten_time_and_features(buffers.input_deltas.default) 93 | dloss = flatten_time_and_features(buffers.output_deltas.loss) 94 | dcee = flatten_time_and_features(buffers.internals.dcee) 95 | targets = flatten_time_and_features(buffers.inputs.targets) 96 | prob = flatten_time_and_features(buffers.outputs.predictions) 97 | 98 | _h.subtract_tt(prob, targets, dcee) # y - t 99 | _h.mult_mv(dcee, dloss, dcee) # out_delta * (y - t) 100 | _h.add_tt(dcee, dinputs, dinputs) 101 | -------------------------------------------------------------------------------- /brainstorm/layers/softmax_ce_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import (LayerValidationError, flatten_all_but_last) 12 | 13 | 14 | def SoftmaxCE(name=None): 15 | """Create a softmax layer with integrated Multinomial Cross Entropy loss. 16 | 17 | Applies the softmax activation function on 'default' input and puts 18 | results (per-class probabilities) in 'predictions'. 19 | 20 | It also takes class indices (0-based) as the 'targets' input, 21 | and computes the multinomial cross-entropy loss. The resulting losses are 22 | stored in the 'loss' output. 23 | 24 | For pixel/voxel-wise classification, the `channel` dimension must be 25 | right-most (known as NHWC or NDHWC format). 26 | 27 | WARNING: 28 | This layer does not compute derivatives wrt the 'targets' input. 29 | It also does not use the deltas coming in from the 'predictions'. 30 | """ 31 | return ConstructionWrapper.create(SoftmaxCELayerImpl, name=name) 32 | 33 | 34 | class SoftmaxCELayerImpl(Layer): 35 | 36 | expected_inputs = {'default': StructureTemplate('T', 'B', '...'), 37 | 'targets': StructureTemplate('T', 'B', '...')} 38 | 39 | computes_no_input_deltas_for = ['targets'] 40 | takes_no_output_deltas_from = ['predictions'] 41 | 42 | def setup(self, kwargs, in_shapes): 43 | in_shape = in_shapes['default'].feature_shape 44 | tar_shape = in_shapes['targets'].feature_shape 45 | 46 | if len(tar_shape) != len(in_shape): 47 | raise LayerValidationError('Default input and targets must have ' 48 | 'the same number of dimensions.') 49 | if tar_shape[:-1] != in_shape[:-1]: 50 | raise LayerValidationError('All dimensions except last must match ' 51 | 'for default input and targets.') 52 | if tar_shape[-1] != 1: 53 | raise LayerValidationError('Last dimension of targets must be ' 54 | 'size 1.') 55 | 56 | outputs = OrderedDict() 57 | outputs['predictions'] = BufferStructure('T', 'B', *in_shape) 58 | outputs['loss'] = BufferStructure('T', 'B', *tar_shape) 59 | 60 | internals = OrderedDict() 61 | internals['t_bin'] = BufferStructure('T', 'B', *in_shape, 62 | is_backward_only=True) 63 | return outputs, OrderedDict(), internals 64 | 65 | def forward_pass(self, buffers, training_pass=True): 66 | # prepare 67 | _h = self.handler 68 | inputs = buffers.inputs.default 69 | targets = buffers.inputs.targets 70 | predictions = buffers.outputs.predictions 71 | loss = buffers.outputs.loss 72 | 73 | # reshape 74 | flat_inputs = flatten_all_but_last(inputs) 75 | flat_probs = flatten_all_but_last(predictions) 76 | flat_loss = flatten_all_but_last(loss) 77 | flat_targets = flatten_all_but_last(targets) 78 | 79 | # softmax 80 | _h.softmax_m(flat_inputs, flat_probs) 81 | 82 | # the multinomial cross entropy error is given by 83 | # - sum over i: p_i * ln(y_i) 84 | # now our targets are indices so all p_i = 0 except for i=t 85 | _h.fill(loss, 0.) 86 | _h.index_m_by_v(flat_probs, flat_targets, flat_loss) 87 | _h.clip_t(flat_loss, 1e-6, 1.0, flat_loss) 88 | _h.log_t(loss, loss) 89 | _h.mult_st(-1, loss, loss) 90 | 91 | def backward_pass(self, buffers): 92 | # prepare 93 | _h = self.handler 94 | targets = buffers.inputs.targets 95 | probs = buffers.outputs.predictions 96 | 97 | dinputs = buffers.input_deltas.default 98 | dloss = buffers.output_deltas.loss 99 | t_bin = buffers.internals.t_bin 100 | 101 | # reshape 102 | flat_probs = flatten_all_but_last(probs) 103 | flat_targets = flatten_all_but_last(targets) 104 | flat_t_bin = flatten_all_but_last(t_bin) 105 | flat_dloss = flatten_all_but_last(dloss) 106 | flat_dinputs = flatten_all_but_last(dinputs) 107 | 108 | # derivative of multinomial cross-entropy error wrt softmax: 109 | # y - t 110 | _h.binarize_v(flat_targets, flat_t_bin) 111 | _h.mult_st(-1, flat_t_bin, flat_t_bin) 112 | _h.add_tt(flat_t_bin, flat_probs, flat_t_bin) 113 | _h.mult_mv(flat_t_bin, flat_dloss, flat_t_bin) 114 | _h.add_tt(flat_t_bin, flat_dinputs, flat_dinputs) 115 | -------------------------------------------------------------------------------- /brainstorm/layers/squared_difference_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import (LayerValidationError, flatten_time_and_features) 12 | 13 | 14 | def SquaredDifference(name=None): 15 | """Create a Squared Difference layer.""" 16 | return ConstructionWrapper.create(SquaredDifferenceLayerImpl, name=name) 17 | 18 | 19 | class SquaredDifferenceLayerImpl(Layer): 20 | 21 | expected_inputs = {'inputs_1': StructureTemplate('T', 'B', '...'), 22 | 'inputs_2': StructureTemplate('T', 'B', '...')} 23 | expected_kwargs = {} 24 | 25 | def setup(self, kwargs, in_shapes): 26 | # 'inputs_1' and 'inputs_2' must have same shape 27 | f_shape1 = in_shapes['inputs_1'].feature_shape 28 | f_shape2 = in_shapes['inputs_2'].feature_shape 29 | if f_shape1 != f_shape2: 30 | raise LayerValidationError( 31 | "{}: inputs_1 and inputs_2 must have same feature shapes but " 32 | "got {} and {}".format(self.name, f_shape1, f_shape2)) 33 | 34 | outputs = OrderedDict() 35 | outputs['default'] = BufferStructure('T', 'B', *f_shape1) 36 | 37 | internals = OrderedDict() 38 | feature_shape = self.in_shapes['inputs_1'].feature_shape 39 | internals['diff'] = BufferStructure('T', 'B', *feature_shape) 40 | return outputs, OrderedDict(), internals 41 | 42 | def forward_pass(self, buffers, training_pass=True): 43 | # prepare 44 | _h = self.handler 45 | inputs_1 = flatten_time_and_features(buffers.inputs.inputs_1) 46 | inputs_2 = flatten_time_and_features(buffers.inputs.inputs_2) 47 | diff = flatten_time_and_features(buffers.internals.diff) 48 | outputs = flatten_time_and_features(buffers.outputs.default) 49 | 50 | # calculate 51 | _h.subtract_tt(inputs_1, inputs_2, out=diff) 52 | _h.mult_tt(diff, diff, out=outputs) 53 | 54 | def backward_pass(self, buffers): 55 | # prepare 56 | _h = self.handler 57 | out_deltas = flatten_time_and_features(buffers.output_deltas.default) 58 | diff = flatten_time_and_features(buffers.internals.diff) 59 | dinputs_1 = flatten_time_and_features(buffers.input_deltas.inputs_1) 60 | dinputs_2 = flatten_time_and_features(buffers.input_deltas.inputs_2) 61 | 62 | tmp = _h.allocate(out_deltas.shape) 63 | # calculate 64 | _h.mult_st(2, out_deltas, out=out_deltas) 65 | _h.mult_add_tt(out_deltas, diff, out=dinputs_1) 66 | _h.mult_st(-1, diff, out=tmp) 67 | _h.mult_add_tt(out_deltas, tmp, out=dinputs_2) 68 | -------------------------------------------------------------------------------- /brainstorm/layers/squared_error_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | from brainstorm.layers.base_layer import Layer 8 | from brainstorm.structure.buffer_structure import (BufferStructure, 9 | StructureTemplate) 10 | from brainstorm.structure.construction import ConstructionWrapper 11 | from brainstorm.utils import (LayerValidationError, flatten_time_and_features) 12 | 13 | 14 | def SquaredError(name=None): 15 | """ 16 | Create a SquaredError layer which computes half of the squared difference 17 | between the inputs `default` and `targets` element-wise. The factor of half 18 | is used to be consistent with common machine learning texts and resources. 19 | 20 | Produces outputs named `predictions` and `loss`. The `loss` output can be 21 | connected to a ``Loss`` layer for typical network training for a 22 | regression task. 23 | 24 | This layer acts similar to ``SigmoidCE`` and ``SoftmaxCE`` layers. Like 25 | the above layers, it does not compute the gradients w.r.t. the `targets` 26 | input and ignores incoming deltas w.r.t. the `predictions` output. 27 | """ 28 | return ConstructionWrapper.create(SquaredErrorLayerImpl, name=name) 29 | 30 | 31 | class SquaredErrorLayerImpl(Layer): 32 | 33 | expected_inputs = {'default': StructureTemplate('T', 'B', '...'), 34 | 'targets': StructureTemplate('T', 'B', '...')} 35 | expected_kwargs = {} 36 | computes_no_input_deltas_for = ['targets'] 37 | takes_no_output_deltas_from = ['predictions'] 38 | 39 | def setup(self, kwargs, in_shapes): 40 | # 'default' and 'targets' must have same shape 41 | in_shape = in_shapes['default'].feature_shape 42 | tar_shape = in_shapes['targets'].feature_shape 43 | if in_shape != tar_shape: 44 | raise LayerValidationError( 45 | "{}: default and targets must have same feature shapes but " 46 | "got {} and {}".format(self.name, in_shape, tar_shape)) 47 | 48 | outputs = OrderedDict() 49 | outputs['predictions'] = BufferStructure('T', 'B', *in_shape) 50 | outputs['loss'] = BufferStructure('T', 'B', *in_shape) 51 | 52 | internals = OrderedDict() 53 | internals['diff'] = BufferStructure('T', 'B', *in_shape) 54 | return outputs, OrderedDict(), internals 55 | 56 | def forward_pass(self, buffers, training_pass=True): 57 | # prepare 58 | _h = self.handler 59 | x = flatten_time_and_features(buffers.inputs.default) 60 | t = flatten_time_and_features(buffers.inputs.targets) 61 | diff = flatten_time_and_features(buffers.internals.diff) 62 | y = flatten_time_and_features(buffers.outputs.predictions) 63 | loss = flatten_time_and_features(buffers.outputs.loss) 64 | 65 | # calculate 66 | _h.copy_to(x, y) 67 | _h.subtract_tt(x, t, out=diff) 68 | _h.mult_tt(diff, diff, out=loss) 69 | _h.mult_st(0.5, loss, out=loss) 70 | 71 | def backward_pass(self, buffers): 72 | # prepare 73 | _h = self.handler 74 | dloss = flatten_time_and_features(buffers.output_deltas.loss) 75 | diff = flatten_time_and_features(buffers.internals.diff) 76 | dx = flatten_time_and_features(buffers.input_deltas.default) 77 | 78 | # calculate 79 | _h.mult_add_tt(dloss, diff, dx) 80 | -------------------------------------------------------------------------------- /brainstorm/optional.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | import sys 5 | import six 6 | 7 | 8 | class MissingDependencyMock(object): 9 | def __init__(self, error): 10 | self.error = error 11 | 12 | def __getattribute__(self, item): 13 | if item == "__doc__": 14 | return 15 | if item == "__class__": 16 | return MissingDependencyMock 17 | else: 18 | print("Attempted to access '{}' of a MissingDependencyMock object." 19 | " Reraising the import error:".format(item), file=sys.stderr) 20 | six.reraise(*object.__getattribute__(self, 'error')) 21 | 22 | def __call__(self, *args, **kwargs): 23 | print("Attempted to call MissingDependencyMock object. " 24 | "Reraising the import error:", file=sys.stderr) 25 | six.reraise(*object.__getattribute__(self, 'error')) 26 | 27 | 28 | try: 29 | import pycuda 30 | from pycuda import gpuarray, cumath 31 | import pycuda.driver as drv 32 | import pycuda.autoinit 33 | from pycuda.elementwise import ElementwiseKernel 34 | from pycuda.compiler import SourceModule 35 | from pycuda.curandom import XORWOWRandomNumberGenerator 36 | import skcuda.linalg as culinalg 37 | import skcuda.misc as cumisc 38 | has_pycuda = True 39 | pycuda_mock = None 40 | except ImportError as e: 41 | has_pycuda = False 42 | pycuda_mock = MissingDependencyMock(sys.exc_info()) 43 | 44 | 45 | try: 46 | import bokeh 47 | has_bokeh = True 48 | except ImportError as e: 49 | has_bokeh = False 50 | bokeh_mock = MissingDependencyMock(sys.exc_info()) 51 | 52 | 53 | __all__ = ['has_pycuda', 'pycuda_mock', 'has_bokeh', 'bokeh_mock', 54 | 'MissingDependencyMock'] 55 | -------------------------------------------------------------------------------- /brainstorm/randomness.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | 7 | from brainstorm.describable import Describable 8 | 9 | 10 | class RandomState(np.random.RandomState): 11 | """ 12 | An extension of the numpy RandomState that saves it's own seed 13 | and offers convenience methods to generate seeds and other RandomStates. 14 | """ 15 | 16 | seed_range = (0, 1000000000) 17 | 18 | def __init__(self, seed=None): 19 | if seed is None: 20 | seed = global_rnd.generate_seed() 21 | super(RandomState, self).__init__(seed) 22 | self._seed = seed 23 | 24 | def seed(self, seed=None): 25 | """ 26 | Set the seed of this RandomState. 27 | This method is kept for compatibility with the numpy RandomState. But 28 | for better readability you are encouraged to use the set_seed() method 29 | instead. 30 | 31 | Args: 32 | seed (int): 33 | the seed to reseed this random state with. 34 | """ 35 | super(RandomState, self).seed(seed) 36 | self._seed = seed 37 | 38 | def get_seed(self): 39 | """ 40 | Return the seed of this RandomState. 41 | """ 42 | return self._seed 43 | 44 | def set_seed(self, seed): 45 | """ 46 | Set the seed of this RandomState. 47 | 48 | Args: 49 | seed (int): 50 | the seed to reseed this random state with. 51 | """ 52 | self.seed(seed) 53 | 54 | def reset(self): 55 | """ 56 | Reset the internal state of this RandomState. 57 | """ 58 | self.seed(self._seed) 59 | 60 | def generate_seed(self): 61 | """ 62 | Generate a random seed. 63 | """ 64 | return self.randint(*RandomState.seed_range) 65 | 66 | def create_random_state(self, seed=None): 67 | """ 68 | Create and return new RandomState object. If seed is given this is 69 | equivalent to RandomState(seed). Otherwise this will first generate a 70 | seed and initialize the new RandomState with that. 71 | 72 | Args: 73 | seed (Optional(int)): 74 | the seed to initialize the generated RandomState with. 75 | Defaults to None in which case a seed is first generated by 76 | the self.generate_seed() method. 77 | """ 78 | if seed is None: 79 | seed = self.generate_seed() 80 | return RandomState(seed) 81 | 82 | def __reduce__(self): 83 | # Note: We need to override __reduce__ and __setstate__ 84 | # because numpys RandomState implements them such that if pickled and 85 | # unpickled it would yield a numpy RandomState instead of a 86 | # brainstorm RandomState. 87 | return self.__class__, (self._seed,), self.get_state() 88 | 89 | def __setstate__(self, state): 90 | self.set_state(state) 91 | 92 | 93 | class Seedable(Describable): 94 | """ 95 | Base class for all objects that use randomness. 96 | It offers a self.rnd which is a RandomState. 97 | 98 | Dev-note: It inherits from Describable in order to implement 99 | __init_from_description__ and to make rnd undescribed. 100 | 101 | """ 102 | __undescribed__ = {'rnd'} 103 | 104 | def __init__(self, seed=None): 105 | self.rnd = RandomState(seed) 106 | 107 | def __init_from_description__(self, description): 108 | Seedable.__init__(self) 109 | 110 | 111 | global_rnd = RandomState(np.random.randint(*RandomState.seed_range)) 112 | -------------------------------------------------------------------------------- /brainstorm/scorers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | 7 | import numpy as np 8 | 9 | from brainstorm.describable import Describable 10 | 11 | 12 | # ----------------------------- Base Class ---------------------------------- # 13 | 14 | class Scorer(Describable): 15 | def __init__(self, out_name='', targets_name='targets', mask_name='', 16 | name=None): 17 | self.out_name = out_name 18 | self.targets_name = targets_name 19 | self.mask_name = mask_name 20 | self.__name__ = name if name is not None else self.__class__.__name__ 21 | 22 | def __call__(self, true_labels, predicted, mask=None): 23 | pass 24 | 25 | @staticmethod 26 | def aggregate(errors): 27 | errors = np.array(errors) 28 | assert errors.ndim == 2 and errors.shape[1] == 2 29 | return np.sum(errors[:, 1]) / np.sum(errors[:, 0]) 30 | 31 | 32 | # ------------------------- Scoring Functions ------------------------------- # 33 | 34 | def gather_losses_and_scores(net, scorers, scores, out_name='', 35 | targets_name='targets', mask_name=''): 36 | ls = net.get_loss_values() 37 | for name, loss in ls.items(): 38 | scores[name].append((net._buffer_manager.batch_size, loss)) 39 | 40 | for sc in scorers: 41 | name = sc.__name__ 42 | predicted = net.get(sc.out_name or out_name or net.output_name) 43 | true_labels = net.get_input(sc.targets_name) if sc.targets_name\ 44 | else net.get_input(targets_name) 45 | mask = net.get_input(sc.mask_name) if sc.mask_name\ 46 | else (net.get_input(mask_name) if mask_name else None) 47 | 48 | predicted = _flatten_all_but_last(predicted) 49 | true_labels = _flatten_all_but_last(true_labels) 50 | mask = _flatten_all_but_last(mask) 51 | weight = mask.sum() if mask is not None else predicted.shape[0] 52 | 53 | scores[name].append((weight, sc(true_labels, predicted, mask))) 54 | 55 | 56 | def aggregate_losses_and_scores(scores, net, scorers): 57 | results = OrderedDict() 58 | for name in net.get_loss_values(): 59 | results[name] = _weighted_average(scores[name]) 60 | for sc in scorers: 61 | results[sc.__name__] = sc.aggregate(scores[sc.__name__]) 62 | return results 63 | 64 | 65 | # ------------------------------- Scorers ----------------------------------- # 66 | 67 | class Accuracy(Scorer): 68 | def __call__(self, true_labels, predicted, mask=None): 69 | if predicted.shape[1] > 1: 70 | predicted = predicted.argmax(1).reshape(-1, 1) 71 | correct = (predicted == true_labels).astype(np.float) 72 | if mask is not None: 73 | correct *= mask 74 | return np.sum(correct) 75 | 76 | 77 | class Hamming(Scorer): 78 | def __init__(self, threshold=0.5, out_name='', targets_name='targets', 79 | mask_name='', name=None): 80 | super(Hamming, self).__init__(out_name, targets_name, mask_name, name) 81 | self.threshold = threshold 82 | 83 | def __call__(self, true_labels, predicted, mask=None): 84 | correct = np.logical_xor(predicted < self.threshold, 85 | true_labels).astype(np.float) 86 | if mask is not None: 87 | correct *= mask 88 | return np.sum(correct) / true_labels.shape[1] 89 | 90 | 91 | class MeanSquaredError(Scorer): 92 | def __call__(self, true_labels, predicted, mask=None): 93 | errors = (true_labels - predicted) ** 2 94 | if mask is not None: 95 | errors *= mask 96 | return 0.5 * np.sum(errors) 97 | 98 | 99 | # ---------------------------- Helper Functions ----------------------------- # 100 | 101 | def _flatten_all_but_last(a): 102 | if a is None: 103 | return None 104 | return a.reshape(-1, a.shape[-1]) 105 | 106 | 107 | def _weighted_average(errors): 108 | errors = np.array(errors) 109 | assert errors.ndim == 2 and errors.shape[1] == 2 110 | return np.sum(errors[:, 1] * errors[:, 0] / np.sum(errors[:, 0])) 111 | -------------------------------------------------------------------------------- /brainstorm/structure/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function 4 | from brainstorm.structure.network import Network 5 | from brainstorm.structure.architecture import generate_architecture 6 | 7 | 8 | __all__ = ['Network', 'generate_architecture'] 9 | -------------------------------------------------------------------------------- /brainstorm/structure/buffer_views.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from brainstorm.utils import get_by_path 6 | 7 | 8 | class BufferView(list): 9 | def __init__(self, buffer_names, buffers, full_buffer=None): 10 | super(BufferView, self).__init__(buffers) 11 | if not len(buffers) == len(buffer_names): 12 | raise ValueError("Length mismatch between buffers and names ({} !=" 13 | " {})".format(len(buffers), len(buffer_names))) 14 | self._full_buffer = full_buffer 15 | self._buffer_names = tuple(buffer_names) 16 | self._keys = set(buffer_names) 17 | for i, n in enumerate(buffer_names): 18 | self.__dict__[n] = self[i] 19 | 20 | def adjust(self, buffer_names, buffers, full_buffer=None): 21 | assert self._buffer_names == tuple(buffer_names) 22 | self._full_buffer = full_buffer 23 | for i, (n, b) in enumerate(zip(buffer_names, buffers)): 24 | self[i] = b 25 | self.__dict__[n] = self[i] 26 | return self 27 | 28 | def _asdict(self): 29 | return dict(zip(self._buffer_names, self)) 30 | 31 | def items(self): 32 | return self._asdict().items() 33 | 34 | def keys(self): 35 | return self._asdict().keys() 36 | 37 | def values(self): 38 | return self._asdict().values() 39 | 40 | def __getitem__(self, item): 41 | if isinstance(item, int): 42 | return super(BufferView, self).__getitem__(item) 43 | if item in self._keys: 44 | return self.__dict__[item] 45 | elif '.' in item: 46 | return get_by_path(self, item) 47 | 48 | raise KeyError('{} is not present. Available items are [{}]' 49 | .format(item, ", ".join(sorted(self._keys)))) 50 | 51 | def __contains__(self, item): 52 | return item in self._buffer_names 53 | -------------------------------------------------------------------------------- /brainstorm/structure/buffers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from __future__ import division, print_function, unicode_literals 5 | 6 | import numpy as np 7 | 8 | from brainstorm.handlers import default_handler 9 | from brainstorm.structure.buffer_structure import BufferStructure 10 | from brainstorm.structure.buffer_views import BufferView 11 | from brainstorm.utils import sort_by_index_key 12 | 13 | 14 | def create_buffer_views_from_layout(layout, buffers, hubs, existing_view=None): 15 | if '@slice' in layout: 16 | buffer_nr = layout['@hub'] 17 | feature_slice = slice(*layout['@slice']) 18 | structure = BufferStructure.from_layout(layout) 19 | full_buffer = structure.create_from_buffer_hub( 20 | buffers[buffer_nr], hubs[buffer_nr], feature_slice) 21 | else: 22 | full_buffer = None 23 | 24 | if layout['@type'] == 'BufferView': 25 | names, child_buffers = [], [] 26 | for n, sub_node in sorted(layout.items(), key=sort_by_index_key): 27 | if n.startswith('@'): 28 | continue 29 | if existing_view: 30 | assert n in existing_view 31 | c = create_buffer_views_from_layout( 32 | sub_node, buffers, hubs, existing_view=existing_view[n]) 33 | else: 34 | c = create_buffer_views_from_layout(sub_node, buffers, hubs) 35 | names.append(n) 36 | child_buffers.append(c) 37 | 38 | if existing_view: 39 | return existing_view.adjust(names, child_buffers, full_buffer) 40 | else: 41 | return BufferView(names, child_buffers, full_buffer) 42 | else: # layout['@type'] == 'array': 43 | assert full_buffer is not None, layout 44 | return full_buffer 45 | 46 | 47 | def get_total_size_slices_and_shapes(hubs, time_size, batch_size): 48 | shapes = [h.get_shape(time_size, batch_size) for h in hubs] 49 | totals = np.cumsum([0] + [int(np.prod(s)) for s in shapes]) 50 | size = int(totals[-1]) 51 | slices = [slice(int(i), int(j)) 52 | for i, j in zip(totals[:-1], totals[1:])] 53 | return size, slices, shapes 54 | 55 | 56 | class BufferManager(object): 57 | def __init__(self, layout, hubs, handler=default_handler): 58 | self.hubs = hubs 59 | self.handler = handler 60 | self.layout = layout 61 | self.time_size = -1 62 | self.batch_size = -1 63 | self.size = -1 64 | self.full_buffer = None 65 | self.buffers = [] 66 | self.views = None 67 | self.resize(0, 0) 68 | 69 | def resize(self, time_size, batch_size): 70 | if time_size == self.time_size and batch_size == self.batch_size: 71 | return self.views # lazy 72 | 73 | self.time_size = time_size 74 | self.batch_size = batch_size 75 | total_size, slices, shapes = get_total_size_slices_and_shapes( 76 | self.hubs, time_size, batch_size) 77 | 78 | if total_size > self.size: 79 | self.full_buffer = self.handler.allocate((total_size,)) 80 | self.size = total_size 81 | 82 | self.buffers = [self.full_buffer[slices[i]].reshape(shapes[i]) 83 | for i in range(len(self.hubs))] 84 | 85 | parameters = None 86 | if self.views is not None: 87 | # copy the parameters 88 | parameters = self.handler.get_numpy_copy(self.views.parameters) 89 | 90 | self.views = create_buffer_views_from_layout( 91 | self.layout, self.buffers, self.hubs, existing_view=self.views) 92 | 93 | if parameters is not None: 94 | self.handler.set_from_numpy(self.views.parameters, parameters) 95 | 96 | return self.views 97 | 98 | def set_handler(self, new_handler): 99 | self.full_buffer = None 100 | self.size = -1 101 | self.time_size = -1 102 | self.batch_size = -1 103 | parameters = None 104 | if self.views is not None: 105 | parameters = self.handler.get_numpy_copy(self.views.parameters) 106 | self.views = None 107 | self.handler = new_handler 108 | self.resize(0, 0) 109 | if parameters is not None: 110 | self.handler.set_from_numpy(self.views.parameters, parameters) 111 | 112 | def get_context(self): 113 | if self.buffers is None: 114 | return None 115 | context = [] 116 | for hub, buf in zip(self.hubs, self.buffers): 117 | if hub.btype != 2 or hub.context_size == 0: 118 | context.append(None) 119 | else: 120 | c = self.handler.zeros( 121 | (hub.context_size, self.batch_size, hub.size)) 122 | 123 | context_start_idx = self.time_size - hub.context_size 124 | context_stop_idx = self.time_size 125 | 126 | self.handler.copy_to(buf[context_start_idx:context_stop_idx], 127 | c) 128 | context.append(c) 129 | 130 | return context 131 | 132 | def apply_context(self, context): 133 | for c, buf in zip(context, self.buffers): 134 | if c is None: 135 | continue 136 | self.handler.copy_to(c, buf[self.time_size:]) 137 | 138 | def clear_context(self): 139 | if self.buffers is None: 140 | return None 141 | for hub, buf in zip(self.hubs, self.buffers): 142 | if hub.btype != 2 or not hub.context_size: 143 | continue 144 | self.handler.fill( 145 | buf[self.time_size - hub.context_size:], 0.) 146 | 147 | def clear_backward_buffers(self): 148 | for h, b in zip(self.hubs, self.buffers): 149 | if h.is_backward_only: 150 | self.handler.fill(b, 0.) 151 | -------------------------------------------------------------------------------- /brainstorm/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | -------------------------------------------------------------------------------- /brainstorm/tests/conftest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import pytest 6 | 7 | from brainstorm.structure.architecture import \ 8 | instantiate_layers_from_architecture 9 | 10 | 11 | def pytest_addoption(parser): 12 | parser.addoption("--skipslow", action="store_true", 13 | help="skip slow tests") 14 | 15 | 16 | def pytest_runtest_setup(item): 17 | if 'slow' in item.keywords and item.config.getoption("--skipslow"): 18 | pytest.skip("skipped because of --skipslow option") 19 | 20 | 21 | # /--- A -- C-- 22 | # Input - / \ 23 | # \--- B ------- D 24 | 25 | @pytest.fixture 26 | def layers(): 27 | arch = { 28 | 'Input': { 29 | '@type': 'Input', 30 | 'out_shapes': {'default': ('T', 'B', 2)}, 31 | '@outgoing_connections': ['A', 'B'] 32 | }, 33 | 'A': { 34 | '@type': 'FullyConnected', 35 | 'size': 3, 36 | '@outgoing_connections': ['C'] 37 | }, 38 | 'B': { 39 | '@type': 'FullyConnected', 40 | 'size': 5, 41 | '@outgoing_connections': ['C', 'D'] 42 | }, 43 | 'C': { 44 | '@type': 'FullyConnected', 45 | 'size': 7, 46 | '@outgoing_connections': ['D'] 47 | }, 48 | 'D': { 49 | '@type': 'FullyConnected', 50 | 'size': 11, 51 | '@outgoing_connections': [] 52 | } 53 | } 54 | return instantiate_layers_from_architecture(arch) 55 | -------------------------------------------------------------------------------- /brainstorm/tests/test_array.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | from brainstorm.handlers.debug_handler import DebugArray 9 | 10 | 11 | @pytest.fixture 12 | def array(): 13 | return DebugArray(np.zeros((11, 7, 5, 3, 2))) 14 | 15 | 16 | def test_array_shape(array): 17 | assert isinstance(array.shape, tuple) 18 | assert array.shape == (11, 7, 5, 3, 2) 19 | 20 | 21 | def test_simple_single_indexing(array): 22 | b = array[1] 23 | assert isinstance(b, type(array)) 24 | assert b.shape == (7, 5, 3, 2) 25 | 26 | with pytest.raises(IndexError): 27 | _ = array[11] 28 | 29 | 30 | def test_simple_double_indexing(array): 31 | b = array[1, 4] 32 | assert isinstance(b, type(array)) 33 | assert b.shape == (5, 3, 2) 34 | 35 | with pytest.raises(IndexError): 36 | _ = array[1, 7] 37 | 38 | 39 | def test_simple_triple_indexing(array): 40 | b = array[1, 4, 0] 41 | assert isinstance(b, type(array)) 42 | assert b.shape == (3, 2) 43 | 44 | with pytest.raises(IndexError): 45 | _ = array[1, 2, 5] 46 | 47 | 48 | def test_simple_quad_indexing(array): 49 | b = array[3, 4, 1, 2] 50 | assert isinstance(b, type(array)) 51 | assert b.shape == (2, ) 52 | 53 | with pytest.raises(IndexError): 54 | _ = array[1, 2, 4, 3] 55 | 56 | 57 | def test_simple_quint_indexing(array): 58 | b = array[3, 4, 1, 2, 0] 59 | assert isinstance(b, type(array)) 60 | assert b.shape == () 61 | 62 | with pytest.raises(IndexError): 63 | _ = array[1, 2, 4, 2, 2] 64 | -------------------------------------------------------------------------------- /brainstorm/tests/test_finite_differences/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | -------------------------------------------------------------------------------- /brainstorm/tests/test_finite_differences/test_deltas.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | from brainstorm.handlers import NumpyHandler 9 | from brainstorm.initializers import Gaussian 10 | from brainstorm.randomness import global_rnd 11 | from brainstorm.structure.network import Network 12 | 13 | from brainstorm.tests.helpers import approx_fprime 14 | 15 | 16 | @pytest.fixture(scope='module') 17 | def input_data(): 18 | global_rnd.set_seed(1337) 19 | return global_rnd.randn(7, 5, 3) 20 | 21 | 22 | @pytest.fixture(scope='module') 23 | def targets(): 24 | global_rnd.set_seed(7331) 25 | return global_rnd.randn(7, 5, 2) 26 | 27 | architectures = [{ 28 | 'Input': { 29 | '@type': 'Input', 30 | 'out_shapes': {'default': ('T', 'B', 3,)}, 31 | '@outgoing_connections': { 32 | 'default': ['Output'], 33 | }}, 34 | 'Output': { 35 | '@type': 'FullyConnected', 36 | 'size': 2, 37 | '@outgoing_connections': { 38 | 'default': ['Loss'] 39 | }}, 40 | 'Loss': { 41 | '@type': 'Loss', 42 | '@outgoing_connections': {} 43 | }} 44 | ] 45 | 46 | 47 | @pytest.fixture(scope='module', params=architectures) 48 | def net(request): 49 | n = Network.from_architecture(request.param) 50 | n.set_handler(NumpyHandler(dtype=np.float64)) 51 | n.initialize(Gaussian(1), seed=235) 52 | return n 53 | 54 | 55 | def test_deltas_finite_differences(net, input_data): 56 | # ######## calculate deltas ########## 57 | net.provide_external_data({'default': input_data}) 58 | net.forward_pass(training_pass=True) 59 | net.backward_pass() 60 | delta_calc = net.buffer.Input.output_deltas.default.flatten() 61 | 62 | # ######## estimate deltas ########## 63 | def f(x): 64 | net.provide_external_data({'default': x.reshape(input_data.shape)}) 65 | net.forward_pass() 66 | return net.get_loss_values()['total_loss'] 67 | delta_approx = approx_fprime(input_data.copy().flatten(), f, 1e-5) 68 | 69 | # ######## compare them ############# 70 | nr_sequences = input_data.shape[1] 71 | mse = np.sum((delta_approx - delta_calc) ** 2) / nr_sequences 72 | if mse > 1e-4: 73 | diff = (delta_approx - delta_calc).reshape(input_data.shape) 74 | for t in range(diff.shape[0]): 75 | print("======== t=%d =========" % t) 76 | print(diff[t]) 77 | # print("Checking Deltas = %0.4f" % mse) 78 | 79 | assert mse < 1e-4 80 | 81 | 82 | def test_gradient_finite_differences(net, input_data): 83 | # ######## calculate deltas ########## 84 | net.provide_external_data({'default': input_data}) 85 | net.forward_pass(training_pass=True) 86 | net.backward_pass() 87 | gradient_calc = net.buffer.gradients 88 | 89 | # ######## estimate deltas ########## 90 | def f(x): 91 | net.buffer.parameters[:] = x 92 | net.forward_pass() 93 | return net.get_loss_values()['total_loss'] 94 | initial_weigths = net.buffer.parameters.copy() 95 | gradient_approx = approx_fprime(initial_weigths, f, 1e-6) 96 | 97 | # ######## compare them ############# 98 | nr_sequences = input_data.shape[1] 99 | diff = gradient_approx - gradient_calc 100 | mse = np.sum(diff ** 2) / nr_sequences 101 | if mse > 1e-4: 102 | # Hijack the network gradient buffer for the view 103 | net.buffer.gradients[:] = diff 104 | for layer_name in net.layers: 105 | if not net.buffer[layer_name]: 106 | continue 107 | print("============= Layer: {} =============".format(layer_name)) 108 | for view_name in net.buffer[layer_name].gradients.keys(): 109 | print("------------- {} -------------".format(view_name)) 110 | print(net.buffer[layer_name].gradients[view_name]) 111 | 112 | # print(">> Checking Gradient = %0.4f" % mse) 113 | assert mse < 1e-4 114 | -------------------------------------------------------------------------------- /brainstorm/tests/test_initializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import mock 6 | import numpy as np 7 | import pytest 8 | 9 | from brainstorm.initializers import (DenseSqrtFanIn, DenseSqrtFanInOut, 10 | EchoState, Gaussian, InitializationError, 11 | Initializer, SparseInputs, SparseOutputs, 12 | Uniform, evaluate_initializer) 13 | 14 | 15 | @pytest.mark.parametrize("initializer", [Gaussian(), 16 | Uniform(), 17 | DenseSqrtFanIn(), 18 | DenseSqrtFanInOut()]) 19 | def test_initializers_return_correct_shape(initializer): 20 | shape = (4, 7) 21 | result = initializer(shape) 22 | assert isinstance(result, np.ndarray) 23 | assert result.shape == shape 24 | 25 | 26 | @pytest.mark.parametrize("initializer", [Gaussian(), 27 | Uniform()]) 28 | def test_universal_initializers_work_on_biases(initializer): 29 | shape = (5,) 30 | result = initializer(shape) 31 | assert isinstance(result, np.ndarray) 32 | assert result.shape == shape 33 | 34 | 35 | @pytest.mark.parametrize("initializer", [DenseSqrtFanInOut(), 36 | DenseSqrtFanIn(), 37 | SparseInputs(0), 38 | SparseOutputs(0), 39 | EchoState()]) 40 | def test_matrix_initializers_raise_on_1d_matrices(initializer): 41 | with pytest.raises(InitializationError): 42 | initializer((20,)) 43 | 44 | 45 | # ################ EchoState ################################################## 46 | 47 | def test_echo_state_raises_if_matrix_non_square(): 48 | with pytest.raises(InitializationError): 49 | EchoState()((3, 5)) 50 | 51 | 52 | def test_echostate_returns_correct_shape(): 53 | result = EchoState()((4, 4)) 54 | assert isinstance(result, np.ndarray) 55 | assert result.shape == (4, 4) 56 | 57 | 58 | def test_echostate_has_correct_spectral_radius(): 59 | result = EchoState(spectral_radius=1.7)((4, 4)) 60 | spectral_radius = max(abs(np.linalg.eig(result)[0])) 61 | assert abs(spectral_radius - 1.7) < 1e-12 62 | 63 | 64 | # ################ SparseInputs & SparseOutputs ############################### 65 | 66 | def test_sparse_inputs_raises_if_input_dim_too_small(): 67 | with pytest.raises(InitializationError): 68 | SparseInputs(1, connections=17)((15, 20)) 69 | 70 | 71 | def test_sparse_outputs_raises_if_output_dim_too_small(): 72 | with pytest.raises(InitializationError): 73 | SparseOutputs(1, connections=17)((20, 15)) 74 | 75 | 76 | def test_sparse_inputs_has_correct_number_of_nonzero_rows(): 77 | res = SparseInputs(1, connections=17)((25, 12)) 78 | assert np.all(np.sum(res > 0, axis=0) == 17) 79 | 80 | 81 | def test_sparse_outputs_has_correct_number_of_nonzero_cols(): 82 | res = SparseOutputs(1, connections=17)((12, 25)) 83 | assert np.all(np.sum(res > 0, axis=1) == 17) 84 | 85 | 86 | # ########################## evaluate_initializer ############################# 87 | 88 | def test_evaluate_initializer_with_number(): 89 | assert np.all(evaluate_initializer(1.4, (7, 5)) == 1.4) 90 | 91 | 92 | def test_evaluate_initializer_calls_initializer(): 93 | init = mock.create_autospec(Initializer()) 94 | init.side_effect = lambda x: np.array(1) 95 | evaluate_initializer(init, (7, 5)) 96 | init.assert_called_once_with((7, 5)) 97 | 98 | 99 | def test_evaluate_initializer_without_fallback_propagates_error(): 100 | init = mock.create_autospec(Initializer()) 101 | init.side_effect = InitializationError 102 | with pytest.raises(InitializationError): 103 | evaluate_initializer(init, (7, 5)) 104 | 105 | 106 | def test_evaluate_initializer_with_fallback_calls_fallback(): 107 | init = mock.create_autospec(Initializer()) 108 | fallback = mock.create_autospec(Initializer()) 109 | fallback.side_effect = lambda x: np.array(1) 110 | init.side_effect = InitializationError 111 | evaluate_initializer(init, (7, 5), fallback) 112 | fallback.assert_called_once_with((7, 5)) 113 | -------------------------------------------------------------------------------- /brainstorm/tests/test_integration/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | -------------------------------------------------------------------------------- /brainstorm/tests/test_integration/test_initialization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | import brainstorm as bs 9 | from brainstorm.utils import NetworkValidationError 10 | 11 | 12 | @pytest.fixture 13 | def net(): 14 | net = bs.Network.from_layer( 15 | bs.layers.Input(out_shapes={'default': ('T', 'B', 1)}) >> 16 | bs.layers.FullyConnected(2) >> 17 | bs.layers.FullyConnected(3) >> 18 | bs.layers.FullyConnected(1, name='OutputLayer') 19 | ) 20 | return net 21 | 22 | 23 | def test_initialize_default(net): 24 | net.initialize(7) 25 | assert np.all(net.buffer.parameters == 7) 26 | 27 | 28 | def test_initialze_layerwise_dict(net): 29 | net.initialize({ 30 | 'FullyConnected_1': 1, 31 | 'FullyConnected_2': 2, 32 | 'OutputLayer': 3}) 33 | 34 | assert np.all(net.buffer.FullyConnected_1.parameters.W == 1) 35 | assert np.all(net.buffer.FullyConnected_2.parameters.W == 2) 36 | assert np.all(net.buffer.OutputLayer.parameters.W == 3) 37 | 38 | 39 | def test_initialize_layerwise_kwargs(net): 40 | net.initialize( 41 | FullyConnected_1=1, 42 | FullyConnected_2=2, 43 | OutputLayer=3) 44 | 45 | assert np.all(net.buffer.FullyConnected_1.parameters.W == 1) 46 | assert np.all(net.buffer.FullyConnected_2.parameters.W == 2) 47 | assert np.all(net.buffer.OutputLayer.parameters.W == 3) 48 | 49 | 50 | def test_initialize_layerwise_plus_default_kwargs(net): 51 | net.initialize(7, 52 | FullyConnected_1=1, 53 | OutputLayer=3) 54 | 55 | assert np.all(net.buffer.FullyConnected_1.parameters.W == 1) 56 | assert np.all(net.buffer.FullyConnected_2.parameters.W == 7) 57 | assert np.all(net.buffer.OutputLayer.parameters.W == 3) 58 | 59 | 60 | def test_initialize_weightwise(net): 61 | net.initialize(7, 62 | FullyConnected_1={'W': 1}, 63 | FullyConnected_2={'W': 2, 'default': 3}, 64 | OutputLayer={'bias': 4}) 65 | assert np.all(net.buffer.FullyConnected_1.parameters.W == 1) 66 | assert np.all(net.buffer.FullyConnected_1.parameters.bias == 7) 67 | assert np.all(net.buffer.FullyConnected_2.parameters.W == 2) 68 | assert np.all(net.buffer.FullyConnected_2.parameters.bias == 3) 69 | assert np.all(net.buffer.OutputLayer.parameters.W == 7) 70 | assert np.all(net.buffer.OutputLayer.parameters.bias == 4) 71 | 72 | 73 | def test_initialize_with_array(net): 74 | net.initialize(0, 75 | FullyConnected_1={'bias': [1, 2]}, 76 | FullyConnected_2={'bias': np.array([3, 4, 5]), 77 | 'W': [[6, 7], 78 | [8, 9], 79 | [10, 11]]}, 80 | OutputLayer={'bias': [12]}) 81 | 82 | assert np.all(net.buffer.FullyConnected_1.parameters.W == 0) 83 | assert np.all(net.buffer.FullyConnected_1.parameters.bias == 84 | [1, 2]) 85 | assert np.all(net.buffer.FullyConnected_2.parameters.bias == 86 | [3, 4, 5]) 87 | assert np.all(net.buffer.FullyConnected_2.parameters.W == 88 | [[6, 7], [8, 9], [10, 11]]) 89 | assert np.all(net.buffer.OutputLayer.parameters.W == 0) 90 | assert np.all(net.buffer.OutputLayer.parameters.bias == 12) 91 | 92 | 93 | def test_initialize_with_initializer(net): 94 | net.initialize( 95 | default=bs.initializers.Uniform(0, 1), 96 | FullyConnected_1=bs.initializers.Uniform(1, 2), 97 | FullyConnected_2={'W': bs.initializers.Uniform(2, 3)}, 98 | OutputLayer={'W': bs.initializers.Uniform(3, 4), 99 | 'bias': bs.initializers.Uniform(4, 5)} 100 | ) 101 | 102 | layer1 = net.buffer.FullyConnected_1.parameters 103 | layer2 = net.buffer.FullyConnected_2.parameters 104 | layer3 = net.buffer.OutputLayer.parameters 105 | 106 | assert np.all(1 <= layer1.W) and np.all(layer1.W <= 2) 107 | assert np.all(1 <= layer1.bias) and np.all(layer1.bias <= 2) 108 | assert np.all(2 <= layer2.W) and np.all(layer2.W <= 3) 109 | assert np.all(0 <= layer2.bias) and np.all(layer2.bias <= 1) 110 | assert np.all(3 <= layer3.W) and np.all(layer3.W <= 4) 111 | assert np.all(4 <= layer3.bias) and np.all(layer3.bias <= 5) 112 | 113 | 114 | def test_initialize_with_layer_pattern(net): 115 | net.initialize({'default': 0, 'Fully*': 1}) 116 | 117 | assert np.all(net.buffer.FullyConnected_1.parameters.W == 1) 118 | assert np.all(net.buffer.FullyConnected_2.parameters.W == 1) 119 | assert np.all(net.buffer.OutputLayer.parameters.W == 0) 120 | 121 | 122 | def test_initialize_with_nonmatching_pattern_raises(net): 123 | with pytest.raises(NetworkValidationError): 124 | net.initialize({'default': 0, 'LSTM*': 1}) 125 | 126 | with pytest.raises(NetworkValidationError): 127 | net.initialize({'default': 0, 'OutputLayer': {'*_bias': 1}}) 128 | 129 | 130 | def test_initialize_raises_on_missing_initializers(net): 131 | with pytest.raises(NetworkValidationError): 132 | net.initialize({'FullyConnected*': 1}) 133 | 134 | with pytest.raises(NetworkValidationError): 135 | net.initialize( 136 | FullyConnected_1=1, 137 | FullyConnected_2={'b': 2}, 138 | OutputLayer=3) 139 | -------------------------------------------------------------------------------- /brainstorm/tests/test_integration/test_xor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | from brainstorm import Network, Trainer 9 | from brainstorm.data_iterators import Undivided 10 | from brainstorm.hooks import StopAfterEpoch 11 | from brainstorm.initializers import Gaussian 12 | from brainstorm.layers import * 13 | from brainstorm.training import SgdStepper 14 | 15 | 16 | @pytest.mark.slow 17 | def test_learn_xor_function(): 18 | # set up the network 19 | inp = Input(out_shapes={'default': ('T', 'B', 2), 20 | 'targets': ('T', 'B', 1)}) 21 | error_func = BinomialCrossEntropy() 22 | 23 | (inp >> 24 | FullyConnected(2, activation='sigmoid') >> 25 | FullyConnected(1, activation='sigmoid', name='OutLayer') >> 26 | error_func >> 27 | Loss()) 28 | 29 | net = Network.from_layer(inp - 'targets' >> 'targets' - error_func) 30 | # net.set_handler(PyCudaHandler()) 31 | net.initialize(Gaussian(1.0), seed=42) # high weight-init needed 32 | # print(net.buffer.parameters) 33 | 34 | # set up the trainer 35 | tr = Trainer(SgdStepper(learning_rate=4.0), verbose=False) 36 | tr.add_hook(StopAfterEpoch(300)) 37 | 38 | # generate the data 39 | data = np.array([ 40 | [0., 0.], 41 | [0., 1.], 42 | [1., 0.], 43 | [1., 1.] 44 | ]).reshape((1, 4, 2)) 45 | targets = np.array([0., 1., 1., 0.]).reshape((1, 4, 1)) 46 | 47 | tr.train(net, Undivided(default=data, targets=targets)) 48 | 49 | out = net.buffer.OutLayer.outputs.default 50 | success = np.all(np.round(out) == targets) 51 | if not success: 52 | print('Network output:', out.flatten()) 53 | print('Rounded output:', np.round(out.flatten())) 54 | print('Targets :', targets.flatten()) 55 | raise AssertionError("Network training did not succeed.") 56 | assert min(tr.logs['rolling_training']['total_loss']) < 0.5 57 | -------------------------------------------------------------------------------- /brainstorm/tests/test_network.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from __future__ import division, print_function, unicode_literals 5 | 6 | import numpy as np 7 | import pytest 8 | 9 | from brainstorm import Network 10 | from brainstorm.data_iterators import Undivided 11 | from brainstorm.initializers import Gaussian 12 | from brainstorm.layers import SoftmaxCE, Input, Lstm, Recurrent, FullyConnected 13 | from brainstorm.training.utils import run_network 14 | 15 | from brainstorm.tests.helpers import HANDLER 16 | 17 | np.random.seed(1234) 18 | 19 | NO_CON = set() 20 | 21 | 22 | def simple_recurrent_net(): 23 | inp = Input(out_shapes={'default': ('T', 'B', 2)}) 24 | net = Network.from_layer(inp >> Recurrent(3, name='out')) 25 | return net 26 | 27 | 28 | def lstm_net(): 29 | inp = Input(out_shapes={'default': ('T', 'B', 2)}) 30 | net = Network.from_layer(inp >> Lstm(3, name='out')) 31 | return net 32 | 33 | 34 | layers_to_test_with_context = [ 35 | simple_recurrent_net, 36 | lstm_net 37 | ] 38 | 39 | ids = [f.__name__ for f in layers_to_test_with_context] 40 | 41 | 42 | @pytest.fixture(params=layers_to_test_with_context, ids=ids) 43 | def net_with_context(request): 44 | net = request.param() 45 | return net 46 | 47 | 48 | def test_context_slice_allows_continuing_forward_pass(net_with_context): 49 | net = net_with_context 50 | net.set_handler(HANDLER) 51 | net.initialize(Gaussian(0.1), seed=1234) 52 | all_data = np.random.randn(4, 1, 2) 53 | 54 | # First do a pass on all the data 55 | net.provide_external_data({'default': all_data}) 56 | net.forward_pass() 57 | final_context = [HANDLER.get_numpy_copy(x) if x is not None else None 58 | for x in net.get_context()] 59 | final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default) 60 | 61 | # Pass only part of data 62 | data_a = all_data[:2] 63 | net.provide_external_data({'default': data_a}) 64 | net.forward_pass() 65 | 66 | # Pass rest of data with context 67 | data_b = all_data[2:] 68 | net.provide_external_data({'default': data_b}) 69 | net.forward_pass(context=net.get_context()) 70 | context = [HANDLER.get_numpy_copy(x) if x is not None else None 71 | for x in net.get_context()] 72 | outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default) 73 | 74 | # Check if outputs are the same as final_outputs 75 | success = np.allclose(outputs[:-1], final_outputs[2:-1]) 76 | if not success: 77 | print("Outputs:\n", outputs[:-1]) 78 | print("Should match:\n", final_outputs[2:-1]) 79 | raise AssertionError("Outputs did not match.") 80 | 81 | # Check if context is same as final_context 82 | assert len(context) == len(final_context), "Context list sizes mismatch!" 83 | 84 | for (x, y) in zip(context, final_context): 85 | if x is None: 86 | assert y is None 87 | else: 88 | # print("Context:\n", x) 89 | # print("Should match:\n", y) 90 | assert np.allclose(x, y) 91 | 92 | 93 | inp = Input(out_shapes={'default': ('T', 'B', 4), 94 | 'targets': ('T', 'B', 1)}) 95 | hid = FullyConnected(2, name="Hid") 96 | out = SoftmaxCE(name='Output') 97 | (inp - 'targets' >> 'targets' - out) 98 | simple_net = Network.from_layer(inp >> hid >> out) 99 | 100 | 101 | def test_forward_pass_with_missing_data(): 102 | it = Undivided(default=np.random.randn(3, 2, 4))(simple_net.handler) 103 | 104 | with pytest.raises(KeyError): 105 | for _ in run_network(simple_net, it): 106 | pass 107 | 108 | for _ in run_network(simple_net, it, all_inputs=False): 109 | pass 110 | -------------------------------------------------------------------------------- /brainstorm/tests/test_randomness.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import pickle 6 | from copy import copy 7 | 8 | import pytest 9 | 10 | from brainstorm.describable import (Describable, create_from_description, 11 | get_description) 12 | from brainstorm.randomness import RandomState, Seedable, global_rnd 13 | 14 | 15 | # ##################### RandomState ########################################### 16 | 17 | @pytest.fixture 18 | def rnd(): 19 | return RandomState(1) 20 | 21 | 22 | def test_randomstate_constructor_without_arg(): 23 | rnd1 = RandomState() 24 | rnd2 = RandomState() 25 | assert rnd1.get_seed() != rnd2.get_seed() 26 | 27 | 28 | def test_randomstate_constructor_with_seed(): 29 | rnd1 = RandomState(2) 30 | assert rnd1.get_seed() == 2 31 | 32 | 33 | def test_randomstate_set_seed(rnd): 34 | rnd.set_seed(23) 35 | assert rnd.get_seed() == 23 36 | 37 | 38 | def test_randomstate_randint_randomness(rnd): 39 | a = rnd.randint(10000) 40 | b = rnd.randint(10000) 41 | assert a != b 42 | 43 | 44 | def test_randomstate_seeded_randint_deterministic(rnd): 45 | rnd.set_seed(1) 46 | a = rnd.randint(10000) 47 | rnd.set_seed(1) 48 | b = rnd.randint(10000) 49 | assert a == b 50 | 51 | 52 | def test_randomstate_reset_randint_deterministic(rnd): 53 | a = rnd.randint(10000) 54 | rnd.reset() 55 | b = rnd.randint(10000) 56 | assert a == b 57 | 58 | 59 | def test_randomstate_get_new_random_state_randomness(rnd): 60 | rnd1 = rnd.create_random_state() 61 | rnd2 = rnd.create_random_state() 62 | assert rnd1.get_seed() != rnd2.get_seed 63 | 64 | 65 | def test_randomstate_seeded_get_new_random_state_deterministic(rnd): 66 | rnd1 = rnd.create_random_state() 67 | rnd.reset() 68 | rnd2 = rnd.create_random_state() 69 | assert rnd1.get_seed() == rnd2.get_seed() 70 | 71 | 72 | # ################## global_rnd ############################################### 73 | 74 | def test_global_rnd_exists(): 75 | assert isinstance(global_rnd, RandomState) 76 | 77 | 78 | # ################## Seedable ################################################# 79 | 80 | def test_seedable_constructor_without_seed(): 81 | seedable1 = Seedable() 82 | seedable2 = Seedable() 83 | assert seedable1.rnd.get_seed() != seedable2.rnd.get_seed() 84 | 85 | 86 | def test_seedable_constructor_with_seed(): 87 | seedable = Seedable(1) 88 | assert seedable.rnd.get_seed() == 1 89 | 90 | 91 | def test_seedable_description_does_not_include_rnd1(): 92 | class Foo0(Seedable): 93 | pass 94 | 95 | assert get_description(Foo0()) == {'@type': 'Foo0'} 96 | 97 | 98 | def test_seedable_description_does_not_include_rnd(): 99 | class Foo1(Seedable, Describable): 100 | pass 101 | 102 | assert get_description(Foo1()) == {'@type': 'Foo1'} 103 | 104 | 105 | def test_seedable_initializes_from_description1(): 106 | class Foo2(Seedable, Describable): 107 | pass 108 | 109 | f = create_from_description({'@type': 'Foo2'}) 110 | assert hasattr(f, 'rnd') 111 | assert isinstance(f.rnd, RandomState) 112 | f.rnd.randint(100) # assert no throw 113 | 114 | 115 | def test_seedable_initializes_from_description2(): 116 | class Foo3(Seedable, Describable): 117 | def __init_from_description__(self, description): 118 | super(Foo3, self).__init_from_description__(description) 119 | 120 | f = create_from_description({'@type': 'Foo3'}) 121 | assert hasattr(f, 'rnd') 122 | assert isinstance(f.rnd, RandomState) 123 | f.rnd.randint(100) # assert no throw 124 | 125 | 126 | def test_random_state_copyable(): 127 | r = RandomState(127) 128 | _ = r.randint(10) 129 | r2 = copy(r) 130 | assert r.get_seed() == r2.get_seed() 131 | assert r.generate_seed() == r2.generate_seed() 132 | 133 | 134 | def test_random_state_pickleable(): 135 | r = RandomState(127) 136 | _ = r.randint(10) 137 | s = pickle.dumps(r) 138 | r2 = pickle.loads(s) 139 | assert r.get_seed() == r2.get_seed() 140 | assert r.generate_seed() == r2.generate_seed() 141 | -------------------------------------------------------------------------------- /brainstorm/tests/test_schedules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from __future__ import division, print_function, unicode_literals 5 | 6 | import pytest 7 | import six 8 | import numpy as np 9 | 10 | from brainstorm.training.schedules import Exponential, Linear, MultiStep, DecreaseAfterEpoch 11 | 12 | 13 | def test_linear(): 14 | sch = Linear(initial_value=1.0, final_value=0.5, num_changes=5) 15 | epochs = [0] * 2 + [1] * 2 + [2] * 2 + [3] * 2 + [4] * 2 16 | updates = range(10) 17 | 18 | values = [sch(epoch, update, 'epoch', 1, None, None, None) 19 | for epoch, update in six.moves.zip(epochs, updates)] 20 | assert values == [1.0, 1.0, 0.9, 0.9, 0.8, 0.8, 0.7, 0.7, 0.6, 0.6] 21 | 22 | values = [sch(epoch, update, 'update', 1, None, None, None) 23 | for epoch, update in six.moves.zip(epochs, updates)] 24 | assert values == [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.5, 0.5, 0.5, 0.5] 25 | 26 | values = [sch(epoch, update, 'update', 3, None, None, None) 27 | for epoch, update in six.moves.zip(epochs, updates)] 28 | assert values == [1.0, 1.0, 1.0, 0.9, 0.9, 0.9, 0.8, 0.8, 0.8, 0.7] 29 | 30 | 31 | def test_exponential(): 32 | sch = Exponential(initial_value=1.0, factor=0.99, minimum=0.97) 33 | epochs = [0] * 4 + [1] * 4 + [2] * 4 34 | updates = range(12) 35 | 36 | values = [sch(epoch, update, 'epoch', 1, None, None, None) 37 | for epoch, update in six.moves.zip(epochs, updates)] 38 | assert values == [1.0] * 4 + [0.99] * 4 + [0.99 * 0.99] * 4 39 | 40 | values = [sch(epoch, update, 'update', 1, None, None, None) 41 | for epoch, update in six.moves.zip(epochs, updates)] 42 | assert values == [1.0 * (0.99 ** x) for x in range(4)] + [0.97] * 8 43 | 44 | values = [sch(epoch, update, 'update', 3, None, None, None) 45 | for epoch, update in six.moves.zip(epochs, updates)] 46 | assert values == [1.0] * 3 + [0.99] * 3 + [0.9801] * 3 + [0.99 ** 3] * 3 47 | 48 | 49 | def test_multistep(): 50 | sch = MultiStep(initial_value=1.0, steps=[3, 5, 8], 51 | values=[0.1, 0.01, 0.001]) 52 | epochs = [0] * 2 + [1] * 2 + [2] * 2 + [3] * 2 + [4] * 2 53 | updates = range(10) 54 | 55 | values = [sch(epoch, update, 'epoch', 1, None, None, None) 56 | for epoch, update in six.moves.zip(epochs, updates)] 57 | assert values == [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.1, 0.1, 0.1, 0.1] 58 | 59 | values = [sch(epoch, update, 'update', 1, None, None, None) 60 | for epoch, update in six.moves.zip(epochs, updates)] 61 | assert values == [1.0, 1.0, 1.0, 0.1, 0.1, 0.01, 0.01, 0.01, 0.001, 0.001] 62 | 63 | with pytest.raises(AssertionError): 64 | _ = sch(0, 0, 'update', 3, None, None, None) 65 | 66 | 67 | def test_decrease_after_epoch(): 68 | sch = DecreaseAfterEpoch(initial_value=0.02, T=6) 69 | epochs = range(10) 70 | updates = range(10) 71 | 72 | values = [sch(epoch, update, 'epoch', 0, None, None, None) 73 | for epoch, update in six.moves.zip(epochs, updates)] 74 | assert np.allclose(values, [0.02] * 6 + [0.0171428, 0.015, 0.0133333, 0.012]) 75 | -------------------------------------------------------------------------------- /brainstorm/tests/test_scorers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | from brainstorm.scorers import Accuracy, Hamming, MeanSquaredError 9 | 10 | 11 | def accuracy(): 12 | true_labels = np.array([0, 3, 1, 2, 3, 2, 0, 1]).reshape(-1, 1) 13 | predictions = np.eye(4)[[0, 1, 1, 2, 3, 3, 1, 2]] 14 | scorer = Accuracy() 15 | expected = 0.5 16 | return scorer, true_labels, predictions, expected 17 | 18 | 19 | def hamming(): 20 | true_labels = np.array([[0, 1, 1], 21 | [0, 1, 0], 22 | [0, 0, 0], 23 | [1, 1, 0]]) 24 | predictions = np.array([[0.1, 0.7, 0.4], 25 | [0.9, 0.3, .1], 26 | [0.8, 0.2, 0.1], 27 | [0.5, 0.4, 0.2]]) # 7 / 12 correct 28 | 29 | scorer = Hamming(threshold=0.5) 30 | expected = 7./12. 31 | return scorer, true_labels, predictions, expected 32 | 33 | 34 | def mean_squared_error(): 35 | true_labels = np.array([0, 1, 2, 3, 4, 5]).reshape(-1, 1) 36 | predictions = np.array([2, 1, 2, 3.5, 4, 4]).reshape(-1, 1) 37 | scorer = MeanSquaredError() 38 | expected = 0.5 * (4 + 0 + 0 + 0.25 + 0 + 1) / 6 39 | return scorer, true_labels, predictions, expected 40 | 41 | 42 | scorers_to_test = [ 43 | accuracy, 44 | hamming, 45 | mean_squared_error 46 | ] 47 | 48 | scorer_ids = [s.__name__ for s in scorers_to_test] 49 | 50 | 51 | @pytest.fixture(params=scorers_to_test, ids=scorer_ids) 52 | def scorer_test_case(request): 53 | return request.param() 54 | 55 | 56 | @pytest.mark.parametrize('batch_size', [1, 2, 3, 100]) 57 | def test_scorer_with_different_batchsizes(scorer_test_case, batch_size): 58 | scorer, true_labels, predictions, expected = scorer_test_case 59 | errors = [] 60 | for i in range(0, true_labels.shape[0], batch_size): 61 | t = true_labels[i:i+batch_size] 62 | p = predictions[i:i+batch_size] 63 | errors.append((t.shape[0], scorer(t, p))) 64 | assert np.allclose(scorer.aggregate(errors), expected) 65 | 66 | 67 | def test_scorer_with_mask(scorer_test_case): 68 | scorer, true_labels, predictions, expected = scorer_test_case 69 | errors1 = [(3, scorer(true_labels[:3], predictions[:3]))] 70 | mask = np.zeros((true_labels.shape[0], 1)) 71 | mask[:3] = 1.0 72 | errors2 = [(3, scorer(true_labels, predictions, mask=mask))] 73 | 74 | assert scorer.aggregate(errors1) == scorer.aggregate(errors2) 75 | -------------------------------------------------------------------------------- /brainstorm/tests/test_structure/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | -------------------------------------------------------------------------------- /brainstorm/tests/test_structure/test_buffer_view.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | from brainstorm.structure.buffers import BufferView 9 | 10 | 11 | @pytest.fixture 12 | def buffer_view(): 13 | buffer_names = ['W', 'R', 'b'] 14 | full_buffer = np.zeros(27) 15 | buffers = [full_buffer[:15].reshape((5, 3)), 16 | full_buffer[15:24].reshape((3, 3)), 17 | full_buffer[24:27].reshape((3,))] 18 | return BufferView(buffer_names, buffers, full_buffer) 19 | 20 | 21 | def test_buffer_view_retains_full_buffer(buffer_view): 22 | assert buffer_view._full_buffer.size == 27 23 | assert isinstance(buffer_view._full_buffer, np.ndarray) 24 | 25 | 26 | def test_buffer_view_buffer_slicing(): 27 | buffer_names = ['W', 'R', 'b'] 28 | full_buffer = np.arange(11) 29 | buffers = [full_buffer[:6].reshape((3, 2)), 30 | full_buffer[6:10].reshape((2, 2)), 31 | full_buffer[10:].reshape((1,))] 32 | bv = BufferView(buffer_names, buffers, full_buffer) 33 | 34 | assert bv._full_buffer is full_buffer 35 | assert np.all(bv[0] == np.array([0, 1, 2, 3, 4, 5]).reshape(3, 2)) 36 | assert np.all(bv[1] == np.array([6, 7, 8, 9]).reshape(2, 2)) 37 | assert np.all(bv[2] == np.array([10])) 38 | 39 | 40 | def test_empty_list_param_view(): 41 | buff = np.zeros(0) 42 | empty = BufferView([], [], buff) 43 | assert empty._full_buffer is buff 44 | assert empty._buffer_names == () 45 | assert set(empty.__dict__.keys()) == {'_full_buffer', '_buffer_names', 46 | '_keys'} 47 | 48 | 49 | def test_buffer_view_tuple_upacking(buffer_view): 50 | W, R, b = buffer_view 51 | assert isinstance(W, np.ndarray) 52 | assert W.shape == (5, 3) 53 | assert isinstance(R, np.ndarray) 54 | assert R.shape == (3, 3) 55 | assert isinstance(b, np.ndarray) 56 | assert b.shape == (3,) 57 | 58 | 59 | def test_buffer_view_len(buffer_view): 60 | assert len(buffer_view) == 3 61 | 62 | 63 | def test_buffer_view_tuple_getitem(buffer_view): 64 | W, R, b = buffer_view 65 | assert buffer_view[0] is W 66 | assert buffer_view[1] is R 67 | assert buffer_view[2] is b 68 | 69 | with pytest.raises(IndexError): 70 | _ = buffer_view[3] 71 | 72 | 73 | def test_buffer_view_dict_getitem(buffer_view): 74 | W, R, b = buffer_view 75 | assert buffer_view['W'] is W 76 | assert buffer_view['R'] is R 77 | assert buffer_view['b'] is b 78 | 79 | with pytest.raises(KeyError): 80 | _ = buffer_view['nonexisting'] 81 | 82 | 83 | def test_buffer_view_contains(buffer_view): 84 | 85 | assert 'W' in buffer_view 86 | assert 'R' in buffer_view 87 | assert 'b' in buffer_view 88 | 89 | assert 'foo' not in buffer_view 90 | assert 'Q' not in buffer_view 91 | 92 | 93 | def test_buffer_view_getattr(buffer_view): 94 | W, R, b = buffer_view 95 | assert buffer_view.W is W 96 | assert buffer_view.R is R 97 | assert buffer_view.b is b 98 | 99 | with pytest.raises(AttributeError): 100 | _ = buffer_view.nonexisting 101 | 102 | 103 | def test_buffer_view_dict_for_autocompletion(buffer_view): 104 | assert set(buffer_view.__dict__.keys()) == {'W', 'R', 'b', '_buffer_names', 105 | '_full_buffer', '_keys'} 106 | 107 | 108 | def test_buffer_view_as_dict(buffer_view): 109 | assert buffer_view._asdict() == { 110 | 'W': buffer_view.W, 111 | 'R': buffer_view.R, 112 | 'b': buffer_view.b 113 | } 114 | 115 | 116 | def test_buffer_view_dict_interface(buffer_view): 117 | assert list(buffer_view.keys()) == list(buffer_view._asdict().keys()) 118 | assert list(buffer_view.values()) == list(buffer_view._asdict().values()) 119 | assert list(buffer_view.items()) == list(buffer_view._asdict().items()) 120 | 121 | 122 | def test_deep_copying_of_buffer_view(buffer_view): 123 | foo_names = ['a', 'b'] 124 | foo_buffers = [np.ones(2), np.zeros(3)] 125 | foo_view = BufferView(foo_names, foo_buffers) 126 | 127 | names = ['other', 'foo'] 128 | buffers = [buffer_view, foo_view] 129 | my_buffer = BufferView(names, buffers) 130 | 131 | from copy import deepcopy 132 | 133 | my_buffer_copy = deepcopy(my_buffer) 134 | assert my_buffer_copy.foo is not foo_view 135 | assert my_buffer_copy.other is not buffer_view 136 | 137 | foo_view.a[:] = 7 138 | assert np.all(my_buffer_copy.foo.a == np.ones(2)) 139 | -------------------------------------------------------------------------------- /brainstorm/tests/test_structure/test_construction_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import pytest 6 | 7 | from brainstorm.structure.construction import (ConstructionWrapper, 8 | NetworkValidationError) 9 | 10 | 11 | @pytest.fixture 12 | def layers(): 13 | return [ConstructionWrapper.create('DummyLayerImpl') for _ in range(5)] 14 | 15 | 16 | def test_constructor(): 17 | cl = ConstructionWrapper.create('FooLayerImpl') 18 | assert cl.layer.layer_type == 'Foo' 19 | assert repr(cl) == "" 20 | 21 | 22 | def test_raises_on_invalid_layer_type(): 23 | with pytest.raises(NetworkValidationError): 24 | i = ConstructionWrapper.create('not valid!') 25 | 26 | 27 | def test_raises_on_invalid_layer_name(): 28 | with pytest.raises(NetworkValidationError): 29 | i = ConstructionWrapper.create('layertype', name='also invalid.') 30 | 31 | 32 | def test_connecting_two_layers_sets_sinks_and_sources(layers): 33 | l1, l2, l3, l4, l5 = layers 34 | _ = l1 >> l2 >> l3 35 | assert (l1.layer, 'default', 'default') in l2.layer.incoming 36 | assert ('default', 'default', l2.layer) in l1.layer.outgoing 37 | assert ('default', 'default', l3.layer) in l2.layer.outgoing 38 | 39 | 40 | def test_connect_multiple_targets(layers): 41 | l1, l2, l3, l4, l5 = layers 42 | _ = l1 >> l2 43 | _ = l1 >> l3 44 | _ = l1 >> l4 45 | assert ('default', 'default', l2.layer) in l1.layer.outgoing 46 | assert ('default', 'default', l3.layer) in l1.layer.outgoing 47 | assert ('default', 'default', l4.layer) in l1.layer.outgoing 48 | 49 | 50 | def test_connect_multiple_sources(layers): 51 | l1, l2, l3, l4, l5 = layers 52 | _ = l2 >> l1 53 | _ = l3 >> l1 54 | _ = l4 >> l1 55 | assert (l2.layer, 'default', 'default') in l1.layer.incoming 56 | assert (l3.layer, 'default', 'default') in l1.layer.incoming 57 | assert (l4.layer, 'default', 'default') in l1.layer.incoming 58 | 59 | 60 | def test_connect_named_output(layers): 61 | l1, l2, l3, l4, l5 = layers 62 | _ = l1 >> l2 - 'out1' >> l3 63 | 64 | assert ('default', 'default', l2.layer) in l1.layer.outgoing 65 | assert (l1.layer, 'default', 'default') in l2.layer.incoming 66 | assert (l2.layer, 'out1', 'default') in l3.layer.incoming 67 | assert ('out1', 'default', l3.layer) in l2.layer.outgoing 68 | 69 | 70 | def test_connect_named_input(layers): 71 | l1, l2, l3, l4, l5 = layers 72 | _ = l1 >> "in1" - l2 >> l3 73 | 74 | assert (l1.layer, 'default', 'in1') in l2.layer.incoming 75 | assert ('default', 'in1', l2.layer) in l1.layer.outgoing 76 | assert ('default', 'default', l3.layer) in l2.layer.outgoing 77 | assert (l2.layer, 'default', 'default') in l3.layer.incoming 78 | 79 | 80 | def test_connect_named_output_to_name_input(layers): 81 | l1, l2, l3, l4, l5 = layers 82 | _ = l1 >> l2 - "out1" >> "in1" - l3 >> l4 83 | 84 | assert ('default', 'default', l2.layer) in l1.layer.outgoing 85 | assert (l1.layer, 'default', 'default') in l2.layer.incoming 86 | 87 | assert ('out1', 'in1', l3.layer) in l2.layer.outgoing 88 | assert (l2.layer, 'out1', 'in1') in l3.layer.incoming 89 | 90 | assert ('default', 'default', l4.layer) in l3.layer.outgoing 91 | assert (l3.layer, 'default', 'default') in l4.layer.incoming 92 | 93 | 94 | def test_connect_named_output_to_name_input_in_chain(layers): 95 | l1, l2, l3, l4, l5 = layers 96 | _ = l1 >> "in1" - l2 - "out1" >> l3 97 | 98 | assert ('default', 'in1', l2.layer) in l1.layer.outgoing 99 | assert (l1.layer, 'default', 'in1') in l2.layer.incoming 100 | assert ('out1', 'default', l3.layer) in l2.layer.outgoing 101 | assert (l2.layer, 'out1', 'default') in l3.layer.incoming 102 | 103 | 104 | def test_collect_connected_layers(layers): 105 | l1, l2, l3, l4, l5 = layers 106 | _ = l1 >> l2 >> l3 >> l4 >> l5 107 | layer_set = {l.layer for l in layers} 108 | assert l1.layer.collect_connected_layers() == layer_set 109 | assert l5.layer.collect_connected_layers() == layer_set 110 | 111 | 112 | def test_collect_connected_layers2(layers): 113 | l1, l2, l3, l4, l5 = layers 114 | _ = l1 >> l2 >> l3 >> l4 115 | _ = l1 >> l5 >> l4 116 | layer_set = {l.layer for l in layers} 117 | assert l1.layer.collect_connected_layers() == layer_set 118 | assert l4.layer.collect_connected_layers() == layer_set 119 | assert l5.layer.collect_connected_layers() == layer_set 120 | 121 | 122 | def test_name(): 123 | l = ConstructionWrapper.create('FooLayerImpl', name='bar') 124 | assert l.layer.name == 'bar' 125 | 126 | 127 | def test_default_name(): 128 | l = ConstructionWrapper.create('FooLayerImpl') 129 | assert l.layer.name == 'Foo' 130 | 131 | 132 | def test_name_unconnected(): 133 | l1 = ConstructionWrapper.create('FooLayerImpl', name='bar') 134 | l2 = ConstructionWrapper.create('FooLayerImpl', name='bar') 135 | assert l1.layer.name == 'bar' 136 | assert l2.layer.name == 'bar' 137 | 138 | 139 | def test_name_connected(): 140 | l1 = ConstructionWrapper.create('FooLayerImpl', name='bar') 141 | l2 = ConstructionWrapper.create('FooLayerImpl', name='bar') 142 | _ = l1 >> l2 143 | assert l1.layer.name == 'bar_1' 144 | assert l2.layer.name == 'bar_2' 145 | 146 | 147 | def test_name_connected_complex(layers): 148 | l1, l2, l3, l4, l5 = layers 149 | _ = l3 >> l4 150 | _ = l2 >> l1 151 | _ = l5 >> l2 >> l3 152 | assert l1.layer.name == 'Dummy_1' 153 | assert l2.layer.name == 'Dummy_2' 154 | assert l3.layer.name == 'Dummy_3' 155 | assert l4.layer.name == 'Dummy_4' 156 | assert l5.layer.name == 'Dummy_5' 157 | -------------------------------------------------------------------------------- /brainstorm/tests/test_structure/test_generate_architecture.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from brainstorm.layers.input_layer import InputLayerImpl 6 | from brainstorm.layers.noop_layer import NoOpLayerImpl 7 | from brainstorm.structure.architecture import ( 8 | generate_architecture, get_layer_description, 9 | instantiate_layers_from_architecture) 10 | from brainstorm.structure.buffer_structure import BufferStructure 11 | from brainstorm.structure.construction import ConstructionWrapper 12 | 13 | 14 | def test_get_layer_description(): 15 | l = ConstructionWrapper.create('FooLayerImpl', name='foo') 16 | l2 = ConstructionWrapper.create('FooLayerImpl', name='bar') 17 | l3 = ConstructionWrapper.create('FooLayerImpl', name='baz') 18 | _ = l >> l2 19 | _ = l >> l3 20 | descr = get_layer_description(l.layer) 21 | assert descr == { 22 | '@type': 'Foo', 23 | '@outgoing_connections': { 24 | 'default': ['bar.default', 'baz.default'] 25 | } 26 | } 27 | 28 | 29 | def test_get_layer_description_named_inputs_outputs(): 30 | l = ConstructionWrapper.create('FooLayerImpl', name='foo') 31 | l2 = ConstructionWrapper.create('FooLayerImpl', name='bar') 32 | l3 = ConstructionWrapper.create('FooLayerImpl', name='baz') 33 | _ = l - 'out1' >> l2 34 | _ = l >> 'A' - l3 35 | descr = get_layer_description(l.layer) 36 | assert descr == { 37 | '@type': 'Foo', 38 | '@outgoing_connections': { 39 | 'default': ['baz.A'], 40 | 'out1': ['bar.default'] 41 | } 42 | } 43 | 44 | 45 | def test_layer_with_kwargs(): 46 | l = ConstructionWrapper.create('FooLayerImpl', name='foo', a=2, b=3) 47 | descr = get_layer_description(l.layer) 48 | assert descr == { 49 | '@type': 'Foo', 50 | '@outgoing_connections': {}, 51 | 'a': 2, 52 | 'b': 3 53 | } 54 | 55 | 56 | def test_generate_architecture(): 57 | l1 = ConstructionWrapper.create('InputLayerImpl') 58 | l2 = ConstructionWrapper.create('FooLayerImpl', name='bar') 59 | l3 = ConstructionWrapper.create('FooLayerImpl', name='baz') 60 | l4 = ConstructionWrapper.create('FooLayerImpl', name='out') 61 | _ = l1 - 'foo' >> l2 >> 'A' - l4 62 | _ = l1 - 'bar' >> l3 >> 'B' - l4 63 | 64 | arch1 = generate_architecture(l1.layer) 65 | arch2 = generate_architecture(l2.layer) 66 | arch3 = generate_architecture(l3.layer) 67 | assert arch1 == arch2 68 | assert arch1 == arch3 69 | assert arch1 == { 70 | 'Input': { 71 | '@type': 'Input', 72 | '@outgoing_connections': { 73 | 'foo': ['bar.default'], 74 | 'bar': ['baz.default'], 75 | } 76 | 77 | }, 78 | 'bar': { 79 | '@type': 'Foo', 80 | '@outgoing_connections': { 81 | 'default': ['out.A'], 82 | } 83 | }, 84 | 'baz': { 85 | '@type': 'Foo', 86 | '@outgoing_connections': { 87 | 'default': ['out.B'], 88 | } 89 | }, 90 | 'out': { 91 | '@type': 'Foo', 92 | '@outgoing_connections': {} 93 | } 94 | } 95 | 96 | 97 | def test_instantiate_layers_from_architecture(): 98 | arch = { 99 | 'Input': { 100 | '@type': 'Input', 101 | 'out_shapes': {'default': ('T', 'B', 10,)}, 102 | '@outgoing_connections': ['A', 'B', 'C'] 103 | }, 104 | 'A': { 105 | '@type': 'NoOp', 106 | '@outgoing_connections': ['B'] 107 | }, 108 | 'B': { 109 | '@type': 'NoOp', 110 | '@outgoing_connections': ['D'] 111 | }, 112 | 'C': { 113 | '@type': 'NoOp', 114 | '@outgoing_connections': ['D'] 115 | }, 116 | 'D': { 117 | '@type': 'NoOp', 118 | '@outgoing_connections': [] 119 | } 120 | } 121 | layers = instantiate_layers_from_architecture(arch) 122 | assert set(arch.keys()) == set(layers.keys()) 123 | 124 | assert isinstance(layers['Input'], InputLayerImpl) 125 | assert type(layers['A']) == NoOpLayerImpl 126 | assert type(layers['B']) == NoOpLayerImpl 127 | assert type(layers['C']) == NoOpLayerImpl 128 | assert type(layers['D']) == NoOpLayerImpl 129 | 130 | assert layers['Input'].in_shapes == {} 131 | assert layers['Input'].out_shapes == {'default': 132 | BufferStructure('T', 'B', 10)} 133 | assert layers['A'].in_shapes == {'default': BufferStructure('T', 'B', 10)} 134 | assert layers['A'].out_shapes == {'default': BufferStructure('T', 'B', 10)} 135 | assert layers['B'].in_shapes == {'default': BufferStructure('T', 'B', 20)} 136 | assert layers['B'].out_shapes == {'default': BufferStructure('T', 'B', 20)} 137 | assert layers['C'].in_shapes == {'default': BufferStructure('T', 'B', 10)} 138 | assert layers['C'].out_shapes == {'default': BufferStructure('T', 'B', 10)} 139 | assert layers['D'].in_shapes == {'default': BufferStructure('T', 'B', 30)} 140 | assert layers['D'].out_shapes == {'default': BufferStructure('T', 'B', 30)} 141 | -------------------------------------------------------------------------------- /brainstorm/tests/test_uniquely_named.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from brainstorm.structure.construction import UniquelyNamed 5 | 6 | 7 | def test_basename(): 8 | n = UniquelyNamed('my_basename') 9 | assert n.name == 'my_basename' 10 | 11 | 12 | def test_merging_scopes_no_conflict(): 13 | n1 = UniquelyNamed('A') 14 | n2 = UniquelyNamed('B') 15 | n1.merge_scopes(n2) 16 | assert n1.name == 'A' 17 | assert n2.name == 'B' 18 | 19 | 20 | def test_multimerging_no_conflict(): 21 | n1 = UniquelyNamed('A') 22 | n2 = UniquelyNamed('B') 23 | n1.merge_scopes(n2) 24 | n2.merge_scopes(n1) 25 | n2.merge_scopes(n1) 26 | n1.merge_scopes(n2) 27 | assert n1.name == 'A' 28 | assert n2.name == 'B' 29 | 30 | 31 | def test_merging_scopes(): 32 | n1 = UniquelyNamed('my_basename') 33 | n2 = UniquelyNamed('my_basename') 34 | n1.merge_scopes(n2) 35 | assert n1.name == 'my_basename_1' 36 | assert n2.name == 'my_basename_2' 37 | 38 | 39 | def test_merging_scopes_symmetric(): 40 | n1 = UniquelyNamed('my_basename') 41 | n2 = UniquelyNamed('my_basename') 42 | n2.merge_scopes(n1) 43 | assert n1.name == 'my_basename_1' 44 | assert n2.name == 'my_basename_2' 45 | 46 | 47 | def test_merging_scopes_transitiv(): 48 | n1 = UniquelyNamed('my_basename') 49 | n2 = UniquelyNamed('my_basename') 50 | n3 = UniquelyNamed('my_basename') 51 | n1.merge_scopes(n2) 52 | n2.merge_scopes(n3) 53 | assert n1.name == 'my_basename_1' 54 | assert n2.name == 'my_basename_2' 55 | assert n3.name == 'my_basename_3' 56 | 57 | 58 | def test_merging_scopes_transitiv2(): 59 | n1 = UniquelyNamed('my_basename') 60 | n2 = UniquelyNamed('my_basename') 61 | n3 = UniquelyNamed('my_basename') 62 | n4 = UniquelyNamed('my_basename') 63 | n1.merge_scopes(n2) 64 | n3.merge_scopes(n4) 65 | n2.merge_scopes(n4) 66 | assert n1.name == 'my_basename_1' 67 | assert n2.name == 'my_basename_2' 68 | assert n3.name == 'my_basename_3' 69 | assert n4.name == 'my_basename_4' 70 | 71 | 72 | def test_sneaky_name_collision(): 73 | n1 = UniquelyNamed('A_2') 74 | n2 = UniquelyNamed('A') 75 | n3 = UniquelyNamed('A') 76 | n4 = UniquelyNamed('A') 77 | n1.merge_scopes(n2) 78 | n2.merge_scopes(n3) 79 | n3.merge_scopes(n4) 80 | 81 | assert n1.name == 'A_2' 82 | assert n2.name == 'A_1' 83 | assert n3.name == 'A_3' 84 | assert n4.name == 'A_4' 85 | 86 | 87 | def test_no_sneaky_name_collision(): 88 | n0 = UniquelyNamed('A_2') 89 | n1 = UniquelyNamed('A_2') 90 | n2 = UniquelyNamed('A') 91 | n3 = UniquelyNamed('A') 92 | n4 = UniquelyNamed('A') 93 | n0.merge_scopes(n1) 94 | n1.merge_scopes(n2) 95 | n2.merge_scopes(n3) 96 | n3.merge_scopes(n4) 97 | 98 | assert n0.name == 'A_2_1' 99 | assert n1.name == 'A_2_2' 100 | assert n2.name == 'A_1' 101 | assert n3.name == 'A_2' 102 | assert n4.name == 'A_3' 103 | 104 | 105 | def test_separate_scopes(): 106 | n0 = UniquelyNamed('A') 107 | n1 = UniquelyNamed('A') 108 | n2 = UniquelyNamed('A') 109 | n3 = UniquelyNamed('A') 110 | 111 | n1.merge_scopes(n0) 112 | n2.merge_scopes(n3) 113 | assert n0.name == 'A_1' 114 | assert n1.name == 'A_2' 115 | assert n2.name == 'A_1' 116 | assert n3.name == 'A_2' 117 | -------------------------------------------------------------------------------- /brainstorm/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from __future__ import division, print_function, unicode_literals 5 | 6 | import numpy as np 7 | 8 | from brainstorm.handlers import NumpyHandler 9 | from brainstorm.utils import (convert_to_nested_indices, flatten, flatten_keys, 10 | flatten_time, flatten_time_and_features, 11 | get_inheritors, progress_bar) 12 | 13 | 14 | def test_get_inheritors(): 15 | class A(object): 16 | pass 17 | 18 | class B(A): 19 | pass 20 | 21 | class C(B): 22 | pass 23 | 24 | class D(A): 25 | pass 26 | 27 | class E(object): 28 | pass 29 | 30 | assert get_inheritors(A) == {B, C, D} 31 | 32 | 33 | def test_flatten(): 34 | assert list(flatten([0, (1, 2, 3), 4, [5, (6, 7), 8]])) == list(range(9)) 35 | 36 | 37 | def test_convert_to_nested_indices(): 38 | assert list(convert_to_nested_indices( 39 | ['a', ('b', 'c', 'a'), 'b', ['c', ('c', 'c'), 'b']])) == \ 40 | [0, [1, 2, 3], 4, [5, [6, 7], 8]] 41 | 42 | 43 | def test_flatten_time(): 44 | # Testing for NumpyHandler only 45 | _h = NumpyHandler(np.float64) 46 | shape = (2, 3, 2, 4) 47 | x = np.random.randn(*shape) 48 | y = flatten_time(x).copy() 49 | yp = x.reshape((6, 2, 4)) 50 | assert np.allclose(y, yp) 51 | 52 | 53 | def test_flatten_time_and_features(): 54 | # Testing for NumpyHandler only 55 | _h = NumpyHandler(np.float64) 56 | shape = (2, 3, 2, 4) 57 | x = np.random.randn(*shape) 58 | y = flatten_time_and_features(x).copy() 59 | yp = x.reshape((6, 8)) 60 | assert np.allclose(y, yp) 61 | 62 | 63 | def test_flatten_keys(): 64 | d = {'training_loss': None, 65 | 'validation': {'accuracy': [0], 66 | 'loss': [0]}} 67 | assert set(flatten_keys(d)) == {'training_loss', 'validation.accuracy', 68 | 'validation.loss'} 69 | 70 | d = {'default': None, 71 | 'a': [1, 2], 72 | 'b': {'i': None, 73 | 'j': [0, 1], 74 | 'k': {'x': 'default', 75 | 'y': True} 76 | } 77 | } 78 | assert set(flatten_keys(d)) == {'default', 'a', 'b.i', 'b.j', 'b.k.x', 79 | 'b.k.y'} 80 | 81 | 82 | def test_progress_bar(): 83 | prefix = '<<' 84 | bar = '1234567890' 85 | suffix = '>>' 86 | p = progress_bar(10, prefix, bar, suffix) 87 | assert next(p) == prefix 88 | assert p.send(4) == '1234' 89 | assert p.send(4) == '' 90 | assert p.send(9) == '56789' 91 | assert p.send(9.999) == '' 92 | assert p.send(10) == '0' + suffix 93 | -------------------------------------------------------------------------------- /brainstorm/tests/test_weight_modifiers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import numpy as np 6 | 7 | from brainstorm.handlers import default_handler 8 | from brainstorm.optional import has_pycuda 9 | from brainstorm.value_modifiers import ConstrainL2Norm 10 | 11 | non_default_handlers = [] 12 | if has_pycuda: 13 | from brainstorm.handlers import PyCudaHandler 14 | non_default_handlers.append(PyCudaHandler()) 15 | 16 | 17 | def test_limit_incoming_weights_squared(): 18 | for orig in (np.random.rand(4, 5), np.random.randn(3, 5, 4, 6)): 19 | for limit in [0.00001, 1, 10, 10000]: 20 | x = orig.reshape(orig.shape[0], int(orig.size / orig.shape[0])).copy() 21 | divisor = (x * x).sum(axis=1, keepdims=True) ** 0.5 / limit 22 | divisor[divisor < 1] = 1 23 | out = (x / divisor).reshape(orig.shape) 24 | 25 | y = orig.copy() 26 | mod = ConstrainL2Norm(limit) 27 | mod(default_handler, y) 28 | assert np.allclose(y, out) 29 | 30 | for handler in non_default_handlers: 31 | y = handler.create_from_numpy(orig) 32 | mod(handler, y) 33 | assert np.allclose(handler.get_numpy_copy(y), out) 34 | -------------------------------------------------------------------------------- /brainstorm/tests/tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from brainstorm.tests.test_layers import ( 6 | spec_list, test_deltas_calculation_of_layer, test_layer_add_to_deltas, 7 | test_layer_backward_pass_insensitive_to_internal_state_init, 8 | test_layer_forward_pass_insensitive_to_internal_state_init, 9 | test_gradients_for_layer) 10 | 11 | 12 | def get_test_configurations(): 13 | for spec in spec_list: 14 | time_steps, batch_size, activation = spec 15 | yield { 16 | 'time_steps': time_steps, 17 | 'batch_size': batch_size, 18 | 'activation': activation 19 | } 20 | 21 | 22 | def run_layer_tests(layer, spec): 23 | spec_str = "time_steps={time_steps}, batch_size={batch_size}," \ 24 | " activation={activation}".format(**spec) 25 | print('======= Testing {} for {} ====='.format(layer.name, spec_str)) 26 | print('Testing Delta Calculations ...') 27 | test_deltas_calculation_of_layer((layer, spec)) 28 | print('Testing Gradient Calculations ...') 29 | test_gradients_for_layer((layer, spec)) 30 | print('Verifying that layer ADDS to deltas ...') 31 | test_layer_add_to_deltas((layer, spec)) 32 | print('Verifying that the forward pass is insensitive to initialization of' 33 | ' internals ...') 34 | test_layer_forward_pass_insensitive_to_internal_state_init((layer, spec)) 35 | print('Verifying that the backward pass is insensitive to initialization' 36 | ' of internals ...') 37 | test_layer_backward_pass_insensitive_to_internal_state_init((layer, spec)) 38 | print("") 39 | -------------------------------------------------------------------------------- /brainstorm/training/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function 4 | 5 | from brainstorm.training.trainer import Trainer 6 | from brainstorm.training.steppers import ( 7 | SgdStepper, MomentumStepper, NesterovStepper) 8 | from brainstorm.training.schedules import Linear, Exponential, MultiStep 9 | 10 | __all__ = ['Trainer', 'SgdStepper', 'MomentumStepper', 'NesterovStepper', 11 | 'Linear', 'Exponential', 'MultiStep'] 12 | -------------------------------------------------------------------------------- /brainstorm/training/steppers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from brainstorm.describable import Describable 6 | 7 | 8 | # ########################### Base Class ###################################### 9 | 10 | class TrainingStepper(Describable): 11 | """ 12 | Base class for all training steps. Defines the common interface 13 | """ 14 | __undescribed__ = {'net'} 15 | 16 | def __init__(self): 17 | self.net = None 18 | 19 | def start(self, net): 20 | self.net = net 21 | 22 | def run(self): 23 | pass 24 | 25 | 26 | # ########################## Training Steps ################################### 27 | 28 | class ForwardStepper(TrainingStepper): 29 | """ 30 | Only runs the forward pass and returns the error. Does not train the 31 | network at all. 32 | This step is usually used for validation. If this step is used during 33 | training it should be initialized with the use_training_pass flag set to 34 | true. 35 | """ 36 | 37 | def __init__(self, use_training_pass=False): 38 | super(ForwardStepper, self).__init__() 39 | self.use_training_pass = use_training_pass 40 | 41 | def run(self): 42 | self.net.forward_pass(training_pass=self.use_training_pass) 43 | return self.net.get_loss_value() 44 | 45 | 46 | class SgdStepper(TrainingStepper): 47 | """ 48 | Stochastic Gradient Descent. 49 | """ 50 | __undescribed__ = {'update'} 51 | 52 | def __init__(self, learning_rate=0.1): 53 | super(SgdStepper, self).__init__() 54 | self.learning_rate = learning_rate 55 | self.update = None 56 | 57 | def start(self, net): 58 | super(SgdStepper, self).start(net) 59 | self.update = net.handler.zeros(net.buffer.parameters.shape) 60 | 61 | def run(self): 62 | self.net.forward_pass(training_pass=True) 63 | self.net.backward_pass() 64 | self.net.handler.mult_st(-self.learning_rate, 65 | self.net.buffer.gradients, 66 | out=self.update) 67 | self.net.handler.add_tt(self.update, 68 | self.net.buffer.parameters, 69 | out=self.net.buffer.parameters) 70 | 71 | 72 | class MomentumStepper(TrainingStepper): 73 | """ 74 | Stochastic Gradient Descent with a momentum term. 75 | learning_rate and momentum can be scheduled using 76 | brainstorm.training.schedules 77 | If scale_learning_rate is True (default), 78 | learning_rate is multiplied by (1 - momentum) when used. 79 | """ 80 | __undescribed__ = {'velocity'} 81 | __default_values__ = {'scale_learning_rate': True} 82 | 83 | def __init__(self, learning_rate=0.1, momentum=0.0, 84 | scale_learning_rate=True): 85 | super(MomentumStepper, self).__init__() 86 | self.velocity = None 87 | self.learning_rate = learning_rate 88 | self.momentum = momentum 89 | assert isinstance(scale_learning_rate, bool), \ 90 | "scale_learning_rate must be True or False." 91 | self.scale_learning_rate = scale_learning_rate 92 | 93 | def start(self, net): 94 | super(MomentumStepper, self).start(net) 95 | self.velocity = net.handler.zeros(net.buffer.parameters.shape) 96 | 97 | def run(self): 98 | learning_rate = self.learning_rate 99 | momentum = self.momentum 100 | if self.scale_learning_rate: 101 | learning_rate *= (1 - momentum) 102 | 103 | self.net.forward_pass(training_pass=True) 104 | self.net.backward_pass() 105 | 106 | self.net.handler.mult_st(momentum, 107 | self.velocity, 108 | out=self.velocity) 109 | self.net.handler.mult_add_st(-learning_rate, 110 | self.net.buffer.gradients, 111 | out=self.velocity) 112 | self.net.handler.add_tt(self.velocity, 113 | self.net.buffer.parameters, 114 | out=self.net.buffer.parameters) 115 | 116 | 117 | class NesterovStepper(MomentumStepper): 118 | """ 119 | Stochastic Gradient Descent with a Nesterov-style momentum term. 120 | learning_rate and momentum can be scheduled using 121 | brainstorm.training.schedules 122 | If scale_learning_rate is True (default), 123 | learning_rate is multiplied by (1 - momentum) when used. 124 | """ 125 | def run(self): 126 | learning_rate = self.learning_rate 127 | momentum = self.momentum 128 | if self.scale_learning_rate: 129 | learning_rate *= (1 - momentum) 130 | 131 | self.net.handler.mult_st(momentum, 132 | self.velocity, 133 | out=self.velocity) 134 | self.net.handler.add_tt(self.velocity, 135 | self.net.buffer.parameters, 136 | out=self.net.buffer.parameters) 137 | self.net.forward_pass(training_pass=True) 138 | self.net.backward_pass() 139 | 140 | self.net.handler.mult_add_st(-learning_rate, 141 | self.net.buffer.gradients, 142 | self.velocity) 143 | self.net.handler.mult_add_st(-learning_rate, 144 | self.net.buffer.gradients, 145 | self.net.buffer.parameters) 146 | -------------------------------------------------------------------------------- /brainstorm/training/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | 6 | def run_network(net, iterator, all_inputs=True): 7 | for i, data in enumerate(iterator): 8 | net.provide_external_data(data, all_inputs=all_inputs) 9 | yield i 10 | -------------------------------------------------------------------------------- /data/create_cifar10.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from six.moves.urllib.request import urlretrieve 5 | import numpy as np 6 | import tarfile 7 | import h5py 8 | import os 9 | 10 | bs_data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '.') 11 | url = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' 12 | cifar10_file = os.path.join(bs_data_dir, 'cifar-10-binary.tar.gz') 13 | hdf_file = os.path.join(bs_data_dir, 'CIFAR-10.hdf5') 14 | 15 | print("Using data directory:", bs_data_dir) 16 | if not os.path.exists(cifar10_file): 17 | print("Downloading CIFAR-10 data ...") 18 | urlretrieve(url, cifar10_file) 19 | print("Done.") 20 | 21 | archive_paths = [ 22 | 'cifar-10-batches-bin/data_batch_1.bin', 23 | 'cifar-10-batches-bin/data_batch_2.bin', 24 | 'cifar-10-batches-bin/data_batch_3.bin', 25 | 'cifar-10-batches-bin/data_batch_4.bin', 26 | 'cifar-10-batches-bin/data_batch_5.bin', 27 | 'cifar-10-batches-bin/test_batch.bin' 28 | ] 29 | 30 | print("Extracting CIFAR-10 data ...") 31 | with tarfile.open(cifar10_file) as f: 32 | res = [] 33 | for fn in archive_paths: 34 | buf = f.extractfile(fn).read() 35 | tmp = np.fromstring(buf, dtype=np.uint8).reshape(-1, 1024 * 3 + 1) 36 | res.append(tmp) 37 | print("Done.") 38 | 39 | ds = np.concatenate(res) 40 | num_tr = 40000 41 | x_tr = ds[:num_tr, 1:].reshape((1, num_tr, 3, 32, 32)) 42 | x_tr = x_tr.transpose([0, 1, 3, 4, 2]) 43 | y_tr = ds[:num_tr, 0].reshape((1, num_tr, 1)) 44 | 45 | x_va = ds[num_tr: 50000, 1:].reshape((1, 10000, 3, 32, 32)) 46 | x_va = x_va.transpose([0, 1, 3, 4, 2]) 47 | y_va = ds[num_tr: 50000, 0].reshape((1, 10000, 1)) 48 | 49 | x_te = ds[50000:, 1:].reshape((1, 10000, 3, 32, 32)) 50 | x_te = x_te.transpose([0, 1, 3, 4, 2]) 51 | y_te = ds[50000:, 0].reshape((1, 10000, 1)) 52 | 53 | tr_mean = x_tr.squeeze().mean(axis=0) 54 | tr_std = x_tr.squeeze().std(axis=0) 55 | x_tr = (x_tr - tr_mean) / tr_std 56 | x_va = (x_va - tr_mean) / tr_std 57 | x_te = (x_te - tr_mean) / tr_std 58 | 59 | print("Creating CIFAR-10 HDF5 dataset ...") 60 | f = h5py.File(hdf_file, 'w') 61 | description = """ 62 | The CIFAR-10 dataset is a labeled subset of the 80 million tiny images 63 | dataset. It consists of 60000 32x32 colour images in 10 classes, with 6000 64 | images per class. There are 50000 training images and 10000 test images. 65 | 66 | The dataset was obtained from the link: 67 | http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz 68 | 69 | Attributes 70 | ========== 71 | 72 | description: This description. 73 | 74 | mean: The pixel-wise mean of the first 40000 training set images. 75 | 76 | std: The pixel-wise standard deviation of the first 40000 training set images. 77 | 78 | Variants 79 | ======== 80 | 81 | normalized_full: Contains 'training' and 'test' sets. Image data has been 82 | normalized to have zero mean and unit standard deviation. 83 | 84 | normalized_split: Contains 'training' (first 40K out of the full training 85 | set), 'validation' (remaining 10K out of the full training set) and 86 | 'test' sets. Image data has been normalized to have zero mean and unit 87 | standard deviation. 88 | 89 | """ 90 | f.attrs['description'] = description 91 | f.attrs['mean'] = tr_mean 92 | f.attrs['std'] = tr_std 93 | 94 | variant = f.create_group('normalized_split') 95 | group = variant.create_group('training') 96 | group.create_dataset(name='default', data=x_tr, compression='gzip') 97 | group.create_dataset(name='targets', data=y_tr, compression='gzip') 98 | 99 | group = variant.create_group('validation') 100 | group.create_dataset(name='default', data=x_va, compression='gzip') 101 | group.create_dataset(name='targets', data=y_va, compression='gzip') 102 | 103 | group = variant.create_group('test') 104 | group.create_dataset(name='default', data=x_te, compression='gzip') 105 | group.create_dataset(name='targets', data=y_te, compression='gzip') 106 | 107 | nr_tr = 50000 108 | x_tr = ds[:nr_tr, 1:].reshape((1, nr_tr, 3, 32, 32)).transpose([0, 1, 3, 4, 2]) 109 | x_tr = (x_tr - tr_mean) / tr_std 110 | y_tr = ds[:nr_tr, 0].reshape((1, nr_tr, 1)) 111 | 112 | variant = f.create_group('normalized_full') 113 | group = variant.create_group('training') 114 | group.create_dataset(name='default', data=x_tr, compression='gzip') 115 | group.create_dataset(name='targets', data=y_tr, compression='gzip') 116 | 117 | group = variant.create_group('test') 118 | group.create_dataset(name='default', data=x_te, compression='gzip') 119 | group.create_dataset(name='targets', data=y_te, compression='gzip') 120 | 121 | f.close() 122 | print("Done.") 123 | -------------------------------------------------------------------------------- /data/create_cifar100.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from six.moves.urllib.request import urlretrieve 5 | import numpy as np 6 | import tarfile 7 | import h5py 8 | import os 9 | 10 | bs_data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '.') 11 | url = 'http://www.cs.toronto.edu/~kriz/cifar-100-binary.tar.gz' 12 | cifar100_file = os.path.join(bs_data_dir, 'cifar-100-binary.tar.gz') 13 | hdf_file = os.path.join(bs_data_dir, 'CIFAR-100.hdf5') 14 | 15 | print("Using data directory:", bs_data_dir) 16 | if not os.path.exists(cifar100_file): 17 | print("Downloading CIFAR-100 data ...") 18 | urlretrieve(url, cifar100_file) 19 | print("Done.") 20 | 21 | archive_paths = [ 22 | 'cifar-100-binary/train.bin', 23 | 'cifar-100-binary/test.bin' 24 | ] 25 | 26 | print("Extracting CIFAR-100 data ...") 27 | with tarfile.open(cifar100_file) as f: 28 | res = [] 29 | for fn in archive_paths: 30 | buf = f.extractfile(fn).read() 31 | tmp = np.fromstring(buf, dtype=np.uint8) 32 | tmp = tmp.reshape(-1, 1024 * 3 + 2) 33 | res.append(tmp) 34 | print("Done.") 35 | 36 | ds = np.concatenate(res) 37 | nr_tr = 40000 38 | x_tr = ds[:nr_tr, 2:].reshape((1, nr_tr, 3, 32, 32)) 39 | x_tr = x_tr.transpose([0, 1, 3, 4, 2]) 40 | y_tr = ds[:nr_tr, 1] .reshape((1, nr_tr, 1)) 41 | 42 | x_va = ds[40000: 50000, 2:].reshape((1, 10000, 3, 32, 32)) 43 | x_va = x_va.transpose([0, 1, 3, 4, 2]) 44 | y_va = ds[40000: 50000, 1].reshape((1, 10000, 1)) 45 | 46 | x_te = ds[50000:, 2:].reshape((1, 10000, 3, 32, 32)) 47 | x_te = x_te.transpose([0, 1, 3, 4, 2]) 48 | y_te = ds[50000:, 1].reshape((1, 10000, 1)) 49 | 50 | tr_mean = x_tr.squeeze().mean(axis=0) 51 | tr_std = x_tr.squeeze().std(axis=0) 52 | x_tr = (x_tr - tr_mean) / tr_std 53 | x_va = (x_va - tr_mean) / tr_std 54 | x_te = (x_te - tr_mean) / tr_std 55 | 56 | print("Creating CIFAR-100 HDF5 dataset ...") 57 | f = h5py.File(hdf_file, 'w') 58 | description = """ 59 | The CIFAR-100 dataset is a labeled subset of the 80 million tiny images 60 | dataset. It consists of 60000 32x32 colour images in 100 classes, with 600 61 | images per class. There are 50000 training images and 10000 test images. 62 | 63 | The dataset was obtained from the link: 64 | http://www.cs.toronto.edu/~kriz/cifar-100-binary.tar.gz 65 | 66 | Attributes 67 | ========== 68 | 69 | description: This description. 70 | 71 | mean: The pixel-wise mean of the first 40000 training set images. 72 | 73 | std: The pixel-wise standard deviation of the first 40000 training set images. 74 | 75 | std: 76 | 77 | Variants 78 | ======== 79 | 80 | normalized_full: Contains 'training' and 'test' sets. Image data has been 81 | normalized to have zero mean and unit standard deviation. 82 | 83 | normalized_split: Contains 'training' (first 40K out of the full training 84 | set), 'validation' (remaining 10K out of the full training set) and 85 | 'test' sets. Image data has been normalized to have zero mean and unit 86 | standard deviation. 87 | 88 | """ 89 | f.attrs['description'] = description 90 | f.attrs['mean'] = tr_mean 91 | f.attrs['std'] = tr_std 92 | 93 | variant = f.create_group('normalized_split') 94 | group = variant.create_group('training') 95 | group.create_dataset(name='default', data=x_tr, compression='gzip') 96 | group.create_dataset(name='targets', data=y_tr, compression='gzip') 97 | 98 | group = variant.create_group('validation') 99 | group.create_dataset(name='default', data=x_va, compression='gzip') 100 | group.create_dataset(name='targets', data=y_va, compression='gzip') 101 | 102 | group = variant.create_group('test') 103 | group.create_dataset(name='default', data=x_te, compression='gzip') 104 | group.create_dataset(name='targets', data=y_te, compression='gzip') 105 | 106 | nr_tr = 50000 107 | x_tr = ds[:nr_tr, 2:].reshape((1, nr_tr, 3, 32, 32)).transpose([0, 1, 3, 4, 2]) 108 | x_tr = (x_tr - tr_mean) / tr_std 109 | y_tr = ds[:nr_tr, 1].reshape((1, nr_tr, 1)) 110 | 111 | variant = f.create_group('normalized_full') 112 | group = variant.create_group('training') 113 | group.create_dataset(name='default', data=x_tr, compression='gzip') 114 | group.create_dataset(name='targets', data=y_tr, compression='gzip') 115 | 116 | group = variant.create_group('test') 117 | group.create_dataset(name='default', data=x_te, compression='gzip') 118 | group.create_dataset(name='targets', data=y_te, compression='gzip') 119 | 120 | f.close() 121 | print("Done.") 122 | -------------------------------------------------------------------------------- /data/create_hutter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | from six.moves.urllib.request import urlretrieve 5 | import numpy as np 6 | import zipfile 7 | import h5py 8 | import os 9 | 10 | 11 | def convert_to_batches(serial_data, length, bs): 12 | assert serial_data.size % length == 0 13 | num_sequences = serial_data.size // length 14 | assert num_sequences % bs == 0 15 | num_batches = num_sequences // bs 16 | serial_data = serial_data.reshape((bs, num_batches * length)) 17 | serial_data = np.vstack(np.hsplit(serial_data, num_batches)).T[:, :, None] 18 | return serial_data 19 | 20 | batch_size = 100 21 | # Batch size which will be used for training. 22 | # Needed to maintain continuity of data across batches. 23 | seq_len = 100 24 | # Number of characters in each sub-sequence. 25 | # Limits the number of time-steps that the gradient is back-propagated. 26 | num_test_chars = 5000000 27 | # Number of characters which will be used for testing. 28 | # An equal number of characters will be used for validation. 29 | 30 | bs_data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '.') 31 | url = 'http://mattmahoney.net/dc/enwik8.zip' 32 | hutter_file = os.path.join(bs_data_dir, 'enwik8.zip') 33 | hdf_file = os.path.join(bs_data_dir, 'HutterPrize.hdf5') 34 | 35 | print("Using data directory:", bs_data_dir) 36 | if not os.path.exists(hutter_file): 37 | print("Downloading Hutter Prize data ...") 38 | urlretrieve(url, hutter_file) 39 | print("Done.") 40 | 41 | print("Extracting Hutter Prize data ...") 42 | raw_data = zipfile.ZipFile(hutter_file).read('enwik8') 43 | print("Done.") 44 | 45 | print("Preparing data for Brainstorm ...") 46 | raw_data = np.fromstring(raw_data, dtype=np.uint8) 47 | unique, data = np.unique(raw_data, return_inverse=True) 48 | 49 | train_data = data[: -2 * num_test_chars] 50 | train_targets = data[1: -2 * num_test_chars + 1] 51 | valid_data = data[-2 * num_test_chars: -num_test_chars] 52 | valid_targets = data[-2 * num_test_chars + 1: -num_test_chars + 1] 53 | test_data = data[-num_test_chars:] 54 | test_targets = np.append(data[-num_test_chars + 1:], [0]) 55 | 56 | # Convert to batches 57 | train_data = convert_to_batches(train_data, seq_len, batch_size) 58 | train_targets = convert_to_batches(train_targets, seq_len, batch_size) 59 | 60 | valid_data = convert_to_batches(valid_data, seq_len, batch_size) 61 | valid_targets = convert_to_batches(valid_targets, seq_len, batch_size) 62 | 63 | test_data = convert_to_batches(test_data, seq_len, batch_size) 64 | test_targets = convert_to_batches(test_targets, seq_len, batch_size) 65 | print("Done.") 66 | 67 | print("Creating Hutter Prize character-level HDF5 dataset ...") 68 | f = h5py.File(hdf_file, 'w') 69 | description = """ 70 | The Hutter Prize Wikipedia dataset, prepared for character-level language 71 | modeling. 72 | 73 | The data was obtained from the link: 74 | http://mattmahoney.net/dc/enwik8.zip 75 | 76 | Attributes 77 | ========== 78 | 79 | description: This description. 80 | 81 | unique: A 1-D array of unique characters (0-255 ASCII values) in the dataset. 82 | The index of each character was used as the class ID for preparing the data. 83 | 84 | Variants 85 | ======== 86 | 87 | split: Split into 'training', 'validation' and 'test' tests of size 90, 10 and 88 | 10 million characters respectively. Each sequence is {} characters long. The 89 | dataset has been prepared expecting minibatches of {} sequences. 90 | """.format(seq_len, batch_size) 91 | f.attrs['description'] = description 92 | f.attrs['unique'] = unique 93 | 94 | variant = f.create_group('split') 95 | group = variant.create_group('training') 96 | group.create_dataset(name='default', data=train_data, compression='gzip') 97 | group.create_dataset(name='targets', data=train_targets, compression='gzip') 98 | 99 | group = variant.create_group('validation') 100 | group.create_dataset(name='default', data=valid_data, compression='gzip') 101 | group.create_dataset(name='targets', data=valid_targets, compression='gzip') 102 | 103 | group = variant.create_group('test') 104 | group.create_dataset(name='default', data=test_data, compression='gzip') 105 | group.create_dataset(name='targets', data=test_targets, compression='gzip') 106 | 107 | f.close() 108 | print("Done.") 109 | -------------------------------------------------------------------------------- /data/create_mnist.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | import gzip 5 | import pickle 6 | from six.moves.urllib.request import urlretrieve 7 | import numpy as np 8 | import h5py 9 | import os 10 | import sys 11 | 12 | bs_data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '.') 13 | url = 'http://deeplearning.net/data/mnist/mnist.pkl.gz' 14 | mnist_file = os.path.join(bs_data_dir, 'mnist.pkl.gz') 15 | hdf_file = os.path.join(bs_data_dir, 'MNIST.hdf5') 16 | 17 | print("Using data directory:", bs_data_dir) 18 | if not os.path.exists(mnist_file): 19 | print("Downloading MNIST data ...") 20 | urlretrieve(url, mnist_file) 21 | print("Done.") 22 | 23 | print("Extracting MNIST data ...") 24 | with gzip.open(mnist_file, 'rb') as f: 25 | if sys.version_info < (3,): 26 | ds = pickle.load(f) 27 | else: 28 | ds = pickle.load(f, encoding='latin1') 29 | print("Done.") 30 | 31 | train_inputs, train_targets = \ 32 | ds[0][0].reshape((1, 50000, 28, 28, 1)), ds[0][1].reshape((1, 50000, 1)) 33 | valid_inputs, valid_targets = \ 34 | ds[1][0].reshape((1, 10000, 28, 28, 1)), ds[1][1].reshape((1, 10000, 1)) 35 | test_inputs, test_targets = \ 36 | ds[2][0].reshape((1, 10000, 28, 28, 1)), ds[2][1].reshape((1, 10000, 1)) 37 | 38 | print("Creating HDF5 dataset ...") 39 | f = h5py.File(hdf_file, 'w') 40 | description = """ 41 | The MNIST handwritten digit database is a preprocessed subset of NIST's 42 | Special Database 3 and Special Database 1. It consists of 70000 28x28 binary 43 | images of handwritten digits (0-9), with approximately 7000 images per class. 44 | There are 60000 training images and 10000 test images. 45 | 46 | The dataset was obtained from the link: 47 | http://deeplearning.net/data/mnist/mnist.pkl.gz 48 | which hosts a normalized version of the data originally from: 49 | http://yann.lecun.com/exdb/mnist/ 50 | 51 | Attributes 52 | ========== 53 | 54 | description: This description. 55 | 56 | Variants 57 | ======== 58 | 59 | normalized_full: Contains 'training' and 'test' sets. Image data has been 60 | normalized by dividing all pixel values by 255. 61 | 62 | normalized_split: Contains 'training' (first 50K out of the full training 63 | set), 'validation' (remaining 10K out of the full training set) and 64 | 'test' sets. Image data has been normalized by dividing all pixel values by 65 | 255. 66 | 67 | """ 68 | f.attrs['description'] = description 69 | 70 | variant = f.create_group('normalized_split') 71 | group = variant.create_group('training') 72 | group.create_dataset(name='default', data=train_inputs, compression='gzip') 73 | group.create_dataset(name='targets', data=train_targets, compression='gzip') 74 | 75 | group = variant.create_group('validation') 76 | group.create_dataset(name='default', data=valid_inputs, compression='gzip') 77 | group.create_dataset(name='targets', data=valid_targets, compression='gzip') 78 | 79 | group = variant.create_group('test') 80 | group.create_dataset(name='default', data=test_inputs, compression='gzip') 81 | group.create_dataset(name='targets', data=test_targets, compression='gzip') 82 | 83 | 84 | train_inputs = np.concatenate((train_inputs, valid_inputs), axis=1) 85 | train_targets = np.concatenate((train_targets, valid_targets), axis=1) 86 | 87 | variant = f.create_group('normalized_full') 88 | group = variant.create_group('training') 89 | group.create_dataset(name='default', data=train_inputs, compression='gzip') 90 | group.create_dataset(name='targets', data=train_targets, compression='gzip') 91 | 92 | group = variant.create_group('test') 93 | group.create_dataset(name='default', data=test_inputs, compression='gzip') 94 | group.create_dataset(name='targets', data=test_targets, compression='gzip') 95 | 96 | f.close() 97 | print("Done.") 98 | -------------------------------------------------------------------------------- /doc_requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | six 3 | mock 4 | -------------------------------------------------------------------------------- /docs/apidoc.rst: -------------------------------------------------------------------------------- 1 | ################# 2 | API Documentation 3 | ################# 4 | This is a construction site... 5 | 6 | ******* 7 | Network 8 | ******* 9 | 10 | .. autoclass:: brainstorm.structure.network.Network 11 | :members: 12 | :special-members: __init__ 13 | 14 | ******* 15 | Trainer 16 | ******* 17 | .. autoclass:: brainstorm.training.trainer.Trainer 18 | :members: 19 | :special-members: __init__ 20 | 21 | ***** 22 | Tools 23 | ***** 24 | .. automodule:: brainstorm.tools 25 | :members: 26 | 27 | ************** 28 | Data Iterators 29 | ************** 30 | .. automodule:: brainstorm.data_iterators 31 | :members: 32 | 33 | ************ 34 | Initializers 35 | ************ 36 | .. automodule:: brainstorm.initializers 37 | :members: 38 | 39 | ***** 40 | Hooks 41 | ***** 42 | .. automodule:: brainstorm.hooks 43 | :members: 44 | 45 | *************** 46 | Value Modifiers 47 | *************** 48 | 49 | .. automodule:: brainstorm.value_modifiers 50 | :members: 51 | 52 | ******* 53 | Scorers 54 | ******* 55 | .. automodule:: brainstorm.scorers 56 | :members: 57 | 58 | ******* 59 | Handler 60 | ******* 61 | .. autoclass:: brainstorm.handlers.base_handler.Handler 62 | :members: 63 | :inherited-members: 64 | 65 | ************ 66 | Describables 67 | ************ 68 | .. automodule:: brainstorm.describable 69 | 70 | Conversion to and from descriptions 71 | =================================== 72 | .. autofunction:: create_from_description 73 | .. autofunction:: get_description 74 | 75 | Describable Base Class 76 | ====================== 77 | .. autoclass:: Describable 78 | :members: 79 | 80 | .. autoattribute:: __undescribed__ 81 | .. autoattribute:: __default_values__ 82 | .. automethod:: __init_from_description__ 83 | .. automethod:: __describe__ 84 | .. automethod:: __new_from_description__ 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst -------------------------------------------------------------------------------- /docs/credits.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Klaus Greff 9 | * Rupesh Srivastava 10 | 11 | Contributors 12 | ------------ 13 | 14 | * Thomas Unterthiner 15 | * Julian Zilly 16 | * Sjoerd van Steenkiste 17 | -------------------------------------------------------------------------------- /docs/data_format.rst: -------------------------------------------------------------------------------- 1 | .. _data_format: 2 | 3 | ########### 4 | Data Format 5 | ########### 6 | 7 | *********** 8 | Data Shapes 9 | *********** 10 | All data passed to a network in Brainstorm by a data iterator must match 11 | the template ``(T, B, ...)`` where ``T`` is the maximum sequence length and 12 | ``B`` is the number of sequences (or batch size, in other words). 13 | 14 | To simplify handling both sequential and non-sequential data, 15 | these shapes should also be used when the data is not sequential. In such cases 16 | the shape simply becomes ``(1, B, ...)``. As an example, the MNIST training images 17 | for classification with an MLP should be shaped ``(1, 60000, 784)`` and the 18 | corresponding targets should be shaped ``(1, 60000, 1)``. 19 | 20 | Data for images/videos should be stored in the ``TNHWC`` format. For 21 | example, the training images for CIFAR-10 should be shaped 22 | ``(1, 50000, 32, 32, 3)`` and the targets should be shaped ``(1, 50000, 1)``. 23 | 24 | ******* 25 | Example 26 | ******* 27 | 28 | A network in brainstorm accepts a dictionary of named data items as input. 29 | The keys of this dictionary and the shapes of the data should match those 30 | which were specified when the network was built. 31 | 32 | Consider a simple network built as follows: 33 | 34 | .. code-block:: python 35 | 36 | import numpy as np 37 | from brainstorm import Network, layers 38 | 39 | inp = layers.Input({'my_inputs': ('T', 'B', 50), 40 | 'my_targets': ('T','B', 2)}) 41 | hid = layers.FullyConnected(100, name='Hidden') 42 | out = layers.SoftmaxCE(name='Output') 43 | loss = layers.Loss() 44 | inp - 'my_inputs' >> hid >> out 45 | inp - 'my_targets' >> 'targets' - out - 'loss' >> loss 46 | network = Network.from_layer(loss) 47 | 48 | The same network can be quickly build 49 | 50 | 51 | Here's how you can provide some data to a network in brainstorm and run a 52 | forward pass on it. 53 | 54 | *********** 55 | File Format 56 | *********** 57 | There is no requirement on how to store the data in ``brainstorm``, but we 58 | highly recommend the HDF5 format using the h5py library. 59 | 60 | It is very simple to create hdf5 files: 61 | 62 | .. code-block:: python 63 | 64 | import h5py 65 | import numpy as np 66 | 67 | with h5py.File('demo.hdf5', 'w') as f: 68 | f['training/input_data'] = np.random.randn(7, 100, 15) 69 | f['training/targets'] = np.random.randn(7, 100, 2) 70 | f['training/static_data'] = np.random.randn(1, 100, 4) 71 | 72 | Having such a file available you can then set-up your data iterator like this: 73 | 74 | .. code-block:: python 75 | 76 | import h5py 77 | import brainstorm as bs 78 | 79 | ds = h5py.File('demo.hdf5', 'r') 80 | 81 | online_train_iter = bs.Online(**ds['training']) 82 | minibatch_train_iter = bs.Minibatches(100, **ds['training']) 83 | 84 | These iterators will then provide named data items (a dictionary) to the 85 | network with names 'input_data', 'targets' and 'static_data'. 86 | 87 | H5py offers many more features, which can be utilized to improve data 88 | storage and access such as chunking and compression. 89 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | ######## 2 | Examples 3 | ######## 4 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst -------------------------------------------------------------------------------- /docs/hooks.rst: -------------------------------------------------------------------------------- 1 | ##### 2 | Hooks 3 | ##### 4 | 5 | This is a construction site. 6 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Brainstorm: fast, flexible and fun neural networks. 2 | 3 | 4 | Brainstorm is under active development, so the documentation may be spotty 5 | (though we are working on it!). The API documentation is fairly complete, 6 | and help may be found on the mailing list. 7 | 8 | Contents: 9 | ========= 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | installation 15 | walkthrough 16 | data_format 17 | network 18 | layers 19 | trainer 20 | hooks 21 | apidoc 22 | internals 23 | contributing 24 | credits 25 | history 26 | 27 | Feedback 28 | ======== 29 | 30 | If you have any suggestions or questions about **brainstorm** feel free to 31 | email us at mailstorm@googlegroups.com. 32 | 33 | If you encounter any errors or problems with **brainstorm**, please let us know! 34 | Open an Issue at the GitHub http://github.com/IDSIA/brainstorm main repository. -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ############ 2 | Installation 3 | ############ 4 | 5 | Common Install Notes 6 | ==================== 7 | 8 | A basic requirement to use Brainstorm is Numpy, and we recommend that you make sure that you have a fast `BLAS `_ installation which Numpy will use. 9 | OpenBLAS is excellent, and can be installed on a Debian family system easily: ``sudo apt-get install libopenblas-dev``. 10 | 11 | Brainstorm provide a ``PyCudaHandler`` which can be used to accelerate neural networks using Nvidia GPUs. 12 | In order to use it, you need to have CUDA 7.0 or later already installed and setup from https://developer.nvidia.com/cuda-downloads 13 | 14 | Installation variants 15 | ===================== 16 | 17 | When installing from PyPI or GitHub, you can specify the following installation variants to additionally install optional dependencies: 18 | 19 | all 20 | pycuda 21 | tests 22 | live_viz 23 | draw_net 24 | 25 | ****** 26 | Ubuntu 27 | ****** 28 | 29 | Install prerequisites: 30 | 31 | .. code-block:: bash 32 | 33 | sudo apt-get install python-dev libhdf5-dev libopenblas-dev 34 | 35 | Install the latest stable release from PyPI, including all optional dependencies: 36 | 37 | .. code-block:: bash 38 | 39 | pip install brainstorm[all] 40 | 41 | which will additionally install pycuda, scikit-cuda, pygraphviz and bokeh. 42 | 43 | To install the latest master branch, you can do: 44 | 45 | .. code-block:: bash 46 | 47 | pip install git+git@github.com:IDSIA/brainstorm.git#egg=brainstorm[all] 48 | 49 | **** 50 | OS X 51 | **** 52 | 53 | Instructions coming soon. 54 | 55 | ******* 56 | Windows 57 | ******* 58 | 59 | Instructions coming soon. 60 | 61 | ****** 62 | Docker 63 | ****** 64 | 65 | Builds of Brainstorm are available as `Docker `_ 66 | images: `Brainstorm Docker (CPU) `_ or 67 | `Brainstorm Docker (CUDA) `_. These 68 | are updated on a weekly basis with bleeding-edge builds of Brainstorm. 69 | Examples of running bash in a Docker container are as follows: 70 | 71 | .. code-block:: bash 72 | 73 | sudo docker run -it kaixhin/brainstorm 74 | sudo nvidia-docker run -it kaixhin/cuda-brainstorm:7.0 75 | 76 | For a guide to Docker, see the `official docs `_. 77 | CUDA support requires `NVIDIA Docker `_. 78 | For more details on how to use the Brainstorm Docker images, 79 | consult the `source project `_. 80 | -------------------------------------------------------------------------------- /docs/layers.rst: -------------------------------------------------------------------------------- 1 | ###### 2 | Layers 3 | ###### 4 | 5 | This is a construction site. -------------------------------------------------------------------------------- /docs/network.rst: -------------------------------------------------------------------------------- 1 | ####### 2 | Network 3 | ####### 4 | Networks are the central structure in brainstorm. They contain and manage a 5 | directed acyclic graph of layers, manage the memory and provide access to all 6 | the internal states. 7 | 8 | ******** 9 | Creating 10 | ******** 11 | There are essentially 4 different ways of creating a network in brainstorm. 12 | 13 | 1. with the ``create_network_from_spec`` tool 14 | 2. using layer wiring in python (with and without helpers) 15 | 3. writing an architecture yourself (advanced) 16 | 4. instantiating the layers by hand and setting up a layout (don't do this) 17 | 18 | Setting the Handler 19 | =================== 20 | If you want to run on CPU in 32bit mode you don't need to do anything. 21 | For GPU you need to do: 22 | 23 | .. code-block:: python 24 | 25 | from brainstorm.handlers import PyCudaHandler 26 | net.set_handler(PyCudaHandler()) 27 | 28 | 29 | Initializing 30 | ============ 31 | 32 | Just use the :meth:`~brainstorm.structure.network.Network.initialize` method. 33 | 34 | 35 | Weight and Gradient Modifiers 36 | ============================= 37 | 38 | ``net.set_weight_modifiers()`` 39 | 40 | ``net.set_gradient_modifiers()`` 41 | 42 | ******* 43 | Running 44 | ******* 45 | Normally a trainer will run the network for you. But if you want to run a 46 | network yourself you have to do this in order: 47 | 48 | 1. ``net.provide_external_data(my_data)`` 49 | 2. ``net.forward_pass()`` 50 | 3. (optional) ``net.backward_pass()`` 51 | 52 | ******************* 53 | Accessing Internals 54 | ******************* 55 | 56 | The recommended way is to always use ``net.get(PATH)`` because that returns 57 | a copy of the buffer in numpy format. If you for some reason want to tamper 58 | with the memory that the network is actually using you can get access with: 59 | ``net.buffer[PATH]``. 60 | 61 | Parameters 62 | ========== 63 | * ``'parameters'`` for an array of all parameters 64 | * ``'LAYER_NAME.parameters.PARAM_NAME'`` for a specific parameter buffer 65 | 66 | For the corresponding derivatives calculated during the backward pass: 67 | * ``'gradients'`` 68 | * ``'LAYER_NAME.gradients.GRAD_NAME'`` 69 | 70 | Inputs and Outputs 71 | ================== 72 | To access the inputs that have been passed to the network you can use the 73 | shortcut ``net.get_inputs(IN_NAME)``. 74 | 75 | To access inputs and outputs of layers use the following paths: 76 | 77 | * ``'LAYER_NAME.inputs.IN_NAME'`` (often IN_NAME = default) 78 | * ``'LAYER_NAME.outputs.OUT_NAME'`` (often OUT_NAME = default) 79 | 80 | For the corresponding derivatives calculated during the backward pass: 81 | 82 | * ``'LAYER_NAME.input_deltas.IN_NAME'`` 83 | * ``'LAYER_NAME.output_deltas.OUT_NAME'`` 84 | 85 | Internals 86 | ========= 87 | Some layers also expose some internal buffers. You can access them with this 88 | path: 89 | 90 | * ``'LAYER_NAME.internals.INTERNAL_NAME'`` 91 | 92 | 93 | ****************** 94 | Loading and Saving 95 | ****************** 96 | 97 | ``net.save_as_hdf5(filename)`` 98 | 99 | ``net = Network.from_hdf5(filename)`` 100 | -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | ########## 2 | Quickstart 3 | ########## 4 | 5 | To use brainstorm in a project:: 6 | 7 | import brainstorm 8 | 9 | -------------------------------------------------------------------------------- /docs/trainer.rst: -------------------------------------------------------------------------------- 1 | ####### 2 | Trainer 3 | ####### 4 | 5 | This is a construction site. 6 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | These examples have been included to quickly present some basic features of Brainstorm. 5 | Prior to running an example, you should prepare the required dataset using the corresponding data preparation script in the ``data`` directory. 6 | For each example, you can uncomment one line (indicated in the code) to make the training faster using your GPU. 7 | 8 | mnist_pi 9 | -------- 10 | 11 | This example trains a simple neural network with 2 fully connected hidden layers and dropout on the MNIST dataset. 12 | 13 | 14 | cifar10_cnn 15 | ----------- 16 | 17 | This example trains a simple convolutional neural network on the CIFAR-10 dataset. 18 | Initialization is one of the most crucial aspects of training neural networks. This example shows how Brainstorm let us flexibly specify the initialization for different layers of the network. 19 | 20 | 21 | hutter_lstm 22 | ----------- 23 | 24 | This example trains a simple character-level LSTM recurrent neural network on the Hutter Prize dataset. It also shows a simple use-case of the ``create_net_from_spec`` tool which is useful for quickly building networks. 25 | 26 | 27 | custom_layer 28 | ------------ 29 | 30 | This example shows how one can write a self-contained custom layer in a single file using Brainstorm. 31 | It also shows how Brainstorm tests can be run on the layer to make sure that the implementation is correct. 32 | Finally, it uses the newly created custom layer in a simple example like any other layer. 33 | -------------------------------------------------------------------------------- /examples/cifar10_cnn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import os 6 | 7 | import h5py 8 | 9 | import brainstorm as bs 10 | from brainstorm.data_iterators import Minibatches 11 | from brainstorm.handlers import PyCudaHandler 12 | from brainstorm.initializers import Gaussian 13 | 14 | bs.global_rnd.set_seed(42) 15 | 16 | # ----------------------------- Set up Iterators ---------------------------- # 17 | 18 | data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '../data') 19 | data_file = os.path.join(data_dir, 'CIFAR-10.hdf5') 20 | ds = h5py.File(data_file, 'r')['normalized_split'] 21 | 22 | getter_tr = Minibatches(100, default=ds['training']['default'][:], 23 | targets=ds['training']['targets'][:]) 24 | getter_va = Minibatches(100, default=ds['validation']['default'][:], 25 | targets=ds['validation']['targets'][:]) 26 | 27 | # ------------------------------ Set up Network ----------------------------- # 28 | 29 | inp, fc = bs.tools.get_in_out_layers('classification', (32, 32, 3), 10) 30 | 31 | (inp >> 32 | bs.layers.Convolution2D(32, kernel_size=(5, 5), padding=2, name='Conv1') >> 33 | bs.layers.Pooling2D(type="max", kernel_size=(3, 3), stride=(2, 2)) >> 34 | bs.layers.Convolution2D(32, kernel_size=(5, 5), padding=2, name='Conv2') >> 35 | bs.layers.Pooling2D(type="max", kernel_size=(3, 3), stride=(2, 2)) >> 36 | bs.layers.Convolution2D(64, kernel_size=(5, 5), padding=2, name='Conv3') >> 37 | bs.layers.Pooling2D(type="max", kernel_size=(3, 3), stride=(2, 2)) >> 38 | bs.layers.FullyConnected(64, name='FC') >> 39 | fc) 40 | 41 | network = bs.Network.from_layer(fc) 42 | # Uncomment next line to use GPU 43 | # network.set_handler(PyCudaHandler()) 44 | network.initialize({'Conv*': {'W': Gaussian(0.01), 'bias': 0}, 45 | 'FC': {'W': Gaussian(0.1), 'bias': 0}, 46 | 'Output_projection': {'W': Gaussian(0.1), 'bias': 0}}) 47 | 48 | # ------------------------------ Set up Trainer ----------------------------- # 49 | 50 | trainer = bs.Trainer(bs.training.MomentumStepper(learning_rate=0.01, momentum=0.9)) 51 | trainer.add_hook(bs.hooks.ProgressBar()) 52 | scorers = [bs.scorers.Accuracy(out_name='Output.outputs.predictions')] 53 | trainer.train_scorers = scorers 54 | trainer.add_hook(bs.hooks.MonitorScores('valid_getter', scorers, 55 | name='validation')) 56 | trainer.add_hook(bs.hooks.SaveBestNetwork('validation.Accuracy', 57 | filename='cifar10_cnn_best.hdf5', 58 | name='best weights', 59 | criterion='max')) 60 | trainer.add_hook(bs.hooks.StopAfterEpoch(20)) 61 | 62 | # --------------------------------- Train ----------------------------------- # 63 | 64 | trainer.train(network, getter_tr, valid_getter=getter_va) 65 | print("Best validation accuracy:", max(trainer.logs["validation"]["Accuracy"])) 66 | -------------------------------------------------------------------------------- /examples/custom_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | from collections import OrderedDict 6 | import os 7 | 8 | import h5py 9 | 10 | import brainstorm as bs 11 | from brainstorm.layers import FullyConnected 12 | from brainstorm.data_iterators import Minibatches 13 | from brainstorm.layers.base_layer import Layer 14 | from brainstorm.structure.buffer_structure import (StructureTemplate, 15 | BufferStructure) 16 | from brainstorm.structure.construction import ConstructionWrapper 17 | 18 | bs.global_rnd.set_seed(42) 19 | 20 | 21 | # ------------------------------ Custom Layer ------------------------------- # 22 | 23 | # This function will be used for wiring the layers 24 | def Square(name=None): 25 | """Create a layer that squares its inputs elementwise""" 26 | return ConstructionWrapper.create(SquareLayerImpl, name=name) 27 | 28 | 29 | # Layer implementations need to inherit from brainstorm.layers.base_layer.Layer 30 | # And their class name needs to end with 'LayerImpl' 31 | class SquareLayerImpl(Layer): 32 | # accept inputs in any format 33 | expected_inputs = {'default': StructureTemplate('...')} 34 | # no kwargs supported 35 | expected_kwargs = {} 36 | 37 | # For a custom layer we need to implement the following 3 methods: 38 | 39 | def setup(self, kwargs, in_shapes): 40 | # In this method we set up the buffer structure of the layer 41 | # we can use the kwargs passed to this layer (here we don't) 42 | # and the shapes of the inputs (an OrderedDict[str, BufferStructure]) 43 | 44 | # This layer is elementwise so the output shapes should be the same as 45 | # the input shapes 46 | outputs = in_shapes 47 | parameters = OrderedDict() # No parameters so this is empty 48 | internals = OrderedDict() # Also no need for internal buffers 49 | return outputs, parameters, internals 50 | 51 | def forward_pass(self, buffers, training_pass=True): 52 | inputs = buffers.inputs.default 53 | outputs = buffers.outputs.default 54 | self.handler.mult_tt(inputs, inputs, outputs) 55 | self.handler.mult_st(0.5, outputs, outputs) 56 | 57 | def backward_pass(self, buffers): 58 | inputs = buffers.inputs.default 59 | output_deltas = buffers.output_deltas.default 60 | input_deltas = buffers.input_deltas.default 61 | self.handler.mult_add_tt(inputs, output_deltas, input_deltas) 62 | 63 | # --------------------------- Testing the Layer ----------------------------- # 64 | # Testing doesn't need to happen before every run. We recommend 65 | # having a layer implementation + the tests in a separate file. 66 | 67 | from brainstorm.tests.tools import get_test_configurations, run_layer_tests 68 | 69 | for cfg in get_test_configurations(): 70 | layer = SquareLayerImpl('Square', 71 | {'default': BufferStructure('T', 'B', 3)}, 72 | set(), set()) 73 | run_layer_tests(layer, cfg) 74 | 75 | 76 | # ------------------------------ Demo Example ------------------------------- # 77 | 78 | # ---------------------------- Set up Iterators ----------------------------- # 79 | 80 | data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '../data') 81 | data_file = os.path.join(data_dir, 'MNIST.hdf5') 82 | ds = h5py.File(data_file, 'r')['normalized_split'] 83 | x_tr, y_tr = ds['training']['default'][:], ds['training']['targets'][:] 84 | x_va, y_va = ds['validation']['default'][:], ds['validation']['targets'][:] 85 | 86 | getter_tr = Minibatches(100, default=x_tr, targets=y_tr) 87 | getter_va = Minibatches(100, default=x_va, targets=y_va) 88 | 89 | # ----------------------------- Set up Network ------------------------------ # 90 | 91 | inp, out = bs.tools.get_in_out_layers('classification', (28, 28, 1), 10) 92 | network = bs.Network.from_layer( 93 | inp >> 94 | FullyConnected(500, name='Hid1', activation='linear') >> 95 | Square(name='MySquareLayer') >> 96 | out 97 | ) 98 | 99 | network.initialize(bs.initializers.Gaussian(0.01)) 100 | 101 | # ----------------------------- Set up Trainer ------------------------------ # 102 | 103 | trainer = bs.Trainer(bs.training.MomentumStepper(learning_rate=0.01, 104 | momentum=0.9)) 105 | trainer.add_hook(bs.hooks.ProgressBar()) 106 | scorers = [bs.scorers.Accuracy(out_name='Output.outputs.predictions')] 107 | trainer.add_hook(bs.hooks.MonitorScores('valid_getter', scorers, 108 | name='validation')) 109 | trainer.add_hook(bs.hooks.EarlyStopper('validation.Accuracy', patience=10, criterion='max')) 110 | trainer.add_hook(bs.hooks.StopAfterEpoch(500)) 111 | 112 | # -------------------------------- Train ------------------------------------ # 113 | 114 | trainer.train(network, getter_tr, valid_getter=getter_va) 115 | print("Best validation set accuracy:", max(trainer.logs["validation"]["Accuracy"])) 116 | -------------------------------------------------------------------------------- /examples/hutter_lstm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import os 6 | 7 | import h5py 8 | 9 | import brainstorm as bs 10 | from brainstorm.data_iterators import OneHot, Minibatches 11 | from brainstorm.handlers import PyCudaHandler 12 | 13 | bs.global_rnd.set_seed(42) 14 | 15 | 16 | # ---------------------------- Set up Iterators ----------------------------- # 17 | 18 | data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '../data') 19 | data_file = os.path.join(data_dir, 'HutterPrize.hdf5') 20 | ds = h5py.File(data_file, 'r')['split'] 21 | x_tr, y_tr = ds['training']['default'][:], ds['training']['targets'][:] 22 | x_va, y_va = ds['validation']['default'][:], ds['validation']['targets'][:] 23 | 24 | getter_tr = OneHot(Minibatches(100, default=x_tr, targets=y_tr, shuffle=False), 25 | {'default': 205}) 26 | getter_va = OneHot(Minibatches(100, default=x_va, targets=y_va, shuffle=False), 27 | {'default': 205}) 28 | 29 | # ----------------------------- Set up Network ------------------------------ # 30 | 31 | network = bs.tools.create_net_from_spec('classification', 205, 205, 32 | 'L1000') 33 | 34 | # Uncomment next line to use the GPU 35 | # network.set_handler(PyCudaHandler()) 36 | network.initialize(bs.initializers.Gaussian(0.01)) 37 | 38 | # ----------------------------- Set up Trainer ------------------------------ # 39 | 40 | trainer = bs.Trainer(bs.training.MomentumStepper(learning_rate=0.01, 41 | momentum=0.9)) 42 | trainer.add_hook(bs.hooks.ProgressBar()) 43 | scorers = [bs.scorers.Accuracy(out_name='Output.outputs.predictions')] 44 | trainer.add_hook(bs.hooks.MonitorScores('valid_getter', scorers, 45 | name='validation', interval=3000, 46 | timescale='update')) 47 | trainer.add_hook(bs.hooks.SaveBestNetwork('validation.total_loss', 48 | filename='hutter_lstm_best.hdf5', 49 | name='best weights', 50 | criterion='min')) 51 | trainer.add_hook(bs.hooks.StopAfterEpoch(500)) 52 | 53 | # -------------------------------- Train ------------------------------------ # 54 | 55 | trainer.train(network, getter_tr, valid_getter=getter_va) 56 | print("Best validation set loss:", max(trainer.logs["validation"]["total_loss"])) 57 | -------------------------------------------------------------------------------- /examples/mnist_pi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from __future__ import division, print_function, unicode_literals 4 | 5 | import os 6 | 7 | import h5py 8 | 9 | import brainstorm as bs 10 | from brainstorm.data_iterators import Minibatches 11 | from brainstorm.handlers import PyCudaHandler 12 | 13 | bs.global_rnd.set_seed(42) 14 | 15 | # ---------------------------- Set up Iterators ----------------------------- # 16 | 17 | data_dir = os.environ.get('BRAINSTORM_DATA_DIR', '../data') 18 | data_file = os.path.join(data_dir, 'MNIST.hdf5') 19 | ds = h5py.File(data_file, 'r')['normalized_split'] 20 | x_tr, y_tr = ds['training']['default'][:], ds['training']['targets'][:] 21 | x_va, y_va = ds['validation']['default'][:], ds['validation']['targets'][:] 22 | 23 | getter_tr = Minibatches(100, default=x_tr, targets=y_tr) 24 | getter_va = Minibatches(100, default=x_va, targets=y_va) 25 | 26 | # ----------------------------- Set up Network ------------------------------ # 27 | 28 | inp, fc = bs.tools.get_in_out_layers('classification', (28, 28, 1), 10, projection_name='FC') 29 | network = bs.Network.from_layer( 30 | inp >> 31 | bs.layers.Dropout(drop_prob=0.2) >> 32 | bs.layers.FullyConnected(1200, name='Hid1', activation='rel') >> 33 | bs.layers.Dropout(drop_prob=0.5) >> 34 | bs.layers.FullyConnected(1200, name='Hid2', activation='rel') >> 35 | bs.layers.Dropout(drop_prob=0.5) >> 36 | fc 37 | ) 38 | 39 | # Uncomment next line to use GPU 40 | # network.set_handler(PyCudaHandler()) 41 | network.initialize(bs.initializers.Gaussian(0.01)) 42 | network.set_weight_modifiers({"FC": bs.value_modifiers.ConstrainL2Norm(1)}) 43 | 44 | # ----------------------------- Set up Trainer ------------------------------ # 45 | 46 | trainer = bs.Trainer(bs.training.MomentumStepper(learning_rate=0.1, momentum=0.9)) 47 | trainer.add_hook(bs.hooks.ProgressBar()) 48 | scorers = [bs.scorers.Accuracy(out_name='Output.outputs.predictions')] 49 | trainer.add_hook(bs.hooks.MonitorScores('valid_getter', scorers, 50 | name='validation')) 51 | trainer.add_hook(bs.hooks.SaveBestNetwork('validation.Accuracy', 52 | filename='mnist_pi_best.hdf5', 53 | name='best weights', 54 | criterion='max')) 55 | trainer.add_hook(bs.hooks.StopAfterEpoch(500)) 56 | 57 | # -------------------------------- Train ------------------------------------ # 58 | 59 | trainer.train(network, getter_tr, valid_getter=getter_va) 60 | print("Best validation accuracy:", max(trainer.logs["validation"]["Accuracy"])) 61 | -------------------------------------------------------------------------------- /pycuda_requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/inducer/pycuda#egg=pycuda 2 | git+https://github.com/lebedov/scikit-cuda#egg=scikit-cuda 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | cython 3 | six 4 | wheel>=0.22 5 | mock 6 | h5py 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | [wheel] 4 | universal = 1 5 | [pytest] 6 | norecursedirs = build dist *.egg_info .tox .cache .git docs 7 | [flake8] 8 | ignore = D100,D101,D102,D103 9 | exclude = *__init__.py,*optional.py 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | from setuptools import setup, Extension 6 | from setuptools.command.build_ext import build_ext as _build_ext 7 | from setuptools.command.test import test as TestCommand 8 | from distutils.errors import CompileError 9 | from warnings import warn 10 | 11 | # Check if we are going to use Cython to compile the pyx extension files 12 | try: 13 | from Cython.Distutils import build_ext as _build_ext 14 | except ImportError: 15 | use_cython = False 16 | else: 17 | use_cython = True 18 | 19 | # Try to get __about__ information 20 | try: 21 | from brainstorm import __about__ 22 | about = __about__.__dict__ 23 | except ImportError: 24 | # installing - dependencies are not there yet 25 | ext_modules = [] 26 | # Manually extract the __about__ 27 | about = dict() 28 | exec(open("brainstorm/__about__.py").read(), about) 29 | 30 | 31 | if sys.argv[-1] == 'publish': 32 | os.system('python setup.py sdist upload') 33 | sys.exit() 34 | 35 | 36 | # Add includes for building extensions 37 | class build_ext(_build_ext): 38 | def finalize_options(self): 39 | _build_ext.finalize_options(self) 40 | __builtins__.__NUMPY_SETUP__ = False 41 | import numpy as np 42 | self.include_dirs.append(np.get_include()) 43 | 44 | def run(self): 45 | try: 46 | _build_ext.run(self) 47 | except CompileError: 48 | warn('Failed to build optional extension modules') 49 | 50 | # Cythonize pyx if possible, else compile C 51 | if use_cython: 52 | from Cython.Build import cythonize 53 | extensions = cythonize([Extension('brainstorm.handlers._cpuop', 54 | ['brainstorm/handlers/_cpuop.pyx'])]) 55 | 56 | else: 57 | extensions = [ 58 | Extension( 59 | 'brainstorm.handlers._cpuop', ['brainstorm/handlers/_cpuop.c'], 60 | extra_compile_args=['-w', '-Ofast']), 61 | ] 62 | 63 | 64 | # Setup testing 65 | class PyTest(TestCommand): 66 | user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] 67 | 68 | def initialize_options(self): 69 | TestCommand.initialize_options(self) 70 | self.pytest_args = [] 71 | 72 | def finalize_options(self): 73 | TestCommand.finalize_options(self) 74 | self.test_args = [] 75 | self.test_suite = True 76 | 77 | def run_tests(self): 78 | # import here, cause outside the eggs aren't loaded 79 | import pytest 80 | errno = pytest.main(self.pytest_args) 81 | sys.exit(errno) 82 | 83 | 84 | doclink = """ 85 | Documentation 86 | ------------- 87 | 88 | The full documentation is at http://brainstorm.rtfd.org.""" 89 | history = open('HISTORY.rst').read().replace('.. :changelog:', '') 90 | live_viz = ['bokeh'] 91 | draw_net = ['pygraphviz'] 92 | tests = ['pytest', 'mock'] 93 | pycuda = ['pycuda>=2015.1.3', 'scikit-cuda>=0.5.1'] 94 | all_deps = live_viz + draw_net + tests + pycuda 95 | 96 | setup( 97 | name='brainstorm', 98 | version=about['__version__'], 99 | description='Fast, flexible and fun neural networks.', 100 | long_description=doclink + '\n\n' + history, 101 | author=about['__author__'], 102 | author_email="mailstorm@googlemail.com", 103 | url=about['__url__'], 104 | packages=['brainstorm', 105 | 'brainstorm.structure', 106 | 'brainstorm.layers', 107 | 'brainstorm.training', 108 | 'brainstorm.handlers'], 109 | setup_requires=['cython', 'numpy>=1.8'], 110 | install_requires=['cython', 'h5py', 'mock', 'numpy>=1.8', 'six'], 111 | extras_require={ 112 | 'live_viz': live_viz, 113 | 'draw_net': draw_net, 114 | 'test': tests, 115 | 'pycuda': pycuda, 116 | 'all': all_deps 117 | }, 118 | tests_require=tests, 119 | cmdclass={'test': PyTest, 'build_ext': build_ext}, 120 | license=about['__license__'], 121 | classifiers=[ 122 | 'Development Status :: 4 - Beta', 123 | 'Intended Audience :: Developers', 124 | 'Intended Audience :: Science/Research', 125 | 'Intended Audience :: Education', 126 | 'License :: OSI Approved :: MIT License', 127 | 'Natural Language :: English', 128 | 'Programming Language :: Python :: 2.7', 129 | 'Programming Language :: Python :: 3.3', 130 | ], 131 | 132 | ext_modules=extensions, 133 | ) 134 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py32, py34, flake8, pep8 3 | 4 | [testenv] 5 | setenv = 6 | PYTHONPATH = {toxinidir}:{toxinidir}/brainstorm 7 | deps = 8 | -rrequirements.txt 9 | pytest 10 | 11 | commands = 12 | py.test --basetemp={envtmpdir} 13 | 14 | [testenv:flake8] 15 | basepython=python 16 | deps= 17 | flake8 18 | pep8-naming 19 | mccabe 20 | # flake8-docstrings 21 | commands= 22 | flake8 --max-complexity 8 brainstorm 23 | 24 | [testenv:pep8] 25 | basepython=python 26 | deps= 27 | -rrequirements.txt 28 | pytest 29 | pytest-pep8 30 | commands= 31 | py.test --pep8 -m pep8 32 | 33 | [testenv:docs] 34 | changedir=docs/ 35 | deps = 36 | -r{toxinidir}/requirements.txt 37 | sphinx 38 | commands = 39 | sphinx-build -b linkcheck ./ _build/ 40 | sphinx-build -b html ./ _build/ 41 | 42 | [testenv:coverage] 43 | basepython=python 44 | deps= 45 | pytest-cov 46 | python-coveralls 47 | commands= 48 | py.test \ 49 | --cov sacred \ 50 | {posargs} 51 | coveralls --------------------------------------------------------------------------------