├── .editorconfig ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── docs ├── Makefile ├── authors.rst ├── conf.py ├── contributing.rst ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── readme.rst └── usage.rst ├── examples ├── Comparison.ipynb ├── Convolutional Variational AE.ipynb ├── DRAW.ipynb ├── Differentiable RAM.ipynb ├── NTM.ipynb ├── Neural Stacks and Queues.ipynb ├── Spatial Transformer Networks.ipynb ├── Winner-take-all autoencoders.ipynb ├── babi_memn2n.py ├── fista_sc.ipynb ├── imdb_brnn.py ├── imdb_readout.py ├── main_img.gif ├── stateful_text_generation.py ├── wip-AA.ipynb └── wip-GAN.ipynb ├── requirements.txt ├── setup.cfg ├── setup.py ├── seya ├── __init__.py ├── callbacks.py ├── data_utils.py ├── layers │ ├── __init__.py │ ├── attention.py │ ├── base.py │ ├── coding.py │ ├── containers.py │ ├── conv_rnn.py │ ├── convolutional.py │ ├── deep_recurrent.py │ ├── draw.py │ ├── embedding.py │ ├── imageproc.py │ ├── memnn.py │ ├── memnn2.py │ ├── normalization.py │ ├── ntm.py │ ├── readout.py │ ├── recurrent.py │ ├── regularization.py │ ├── stack.py │ ├── temp.py │ ├── tensor.py │ └── variational.py ├── models.py ├── objectives.py ├── optimizers.py ├── parzen.py ├── preprocessing │ ├── README.md │ ├── __init__.py │ ├── features_to_hdf5.py │ ├── image.py │ └── videos_to_hdf5.py ├── regularizers.py ├── seya.py └── utils.py ├── tests ├── __init__.py ├── test_apply.py ├── test_base.py ├── test_containers.py ├── test_conv_rnn.py ├── test_memnn.py ├── test_recurrent.py ├── test_seya.py ├── test_utils.py └── test_variational.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.*ipynb_checkpoints 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | htmlcov 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Complexity 40 | output/*.html 41 | output/*/index.html 42 | 43 | # Sphinx 44 | docs/_build 45 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: python 4 | python: 5 | - "2.7" 6 | - "3.4" 7 | install: 8 | # code below is taken from http://conda.pydata.org/docs/travis.html 9 | # We do this conditionally because it saves us some downloading if the 10 | # version is the same. 11 | - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then 12 | wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; 13 | else 14 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 15 | fi 16 | - bash miniconda.sh -b -p $HOME/miniconda 17 | - export PATH="$HOME/miniconda/bin:$PATH" 18 | - hash -r 19 | - conda config --set always_yes yes --set changeps1 no 20 | - conda update -q conda 21 | # Useful for debugging any issues with conda 22 | - conda info -a 23 | 24 | - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib pandas pytest h5py 25 | - source activate test-environment 26 | - pip install pytest-cov python-coveralls 27 | - pip install git+git://github.com/Theano/Theano.git 28 | - python setup.py install 29 | - git clone https://github.com/fchollet/keras.git@654404c2ed8db47a5361a3 $PWD/keras 30 | - export PYTHONPATH=$PWD/keras 31 | 32 | # install TensorFlow 33 | # - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then 34 | # pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.5.0-cp27-none-linux_x86_64.whl; 35 | # fi 36 | # command to run tests 37 | script: 38 | - PYTHONPATH=$PWD:$PYTHONPATH py.test -v --cov-report term-missing --cov seya tests/ 39 | after_success: 40 | - coveralls 41 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Eder Santana 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/edersantana/seya/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whoever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whoever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | Seya could always use more documentation, whether as part of the 40 | official Seya docs, in docstrings, or even on the web in blog posts, 41 | articles, and such. 42 | 43 | Submit Feedback 44 | ~~~~~~~~~~~~~~~ 45 | 46 | The best way to send feedback is to file an issue at https://github.com/edersantana/seya/issues. 47 | 48 | If you are proposing a feature: 49 | 50 | * Explain in detail how it would work. 51 | * Keep the scope as narrow as possible, to make it easier to implement. 52 | * Remember that this is a volunteer-driven project, and that contributions 53 | are welcome :) 54 | 55 | Get Started! 56 | ------------ 57 | 58 | Ready to contribute? Here's how to set up `seya` for local development. 59 | 60 | 1. Fork the `seya` repo on GitHub. 61 | 2. Clone your fork locally:: 62 | 63 | $ git clone git@github.com:your_name_here/seya.git 64 | 65 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 66 | 67 | $ mkvirtualenv seya 68 | $ cd seya/ 69 | $ python setup.py develop 70 | 71 | 4. Create a branch for local development:: 72 | 73 | $ git checkout -b name-of-your-bugfix-or-feature 74 | 75 | Now you can make your changes locally. 76 | 77 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 78 | 79 | $ flake8 seya tests 80 | $ python setup.py test 81 | $ tox 82 | 83 | To get flake8 and tox, just pip install them into your virtualenv. 84 | 85 | 6. Commit your changes and push your branch to GitHub:: 86 | 87 | $ git add . 88 | $ git commit -m "Your detailed description of your changes." 89 | $ git push origin name-of-your-bugfix-or-feature 90 | 91 | 7. Submit a pull request through the GitHub website. 92 | 93 | Pull Request Guidelines 94 | ----------------------- 95 | 96 | Before you submit a pull request, check that it meets these guidelines: 97 | 98 | 1. The pull request should include tests. 99 | 2. If the pull request adds functionality, the docs should be updated. Put 100 | your new functionality into a function with a docstring, and add the 101 | feature to the list in README.rst. 102 | 3. The pull request should work for Python 2.6, 2.7, 3.3, and 3.4, and for PyPy. Check 103 | https://travis-ci.org/edersantana/seya/pull_requests 104 | and make sure that the tests pass for all supported Python versions. 105 | 106 | Tips 107 | ---- 108 | 109 | To run a subset of tests:: 110 | 111 | $ python -m unittest tests.test_seya 112 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | 0.1.0 (2015-07-07) 7 | --------------------- 8 | 9 | * First release on PyPI. 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Eder Santana 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | * Neither the name of Seya nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | 14 | OBS: 15 | This library uses code from pylearn2 which is under BSD clause-3 license. 16 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs clean 2 | 3 | help: 4 | @echo "clean - remove all build, test, coverage and Python artifacts" 5 | @echo "clean-build - remove build artifacts" 6 | @echo "clean-pyc - remove Python file artifacts" 7 | @echo "clean-test - remove test and coverage artifacts" 8 | @echo "lint - check style with flake8" 9 | @echo "test - run tests quickly with the default Python" 10 | @echo "test-all - run tests on every Python version with tox" 11 | @echo "coverage - check code coverage quickly with the default Python" 12 | @echo "docs - generate Sphinx HTML documentation, including API docs" 13 | @echo "release - package and upload a release" 14 | @echo "dist - package" 15 | @echo "install - install the package to the active Python's site-packages" 16 | 17 | clean: clean-build clean-pyc clean-test 18 | 19 | clean-build: 20 | rm -fr build/ 21 | rm -fr dist/ 22 | rm -fr .eggs/ 23 | find . -name '*.egg-info' -exec rm -fr {} + 24 | find . -name '*.egg' -exec rm -f {} + 25 | 26 | clean-pyc: 27 | find . -name '*.pyc' -exec rm -f {} + 28 | find . -name '*.pyo' -exec rm -f {} + 29 | find . -name '*~' -exec rm -f {} + 30 | find . -name '__pycache__' -exec rm -fr {} + 31 | 32 | clean-test: 33 | rm -fr .tox/ 34 | rm -f .coverage 35 | rm -fr htmlcov/ 36 | 37 | lint: 38 | flake8 seya tests 39 | 40 | test: 41 | python setup.py test 42 | 43 | test-all: 44 | tox 45 | 46 | coverage: 47 | coverage run --source seya setup.py test 48 | coverage report -m 49 | coverage html 50 | open htmlcov/index.html 51 | 52 | docs: 53 | rm -f docs/seya.rst 54 | rm -f docs/modules.rst 55 | sphinx-apidoc -o docs/ seya 56 | $(MAKE) -C docs clean 57 | $(MAKE) -C docs html 58 | open docs/_build/html/index.html 59 | 60 | release: clean 61 | python setup.py sdist upload 62 | python setup.py bdist_wheel upload 63 | 64 | dist: clean 65 | python setup.py sdist 66 | python setup.py bdist_wheel 67 | ls -l dist 68 | 69 | install: clean 70 | python setup.py install 71 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | Seya 3 | =============================== 4 | 5 | Bringing up some extra Cosmo to Keras. 6 | 7 | * Free software: BSD license 8 | * Documentation: https://seya.readthedocs.org. 9 | 10 | Note: This code base was built with Keras 0.3. There is a branch called `keras1` where some of the layers were updated to work 11 | with the new API. To makes things easier to maintain, I'll start publishing individual models as self-contained github 12 | repositories and cross-reference them here. 13 | 14 | Why Seya? 15 | --------- 16 | Those smart guys may know a lot about ancient greek mathematicians and poetry. 17 | But the first thing I remember about Greece is my childhood favorite anime, Saint Seiya. 18 | I misspelled it as Seya and now the name is 1/5 shorter. 19 | 20 | Features 21 | -------- 22 | 23 | * Spatial Transformer Networks example 24 | * Recursive graph WIP_ 25 | * Sparse coding 26 | * Bidirectional RNN example_ 27 | * DRAW_ 28 | * `Neural Turing Machines`_ 29 | 30 | .. _WIP: https://github.com/fchollet/keras/issues/620 31 | .. _example: https://github.com/EderSantana/seya/blob/master/examples/imdb_brnn.py 32 | .. _DRAW: https://github.com/EderSantana/seya/blob/master/examples/DRAW.ipynb 33 | .. _Neural Turing Machines: https://github.com/EderSantana/seya/blob/master/examples/NTM.ipynb 34 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/seya.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/seya.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/seya" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/seya" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # seya documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another 20 | # directory, add these directories to sys.path here. If the directory is 21 | # relative to the documentation root, use os.path.abspath to make it 22 | # absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # Get the project root dir, which is the parent dir of this 26 | cwd = os.getcwd() 27 | project_root = os.path.dirname(cwd) 28 | 29 | # Insert the project root dir as the first element in the PYTHONPATH. 30 | # This lets us ensure that the source package is imported, and that its 31 | # version is used. 32 | sys.path.insert(0, project_root) 33 | 34 | import seya 35 | 36 | # -- General configuration --------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | #needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 43 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # The suffix of source filenames. 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | #source_encoding = 'utf-8-sig' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # General information about the project. 58 | project = u'Seya' 59 | copyright = u'2015, Eder Santana' 60 | 61 | # The version info for the project you're documenting, acts as replacement 62 | # for |version| and |release|, also used in various other places throughout 63 | # the built documents. 64 | # 65 | # The short X.Y version. 66 | version = seya.__version__ 67 | # The full version, including alpha/beta/rc tags. 68 | release = seya.__version__ 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | #language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to 75 | # some non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = ['_build'] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built 106 | # documents. 107 | #keep_warnings = False 108 | 109 | 110 | # -- Options for HTML output ------------------------------------------- 111 | 112 | # The theme to use for HTML and HTML Help pages. See the documentation for 113 | # a list of builtin themes. 114 | html_theme = 'default' 115 | 116 | # Theme options are theme-specific and customize the look and feel of a 117 | # theme further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | #html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as 129 | # html_title. 130 | #html_short_title = None 131 | 132 | # The name of an image file (relative to this directory) to place at the 133 | # top of the sidebar. 134 | #html_logo = None 135 | 136 | # The name of an image file (within the static path) to use as favicon 137 | # of the docs. This file should be a Windows icon file (.ico) being 138 | # 16x16 or 32x32 pixels large. 139 | #html_favicon = None 140 | 141 | # Add any paths that contain custom static files (such as style sheets) 142 | # here, relative to this directory. They are copied after the builtin 143 | # static files, so a file named "default.css" will overwrite the builtin 144 | # "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page 148 | # bottom, using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names 159 | # to template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. 175 | # Default is True. 176 | #html_show_sphinx = True 177 | 178 | # If true, "(C) Copyright ..." is shown in the HTML footer. 179 | # Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages 183 | # will contain a tag referring to it. The value of this option 184 | # must be the base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Output file base name for HTML help builder. 191 | htmlhelp_basename = 'seyadoc' 192 | 193 | 194 | # -- Options for LaTeX output ------------------------------------------ 195 | 196 | latex_elements = { 197 | # The paper size ('letterpaper' or 'a4paper'). 198 | #'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | #'pointsize': '10pt', 202 | 203 | # Additional stuff for the LaTeX preamble. 204 | #'preamble': '', 205 | } 206 | 207 | # Grouping the document tree into LaTeX files. List of tuples 208 | # (source start file, target name, title, author, documentclass 209 | # [howto/manual]). 210 | latex_documents = [ 211 | ('index', 'seya.tex', 212 | u'Seya Documentation', 213 | u'Eder Santana', 'manual'), 214 | ] 215 | 216 | # The name of an image file (relative to this directory) to place at 217 | # the top of the title page. 218 | #latex_logo = None 219 | 220 | # For "manual" documents, if this is true, then toplevel headings 221 | # are parts, not chapters. 222 | #latex_use_parts = False 223 | 224 | # If true, show page references after internal links. 225 | #latex_show_pagerefs = False 226 | 227 | # If true, show URL addresses after external links. 228 | #latex_show_urls = False 229 | 230 | # Documents to append as an appendix to all manuals. 231 | #latex_appendices = [] 232 | 233 | # If false, no module index is generated. 234 | #latex_domain_indices = True 235 | 236 | 237 | # -- Options for manual page output ------------------------------------ 238 | 239 | # One entry per manual page. List of tuples 240 | # (source start file, name, description, authors, manual section). 241 | man_pages = [ 242 | ('index', 'seya', 243 | u'Seya Documentation', 244 | [u'Eder Santana'], 1) 245 | ] 246 | 247 | # If true, show URL addresses after external links. 248 | #man_show_urls = False 249 | 250 | 251 | # -- Options for Texinfo output ---------------------------------------- 252 | 253 | # Grouping the document tree into Texinfo files. List of tuples 254 | # (source start file, target name, title, author, 255 | # dir menu entry, description, category) 256 | texinfo_documents = [ 257 | ('index', 'seya', 258 | u'Seya Documentation', 259 | u'Eder Santana', 260 | 'seya', 261 | 'One line description of project.', 262 | 'Miscellaneous'), 263 | ] 264 | 265 | # Documents to append as an appendix to all manuals. 266 | #texinfo_appendices = [] 267 | 268 | # If false, no module index is generated. 269 | #texinfo_domain_indices = True 270 | 271 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 272 | #texinfo_show_urls = 'footnote' 273 | 274 | # If true, do not generate a @detailmenu in the "Top" node's menu. 275 | #texinfo_no_detailmenu = False 276 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. seya documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Seya's documentation! 7 | ====================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | contributing 18 | authors 19 | history 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | 28 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | At the command line:: 6 | 7 | $ easy_install seya 8 | 9 | Or, if you have virtualenvwrapper installed:: 10 | 11 | $ mkvirtualenv seya 12 | $ pip install seya 13 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\seya.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\seya.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Usage 3 | ======== 4 | 5 | To use Seya in a project:: 6 | 7 | import seya 8 | -------------------------------------------------------------------------------- /examples/babi_memn2n.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras.models import Sequential 3 | from keras.layers.core import Lambda, Dense 4 | from keras.layers.recurrent import LSTM 5 | from keras.utils.data_utils import get_file 6 | from keras.preprocessing.sequence import pad_sequences 7 | from functools import reduce 8 | import tarfile 9 | import numpy as np 10 | import re 11 | 12 | from seya.layers.memnn2 import MemN2N 13 | 14 | 15 | def tokenize(sent): 16 | '''Return the tokens of a sentence including punctuation. 17 | >>> tokenize('Bob dropped the apple. Where is the apple?') 18 | ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?'] 19 | ''' 20 | return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()] 21 | 22 | 23 | def parse_stories(lines, only_supporting=False): 24 | '''Parse stories provided in the bAbi tasks format 25 | If only_supporting is true, only the sentences that support the answer are kept. 26 | ''' 27 | data = [] 28 | story = [] 29 | for line in lines: 30 | line = line.decode('utf-8').strip() 31 | nid, line = line.split(' ', 1) 32 | nid = int(nid) 33 | if nid == 1: 34 | story = [] 35 | if '\t' in line: 36 | q, a, supporting = line.split('\t') 37 | q = tokenize(q) 38 | substory = None 39 | if only_supporting: 40 | # Only select the related substory 41 | supporting = map(int, supporting.split()) 42 | substory = [story[i - 1] for i in supporting] 43 | else: 44 | # Provide all the substories 45 | substory = [x for x in story if x] 46 | data.append((substory, q, a)) 47 | story.append('') 48 | else: 49 | sent = tokenize(line) 50 | story.append(sent) 51 | return data 52 | 53 | 54 | def get_stories(f, only_supporting=False, max_length=None): 55 | '''Given a file name, read the file, retrieve the stories, and then convert the sentences into a single story. 56 | If max_length is supplied, any stories longer than max_length tokens will be discarded. 57 | ''' 58 | data = parse_stories(f.readlines(), only_supporting=only_supporting) 59 | return data 60 | 61 | 62 | def vectorize_stories(data, word_idx, memory_length, input_length): 63 | X = [] 64 | Xq = [] 65 | Y = [] 66 | for stories, query, answer in data: 67 | x = [[word_idx[w] for w in story] for story in stories] 68 | xq = [word_idx[w] for w in query] 69 | y = np.zeros(len(word_idx) + 1) # let's not forget that index 0 is reserved 70 | y[word_idx[answer]] = 1 71 | X.append(x) 72 | Xq.append(xq) 73 | Y.append(y) 74 | padded_X = [pad_sequences(xx, maxlen=input_length) for xx in X] 75 | out = [] 76 | # pad stories 77 | for story in padded_X: 78 | Z = np.zeros((memory_length, input_length)) 79 | Z[-len(story):] = story 80 | out.append(Z) 81 | return (np.asarray(out), 82 | pad_sequences(Xq, maxlen=input_length), np.array(Y)) 83 | 84 | path = get_file('babi-tasks-v1-2.tar.gz', 85 | origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz') 86 | tar = tarfile.open(path) 87 | 88 | challenges = { 89 | # QA1 with 10,000 samples 90 | 'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt', 91 | # QA2 with 10,000 samples 92 | 'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt', 93 | } 94 | 95 | # challenge_type = 'two_supporting_facts_10k' 96 | challenge_type = 'single_supporting_fact_10k' 97 | challenge = challenges[challenge_type] 98 | 99 | print('Extracting stories for the challenge:', challenge_type) 100 | train_stories = get_stories(tar.extractfile(challenge.format('train'))) 101 | test_stories = get_stories(tar.extractfile(challenge.format('test'))) 102 | 103 | vocab = sorted( 104 | reduce( 105 | lambda x, y: x | y, ( 106 | set( 107 | reduce(lambda x, y: x + y, stories) + q + [answer]) 108 | for stories, q, answer in train_stories + test_stories))) 109 | 110 | vocab_size = len(vocab) + 1 111 | memory_length = max(map(len, (x for x, _, _ in train_stories + test_stories))) 112 | # query_maxlen = max(map(len, (x for _, x, _ in train_stories + test_stories))) 113 | input_length = max(map(len, (x[0] for x, _, _ in train_stories + test_stories))) 114 | word_idx = dict((c, i+1) for i, c in enumerate(vocab)) 115 | output_dim = 64 116 | 117 | inputs_train, queries_train, answers_train = vectorize_stories( 118 | train_stories, word_idx, memory_length, input_length) 119 | inputs_test, queries_test, answers_test = vectorize_stories( 120 | test_stories, word_idx, memory_length, input_length) 121 | 122 | queries_train = queries_train[:, None, :] 123 | queries_test = queries_test[:, None, :] 124 | 125 | # Model definition 126 | facts = Sequential() 127 | facts.add(Lambda(lambda x: x, input_shape=(memory_length, vocab_size), 128 | output_shape=(memory_length, vocab_size))) 129 | question = Sequential() 130 | question.add(Lambda(lambda x: x, input_shape=(1, vocab_size), 131 | output_shape=(1, vocab_size))) 132 | 133 | memnn = MemN2N([facts, question], output_dim=output_dim, input_dim=vocab_size, 134 | input_length=input_length, 135 | memory_length=memory_length, hops=1, output_shape=(vocab_size,)) 136 | memnn.build() 137 | 138 | model = Sequential() 139 | model.add(memnn) 140 | model.add(LSTM(32)) 141 | model.add(Dense(vocab_size, activation="softmax")) 142 | 143 | W = model.trainable_weights[0].get_value() 144 | model.compile("rmsprop", "categorical_crossentropy") 145 | model.fit([inputs_train, queries_train], answers_train, 146 | batch_size=32, 147 | nb_epoch=100, 148 | show_accuracy=True, 149 | validation_data=([inputs_test, queries_test], answers_test)) 150 | 151 | # print(W - model.trainable_weights[0].get_value()) 152 | -------------------------------------------------------------------------------- /examples/imdb_brnn.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import numpy as np 4 | np.random.seed(1337) # for reproducibility 5 | 6 | from keras.preprocessing import sequence 7 | from keras.optimizers import SGD, RMSprop, Adagrad 8 | from keras.utils import np_utils 9 | from keras.models import Sequential 10 | from keras.layers.core import Dense, Dropout, Activation 11 | from keras.layers.embeddings import Embedding 12 | from keras.layers.recurrent import LSTM, GRU 13 | from keras.datasets import imdb 14 | 15 | from seya.layers.recurrent import Bidirectional 16 | 17 | ''' 18 | Train a Bidirectional-LSTM on the IMDB sentiment classification task. 19 | Code borrowed from Keras/examples 20 | 21 | The dataset is actually too small for LSTM to be of any advantage 22 | compared to simpler, much faster methods such as TF-IDF+LogReg. 23 | 24 | Notes: 25 | 26 | - RNNs are tricky. Choice of batch size is important, 27 | choice of loss and optimizer is critical, etc. 28 | Some configurations won't converge. 29 | 30 | - LSTM loss decrease patterns during training can be quite different 31 | from what you see with CNNs/MLPs/etc. 32 | 33 | GPU command: 34 | THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python imdb_lstm.py 35 | ''' 36 | 37 | max_features = 20000 38 | maxlen = 100 # cut texts after this number of words (among top max_features most common words) 39 | batch_size = 32 40 | 41 | print("Loading data...") 42 | (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features, test_split=0.2) 43 | print(len(X_train), 'train sequences') 44 | print(len(X_test), 'test sequences') 45 | 46 | print("Pad sequences (samples x time)") 47 | X_train = sequence.pad_sequences(X_train, maxlen=maxlen) 48 | X_test = sequence.pad_sequences(X_test, maxlen=maxlen) 49 | print('X_train shape:', X_train.shape) 50 | print('X_test shape:', X_test.shape) 51 | 52 | lstm = LSTM(output_dim=64) 53 | gru = GRU(output_dim=64) # original examples was 128, we divide by 2 because results will be concatenated 54 | brnn = Bidirectional(forward=lstm, backward=gru) 55 | 56 | print('Build model...') 57 | model = Sequential() 58 | model.add(Embedding(max_features, 128, input_length=maxlen)) 59 | model.add(brnn) # try using another Bidirectional RNN inside the Bidirectional RNN. Inception meets callback hell. 60 | model.add(Dropout(0.5)) 61 | model.add(Dense(1)) 62 | model.add(Activation('sigmoid')) 63 | 64 | # try using different optimizers and different optimizer configs 65 | model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary") 66 | 67 | print("Train...") 68 | model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=4, validation_data=(X_test, y_test), show_accuracy=True) 69 | score, acc = model.evaluate(X_test, y_test, batch_size=batch_size, show_accuracy=True) 70 | print('Test score:', score) 71 | print('Test accuracy:', acc) 72 | -------------------------------------------------------------------------------- /examples/imdb_readout.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import numpy as np 4 | np.random.seed(1337) # for reproducibility 5 | 6 | from keras.preprocessing import sequence 7 | from keras.optimizers import SGD, RMSprop, Adagrad 8 | from keras.utils import np_utils 9 | from keras.models import Sequential 10 | from keras.layers.core import Dense, Dropout, Activation 11 | from keras.layers.embeddings import Embedding 12 | from keras.layers.recurrent import LSTM, GRU 13 | from keras.datasets import imdb 14 | 15 | from seya.layers.readout import GRUwithReadout 16 | 17 | ''' 18 | Train a GRU with readout on the IMDB sentiment classification task. 19 | Code borrowed from Keras/examples. 20 | 21 | When doing sequence generation is usually useful to feedback the final 22 | prediction alongside the hidden states. This example is not a really good 23 | use case, but at least it shows how to use the new layer. 24 | 25 | The dataset is actually too small for LSTM to be of any advantage 26 | compared to simpler, much faster methods such as TF-IDF+LogReg. 27 | 28 | Notes: 29 | 30 | - RNNs are tricky. Choice of batch size is important, 31 | choice of loss and optimizer is critical, etc. 32 | Some configurations won't converge. 33 | 34 | GPU command: 35 | THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python imdb_readout.py 36 | ''' 37 | 38 | max_features = 20000 39 | maxlen = 100 # cut texts after this number of words (among top max_features most common words) 40 | batch_size = 32 41 | 42 | print("Loading data...") 43 | (X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features, test_split=0.2) 44 | print(len(X_train), 'train sequences') 45 | print(len(X_test), 'test sequences') 46 | 47 | print("Pad sequences (samples x time)") 48 | X_train = sequence.pad_sequences(X_train, maxlen=maxlen) 49 | X_test = sequence.pad_sequences(X_test, maxlen=maxlen) 50 | print('X_train shape:', X_train.shape) 51 | print('X_test shape:', X_test.shape) 52 | 53 | state_dim = 128 54 | readout = Sequential() 55 | readout.add(Dense(1, input_shape=(state_dim,), activation='sigmoid')) 56 | 57 | # The GRU's actual input dim is data_dim + readout.output_dim. 58 | # This is estimated internally 59 | gru_wr = GRUwithReadout(readout) 60 | 61 | print('Build model...') 62 | model = Sequential() 63 | model.add(Embedding(max_features, 128, input_length=maxlen)) 64 | model.add(gru_wr) # try using another Bidirectional RNN inside the Bidirectional RNN. Inception meets callback hell. 65 | 66 | # try using different optimizers and different optimizer configs 67 | model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary") 68 | 69 | print("Train...") 70 | model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=4, validation_data=(X_test, y_test), show_accuracy=True) 71 | score, acc = model.evaluate(X_test, y_test, batch_size=batch_size, show_accuracy=True) 72 | print('Test score:', score) 73 | print('Test accuracy:', acc) 74 | -------------------------------------------------------------------------------- /examples/main_img.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EderSantana/seya/a48bfc3af54dace95a399ace3a6266afe64b3280/examples/main_img.gif -------------------------------------------------------------------------------- /examples/stateful_text_generation.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.layers.recurrent import LSTM 5 | from keras.datasets.data_utils import get_file 6 | import numpy as np 7 | import random, sys 8 | 9 | from seya.layers.recurrent import StatefulGRU as SGRU 10 | from seya.callbacks import ResetRNNState 11 | ''' 12 | Example modified from Keras's script to generate text from Nietzsche's writings. 13 | 14 | At least 20 epochs are required before the generated text 15 | starts sounding coherent. 16 | 17 | It is recommended to run this script on GPU, as recurrent 18 | networks are quite computationally intensive. 19 | 20 | If you try this script on new data, make sure your corpus 21 | has at least ~100k characters. ~1M is better. 22 | ''' 23 | 24 | path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/nietzsche.txt") 25 | text = open(path).read().lower() 26 | print('corpus length:', len(text)) 27 | 28 | chars = set(text) 29 | print('total chars:', len(chars)) 30 | char_indices = dict((c, i) for i, c in enumerate(chars)) 31 | indices_char = dict((i, c) for i, c in enumerate(chars)) 32 | 33 | # cut the text in semi-redundant sequences of maxlen characters 34 | batch_size = 128 35 | maxlen = 20 36 | step = 3 37 | sentences = [] 38 | next_chars = [] 39 | for i in range(0, len(text) - maxlen, step): 40 | sentences.append(text[i : i + maxlen]) 41 | next_chars.append(text[i + maxlen]) 42 | print('nb sequences:', len(sentences)) 43 | 44 | print('Vectorization...') 45 | X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool) 46 | y = np.zeros((len(sentences), len(chars)), dtype=np.bool) 47 | for i, sentence in enumerate(sentences): 48 | for t, char in enumerate(sentence): 49 | X[i, t, char_indices[char]] = 1 50 | y[i, char_indices[next_chars[i]]] = 1 51 | 52 | 53 | # build the model: 2 stacked Stateful GRUs 54 | print('Build model...') 55 | model = Sequential() 56 | model.add(SGRU(input_shape=(None, len(chars)), batch_size=batch_size, 57 | output_dim=512, return_sequences=True)) 58 | model.add(Dropout(0.2)) 59 | model.add(SGRU(batch_size=batch_size, output_dim=512, return_sequences=False)) 60 | model.add(Dropout(0.2)) 61 | model.add(Dense(len(chars))) 62 | model.add(Activation('softmax')) 63 | 64 | # define callbacks to reset state after every 4 batches 65 | h1_reset = ResetRNNState(model.layers[0].h, lambda batches, logs: batches%4 == 0) 66 | h2_reset = ResetRNNState(model.layers[2].h, lambda batches, logs: batches%4 == 0) 67 | 68 | model.compile(loss='categorical_crossentropy', optimizer='rmsprop') 69 | 70 | # helper function to sample an index from a probability array 71 | def sample(a, temperature=1.0): 72 | a = np.log(a)/temperature 73 | a = np.exp(a)/np.sum(np.exp(a)) 74 | return np.argmax(np.random.multinomial(1,a,1)) 75 | 76 | # train the model, output generated text after each iteration 77 | for iteration in range(1, 60): 78 | print() 79 | print('-' * 50) 80 | print('Iteration', iteration) 81 | model.fit(X, y, batch_size=batch_size, nb_epoch=1, callbacks=[h1_reset, h2_reset]) 82 | 83 | start_index = random.randint(0, len(text) - maxlen - 1) 84 | 85 | for diversity in [0.2, 0.5, 1.0, 1.2]: 86 | print() 87 | print('----- diversity:', diversity) 88 | 89 | generated = '' 90 | sentence = text[start_index : start_index + maxlen] 91 | generated += sentence 92 | print('----- Generating with seed: "' + sentence + '"') 93 | sys.stdout.write(generated) 94 | 95 | for iteration in range(400): 96 | x = np.zeros((1, maxlen, len(chars))) 97 | for t, char in enumerate(sentence): 98 | x[0, t, char_indices[char]] = 1. 99 | 100 | preds = model.predict(x, verbose=0)[0] 101 | next_index = sample(preds, diversity) 102 | next_char = indices_char[next_index] 103 | 104 | generated += next_char 105 | sentence = sentence[1:] + next_char 106 | 107 | sys.stdout.write(next_char) 108 | sys.stdout.flush() 109 | print() 110 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wheel==0.23.0 2 | git+https://github.com/Theano/Theano.git@15c90dd3#egg=Theano==0.8.git 3 | keras==0.3.2 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | # from setuptools.command.test import test as TestCommand 5 | from setuptools import find_packages 6 | import codecs 7 | import re 8 | # import sys 9 | 10 | try: 11 | from setuptools import setup 12 | except ImportError: 13 | from distutils.core import setup 14 | 15 | here = os.path.abspath(os.path.dirname(__file__)) 16 | 17 | 18 | def read(*parts): 19 | # intentionally *not* adding an encoding option to open 20 | return codecs.open(os.path.join(here, *parts), 'r').read() 21 | 22 | 23 | def find_version(*file_paths): 24 | version_file = read(*file_paths) 25 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 26 | version_file, re.M) 27 | if version_match: 28 | return version_match.group(1) 29 | raise RuntimeError("Unable to find version string.") 30 | 31 | 32 | # class PyTest(TestCommand): 33 | # def finalize_options(self): 34 | # TestCommand.finalize_options(self) 35 | # self.test_args = ['--strict', '--verbose', '--tb=long', 'tests'] 36 | # self.test_suite = True 37 | # 38 | # def run_tests(self): 39 | # import pytest 40 | # errno = pytest.main(self.test_args) 41 | # sys.exit(errno) 42 | 43 | long_description = read('README.rst') 44 | 45 | 46 | with open('README.rst') as readme_file: 47 | readme = readme_file.read() 48 | 49 | with open('HISTORY.rst') as history_file: 50 | history = history_file.read().replace('.. :changelog:', '') 51 | 52 | requirements = [ 53 | # TODO: put package requirements here 54 | ] 55 | 56 | test_requirements = [ 57 | # TODO: put package test requirements here 58 | ] 59 | 60 | setup( 61 | name='seya', 62 | version='0.1.0', 63 | description="Deep Learning models and utility functions for Keras", 64 | long_description=readme + '\n\n' + history, 65 | author="Eder Santana", 66 | author_email='edercsjr@gmail.com', 67 | url='https://github.com/edersantana/seya', 68 | packages=find_packages(), 69 | # package_dir={'seya': 'seya'}, 70 | include_package_data=True, 71 | install_requires=requirements, 72 | license="MIT", 73 | zip_safe=False, 74 | keywords='seya', 75 | classifiers=[ 76 | 'Development Status :: 2 - Pre-Alpha', 77 | 'Intended Audience :: Developers, Science/Research', 78 | 'License :: The MIT License', 79 | 'Natural Language :: English', 80 | "Programming Language :: Python :: 2", 81 | 'Programming Language :: Python :: 2.6', 82 | 'Programming Language :: Python :: 2.7', 83 | 'Programming Language :: Python :: 3', 84 | 'Programming Language :: Python :: 3.3', 85 | 'Programming Language :: Python :: 3.4', 86 | ], 87 | # cmdclass={'test': PyTest}, 88 | extras_require={ 89 | 'testing': ['pytest'], 90 | }, 91 | test_suite='tests', 92 | tests_require=test_requirements 93 | ) 94 | -------------------------------------------------------------------------------- /seya/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __author__ = 'Eder Santana' 4 | __email__ = 'edercsjr@gmail.com' 5 | __version__ = '0.1.0' 6 | -------------------------------------------------------------------------------- /seya/callbacks.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import keras.backend as K 3 | from keras.callbacks import Callback 4 | 5 | 6 | class ResetRNNState(Callback): 7 | """ 8 | This is supposed to be used with stateful RNNs like 9 | seya.layers.recurrent.StatefulGRU 10 | 11 | h: the rnn state 12 | func: a function that returns true when the state should be reset to zero 13 | """ 14 | def __init__(self, h, func): 15 | self.h = h 16 | self.func = func 17 | 18 | def on_batch_end(self, batch, logs={}): 19 | if self.func(batch, logs): 20 | self.h.set_value(self.h.get_value()*0) 21 | 22 | 23 | class RenormalizeWeight(Callback): 24 | def __init__(self, W, transpose=False): 25 | Callback.__init__(self) 26 | self.W = W 27 | self.W_shape = K.get_value(self.W).shape 28 | self.transpose = transpose 29 | 30 | def on_batch_begin(self, batch, logs={}): 31 | W = self.W.get_value() 32 | if len(self.W_shape) == 4: 33 | if self.transpose: 34 | W = W.transpose(1, 0, 2, 3) 35 | this_shape = W.shape 36 | W = W.reshape((self.W_shape[0], -1)) 37 | elif self.transpose: 38 | W = W.T 39 | norm = np.sqrt((W**2).sum(axis=-1)) 40 | W /= norm[:, None] 41 | if self.transpose: 42 | if len(self.W_shape) == 4: 43 | W = W.reshape(this_shape) 44 | W = W.transpose(1, 0, 2, 3) 45 | else: 46 | W = W.T 47 | self.W.set_value(W) 48 | -------------------------------------------------------------------------------- /seya/data_utils.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import numpy as np 3 | import h5py 4 | import threading 5 | 6 | from keras.datasets import mnist 7 | from keras.utils import np_utils 8 | 9 | from collections import defaultdict 10 | from skimage.transform import rotate 11 | floatX = theano.config.floatX 12 | 13 | 14 | def load_rotated_mnist(): 15 | (_, y_train), (_, y_test) = mnist.load_data() 16 | 17 | X_train = HDF5Tensor('/home/eders/python/blog/rotated_mnist_train.hdf5', 'X_train', 0, 50000, 0, 19) 18 | X2_train = HDF5Tensor('/home/eders/python/blog/rotated_mnist_train.hdf5', 'X_train', 0, 50000, 1, 20) 19 | 20 | X_valid = HDF5Tensor('/home/eders/python/blog/rotated_mnist_train.hdf5', 'X_valid', 0, 10000-16, 0, 19) 21 | X2_valid = HDF5Tensor('/home/eders/python/blog/rotated_mnist_train.hdf5', 'X_valid', 0, 10000-16, 1, 20) 22 | 23 | X_test = HDF5Tensor('/home/eders/python/blog/rotated_mnist_train.hdf5', 'X_test', 0, 10000-16, 0, 19) 24 | X2_test = HDF5Tensor('/home/eders/python/blog/rotated_mnist_train.hdf5', 'X_test', 0, 10000-16, 1, 20) 25 | 26 | # convert class vectors to binary class matrices 27 | Y_train = np_utils.to_categorical(y_train, 10) 28 | Y_test = np_utils.to_categorical(y_test, 10) 29 | return (X_train, X2_train, Y_train[:50000], 30 | X_valid, X2_valid, Y_train[50000:], 31 | X_test, X2_test, Y_test) 32 | 33 | 34 | class TransformedDataset(): 35 | def __init__(self, data, transformer): 36 | self.data = data 37 | self.transformer = transformer 38 | samp_out = transformer.transform(data[:2]) 39 | self.shape = (data.shape[0],) + samp_out.shape[1:] 40 | 41 | def __len__(self): 42 | return self.shape[0] 43 | 44 | def __getitem__(self, key): 45 | b = self.data.__getitem__(key) 46 | b = self.transformer.transform(b) 47 | return b 48 | 49 | 50 | class DataTransformer(): 51 | def __init__(self): 52 | pass 53 | 54 | def fit(self): 55 | pass 56 | 57 | def transform(self, X): 58 | pass 59 | 60 | 61 | class RotateData(DataTransformer): 62 | '''Generate rotated images. 63 | This functions is supposed to be used with 64 | `seya.data_utils.TransformedDataset` 65 | 66 | Expected data shape batch_size x dim 67 | ''' 68 | def __init__(self, n_steps, img_shape=(28, 28), final_angle=180): 69 | self.n_steps = n_steps 70 | self.img_shape = img_shape 71 | self.final_angle = final_angle 72 | 73 | def _allrotations(self, image): 74 | angles = np.linspace(0, self.final_angle, self.n_steps) 75 | R = np.zeros((self.n_steps, np.prod(self.img_shape))) 76 | for i in xrange(self.n_steps): 77 | img = rotate(image, angles[i]) 78 | if len(self.img_shape) == 3: 79 | img = img.transpose(2, 0, 1) 80 | R[i] = img.flatten() 81 | return R 82 | 83 | def transform(self, X): 84 | Rval = np.zeros((X.shape[0], self.n_steps, X.shape[1])) 85 | for i, sample in enumerate(X): 86 | if len(self.img_shape) == 3: 87 | I = sample.reshape(self.img_shape).transpose(1, 2, 0) 88 | else: 89 | I = sample.reshape(self.img_shape) 90 | Rval[i] = self._allrotations(I) 91 | Rval = Rval.astype(floatX) 92 | return Rval 93 | 94 | 95 | class HDF5Tensor(): 96 | def __init__(self, datapath, dataset, start, end, 97 | time_start, time_end, time_step=1, normalizer=None): 98 | self.refs = defaultdict(int) 99 | if datapath not in list(self.refs.keys()): 100 | f = h5py.File(datapath) 101 | self.refs[datapath] = f 102 | else: 103 | f = self.refs[datapath] 104 | self.start = start 105 | self.end = end 106 | self.data = f[dataset] 107 | self.normalizer = normalizer 108 | self.time_start = time_start 109 | self.time_end = time_end 110 | self.time_step = time_step 111 | 112 | def __len__(self): 113 | return self.end - self.start 114 | 115 | def __getitem__(self, key): 116 | if isinstance(key, slice): 117 | if key.stop + self.start <= self.end: 118 | idx = slice(key.start+self.start, key.stop + self.start) 119 | else: 120 | raise IndexError 121 | elif isinstance(key, int): 122 | if key + self.start < self.end: 123 | idx = key+self.start 124 | else: 125 | raise IndexError 126 | elif isinstance(key, np.ndarray): 127 | if np.max(key) + self.start < self.end: 128 | idx = (self.start + key).tolist() 129 | else: 130 | raise IndexError 131 | elif isinstance(key, list): 132 | if max(key) + self.start < self.end: 133 | idx = [x + self.start for x in key] 134 | else: 135 | raise IndexError 136 | if self.normalizer is not None: 137 | return self.normalizer( 138 | self.data[idx, self.time_start:self.time_end:self.time_step]) 139 | else: 140 | return self.data[idx, self.time_start:self.time_end:self.time_step] 141 | 142 | @property 143 | def shape(self): 144 | return tuple((self.end - self.start, 145 | (self.time_end - self.time_start)/self.time_step) + 146 | self.data.shape[2:]) 147 | 148 | @property 149 | def ndim(self): 150 | return len(self.data.shape) 151 | 152 | 153 | class IndexedGenerator(object): 154 | """IndexedGenerator 155 | Handles datasets with arbitrary dataset indeces 156 | 157 | Usage: 158 | ------ 159 | Assume that you have a list of valid indices ids = [0, 10, 11] 160 | Create `datagen = IndexedH5(ids)` 161 | Define data `X = h5py.File("mydataset.h5")['dataset'] # works even with HDF5` 162 | Fit model `model.fit_generator(datagen.flow(X, Labels, batch_size=32), 163 | samples_per_epoch=len(ids), 164 | nb_epoch=1, show_accuracy=True, 165 | nb_worker=8)) 166 | 167 | Parameters: 168 | ----------- 169 | indices: array of int, numpy array of dataset indices 170 | 171 | """ 172 | def __init__(self, indices=None, callback=None): 173 | if indices is not None: 174 | self.indices = indices 175 | else: 176 | self.indices = range(len(indices)) 177 | # self.data = f[dataset] 178 | self.callback = callback 179 | self.lock = threading.Lock() 180 | 181 | def _flow_index(self, N, batch_size=32, shuffle=True, seed=None): 182 | b = 0 183 | total_b = 0 184 | while 1: 185 | if b == 0: 186 | if seed is not None: 187 | np.random.seed(seed + total_b) 188 | 189 | if shuffle: 190 | index_array = np.random.shuffle(self.indices) 191 | else: 192 | index_array = np.arange(len(self.indices)) 193 | 194 | current_index = (b * batch_size) % N 195 | if N >= current_index + batch_size: 196 | current_batch_size = batch_size 197 | else: 198 | current_batch_size = N - current_index 199 | 200 | if current_batch_size == batch_size: 201 | b += 1 202 | else: 203 | b = 0 204 | total_b += 1 205 | yield (index_array[current_index: current_index + current_batch_size], 206 | current_index, current_batch_size) 207 | 208 | def flow(self, X, y, batch_size=32, shuffle=False, seed=None): 209 | assert len(X) == len(y) 210 | self.X = X 211 | self.y = y 212 | self.flow_generator = self._flow_index(self.indices.shape[0], batch_size, 213 | shuffle, seed) 214 | return self 215 | 216 | def __iter__(self): 217 | # needed if we want to do something like: 218 | # for x, y in data_gen.flow(...): 219 | return self 220 | 221 | def next(self): 222 | # for python 2.x. 223 | # Keeps under lock only the mechanism which advances 224 | # the indexing of each batch 225 | # see # http://anandology.com/blog/using-iterators-and-generators/ 226 | with self.lock: 227 | index_array, current_index, current_batch_size = next(self.flow_generator) 228 | idx = sorted(self.indices[index_array.tolist()].tolist()) 229 | # import pdb; pdb.set_trace() 230 | bX = self.X[idx] 231 | if self.callback is not None: 232 | bX = self.callback(bX) 233 | bY = self.y[idx] 234 | return bX, bY 235 | 236 | def __next__(self): 237 | # for python 3.x. 238 | return self.next() 239 | -------------------------------------------------------------------------------- /seya/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EderSantana/seya/a48bfc3af54dace95a399ace3a6266afe64b3280/seya/layers/__init__.py -------------------------------------------------------------------------------- /seya/layers/base.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | import theano.tensor as T 4 | 5 | from keras.layers.core import MaskedLayer, Layer 6 | from keras import backend as K 7 | from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams 8 | 9 | floatX = theano.config.floatX 10 | 11 | 12 | class WinnerTakeAll2D(Layer): 13 | def __init__(self, **kwargs): 14 | super(WinnerTakeAll2D, self).__init__(**kwargs) 15 | 16 | def get_output(self, train=False): 17 | X = self.get_input(train) 18 | if train: 19 | M = K.max(X, axis=(2, 3), keepdims=True) 20 | R = K.switch(K.equal(X, M), X, 0.) 21 | return R 22 | else: 23 | return X 24 | 25 | 26 | class Lambda(MaskedLayer): 27 | def __init__(self, func, output_shape, ndim=2, **kwargs): 28 | super(Lambda, self).__init__(**kwargs) 29 | self.input = K.placeholder(ndim=ndim) 30 | self.func = func 31 | self._output_shape = output_shape 32 | 33 | def get_output(self, train=False): 34 | X = self.get_input(train) 35 | return self.func(X) 36 | 37 | @property 38 | def output_shape(self): 39 | return self._output_shape 40 | 41 | 42 | class Pass(MaskedLayer): 43 | ''' Do literally nothing 44 | It can the first layer 45 | ''' 46 | def __init__(self, ndim=2, **kwargs): 47 | super(Pass, self).__init__(**kwargs) 48 | self.input = K.placeholder(ndim=ndim) 49 | 50 | def get_output(self, train=False): 51 | X = self.get_input(train) 52 | return X 53 | 54 | 55 | class GaussianProd(MaskedLayer): 56 | ''' 57 | Multiply by Gaussian noise. 58 | Similar to dropout but with gaussians instead of binomials. 59 | The way they have this at Keras is not the way we need for 60 | Variational AutoEncoders. 61 | ''' 62 | def __init__(self, avg=0., std=1., **kwargs): 63 | super(GaussianProd, self).__init__(**kwargs) 64 | self.std = std 65 | self.avg = avg 66 | self.srng = RandomStreams(seed=np.random.randint(10e6)) 67 | 68 | def get_output(self, train=False): 69 | X = self.get_input(train) 70 | X *= self.srng.normal(size=X.shape, 71 | avg=self.avg, 72 | std=self.std, 73 | dtype=floatX) 74 | return X 75 | 76 | def get_config(self): 77 | return {"name": self.__class__.__name__, 78 | "avg": self.avg, 79 | "std": self.std} 80 | 81 | 82 | class Replicator(MaskedLayer): 83 | ''' 84 | WARN: use `keras.layer.RepeatVector` instead. 85 | 86 | Replicates an input matrix across a new second dimension. 87 | Originally useful for broadcasting a fixed input into a scan loop. 88 | Think conditional RNNs without the need to rewrite the RNN class. 89 | ''' 90 | def __init__(self, leng): 91 | super(Replicator, self).__init__() 92 | raise ValueError("Deprecated. Use `keras.layers.RepeatVector instead`") 93 | self.ones = T.ones((leng,)) 94 | self.input = T.matrix() 95 | 96 | def get_output(self, train=False): 97 | X = self.get_input(train) 98 | output = X[:, None, :] * self.ones[None, :, None] 99 | return output 100 | 101 | 102 | class Unpool(Layer): 103 | '''Unpooling layer for convolutional autoencoders 104 | inspired by: https://github.com/mikesj-public/convolutional_autoencoder/blob/master/mnist_conv_autoencode.py 105 | 106 | Parameter: 107 | ---------- 108 | ds: list with two values each one defines how much that dimension will 109 | be upsampled. 110 | ''' 111 | def __init__(self, ds): 112 | raise ValueError("Deprecated. Use `keras.layers.convolutional.UpSample instead`") 113 | super(Unpool, self).__init__() 114 | self.input = T.tensor4() 115 | self.ds = ds 116 | 117 | def get_output(self, train=False): 118 | X = self.get_input(train) 119 | output = X.repeat(self.ds[0], axis=2).repeat(self.ds[1], axis=3) 120 | return output 121 | 122 | 123 | class TimePicker(MaskedLayer): 124 | def __init__(self, time=-1): 125 | '''Picks a single value in time from a recurrent layer 126 | without forgeting its input mask''' 127 | super(TimePicker, self).__init__() 128 | self.time = time 129 | self.input = T.tensor3() 130 | 131 | def get_output(self, train=False): 132 | X = self.get_input(train) 133 | return X[:, self.time, :] 134 | 135 | @property 136 | def output_shape(self): 137 | return self.input_shape[0, 2] 138 | -------------------------------------------------------------------------------- /seya/layers/convolutional.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.layers.core import Layer, MaskedLayer 3 | from theano import tensor as T # , scan 4 | 5 | 6 | class GlobalPooling2D(Layer): 7 | """ 8 | By @entron 9 | Borrowed and modified from here: https://github.com/fchollet/keras/pull/522 10 | """ 11 | def __init__(self, pooling_function='average'): 12 | super(GlobalPooling2D, self).__init__() 13 | if pooling_function not in {'average', 'max'}: 14 | raise Exception('Invalid pooling function for GlobalPooling2D:', pooling_function) 15 | if pooling_function == 'average': 16 | self.pooling_function = K.mean 17 | else: 18 | self.pooling_function = K.max 19 | self.input = K.placeholder(ndim=4) 20 | 21 | def get_output(self, train): 22 | X = self.get_input(train) 23 | return self.pooling_function(self.pooling_function(X, axis=-1), axis=-1) 24 | 25 | @property 26 | def output_shape(self): 27 | return self.input_shape[:2] 28 | 29 | def get_config(self): 30 | return {"name": self.__class__.__name__, 31 | "pooling_function": self.pooling_function.__name__} 32 | 33 | 34 | class ChannelDropout(MaskedLayer): 35 | def __init__(self, p, **kwargs): 36 | super(ChannelDropout, self).__init__(**kwargs) 37 | self.p = p 38 | 39 | def channel_dropout(self, X): 40 | # TODO write channel_dropout at Keras.backend 41 | from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams 42 | rng = RandomStreams() 43 | r = rng.binomial(X.shape[1:2], p=1-self.p, dtype=X.dtype) 44 | r = r.dimshuffle('x', 0, 'x', 'x').repeat(X.shape[0], 45 | axis=0).repeat(X.shape[2], 46 | axis=2).repeat(X.shape[3], 47 | axis=3) 48 | X = X * r 49 | return X 50 | 51 | def get_output(self, train=False): 52 | X = self.get_input(train) 53 | if self.p > 0.: 54 | if train: 55 | X = self.channel_dropout(X) 56 | return X 57 | 58 | def get_config(self): 59 | config = {'name': self.__class__.__name__, 60 | 'p': self.p} 61 | base_config = super(ChannelDropout, self).get_config() 62 | return dict(list(base_config.items()) + list(config.items())) 63 | 64 | 65 | class WinnerTakeAll2D(Layer): 66 | """Spatial Winner-Take-All 67 | ref: Winner-Take-All Autoencoders by Alireza Makhzani, Brendan Frey 68 | 69 | Parameters: 70 | ----------- 71 | spatial: int, controls spatial sparsity, defines maximum number of non zeros 72 | in a spatial map 73 | lifetime, int, controls lifetime sparsity, defines maximum number of non 74 | zeros in a dimension throughout the batches 75 | n_largest: int, global sparsity, defines maximum number of non zeros in the 76 | output tensor. 77 | previous_mode: bool, flag to use legacy behavior of this layer 78 | 79 | NOTE: 80 | ===== 81 | This is a Theano only layer 82 | 83 | """ 84 | def __init__(self, n_largest=None, 85 | spatial=5, lifetime=5, previous_mode=True, **kwargs): 86 | if K._BACKEND == "tensorflow" and not previous_mode: 87 | raise ValueError("This is a Theano-only layer") 88 | super(WinnerTakeAll2D, self).__init__(**kwargs) 89 | self.n_largest = n_largest 90 | self.spatial = spatial 91 | self.lifetime = lifetime 92 | self.previous_mode = previous_mode 93 | 94 | def wta_largest(self, c, n=1): 95 | s = T.sort(c.flatten()) 96 | nl = s[-n] 97 | c = T.switch(T.ge(c, nl), c, 0.) 98 | return c 99 | 100 | def wta_lifetime(self, c, n=1): 101 | s = T.sort(c, axis=0)[-n].dimshuffle('x', 0, 1, 2) 102 | r = K.switch(T.ge(c, s), c, 0.) 103 | return r 104 | 105 | def wta_spatial(self, c, n=1): 106 | c = c.reshape((c.shape[0], c.shape[1], -1)) 107 | s = T.sort(c, axis=2)[:, :, -n].dimshuffle(0, 1, 'x', 'x') 108 | r = K.switch(T.ge(c, s), c, 0.) 109 | return r 110 | 111 | def winner_take_all(self, X): 112 | M = K.max(X, axis=(2, 3), keepdims=True) 113 | R = K.switch(K.equal(X, M), X, 0.) 114 | return R 115 | 116 | def get_output(self, train=True): 117 | X = self.get_input(train) 118 | if train is False: 119 | return X 120 | elif self.previous_mode: 121 | return self.winner_take_all(X) 122 | else: 123 | Y = X 124 | if self.n_largest: 125 | Y = self.wta_largest(Y, self.n_largest) 126 | if self.wta_spatial: 127 | Y = self.wta_lifetime(Y, self.spatial) 128 | if self.wta_lifetime: 129 | Y = self.wta_spatial(Y, self.lifetime) 130 | return Y 131 | 132 | def get_config(self): 133 | config = {'name': self.__class__.__name__, 134 | 'n_largest': self.n_largest, 135 | 'spatial': self.spatial, 136 | 'lifetime': self.lifetime, 137 | 'previous_mode': self.previous_mode} 138 | base_config = super(WinnerTakeAll2D, self).get_config() 139 | return dict(list(base_config.items()) + list(config.items())) 140 | -------------------------------------------------------------------------------- /seya/layers/deep_recurrent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras.layers.core import Dense 4 | from keras.layers.recurrent import Recurrent, LSTM 5 | from keras import initializations, activations 6 | from keras import backend as K 7 | 8 | # from theano.printing import Print 9 | 10 | 11 | class DeepLSTM(Recurrent): 12 | '''Seq2Seq Deep Long-Short Term Memory unit. 13 | Inspired byt Sutskever et. al 2014 14 | 15 | This layer outputs ALL the states and cells like [h_0, c_0, ..., h_deeper, c_deeper]. 16 | If you need only the very last states, use a Lambada layer to narrow 17 | output[:, -2*output_dim:-output_dim] 18 | 19 | Args: similar to regular LSTM 20 | depth: number of LSTMs to stack 21 | readout: int, if we should a final Dense layer on top or not. readout is 22 | this Dense's output_dim 23 | ''' 24 | def __init__(self, output_dim, depth=1, readout=False, dropout=.5, 25 | init='glorot_uniform', inner_init='orthogonal', 26 | forget_bias_init='one', activation='tanh', 27 | inner_activation='hard_sigmoid', **kwargs): 28 | self.output_dim = output_dim 29 | self.depth = depth 30 | self.readout = readout 31 | self.dropout = dropout 32 | self.init = initializations.get(init) 33 | self.inner_init = initializations.get(inner_init) 34 | self.forget_bias_init = initializations.get(forget_bias_init) 35 | self.activation = activations.get(activation) 36 | self.inner_activation = activations.get(inner_activation) 37 | self._kwargs = kwargs 38 | super(DeepLSTM, self).__init__(**kwargs) 39 | 40 | def build(self): 41 | self.lstms = [] 42 | for i in range(self.depth): 43 | if i == 0: 44 | self.lstms.append(LSTM(self.output_dim, self.init, self.inner_init, 45 | self.forget_bias_init, self.activation, 46 | self.inner_activation, **self._kwargs)) 47 | else: 48 | self._kwargs['input_dim'] = self.output_dim 49 | self.lstms.append(LSTM(self.output_dim, self.init, self.inner_init, 50 | self.forget_bias_init, self.activation, 51 | self.inner_activation, **self._kwargs)) 52 | 53 | [lstm.build() for lstm in self.lstms] 54 | 55 | # Get a flat list of trainable_weights 56 | self.trainable_weights = [weights for lstm in self.lstms for weights in 57 | lstm.trainable_weights] 58 | 59 | if self.readout: 60 | self.readout_layer = Dense(self.readout, input_dim=self.output_dim, 61 | activation='softmax') 62 | self.readout_layer.build() 63 | self.trainable_weights.extend(self.readout_layer.trainable_weights) 64 | 65 | if self.initial_weights is not None: 66 | self.set_weights(self.initial_weights) 67 | del self.initial_weights 68 | 69 | if self.stateful is not None: 70 | self.states = [] 71 | for lstm in self.lstms: 72 | self.states.extend(lstm.states) 73 | else: 74 | self.states = [None, None] * self.depth 75 | 76 | def reset_states(self): 77 | [lstm.reset_states() for lstm in self.lstms] 78 | 79 | def get_initial_states(self, X): 80 | states = super(DeepLSTM, self).get_initial_states(X) 81 | if self.readout: 82 | initial_state = K.zeros_like(X) # (samples, timesteps, input_dim) 83 | initial_state = K.sum(initial_state, axis=1) # (samples, input_dim) 84 | reducer = K.zeros((self.input_dim, self.readout)) 85 | initial_state = K.dot(initial_state, reducer) # (samples, output_dim) 86 | states += [initial_state] 87 | return states 88 | 89 | def step(self, x, states): 90 | if self.readout: 91 | assert len(states) == 2*self.depth+1 92 | states = states[:-1] 93 | else: 94 | assert len(states) == 2*self.depth 95 | 96 | h = [] 97 | # P = Print('[debug] X value: ', attrs=("shape",)) 98 | for i, (h_tm1, c_tm1) in enumerate(zip(states[:-1:2], states[1::2])): 99 | # x = P(x) 100 | x, new_states = self.lstms[i].step(x, [h_tm1, c_tm1]) 101 | h.extend(new_states) 102 | # x = K.dropout(x, self.dropout) # no dropout on the first layer inputs 103 | 104 | if self.readout: 105 | h += [self.readout_layer(h[-2])] 106 | 107 | return K.concatenate(h, axis=-1), h 108 | 109 | def dream(self, length=140): 110 | def _dream_step(x, states): 111 | # input + states 112 | assert len(states) == 2*self.depth + 1 113 | x = states[-1] 114 | x = K.switch(K.equal(x, K.max(x, axis=-1, 115 | keepdims=True)), 1., 0.) 116 | states = states[:-1] 117 | 118 | h = [] 119 | for i, (h_tm1, c_tm1) in enumerate(zip(states[:-1:2], states[1::2])): 120 | x, new_states = self.lstms[i].step(x, [h_tm1, c_tm1]) 121 | h.extend(new_states) 122 | 123 | if self.readout: 124 | h += [self.readout_layer(h[-2])] 125 | final = h[-1] 126 | else: 127 | h += [h[-2]] 128 | final = h[-2] 129 | 130 | return final, h 131 | 132 | # input shape: (nb_samples, time (padded with zeros), input_dim) 133 | # Only the very first time point of the input is used, the others only 134 | # server to count the lenght of the output sequence 135 | X = self.get_input(train=False) 136 | mask = self.get_input_mask(train=False) 137 | 138 | assert K.ndim(X) == 3 139 | if K._BACKEND == 'tensorflow': 140 | if not self.input_shape[1]: 141 | raise Exception('When using TensorFlow, you should define ' + 142 | 'explicitly the number of timesteps of ' + 143 | 'your sequences.\n' + 144 | 'If your first layer is an Embedding, ' + 145 | 'make sure to pass it an "input_length" ' + 146 | 'argument. Otherwise, make sure ' + 147 | 'the first layer has ' + 148 | 'an "input_shape" or "batch_input_shape" ' + 149 | 'argument, including the time axis.') 150 | # if self.stateful: 151 | # initial_states = self.states 152 | # else: 153 | # initial_states = self.get_initial_states(X) 154 | 155 | s = self.get_output(train=False)[:, -1] 156 | idx = [0, ] + list(np.cumsum([self.output_dim]*2*self.depth + 157 | [self.readout, ])) 158 | initial_states = [s[:, idx[i]:idx[i+1]] for i in range(len(idx)-1)] 159 | 160 | # if self.readout: 161 | # initial_states.pop(-1) 162 | # initial_states.append(X[:, 0]) 163 | 164 | last_output, outputs, states = K.rnn(_dream_step, K.zeros((1, length, 1)), 165 | initial_states, 166 | go_backwards=self.go_backwards, 167 | mask=mask) 168 | if self.stateful: 169 | self.updates = [] 170 | for i in range(len(states)): 171 | self.updates.append((self.states[i], states[i])) 172 | 173 | return outputs 174 | 175 | def get_config(self): 176 | config = {"output_dim": self.output_dim, 177 | "depth": self.depth, 178 | "readout": self.readout, 179 | "dropout": self.dropout, 180 | "init": self.init.__name__, 181 | "inner_init": self.inner_init.__name__, 182 | "forget_bias_init": self.forget_bias_init.__name__, 183 | "activation": self.activation.__name__, 184 | "inner_activation": self.inner_activation.__name__} 185 | base_config = super(LSTM, self).get_config() 186 | return dict(list(base_config.items()) + list(config.items())) 187 | 188 | @property 189 | def output_shape(self): 190 | input_shape = self.input_shape 191 | if self.readout: 192 | return input_shape[:2] + [self.output_dim*2*self.depth + 193 | self.readout] 194 | else: 195 | return input_shape[:2] + tuple([self.output_dim*2*self.depth]) 196 | -------------------------------------------------------------------------------- /seya/layers/embedding.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from keras import backend as K 4 | from keras import initializations, regularizers, constraints 5 | from keras.layers.core import Layer 6 | 7 | 8 | class BagEmbedding(Layer): 9 | '''Apply keras.layers.embedding.Embeddinig to a sequence of phrases and 10 | compresses each phrase into a vector. Output is thus a bag-of-words per 11 | sequence. 12 | This layer can only be used as the first layer in a model. 13 | # Input shape 14 | 3D tensor with shape: `(nb_samples, nb_sequences, sequence_length)`. 15 | # Output shape 16 | 3D tensor with shape: `(nb_samples, nb_sequences, output_dim)`. 17 | # Arguments 18 | input_dim: int >= 0. Size of the vocabulary, ie. 19 | 1 + maximum integer index occurring in the input data. 20 | nb_words: Maximum number of words per input sequence 21 | output_dim: int >= 0. Dimension of the dense embedding. 22 | input_length: Number of sequences in the input. 23 | This argument is required if you are going to connect 24 | `Flatten` then `Dense` layers upstream 25 | (without it, the shape of the dense outputs cannot be computed). 26 | bow_mode: How to reduce sequence into a `bag of words`. 27 | modes are "bow" for conventional sum reduce... TODO 28 | init: name of initialization function for the weights 29 | of the layer (see: [initializations](../initializations.md)), 30 | or alternatively, Theano function to use for weights initialization. 31 | This parameter is only relevant if you don't pass a `weights` 32 | argument. 33 | weights: list of numpy arrays to set as initial weights. 34 | The list should have 1 element, of shape `(input_dim, output_dim)`. 35 | W_regularizer: instance of the [regularizers](../regularizers.md) module 36 | (eg. L1 or L2 regularization), applied to the embedding matrix. 37 | W_constraint: instance of the [constraints](../constraints.md) module 38 | (eg. maxnorm, nonneg), applied to the embedding matrix. 39 | mask_zero: Whether or not the input value 0 is a special "padding" 40 | value that should be masked out. 41 | This is useful for [recurrent layers](recurrent.md) which may take 42 | variable length input. If this is `True` then all subsequent layers 43 | in the model need to support masking or an exception will be raised. 44 | dropout: float between 0 and 1. Fraction of the embeddings to drop. 45 | # References 46 | - [End-To-End Memory Networks](http://arxiv.org/pdf/1503.08895v5.pdf) 47 | ''' 48 | input_ndim = 3 49 | 50 | def __init__(self, input_dim, nb_words, output_dim, input_length, 51 | bow_mode="bow", init='uniform', 52 | W_regularizer=None, activity_regularizer=None, 53 | W_constraint=None, 54 | mask_zero=False, 55 | weights=None, dropout=0., **kwargs): 56 | self.input_dim = input_dim 57 | self.output_dim = output_dim 58 | self.bow_mode = bow_mode 59 | self.init = initializations.get(init) 60 | self.input_length = input_length 61 | self.nb_words = nb_words 62 | self.mask_zero = mask_zero 63 | self.dropout = dropout 64 | 65 | self.W_constraint = constraints.get(W_constraint) 66 | self.constraints = [self.W_constraint] 67 | 68 | self.W_regularizer = regularizers.get(W_regularizer) 69 | self.activity_regularizer = regularizers.get(activity_regularizer) 70 | 71 | self.initial_weights = weights 72 | kwargs['input_shape'] = (self.input_dim, self.nb_words) 73 | super(BagEmbedding, self).__init__(**kwargs) 74 | 75 | def build(self): 76 | self.input = K.placeholder(shape=( 77 | self.input_shape[0], self.input_length, self.nb_words), 78 | dtype='int32') 79 | self.W = self.init((self.input_dim, self.output_dim), 80 | name='{}_W'.format(self.name)) 81 | # set mask weights to a vector of zeros, doesn't really matter due to 82 | # masking, but checking that they don't change is useful for debugging. 83 | W = K.get_value(self.W) 84 | W[0] = 0. * W[0] 85 | K.set_value(self.W, W) 86 | 87 | self.trainable_weights = [self.W] 88 | self.regularizers = [] 89 | if self.W_regularizer: 90 | self.W_regularizer.set_param(self.W) 91 | self.regularizers.append(self.W_regularizer) 92 | 93 | if self.activity_regularizer: 94 | self.activity_regularizer.set_layer(self) 95 | self.regularizers.append(self.activity_regularizer) 96 | 97 | if self.initial_weights is not None: 98 | self.set_weights(self.initial_weights) 99 | 100 | def get_output_mask(self, train=None): 101 | X = self.get_input(train) 102 | if not self.mask_zero: 103 | return None 104 | else: 105 | return K.not_equal(X, 0) 106 | 107 | @property 108 | def output_shape(self): 109 | return (self.input_shape[0], self.input_length, self.output_dim) 110 | 111 | def get_output(self, train=False): 112 | X = self.get_input(train) 113 | retain_p = 1. - self.dropout 114 | if train and self.dropout > 0: 115 | B = K.random_binomial((self.input_dim,), p=retain_p) 116 | else: 117 | B = K.ones((self.input_dim)) * retain_p 118 | # we zero-out rows of W at random 119 | Xs = K.cast(K.reshape(X, (-1, self.nb_words)), 'int32') 120 | 121 | # (samples*input_length, nb_words, dim) 122 | out = K.gather(self.W * K.expand_dims(B), Xs) 123 | out = K.reshape(out, (-1, self.input_length, self.nb_words, 124 | self.output_dim)) 125 | # (samples, input_length, nb_words, dim) 126 | out = out * K.expand_dims(K.not_equal(X, 0), dim=-1) 127 | if self.bow_mode == "bow": 128 | out = K.sum(out, axis=2) 129 | return out 130 | 131 | def get_config(self): 132 | config = {"name": self.__class__.__name__, 133 | "input_dim": self.input_dim, 134 | "nb_words": self.nb_words, 135 | "bow_mode": self.bow_mode, 136 | "output_dim": self.output_dim, 137 | "init": self.init.__name__, 138 | "input_length": self.input_length, 139 | "mask_zero": self.mask_zero, 140 | "activity_regularizer": self.activity_regularizer.get_config() if self.activity_regularizer else None, 141 | "W_regularizer": self.W_regularizer.get_config() if self.W_regularizer else None, 142 | "W_constraint": self.W_constraint.get_config() if self.W_constraint else None, 143 | "dropout": self.dropout} 144 | base_config = super(BagEmbedding, self).get_config() 145 | return dict(list(base_config.items()) + list(config.items())) 146 | -------------------------------------------------------------------------------- /seya/layers/imageproc.py: -------------------------------------------------------------------------------- 1 | """Note: this code was modified from: 2 | 3 | https://github.com/lpigou/Theano-3D-ConvNet/blob/master/LICENSE 4 | by @lpigou and collaborators 5 | """ 6 | import numpy as np 7 | import theano.tensor as T 8 | import keras.backend as K 9 | from keras.layers.core import Layer 10 | 11 | 12 | class NormLayer(Layer): 13 | """ Normalization layer """ 14 | 15 | def __init__(self, method="lcn", kernel_size=9, threshold=1e-4, 16 | nb_channels=3, 17 | use_divisor=True, **kwargs): 18 | """ 19 | method: "lcn", "gcn", "mean" 20 | LCN: local contrast normalization 21 | kwargs: 22 | kernel_size=9, threshold=1e-4, use_divisor=True 23 | GCN: global contrast normalization 24 | kwargs: 25 | scale=1., subtract_mean=True, use_std=False, sqrt_bias=0., 26 | min_divisor=1e-8 27 | MEAN: local mean subtraction 28 | kwargs: 29 | kernel_size=5 30 | """ 31 | 32 | super(NormLayer, self).__init__(**kwargs) 33 | self.method = method 34 | self.kernel_size = kernel_size 35 | self.threshold = threshold 36 | self.use_divisor = use_divisor 37 | self.nb_channels = nb_channels 38 | self.input = K.placeholder(ndim=4) 39 | 40 | def get_output(self, train=False): 41 | X = self.get_input() 42 | out = [] 43 | if self.method == "lcn": 44 | for i in range(self.nb_channels): 45 | XX = X[:, i:i+1, :, :] 46 | out += [self.lecun_lcn(XX, self.kernel_size, self.threshold, 47 | self.use_divisor)] 48 | out = K.concatenate(out, axis=1) 49 | elif self.method == "gcn": 50 | out = self.global_contrast_normalize(X) 51 | elif self.method == "mean": 52 | out = self.local_mean_subtraction(X, self.kernel_size) 53 | else: 54 | raise NotImplementedError() 55 | return out 56 | 57 | def lecun_lcn(self, X, kernel_size=7, threshold=1e-4, use_divisor=True): 58 | """ 59 | Yann LeCun's local contrast normalization 60 | Orginal code in Theano by: Guillaume Desjardins 61 | """ 62 | 63 | filter_shape = (1, 1, kernel_size, kernel_size) 64 | filters = self.gaussian_filter( 65 | kernel_size).reshape(filter_shape) 66 | # filters = shared(_asarray(filters, dtype=floatX), borrow=True) 67 | filters = K.variable(filters) 68 | 69 | convout = K.conv2d(X, filters, filter_shape=filter_shape, 70 | border_mode='same') 71 | 72 | # For each pixel, remove mean of kernel_sizexkernel_size neighborhood 73 | new_X = X - convout 74 | 75 | if use_divisor: 76 | # Scale down norm of kernel_sizexkernel_size patch 77 | sum_sqr_XX = K.conv2d(K.pow(K.abs(new_X), 2), filters, 78 | filter_shape=filter_shape, border_mode='same') 79 | 80 | denom = T.sqrt(sum_sqr_XX) 81 | per_img_mean = denom.mean(axis=[2, 3]) 82 | divisor = T.largest(per_img_mean.dimshuffle(0, 1, 'x', 'x'), denom) 83 | divisor = T.maximum(divisor, threshold) 84 | 85 | new_X /= divisor 86 | 87 | return new_X 88 | 89 | def local_mean_subtraction(self, X, kernel_size=5): 90 | 91 | filter_shape = (1, 1, kernel_size, kernel_size) 92 | filters = self.mean_filter(kernel_size).reshape(filter_shape) 93 | filters = K.variable(filters) 94 | 95 | mean = K.conv2d(X, filters, filter_shape=filter_shape, 96 | border_mode='same') 97 | return X - mean 98 | 99 | def global_contrast_normalize(self, X, scale=1., subtract_mean=True, 100 | use_std=False, sqrt_bias=0., min_divisor=1e-6): 101 | 102 | ndim = X.ndim 103 | if ndim not in [3, 4]: 104 | raise NotImplementedError("X.dim>4 or X.ndim<3") 105 | 106 | scale = float(scale) 107 | mean = X.mean(axis=ndim-1) 108 | new_X = X.copy() 109 | 110 | if subtract_mean: 111 | if ndim == 3: 112 | new_X = X - mean[:, :, None] 113 | else: 114 | new_X = X - mean[:, :, :, None] 115 | 116 | if use_std: 117 | normalizers = T.sqrt(sqrt_bias + X.var(axis=ndim-1)) / scale 118 | else: 119 | normalizers = T.sqrt(sqrt_bias + (new_X ** 2).sum(axis=ndim-1)) / scale 120 | 121 | # Don't normalize by anything too small. 122 | T.set_subtensor(normalizers[(normalizers < min_divisor).nonzero()], 1.) 123 | 124 | if ndim == 3: 125 | new_X /= (normalizers[:, :, None] + 1e-6) 126 | else: 127 | new_X /= (normalizers[:, :, :, None] + 1e-6) 128 | 129 | return new_X 130 | 131 | def gaussian_filter(self, kernel_shape): 132 | 133 | x = np.zeros((kernel_shape, kernel_shape), dtype='float32') 134 | 135 | def gauss(x, y, sigma=2.0): 136 | Z = 2 * np.pi * sigma**2 137 | return 1./Z * np.exp(-(x**2 + y**2) / (2. * sigma**2)) 138 | mid = np.floor(kernel_shape / 2.) 139 | for i in xrange(0, kernel_shape): 140 | for j in xrange(0, kernel_shape): 141 | x[i, j] = gauss(i-mid, j-mid) 142 | return x / sum(x) 143 | 144 | def mean_filter(self, kernel_size): 145 | s = kernel_size**2 146 | x = np.repeat(1./s, s).reshape((kernel_size, kernel_size)) 147 | return x 148 | -------------------------------------------------------------------------------- /seya/layers/memnn.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.layers.core import Layer, Dense 3 | from keras import activations 4 | 5 | from seya.layers.embedding import BagEmbedding 6 | 7 | import theano.tensor as T 8 | 9 | 10 | class MemNN(Layer): 11 | """End-To-End Memory Networks 12 | 13 | # Parameters 14 | hops : int >= 1 15 | mode : str in {'untied', 'adjacent', 'rnn'} 16 | 17 | # Reference 18 | - [End-To-End Memory Networks](http://arxiv.org/abs/1503.08895) 19 | """ 20 | def __init__(self, output_dim, input_dim, input_length, f_nb_words, 21 | q_nb_words, 22 | hops=1, mode="untied", bow_mode="bow", dropout=0., 23 | mask_zero=False, 24 | inner_activation='relu', activation='softmax', **kwargs): 25 | 26 | if mode not in {'untied', 'adjacent', 'rnn'}: 27 | # TODO implement adjacent and rnn 28 | raise Exception('Invalid merge mode: ' + str(mode)) 29 | assert mode == 'untied' 30 | 31 | self.input_dim = input_dim 32 | self.input_length = input_length 33 | self.f_nb_words = f_nb_words 34 | self.q_nb_words = q_nb_words 35 | self.dropout = dropout 36 | self.hops = hops 37 | self.mode = mode 38 | self.bow_mode = bow_mode 39 | self.mask_zero = mask_zero 40 | self.output_dim = output_dim 41 | self.inner_activation = activations.get(inner_activation) 42 | self.activation = activations.get(activation) 43 | super(MemNN, self).__init__(**kwargs) 44 | 45 | def build(self): 46 | # list of embedding layers 47 | self.question = [] 48 | self.facts = [] 49 | self.memory = [] 50 | self.Ws = [] 51 | self.trainable_weights = [] 52 | for i in range(self.hops): 53 | q = BagEmbedding(self.input_dim, self.q_nb_words, self.output_dim, 54 | 1, bow_mode=self.bow_mode, 55 | mask_zero=self.mask_zero, dropout=self.dropout) 56 | q.build() 57 | f = BagEmbedding(self.input_dim, self.f_nb_words, self.output_dim, 58 | self.input_length, bow_mode=self.bow_mode, 59 | mask_zero=self.mask_zero, dropout=self.dropout) 60 | f.build() 61 | m = BagEmbedding(self.input_dim, self.f_nb_words, self.output_dim, 62 | self.input_length, bow_mode=self.bow_mode, 63 | mask_zero=self.mask_zero, dropout=self.dropout) 64 | m.build() 65 | self.question.append(q) 66 | self.facts.append(f) 67 | self.memory.append(m) 68 | if i == self.hops-1: 69 | w = Dense(self.output_dim, input_dim=self.output_dim, 70 | activation=self.activation) 71 | else: 72 | w = Dense(self.output_dim, input_dim=self.output_dim, 73 | activation=self.inner_activation) 74 | w.build() 75 | self.Ws.append(w) 76 | for l in (q, f, m, w): 77 | self.trainable_weights += l.trainable_weights 78 | 79 | def get_input(self, train=False): 80 | res = [] 81 | # question 82 | q = K.placeholder(shape=( 83 | self.input_shape[1][0], 1, self.q_nb_words)) 84 | res.append(q) 85 | # facts 86 | f = K.placeholder(shape=( 87 | self.input_shape[0][0], self.input_length, self.f_nb_words)) 88 | res.append(f) 89 | return res 90 | 91 | def get_output(self, train=False): 92 | facts, question = self.get_input() 93 | # facts = facts[0] 94 | # question = question[0] 95 | u = K.sum(question, axis=1) 96 | for q, f, m, w in zip(self.question, self.facts, self.memory, self.Ws): 97 | C = f(facts, train=train) 98 | A = m(facts, train=train) 99 | B = q(u, train=train) 100 | match = self._match(A, B) 101 | att = self._attention(C, match) 102 | u = w(u + att, train=train) 103 | return u 104 | 105 | def _attention(self, A, B): 106 | att = K.permute_dimensions(B, (0, 2, 1)) 107 | att = K.sum(att * B, axis=1) 108 | return att 109 | 110 | def _match(self, A, B): 111 | return T.batched_tensordot(A, B, axes=(2, 2)) 112 | 113 | @property 114 | def input_shape(self): 115 | question_shape = (None, 1, self.q_nb_words) 116 | facts_shape = (None, self.input_length, self.f_nb_words) 117 | return [facts_shape, question_shape] 118 | 119 | @property 120 | def output_shape(self): 121 | return self.input_shape[0][0], self.output_dim 122 | 123 | @property 124 | def input(self): 125 | return self.get_input() 126 | 127 | def supports_masked_input(self): 128 | return False 129 | 130 | def get_output_mask(self, train=None): 131 | return None 132 | 133 | def get_weights(self): 134 | weights = [] 135 | for q, f, m, w in zip(self.question, self.facts, self.memory, self.Ws): 136 | for l in (q, f, m, w): 137 | weights += l.get_weights() 138 | return weights 139 | 140 | def set_weights(self, weights): 141 | for q, f, m, w in zip(self.question, self.facts, self.memory, self.Ws): 142 | for l in (q, f, m): 143 | l.set_weights(weights[0]) 144 | weights.pop[0] 145 | w.set_weights(weights[:2]) 146 | weights.pop[0] 147 | weights.pop[1] 148 | 149 | def get_config(self): 150 | config = {'name': self.__class__.__name__, 151 | 'output_dim': self.output_dim, 152 | 'inner_activation': self.inner_activation, 153 | 'activation': self.activation, 154 | 'input_length': self.input_length, 155 | 'f_nb_words': self.f_nb_words, 156 | 'q_nb_words': self.q_nb_words, 157 | 'dropout': self.dropout, 158 | 'bow_mode': self.bow_mode, 159 | 'mask_zero': self.mask_zero, 160 | 'hops': self.hops, 161 | 'mode': self.mode} 162 | base_config = super(MemNN, self).get_config() 163 | return dict(list(base_config.items()) + list(config.items())) 164 | -------------------------------------------------------------------------------- /seya/layers/memnn2.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as T 2 | import keras.backend as K 3 | from keras.layers.core import LambdaMerge 4 | from keras import initializations 5 | 6 | 7 | class MemN2N(LambdaMerge): 8 | def __init__(self, layers, output_dim, input_dim, input_length, 9 | memory_length, hops=3, bow_mode="bow", mode="adjacent", 10 | emb_init="uniform", init="glorot_uniform", **kwargs): 11 | 12 | self.output_dim = output_dim 13 | self.input_dim = input_dim 14 | self.input_length = input_length 15 | self.memory_length = memory_length 16 | self.hops = hops 17 | self.bow_mode = bow_mode 18 | self.mode = mode 19 | self.init = initializations.get(init) 20 | self.emb_init = initializations.get(emb_init) 21 | output_shape = (self.output_dim, ) 22 | 23 | super(MemN2N, self).__init__(layers, lambda x: x, output_shape) 24 | 25 | def build(self): 26 | # list of embedding layers 27 | self.outputs = [] 28 | self.memory = [] 29 | # self.Hs = [] # if self.mode == "rnn" 30 | self.trainable_weights = [] 31 | for i in range(self.hops): 32 | # memory embedding - A 33 | if self.mode == "adjacent" and i > 0: 34 | A = self.outputs[-1] 35 | else: 36 | A = self.emb_init((self.input_dim, self.output_dim), 37 | name="{}_A_{}".format(self.name, i)) 38 | self.trainable_weights += [A] 39 | self.memory.append(A) 40 | 41 | # outputs embedding - C 42 | # if self.mode == "adjacent" and i > 1: 43 | # Wo = self.outputs[-1] 44 | # elif self.mode == "untied" or i == 0: 45 | C = self.emb_init((self.input_dim, self.output_dim), 46 | name="{}_C_{}".format(self.name, i)) 47 | self.trainable_weights += [C] 48 | self.outputs.append(C) 49 | 50 | # if self.mode == "rnn" 51 | # H = self.init((self.output_dim, self.output_dim), 52 | # name="{}_H_{}".format(self.name, i)) 53 | # self.trainable_weights += [H] 54 | # b = K.zeros((self.input_dim,), 55 | # name="{}_b_{}".format(self.name, i)) 56 | # self.Hs += [H] 57 | # self.trainable_weights += [H] 58 | 59 | if self.mode == "adjacent": 60 | self.W = self.outputs[-1].T 61 | self.b = K.zeros((self.input_dim,), name="{}_b".format(self.name)) 62 | # self.trainable_weights += [self.b] 63 | 64 | # question embedding - B 65 | self.B = self.emb_init((self.input_dim, self.output_dim), 66 | name="{}_B".format(self.name)) 67 | self.trainable_weights += [self.B] 68 | 69 | # Temporal embedding 70 | self.Te = self.emb_init((self.input_length, self.output_dim)) 71 | self.trainable_weights += [self.Te] 72 | 73 | def get_output(self, train=False): 74 | inputs = [layer.get_output(train) for layer in self.layers] 75 | facts, question = inputs 76 | # WARN make sure input layers are Embedding layers with identity init 77 | # facts = K.argmax(facts, axis=-1) 78 | # question = K.argmax(question, axis=-1) 79 | u, mask_q = self.lookup(question, self.B, 1) # just 1 question 80 | for A, C in zip(self.memory, self.outputs): 81 | m, mask_m = self.lookup(facts, A, self.memory_length) 82 | c, mask_c = self.lookup(facts, C, self.memory_length) 83 | 84 | # attention weights 85 | p = self.attention(m, u, mask_m) 86 | 87 | # output 88 | o = self.calc_output(c, p) 89 | u = o + u 90 | # u = K.dot(u[:, 0, :], self.W) + self.b 91 | return u[:, 0, :] # K.softmax(u) 92 | 93 | def lookup(self, x, W, memory_length): 94 | # shape: (batch*memory_length, input_length) 95 | x = K.cast(K.reshape(x, (-1, self.input_length)), 'int32') 96 | mask = K.expand_dims(K.not_equal(x, 0.), dim=-1) 97 | # shape: (batch*memory_length, input_length, output_dim) 98 | X = K.gather(W, x) 99 | if self.bow_mode == "bow": 100 | # shape: (batch*memory_length, output_dim) 101 | X = K.sum(X + K.expand_dims(self.Te, 0), axis=1) 102 | # shape: (batch, memory_length, output_dim) 103 | X = K.reshape(X, (-1, memory_length, self.output_dim)) 104 | return X, mask 105 | 106 | def attention(self, m, q, mask): 107 | # mask original shape is (batch*memory_length, input_length, 1) 108 | # shape (batch, memory) 109 | mask = K.reshape(mask[:, 0], (-1, self.memory_length)) 110 | # shape: (batch, memory_length, 1) 111 | p = T.batched_tensordot(m, q, (2, 2)) 112 | # shape: (batch, memory_length) 113 | p = K.softmax(p[:, :, 0]) # * K.cast(mask, 'float32') 114 | # shape: (batch, 1, memory_length) 115 | return K.expand_dims(p, dim=1) 116 | 117 | def calc_output(self, c, p): 118 | # shape: (batch, memory_length, 1) 119 | p = K.permute_dimensions(p, (0, 2, 1)) 120 | # shape: (batch, output_dim) 121 | o = K.sum(c * p, axis=1) 122 | # if self.mode == "rnn": 123 | # import theano 124 | # W = theano.printing.Print('[Debug] W shape: ', attrs=("shape",))(W) 125 | # o = K.dot(o, W) + b 126 | # shape: (batch, 1, output_dim) 127 | return K.expand_dims(o, dim=1) 128 | -------------------------------------------------------------------------------- /seya/layers/normalization.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.layers.normalization import BatchNormalization 3 | from keras.layers.core import MaskedLayer, Layer, ActivityRegularization 4 | from keras.layers.convolutional import MaxPooling2D 5 | 6 | from seya.regularizers import LambdaRegularizer 7 | 8 | 9 | class MaskedBN(MaskedLayer, BatchNormalization): 10 | def __init__(self, *args, **kwargs): 11 | super(MaskedBN, self).__init__(*args, **kwargs) 12 | -------------------------------------------------------------------------------- /seya/layers/ntm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import theano 4 | import theano.tensor as T 5 | floatX = theano.config.floatX 6 | 7 | from keras.layers.recurrent import Recurrent, GRU, LSTM 8 | from keras import backend as K 9 | 10 | from seya.utils import rnn_states 11 | tol = 1e-4 12 | 13 | 14 | def _wta(X): 15 | M = K.max(X, axis=-1, keepdims=True) 16 | R = K.switch(K.equal(X, M), X, 0.) 17 | return R 18 | 19 | 20 | def _update_controller(self, inp, h_tm1, M): 21 | """We have to update the inner RNN inside the NTM, this 22 | is the function to do it. Pretty much copy+pasta from Keras 23 | """ 24 | x = T.concatenate([inp, M], axis=-1) 25 | #1 is for gru, 2 is for lstm 26 | if len(h_tm1) in [1,2]: 27 | if hasattr(self.rnn,"get_constants"): 28 | BW,BU = self.rnn.get_constants(x) 29 | h_tm1 += (BW,BU) 30 | # update state 31 | _, h = self.rnn.step(x, h_tm1) 32 | 33 | return h 34 | 35 | 36 | def _circulant(leng, n_shifts): 37 | """ 38 | I confess, I'm actually proud of this hack. I hope you enjoy! 39 | This will generate a tensor with `n_shifts` of rotated versions the 40 | identity matrix. When this tensor is multiplied by a vector 41 | the result are `n_shifts` shifted versions of that vector. Since 42 | everything is done with inner products, everything is differentiable. 43 | 44 | Paramters: 45 | ---------- 46 | leng: int > 0, number of memory locations 47 | n_shifts: int > 0, number of allowed shifts (if 1, no shift) 48 | 49 | Returns: 50 | -------- 51 | shift operation, a tensor with dimensions (n_shifts, leng, leng) 52 | """ 53 | eye = np.eye(leng) 54 | shifts = range(n_shifts//2, -n_shifts//2, -1) 55 | C = np.asarray([np.roll(eye, s, axis=1) for s in shifts]) 56 | return theano.shared(C.astype(theano.config.floatX)) 57 | 58 | 59 | def _renorm(x): 60 | return x / (x.sum(axis=1, keepdims=True)) 61 | 62 | 63 | def _softmax(x): 64 | wt = x.flatten(ndim=2) 65 | w = T.nnet.softmax(wt) 66 | return w.reshape(x.shape) # T.clip(s, 0, 1) 67 | 68 | 69 | def _cosine_distance(M, k): 70 | dot = (M * k[:, None, :]).sum(axis=-1) 71 | nM = T.sqrt((M**2).sum(axis=-1)) 72 | nk = T.sqrt((k**2).sum(axis=-1, keepdims=True)) 73 | return dot / (nM * nk) 74 | 75 | 76 | class NeuralTuringMachine(Recurrent): 77 | """ Neural Turing Machines 78 | 79 | Non obvious parameter: 80 | ---------------------- 81 | shift_range: int, number of available shifts, ex. if 3, avilable shifts are 82 | (-1, 0, 1) 83 | n_slots: number of memory locations 84 | m_length: memory length at each location 85 | 86 | Known issues: 87 | ------------- 88 | Theano may complain when n_slots == 1. 89 | 90 | """ 91 | def __init__(self, output_dim, n_slots, m_length, shift_range=3, 92 | inner_rnn='gru', 93 | init='glorot_uniform', inner_init='orthogonal', 94 | input_dim=None, input_length=None, **kwargs): 95 | self.output_dim = output_dim 96 | self.n_slots = n_slots 97 | self.m_length = m_length 98 | self.shift_range = shift_range 99 | self.init = init 100 | self.inner_init = inner_init 101 | self.inner_rnn = inner_rnn 102 | 103 | self.input_dim = input_dim 104 | self.input_length = input_length 105 | if self.input_dim: 106 | kwargs['input_shape'] = (self.input_length, self.input_dim) 107 | super(NeuralTuringMachine, self).__init__(**kwargs) 108 | 109 | def build(self): 110 | input_leng, input_dim = self.input_shape[1:] 111 | self.input = T.tensor3() 112 | 113 | if self.inner_rnn == 'gru': 114 | self.rnn = GRU( 115 | activation='relu', 116 | input_dim=input_dim+self.m_length, 117 | input_length=input_leng, 118 | output_dim=self.output_dim, init=self.init, 119 | inner_init=self.inner_init) 120 | elif self.inner_rnn == 'lstm': 121 | self.rnn = LSTM( 122 | input_dim=input_dim+self.m_length, 123 | input_length=input_leng, 124 | output_dim=self.output_dim, init=self.init, 125 | forget_bias_init='zero', 126 | inner_init=self.inner_init) 127 | else: 128 | raise ValueError('this inner_rnn is not implemented yet.') 129 | 130 | self.rnn.build() 131 | 132 | # initial memory, state, read and write vecotrs 133 | self.M = theano.shared((.001*np.ones((1,)).astype(floatX))) 134 | self.init_h = K.zeros((self.output_dim)) 135 | self.init_wr = self.rnn.init((self.n_slots,)) 136 | self.init_ww = self.rnn.init((self.n_slots,)) 137 | 138 | # write 139 | self.W_e = self.rnn.init((self.output_dim, self.m_length)) # erase 140 | self.b_e = K.zeros((self.m_length)) 141 | self.W_a = self.rnn.init((self.output_dim, self.m_length)) # add 142 | self.b_a = K.zeros((self.m_length)) 143 | 144 | # get_w parameters for reading operation 145 | self.W_k_read = self.rnn.init((self.output_dim, self.m_length)) 146 | self.b_k_read = self.rnn.init((self.m_length, )) 147 | self.W_c_read = self.rnn.init((self.output_dim, 3)) # 3 = beta, g, gamma see eq. 5, 7, 9 148 | self.b_c_read = K.zeros((3)) 149 | self.W_s_read = self.rnn.init((self.output_dim, self.shift_range)) 150 | self.b_s_read = K.zeros((self.shift_range)) # b_s lol! not intentional 151 | 152 | # get_w parameters for writing operation 153 | self.W_k_write = self.rnn.init((self.output_dim, self.m_length)) 154 | self.b_k_write = self.rnn.init((self.m_length, )) 155 | self.W_c_write = self.rnn.init((self.output_dim, 3)) # 3 = beta, g, gamma see eq. 5, 7, 9 156 | self.b_c_write = K.zeros((3)) 157 | self.W_s_write = self.rnn.init((self.output_dim, self.shift_range)) 158 | self.b_s_write = K.zeros((self.shift_range)) 159 | 160 | self.C = _circulant(self.n_slots, self.shift_range) 161 | 162 | self.trainable_weights = self.rnn.trainable_weights + [ 163 | self.W_e, self.b_e, 164 | self.W_a, self.b_a, 165 | self.W_k_read, self.b_k_read, 166 | self.W_c_read, self.b_c_read, 167 | self.W_s_read, self.b_s_read, 168 | self.W_k_write, self.b_k_write, 169 | self.W_s_write, self.b_s_write, 170 | self.W_c_write, self.b_c_write, 171 | self.M, 172 | self.init_h, self.init_wr, self.init_ww] 173 | 174 | if self.inner_rnn == 'lstm': 175 | self.init_c = K.zeros((self.output_dim)) 176 | self.trainable_weights = self.trainable_weights + [self.init_c, ] 177 | 178 | def _read(self, w, M): 179 | return (w[:, :, None]*M).sum(axis=1) 180 | 181 | def _write(self, w, e, a, M): 182 | Mtilda = M * (1 - w[:, :, None]*e[:, None, :]) 183 | Mout = Mtilda + w[:, :, None]*a[:, None, :] 184 | return Mout 185 | 186 | def _get_content_w(self, beta, k, M): 187 | num = beta[:, None] * _cosine_distance(M, k) 188 | return _softmax(num) 189 | 190 | def _get_location_w(self, g, s, C, gamma, wc, w_tm1): 191 | wg = g[:, None] * wc + (1-g[:, None])*w_tm1 192 | Cs = (C[None, :, :, :] * wg[:, None, None, :]).sum(axis=3) 193 | wtilda = (Cs * s[:, :, None]).sum(axis=1) 194 | wout = _renorm(wtilda ** gamma[:, None]) 195 | return wout 196 | 197 | def _get_controller_output(self, h, W_k, b_k, W_c, b_c, W_s, b_s): 198 | k = T.tanh(T.dot(h, W_k) + b_k) # + 1e-6 199 | c = T.dot(h, W_c) + b_c 200 | beta = T.nnet.relu(c[:, 0]) + 1e-4 201 | g = T.nnet.sigmoid(c[:, 1]) 202 | gamma = T.nnet.relu(c[:, 2]) + 1.0001 203 | s = T.nnet.softmax(T.dot(h, W_s) + b_s) 204 | return k, beta, g, gamma, s 205 | 206 | def get_initial_states(self, X): 207 | batch_size = X.shape[0] 208 | init_M = self.M.dimshuffle(0, 'x', 'x').repeat( 209 | batch_size, axis=0).repeat(self.n_slots, axis=1).repeat( 210 | self.m_length, axis=2) 211 | init_M = init_M.flatten(ndim=2) 212 | 213 | init_h = self.init_h.dimshuffle(('x', 0)).repeat(batch_size, axis=0) 214 | init_wr = self.init_wr.dimshuffle(('x', 0)).repeat(batch_size, axis=0) 215 | init_ww = self.init_ww.dimshuffle(('x', 0)).repeat(batch_size, axis=0) 216 | if self.inner_rnn == 'lstm': 217 | init_c = self.init_c.dimshuffle(('x', 0)).repeat(batch_size, axis=0) 218 | return [init_M, T.nnet.softmax(init_wr), T.nnet.softmax(init_ww), 219 | init_h, init_c] 220 | else: 221 | return [init_M, T.nnet.softmax(init_wr), T.nnet.softmax(init_ww), 222 | init_h] 223 | 224 | @property 225 | def output_shape(self): 226 | input_shape = self.input_shape 227 | if self.return_sequences: 228 | return input_shape[0], input_shape[1], self.output_dim 229 | else: 230 | return input_shape[0], self.output_dim 231 | 232 | def get_full_output(self, train=False): 233 | """ 234 | This method is for research and visualization purposes. Use it as 235 | X = model.get_input() # full model 236 | Y = ntm.get_output() # this layer 237 | F = theano.function([X], Y, allow_input_downcast=True) 238 | [memory, read_address, write_address, rnn_state] = F(x) 239 | 240 | if inner_rnn == "lstm" use it as 241 | [memory, read_address, write_address, rnn_cell, rnn_state] = F(x) 242 | 243 | """ 244 | # input shape: (nb_samples, time (padded with zeros), input_dim) 245 | X = self.get_input(train) 246 | assert K.ndim(X) == 3 247 | if K._BACKEND == 'tensorflow': 248 | if not self.input_shape[1]: 249 | raise Exception('When using TensorFlow, you should define ' + 250 | 'explicitely the number of timesteps of ' + 251 | 'your sequences. Make sure the first layer ' + 252 | 'has a "batch_input_shape" argument ' + 253 | 'including the samples axis.') 254 | 255 | mask = self.get_output_mask(train) 256 | if mask: 257 | # apply mask 258 | X *= K.cast(K.expand_dims(mask), X.dtype) 259 | masking = True 260 | else: 261 | masking = False 262 | 263 | if self.stateful: 264 | initial_states = self.states 265 | else: 266 | initial_states = self.get_initial_states(X) 267 | 268 | states = rnn_states(self.step, X, initial_states, 269 | go_backwards=self.go_backwards, 270 | masking=masking) 271 | return states 272 | 273 | def step(self, x, states): 274 | M_tm1, wr_tm1, ww_tm1 = states[:3] 275 | # reshape 276 | M_tm1 = M_tm1.reshape((x.shape[0], self.n_slots, self.m_length)) 277 | # read 278 | h_tm1 = states[3:] 279 | k_read, beta_read, g_read, gamma_read, s_read = self._get_controller_output( 280 | h_tm1[0], self.W_k_read, self.b_k_read, self.W_c_read, self.b_c_read, 281 | self.W_s_read, self.b_s_read) 282 | wc_read = self._get_content_w(beta_read, k_read, M_tm1) 283 | wr_t = self._get_location_w(g_read, s_read, self.C, gamma_read, 284 | wc_read, wr_tm1) 285 | M_read = self._read(wr_t, M_tm1) 286 | 287 | # update controller 288 | h_t = _update_controller(self, x, h_tm1, M_read) 289 | 290 | # write 291 | k_write, beta_write, g_write, gamma_write, s_write = self._get_controller_output( 292 | h_t[0], self.W_k_write, self.b_k_write, self.W_c_write, 293 | self.b_c_write, self.W_s_write, self.b_s_write) 294 | wc_write = self._get_content_w(beta_write, k_write, M_tm1) 295 | ww_t = self._get_location_w(g_write, s_write, self.C, gamma_write, 296 | wc_write, ww_tm1) 297 | e = T.nnet.sigmoid(T.dot(h_t[0], self.W_e) + self.b_e) 298 | a = T.tanh(T.dot(h_t[0], self.W_a) + self.b_a) 299 | M_t = self._write(ww_t, e, a, M_tm1) 300 | 301 | M_t = M_t.flatten(ndim=2) 302 | 303 | return h_t[0], [M_t, wr_t, ww_t] + h_t 304 | -------------------------------------------------------------------------------- /seya/layers/readout.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as T 3 | 4 | from keras.layers.recurrent import GRU 5 | from keras.utils.theano_utils import shared_zeros 6 | 7 | from seya.utils import apply_model 8 | 9 | 10 | def _masking(h_t, h_tm1, mask): 11 | mask = mask[:, 0].dimshuffle(0, 'x') 12 | return mask * h_t + (1-mask) * h_tm1 13 | 14 | 15 | class GRUwithReadout(GRU): 16 | """ 17 | GRUwithReadout 18 | GRU with a last layer whose output is also fedback with 19 | the dynamic state. 20 | 21 | Extra Parameter: 22 | ---------------- 23 | readout: `keras.model.sequential` 24 | state_dim: dimension of the inner GRU 25 | 26 | Notes: 27 | ------ 28 | output_dim == readout.output_dim 29 | 30 | """ 31 | def __init__(self, readout, 32 | init='glorot_uniform', inner_init='orthogonal', 33 | activation='sigmoid', inner_activation='hard_sigmoid', 34 | weights=None, truncate_gradient=-1, 35 | return_sequences=False, 36 | input_dim=None, input_length=None, **kwargs): 37 | 38 | self.readout = readout 39 | self.state_dim = readout.layers[0].input_shape[1] # state_dim 40 | input_dim = readout.layers[0].input_shape[1] 41 | super(GRUwithReadout, self).__init__( 42 | self.state_dim, 43 | init=init, inner_init=inner_init, 44 | activation=activation, inner_activation=inner_activation, 45 | weights=weights, truncate_gradient=truncate_gradient, 46 | return_sequences=return_sequences, input_dim=input_dim, 47 | **kwargs) 48 | self.output_dim = self.readout.output_shape[1] 49 | 50 | def build(self): 51 | self.readout.build() 52 | self.init_h = shared_zeros((self.state_dim,)) 53 | # here is difference on the sizes 54 | input_dim = self.input_shape[2] + self.readout.output_shape[1] 55 | 56 | # copy-paste from keras.recurrent.GRU 57 | self.W_z = self.init((input_dim, self.state_dim)) 58 | self.U_z = self.inner_init((self.state_dim, self.state_dim)) 59 | self.b_z = shared_zeros((self.state_dim)) 60 | 61 | self.W_r = self.init((input_dim, self.state_dim)) 62 | self.U_r = self.inner_init((self.state_dim, self.state_dim)) 63 | self.b_r = shared_zeros((self.state_dim)) 64 | 65 | self.W_h = self.init((input_dim, self.state_dim)) 66 | self.U_h = self.inner_init((self.state_dim, self.state_dim)) 67 | self.b_h = shared_zeros((self.state_dim)) 68 | 69 | self.trainable_weights = [ 70 | self.init_h, 71 | self.W_z, self.U_z, self.b_z, 72 | self.W_r, self.U_r, self.b_r, 73 | self.W_h, self.U_h, self.b_h, 74 | ] 75 | 76 | if self.initial_weights is not None: 77 | self.set_weights(self.initial_weights) 78 | del self.initial_weights 79 | 80 | def _get_initial_states(self, batch_size): 81 | init_h = T.repeat(self.init_h.dimshuffle('x', 0), batch_size, axis=0) 82 | init_o = apply_model(self.readout, init_h) 83 | return init_h, init_o 84 | 85 | def _step(self, 86 | x_t, mask_tm1, 87 | h_tm1, o_tm1, 88 | u_z, u_r, u_h, *args): 89 | xo = T.concatenate([x_t, o_tm1], axis=-1) 90 | xz_t = T.dot(xo, self.W_z) + self.b_z 91 | xr_t = T.dot(xo, self.W_r) + self.b_r 92 | xh_t = T.dot(xo, self.W_h) + self.b_h 93 | z = self.inner_activation(xz_t + T.dot(h_tm1, u_z)) 94 | r = self.inner_activation(xr_t + T.dot(h_tm1, u_r)) 95 | hh_t = self.activation(xh_t + T.dot(r * h_tm1, u_h)) 96 | h_t = z * h_tm1 + (1 - z) * hh_t 97 | o_t = apply_model(self.readout, h_t) 98 | 99 | return _masking(h_t, h_tm1, mask_tm1), _masking(o_t, o_tm1, mask_tm1) 100 | 101 | def get_output(self, train=False): 102 | X = self.get_input(train) 103 | padded_mask = self.get_padded_shuffled_mask(train, X, pad=1) 104 | X = X.dimshuffle((1, 0, 2)) 105 | 106 | init_h, init_o = self._get_initial_states(X.shape[1]) 107 | outputs, updates = theano.scan( 108 | self._step, 109 | sequences=[X, padded_mask], 110 | outputs_info=[init_h, init_o], 111 | non_sequences=[self.U_z, self.U_r, self.U_h] + self.readout.trainable_weights, 112 | truncate_gradient=self.truncate_gradient) 113 | 114 | if self.return_sequences: 115 | return outputs[1].dimshuffle((1, 0, 2)) 116 | return outputs[1][-1] 117 | -------------------------------------------------------------------------------- /seya/layers/recurrent.py: -------------------------------------------------------------------------------- 1 | import types 2 | import theano 3 | import theano.tensor as T 4 | 5 | from keras.layers.recurrent import Recurrent, GRU 6 | from keras import backend as K 7 | 8 | 9 | def _get_reversed_input(self, train=False): 10 | if hasattr(self, 'previous'): 11 | X = self.previous.get_output(train=train) 12 | else: 13 | X = self.input 14 | return X[::-1] 15 | 16 | 17 | class Bidirectional(Recurrent): 18 | def __init__(self, forward=None, backward=None, return_sequences=False, 19 | forward_conf=None, backward_conf=None): 20 | assert forward is not None or forward_conf is not None, "Must provide a forward RNN or a forward configuration" 21 | assert backward is not None or backward_conf is not None, "Must provide a backward RNN or a backward configuration" 22 | super(Bidirectional, self).__init__() 23 | if forward is not None: 24 | self.forward = forward 25 | else: 26 | # Must import inside the function, because in order to support loading 27 | # we must import this module inside layer_utils... ugly 28 | from keras.utils.layer_utils import container_from_config 29 | self.forward = container_from_config(forward_conf) 30 | if backward is not None: 31 | self.backward = backward 32 | else: 33 | from keras.utils.layer_utils import container_from_config 34 | self.backward = container_from_config(backward_conf) 35 | self.return_sequences = return_sequences 36 | self.output_dim = self.forward.output_dim + self.backward.output_dim 37 | 38 | if not (self.return_sequences == self.forward.return_sequences == self.backward.return_sequences): 39 | raise ValueError("Make sure 'return_sequences' is equal for self," 40 | " forward and backward.") 41 | 42 | def build(self): 43 | self.input = T.tensor3() 44 | self.forward.input = self.input 45 | self.backward.input = self.input 46 | self.forward.build() 47 | self.backward.build() 48 | self.trainable_weights = self.forward.trainable_weights + self.backward.trainable_weights 49 | 50 | def set_previous(self, layer): 51 | assert self.nb_input == layer.nb_output == 1, "Cannot connect layers: input count and output count should be 1." 52 | if hasattr(self, 'input_ndim'): 53 | assert self.input_ndim == len(layer.output_shape), "Incompatible shapes: layer expected input with ndim=" +\ 54 | str(self.input_ndim) + " but previous layer has output_shape " + str(layer.output_shape) 55 | self.forward.set_previous(layer) 56 | self.backward.set_previous(layer) 57 | self.backward.get_input = types.MethodType(_get_reversed_input, self.backward) 58 | self.previous = layer 59 | self.build() 60 | 61 | @property 62 | def output_shape(self): 63 | input_shape = self.input_shape 64 | output_dim = self.output_dim 65 | if self.return_sequences: 66 | return (input_shape[0], input_shape[1], output_dim) 67 | else: 68 | return (input_shape[0], output_dim) 69 | 70 | def get_output(self, train=False): 71 | Xf = self.forward.get_output(train) 72 | Xb = self.backward.get_output(train) 73 | Xb = Xb[::-1] 74 | return T.concatenate([Xf, Xb], axis=-1) 75 | 76 | def get_config(self): 77 | return {'name': self.__class__.__name__, 78 | 'forward_conf': self.forward.get_config(), 79 | 'backward_conf': self.backward.get_config(), 80 | 'return_sequences': self.return_sequences} 81 | 82 | 83 | class StatefulGRU(GRU): 84 | def __init__(self, batch_size, output_dim=128, 85 | init='glorot_uniform', inner_init='orthogonal', 86 | activation='sigmoid', inner_activation='hard_sigmoid', 87 | weights=None, return_sequences=False, 88 | input_dim=None, input_length=None, **kwargs): 89 | 90 | self.batch_size = batch_size 91 | self.input_dim = input_dim 92 | self.input_length = input_length 93 | if self.input_dim: 94 | kwargs['input_shape'] = (self.input_length, self.input_dim) 95 | 96 | super(StatefulGRU, self).__init__( 97 | output_dim, init=init, inner_init=inner_init, 98 | activation=activation, inner_activation=inner_activation, 99 | weights=weights, 100 | return_sequences=return_sequences, 101 | input_dim=input_dim, input_length=input_length, **kwargs) 102 | 103 | def build(self): 104 | super(StatefulGRU, self).build() 105 | self.h = K.zeros((self.batch_size, self.output_dim)) # Here is the state 106 | 107 | def get_output(self, train=False): 108 | X = self.get_input(train) 109 | padded_mask = self.get_padded_shuffled_mask(train, X, pad=1) 110 | X = X.dimshuffle((1, 0, 2)) 111 | 112 | x_z = T.dot(X, self.W_z) + self.b_z 113 | x_r = T.dot(X, self.W_r) + self.b_r 114 | x_h = T.dot(X, self.W_h) + self.b_h 115 | outputs, updates = theano.scan( 116 | self._step, 117 | sequences=[x_z, x_r, x_h, padded_mask], 118 | outputs_info=self.h[:X.shape[1]], 119 | non_sequences=[self.U_z, self.U_r, self.U_h]) 120 | 121 | self.updates = ((self.h, outputs[-1]), ) # initial state of next batch 122 | # is the last state of this 123 | # batch 124 | if self.return_sequences: 125 | return outputs.dimshuffle((1, 0, 2)) 126 | return outputs[-1] 127 | 128 | def init_updates(self): 129 | self.get_output(train=True) 130 | 131 | def get_config(self): 132 | return {"name": self.__class__.__name__, 133 | "input_dim": self.input_dim, 134 | "output_dim": self.output_dim, 135 | "init": self.init.__name__, 136 | "inner_init": self.inner_init.__name__, 137 | "activation": self.activation.__name__, 138 | "inner_activation": self.inner_activation.__name__, 139 | "return_sequences": self.return_sequences} 140 | -------------------------------------------------------------------------------- /seya/layers/regularization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras import backend as K 4 | from keras.layers.core import Layer 5 | 6 | from seya.regularizers import LambdaRegularizer 7 | 8 | 9 | class ITLRegularizer(Layer): 10 | '''ITL regularization layer using shared variables 11 | 12 | Parameters: 13 | ----------- 14 | scale: float, (default, 1) how important is this regularization to the cost 15 | function. 16 | distribution: function, random number generator from the prior distribution 17 | ksize: float, (default, 1) Parzen window size. 18 | 19 | ''' 20 | def default_distribution(shape): 21 | return K.random_normal(shape, std=5) 22 | 23 | def __init__(self, scale=1., distribution=default_distribution, distance='euclidean', ksize=1., **kwargs): 24 | super(ITLRegularizer, self).__init__(**kwargs) 25 | self.scale = scale 26 | self.distribution = distribution 27 | self.ksize = ksize 28 | self.distance = distance 29 | 30 | def _get_kernel(self, X, Z): 31 | G = K.sum((K.expand_dims(X, dim=1) - Z)**2, axis=-1) # Gram matrix 32 | G = K.exp(-G/(self.ksize)) / K.sqrt(2*np.pi*self.ksize) 33 | return G 34 | 35 | def build(self): 36 | X = self.get_input() 37 | # if self.distribution == 'normal': 38 | # Z = K.random_normal(K.shape(X), std=5) 39 | Z = self.distribution(K.shape(X)) 40 | 41 | Gxx = self._get_kernel(X, X) 42 | Gzz = self._get_kernel(Z, Z) 43 | Gxz = self._get_kernel(X, Z) 44 | if self.distance == 'euclidean': # same as minimum mean discrepancy 45 | itl_divergence = K.mean(Gxx) + K.mean(Gzz) - 2*K.mean(Gxz) 46 | 47 | elif self.distance == 'csd': # Cauchy-Schwarz divergence 48 | itl_divergence = K.log(K.sqrt(K.mean(Gxx)*K.mean(Gzz)) / 49 | K.mean(Gxz)) 50 | 51 | elif self.distance == 'entropy': # Maximum entropy 52 | itl_divergence = K.mean(Gxx) 53 | 54 | elif self.distance == 'cip': # cross-information potential 55 | itl_divergence = -K.mean(Gxz) 56 | 57 | self.regularizers = [LambdaRegularizer(self.scale * itl_divergence), ] 58 | 59 | 60 | class ITLextInput(ITLRegularizer): 61 | """ 62 | Same as ITLRegularizer but expecting random noise as an external input. 63 | This is meant to be used as part of a models.Graph 64 | """ 65 | def __init__(self, code_size, scale=1., distance='euclidean', ksize=1., **kwargs): 66 | super(ITLextInput, self).__init__(scale=scale, distance=distance, 67 | ksize=ksize, **kwargs) 68 | self.code_size = code_size 69 | 70 | def build(self): 71 | Inp = self.get_input() 72 | X = Inp[:, :self.code_size] 73 | Z = Inp[:, self.code_size:] 74 | 75 | Gxx = self._get_kernel(X, X) 76 | Gzz = self._get_kernel(Z, Z) 77 | Gxz = self._get_kernel(X, Z) 78 | if self.distance == 'euclidean': # same as minimum mean discrepancy 79 | itl_divergence = K.mean(Gxx) + K.mean(Gzz) - 2*K.mean(Gxz) 80 | 81 | elif self.distance == 'csd': # Cauchy-Schwarz divergence 82 | itl_divergence = K.log(K.sqrt(K.mean(Gxx)*K.mean(Gzz)) / 83 | K.mean(Gxz)) 84 | 85 | elif self.distance == 'entropy': # Maximum entropy 86 | itl_divergence = K.mean(Gxx) 87 | 88 | self.regularizers = [LambdaRegularizer(self.scale * itl_divergence), ] 89 | 90 | def get_output(self, train=False): 91 | Inp = self.get_input(train) 92 | X = Inp[:, :self.code_size] 93 | Z = Inp[:, self.code_size:] 94 | Y = X 95 | return Y + 0*Z 96 | 97 | @property 98 | def output_shape(self): 99 | return (None, self.code_size) 100 | -------------------------------------------------------------------------------- /seya/layers/stack.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import theano 4 | import theano.tensor as T 5 | 6 | floatX = theano.config.floatX 7 | 8 | from keras.layers.recurrent import Recurrent, GRU, LSTM 9 | from keras import backend as K 10 | 11 | tol = 1e-4 12 | 13 | def _update_controller(self, inp , h_tm1): 14 | """We have to update the inner RNN inside the NTM, this 15 | is the function to do it. Pretty much copy+pasta from Keras 16 | """ 17 | 18 | #1 is for gru, 2 is for lstm 19 | if len(h_tm1) in [1,2]: 20 | if hasattr(self.rnn,"get_constants"): 21 | BW,BU = self.rnn.get_constants(inp) 22 | h_tm1 += (BW,BU) 23 | # update state 24 | 25 | op_t, h = self.rnn.step(inp, h_tm1) 26 | 27 | return op_t , h 28 | 29 | def _update_neural_stack(self, V_tm1, s_tm1, d_t, u_t, v_t, time,stack=True): 30 | 31 | ############################################################ 32 | #Equation 1 33 | 34 | V_t = V_tm1 35 | 36 | V_t = T.set_subtensor(V_t[::,time,::],v_t) 37 | 38 | ############################################################ 39 | #equation 2 40 | if stack: 41 | s_op = T.cumsum(s_tm1[::,1:time][::,::-1],axis=1) #Size t-2 42 | s_op = s_op[::,::-1] 43 | #padding 44 | input_shape = s_op.shape 45 | output_shape = (input_shape[0], 46 | input_shape[1] + 1) 47 | 48 | output = T.zeros(output_shape) 49 | s_op = T.set_subtensor(output[:, :input_shape[1]], s_op) 50 | else: 51 | s_op = T.cumsum(s_tm1[::,:time-1],axis=1) #Size t-2 52 | #padding 53 | input_shape = s_op.shape 54 | output_shape = (input_shape[0], 55 | input_shape[1] + 1) 56 | 57 | output = T.zeros(output_shape) 58 | s_op = T.set_subtensor(output[:, 1:input_shape[1]+1], s_op) 59 | 60 | s_op = u_t.dimshuffle(0,"x") - s_op 61 | 62 | s_op = T.maximum(s_op,0) 63 | 64 | 65 | #ifelse to deal with time == 0 66 | #m = T.max() 67 | #ifelse(T.ge(time,1),time,T.cast(1,"int32")) 68 | 69 | s_op = s_tm1[::,:time]-s_op 70 | 71 | s_op = T.maximum(s_op,0) 72 | 73 | s_t = s_tm1 74 | 75 | 76 | s_t = T.set_subtensor(s_t[::,:time], s_op) 77 | 78 | s_t = T.set_subtensor(s_t[::,time], d_t) 79 | 80 | 81 | 82 | ############################################################ 83 | #equation 3 84 | 85 | 86 | if stack: 87 | s_op = T.cumsum(s_t[::,1:time+1][::,::-1],axis=1) #Size t-1 88 | s_op = s_op[::,::-1] 89 | #left padding 90 | input_shape = s_op.shape 91 | output_shape = (input_shape[0], 92 | input_shape[1] + 1) 93 | 94 | output = T.zeros(output_shape) 95 | s_op = T.set_subtensor(output[:, :input_shape[1]], s_op) 96 | else: 97 | s_op = T.cumsum(s_t[::,:time],axis=1) #Size t-1 98 | #left padding 99 | input_shape = s_op.shape 100 | output_shape = (input_shape[0], 101 | input_shape[1] + 1) 102 | 103 | output = T.zeros(output_shape) 104 | s_op = T.set_subtensor(output[:,1:1+input_shape[1]], s_op) 105 | 106 | # Max operation 107 | s_op = 1 - s_op 108 | s_op = T.maximum(s_op,0) 109 | 110 | #Min operation 111 | s_op = T.minimum(s_t[::,:time+1],s_op) 112 | 113 | 114 | r_t = T.sum(s_op[::,:time+1].dimshuffle(0,1,"x")*V_t[::,:time+1,::],axis=1) 115 | 116 | return V_t, s_t,r_t 117 | 118 | 119 | class Stack(Recurrent): 120 | """ Stack and queue network 121 | 122 | 123 | output_dim = output dimension 124 | n_slots = number of memory slot 125 | m_length = dimention of the memory 126 | rnn_size = output length of the memory controler 127 | inner_rnn = "lstm" only lstm is supported 128 | stack = True to create neural stack or False to create neural queue 129 | 130 | 131 | from Learning to Transduce with Unbounded Memory 132 | [[http://arxiv.org/pdf/1506.02516.pdf]] 133 | """ 134 | def __init__(self, output_dim, n_slots, m_length, 135 | inner_rnn='lstm',rnn_size=64, stack=True, 136 | init='glorot_uniform', inner_init='orthogonal', 137 | input_dim=None, input_length=None, **kwargs): 138 | self.output_dim = output_dim 139 | self.n_slots = n_slots + 1 # because we start at time 1 140 | self.m_length = m_length 141 | self.init = init 142 | self.inner_init = inner_init 143 | if inner_rnn != "lstm": 144 | print "Only lstm is supported" 145 | raise 146 | self.inner_rnn = inner_rnn 147 | self.rnn_size = rnn_size 148 | self.stack = stack 149 | 150 | self.input_dim = input_dim 151 | self.input_length = input_length 152 | if self.input_dim: 153 | kwargs['input_shape'] = (self.input_length, self.input_dim) 154 | super(Stack, self).__init__(**kwargs) 155 | 156 | def build(self): 157 | input_leng, input_dim = self.input_shape[1:] 158 | self.input = T.tensor3() 159 | 160 | if self.inner_rnn == 'gru': 161 | self.rnn = GRU( 162 | activation='relu', 163 | input_dim=input_dim+self.m_length, 164 | input_length=input_leng, 165 | output_dim=self.output_dim, init=self.init, 166 | inner_init=self.inner_init) 167 | elif self.inner_rnn == 'lstm': 168 | self.rnn = LSTM( 169 | input_dim=input_dim+self.m_length, 170 | input_length=input_leng, 171 | output_dim=self.rnn_size, init=self.init, 172 | forget_bias_init='zero', 173 | inner_init=self.inner_init) 174 | else: 175 | raise ValueError('this inner_rnn is not implemented yet.') 176 | 177 | self.rnn.build() 178 | 179 | 180 | self.init_h = K.zeros((self.rnn_size)) 181 | 182 | self.W_d = self.rnn.init((self.rnn_size,1)) 183 | self.W_u = self.rnn.init((self.rnn_size,1)) 184 | 185 | self.W_v = self.rnn.init((self.rnn_size,self.m_length)) 186 | self.W_o = self.rnn.init((self.rnn_size,self.output_dim)) 187 | 188 | self.b_d = K.zeros((1,),name="b_d") 189 | self.b_u = K.zeros((1,),name="b_u") 190 | self.b_v = K.zeros((self.m_length,)) 191 | self.b_o = K.zeros((self.output_dim,)) 192 | 193 | 194 | self.trainable_weights = self.rnn.trainable_weights + [ 195 | self.W_d, self.b_d, 196 | self.W_v, self.b_v, 197 | self.W_u, self.b_u, 198 | self.W_o, self.b_o, self.init_h] 199 | 200 | if self.inner_rnn == 'lstm': 201 | self.init_c = K.zeros((self.rnn_size)) 202 | self.trainable_weights = self.trainable_weights + [self.init_c, ] 203 | #self.trainable_weights =[self.W_d] 204 | 205 | 206 | def get_initial_states(self, X): 207 | 208 | 209 | batch_size = X.shape[0] 210 | 211 | init_r = K.zeros((self.m_length)).dimshuffle('x',0).repeat(batch_size,axis=0) 212 | init_V = K.zeros((self.n_slots,self.m_length)).dimshuffle('x',0,1).repeat(batch_size,axis=0) 213 | init_S = K.zeros((self.n_slots)).dimshuffle('x',0).repeat(batch_size,axis=0) 214 | init_h = self.init_h.dimshuffle(('x', 0)).repeat(batch_size, axis=0) 215 | 216 | itime = K.zeros((1,),dtype=np.int32) 217 | 218 | 219 | if self.inner_rnn == 'lstm': 220 | init_c = self.init_c.dimshuffle(('x', 0)).repeat(batch_size, axis=0) 221 | return [init_r , init_V,init_S,itime,init_h,init_c] 222 | 223 | @property 224 | def output_shape(self): 225 | input_shape = self.input_shape 226 | if self.return_sequences: 227 | return input_shape[0], input_shape[1], self.output_dim 228 | else: 229 | return input_shape[0], self.output_dim 230 | 231 | def step(self, x, states): 232 | 233 | r_tm1, V_tm1,s_tm1,time = states[:4] 234 | h_tm1 = states[4:] 235 | 236 | 237 | 238 | r_tm1 = r_tm1 239 | 240 | op_t, h_t = _update_controller(self, T.concatenate([x, r_tm1], axis=-1), 241 | h_tm1) 242 | 243 | # op_t = op_t + print_name_shape("W_d",self.W_d.get_value()) 244 | op_t = op_t 245 | #op_t = op_t[:,0,:] 246 | d_t = K.sigmoid( K.dot(op_t, self.W_d) + self.b_d) 247 | u_t = K.sigmoid(K.dot(op_t, self.W_u) + self.b_u) 248 | v_t = K.tanh(K.dot(op_t, self.W_v) + self.b_v) 249 | o_t = K.tanh(K.dot(op_t, self.W_o) + self.b_o) 250 | 251 | 252 | time = time + 1 253 | V_t, s_t, r_t = _update_neural_stack(self, V_tm1, s_tm1, d_t[::,0], 254 | u_t[::,0], v_t,time[0],stack=self.stack) 255 | 256 | 257 | 258 | return o_t, [r_t, V_t, s_t, time] + h_t 259 | 260 | 261 | -------------------------------------------------------------------------------- /seya/layers/temp.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from keras import backend as K 4 | from keras.layers.core import Layer 5 | from keras import initializations, activations 6 | from keras.layers.convolutional import MaxPooling1D 7 | 8 | from seya.regularizers import GaussianKL, LambdaRegularizer 9 | 10 | 11 | class VariationalDense(Layer): 12 | """VariationalDense 13 | Hidden layer for Variational Autoencoding Bayes method [1]. 14 | This layer projects the input twice to calculate the mean and variance 15 | of a Gaussian distribution. During training, the output is sampled from 16 | that distribution as mean + random_noise * variance, during testing the 17 | output is the mean, i.e the expected value of the encoded distribution. 18 | 19 | Parameters: 20 | ----------- 21 | batch_size: Both Keras backends need the batch_size to be defined before 22 | hand for sampling random numbers. Make sure your batch size is kept 23 | fixed during training. You can use any batch size for testing. 24 | 25 | regularizer_scale: By default the regularization is already proberly 26 | scaled if you use binary or categorical crossentropy cost functions. 27 | In most cases this regularizers should be kept fixed at one. 28 | 29 | """ 30 | def __init__(self, output_dim, batch_size, init='glorot_uniform', 31 | activation='tanh', 32 | weights=None, input_dim=None, regularizer_scale=1, **kwargs): 33 | self.regularizer_scale = regularizer_scale 34 | self.batch_size = batch_size 35 | self.init = initializations.get(init) 36 | self.activation = activations.get(activation) 37 | self.output_dim = output_dim 38 | self.initial_weights = weights 39 | self.input_dim = input_dim 40 | if self.input_dim: 41 | kwargs['input_shape'] = (self.input_dim,) 42 | self.input = K.placeholder(ndim=2) 43 | super(VariationalDense, self).__init__(**kwargs) 44 | 45 | def build(self): 46 | input_dim = self.input_shape[1] 47 | 48 | self.W_mean = self.init((input_dim, self.output_dim)) 49 | self.b_mean = K.zeros((self.output_dim,)) 50 | self.W_logsigma = self.init((input_dim, self.output_dim)) 51 | self.b_logsigma = K.zeros((self.output_dim,)) 52 | 53 | self.trainable_weights = [self.W_mean, self.b_mean, self.W_logsigma, 54 | self.b_logsigma] 55 | 56 | self.regularizers = [] 57 | reg = self.get_variational_regularization(self.get_input()) 58 | self.regularizers.append(reg) 59 | 60 | def get_variational_regularization(self, X): 61 | mean = self.activation(K.dot(X, self.W_mean) + self.b_mean) 62 | logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma) 63 | return GaussianKL(mean, logsigma, regularizer_scale=self.regularizer_scale) 64 | 65 | def get_mean_logsigma(self, X): 66 | mean = self.activation(K.dot(X, self.W_mean) + self.b_mean) 67 | logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma) 68 | return mean, logsigma 69 | 70 | def _get_output(self, X, train=False): 71 | mean, logsigma = self.get_mean_logsigma(X) 72 | if train: 73 | eps = K.random_normal((self.batch_size, self.output_dim)) 74 | return mean + K.exp(logsigma) * eps 75 | else: 76 | return mean 77 | 78 | def get_output(self, train=False): 79 | X = self.get_input() 80 | return self._get_output(X, train) 81 | 82 | @property 83 | def output_shape(self): 84 | return (self.input_shape[0], self.output_dim) 85 | 86 | 87 | class SlowSiamese(Layer): 88 | def __init__(self, encoder, decoder, code_dim, batch_size, 89 | beta=0.5, subsample=2, regularizer_scale=0.5, 90 | init='glorot_uniform', activation='linear', 91 | weights=None, input_dim=None, **kwargs): 92 | self.regularizer_scale = regularizer_scale 93 | self.beta = beta 94 | self.max_pool = MaxPooling1D(subsample) 95 | self.encoder = encoder 96 | self.decoder = decoder 97 | self.variational = VariationalDense(code_dim, batch_size, 98 | input_dim=self.encoder.output_shape[1], 99 | regularizer_scale=regularizer_scale) 100 | self.batch_size = batch_size 101 | self.init = initializations.get(init) 102 | self.activation = activations.get(activation) 103 | self.code_dim = code_dim 104 | self.initial_weights = weights 105 | self.input_dim = input_dim 106 | if self.input_dim: 107 | kwargs['input_shape'] = (self.input_dim,) 108 | self.input = K.placeholder(ndim=4) 109 | super(SlowSiamese, self).__init__(**kwargs) 110 | 111 | def build(self): 112 | self.encoder.build() 113 | self.decoder.build() 114 | self.variational.build() 115 | 116 | outputs = [] 117 | self.regularizers = [] 118 | input_list = self.get_input() 119 | if isinstance(input_list, OrderedDict): 120 | assert len(input_list) == 2 121 | for X in input_list.values(): 122 | Y = self.encoder(X) 123 | reg = self.variational.get_variational_regularization(Y) 124 | self.regularizers.append(reg) 125 | Y = self.variational._get_output(Y, train=True) 126 | Y = self.decoder(Y) 127 | outputs.append(Y) 128 | pool0 = self.max_pool(K.expand_dims(outputs[0], 2)) 129 | pool1 = self.max_pool(K.expand_dims(outputs[1], 2)) 130 | slow = self.beta * ((pool0 - pool1)**2).mean() 131 | self.regularizers.append(LambdaRegularizer(slow)) 132 | else: 133 | Y = self.encoder(input_list) 134 | reg = self.variational.get_variational_regularization(Y) 135 | self.regularizers.append(reg) 136 | Y = self.variational._get_output(Y, train=True) 137 | Y = self.decoder(Y) 138 | 139 | self.trainable_weights = self.encoder.trainable_weights + self.variational.trainable_weights + self.decoder.trainable_weights 140 | 141 | def get_output(self, train=False): 142 | input_list = self.get_input() 143 | outputs = [] 144 | if isinstance(input_list, OrderedDict): 145 | assert len(input_list) == 2 146 | for X in input_list.values(): 147 | Y = self.encoder(X) 148 | Y = self.variational._get_output(Y, train) 149 | Y = self.decoder(Y) 150 | outputs.append(Y) 151 | else: 152 | Y = self.encoder(input_list) 153 | Y = self.variational._get_output(Y, train) 154 | Y = self.decoder(Y) 155 | outputs.append(Y) 156 | return outputs 157 | 158 | def get_output_at(self, head, train=False): 159 | return self.get_output(train)[head] 160 | 161 | def get_output_shape(self, head): 162 | return self.output_shape 163 | 164 | def encode(self, X): 165 | Y = self.encoder(X) 166 | return self.variational(Y) 167 | 168 | def sample(self, X): 169 | return self.decoder(X) 170 | 171 | @property 172 | def output_shape(self): 173 | return (self.input_shape[0], self.decoder.output_dim) 174 | -------------------------------------------------------------------------------- /seya/layers/variational.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.layers.core import Layer 3 | from keras import initializations, activations 4 | 5 | from seya.regularizers import GaussianKL 6 | 7 | 8 | class VariationalDense(Layer): 9 | """VariationalDense 10 | Hidden layer for Variational Autoencoding Bayes method [1]. 11 | This layer projects the input twice to calculate the mean and variance 12 | of a Gaussian distribution. During training, the output is sampled from 13 | that distribution as mean + random_noise * variance, during testing the 14 | output is the mean, i.e the expected value of the encoded distribution. 15 | 16 | Parameters: 17 | ----------- 18 | batch_size: Both Keras backends need the batch_size to be defined before 19 | hand for sampling random numbers. Make sure your batch size is kept 20 | fixed during training. You can use any batch size for testing. 21 | 22 | regularizer_scale: By default the regularization is already proberly 23 | scaled if you use binary or categorical crossentropy cost functions. 24 | In most cases this regularizers should be kept fixed at one. 25 | 26 | """ 27 | def __init__(self, output_dim, batch_size, init='glorot_uniform', 28 | activation='tanh', 29 | weights=None, input_dim=None, regularizer_scale=1, 30 | prior_mean=0, prior_logsigma=1, **kwargs): 31 | self.prior_mean = prior_mean 32 | self.prior_logsigma = prior_logsigma 33 | self.regularizer_scale = regularizer_scale 34 | self.batch_size = batch_size 35 | self.init = initializations.get(init) 36 | self.activation = activations.get(activation) 37 | self.output_dim = output_dim 38 | self.initial_weights = weights 39 | self.input_dim = input_dim 40 | if self.input_dim: 41 | kwargs['input_shape'] = (self.input_dim,) 42 | self.input = K.placeholder(ndim=2) 43 | super(VariationalDense, self).__init__(**kwargs) 44 | 45 | def build(self): 46 | input_dim = self.input_shape[-1] 47 | 48 | self.W_mean = self.init((input_dim, self.output_dim)) 49 | self.b_mean = K.zeros((self.output_dim,)) 50 | self.W_logsigma = self.init((input_dim, self.output_dim)) 51 | self.b_logsigma = K.zeros((self.output_dim,)) 52 | 53 | self.trainable_weights = [self.W_mean, self.b_mean, self.W_logsigma, 54 | self.b_logsigma] 55 | 56 | self.regularizers = [] 57 | reg = self.get_variational_regularization(self.get_input()) 58 | self.regularizers.append(reg) 59 | 60 | def get_variational_regularization(self, X): 61 | mean = self.activation(K.dot(X, self.W_mean) + self.b_mean) 62 | logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma) 63 | return GaussianKL(mean, logsigma, 64 | regularizer_scale=self.regularizer_scale, 65 | prior_mean=self.prior_mean, 66 | prior_logsigma=self.prior_logsigma) 67 | 68 | def get_mean_logsigma(self, X): 69 | mean = self.activation(K.dot(X, self.W_mean) + self.b_mean) 70 | logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma) 71 | return mean, logsigma 72 | 73 | def _get_output(self, X, train=False): 74 | mean, logsigma = self.get_mean_logsigma(X) 75 | if train: 76 | if K._BACKEND == 'theano': 77 | eps = K.random_normal((X.shape[0], self.output_dim)) 78 | else: 79 | eps = K.random_normal((self.batch_size, self.output_dim)) 80 | return mean + K.exp(logsigma) * eps 81 | else: 82 | return mean 83 | 84 | def get_output(self, train=False): 85 | X = self.get_input() 86 | return self._get_output(X, train) 87 | 88 | @property 89 | def output_shape(self): 90 | return (self.input_shape[0], self.output_dim) 91 | 92 | 93 | class TimeDistributedVAE(VariationalDense): 94 | def __init__(self, time_length, **kwargs): 95 | super(TimeDistributedVAE, self).__init__(**kwargs) 96 | self.input = K.placeholder(ndim=3) 97 | self.time_length = time_length 98 | self.batch_size = self.batch_size * self.time_length 99 | 100 | def get_variational_regularization(self, X): 101 | X = K.reshape(X, (-1, self.input_shape[-1])) 102 | mean = self.activation(K.dot(X, self.W_mean) + self.b_mean) 103 | logsigma = self.activation(K.dot(X, self.W_logsigma) + self.b_logsigma) 104 | return GaussianKL(mean, logsigma, 105 | regularizer_scale=self.regularizer_scale, 106 | prior_mean=self.prior_mean, 107 | prior_logsigma=self.prior_logsigma) 108 | 109 | def get_output(self, train=False): 110 | X = self.get_input() 111 | X = K.reshape(X, (-1, self.input_shape[-1])) 112 | outputs = self._get_output(X, train=train) 113 | outputs = K.reshape(X, (-1, self.time_length, self.output_dim)) 114 | return outputs 115 | 116 | @property 117 | def output_shape(self): 118 | return (self.input_shape[0], self.time_length, self.output_dim) 119 | -------------------------------------------------------------------------------- /seya/objectives.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras import backend as K 4 | 5 | 6 | def sum_mse(y_true, y_pred): 7 | return K.sqr(y_true - y_pred).sum() 8 | 9 | 10 | def self_cost(y_true, y_pred): 11 | # output itself is cost and must be minimized 12 | return K.sum(y_pred) + K.sum(y_true)*0 13 | 14 | 15 | def gaussianKL(dumb, y_pred): 16 | # Assumes: y_pred = T.concatenate([mean, logsigma], axis=-1) 17 | dim = y_pred.shape[1] / 2 18 | mean = y_pred[:, :dim] 19 | logsigma = y_pred[:, dim:] 20 | # See Variational Auto-Encoding Bayes by Kingma and Welling. 21 | kl = -.5 - logsigma + .5 * (mean**2 + K.exp(2 * logsigma)) 22 | return K.mean(kl, axis=-1) + 0 * K.sum(dumb) 23 | 24 | 25 | def correntropy(sigma=1.): 26 | def func(y_true, y_pred): 27 | return -K.mean(K.exp(-K.sqr(y_true - y_pred)/sigma), -1) 28 | return func 29 | 30 | 31 | def _get_kernel(X, Z, ksize): 32 | G = K.sum((K.expand_dims(X, dim=1) - 33 | K.expand_dims(Z, dim=0))**2, axis=-1) # Gram matrix 34 | G = K.exp(-G/(ksize)) / K.sqrt(2*np.pi*ksize) 35 | return G 36 | 37 | 38 | def ITLeuclidean(ksize=1.): 39 | def func(y_true, y_pred): 40 | Gxx = _get_kernel(y_true, y_true, ksize) 41 | Gzz = _get_kernel(y_pred, y_pred, ksize) 42 | Gxz = _get_kernel(y_true, y_pred, ksize) 43 | cost = K.mean(Gxx) + K.mean(Gzz) - 2*K.mean(Gxz) 44 | return cost 45 | return func 46 | 47 | 48 | def ITLcsd(ksize=1.): 49 | def func(y_true, y_pred): 50 | Gxx = _get_kernel(y_true, y_true) 51 | Gzz = _get_kernel(y_pred, y_pred) 52 | Gxz = _get_kernel(y_true, y_pred) 53 | cost = K.log(K.sqrt(K.mean(Gxx)*K.mean(Gzz)) / 54 | K.mean(Gxz)) 55 | return cost 56 | return func 57 | 58 | 59 | def gdl(img_shape, alpha=2): 60 | """Image gradient difference loss 61 | 62 | img_shape: (channels, rows, cols) shape to resize the input 63 | vectors, we assume they are input flattened in the spatial dimensions. 64 | alpha: l_alpha norm 65 | 66 | ref: Deep Multi-scale video prediction beyond mean square error, 67 | by Mathieu et. al. 68 | """ 69 | def func(y_true, y_pred): 70 | y_true = K.batch_flatten(y_true) 71 | y_pred = K.batch_flatten(y_pred) 72 | Y_true = K.reshape(y_true, (-1, ) + img_shape) 73 | Y_pred = K.reshape(y_pred, (-1, ) + img_shape) 74 | t1 = K.pow(K.abs(Y_true[:, :, 1:, :] - Y_true[:, :, :-1, :]) - 75 | K.abs(Y_pred[:, :, 1:, :] - Y_pred[:, :, :-1, :]), alpha) 76 | t2 = K.pow(K.abs(Y_true[:, :, :, :-1] - Y_true[:, :, :, 1:]) - 77 | K.abs(Y_pred[:, :, :, :-1] - Y_pred[:, :, :, 1:]), alpha) 78 | out = K.mean(K.batch_flatten(t1 + t2), -1) 79 | return out 80 | return func 81 | 82 | 83 | def gdl_video(img_shape, alpha=2): 84 | """Image gradient difference loss for videos 85 | 86 | img_shape: (time, channels, rows, cols) shape to resize the input 87 | vectors, we assume they are input flattened in the spatial dimensions. 88 | alpha: l_alpha norm 89 | 90 | ref: Deep Multi-scale video prediction beyond mean square error, 91 | by Mathieu et. al. 92 | """ 93 | def func(y_true, y_pred): 94 | Y_true = K.reshape(y_true, (-1, ) + img_shape) 95 | Y_pred = K.reshape(y_pred, (-1, ) + img_shape) 96 | t1 = K.pow(K.abs(Y_true[:, :, :, 1:, :] - Y_true[:, :, :, :-1, :]) - 97 | K.abs(Y_pred[:, :, :, 1:, :] - Y_pred[:, :, :, :-1, :]), alpha) 98 | t2 = K.pow(K.abs(Y_true[:, :, :, :, :-1] - Y_true[:, :, :, :, 1:]) - 99 | K.abs(Y_pred[:, :, :, :, :-1] - Y_pred[:, :, :, :, 1:]), alpha) 100 | out = K.mean(K.batch_flatten(t1 + t2), -1) 101 | return out 102 | return func 103 | -------------------------------------------------------------------------------- /seya/optimizers.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as T 3 | 4 | from keras.optimizers import SGD, Adam 5 | from keras.utils.theano_utils import shared_zeros 6 | 7 | 8 | def _proxOp(x, l, soft=True): 9 | if soft: 10 | return T.maximum(x-l, 0) + T.minimum(x+l, 0) 11 | else: 12 | return T.switch(T.lt(abs(x),l), x, 0) 13 | 14 | 15 | class ISTA(SGD): 16 | 17 | def __init__(self, lambdav=.1, lr=0.01, momentum=0., decay=0., 18 | nesterov=False, soft_threshold=True, *args, **kwargs): 19 | super(ISTA, self).__init__(lr, momentum, decay, nesterov, *args, 20 | **kwargs) 21 | self.lambdav = lambdav 22 | self.soft_threshold = soft_threshold 23 | 24 | def get_updates(self, params, constraints, loss): 25 | grads = self.get_gradients(loss, params) 26 | lr = self.lr * (1.0 / (1.0 + self.decay * self.iterations)) 27 | self.updates = [(self.iterations, self.iterations + 1.)] 28 | 29 | for p, g, c in zip(params, grads, constraints): 30 | m = shared_zeros(p.get_value().shape) # momentum 31 | v = self.momentum * m - lr * g # velocity 32 | self.updates.append((m, v)) 33 | 34 | if self.nesterov: 35 | new_p = p + self.momentum * v - lr * g 36 | else: 37 | new_p = p + v 38 | c_new_p = _proxOp(c(new_p), self.lr * self.lambdav, 39 | self.soft_threshold) 40 | self.updates.append((p, c_new_p)) 41 | return self.updates 42 | 43 | def get_config(self): 44 | return {"name": self.__class__.__name__, 45 | "lr": self.lr, 46 | "lambdav": self.lambdav, 47 | "momentum": self.momentum, 48 | "decay": self.decay, 49 | "nesterov": self.nesterov} 50 | 51 | 52 | class Adamista(Adam): 53 | def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, 54 | lambdav=.0005, *args, **kwargs): 55 | super(Adamista, self).__init__(lr, beta_1, beta_2, epsilon, *args, **kwargs) 56 | self.lambdav = lambdav 57 | 58 | def get_updates(self, params, constraints, loss): 59 | grads = self.get_gradients(loss, params) 60 | self.updates = [(self.iterations, self.iterations+1.)] 61 | 62 | t = self.iterations + 1 63 | lr_t = self.lr * T.sqrt(1-self.beta_2**t)/(1-self.beta_1**t) 64 | 65 | for p, g, c in zip(params, grads, constraints): 66 | m = theano.shared(p.get_value() * 0.) # zero init of moment 67 | v = theano.shared(p.get_value() * 0.) # zero init of velocity 68 | 69 | m_t = (self.beta_1 * m) + (1 - self.beta_1) * g 70 | v_t = (self.beta_2 * v) + (1 - self.beta_2) * (g**2) 71 | p_t = p - lr_t * m_t / (T.sqrt(v_t) + self.epsilon) 72 | 73 | c_p_t = _proxOp(c(p_t), self.lr * self.lambdav) 74 | self.updates.append((m, m_t)) 75 | self.updates.append((v, v_t)) 76 | self.updates.append((p, c_p_t)) # apply constraints 77 | return self.updates 78 | 79 | def get_config(self): 80 | return {"name": self.__class__.__name__, 81 | "lr": float(self.lr.get_value()), 82 | "beta_1": self.beta_1, 83 | "beta_2": self.beta_2, 84 | "epsilon": self.epsilon} 85 | -------------------------------------------------------------------------------- /seya/parzen.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file was copyed from pyleran2.distributions.parzen.py 3 | 4 | Their license is BSD clause-3: https://github.com/lisa-lab/pylearn2/ 5 | """ 6 | import numpy 7 | import theano 8 | T = theano.tensor 9 | 10 | 11 | def log_mean_exp(a): 12 | """ 13 | We need the log-likelihood, this calculates the logarithm 14 | of a Parzen window 15 | """ 16 | max_ = a.max(1) 17 | return max_ + T.log(T.exp(a - max_.dimshuffle(0, 'x')).mean(1)) 18 | 19 | 20 | def make_lpdf(mu, sigma): 21 | """ 22 | Makes a Theano function that allows the evalution of a Parzen windows 23 | estimator (aka kernel density estimator) where the Kernel is a normal 24 | distribution with stddev sigma and with points at mu. 25 | 26 | Parameters 27 | ----------- 28 | mu : numpy matrix 29 | Contains the data points over which this distribution is based. 30 | sigma : scalar 31 | The standard deviation of the normal distribution around each data \ 32 | point. 33 | Returns 34 | ------- 35 | lpdf : callable 36 | Estimator of the log of the probability density under a point. 37 | """ 38 | x = T.matrix() 39 | mu = theano.shared(mu) 40 | 41 | a = (x.dimshuffle(0, 'x', 1) - mu.dimshuffle('x', 0, 1)) / sigma 42 | 43 | E = log_mean_exp(-0.5*(a**2).sum(2)) 44 | 45 | Z = mu.shape[1] * T.log(sigma * numpy.sqrt(numpy.pi * 2)) 46 | 47 | return theano.function([x], E - Z) 48 | 49 | 50 | class ParzenWindows(object): 51 | """ 52 | Parzen Window estimation and log-likelihood calculator. 53 | This is usually used to test generative models as follows: 54 | 1 - Get 10k samples from the generative model 55 | 2 - Contruct a ParzenWindows object with the samples from 1 56 | 3 - Test the log-likelihood on the test set 57 | 58 | Parameters 59 | ---------- 60 | samples : numpy matrix 61 | See description for make_lpdf 62 | sigma : scalar 63 | See description for make_lpdf 64 | """ 65 | def __init__(self, samples, sigma): 66 | # just keeping these for debugging/examination, not needed 67 | self._samples = samples 68 | self._sigma = sigma 69 | 70 | self.lpdf = make_lpdf(samples, sigma) 71 | 72 | def get_ll(self, x, batch_size=10): 73 | """ 74 | Evaluates the log likelihood of a set of datapoints with respect to the 75 | probability distribution. 76 | Parameters 77 | ---------- 78 | x : numpy matrix 79 | The set of points for which you want to evaluate the log \ 80 | likelihood. 81 | """ 82 | inds = range(x.shape[0]) 83 | n_batches = int(numpy.ceil(float(len(inds)) / batch_size)) 84 | 85 | lls = [] 86 | for i in range(n_batches): 87 | lls.extend(self.lpdf(x[inds[i::n_batches]])) 88 | 89 | return numpy.array(lls).mean() 90 | -------------------------------------------------------------------------------- /seya/preprocessing/README.md: -------------------------------------------------------------------------------- 1 | # Preprocessing 2 | 3 | Preprocess data using external libraries. 4 | 5 | ### Eventual Dependencies 6 | * natsort: `pip install natsort` 7 | * Caffe 8 | * [sklearn-theano](https://github.com/sklearn-theano/sklearn-theano) 9 | * [imageio](https://github.com/imageio/imageio.git) 10 | 11 | ### Converting a video dataset to hdf5 12 | Dependencies: `avconv` or `ffmpeg`, `h5py`. 13 | Assume the following folder structure: 14 | ``` 15 | root 16 | |-dataset 17 | |-video_class_1 18 | |-video_1.avi 19 | |-video_2.avi 20 | |-video_class_2 21 | ... 22 | ``` 23 | 24 | We want to convert all the videos to a single hdf5 tensor. That dataset will be 25 | called `'data'` inside the hdf5. To do that, 26 | we use the `convert_dataset_to_hdf5` that calls `convert_videos_to_hdf5` 27 | recursively on each class folder. Also, on the same hdf5 we want a second matrix, called 28 | `'labels'`, with the labels of each video given by the folder name. From the `root` directory we do the 29 | following: 30 | ```python 31 | from videos_to_hdf5 import convert_dataset_to_hdf5 32 | 33 | def label_cbk(f, return_len=False): 34 | # define callback that calculates label from file name. If we don't 35 | # pass a label callback, the next function will only save the frames. 36 | if return_len: 37 | # label_callback must know the number of classes in the dataset 38 | return 51 39 | dirname = f.split('/')[1] 40 | names = [name for name in os.listdir('./dataset') if os.path.isdir(os.path.join('./dataset', name))] 41 | label = np.asarray([w == dirname for w in names]) 42 | return label.astype('float32') 43 | 1 44 | convert_dataset_to_hdf5('final_file.hdf5', 'dataset', ext='*.avi', 45 | label_callback=label_cbk, convert='avconv') 46 | ``` 47 | 48 | The default configuration stores only the first 20 frames of each video center cropped 49 | to 224x224. 50 | If we need to filter the videos to generate train and test splits, it is easier to go through the hdf5 than 51 | the original video dataset. In the example above, the videos order will be 52 | saved to `final_file.hdf5.txt`. 53 | 54 | Note: this code does not provide any exception treatment, use it at your 55 | own risk or consider modifying it if you your dataset is too large. 56 | -------------------------------------------------------------------------------- /seya/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | from features_to_hdf5 import features_to_hdf5 2 | from videos_to_hdf5 import * 3 | -------------------------------------------------------------------------------- /seya/preprocessing/features_to_hdf5.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import h5py 4 | import random 5 | 6 | from PIL import Image 7 | from glob import glob 8 | from sklearn_theano.feature_extraction import OverfeatTransformer as fext 9 | 10 | from keras.utils.generic_utils import Progbar 11 | 12 | 13 | def features_to_hdf5(input_path, output_path='features.hdf5', 14 | feature_extractor=fext, label_callback=None, 15 | img_size=(231, 231)): 16 | ''' Extract features using a pretrained model and store the results to hdf5. 17 | 18 | Parameteres: 19 | ------------ 20 | input_path: str, path to images with appropriate extensions example 21 | "./path/*.jpg" or "./path/*.png" 22 | 23 | output_path: str, path to the resulting hdf5 file. We also create a text 24 | file with the name and order of the files saved to hdf5. 25 | 26 | label_callback: optional function, a callback to get a label from each file 27 | name. Should return a numpy array with labels, even when 28 | using a single binary label. We use the `len` of the 29 | returned value to allocate the labels matrix. 30 | 31 | feature_extractor: sklearn-theano transformer, default: OverfeatTransformer 32 | 33 | img_size: list (int, int): number of rows and columns for the converted 34 | image 35 | 36 | Notes: 37 | ------ 38 | Input files are naturally sorted with natsort and features are saved on 39 | that order. 40 | 41 | This script creates a temporary file called tttemp.hdf5 and deletes 42 | afterward. 43 | 44 | This code converts one image at a time, for 45 | really large datasets you may want to modify it to convert images in batches. 46 | ''' 47 | files = glob(input_path) 48 | random.shuffle(files) 49 | output_txt = file(output_path + '.txt', 'w') 50 | tf = fext(output_layers=(-3,)) 51 | 52 | count = 0 53 | 54 | with h5py.File('tttemp.hdf5', 'w') as h: 55 | feature_size = _load_and_convert(files[0], tf, img_size).shape[-1] 56 | X = h.create_dataset('features', (len(files), feature_size), dtype='f') 57 | if label_callback is not None: 58 | label_size = len(label_callback('test')) 59 | y = h.create_dataset('labels', (len(files), label_size), dtype='f') 60 | 61 | print("Extracting features of a total of {} files...".format(len(files))) 62 | progbar = Progbar(len(files)) 63 | for f in files: 64 | try: 65 | # Some of these files are not reading right... 66 | im2 = _load_and_convert(f, tf, img_size) 67 | X[count] = im2 68 | if label_callback is not None: 69 | y[count] = label_callback(f) 70 | count += 1 71 | output_txt.write(f + "\n") 72 | if count % 100 == 0: 73 | progbar.add(100) 74 | 75 | except: 76 | print("Problem converting: {}".format(f)) 77 | continue 78 | 79 | print("Successfully transformed {} files".format(count)) 80 | with h5py.File(output_path, 'w') as g: 81 | XX = g.create_dataset('features', (count, feature_size), dtype='f') 82 | if label_callback is not None: 83 | yy = g.create_dataset('labels', (count, label_size), dtype='f') 84 | yy[:] = y[:count] 85 | 86 | progbar = Progbar(count) 87 | print("Cleaning up...") 88 | for i in range(count): 89 | progbar.add(1) 90 | XX[i] = X[i] 91 | del tf 92 | output_txt.close() 93 | os.system('rm tttemp.hdf5') 94 | 95 | 96 | def _load_and_convert(f, tf, img_size): 97 | im = Image.open(f, 'r') 98 | I = np.array(im.resize(img_size)) 99 | return tf.transform(I[np.newaxis])[0] 100 | -------------------------------------------------------------------------------- /seya/preprocessing/image.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import keras.backend as K 3 | from seya.layers.imageproc import NormLayer 4 | from seya.utils import batchwise_function 5 | 6 | 7 | def zca_whitening(inputs, epsilon=0.1): 8 | sigma = np.dot(inputs.T, inputs)/inputs.shape[0] # Correlation matrix 9 | U, S, V = np.linalg.svd(sigma) # Singular Value Decomposition 10 | epsilon = 0.1 # Whitening constant, it prevents division by zero 11 | ZCAMatrix = np.dot(np.dot(U, np.diag(1.0/np.sqrt(S + epsilon))), U.T) # ZCA Whitening matrix 12 | return np.dot(ZCAMatrix, inputs.T).T, ZCAMatrix # Data whitening 13 | 14 | 15 | def contrast_normalization(inputs, batch_size=100): 16 | norm = NormLayer() 17 | lcn = K.function([norm.get_input()], norm.get_output()) 18 | return batchwise_function(lcn, inputs, batch_size=batch_size) 19 | 20 | 21 | def global_normalization(inputs, batch_size=100): 22 | norm = NormLayer(method="gcn") 23 | lcn = K.function([norm.get_input()], norm.get_output()) 24 | return batchwise_function(lcn, inputs, batch_size=batch_size) 25 | -------------------------------------------------------------------------------- /seya/preprocessing/videos_to_hdf5.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | # import imageio 4 | import os 5 | import h5py 6 | from glob import glob 7 | 8 | import subprocess 9 | from scipy.misc import imread, imresize 10 | 11 | 12 | def convert_videos_to_hdf5(hdf5file, filepath, 13 | ext='*.avi', 14 | img_shape=(224, 224), 15 | frame_range=range(0, 20), 16 | row_range=slice(120-112, 120+112), 17 | col_range=slice(160-112, 160+112), 18 | label_callback=None, 19 | converter='avconv'): 20 | 21 | """ 22 | Convert all the videos of a folder to images and dump the images to an hdf5 23 | file. 24 | 25 | Parameters: 26 | ----------- 27 | hdf5file: str, path to output hdf5 file 28 | filepath: str, path to folder with videos 29 | ext: str, video extensions (default *.avi) 30 | img_shape: tuple, (row, col) size of the image crop of video 31 | frame_range: list, desired frames of the video 32 | row_range: slice, slice of the image rows 33 | col_rance: slice, slice of the image cols 34 | 35 | Results: 36 | -------- 37 | An hdf5 file with videos stored as array 38 | 39 | """ 40 | rlen = row_range.stop - row_range.start 41 | clen = col_range.stop - col_range.start 42 | files = glob(os.path.join(filepath, ext)) 43 | with h5py.File(hdf5file, 'w') as h5: 44 | # create datasets 45 | X = h5.create_dataset('data', (len(files), len(frame_range), 46 | rlen, clen, 3), dtype='f') 47 | if label_callback is not None: 48 | label_size = label_callback('test', return_len=True) 49 | y = h5.create_dataset('labels', (len(files), label_size), dtype='f') 50 | 51 | for c1, f in enumerate(files): 52 | process = subprocess.Popen('mkdir {}'.format(f[:-4]), shell=True, stdout=subprocess.PIPE) 53 | process.wait() 54 | cmd = converter + " -i {0} -r 1/1 {1}/%03d.jpg".format(f, f[:-4]) 55 | process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) 56 | process.wait() 57 | 58 | imgs = sorted(glob(os.path.join(f[:-4], '*.jpg'))) 59 | for c2, im in enumerate(imgs[:frame_range[-1]]): 60 | I = imread(im)[row_range, col_range] 61 | if I.shape[:2] != img_shape: 62 | I = imresize(imread(im), img_shape) 63 | X[c1, c2] = I 64 | 65 | if label_callback is not None: 66 | y[c1] = label_callback(f) 67 | 68 | process = subprocess.Popen('rm -rf {}'.format(f[:-4]), shell=True, stdout=subprocess.PIPE) 69 | process.wait() 70 | 71 | with open(hdf5file+'.txt', 'w') as file_order: 72 | for f in files: 73 | file_order.write(f+'\n') 74 | 75 | 76 | def convert_dataset_to_hdf5(hdf5file, filepath, 77 | ext='*.avi', 78 | img_shape=(224, 224), 79 | frame_range=range(0, 20), 80 | row_range=slice(120-112, 120+112), 81 | col_range=slice(160-112, 160+112), 82 | label_callback=None): 83 | """ 84 | Convert all the videos of a dataset to images and dump the images to an hdf5 85 | file. Uses convert_videos_to_hdf5 recursively on folder deep in a file 86 | structure. 87 | 88 | Parameters: 89 | ----------- 90 | hdf5file: str, path to output hdf5 file 91 | filepath: str, path to folder with videos 92 | ext: str, video extensions (default *.avi) 93 | img_shape: tuple, (row, col) size of the image crop of video 94 | frame_range: list, desired frames of the video 95 | row_range: slice, slice of the image rows 96 | col_rance: slice, slice of the image cols 97 | 98 | Results: 99 | -------- 100 | An hdf5 file with videos stored as array 101 | 102 | """ 103 | dirs = [d for d in glob(os.path.join(filepath, '*')) if os.path.isdir(d)] 104 | files = glob(os.path.join(filepath, '*', ext)) 105 | rlen = row_range.stop - row_range.start 106 | clen = col_range.stop - col_range.start 107 | 108 | process = subprocess.Popen('mkdir hdf5_files_temp', shell=True, stdout=subprocess.PIPE) 109 | process.wait() 110 | 111 | start = 0 112 | with h5py.File(hdf5file, 'w') as h5: 113 | # create datasets 114 | X = h5.create_dataset('data', (len(files), len(frame_range), 115 | rlen, clen, 3), dtype='f') 116 | if label_callback is not None: 117 | label_size = label_callback('test', return_len=True) 118 | y = h5.create_dataset('labels', (len(files), label_size), dtype='f') 119 | 120 | print('Converting {0} directories, {1} files'.format(len(dirs), len(files))) 121 | for i, d in enumerate(dirs): 122 | print('Converting: {0}/{1} {2}'.format(i+1, len(dirs), d)) 123 | convert_videos_to_hdf5('hdf5_files_temp.h5', d, 124 | ext, 125 | img_shape, 126 | frame_range, 127 | row_range, 128 | col_range, 129 | label_callback) 130 | with h5py.File('hdf5_files_temp.h5', 'r') as temp: 131 | n_samples = temp['data'].shape[0] 132 | X[i+start:i+start+n_samples] = temp['data'] 133 | if label_callback is not None: 134 | y[i+start:i+start+n_samples] = temp['labels'] 135 | start += n_samples 136 | ''' 137 | WORK IN PROGRESSS: waiting for imageio to fix some issues 138 | 139 | def videos_to_hdf5(hdf5file, videospath, 140 | label_callback=None, 141 | ext='*.avi', 142 | frame_range=range(0, 20), 143 | row_range=slice(120-112, 120+112), 144 | col_range=slice(160-112, 160+112)): 145 | """ 146 | Convert all the videos of a nested folder directory of videos 147 | 148 | Problems: 149 | --------- 150 | imageio has problems reading too many videos. 151 | 152 | """ 153 | rlen = row_range.stop - row_range.start 154 | clen = col_range.stop - col_range.start 155 | files = glob(os.path.join(videospath, '*', ext)) 156 | with h5py.File(hdf5file, 'w') as h5: 157 | # create datasets 158 | X = h5.create_dataset('data', (len(files), len(frame_range), 159 | rlen, clen, 3), dtype='f') 160 | 161 | cPickle.dump(files, file('filename.pkl', 'w'), -1) 162 | 163 | if label_callback is not None: 164 | label_size = len(label_callback('test')) 165 | y = h5.create_dataset('labels', (len(files), label_size), dtype='f') 166 | # read files 167 | for c, f in enumerate(files): 168 | with imageio.get_reader(f, 'ffmpeg') as vid: 169 | for i, fr in enumerate(frame_range): 170 | X[c, i] = vid.get_data(fr)[row_range, col_range].astype('float') 171 | if label_callback is not None: 172 | y[c] = label_callback(f) 173 | ''' 174 | -------------------------------------------------------------------------------- /seya/regularizers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras import backend as K 4 | from keras.regularizers import Regularizer 5 | 6 | 7 | class GaussianKL(Regularizer): 8 | """ KL-divergence between two gaussians. 9 | Useful for Variational AutoEncoders. 10 | Use this as an activation regularizer 11 | 12 | Parameters: 13 | ----------- 14 | mean, logsigma: parameters of the input distributions 15 | prior_mean, prior_logsigma: paramaters of the desired distribution (note the 16 | log on logsigma) 17 | regularizer_scale: Rescales the regularization cost. Keep this 1 for most cases. 18 | 19 | Notes: 20 | ------ 21 | See seya.layers.variational.VariationalDense for usage example 22 | 23 | """ 24 | def __init__(self, mean, logsigma, prior_mean=0, prior_logsigma=0, 25 | regularizer_scale=1): 26 | self.regularizer_scale = regularizer_scale 27 | self.mean = mean 28 | self.logsigma = logsigma 29 | self.prior_mean = prior_mean 30 | self.prior_logsigma = prior_logsigma 31 | super(GaussianKL, self).__init__() 32 | 33 | def __call__(self, loss): 34 | # See Variational Auto-Encoding Bayes by Kingma and Welling. 35 | mean, logsigma = self.mean, self.logsigma 36 | kl = (self.prior_logsigma - logsigma + 37 | 0.5 * (-1 + K.exp(2 * logsigma) + (mean - self.prior_mean) ** 2) / 38 | K.exp(2 * self.prior_logsigma)) 39 | loss += K.mean(kl) * self.regularizer_scale 40 | return loss 41 | 42 | def get_config(self): 43 | return {"name": self.__class__.__name__} 44 | 45 | 46 | class ExponentialKL(Regularizer): 47 | """ KL-divergence between two exponentially distrubted random variables. 48 | Useful for Variational AutoEncoders. 49 | Use this as an activation regularizer 50 | 51 | Parameters: 52 | ----------- 53 | _lambda: parameter of the input distributions 54 | prior_lambda: paramater of the desired distribution (scale or rate) 55 | regularizer_scale: Rescales the regularization cost. Keep this 1 for most cases. 56 | 57 | Notes: 58 | ------ 59 | See seya.layers.variational.VariationalExp for usage example 60 | 61 | """ 62 | def __init__(self, _lambda, prior_lambda=1., 63 | regularizer_scale=1): 64 | self.regularizer_scale = regularizer_scale 65 | self._lambda = _lambda 66 | self.prior_lambda = prior_lambda 67 | super(ExponentialKL, self).__init__() 68 | 69 | def __call__(self, loss): 70 | # See Variational Auto-Encoding Bayes by Kingma and Welling. 71 | kl = (K.log(self._lambda) - K.log(self.prior_lambda) + 72 | self.prior_lambda/self._lambda - 1) 73 | loss += K.mean(kl) * self.regularizer_scale 74 | return loss 75 | 76 | def get_config(self): 77 | return {"name": self.__class__.__name__} 78 | 79 | 80 | class LambdaRegularizer(Regularizer): 81 | def __init__(self, cost): 82 | super(LambdaRegularizer, self).__init__() 83 | self.cost = cost 84 | 85 | def __call__(self, loss): 86 | loss += self.cost 87 | return loss 88 | 89 | def get_config(self): 90 | return {"name": self.__class__.__name__} 91 | 92 | 93 | class CorrentropyActivityRegularizer(Regularizer): 94 | def __init__(self, scale, sigma=1.): 95 | super(CorrentropyActivityRegularizer, self).__init__() 96 | self.sigma = sigma 97 | self.scale = scale 98 | 99 | def set_layer(self, layer): 100 | self.layer = layer 101 | 102 | def __call__(self, loss): 103 | output = self.layer.get_output(True) 104 | loss += self.scale * correntropy(output, self.sigma) 105 | return loss 106 | 107 | def get_config(self): 108 | return {"name": self.__class__.__name__, 109 | "sigma": self.sigma, 110 | "scale": self.scale} 111 | 112 | 113 | class CorrentropyWeightRegularizer(Regularizer): 114 | def __init__(self, scale, sigma=1): 115 | super(CorrentropyWeightRegularizer, self).__init__() 116 | self.sigma = sigma 117 | self.scale = scale 118 | 119 | def set_param(self, p): 120 | self.p = p 121 | 122 | def __call__(self, loss): 123 | loss += self.scale * correntropy(self.p, self.sigma) 124 | return loss 125 | 126 | def get_config(self): 127 | return {"name": self.__class__.__name__, 128 | "sigma": self.sigma, 129 | "scale": self.scale} 130 | 131 | 132 | def correntropy(x, sigma): 133 | return -K.sum(K.mean(K.exp(x**2/sigma), axis=0)) / K.sqrt(2*np.pi*sigma) 134 | -------------------------------------------------------------------------------- /seya/seya.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /seya/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import theano 3 | import theano.tensor as T 4 | from theano.sandbox.rng_mrg import MRG_RandomStreams 5 | from keras import backend as K 6 | from keras.utils.generic_utils import Progbar 7 | 8 | 9 | def batched_dot(input, W_list, b_list, sizes_list): 10 | X = input 11 | W = K.concatenate(W_list, axis=-1) 12 | b = K.concatenate(b_list, axis=-1) 13 | Y = K.dot(X, W) + b 14 | sl = [0, ] + list(np.cumsum(sizes_list)) 15 | return [Y[:, sl[i]:sl[i+1]] for i in range(len(sizes_list))] 16 | 17 | 18 | def batchwise_function(func, X, batch_size=100, verbose=1): 19 | # Y = [func([X[i*batch_size:(i+1)*batch_size]]) for i in range( 20 | # 0, X.shape[0]//batch_size)] 21 | Y = [] 22 | progbar = Progbar(X.shape[0]) 23 | for i in range(0, X.shape[0] // batch_size): 24 | Y += [func([X[i*batch_size:(i+1)*batch_size]])] 25 | if verbose > 0: 26 | progbar.add(batch_size) 27 | return np.concatenate(Y, axis=0) 28 | 29 | 30 | def alloc_ones_matrix(*dims): 31 | return T.alloc(np.cast[theano.config.floatX](1.), *dims) 32 | 33 | 34 | def theano_rng(seed=123): 35 | return MRG_RandomStreams(seed=seed) 36 | 37 | 38 | def apply_layer(layer, X, train=False): 39 | tmp = layer.get_input 40 | layer.get_input = lambda _: X 41 | Y = layer.get_output(train=train) 42 | layer.get_input = tmp 43 | return Y 44 | 45 | 46 | def apply_model(model, X): 47 | tmp = model.layers[0].get_input 48 | model.layers[0].get_input = lambda _: X 49 | Y = model.get_output() 50 | model.layers[0].get_input = tmp 51 | 52 | return Y 53 | 54 | 55 | def s2s_to_s2t(sequences, targets): 56 | ''' 57 | Transforms as sequence to sequence problem to a sequence to target. 58 | It does so by replicating the input dataset a lot. 59 | So use this only with small datasets. 60 | Also, there is no way of passing the hidden states from one batch to another, 61 | thus this is not the best way to solve this problem. 62 | ''' 63 | X = [] 64 | Xrev = [] # reversed dataset 65 | y = [] 66 | for seq, tar in zip(sequences, targets): 67 | if not len(seq) == len(tar): 68 | raise ValueError("Sequences and Targets must have the same length.") 69 | for i in range(len(seq)): 70 | X.append(seq[:i+1]) 71 | if i == 0: 72 | Xrev.append(seq[::-1]) 73 | else: 74 | Xrev.append(seq[:i-1:-1]) 75 | # X.append(seq[:i+1]) 76 | # Xrev.append(seq[:i-1:-1]) 77 | y.append(tar[i]) 78 | return X, Xrev, y 79 | 80 | 81 | def pad_md_sequences(sequences, maxlen=None, dtype='int32', padding='pre', truncating='pre', value=0.): 82 | '''Pad multi-dimensional sequence 83 | Similar to keras.preprocesing.sequence but supports a third dimension: 84 | 85 | Pad each sequence to the same length: 86 | the length of the longuest sequence. 87 | 88 | If maxlen is provided, any sequence longer 89 | than maxlen is truncated to maxlen. Truncation happens off either the beginning (default) or 90 | the end of the sequence. 91 | 92 | Supports post-padding and pre-padding (default). 93 | 94 | ''' 95 | lengths = [len(s) for s in sequences] 96 | dim = sequences[0].shape[-1] # the third dimension is assumed equal for all elements 97 | 98 | nb_samples = len(sequences) 99 | if maxlen is None: 100 | maxlen = np.max(lengths) 101 | 102 | x = (np.ones((nb_samples, maxlen, dim)) * value).astype(dtype) 103 | for idx, s in enumerate(sequences): 104 | if truncating == 'pre': 105 | trunc = s[-maxlen:] 106 | elif truncating == 'post': 107 | trunc = s[:maxlen] 108 | else: 109 | raise ValueError("Truncating type '%s' not understood" % padding) 110 | 111 | if padding == 'post': 112 | x[idx, :len(trunc)] = trunc 113 | elif padding == 'pre': 114 | x[idx, -len(trunc):] = trunc 115 | else: 116 | raise ValueError("Padding type '%s' not understood" % padding) 117 | return x 118 | 119 | 120 | def diff_abs(z): 121 | return T.sqrt(T.sqr(z)+1e-6) 122 | 123 | 124 | def unroll_scan(fn, sequences, outputs_info, non_sequences, n_steps, 125 | go_backwards=False): 126 | """ 127 | Copy-pasted from Lasagne 128 | 129 | Helper function to unroll for loops. Can be used to unroll theano.scan. 130 | The parameter names are identical to theano.scan, please refer to here 131 | for more information. 132 | Note that this function does not support the truncate_gradient 133 | setting from theano.scan. 134 | Parameters 135 | ---------- 136 | fn : function 137 | Function that defines calculations at each step. 138 | sequences : TensorVariable or list of TensorVariables 139 | List of TensorVariable with sequence data. The function iterates 140 | over the first dimension of each TensorVariable. 141 | outputs_info : list of TensorVariables 142 | List of tensors specifying the initial values for each recurrent 143 | value. 144 | non_sequences: list of TensorVariables 145 | List of theano.shared variables that are used in the step function. 146 | n_steps: int 147 | Number of steps to unroll. 148 | go_backwards: bool 149 | If true the recursion starts at sequences[-1] and iterates 150 | backwards. 151 | Returns 152 | ------- 153 | List of TensorVariables. Each element in the list gives the recurrent 154 | values at each time step. 155 | """ 156 | if not isinstance(sequences, (list, tuple)): 157 | sequences = [sequences] 158 | 159 | # When backwards reverse the recursion direction 160 | counter = range(n_steps) 161 | if go_backwards: 162 | counter = counter[::-1] 163 | 164 | output = [] 165 | prev_vals = outputs_info 166 | for i in counter: 167 | step_input = [s[i] for s in sequences] + prev_vals + non_sequences 168 | out_ = fn(*step_input) 169 | # The returned values from step can be either a TensorVariable, 170 | # a list, or a tuple. Below, we force it to always be a list. 171 | if isinstance(out_, T.TensorVariable): 172 | out_ = [out_] 173 | if isinstance(out_, tuple): 174 | out_ = list(out_) 175 | output.append(out_) 176 | 177 | prev_vals = output[-1] 178 | 179 | # iterate over each scan output and convert it to same format as scan: 180 | # [[output11, output12,...output1n], 181 | # [output21, output22,...output2n],...] 182 | output_scan = [] 183 | for i in range(len(output[0])): 184 | l = map(lambda x: x[i], output) 185 | output_scan.append(T.stack(*l)) 186 | 187 | return output_scan 188 | 189 | 190 | def rnn_states(step_function, inputs, initial_states, 191 | go_backwards=False, masking=True): 192 | '''Iterates over the time dimension of a tensor. 193 | Parameters 194 | ---------- 195 | inputs: tensor of temporal data of shape (samples, time, ...) 196 | (at least 3D). 197 | step_function: 198 | Parameters: 199 | input: tensor with shape (samples, ...) (no time dimension), 200 | representing input for the batch of samples at a certain 201 | time step. 202 | states: list of tensors. 203 | Returns: 204 | output: tensor with shape (samples, ...) (no time dimension), 205 | new_states: list of tensors, same length and shapes 206 | as 'states'. 207 | initial_states: tensor with shape (samples, ...) (no time dimension), 208 | containing the initial values for the states used in 209 | the step function. 210 | go_backwards: boolean. If True, do the iteration over 211 | the time dimension in reverse order. 212 | masking: boolean. If true, any input timestep inputs[s, i] 213 | that is all-zeros will be skipped (states will be passed to 214 | the next step unchanged) and the corresponding output will 215 | be all zeros. 216 | Returns 217 | ------- 218 | A tuple (last_output, outputs, new_states). 219 | last_output: the latest output of the rnn, of shape (samples, ...) 220 | outputs: tensor with shape (samples, time, ...) where each 221 | entry outputs[s, t] is the output of the step function 222 | at time t for sample s. 223 | new_states: list of tensors, latest states returned by 224 | the step function, of shape (samples, ...). 225 | ''' 226 | inputs = inputs.dimshuffle((1, 0, 2)) 227 | 228 | def _step(input, *states): 229 | output, new_states = step_function(input, states) 230 | if masking: 231 | # if all-zero input timestep, return 232 | # all-zero output and unchanged states 233 | switch = T.any(input, axis=-1, keepdims=True) 234 | output = T.switch(switch, output, 0. * output) 235 | return_states = [] 236 | for state, new_state in zip(states, new_states): 237 | return_states.append(T.switch(switch, new_state, state)) 238 | return [output] + return_states 239 | else: 240 | return [output] + new_states 241 | 242 | results, _ = theano.scan( 243 | _step, 244 | sequences=inputs, 245 | outputs_info=[None] + initial_states, 246 | go_backwards=go_backwards) 247 | 248 | # deal with Theano API inconsistency 249 | if type(results) is list: 250 | states = results[1:] 251 | else: 252 | states = [] 253 | 254 | return states 255 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/test_apply.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """Test seya.layers.recurrent module""" 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | import numpy as np 8 | import theano 9 | 10 | from numpy.testing import assert_allclose 11 | 12 | from keras import backend as K 13 | from keras.layers.core import Dense 14 | from keras.models import Sequential 15 | 16 | from seya.utils import apply_model 17 | floatX = K.common._FLOATX 18 | 19 | 20 | class TestApply(unittest.TestCase): 21 | """Test apply methods""" 22 | 23 | def test_apply_model(self): 24 | """Test keras.models.Sequential.__call__""" 25 | nb_samples, input_dim, output_dim = 3, 10, 5 26 | model = Sequential() 27 | model.add(Dense(output_dim=output_dim, input_dim=input_dim)) 28 | model.compile('sgd', 'mse') 29 | 30 | X = K.placeholder(ndim=2) 31 | Y = apply_model(model, X) 32 | F = theano.function([X], Y) 33 | 34 | x = np.random.randn(nb_samples, input_dim).astype(floatX) 35 | y1 = F(x) 36 | y2 = model.predict(x) 37 | # results of __call__ should match model.predict 38 | assert_allclose(y1, y2) 39 | 40 | 41 | if __name__ == '__main__': 42 | theano.config.exception_verbosity = 'high' 43 | unittest.main(verbosity=2) 44 | -------------------------------------------------------------------------------- /tests/test_base.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #from keras.models import Sequential 4 | #from seya.layers.base import Replicator 5 | 6 | 7 | # def test_replicator(): 8 | # ''' Replicator is deprecated, use keras.layers.core.RepeatVector instead''' 9 | # model = Sequential() 10 | # model.add(Replicator(10)) 11 | # model.compile(loss="mse", optimizer="sgd") 12 | # 13 | # x = np.ones((2, 2)) 14 | # y = model.predict(x).astype('float64') 15 | # np.testing.assert_allclose(y, np.ones((2, 10, 2))) 16 | -------------------------------------------------------------------------------- /tests/test_containers.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Recursive container is temporally disabled 3 | ''' 4 | 5 | # from __future__ import print_function 6 | # import numpy as np 7 | # from numpy.testing import assert_allclose 8 | # 9 | # import unittest 10 | # import theano.tensor as T 11 | # import theano 12 | # from theano import function 13 | # from keras.layers.core import Activation, Dense 14 | # from keras.models import Sequential 15 | # from keras.layers.recurrent import SimpleRNN 16 | # from seya.layers.containers import Recursive 17 | # from seya.layers.base import Lambda 18 | # floatX = theano.config.floatX 19 | 20 | # class TestRecursive(unittest.TestCase): 21 | # def __init__(self, *args, **kwargs): 22 | # super(TestRecursive, self).__init__(*args, **kwargs) 23 | # self.input_dim = 2 24 | # self.state_dim = 2 25 | # self.model = Recursive(return_sequences=True) 26 | # self.model.add_input('input', ndim=3) # Input is 3D tensor 27 | # self.model.add_state('h', dim=self.state_dim) 28 | # self.model.add_node(Dense(self.input_dim + self.state_dim, self.state_dim, 29 | # init='one'), name='rec', 30 | # inputs=['input', 'h'], 31 | # return_state='h') 32 | # self.model.add_node(Activation('linear'), name='out', input='rec', 33 | # create_output=True) 34 | # 35 | # self.model2 = Sequential() 36 | # self.model2.add(SimpleRNN(input_dim=self.input_dim, activation='linear', 37 | # inner_init='one', 38 | # output_dim=self.state_dim, init='one', 39 | # return_sequences=True)) 40 | # 41 | # def test_step(self): 42 | # XX = T.matrix() 43 | # HH = T.matrix() 44 | # A = self.model._step(XX, HH) 45 | # F = function([XX, HH], A, allow_input_downcast=True) 46 | # x = np.ones((1, 2)) 47 | # h = np.ones((1, 2)) 48 | # y = F(x, h) 49 | # r = np.asarray([[4., 4.]]) 50 | # assert_allclose([r, r], y) 51 | # 52 | # def test_get_get_output(self): 53 | # X = self.model.get_input() 54 | # Y = self.model._get_output() 55 | # F = function([X], Y, allow_input_downcast=True) 56 | # 57 | # x = np.ones((3, 5, self.input_dim)).astype(floatX) 58 | # y = F(x) 59 | # print(y) 60 | # 61 | # X2 = self.model2.get_input() 62 | # Y2 = self.model2.get_output() 63 | # F2 = function([X2], Y2) 64 | # y2 = F2(x) 65 | # 66 | # assert_allclose(y2, y[1]) 67 | # 68 | # 69 | # class TestOrthoRNN(unittest.TestCase): 70 | # def __init__(self, *args, **kwargs): 71 | # super(TestOrthoRNN, self).__init__(*args, **kwargs) 72 | # self.input_dim = 2 73 | # self.state_dim = 2 74 | # self.model = Recursive(return_sequences=True) 75 | # self.model.add_input('input', ndim=3) # Input is 3D tensor 76 | # self.model.add_state('h', dim=self.state_dim) 77 | # self.model.add_node(Dense(self.input_dim, self.state_dim, 78 | # init='one'), name='i2h', 79 | # input='input') 80 | # self.model.add_node(Dense(self.state_dim, self.state_dim, 81 | # init='orthogonal'), name='h2h', 82 | # inputs='h') 83 | # self.model.add_node(Lambda(lambda x: x), name='rec', 84 | # inputs=['i2h', 'h2h'], merge_mode='sum', 85 | # return_state='h', 86 | # create_output=True) 87 | # 88 | # self.model2 = Sequential() 89 | # self.model2.add(SimpleRNN(input_dim=self.input_dim, activation='linear', 90 | # inner_init='one', 91 | # output_dim=self.state_dim, init='one', 92 | # return_sequences=True)) 93 | # U = self.model.nodes['h2h'].W.get_value() 94 | # self.model2.layers[0].U.set_value(U) 95 | # 96 | # def test_step(self): 97 | # XX = T.matrix() 98 | # HH = T.matrix() 99 | # A = self.model._step(XX, HH) 100 | # F = function([XX, HH], A, allow_input_downcast=True) 101 | # x = np.ones((1, 2)) 102 | # h = np.ones((1, 2)) 103 | # y = F(x, h) 104 | # assert(y[-1].shape == (1, 2)) 105 | # 106 | # def test__get_output(self): 107 | # X = self.model.get_input() 108 | # Y = self.model._get_output() 109 | # F = function([X], Y, allow_input_downcast=True) 110 | # 111 | # x = np.ones((3, 5, self.input_dim)).astype(floatX) 112 | # y = F(x) 113 | # print(y) 114 | # 115 | # X2 = self.model2.get_input() 116 | # Y2 = self.model2.get_output() 117 | # F2 = function([X2], Y2) 118 | # y2 = F2(x) 119 | # 120 | # assert_allclose(y2, y[-1]) 121 | # 122 | # def test_get_output(self): 123 | # X = self.model.get_input() 124 | # Y = self.model.get_output() 125 | # F = function([X], Y, allow_input_downcast=True) 126 | # 127 | # x = np.ones((3, 5, self.input_dim)).astype(floatX) 128 | # y = F(x) 129 | # 130 | # X2 = self.model2.get_input() 131 | # Y2 = self.model2.get_output() 132 | # F2 = function([X2], Y2) 133 | # y2 = F2(x) 134 | # 135 | # print("y-length: {}".format(len(y))) 136 | # assert_allclose(y2, y) 137 | # 138 | # if __name__ == '__main__': 139 | # print('Test Recursive container') 140 | # unittest.main() 141 | -------------------------------------------------------------------------------- /tests/test_conv_rnn.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """Test seya.layers.recurrent module""" 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | import numpy as np 8 | 9 | from keras.models import Sequential 10 | from keras.layers.core import TimeDistributedDense 11 | from keras.layers.convolutional import Convolution2D 12 | from seya.layers.conv_rnn import ConvRNN, ConvGRU, TimeDistributedModel 13 | 14 | 15 | class TestConvRNNs(unittest.TestCase): 16 | """Test seya.layers.conv_rnn layer""" 17 | 18 | def test_conv_rnn(self): 19 | """Just check that the ConvRNN layer can compile and run""" 20 | nb_samples, timesteps, ndim, filter_dim = 5, 10, 28, 3 21 | input_flat = ndim ** 2 22 | layer = ConvRNN(filter_dim=(1, filter_dim, filter_dim), 23 | reshape_dim=(1, ndim, ndim), 24 | input_shape=(timesteps, input_flat), 25 | return_sequences=True) 26 | model = Sequential() 27 | model.add(layer) 28 | model.add(TimeDistributedDense(10)) 29 | model.compile('sgd', 'mse') 30 | 31 | x = np.random.randn(nb_samples, timesteps, input_flat) 32 | y = model.predict(x) 33 | assert y.shape == (nb_samples, timesteps, 10) 34 | 35 | def test_conv_gru(self): 36 | """Just check that the ConvGRU layer can compile and run""" 37 | nb_samples, timesteps, ndim, filter_dim = 5, 10, 28, 3 38 | input_flat = ndim ** 2 39 | layer = ConvGRU(filter_dim=(1, filter_dim, filter_dim), 40 | reshape_dim=(1, ndim, ndim), 41 | # input_shape=(timesteps, input_flat), 42 | return_sequences=True) 43 | model = Sequential() 44 | model.add(TimeDistributedDense(input_flat, input_dim=input_flat)) 45 | model.add(layer) 46 | model.compile('sgd', 'mse') 47 | 48 | x = np.random.randn(nb_samples, timesteps, input_flat) 49 | y = model.predict(x) 50 | assert y.shape == (nb_samples, timesteps, input_flat) 51 | 52 | def test_time_distributed(self): 53 | """Just check that the TimeDistributedModel layer can compile and run""" 54 | nb_samples, timesteps, ndim, filter_dim = 5, 10, 28, 3 55 | input_flat = ndim ** 2 56 | 57 | inner = Sequential() 58 | inner.add(Convolution2D(1, filter_dim, filter_dim, border_mode='same', 59 | input_shape=(1, ndim, ndim))) 60 | 61 | layer = TimeDistributedModel( 62 | inner, batch_size=nb_samples, input_shape=(timesteps, input_flat)) 63 | model = Sequential() 64 | model.add(layer) 65 | model.add(TimeDistributedDense(10)) 66 | model.compile('sgd', 'mse') 67 | 68 | x = np.random.randn(nb_samples, timesteps, input_flat) 69 | y = model.predict(x) 70 | assert y.shape == (nb_samples, timesteps, 10) 71 | 72 | 73 | if __name__ == '__main__': 74 | unittest.main(verbosity=2) 75 | -------------------------------------------------------------------------------- /tests/test_memnn.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import unittest 4 | from seya.layers.memnn2 import MemN2N 5 | from keras.models import Sequential 6 | from keras.layers.core import Lambda 7 | from keras import backend as K 8 | 9 | import numpy as np 10 | 11 | 12 | class TestMemNN(unittest.TestCase): 13 | """Test seya.layers.memnn layer""" 14 | 15 | def test_memnn(self): 16 | def identity_init(shape, name=None): 17 | dim = max(shape) 18 | I = np.identity(dim)[:shape[0], :shape[1]] 19 | return K.variable(I, name=name) 20 | 21 | input_dim = 20 22 | output_dim = 64 23 | input_length = 9 24 | memory_length = 7 25 | 26 | facts = Sequential() 27 | facts.add(Lambda(lambda x: x, input_shape=(memory_length, input_dim), 28 | output_shape=(memory_length, input_dim))) 29 | question = Sequential() 30 | question.add(Lambda(lambda x: x, input_shape=(1, input_dim), 31 | output_shape=(1, input_dim))) 32 | 33 | memnn = MemN2N([facts, question], output_dim, input_dim, 34 | input_length, memory_length, 35 | output_shape=(output_dim,)) 36 | memnn.build() 37 | 38 | model = Sequential() 39 | model.add(memnn) 40 | model.compile("sgd", "mse") 41 | 42 | inp = np.random.randint(0, input_dim, 43 | (1, memory_length, input_length)) 44 | que = np.random.randint(0, input_dim, (1, 1, input_length)) 45 | assert model.predict([inp, que]).shape == (1, output_dim) 46 | 47 | if __name__ == '__main__': 48 | unittest.main(verbosity=2) 49 | -------------------------------------------------------------------------------- /tests/test_recurrent.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """Test seya.layers.recurrent module""" 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | import numpy as np 8 | import theano 9 | 10 | from keras.layers.recurrent import SimpleRNN 11 | from seya.layers.recurrent import Bidirectional 12 | 13 | 14 | class TestBidirectional(unittest.TestCase): 15 | """Test seya.layers.recurrent.Bidirectional layer""" 16 | 17 | def test_basic(self): 18 | """Just check that the Bidirectional layer can compile and run""" 19 | nb_samples, timesteps, input_dim, output_dim = 3, 3, 10, 5 20 | 21 | for ret_seq in [True, False]: 22 | rnn1 = SimpleRNN(output_dim, return_sequences=ret_seq, 23 | input_shape=(None, input_dim)) 24 | rnn2 = SimpleRNN(output_dim, return_sequences=ret_seq, 25 | input_shape=(None, input_dim)) 26 | layer = Bidirectional(rnn1, rnn2, return_sequences=ret_seq) 27 | layer.input = theano.shared(value=np.ones((nb_samples, timesteps, input_dim))) 28 | rnn1.input = layer.input 29 | rnn2.input = layer.input 30 | _ = layer.get_config() 31 | 32 | for train in [True, False]: 33 | out = layer.get_output(train).eval() 34 | # Make sure the output has the desired shape 35 | if ret_seq: 36 | assert(out.shape == (nb_samples, timesteps, output_dim*2)) 37 | else: 38 | assert(out.shape == (nb_samples, output_dim*2)) 39 | _ = layer.get_output_mask(train) 40 | 41 | 42 | if __name__ == '__main__': 43 | theano.config.exception_verbosity = 'high' 44 | unittest.main(verbosity=2) 45 | -------------------------------------------------------------------------------- /tests/test_seya.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_seya 6 | ---------------------------------- 7 | 8 | Tests for `seya` module. 9 | """ 10 | 11 | import unittest 12 | 13 | from seya import seya 14 | 15 | 16 | class TestSeya(unittest.TestCase): 17 | 18 | def setUp(self): 19 | pass 20 | 21 | def test_something(self): 22 | pass 23 | 24 | def tearDown(self): 25 | pass 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """Test seya.layers.recurrent module""" 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | import numpy as np 8 | import theano 9 | 10 | from seya import utils 11 | from keras import backend as K 12 | floatX = K.common._FLOATX 13 | 14 | 15 | class TestUtils(unittest.TestCase): 16 | """Test seya.utils""" 17 | 18 | def test_batched_dot(self): 19 | """Check if batched dot works""" 20 | samples = 8 21 | xdim = 7 22 | sizes_list = [10, 5] 23 | X = K.placeholder(ndim=2) 24 | W1 = K.placeholder(ndim=2) 25 | W2 = K.placeholder(ndim=2) 26 | B1 = K.placeholder(ndim=1) 27 | B2 = K.placeholder(ndim=1) 28 | Y = utils.batched_dot(X, [W1, W2], [B1, B2], sizes_list) 29 | F = K.function([X, W1, W2, B1, B2], Y) 30 | 31 | x = np.ones((samples, xdim)) 32 | w1 = np.ones((xdim, sizes_list[0])).astype('float32') 33 | w2 = np.ones((xdim, sizes_list[1])).astype('float32') 34 | b1 = np.ones(sizes_list[0]).astype('float32') 35 | b2 = np.ones(sizes_list[1]).astype('float32') 36 | y = F([x, w1, w2, b1, b2]) 37 | print(x.dot(w2)+b2, y[1]) 38 | print(y[0], x.dot(w1)+b1) 39 | #assert np.testing.assert_allclose(y[0], x.dot(w1)+b1, rtol=1e-3) 40 | #assert np.testing.assert_allclose(y[1], x.dot(w2)+b2, rtol=1e-3) 41 | 42 | 43 | if __name__ == '__main__': 44 | theano.config.exception_verbosity = 'high' 45 | unittest.main(verbosity=2) 46 | -------------------------------------------------------------------------------- /tests/test_variational.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """Test seya.layers.recurrent module""" 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | import numpy as np 8 | import theano 9 | 10 | from seya.layers.variational import VariationalDense 11 | from keras import backend as K 12 | floatX = K.common._FLOATX 13 | 14 | 15 | class TestVariational(unittest.TestCase): 16 | """Test seya.layers.variational.VariationalDense layer""" 17 | 18 | def test_basic(self): 19 | """Just check that the Variational layer can compile and run""" 20 | nb_samples, input_dim, output_dim = 3, 10, 5 21 | layer = VariationalDense(input_dim=input_dim, output_dim=output_dim, 22 | batch_size=nb_samples) 23 | X = layer.get_input() 24 | Y1 = layer.get_output(train=True) 25 | Y2 = layer.get_output(train=False) 26 | F = theano.function([X], [Y1, Y2]) 27 | 28 | y1, y2 = F(np.random.randn(nb_samples, input_dim).astype(floatX)) 29 | assert y1.shape == (nb_samples, output_dim) 30 | assert y2.shape == (nb_samples, output_dim) 31 | 32 | 33 | if __name__ == '__main__': 34 | theano.config.exception_verbosity = 'high' 35 | unittest.main(verbosity=2) 36 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27, py33, py34 3 | 4 | [testenv] 5 | setenv = 6 | PYTHONPATH = {toxinidir}:{toxinidir}/seya 7 | commands = python setup.py test 8 | deps = 9 | -r{toxinidir}/requirements.txt 10 | --------------------------------------------------------------------------------