├── .coveragerc ├── .gitignore ├── .travis.yml ├── .travis └── install_cntk.sh ├── CODEOWNERS ├── CONTRIBUTING.md ├── GUIDELINES.md ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── contrib_docs ├── README.md ├── pydocmd.yml └── theme │ ├── 404.html │ ├── base.html │ ├── breadcrumbs.html │ ├── css │ ├── theme.css │ └── theme_extra.css │ ├── footer.html │ ├── js │ ├── jquery-2.1.1.min.js │ ├── modernizr-2.8.3.min.js │ └── theme.js │ ├── main.html │ ├── nav.html │ ├── search.html │ ├── searchbox.html │ ├── toc.html │ └── versions.html ├── convert_to_tf_keras.py ├── examples ├── .gitkeep ├── cifar10_clr.py ├── cifar10_densenet.py ├── cifar10_nasnet.py ├── cifar10_resnet.py ├── cifar10_ror.py ├── cifar10_wide_resnet.py ├── conll2000_chunking_crf.py ├── improved_wgan.py └── jaccard_loss.py ├── keras_contrib ├── __init__.py ├── activations │ ├── __init__.py │ └── squash.py ├── applications │ ├── __init__.py │ ├── densenet.py │ ├── nasnet.py │ ├── resnet.py │ └── wide_resnet.py ├── backend │ ├── __init__.py │ ├── cntk_backend.py │ ├── numpy_backend.py │ ├── tensorflow_backend.py │ └── theano_backend.py ├── callbacks │ ├── __init__.py │ ├── cyclical_learning_rate.py │ ├── dead_relu_detector.py │ ├── snapshot.py │ └── tensorboard.py ├── constraints │ ├── __init__.py │ └── clip.py ├── datasets │ ├── __init__.py │ ├── coco.py │ ├── conll2000.py │ └── pascal_voc.py ├── initializers │ ├── __init__.py │ └── convaware.py ├── layers │ ├── __init__.py │ ├── advanced_activations │ │ ├── __init__.py │ │ ├── pelu.py │ │ ├── sinerelu.py │ │ ├── srelu.py │ │ └── swish.py │ ├── capsule.py │ ├── convolutional │ │ ├── __init__.py │ │ ├── cosineconvolution2d.py │ │ └── subpixelupscaling.py │ ├── core.py │ ├── crf.py │ └── normalization │ │ ├── __init__.py │ │ ├── groupnormalization.py │ │ └── instancenormalization.py ├── losses │ ├── __init__.py │ ├── crf_losses.py │ ├── dssim.py │ └── jaccard.py ├── metrics │ ├── __init__.py │ └── crf_accuracies.py ├── optimizers │ ├── __init__.py │ ├── ftml.py │ ├── lars.py │ ├── padam.py │ └── yogi.py ├── preprocessing │ └── __init__.py ├── regularizers │ └── __init__.py ├── tests │ ├── __init__.py │ ├── activations.py │ ├── metrics.py │ ├── optimizers.py │ └── regularizers.py ├── utils │ ├── __init__.py │ ├── conv_utils.py │ ├── save_load_utils.py │ └── test_utils.py └── wrappers │ └── __init__.py ├── pytest.ini ├── setup.py └── tests ├── conftest.py ├── keras_contrib ├── activations │ ├── .gitkeep │ └── test_squash.py ├── backend │ └── backend_test.py ├── callbacks │ ├── cyclical_learning_rate_test.py │ ├── dead_relu_detector_test.py │ └── tensorboard_test.py ├── constraints_test.py ├── datasets │ └── test_datasets.py ├── initializers_test.py ├── layers │ ├── advanced_activations │ │ ├── test_pelu.py │ │ ├── test_sinerelu.py │ │ ├── test_srelu.py │ │ └── test_swish.py │ ├── convolutional │ │ ├── test_cosineconvolution2d.py │ │ └── test_subpixelupscaling.py │ ├── normalization │ │ ├── test_groupnormalization.py │ │ └── test_instancenormalization.py │ ├── test_capsule.py │ ├── test_core.py │ └── test_crf.py ├── losses │ ├── dssim_test.py │ └── jaccard_test.py ├── metrics │ └── .gitkeep ├── optimizers │ ├── ftml_test.py │ ├── lars_test.py │ ├── padam_test.py │ └── yogi_test.py ├── preprocessing │ └── .gitkeep ├── regularizers │ └── .gitkeep ├── utils │ └── save_load_utils_test.py └── wrappers │ └── .gitkeep └── tooling ├── test_codeowners.py ├── test_doc_auto_generation.py └── test_documentation.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | # Regexes for lines to exclude from consideration 3 | exclude_lines = 4 | os.remove 5 | except ImportError 6 | # Don't complain if tests don't hit defensive assertion code: 7 | raise ImportError 8 | raise NotImplementedError 9 | 10 | # Don't complain if legacy support codes are not performed: 11 | if original_keras_version == '1': 12 | 13 | show_missing = True 14 | omit = 15 | keras_contrib/backend/theano_backend.py 16 | keras_contrib/backend/tensorflow_backend.py 17 | keras_contrib/backend/cntk_backend.py 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # PyCharms 38 | *.xml 39 | *.iml 40 | .idea/ 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *,cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # IPython Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | 93 | # Rope project settings 94 | .ropeproject 95 | 96 | # developer environments 97 | .idea 98 | 99 | *.DS_Store 100 | 101 | contrib_docs/site/* 102 | contrib_docs/sources/* 103 | contrib_docs/theme/fonts/* 104 | contrib_docs/theme/img/* 105 | 106 | .pytest_cache/* 107 | 108 | contrib_docs/_build/ 109 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: python 4 | cache: 5 | directories: 6 | - $HOME/.theano 7 | matrix: 8 | include: 9 | - python: 3.6 10 | env: KERAS_BACKEND=tensorflow TEST_MODE=PEP8_DOC 11 | - python: 2.7 12 | env: KERAS_BACKEND=tensorflow 13 | - python: 3.6 14 | env: KERAS_BACKEND=tensorflow 15 | - python: 3.6 16 | env: KERAS_BACKEND=tensorflow USE_TF_KERAS=1 PYTEST_IGNORE='--ignore=tests/test_doc_auto_generation.py --ignore=tests/keras_contrib/backend --ignore=tests/keras_contrib/utils/save_load_utils_test.py' 17 | - python: 3.6 18 | env: KERAS_BACKEND=theano THEANO_FLAGS=optimizer=fast_compile 19 | # - python: 3.6 20 | # env: KERAS_BACKEND=cntk PYTHONWARNINGS=ignore 21 | install: 22 | # code below is taken from http://conda.pydata.org/docs/travis.html 23 | # We do this conditionally because it saves us some downloading if the 24 | # version is the same. 25 | - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then 26 | wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; 27 | else 28 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 29 | fi 30 | - bash miniconda.sh -b -p $HOME/miniconda 31 | - export PATH="$HOME/miniconda/bin:$PATH" 32 | - hash -r 33 | - conda config --set always_yes yes --set changeps1 no 34 | - conda update -q conda 35 | # Useful for debugging any issues with conda 36 | - conda info -a 37 | 38 | - travis_retry conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION 39 | - source activate test-environment 40 | 41 | - travis_retry pip install --only-binary=numpy,scipy,pandas numpy nose scipy h5py theano pydoc-markdown pytest pytest-pep8 pandas pygithub --progress-bar off 42 | - if [[ "$USE_TF_KERAS" == "" ]]; then 43 | pip install git+https://github.com/keras-team/keras.git --progress-bar off; 44 | fi 45 | 46 | # set library path 47 | - export LD_LIBRARY_PATH=$HOME/miniconda/envs/test-environment/lib/:$LD_LIBRARY_PATH 48 | 49 | - if [[ "$KERAS_BACKEND" == "theano" ]]; then 50 | travis_retry conda install -q mkl mkl-service; 51 | fi 52 | 53 | - if [[ "$USE_TF_KERAS" == "1" ]]; then 54 | python convert_to_tf_keras.py; 55 | fi 56 | - pip install -e .[tests] --progress-bar off 57 | 58 | # install TensorFlow (CPU version). 59 | - pip install tensorflow==1.12 --progress-bar off 60 | 61 | # install cntk 62 | - ./.travis/install_cntk.sh 63 | 64 | # Remove the current backend from the coverage exclusion. 65 | - sed -i "\/keras\/backend\/${KERAS_BACKEND}_backend.py/d" .coveragerc 66 | 67 | # command to run tests 68 | script: 69 | - export MKL_THREADING_LAYER="GNU" 70 | # run keras backend init to initialize backend config 71 | - python -c "import keras_contrib.backend" 72 | # create dataset directory to avoid concurrent directory creation at runtime 73 | - mkdir ~/.keras/datasets 74 | # set up keras backend 75 | - sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json; 76 | - echo -e "Running tests with the following config:\n$(cat ~/.keras/keras.json)" 77 | - if [[ "$TEST_MODE" == "PEP8_DOC" ]]; then 78 | PYTHONPATH=$PWD:$PYTHONPATH py.test --pep8 -m pep8 -n0 && py.test tests/tooling/ convert_to_tf_keras.py && cd contrib_docs && pydocmd build; 79 | else 80 | PYTHONPATH=$PWD:$PYTHONPATH py.test tests/ $PYTEST_IGNORE --ignore=tests/tooling/ 81 | --cov-config .coveragerc --cov=keras_contrib tests/; 82 | fi 83 | -------------------------------------------------------------------------------- /.travis/install_cntk.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | pip install cntk --progress-bar off 3 | 4 | # open mpi is needed for cntk 5 | rm -rf ~/mpi 6 | mkdir ~/mpi 7 | pushd ~/mpi 8 | wget http://cntk.ai/PythonWheel/ForKeras/depends/openmpi_1.10-3.zip 9 | unzip ./openmpi_1.10-3.zip 10 | sudo dpkg -i openmpi_1.10-3.deb 11 | popd 12 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # examples 2 | 3 | 4 | # activations 5 | keras_contrib/activations/squash.py @SriRangaTarun 6 | 7 | # applications 8 | keras_contrib/applications/densenet.py @titu1994 9 | keras_contrib/applications/nasnet.py @titu1994 10 | keras_contrib/applications/wide_resnet.py @titu1994 11 | 12 | # backend 13 | 14 | 15 | # callbacks 16 | keras_contrib/callbacks/tensorboard.py @gabrieldemarmiesse 17 | keras_contrib/callbacks/snapshot.py @titu1994 18 | 19 | # constraints 20 | 21 | 22 | # datasets 23 | 24 | 25 | # initializers 26 | 27 | 28 | # layers 29 | keras_contrib/layers/advanced_activations/sinerelu.py @wilderrodrigues 30 | keras_contrib/layers/advanced_activations/swish.py @gabrieldemarmiesse 31 | keras_contrib/layers/convolutional/subpixelupscaling.py @titu1994 32 | keras_contrib/layers/normalization/groupnormalization.py @titu1994 33 | keras_contrib/layers/capsule.py @SriRangaTarun 34 | 35 | # losses 36 | 37 | 38 | # metrics 39 | 40 | 41 | # optimizers 42 | keras_contrib/optimizers/yogi.py @MarcoAndreaBuchmann 43 | keras_contrib/optimizers/padam.py @MFreidank 44 | 45 | # preprocessing 46 | 47 | 48 | # regularizers 49 | 50 | 51 | # utils 52 | 53 | 54 | # wrappers 55 | -------------------------------------------------------------------------------- /GUIDELINES.md: -------------------------------------------------------------------------------- 1 | # Maintainer Guidelines 2 | 3 | ## Maintainers: 4 | Following are the users with write-access to this repository (maintainers) : 5 | * [athundt](https://www.github.com/athundt) 6 | * [bstriner](https://www.github.com/bstriner) 7 | * [farizrahman4u](https://www.github.com/farizrahman4u) 8 | * [fchollet](https://www.github.com/fchollet) 9 | * [kemaswill](https://www.github.com/kemaswill) 10 | * [lukedeo](https://www.github.com/lukedeo) 11 | * [patyork](https://www.github.com/patyork) 12 | * [tboquet](https://www.github.com/tboquet) 13 | * [the-moliver](https://www.github.com/the-moliver) 14 | 15 | ## Addition of new features 16 | * Addition of new features require submitting a pull request, even for those who have write access to this repository. 17 | * Maintainers should not merge their own pull requests. 18 | * Whenever possible, multiple Maintainers should review a pull requests before it is merged. 19 | * All incoming new features should be accompanied by documentation and unit tests. 20 | 21 | ## Becoming a maintainer 22 | * To become a maintainer, you should be a recognized contributor to either Keras-contrib or Keras core. 23 | * If you think you are eligible to be a maintainer, you can contact one of the existing maintainers to join the team. 24 | 25 | ## Versioning 26 | * Keras-contrib is tested only against the bleeding-edge version of Keras. 27 | * In case the Travis build fails due to a change in Keras core, the maintainers are responsible of rectifying the issue. 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Fariz Rahman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | **- What I did** 7 | 8 | 9 | **- How I did it** 10 | 11 | 12 | **- How you can verify it** 13 | 17 | 18 | 19 | 20 | --- 21 | This pull request fixes #issue_number_here 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # keras-contrib : Keras community contributions 2 | 3 | Keras-contrib is deprecated. Use [TensorFlow Addons](https://github.com/tensorflow/addons). 4 | 5 | ## The future of Keras-contrib: 6 | 7 | We're migrating to [tensorflow/addons](https://github.com/tensorflow/addons). See the announcement [here](https://github.com/keras-team/keras-contrib/issues/519). 8 | 9 | [![Build Status](https://travis-ci.org/keras-team/keras-contrib.svg?branch=master)](https://travis-ci.org/keras-team/keras-contrib) 10 | 11 | This library is the official extension repository for the python deep learning library [Keras](http://www.keras.io). It contains additional layers, activations, loss functions, optimizers, etc. which are not yet available within Keras itself. All of these additional modules can be used in conjunction with core Keras models and modules. 12 | 13 | As the community contributions in Keras-Contrib are tested, used, validated, and their utility proven, they may be integrated into the Keras core repository. In the interest of keeping Keras succinct, clean, and powerfully simple, only the most useful contributions make it into Keras. This contribution repository is both the proving ground for new functionality, and the archive for functionality that (while useful) may not fit well into the Keras paradigm. 14 | 15 | 16 | 17 | 18 | 19 | --- 20 | ## Installation 21 | 22 | #### Install keras_contrib for keras-team/keras 23 | For instructions on how to install Keras, 24 | see [the Keras installation page](https://keras.io/#installation). 25 | 26 | ```shell 27 | git clone https://www.github.com/keras-team/keras-contrib.git 28 | cd keras-contrib 29 | python setup.py install 30 | ``` 31 | 32 | Alternatively, using pip: 33 | 34 | ```shell 35 | sudo pip install git+https://www.github.com/keras-team/keras-contrib.git 36 | ``` 37 | 38 | to uninstall: 39 | ```pip 40 | pip uninstall keras_contrib 41 | ``` 42 | 43 | #### Install keras_contrib for tensorflow.keras 44 | 45 | ```shell 46 | git clone https://www.github.com/keras-team/keras-contrib.git 47 | cd keras-contrib 48 | python convert_to_tf_keras.py 49 | USE_TF_KERAS=1 python setup.py install 50 | ``` 51 | 52 | to uninstall: 53 | ```shell 54 | pip uninstall tf_keras_contrib 55 | ``` 56 | 57 | For contributor guidelines see [CONTRIBUTING.md](https://github.com/keras-team/keras-contrib/blob/master/CONTRIBUTING.md) 58 | 59 | --- 60 | ## Example Usage 61 | 62 | Modules from the Keras-Contrib library are used in the same way as modules within Keras itself. 63 | 64 | ```python 65 | from keras.models import Sequential 66 | from keras.layers import Dense 67 | import numpy as np 68 | 69 | # I wish Keras had the Parametric Exponential Linear activation.. 70 | # Oh, wait..! 71 | from keras_contrib.layers.advanced_activations import PELU 72 | 73 | # Create the Keras model, including the PELU advanced activation 74 | model = Sequential() 75 | model.add(Dense(100, input_shape=(10,))) 76 | model.add(PELU()) 77 | 78 | # Compile and fit on random data 79 | model.compile(loss='mse', optimizer='adam') 80 | model.fit(x=np.random.random((100, 10)), y=np.random.random((100, 100)), epochs=5, verbose=0) 81 | 82 | # Save our model 83 | model.save('example.h5') 84 | ``` 85 | 86 | 87 | ### A Common "Gotcha" 88 | 89 | As Keras-Contrib is external to the Keras core, loading a model requires a bit more work. While a pure Keras model is loadable with nothing more than an import of `keras.models.load_model`, a model which contains a contributed module requires an additional import of `keras_contrib`: 90 | 91 | ```python 92 | # Required, as usual 93 | from keras.models import load_model 94 | 95 | # Recommended method; requires knowledge of the underlying architecture of the model 96 | from keras_contrib.layers import PELU 97 | from keras_contrib.layers import GroupNormalization 98 | 99 | # Load our model 100 | custom_objects = {'PELU': PELU, 'GroupNormalization': GroupNormalization} 101 | model = load_model('example.h5', custom_objects) 102 | ``` 103 | -------------------------------------------------------------------------------- /contrib_docs/README.md: -------------------------------------------------------------------------------- 1 | # Keras-contrib Documentation 2 | 3 | The source for Keras-contrib documentation is in this directory under `sources/`. 4 | Our documentation uses extended Markdown, as implemented by [MkDocs](http://mkdocs.org). 5 | 6 | ## Building the documentation 7 | 8 | - install pydoc-markdown: `pip install pydoc-markdown` 9 | - `cd` to the `contrib_docs/` folder and run: 10 | - `pydocmd serve` # Starts a local webserver: [localhost:8000](localhost:8000) 11 | - `pydocmd build` # Builds a static site in "site" directory 12 | -------------------------------------------------------------------------------- /contrib_docs/pydocmd.yml: -------------------------------------------------------------------------------- 1 | site_name: "Keras-contrib Documentation" 2 | 3 | 4 | generate: 5 | - layers/core.md: 6 | - keras_contrib.layers.CosineDense 7 | - layers/convolutional.md: 8 | - keras_contrib.layers.CosineConv2D 9 | - keras_contrib.layers.SubPixelUpscaling 10 | - layers/normalization.md: 11 | - keras_contrib.layers.InstanceNormalization 12 | - keras_contrib.layers.GroupNormalization 13 | - layers/advanced-activations.md: 14 | - keras_contrib.layers.SineReLU 15 | - keras_contrib.layers.SReLU 16 | - keras_contrib.layers.Swish 17 | - keras_contrib.layers.PELU 18 | - layers/crf.md: 19 | - keras_contrib.layers.CRF 20 | - losses.md: 21 | - keras_contrib.losses: 22 | - keras_contrib.losses.DSSIMObjective 23 | - keras_contrib.losses.jaccard_distance 24 | - keras_contrib.losses.crf_loss 25 | - keras_contrib.losses.crf_nll 26 | - optimizers.md: 27 | - keras_contrib.optimizers: 28 | - keras_contrib.optimizers.FTML 29 | - keras_contrib.optimizers.Padam 30 | - keras_contrib.optimizers.Yogi 31 | - keras_contrib.optimizers.LARS 32 | - callbacks.md: 33 | - keras_contrib.callbacks: 34 | - keras_contrib.callbacks.TensorBoardGrouped 35 | - keras_contrib.callbacks.CyclicLR 36 | - keras_contrib.callbacks.SnapshotCallbackBuilder 37 | - keras_contrib.callbacks.SnapshotModelCheckpoint 38 | - keras_contrib.callbacks.DeadReluDetector 39 | 40 | 41 | pages: 42 | - Home: index.md << ../README.md 43 | - layers: 44 | - Core layers: layers/core.md 45 | - Convolutional layers: layers/convolutional.md 46 | - normalization layers: layers/normalization.md 47 | - Advanced activations layers: layers/advanced-activations.md 48 | - CRF layers: layers/crf.md 49 | - Losses: losses.md 50 | - Optimizers: optimizers.md 51 | - Callbacks: callbacks.md 52 | 53 | 54 | 55 | headers: html 56 | -------------------------------------------------------------------------------- /contrib_docs/theme/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | 5 |

404

6 | 7 |

Page not found

8 | 9 | {% endblock %} 10 | -------------------------------------------------------------------------------- /contrib_docs/theme/breadcrumbs.html: -------------------------------------------------------------------------------- 1 |
2 | 29 | {% if config.theme.prev_next_buttons_location|lower in ['top', 'both'] 30 | and page and (page.next_page or page.previous_page) %} 31 | 39 | {% endif %} 40 |
41 |
42 | -------------------------------------------------------------------------------- /contrib_docs/theme/css/theme_extra.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Wrap inline code samples otherwise they shoot of the side and 3 | * can't be read at all. 4 | * 5 | * https://github.com/mkdocs/mkdocs/issues/313 6 | * https://github.com/mkdocs/mkdocs/issues/233 7 | * https://github.com/mkdocs/mkdocs/issues/834 8 | */ 9 | .rst-content code { 10 | white-space: pre-wrap; 11 | word-wrap: break-word; 12 | padding: 2px 5px; 13 | } 14 | 15 | /** 16 | * Make code blocks display as blocks and give them the appropriate 17 | * font size and padding. 18 | * 19 | * https://github.com/mkdocs/mkdocs/issues/855 20 | * https://github.com/mkdocs/mkdocs/issues/834 21 | * https://github.com/mkdocs/mkdocs/issues/233 22 | */ 23 | .rst-content pre code { 24 | white-space: pre; 25 | word-wrap: normal; 26 | display: block; 27 | padding: 12px; 28 | font-size: 12px; 29 | } 30 | 31 | /* 32 | * Fix link colors when the link text is inline code. 33 | * 34 | * https://github.com/mkdocs/mkdocs/issues/718 35 | */ 36 | a code { 37 | color: #2980B9; 38 | } 39 | a:hover code { 40 | color: #3091d1; 41 | } 42 | a:visited code { 43 | color: #9B59B6; 44 | } 45 | 46 | /* 47 | * The CSS classes from highlight.js seem to clash with the 48 | * ReadTheDocs theme causing some code to be incorrectly made 49 | * bold and italic. 50 | * 51 | * https://github.com/mkdocs/mkdocs/issues/411 52 | */ 53 | pre .cs, pre .c { 54 | font-weight: inherit; 55 | font-style: inherit; 56 | } 57 | 58 | /* 59 | * Fix some issues with the theme and non-highlighted code 60 | * samples. Without and highlighting styles attached the 61 | * formatting is broken. 62 | * 63 | * https://github.com/mkdocs/mkdocs/issues/319 64 | */ 65 | .rst-content .no-highlight { 66 | display: block; 67 | padding: 0.5em; 68 | color: #333; 69 | } 70 | 71 | 72 | /* 73 | * Additions specific to the search functionality provided by MkDocs 74 | */ 75 | 76 | .search-results { 77 | margin-top: 23px; 78 | } 79 | 80 | .search-results article { 81 | border-top: 1px solid #E1E4E5; 82 | padding-top: 24px; 83 | } 84 | 85 | .search-results article:first-child { 86 | border-top: none; 87 | } 88 | 89 | form .search-query { 90 | width: 100%; 91 | border-radius: 50px; 92 | padding: 6px 12px; /* csslint allow: box-model */ 93 | border-color: #D1D4D5; 94 | } 95 | 96 | /* 97 | * Improve inline code blocks within admonitions. 98 | * 99 | * https://github.com/mkdocs/mkdocs/issues/656 100 | */ 101 | .rst-content .admonition code { 102 | color: #404040; 103 | border: 1px solid #c7c9cb; 104 | border: 1px solid rgba(0, 0, 0, 0.2); 105 | background: #f8fbfd; 106 | background: rgba(255, 255, 255, 0.7); 107 | } 108 | 109 | /* 110 | * Account for wide tables which go off the side. 111 | * Override borders to avoid wierdness on narrow tables. 112 | * 113 | * https://github.com/mkdocs/mkdocs/issues/834 114 | * https://github.com/mkdocs/mkdocs/pull/1034 115 | */ 116 | .rst-content .section .docutils { 117 | width: 100%; 118 | overflow: auto; 119 | display: block; 120 | border: none; 121 | } 122 | 123 | td, th { 124 | border: 1px solid #e1e4e5 !important; /* csslint allow: important */ 125 | border-collapse: collapse; 126 | } 127 | 128 | /*Keras extras*/ 129 | 130 | .keras-logo { 131 | max-height: 55px; 132 | width: 100%; 133 | background: #d00000; 134 | font-size: 140%; 135 | color: white; 136 | font-family: "Source Sans Pro", "ff-tisa-web-pro", "Georgia", Arial, sans-serif; 137 | } 138 | 139 | .keras-logo-img { 140 | max-width: 45px; 141 | margin: 10px; 142 | } 143 | 144 | h1, h2, h3, h4, h5, h6, legend { 145 | font-family: "Source Sans Pro", "ff-tisa-web-pro", "Georgia", Arial, sans-serif;, 146 | } 147 | 148 | -------------------------------------------------------------------------------- /contrib_docs/theme/footer.html: -------------------------------------------------------------------------------- 1 | 27 | -------------------------------------------------------------------------------- /contrib_docs/theme/js/theme.js: -------------------------------------------------------------------------------- 1 | /* sphinx_rtd_theme version 0.4.1 | MIT license */ 2 | /* Built 20180727 10:07 */ 3 | require=function n(e,i,t){function o(s,a){if(!i[s]){if(!e[s]){var l="function"==typeof require&&require;if(!a&&l)return l(s,!0);if(r)return r(s,!0);var c=new Error("Cannot find module '"+s+"'");throw c.code="MODULE_NOT_FOUND",c}var u=i[s]={exports:{}};e[s][0].call(u.exports,function(n){var i=e[s][1][n];return o(i||n)},u,u.exports,n,e,i,t)}return i[s].exports}for(var r="function"==typeof require&&require,s=0;s"),n("table.docutils.footnote").wrap("
"),n("table.docutils.citation").wrap("
"),n(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var i=n(this);expand=n(''),expand.on("click",function(n){return e.toggleCurrent(i),n.stopPropagation(),!1}),i.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}i.length>0&&($(".wy-menu-vertical .current").removeClass("current"),i.addClass("current"),i.closest("li.toctree-l1").addClass("current"),i.closest("li.toctree-l1").parent().addClass("current"),i.closest("li.toctree-l1").addClass("current"),i.closest("li.toctree-l2").addClass("current"),i.closest("li.toctree-l3").addClass("current"),i.closest("li.toctree-l4").addClass("current"))}catch(o){console.log("Error expanding nav for anchor",o)}},onScroll:function(){this.winScroll=!1;var n=this.win.scrollTop(),e=n+this.winHeight,i=this.navBar.scrollTop()+(n-this.winPosition);n<0||e>this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav,StickyNav:e.exports.ThemeNav}),function(){for(var n=0,e=["ms","moz","webkit","o"],i=0;i{{ nav_item.title }} 2 | {%- set navlevel = navlevel + 1 %} 3 | {%- if navlevel <= config.theme.navigation_depth 4 | and ((nav_item.is_page and nav_item.toc.items 5 | and (not config.theme.titles_only 6 | and (nav_item == page or not config.theme.collapse_navigation))) 7 | or (nav_item.is_section and nav_item.children)) %} 8 | 9 | {%- if nav_item.is_page %} 10 | {#- Skip first level of toc which is page title. #} 11 | {%- set toc_item = nav_item.toc.items[0] %} 12 | {%- include 'toc.html' %} 13 | {%- elif nav_item.is_section %} 14 | {%- for nav_item in nav_item.children %} 15 |
  • 16 | {%- include 'nav.html' %} 17 |
    • 18 | {%- endfor %} 19 | {%- endif %} 20 | 21 | {%- endif %} 22 | {%- set navlevel = navlevel - 1 %} 23 | -------------------------------------------------------------------------------- /contrib_docs/theme/search.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | 5 |

    Search Results

    6 | 7 |
    8 | 9 | 10 |
    11 | 12 |
    13 | Searching... 14 |
    15 | 16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /contrib_docs/theme/searchbox.html: -------------------------------------------------------------------------------- 1 |
    2 |
    3 | 4 |
    5 |
    6 | -------------------------------------------------------------------------------- /contrib_docs/theme/toc.html: -------------------------------------------------------------------------------- 1 | 2 | {%- for toc_item in toc_item.children %} 3 |
  • {{ toc_item.title }} 4 | {%- set navlevel = navlevel + 1 %} 5 | {%- if navlevel <= config.theme.navigation_depth and toc_item.children %} 6 |
      7 | {%- include 'toc.html' %} 8 |
    9 | {%- endif %} 10 | {%- set navlevel = navlevel - 1 %} 11 |
    • 12 | {%- endfor %} 13 | -------------------------------------------------------------------------------- /contrib_docs/theme/versions.html: -------------------------------------------------------------------------------- 1 |
    2 | 3 | {% if config.repo_name == 'GitHub' %} 4 | GitHub 5 | {% elif config.repo_name == 'Bitbucket' %} 6 | BitBucket 7 | {% elif config.repo_name == 'GitLab' %} 8 | GitLab 9 | {% endif %} 10 | {% if page.previous_page %} 11 | « Previous 12 | {% endif %} 13 | {% if page.next_page %} 14 | Next » 15 | {% endif %} 16 | 17 |
    18 | -------------------------------------------------------------------------------- /convert_to_tf_keras.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | list_conversions = [('import keras.', 'import tensorflow.keras.'), 5 | ('import keras ', 'from tensorflow import keras '), 6 | ('import keras\n', 'from tensorflow import keras\n'), 7 | ('from keras.', 'from tensorflow.keras.'), 8 | ('from keras ', 'from tensorflow.keras ')] 9 | 10 | 11 | def replace_imports_in_text(string, revert): 12 | if revert: 13 | list_imports_to_change = [x[::-1] for x in list_conversions] 14 | else: 15 | list_imports_to_change = list_conversions 16 | 17 | text_updated = string 18 | for old_str, new_str in list_imports_to_change: 19 | text_updated = text_updated.replace(old_str, new_str) 20 | return text_updated 21 | 22 | 23 | def replace_imports_in_file(file_path, revert): 24 | if not file_path.endswith('.py'): 25 | return False 26 | if os.path.abspath(file_path) == os.path.abspath(__file__): 27 | return False 28 | with open(file_path, 'r') as f: 29 | text = f.read() 30 | 31 | text_updated = replace_imports_in_text(text, revert) 32 | 33 | with open(file_path, 'w+') as f: 34 | f.write(text_updated) 35 | 36 | return text_updated != text 37 | 38 | 39 | def convert_codebase(revert): 40 | nb_of_files_changed = 0 41 | keras_dir = os.path.dirname(os.path.abspath(__file__)) 42 | for root, dirs, files in os.walk(keras_dir): 43 | for name in files: 44 | if replace_imports_in_file(os.path.join(root, name), revert): 45 | nb_of_files_changed += 1 46 | print('Changed imports in ' + str(nb_of_files_changed) + ' files.') 47 | print('Those files were found in the directory ' + keras_dir) 48 | 49 | 50 | def convert_to_tf_keras(): 51 | """Convert the codebase to tf.keras""" 52 | convert_codebase(False) 53 | 54 | 55 | def convert_to_keras_team_keras(): 56 | """Convert the codebase from tf.keras to keras-team/keras""" 57 | convert_codebase(True) 58 | 59 | 60 | def test_replace_imports(): 61 | python_code = """ 62 | import keras 63 | from keras import backend as K 64 | import os 65 | import keras_contrib 66 | import keras_contrib.layers as lay 67 | import keras.layers 68 | from keras.layers import Dense 69 | 70 | if K.backend() == 'tensorflow': 71 | import tensorflow as tf 72 | function = tf.max 73 | """ 74 | 75 | expected_code = """ 76 | from tensorflow import keras 77 | from tensorflow.keras import backend as K 78 | import os 79 | import keras_contrib 80 | import keras_contrib.layers as lay 81 | import tensorflow.keras.layers 82 | from tensorflow.keras.layers import Dense 83 | 84 | if K.backend() == 'tensorflow': 85 | import tensorflow as tf 86 | function = tf.max 87 | """ 88 | 89 | code_with_replacement = replace_imports_in_text(python_code, False) 90 | assert expected_code == code_with_replacement 91 | assert python_code == replace_imports_in_text(code_with_replacement, True) 92 | 93 | 94 | if __name__ == '__main__': 95 | if '--revert' in sys.argv: 96 | convert_to_keras_team_keras() 97 | else: 98 | convert_to_tf_keras() 99 | -------------------------------------------------------------------------------- /examples/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/examples/.gitkeep -------------------------------------------------------------------------------- /examples/cifar10_clr.py: -------------------------------------------------------------------------------- 1 | '''Train a simple deep CNN on the CIFAR10 small images dataset using 2 | a triangular cyclic learning rate (CLR) policy. 3 | It gets to 75% validation accuracy in 15 epochs, and 79% after 40 epochs; 4 | compare to 25 and 50 epochs respectively without CLR. 5 | ''' 6 | 7 | from __future__ import print_function 8 | from __future__ import absolute_import 9 | import keras 10 | from keras.datasets import cifar10 11 | from keras.preprocessing.image import ImageDataGenerator 12 | from keras.models import Sequential 13 | from keras.layers import Dense, Dropout, Activation, Flatten 14 | from keras.layers import Conv2D, MaxPooling2D 15 | from keras_contrib.callbacks import CyclicLR 16 | 17 | import os 18 | 19 | batch_size = 100 20 | epochs = 50 21 | num_classes = 10 22 | data_augmentation = True 23 | num_predictions = 20 24 | save_dir = os.path.join(os.getcwd(), 'saved_models') 25 | model_name = 'keras_cifar10_trained_model.h5' 26 | 27 | # The data, split between train and test sets: 28 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 29 | print('x_train shape:', x_train.shape) 30 | print(x_train.shape[0], 'train samples') 31 | print(x_test.shape[0], 'test samples') 32 | 33 | # Convert class vectors to binary class matrices. 34 | y_train = keras.utils.to_categorical(y_train, num_classes) 35 | y_test = keras.utils.to_categorical(y_test, num_classes) 36 | 37 | model = Sequential() 38 | model.add(Conv2D(32, (3, 3), padding='same', 39 | input_shape=x_train.shape[1:])) 40 | model.add(Activation('relu')) 41 | model.add(Conv2D(32, (3, 3))) 42 | model.add(Activation('relu')) 43 | model.add(MaxPooling2D(pool_size=(2, 2))) 44 | model.add(Dropout(0.25)) 45 | 46 | model.add(Conv2D(64, (3, 3), padding='same')) 47 | model.add(Activation('relu')) 48 | model.add(Conv2D(64, (3, 3))) 49 | model.add(Activation('relu')) 50 | model.add(MaxPooling2D(pool_size=(2, 2))) 51 | model.add(Dropout(0.25)) 52 | 53 | model.add(Flatten()) 54 | model.add(Dense(512)) 55 | model.add(Activation('relu')) 56 | model.add(Dropout(0.5)) 57 | model.add(Dense(num_classes)) 58 | model.add(Activation('softmax')) 59 | 60 | # initiate RMSprop optimizer 61 | opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) 62 | 63 | # initiate CyclicLR LR scheduler 64 | clr = CyclicLR( 65 | base_lr=0.0001, 66 | max_lr=0.0005, 67 | step_size=2000, 68 | mode='triangular') 69 | 70 | 71 | # Let's train the model using RMSprop 72 | model.compile(loss='categorical_crossentropy', 73 | optimizer=opt, 74 | metrics=['accuracy']) 75 | 76 | x_train = x_train.astype('float32') 77 | x_test = x_test.astype('float32') 78 | x_train /= 255 79 | x_test /= 255 80 | 81 | if not data_augmentation: 82 | print('Not using data augmentation.') 83 | model.fit(x_train, y_train, 84 | batch_size=batch_size, 85 | epochs=epochs, 86 | validation_data=(x_test, y_test), 87 | callbacks=[clr], 88 | shuffle=True) 89 | else: 90 | print('Using real-time data augmentation.') 91 | # This will do preprocessing and realtime data augmentation: 92 | datagen = ImageDataGenerator( 93 | featurewise_center=False, # set input mean to 0 over the dataset 94 | samplewise_center=False, # set each sample mean to 0 95 | featurewise_std_normalization=False, # divide inputs by std of the dataset 96 | samplewise_std_normalization=False, # divide each input by its std 97 | zca_whitening=False, # apply ZCA whitening 98 | zca_epsilon=1e-06, # epsilon for ZCA whitening 99 | rotation_range=0, 100 | # randomly rotate images in the range (degrees, 0 to 180) 101 | # randomly shift images horizontally (fraction of total width) 102 | width_shift_range=0.1, 103 | # randomly shift images vertically (fraction of total height) 104 | height_shift_range=0.1, 105 | shear_range=0., # set range for random shear 106 | zoom_range=0., # set range for random zoom 107 | channel_shift_range=0., # set range for random channel shifts 108 | # set mode for filling points outside the input boundaries 109 | fill_mode='nearest', 110 | cval=0., # value used for fill_mode = "constant" 111 | horizontal_flip=True, # randomly flip images 112 | vertical_flip=False, # randomly flip images 113 | # set rescaling factor (applied before any other transformation) 114 | rescale=None, 115 | # set function that will be applied on each input 116 | preprocessing_function=None, 117 | # image data format, either "channels_first" or "channels_last" 118 | data_format=None, 119 | # fraction of images reserved for validation (strictly between 0 and 1) 120 | validation_split=0.0) 121 | 122 | # Compute quantities required for feature-wise normalization 123 | # (std, mean, and principal components if ZCA whitening is applied). 124 | datagen.fit(x_train) 125 | 126 | # Fit the model on the batches generated by datagen.flow(). 127 | 128 | model.fit_generator(datagen.flow(x_train, y_train, 129 | batch_size=batch_size), 130 | epochs=epochs, 131 | validation_data=(x_test, y_test), 132 | callbacks=[clr], 133 | workers=4) 134 | 135 | # Save model and weights 136 | if not os.path.isdir(save_dir): 137 | os.makedirs(save_dir) 138 | model_path = os.path.join(save_dir, model_name) 139 | model.save(model_path) 140 | print('Saved trained model at %s ' % model_path) 141 | 142 | # Score trained model. 143 | scores = model.evaluate(x_test, y_test, verbose=1) 144 | print('Test loss:', scores[0]) 145 | print('Test accuracy:', scores[1]) 146 | -------------------------------------------------------------------------------- /examples/cifar10_densenet.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Trains a DenseNet-40-12 model on the CIFAR-10 Dataset. 3 | 4 | Gets a 94.84% accuracy score after 100 epochs. 5 | ''' 6 | from __future__ import absolute_import 7 | from __future__ import print_function 8 | from __future__ import division 9 | 10 | import numpy as np 11 | 12 | from keras import backend as K 13 | from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping 14 | from keras.datasets import cifar10 15 | from keras.optimizers import Adam 16 | from keras.preprocessing.image import ImageDataGenerator 17 | from keras.utils import np_utils 18 | from keras_contrib.applications import DenseNet 19 | 20 | batch_size = 64 21 | nb_classes = 10 22 | epochs = 100 23 | 24 | img_rows, img_cols = 32, 32 25 | img_channels = 3 26 | 27 | # Parameters for the DenseNet model builder 28 | if K.image_data_format() == 'channels_first': 29 | img_dim = (img_channels, img_rows, img_cols) 30 | else: 31 | img_dim = (img_rows, img_cols, img_channels) 32 | depth = 40 33 | nb_dense_block = 3 34 | growth_rate = 12 35 | nb_filter = 16 36 | dropout_rate = 0.0 # 0.0 for data augmentation 37 | 38 | # Create the model (without loading weights) 39 | model = DenseNet(depth=depth, nb_dense_block=nb_dense_block, 40 | growth_rate=growth_rate, nb_filter=nb_filter, 41 | dropout_rate=dropout_rate, 42 | input_shape=img_dim, 43 | weights=None) 44 | print('Model created') 45 | 46 | model.summary() 47 | 48 | optimizer = Adam(lr=1e-3) # Using Adam instead of SGD to speed up training 49 | model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc']) 50 | print('Finished compiling') 51 | 52 | (trainX, trainY), (testX, testY) = cifar10.load_data() 53 | 54 | trainX = trainX.astype('float32') 55 | testX = testX.astype('float32') 56 | 57 | trainX /= 255. 58 | testX /= 255. 59 | 60 | Y_train = np_utils.to_categorical(trainY, nb_classes) 61 | Y_test = np_utils.to_categorical(testY, nb_classes) 62 | 63 | generator = ImageDataGenerator(rotation_range=15, 64 | width_shift_range=5. / 32, 65 | height_shift_range=5. / 32) 66 | 67 | generator.fit(trainX, seed=0) 68 | 69 | weights_file = 'DenseNet-40-12-CIFAR-10.h5' 70 | 71 | lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), 72 | cooldown=0, patience=10, min_lr=0.5e-6) 73 | early_stopper = EarlyStopping(monitor='val_acc', min_delta=1e-4, patience=20) 74 | model_checkpoint = ModelCheckpoint(weights_file, monitor='val_acc', 75 | save_best_only=True, 76 | save_weights_only=True, mode='auto') 77 | 78 | callbacks = [lr_reducer, early_stopper, model_checkpoint] 79 | 80 | model.fit_generator(generator.flow(trainX, Y_train, batch_size=batch_size), 81 | steps_per_epoch=len(trainX) // batch_size, 82 | epochs=epochs, 83 | callbacks=callbacks, 84 | validation_data=(testX, Y_test), 85 | verbose=2) 86 | 87 | scores = model.evaluate(testX, Y_test, batch_size=batch_size) 88 | print('Test loss : ', scores[0]) 89 | print('Test accuracy : ', scores[1]) 90 | -------------------------------------------------------------------------------- /examples/cifar10_nasnet.py: -------------------------------------------------------------------------------- 1 | """ 2 | Adapted from keras example cifar10_cnn.py 3 | Train NASNet-CIFAR on the CIFAR10 small images dataset. 4 | """ 5 | from __future__ import print_function 6 | from keras.datasets import cifar10 7 | from keras.preprocessing.image import ImageDataGenerator 8 | from keras.utils import np_utils 9 | from keras.callbacks import ModelCheckpoint 10 | from keras.callbacks import ReduceLROnPlateau 11 | from keras.callbacks import CSVLogger 12 | from keras.optimizers import Adam 13 | from keras_contrib.applications.nasnet import NASNetCIFAR, preprocess_input 14 | 15 | import numpy as np 16 | 17 | 18 | weights_file = 'NASNet-CIFAR-10.h5' 19 | lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.5), 20 | cooldown=0, 21 | patience=5, 22 | min_lr=0.5e-5) 23 | csv_logger = CSVLogger('NASNet-CIFAR-10.csv') 24 | model_checkpoint = ModelCheckpoint(weights_file, 25 | monitor='val_predictions_acc', 26 | save_best_only=True, 27 | save_weights_only=True, mode='max') 28 | 29 | batch_size = 128 30 | nb_classes = 10 31 | nb_epoch = 600 32 | data_augmentation = True 33 | 34 | # input image dimensions 35 | img_rows, img_cols = 32, 32 36 | # The CIFAR10 images are RGB. 37 | img_channels = 3 38 | 39 | # The data, shuffled and split between train and test sets: 40 | (X_train, y_train), (X_test, y_test) = cifar10.load_data() 41 | 42 | # Convert class vectors to binary class matrices. 43 | Y_train = np_utils.to_categorical(y_train, nb_classes) 44 | Y_test = np_utils.to_categorical(y_test, nb_classes) 45 | 46 | X_train = X_train.astype('float32') 47 | X_test = X_test.astype('float32') 48 | 49 | # preprocess input 50 | X_train = preprocess_input(X_train) 51 | X_test = preprocess_input(X_test) 52 | 53 | # For training, the auxilary branch must be used to correctly train NASNet 54 | model = NASNetCIFAR((img_rows, img_cols, img_channels), use_auxilary_branch=True) 55 | model.summary() 56 | 57 | optimizer = Adam(lr=1e-3, clipnorm=5) 58 | model.compile(loss=['categorical_crossentropy', 'categorical_crossentropy'], 59 | optimizer=optimizer, metrics=['accuracy'], loss_weights=[1.0, 0.4]) 60 | 61 | # model.load_weights('NASNet-CIFAR-10.h5', by_name=True) 62 | 63 | if not data_augmentation: 64 | print('Not using data augmentation.') 65 | model.fit(X_train, [Y_train, Y_train], 66 | batch_size=batch_size, 67 | epochs=nb_epoch, 68 | validation_data=(X_test, [Y_test, Y_test]), 69 | shuffle=True, 70 | verbose=2, 71 | callbacks=[lr_reducer, csv_logger, model_checkpoint]) 72 | else: 73 | print('Using real-time data augmentation.') 74 | # This will do preprocessing and realtime data augmentation: 75 | datagen = ImageDataGenerator( 76 | featurewise_center=False, # set input mean to 0 over the dataset 77 | samplewise_center=False, # set each sample mean to 0 78 | featurewise_std_normalization=False, # divide inputs by std of the dataset 79 | samplewise_std_normalization=False, # divide each input by its std 80 | zca_whitening=False, # apply ZCA whitening 81 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 82 | width_shift_range=0.1, 83 | height_shift_range=0.1, 84 | horizontal_flip=True, # randomly flip images 85 | vertical_flip=False) # randomly flip images 86 | 87 | # Compute quantities required for featurewise normalization 88 | # (std, mean, and principal components if ZCA whitening is applied). 89 | datagen.fit(X_train) 90 | 91 | # wrap the ImageDataGenerator to yield 92 | # two label batches [y, y] for each input batch X 93 | # When training a NASNet model, we have to use its auxilary training head 94 | # Therefore the model is technically a 1 input - 2 output model, and requires 95 | # the label to be duplicated for the auxilary head 96 | def image_data_generator_wrapper(image_datagenerator, batch_size): 97 | iterator = datagen.flow(X_train, Y_train, batch_size=batch_size) 98 | 99 | while True: 100 | X, y = next(iterator) # get the next batch 101 | yield X, [y, y] # duplicate the labels for each batch 102 | 103 | # Fit the model on the batches generated by datagen.flow(). 104 | model.fit_generator(image_data_generator_wrapper(datagen, batch_size), 105 | steps_per_epoch=X_train.shape[0] // batch_size, 106 | validation_data=(X_test, [Y_test, Y_test]), 107 | epochs=nb_epoch, verbose=2, 108 | callbacks=[lr_reducer, csv_logger, model_checkpoint]) 109 | 110 | scores = model.evaluate(X_test, [Y_test, Y_test], batch_size=batch_size) 111 | for score, metric_name in zip(scores, model.metrics_names): 112 | print("%s : %0.4f" % (metric_name, score)) 113 | -------------------------------------------------------------------------------- /examples/cifar10_resnet.py: -------------------------------------------------------------------------------- 1 | """ 2 | Adapted from keras example cifar10_cnn.py and github.com/raghakot/keras-resnet 3 | Train ResNet-18 on the CIFAR10 small images dataset. 4 | 5 | GPU run command with Theano backend (with TensorFlow, the GPU is automatically used): 6 | THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10.py 7 | """ 8 | from __future__ import print_function 9 | from keras.datasets import cifar10 10 | from keras.preprocessing.image import ImageDataGenerator 11 | from keras.utils import np_utils 12 | from keras.callbacks import ModelCheckpoint 13 | from keras.callbacks import ReduceLROnPlateau 14 | from keras.callbacks import CSVLogger 15 | from keras.callbacks import EarlyStopping 16 | from keras_contrib.applications.resnet import ResNet18 17 | 18 | import numpy as np 19 | 20 | 21 | weights_file = 'ResNet18v2-CIFAR-10.h5' 22 | lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, 23 | patience=5, min_lr=0.5e-6) 24 | early_stopper = EarlyStopping(min_delta=0.001, patience=10) 25 | csv_logger = CSVLogger('ResNet18v2-CIFAR-10.csv') 26 | model_checkpoint = ModelCheckpoint(weights_file, monitor='val_acc', save_best_only=True, 27 | save_weights_only=True, mode='auto') 28 | 29 | batch_size = 32 30 | nb_classes = 10 31 | nb_epoch = 200 32 | data_augmentation = True 33 | 34 | # input image dimensions 35 | img_rows, img_cols = 32, 32 36 | # The CIFAR10 images are RGB. 37 | img_channels = 3 38 | 39 | # The data, shuffled and split between train and test sets: 40 | (X_train, y_train), (X_test, y_test) = cifar10.load_data() 41 | 42 | # Convert class vectors to binary class matrices. 43 | Y_train = np_utils.to_categorical(y_train, nb_classes) 44 | Y_test = np_utils.to_categorical(y_test, nb_classes) 45 | 46 | X_train = X_train.astype('float32') 47 | X_test = X_test.astype('float32') 48 | 49 | # subtract mean and normalize 50 | mean_image = np.mean(X_train, axis=0) 51 | X_train -= mean_image 52 | X_test -= mean_image 53 | X_train /= 128. 54 | X_test /= 128. 55 | 56 | model = ResNet18((img_rows, img_cols, img_channels), nb_classes) 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer='adam', 59 | metrics=['accuracy']) 60 | 61 | if not data_augmentation: 62 | print('Not using data augmentation.') 63 | model.fit(X_train, Y_train, 64 | batch_size=batch_size, 65 | nb_epoch=nb_epoch, 66 | validation_data=(X_test, Y_test), 67 | shuffle=True, 68 | callbacks=[lr_reducer, early_stopper, csv_logger, model_checkpoint]) 69 | else: 70 | print('Using real-time data augmentation.') 71 | # This will do preprocessing and realtime data augmentation: 72 | datagen = ImageDataGenerator( 73 | featurewise_center=False, # set input mean to 0 over the dataset 74 | samplewise_center=False, # set each sample mean to 0 75 | featurewise_std_normalization=False, # divide inputs by std of the dataset 76 | samplewise_std_normalization=False, # divide each input by its std 77 | zca_whitening=False, # apply ZCA whitening 78 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 79 | width_shift_range=0.1, # randomly shift images horizontally 80 | height_shift_range=0.1, # randomly shift images vertically 81 | horizontal_flip=True, # randomly flip images 82 | vertical_flip=False) # randomly flip images 83 | 84 | # Compute quantities required for featurewise normalization 85 | # (std, mean, and principal components if ZCA whitening is applied). 86 | datagen.fit(X_train) 87 | 88 | callbacks = [lr_reducer, early_stopper, csv_logger, model_checkpoint] 89 | # Fit the model on the batches generated by datagen.flow(). 90 | model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size), 91 | steps_per_epoch=X_train.shape[0] // batch_size, 92 | validation_data=(X_test, Y_test), 93 | epochs=nb_epoch, verbose=2, 94 | callbacks=callbacks) 95 | 96 | scores = model.evaluate(X_test, Y_test, batch_size=batch_size) 97 | print('Test loss : ', scores[0]) 98 | print('Test accuracy : ', scores[1]) 99 | -------------------------------------------------------------------------------- /examples/cifar10_ror.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Trains a Residual-of-Residual Network (WRN-40-2) model on the CIFAR-10 Dataset. 3 | 4 | Gets a 94.53% accuracy score after 150 epochs. 5 | ''' 6 | 7 | import keras.callbacks as callbacks 8 | import keras.utils.np_utils as kutils 9 | from keras.datasets import cifar10 10 | from keras.preprocessing.image import ImageDataGenerator 11 | from keras.optimizers import Adam 12 | 13 | from keras_contrib.applications import ResidualOfResidual 14 | 15 | batch_size = 64 16 | epochs = 150 17 | img_rows, img_cols = 32, 32 18 | 19 | (trainX, trainY), (testX, testY) = cifar10.load_data() 20 | 21 | trainX = trainX.astype('float32') 22 | testX = testX.astype('float32') 23 | 24 | trainX /= 255 25 | testX /= 255 26 | 27 | tempY = testY 28 | trainY = kutils.to_categorical(trainY) 29 | testY = kutils.to_categorical(testY) 30 | 31 | generator = ImageDataGenerator(rotation_range=15, 32 | width_shift_range=5. / 32, 33 | height_shift_range=5. / 32) 34 | 35 | generator.fit(trainX, seed=0) 36 | 37 | model = ResidualOfResidual(depth=40, width=2, dropout_rate=0.0, weights=None) 38 | 39 | optimizer = Adam(lr=1e-3) 40 | 41 | model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc']) 42 | print('Finished compiling') 43 | 44 | checkpoint = callbacks.ModelCheckpoint('weights/RoR-WRN-40-2-Weights.h5', 45 | monitor='val_acc', 46 | save_best_only=True, 47 | save_weights_only=True) 48 | model.fit_generator(generator.flow(trainX, trainY, batch_size=batch_size), 49 | steps_per_epoch=len(trainX) // batch_size, 50 | epochs=epochs, 51 | callbacks=[checkpoint], 52 | validation_data=(testX, testY), 53 | verbose=2) 54 | 55 | scores = model.evaluate(testX, testY, batch_size) 56 | print('Test loss : ', scores[0]) 57 | print('Test accuracy : ', scores[1]) 58 | -------------------------------------------------------------------------------- /examples/cifar10_wide_resnet.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Trains a WRN-28-8 model on the CIFAR-10 Dataset. 3 | 4 | Performance is slightly less than the paper, since 5 | they use WRN-28-10 model (95.83%). 6 | 7 | Gets a 95.54% accuracy score after 300 epochs. 8 | ''' 9 | from __future__ import absolute_import 10 | from __future__ import print_function 11 | from __future__ import division 12 | 13 | from keras.datasets import cifar10 14 | import keras.callbacks as callbacks 15 | import keras.utils.np_utils as kutils 16 | from keras.preprocessing.image import ImageDataGenerator 17 | 18 | from keras_contrib.applications.wide_resnet import WideResidualNetwork 19 | 20 | batch_size = 64 21 | epochs = 300 22 | img_rows, img_cols = 32, 32 23 | 24 | (trainX, trainY), (testX, testY) = cifar10.load_data() 25 | 26 | trainX = trainX.astype('float32') 27 | trainX /= 255.0 28 | testX = testX.astype('float32') 29 | testX /= 255.0 30 | 31 | tempY = testY 32 | trainY = kutils.to_categorical(trainY) 33 | testY = kutils.to_categorical(testY) 34 | 35 | generator = ImageDataGenerator(rotation_range=10, 36 | width_shift_range=5. / 32, 37 | height_shift_range=5. / 32, 38 | horizontal_flip=True) 39 | 40 | generator.fit(trainX, seed=0, augment=True) 41 | 42 | # We will be training the model, therefore no need to load weights 43 | model = WideResidualNetwork(depth=28, width=8, dropout_rate=0.0, weights=None) 44 | 45 | model.summary() 46 | 47 | model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) 48 | print('Finished compiling') 49 | model_checkpoint = callbacks.ModelCheckpoint('WRN-28-8 Weights.h5', 50 | monitor='val_acc', 51 | save_best_only=True, 52 | save_weights_only=True) 53 | model.fit_generator(generator.flow(trainX, trainY, batch_size=batch_size), 54 | steps_per_epoch=len(trainX) // batch_size, 55 | epochs=epochs, 56 | callbacks=[model_checkpoint], 57 | validation_data=(testX, testY)) 58 | 59 | scores = model.evaluate(testX, testY, batch_size) 60 | print('Test loss : %0.5f' % (scores[0])) 61 | print('Test accuracy = %0.5f' % (scores[1])) 62 | -------------------------------------------------------------------------------- /examples/conll2000_chunking_crf.py: -------------------------------------------------------------------------------- 1 | """Train CRF and BiLSTM-CRF on CONLL2000 chunking data, 2 | similar to https://arxiv.org/pdf/1508.01991v1.pdf. 3 | """ 4 | from __future__ import absolute_import 5 | from __future__ import print_function 6 | from __future__ import division 7 | 8 | import numpy 9 | from collections import Counter 10 | 11 | from keras.models import Sequential 12 | from keras.layers import Embedding, Bidirectional, LSTM 13 | from keras_contrib.layers import CRF 14 | from keras_contrib.losses import crf_loss 15 | from keras_contrib.metrics import crf_viterbi_accuracy 16 | from keras_contrib.datasets import conll2000 17 | 18 | EPOCHS = 10 19 | EMBED_DIM = 200 20 | BiRNN_UNITS = 200 21 | 22 | 23 | def classification_report(y_true, y_pred, labels): 24 | '''Similar to the one in sklearn.metrics, 25 | reports per classs recall, precision and F1 score''' 26 | y_true = numpy.asarray(y_true).ravel() 27 | y_pred = numpy.asarray(y_pred).ravel() 28 | corrects = Counter(yt for yt, yp in zip(y_true, y_pred) if yt == yp) 29 | y_true_counts = Counter(y_true) 30 | y_pred_counts = Counter(y_pred) 31 | report = ((lab, # label 32 | corrects[i] / max(1, y_true_counts[i]), # recall 33 | corrects[i] / max(1, y_pred_counts[i]), # precision 34 | y_true_counts[i] # support 35 | ) for i, lab in enumerate(labels)) 36 | report = [(l, r, p, 2 * r * p / max(1e-9, r + p), s) for l, r, p, s in report] 37 | 38 | print('{:<15}{:>10}{:>10}{:>10}{:>10}\n'.format('', 39 | 'recall', 40 | 'precision', 41 | 'f1-score', 42 | 'support')) 43 | formatter = '{:<15}{:>10.2f}{:>10.2f}{:>10.2f}{:>10d}'.format 44 | for r in report: 45 | print(formatter(*r)) 46 | print('') 47 | report2 = list(zip(*[(r * s, p * s, f1 * s) for l, r, p, f1, s in report])) 48 | N = len(y_true) 49 | print(formatter('avg / total', 50 | sum(report2[0]) / N, 51 | sum(report2[1]) / N, 52 | sum(report2[2]) / N, N) + '\n') 53 | 54 | 55 | # ------ 56 | # Data 57 | # ----- 58 | 59 | # conll200 has two different targets, here will only use 60 | # IBO like chunking as an example 61 | train, test, voc = conll2000.load_data() 62 | (train_x, _, train_y) = train 63 | (test_x, _, test_y) = test 64 | (vocab, _, class_labels) = voc 65 | 66 | # -------------- 67 | # 1. Regular CRF 68 | # -------------- 69 | 70 | print('==== training CRF ====') 71 | 72 | model = Sequential() 73 | model.add(Embedding(len(vocab), EMBED_DIM, mask_zero=True)) # Random embedding 74 | crf = CRF(len(class_labels), sparse_target=True) 75 | model.add(crf) 76 | model.summary() 77 | 78 | # The default `crf_loss` for `learn_mode='join'` is negative log likelihood. 79 | model.compile('adam', loss=crf_loss, metrics=[crf_viterbi_accuracy]) 80 | model.fit(train_x, train_y, epochs=EPOCHS, validation_data=[test_x, test_y]) 81 | 82 | test_y_pred = model.predict(test_x).argmax(-1)[test_x > 0] 83 | test_y_true = test_y[test_x > 0] 84 | 85 | print('\n---- Result of CRF ----\n') 86 | classification_report(test_y_true, test_y_pred, class_labels) 87 | 88 | # ------------- 89 | # 2. BiLSTM-CRF 90 | # ------------- 91 | 92 | print('==== training BiLSTM-CRF ====') 93 | 94 | model = Sequential() 95 | model.add(Embedding(len(vocab), EMBED_DIM, mask_zero=True)) # Random embedding 96 | model.add(Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True))) 97 | crf = CRF(len(class_labels), sparse_target=True) 98 | model.add(crf) 99 | model.summary() 100 | 101 | model.compile('adam', loss=crf_loss, metrics=[crf_viterbi_accuracy]) 102 | model.fit(train_x, train_y, epochs=EPOCHS, validation_data=[test_x, test_y]) 103 | 104 | test_y_pred = model.predict(test_x).argmax(-1)[test_x > 0] 105 | test_y_true = test_y[test_x > 0] 106 | 107 | print('\n---- Result of BiLSTM-CRF ----\n') 108 | classification_report(test_y_true, test_y_pred, class_labels) 109 | -------------------------------------------------------------------------------- /examples/jaccard_loss.py: -------------------------------------------------------------------------------- 1 | import keras 2 | from keras import backend as K 3 | from keras_contrib.losses.jaccard import jaccard_distance 4 | import numpy as np 5 | 6 | # Test and plot 7 | y_pred = np.array([np.arange(-10, 10 + 0.1, 0.1)]).T 8 | y_true = np.zeros(y_pred.shape) 9 | name = 'jaccard_distance_loss' 10 | try: 11 | loss = jaccard_distance_loss( 12 | K.variable(y_true), K.variable(y_pred) 13 | ).eval(session=K.get_session()) 14 | except Exception as e: 15 | print("error plotting", name, e) 16 | else: 17 | plt.title(name) 18 | plt.plot(y_pred, loss) 19 | plt.show() 20 | 21 | print("TYPE |Almost_right |half right |all_wrong") 22 | y_true = np.array([[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1., 0.]]) 23 | y_pred = np.array([[0, 0, 0.9, 0], [0, 0, 0.1, 0], [1, 1, 0.1, 1.]]) 24 | 25 | r = jaccard_distance( 26 | K.variable(y_true), 27 | K.variable(y_pred), 28 | ).eval(session=K.get_session()) 29 | print('jaccard_distance_loss', r) 30 | assert r[0] < r[1] 31 | assert r[1] < r[2] 32 | 33 | r = keras.losses.binary_crossentropy( 34 | K.variable(y_true), 35 | K.variable(y_pred), 36 | ).eval(session=K.get_session()) 37 | print('binary_crossentropy', r) 38 | print('binary_crossentropy_scaled', r / r.max()) 39 | assert r[0] < r[1] 40 | assert r[1] < r[2] 41 | 42 | """ 43 | TYPE |Almost_right |half right |all_wrong 44 | jaccard_distance_loss [ 0.09900928 0.89108944 3.75000238] 45 | binary_crossentropy [ 0.02634021 0.57564634 12.53243446] 46 | binary_crossentropy_scaled [ 0.00210176 0.04593252 1. ] 47 | """ 48 | -------------------------------------------------------------------------------- /keras_contrib/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from . import backend 3 | from . import datasets 4 | from . import layers 5 | from . import preprocessing 6 | from . import utils 7 | from . import wrappers 8 | from . import callbacks 9 | from . import constraints 10 | from . import initializers 11 | from . import metrics 12 | from . import losses 13 | from . import optimizers 14 | from . import regularizers 15 | 16 | __version__ = '0.0.2' 17 | -------------------------------------------------------------------------------- /keras_contrib/activations/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .squash import squash 4 | -------------------------------------------------------------------------------- /keras_contrib/activations/squash.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | 4 | def squash(x, axis=-1): 5 | """ 6 | Squash activation function (generally used in Capsule layers). 7 | """ 8 | s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon() 9 | scale = K.sqrt(s_squared_norm) / (0.5 + s_squared_norm) 10 | return scale * x 11 | -------------------------------------------------------------------------------- /keras_contrib/applications/__init__.py: -------------------------------------------------------------------------------- 1 | from .densenet import DenseNet 2 | from .resnet import ResNet, ResNet18, ResNet34, ResNet50, ResNet101, ResNet152 3 | from .wide_resnet import WideResidualNetwork 4 | from .nasnet import NASNet, NASNetLarge, NASNetMobile 5 | -------------------------------------------------------------------------------- /keras_contrib/backend/__init__.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | # We import all keras backend functions here, 4 | # so that files in this repo can import both 5 | # core and contrib backend functions with a 6 | # single import statement. 7 | 8 | if K.backend() == 'theano': 9 | from .theano_backend import * 10 | elif K.backend() == 'tensorflow': 11 | from .tensorflow_backend import * 12 | elif K.backend() == 'cntk': 13 | from .cntk_backend import * 14 | -------------------------------------------------------------------------------- /keras_contrib/backend/cntk_backend.py: -------------------------------------------------------------------------------- 1 | from keras.backend import cntk_backend as KCN 2 | 3 | 4 | def moments(x, axes, shift=None, keep_dims=False): 5 | ''' Calculates and returns the mean and variance of the input ''' 6 | mean, variant = KCN._moments(x, axes=axes, shift=shift, keep_dims=keep_dims) 7 | return mean, variant 8 | -------------------------------------------------------------------------------- /keras_contrib/backend/numpy_backend.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras import backend as K 3 | 4 | 5 | def extract_image_patches(X, ksizes, strides, 6 | padding='valid', 7 | data_format='channels_first'): 8 | raise NotImplementedError 9 | 10 | 11 | def depth_to_space(input, scale, data_format=None): 12 | raise NotImplementedError 13 | 14 | 15 | def moments(x, axes, shift=None, keep_dims=False): 16 | mean_batch = np.mean(x, axis=tuple(axes), keepdims=keep_dims) 17 | var_batch = np.var(x, axis=tuple(axes), keepdims=keep_dims) 18 | return mean_batch, var_batch 19 | -------------------------------------------------------------------------------- /keras_contrib/backend/tensorflow_backend.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | try: 4 | from tensorflow.python.ops import ctc_ops as ctc 5 | except ImportError: 6 | import tensorflow.contrib.ctc as ctc 7 | import keras.backend as K 8 | 9 | py_all = all 10 | 11 | 12 | def _preprocess_conv2d_input(x, data_format): 13 | """Transpose and cast the input before the conv2d. 14 | 15 | # Arguments 16 | x: input tensor. 17 | data_format: string, `"channels_last"` or `"channels_first"`. 18 | 19 | # Returns 20 | A tensor. 21 | """ 22 | if K.dtype(x) == 'float64': 23 | x = tf.cast(x, 'float32') 24 | if data_format == 'channels_first': 25 | # TF uses the last dimension as channel dimension, 26 | # instead of the 2nd one. 27 | # TH input shape: (samples, input_depth, rows, cols) 28 | # TF input shape: (samples, rows, cols, input_depth) 29 | x = tf.transpose(x, (0, 2, 3, 1)) 30 | return x 31 | 32 | 33 | def _postprocess_conv2d_output(x, data_format): 34 | """Transpose and cast the output from conv2d if needed. 35 | 36 | # Arguments 37 | x: A tensor. 38 | data_format: string, `"channels_last"` or `"channels_first"`. 39 | 40 | # Returns 41 | A tensor. 42 | """ 43 | 44 | if data_format == 'channels_first': 45 | x = tf.transpose(x, (0, 3, 1, 2)) 46 | 47 | if K.floatx() == 'float64': 48 | x = tf.cast(x, 'float64') 49 | return x 50 | 51 | 52 | def _preprocess_padding(padding): 53 | """Convert keras' padding to tensorflow's padding. 54 | 55 | # Arguments 56 | padding: string, `"same"` or `"valid"`. 57 | 58 | # Returns 59 | a string, `"SAME"` or `"VALID"`. 60 | 61 | # Raises 62 | ValueError: if `padding` is invalid. 63 | """ 64 | if padding == 'same': 65 | padding = 'SAME' 66 | elif padding == 'valid': 67 | padding = 'VALID' 68 | else: 69 | raise ValueError('Invalid padding:', padding) 70 | return padding 71 | 72 | 73 | def conv2d(x, kernel, strides=(1, 1), padding='valid', data_format='channels_first', 74 | image_shape=None, filter_shape=None): 75 | """2D convolution. 76 | 77 | # Arguments 78 | x: Input tensor 79 | kernel: kernel tensor. 80 | strides: strides tuple. 81 | padding: string, "same" or "valid". 82 | data_format: 'channels_first' or 'channels_last'. 83 | Whether to use Theano or TensorFlow dimension 84 | ordering in inputs/kernels/ouputs. 85 | image_shape: Optional, the input tensor shape 86 | filter_shape: Optional, the kernel shape. 87 | 88 | # Returns 89 | x convolved with the kernel. 90 | 91 | # Raises 92 | Exception: In case of invalid border mode or data format. 93 | """ 94 | return K.conv2d(x, kernel, strides, padding, data_format) 95 | 96 | 97 | def extract_image_patches(x, ksizes, ssizes, padding='same', 98 | data_format='channels_last'): 99 | """Extract the patches from an image. 100 | 101 | # Arguments 102 | x: The input image 103 | ksizes: 2-d tuple with the kernel size 104 | ssizes: 2-d tuple with the strides size 105 | padding: 'same' or 'valid' 106 | data_format: 'channels_last' or 'channels_first' 107 | 108 | # Returns 109 | The (k_w,k_h) patches extracted 110 | TF ==> (batch_size,w,h,k_w,k_h,c) 111 | TH ==> (batch_size,w,h,c,k_w,k_h) 112 | """ 113 | kernel = [1, ksizes[0], ksizes[1], 1] 114 | strides = [1, ssizes[0], ssizes[1], 1] 115 | padding = _preprocess_padding(padding) 116 | if data_format == 'channels_first': 117 | x = K.permute_dimensions(x, (0, 2, 3, 1)) 118 | bs_i, w_i, h_i, ch_i = K.int_shape(x) 119 | patches = tf.extract_image_patches(x, kernel, strides, [1, 1, 1, 1], 120 | padding) 121 | # Reshaping to fit Theano 122 | bs, w, h, ch = K.int_shape(patches) 123 | reshaped = tf.reshape(patches, [-1, w, h, tf.floordiv(ch, ch_i), ch_i]) 124 | final_shape = [-1, w, h, ch_i, ksizes[0], ksizes[1]] 125 | patches = tf.reshape(tf.transpose(reshaped, [0, 1, 2, 4, 3]), final_shape) 126 | if data_format == 'channels_last': 127 | patches = K.permute_dimensions(patches, [0, 1, 2, 4, 5, 3]) 128 | return patches 129 | 130 | 131 | def depth_to_space(input, scale, data_format=None): 132 | """ Uses phase shift algorithm to convert channels/depth for spatial resolution. 133 | 134 | # Arguments 135 | input: Input tensor 136 | scale: n `int` that is `>= 2`. The size of the spatial block. 137 | data_format: 'channels_first' or 'channels_last'. 138 | Whether to use Theano or TensorFlow dimension 139 | ordering in inputs/kernels/ouputs. 140 | 141 | # Returns 142 | TODO (PR welcome): Filling this section. 143 | """ 144 | if data_format is None: 145 | data_format = K.image_data_format() 146 | data_format = data_format.lower() 147 | input = _preprocess_conv2d_input(input, data_format) 148 | out = tf.depth_to_space(input, scale) 149 | out = _postprocess_conv2d_output(out, data_format) 150 | return out 151 | 152 | 153 | def moments(x, axes, shift=None, keep_dims=False): 154 | ''' Wrapper over tensorflow backend call ''' 155 | 156 | return tf.nn.moments(x, axes, shift=shift, keep_dims=keep_dims) 157 | -------------------------------------------------------------------------------- /keras_contrib/backend/theano_backend.py: -------------------------------------------------------------------------------- 1 | from theano import tensor as T 2 | from theano.sandbox.neighbours import images2neibs 3 | 4 | try: 5 | import theano.sparse as th_sparse_module 6 | except ImportError: 7 | th_sparse_module = None 8 | try: 9 | from theano.tensor.nnet.nnet import softsign as T_softsign 10 | except ImportError: 11 | from theano.sandbox.softsign import softsign as T_softsign 12 | from keras.backend import theano_backend as KTH 13 | from keras.backend.common import image_data_format 14 | from keras.backend.theano_backend import _preprocess_conv2d_input 15 | from keras.backend.theano_backend import _postprocess_conv2d_output 16 | 17 | py_all = all 18 | 19 | 20 | def conv2d(x, kernel, strides=(1, 1), padding='valid', data_format='channels_first', 21 | image_shape=None, filter_shape=None): 22 | ''' 23 | padding: string, "same" or "valid". 24 | ''' 25 | if data_format not in {'channels_first', 'channels_last'}: 26 | raise Exception('Unknown data_format ' + str(data_format)) 27 | 28 | if data_format == 'channels_last': 29 | # TF uses the last dimension as channel dimension, 30 | # instead of the 2nd one. 31 | # TH input shape: (samples, input_depth, rows, cols) 32 | # TF input shape: (samples, rows, cols, input_depth) 33 | # TH kernel shape: (depth, input_depth, rows, cols) 34 | # TF kernel shape: (rows, cols, input_depth, depth) 35 | x = x.dimshuffle((0, 3, 1, 2)) 36 | kernel = kernel.dimshuffle((3, 2, 0, 1)) 37 | if image_shape: 38 | image_shape = (image_shape[0], image_shape[3], 39 | image_shape[1], image_shape[2]) 40 | if filter_shape: 41 | filter_shape = (filter_shape[3], filter_shape[2], 42 | filter_shape[0], filter_shape[1]) 43 | 44 | if padding == 'same': 45 | th_padding = 'half' 46 | np_kernel = kernel.eval() 47 | elif padding == 'valid': 48 | th_padding = 'valid' 49 | else: 50 | raise Exception('Border mode not supported: ' + str(padding)) 51 | 52 | # Theano might not accept long type 53 | def int_or_none(value): 54 | try: 55 | return int(value) 56 | except TypeError: 57 | return None 58 | 59 | if image_shape is not None: 60 | image_shape = tuple(int_or_none(v) for v in image_shape) 61 | 62 | if filter_shape is not None: 63 | filter_shape = tuple(int_or_none(v) for v in filter_shape) 64 | 65 | conv_out = T.nnet.conv2d(x, kernel, 66 | border_mode=th_padding, 67 | subsample=strides, 68 | input_shape=image_shape, 69 | filter_shape=filter_shape) 70 | 71 | if padding == 'same': 72 | if np_kernel.shape[2] % 2 == 0: 73 | end = (x.shape[2] + strides[0] - 1) // strides[0] 74 | conv_out = conv_out[:, :, :end, :] 75 | if np_kernel.shape[3] % 2 == 0: 76 | end = (x.shape[3] + strides[1] - 1) // strides[1] 77 | conv_out = conv_out[:, :, :, :end] 78 | 79 | if data_format == 'channels_last': 80 | conv_out = conv_out.dimshuffle((0, 2, 3, 1)) 81 | return conv_out 82 | 83 | 84 | def extract_image_patches(X, ksizes, strides, 85 | padding='valid', 86 | data_format='channels_first'): 87 | ''' 88 | Extract the patches from an image 89 | Parameters 90 | ---------- 91 | X : The input image 92 | ksizes : 2-d tuple with the kernel size 93 | strides : 2-d tuple with the strides size 94 | padding : 'same' or 'valid' 95 | data_format : 'channels_last' or 'channels_first' 96 | Returns 97 | ------- 98 | The (k_w,k_h) patches extracted 99 | TF ==> (batch_size,w,h,k_w,k_h,c) 100 | TH ==> (batch_size,w,h,c,k_w,k_h) 101 | ''' 102 | patch_size = ksizes[1] 103 | if padding == 'same': 104 | padding = 'ignore_borders' 105 | if data_format == 'channels_last': 106 | X = KTH.permute_dimensions(X, [0, 3, 1, 2]) 107 | # Thanks to https://github.com/awentzonline for the help! 108 | batch, c, w, h = KTH.shape(X) 109 | xs = KTH.shape(X) 110 | num_rows = 1 + (xs[-2] - patch_size) // strides[1] 111 | num_cols = 1 + (xs[-1] - patch_size) // strides[1] 112 | num_channels = xs[-3] 113 | patches = images2neibs(X, ksizes, strides, padding) 114 | # Theano is sorting by channel 115 | new_shape = (batch, num_channels, num_rows * num_cols, patch_size, patch_size) 116 | patches = KTH.reshape(patches, new_shape) 117 | patches = KTH.permute_dimensions(patches, (0, 2, 1, 3, 4)) 118 | # arrange in a 2d-grid (rows, cols, channels, px, py) 119 | new_shape = (batch, num_rows, num_cols, num_channels, patch_size, patch_size) 120 | patches = KTH.reshape(patches, new_shape) 121 | if data_format == 'channels_last': 122 | patches = KTH.permute_dimensions(patches, [0, 1, 2, 4, 5, 3]) 123 | return patches 124 | 125 | 126 | def depth_to_space(input, scale, data_format=None): 127 | """Uses phase shift algorithm to convert 128 | channels/depth for spatial resolution 129 | """ 130 | if data_format is None: 131 | data_format = image_data_format() 132 | data_format = data_format.lower() 133 | input = _preprocess_conv2d_input(input, data_format) 134 | 135 | b, k, row, col = input.shape 136 | out_channels = k // (scale ** 2) 137 | x = T.reshape(input, (b, scale, scale, out_channels, row, col)) 138 | x = T.transpose(x, (0, 3, 4, 1, 5, 2)) 139 | out = T.reshape(x, (b, out_channels, row * scale, col * scale)) 140 | 141 | out = _postprocess_conv2d_output(out, input, None, None, None, data_format) 142 | return out 143 | 144 | 145 | def moments(x, axes, shift=None, keep_dims=False): 146 | ''' Calculates and returns the mean and variance of the input ''' 147 | 148 | mean_batch = KTH.mean(x, axis=axes, keepdims=keep_dims) 149 | var_batch = KTH.var(x, axis=axes, keepdims=keep_dims) 150 | 151 | return mean_batch, var_batch 152 | -------------------------------------------------------------------------------- /keras_contrib/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | from .snapshot import SnapshotCallbackBuilder, SnapshotModelCheckpoint 2 | from .dead_relu_detector import DeadReluDetector 3 | from .cyclical_learning_rate import CyclicLR 4 | from .tensorboard import TensorBoardGrouped 5 | -------------------------------------------------------------------------------- /keras_contrib/callbacks/dead_relu_detector.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras.callbacks import Callback 4 | from keras import backend as K 5 | 6 | 7 | class DeadReluDetector(Callback): 8 | """Reports the number of dead ReLUs after each training epoch 9 | ReLU is considered to be dead if it did not fire once for entire training set 10 | 11 | # Arguments 12 | x_train: Training dataset to check whether or not neurons fire 13 | verbose: verbosity mode 14 | True means that even a single dead neuron triggers a warning message 15 | False means that only significant number of dead neurons (10% or more) 16 | triggers a warning message 17 | """ 18 | 19 | def __init__(self, x_train, verbose=False): 20 | super(DeadReluDetector, self).__init__() 21 | self.x_train = x_train 22 | self.verbose = verbose 23 | self.dead_neurons_share_threshold = 0.1 24 | 25 | @staticmethod 26 | def is_relu_layer(layer): 27 | # Should work for all layers with relu 28 | # activation. Tested for Dense and Conv2D 29 | return layer.get_config().get('activation', None) == 'relu' 30 | 31 | def get_relu_activations(self): 32 | model_input = self.model.input 33 | is_multi_input = isinstance(model_input, list) 34 | if not is_multi_input: 35 | model_input = [model_input] 36 | 37 | funcs = {} 38 | for index, layer in enumerate(self.model.layers): 39 | if not layer.get_weights(): 40 | continue 41 | funcs[index] = K.function(model_input 42 | + [K.learning_phase()], [layer.output]) 43 | 44 | if is_multi_input: 45 | list_inputs = [] 46 | list_inputs.extend(self.x_train) 47 | list_inputs.append(1.) 48 | else: 49 | list_inputs = [self.x_train, 1.] 50 | 51 | layer_outputs = {} 52 | for index, func in funcs.items(): 53 | layer_outputs[index] = func(list_inputs)[0] 54 | 55 | for layer_index, layer_activations in layer_outputs.items(): 56 | if self.is_relu_layer(self.model.layers[layer_index]): 57 | layer_name = self.model.layers[layer_index].name 58 | # layer_weight is a list [W] (+ [b]) 59 | layer_weight = self.model.layers[layer_index].get_weights() 60 | 61 | # with kernel and bias, the weights are saved as a list [W, b]. 62 | # If only weights, it is [W] 63 | if type(layer_weight) is not list: 64 | raise ValueError("'Layer_weight' should be a list, " 65 | "but was {}".format(type(layer_weight))) 66 | 67 | # there are no weights for current layer; skip it 68 | # this is only legitimate if layer is "Activation" 69 | if len(layer_weight) == 0: 70 | continue 71 | 72 | layer_weight_shape = np.shape(layer_weight[0]) 73 | yield [layer_index, 74 | layer_activations, 75 | layer_name, 76 | layer_weight_shape] 77 | 78 | def on_epoch_end(self, epoch, logs={}): 79 | for relu_activation in self.get_relu_activations(): 80 | layer_index = relu_activation[0] 81 | activation_values = relu_activation[1] 82 | layer_name = relu_activation[2] 83 | layer_weight_shape = relu_activation[3] 84 | 85 | shape_act = activation_values.shape 86 | 87 | weight_len = len(layer_weight_shape) 88 | act_len = len(shape_act) 89 | 90 | # should work for both Conv and Flat 91 | if K.image_data_format() == 'channels_last': 92 | # features in last axis 93 | axis_filter = -1 94 | else: 95 | # features before the convolution axis, for weight_ 96 | # len the input and output have to be subtracted 97 | axis_filter = -1 - (weight_len - 2) 98 | 99 | total_featuremaps = shape_act[axis_filter] 100 | 101 | axis = [] 102 | for i in range(act_len): 103 | if (i != axis_filter) and (i != (len(shape_act) + axis_filter)): 104 | axis.append(i) 105 | axis = tuple(axis) 106 | 107 | dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) 108 | 109 | dead_neurons_share = float(dead_neurons) / float(total_featuremaps) 110 | if ((self.verbose and dead_neurons > 0) 111 | or dead_neurons_share >= self.dead_neurons_share_threshold): 112 | str_warning = ('Layer {} (#{}) has {} ' 113 | 'dead neurons ({:.2%})!').format(layer_name, 114 | layer_index, 115 | dead_neurons, 116 | dead_neurons_share) 117 | print(str_warning) 118 | -------------------------------------------------------------------------------- /keras_contrib/callbacks/snapshot.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | 4 | import os 5 | 6 | import numpy as np 7 | 8 | from keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler 9 | 10 | try: 11 | import requests 12 | except ImportError: 13 | requests = None 14 | 15 | 16 | class SnapshotModelCheckpoint(Callback): 17 | """Callback that saves the snapshot weights of the model. 18 | 19 | Saves the model weights on certain epochs (which can be considered the 20 | snapshot of the model at that epoch). 21 | 22 | Should be used with the cosine annealing learning rate schedule to save 23 | the weight just before learning rate is sharply increased. 24 | 25 | # Arguments: 26 | nb_epochs: total number of epochs that the model will be trained for. 27 | nb_snapshots: number of times the weights of the model will be saved. 28 | fn_prefix: prefix for the filename of the weights. 29 | """ 30 | 31 | def __init__(self, nb_epochs, nb_snapshots, fn_prefix='Model'): 32 | super(SnapshotModelCheckpoint, self).__init__() 33 | 34 | self.check = nb_epochs // nb_snapshots 35 | self.fn_prefix = fn_prefix 36 | 37 | def on_epoch_end(self, epoch, logs={}): 38 | if epoch != 0 and (epoch + 1) % self.check == 0: 39 | filepath = self.fn_prefix + '-%d.h5' % ((epoch + 1) // self.check) 40 | self.model.save_weights(filepath, overwrite=True) 41 | # print("Saved snapshot at weights/%s_%d.h5" % (self.fn_prefix, epoch)) 42 | 43 | 44 | class SnapshotCallbackBuilder: 45 | """Callback builder for snapshot ensemble training of a model. 46 | From the paper "Snapshot Ensembles: Train 1, Get M For Free" ( 47 | https://openreview.net/pdf?id=BJYwwY9ll) 48 | 49 | Creates a list of callbacks, which are provided when training a model 50 | so as to save the model weights at certain epochs, and then sharply 51 | increase the learning rate. 52 | """ 53 | 54 | def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1): 55 | """ 56 | Initialize a snapshot callback builder. 57 | 58 | # Arguments: 59 | nb_epochs: total number of epochs that the model will be trained for. 60 | nb_snapshots: number of times the weights of the model will be saved. 61 | init_lr: initial learning rate 62 | """ 63 | self.T = nb_epochs 64 | self.M = nb_snapshots 65 | self.alpha_zero = init_lr 66 | 67 | def get_callbacks(self, model_prefix='Model'): 68 | """ 69 | Creates a list of callbacks that can be used during training to create a 70 | snapshot ensemble of the model. 71 | 72 | Args: 73 | model_prefix: prefix for the filename of the weights. 74 | 75 | Returns: list of 3 callbacks [ModelCheckpoint, LearningRateScheduler, 76 | SnapshotModelCheckpoint] which can be provided to the 'fit' function 77 | """ 78 | if not os.path.exists('weights/'): 79 | os.makedirs('weights/') 80 | 81 | callback_list = [ModelCheckpoint('weights/%s-Best.h5' % model_prefix, 82 | monitor='val_acc', 83 | save_best_only=True, save_weights_only=True), 84 | LearningRateScheduler(schedule=self._cosine_anneal_schedule), 85 | SnapshotModelCheckpoint(self.T, 86 | self.M, 87 | fn_prefix='weights/%s' % model_prefix)] 88 | 89 | return callback_list 90 | 91 | def _cosine_anneal_schedule(self, t): 92 | cos_inner = np.pi * (t % (self.T // self.M)) 93 | cos_inner /= self.T // self.M 94 | cos_out = np.cos(cos_inner) + 1 95 | return float(self.alpha_zero / 2 * cos_out) 96 | -------------------------------------------------------------------------------- /keras_contrib/callbacks/tensorboard.py: -------------------------------------------------------------------------------- 1 | from keras.callbacks import TensorBoard 2 | import numpy as np 3 | import os 4 | 5 | 6 | class TensorBoardGrouped(TensorBoard): 7 | """TensorBoard basic visualizations. 8 | 9 | [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) 10 | is a visualization tool provided with TensorFlow. 11 | 12 | This callback is a subclass of `keras.callbacks.TensorBoard`. 13 | The only difference is that the training and validation logs are 14 | grouped and written to the same plot. 15 | 16 | It's a drop-in replacement for the keras callback. 17 | The arguments are the same. 18 | """ 19 | 20 | def __init__(self, log_dir='./logs', *args, **kwargs): 21 | self.base_log_dir = log_dir 22 | self.train_log_dir = os.path.join(log_dir, 'train') 23 | self.val_log_dir = os.path.join(log_dir, 'val') 24 | super(TensorBoardGrouped, self).__init__(self.train_log_dir, 25 | *args, 26 | **kwargs) 27 | 28 | def set_model(self, model): 29 | super(TensorBoardGrouped, self).set_model(model) 30 | import tensorflow as tf 31 | self.val_writer = tf.summary.FileWriter(self.val_log_dir) 32 | 33 | def _write_logs(self, logs, index): 34 | import tensorflow as tf 35 | for name, value in logs.items(): 36 | if name in ['batch', 'size']: 37 | continue 38 | if name.startswith('val_'): 39 | writer = self.val_writer 40 | name = name[4:] # remove val_ 41 | else: 42 | writer = self.writer 43 | summary = tf.Summary() 44 | summary_value = summary.value.add() 45 | if isinstance(value, np.ndarray): 46 | summary_value.simple_value = value.item() 47 | else: 48 | summary_value.simple_value = value 49 | summary_value.tag = name 50 | writer.add_summary(summary, index) 51 | self.writer.flush() 52 | self.val_writer.flush() 53 | 54 | def on_train_end(self, _): 55 | self.writer.close() 56 | self.val_writer.flush() 57 | -------------------------------------------------------------------------------- /keras_contrib/constraints/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .clip import Clip 4 | 5 | # Aliases. 6 | 7 | clip = Clip 8 | -------------------------------------------------------------------------------- /keras_contrib/constraints/clip.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from keras import backend as K 3 | from keras.constraints import Constraint 4 | 5 | 6 | class Clip(Constraint): 7 | """Clips weights to [-c, c]. 8 | 9 | # Arguments 10 | c: Clipping parameter. 11 | """ 12 | 13 | def __init__(self, c=0.01): 14 | self.c = c 15 | 16 | def __call__(self, p): 17 | return K.clip(p, -self.c, self.c) 18 | 19 | def get_config(self): 20 | return {'name': self.__class__.__name__, 21 | 'c': self.c} 22 | -------------------------------------------------------------------------------- /keras_contrib/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/datasets/__init__.py -------------------------------------------------------------------------------- /keras_contrib/datasets/conll2000.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy 3 | from keras.utils.data_utils import get_file 4 | from zipfile import ZipFile 5 | from collections import Counter 6 | from keras.preprocessing.sequence import pad_sequences 7 | 8 | 9 | def load_data(path='conll2000.zip', min_freq=2): 10 | path = get_file(path, 11 | origin='https://raw.githubusercontent.com/nltk' 12 | '/nltk_data/gh-pages/packages/corpora/conll2000.zip') 13 | print(path) 14 | archive = ZipFile(path, 'r') 15 | train = _parse_data(archive.open('conll2000/train.txt')) 16 | test = _parse_data(archive.open('conll2000/test.txt')) 17 | archive.close() 18 | 19 | word_counts = Counter(row[0].lower() for sample in train for row in sample) 20 | vocab = ['', ''] 21 | vocab += [w for w, f in iter(word_counts.items()) if f >= min_freq] 22 | # in alphabetic order 23 | pos_tags = sorted(list(set(row[1] for sample in train + test for row in sample))) 24 | # in alphabetic order 25 | chunk_tags = sorted(list(set(row[2] for sample in train + test for row in sample))) 26 | 27 | train = _process_data(train, vocab, pos_tags, chunk_tags) 28 | test = _process_data(test, vocab, pos_tags, chunk_tags) 29 | return train, test, (vocab, pos_tags, chunk_tags) 30 | 31 | 32 | def _parse_data(fh): 33 | string = fh.read() 34 | data = [] 35 | for sample in string.decode().strip().split('\n\n'): 36 | data.append([row.split() for row in sample.split('\n')]) 37 | fh.close() 38 | return data 39 | 40 | 41 | def _process_data(data, vocab, pos_tags, chunk_tags, maxlen=None, onehot=False): 42 | if maxlen is None: 43 | maxlen = max(len(s) for s in data) 44 | word2idx = dict((w, i) for i, w in enumerate(vocab)) 45 | # set to (index 1) if not in vocab 46 | x = [[word2idx.get(w[0].lower(), 1) for w in s] for s in data] 47 | 48 | y_pos = [[pos_tags.index(w[1]) for w in s] for s in data] 49 | y_chunk = [[chunk_tags.index(w[2]) for w in s] for s in data] 50 | 51 | x = pad_sequences(x, maxlen) # left padding 52 | 53 | # lef padded with -1. Indeed, any integer works as it will be masked 54 | y_pos = pad_sequences(y_pos, maxlen, value=-1) 55 | y_chunk = pad_sequences(y_chunk, maxlen, value=-1) 56 | 57 | if onehot: 58 | y_pos = numpy.eye(len(pos_tags), dtype='float32')[y] 59 | y_chunk = numpy.eye(len(chunk_tags), dtype='float32')[y] 60 | else: 61 | y_pos = numpy.expand_dims(y_pos, 2) 62 | y_chunk = numpy.expand_dims(y_chunk, 2) 63 | return x, y_pos, y_chunk 64 | -------------------------------------------------------------------------------- /keras_contrib/initializers/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .convaware import ConvolutionAware 4 | -------------------------------------------------------------------------------- /keras_contrib/initializers/convaware.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import numpy as np 3 | from keras import backend as K 4 | from keras.initializers import Initializer, Orthogonal 5 | 6 | 7 | class ConvolutionAware(Initializer): 8 | """ 9 | Initializer that generates orthogonal convolution filters in the fourier 10 | space. If this initializer is passed a shape that is not 3D or 4D, 11 | orthogonal initialization will be used. 12 | # Arguments 13 | eps_std: Standard deviation for the random normal noise used to break 14 | symmetry in the inverse fourier transform. 15 | seed: A Python integer. Used to seed the random generator. 16 | # References 17 | Armen Aghajanyan, https://arxiv.org/abs/1702.06295 18 | """ 19 | 20 | def __init__(self, eps_std=0.05, seed=None): 21 | self.eps_std = eps_std 22 | self.seed = seed 23 | self.orthogonal = Orthogonal() 24 | 25 | def __call__(self, shape): 26 | rank = len(shape) 27 | 28 | if self.seed is not None: 29 | np.random.seed(self.seed) 30 | 31 | fan_in, fan_out = _compute_fans(shape, K.image_data_format()) 32 | variance = 2 / fan_in 33 | 34 | if rank == 3: 35 | row, stack_size, filters_size = shape 36 | 37 | transpose_dimensions = (2, 1, 0) 38 | kernel_shape = (row,) 39 | correct_ifft = lambda shape, s=[None]: np.fft.irfft(shape, s[0]) 40 | correct_fft = np.fft.rfft 41 | 42 | elif rank == 4: 43 | row, column, stack_size, filters_size = shape 44 | 45 | transpose_dimensions = (2, 3, 0, 1) 46 | kernel_shape = (row, column) 47 | correct_ifft = np.fft.irfft2 48 | correct_fft = np.fft.rfft2 49 | 50 | elif rank == 5: 51 | x, y, z, stack_size, filters_size = shape 52 | 53 | transpose_dimensions = (3, 4, 0, 1, 2) 54 | kernel_shape = (x, y, z) 55 | correct_fft = np.fft.rfftn 56 | correct_ifft = np.fft.irfftn 57 | else: 58 | return K.variable(self.orthogonal(shape), dtype=K.floatx()) 59 | 60 | kernel_fourier_shape = correct_fft(np.zeros(kernel_shape)).shape 61 | 62 | init = [] 63 | for i in range(filters_size): 64 | basis = self._create_basis( 65 | stack_size, np.prod(kernel_fourier_shape)) 66 | basis = basis.reshape((stack_size,) + kernel_fourier_shape) 67 | 68 | filters = [correct_ifft(x, kernel_shape) + 69 | np.random.normal(0, self.eps_std, kernel_shape) for 70 | x in basis] 71 | 72 | init.append(filters) 73 | 74 | # Format of array is now: filters, stack, row, column 75 | init = np.array(init) 76 | init = self._scale_filters(init, variance) 77 | return init.transpose(transpose_dimensions) 78 | 79 | def _create_basis(self, filters, size): 80 | if size == 1: 81 | return np.random.normal(0.0, self.eps_std, (filters, size)) 82 | 83 | nbb = filters // size + 1 84 | li = [] 85 | for i in range(nbb): 86 | a = np.random.normal(0.0, 1.0, (size, size)) 87 | a = self._symmetrize(a) 88 | u, _, v = np.linalg.svd(a) 89 | li.extend(u.T.tolist()) 90 | p = np.array(li[:filters], dtype=K.floatx()) 91 | return p 92 | 93 | def _symmetrize(self, a): 94 | return a + a.T - np.diag(a.diagonal()) 95 | 96 | def _scale_filters(self, filters, variance): 97 | c_var = np.var(filters) 98 | p = np.sqrt(variance / c_var) 99 | return filters * p 100 | 101 | def get_config(self): 102 | return { 103 | 'eps_std': self.eps_std, 104 | 'seed': self.seed 105 | } 106 | 107 | 108 | def _compute_fans(shape, data_format='channels_last'): 109 | """Computes the number of input and output units for a weight shape. 110 | 111 | # Arguments 112 | shape: Integer shape tuple. 113 | data_format: Image data format to use for convolution kernels. 114 | Note that all kernels in Keras are standardized on the 115 | `channels_last` ordering (even when inputs are set 116 | to `channels_first`). 117 | 118 | # Returns 119 | A tuple of scalars, `(fan_in, fan_out)`. 120 | 121 | # Raises 122 | ValueError: in case of invalid `data_format` argument. 123 | """ 124 | if len(shape) == 2: 125 | fan_in = shape[0] 126 | fan_out = shape[1] 127 | elif len(shape) in {3, 4, 5}: 128 | # Assuming convolution kernels (1D, 2D or 3D). 129 | # TH kernel shape: (depth, input_depth, ...) 130 | # TF kernel shape: (..., input_depth, depth) 131 | if data_format == 'channels_first': 132 | receptive_field_size = np.prod(shape[2:]) 133 | fan_in = shape[1] * receptive_field_size 134 | fan_out = shape[0] * receptive_field_size 135 | elif data_format == 'channels_last': 136 | receptive_field_size = np.prod(shape[:-2]) 137 | fan_in = shape[-2] * receptive_field_size 138 | fan_out = shape[-1] * receptive_field_size 139 | else: 140 | raise ValueError('Invalid data_format: ' + data_format) 141 | else: 142 | # No specific assumptions. 143 | fan_in = np.sqrt(np.prod(shape)) 144 | fan_out = np.sqrt(np.prod(shape)) 145 | return fan_in, fan_out 146 | -------------------------------------------------------------------------------- /keras_contrib/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .advanced_activations.pelu import PELU 4 | from .advanced_activations.srelu import SReLU 5 | from .advanced_activations.swish import Swish 6 | from .advanced_activations.sinerelu import SineReLU 7 | 8 | from .convolutional.cosineconvolution2d import CosineConv2D 9 | from .convolutional.cosineconvolution2d import CosineConvolution2D 10 | from .convolutional.subpixelupscaling import SubPixelUpscaling 11 | 12 | from .core import CosineDense 13 | 14 | from .crf import CRF 15 | 16 | from .capsule import Capsule 17 | 18 | from .normalization.instancenormalization import InstanceNormalization 19 | from .normalization.groupnormalization import GroupNormalization 20 | -------------------------------------------------------------------------------- /keras_contrib/layers/advanced_activations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/layers/advanced_activations/__init__.py -------------------------------------------------------------------------------- /keras_contrib/layers/advanced_activations/pelu.py: -------------------------------------------------------------------------------- 1 | from keras.layers import Layer, InputSpec 2 | from keras import initializers, regularizers, constraints 3 | import keras.backend as K 4 | from keras_contrib.utils.test_utils import to_tuple 5 | 6 | 7 | class PELU(Layer): 8 | """Parametric Exponential Linear Unit. 9 | 10 | It follows: 11 | `f(x) = alphas * (exp(x / betas) - 1) for x < 0`, 12 | `f(x) = (alphas / betas) * x for x >= 0`, 13 | where `alphas` & `betas` are learned arrays with the same shape as x. 14 | 15 | # Input shape 16 | Arbitrary. Use the keyword argument `input_shape` 17 | (tuple of integers, does not include the samples axis) 18 | when using this layer as the first layer in a model. 19 | 20 | # Output shape 21 | Same shape as the input. 22 | 23 | # Arguments 24 | alphas_initializer: initialization function for the alpha variable weights. 25 | betas_initializer: initialization function for the beta variable weights. 26 | weights: initial weights, as a list of a single Numpy array. 27 | shared_axes: the axes along which to share learnable 28 | parameters for the activation function. 29 | For example, if the incoming feature maps 30 | are from a 2D convolution 31 | with output shape `(batch, height, width, channels)`, 32 | and you wish to share parameters across space 33 | so that each filter only has one set of parameters, 34 | set `shared_axes=[1, 2]`. 35 | 36 | # References 37 | - [Parametric exponential linear unit for deep convolutional neural networks]( 38 | https://arxiv.org/abs/1605.09332v3) 39 | """ 40 | 41 | def __init__(self, alpha_initializer='ones', 42 | alpha_regularizer=None, 43 | alpha_constraint=None, 44 | beta_initializer='ones', 45 | beta_regularizer=None, 46 | beta_constraint=None, 47 | shared_axes=None, 48 | **kwargs): 49 | super(PELU, self).__init__(**kwargs) 50 | self.supports_masking = True 51 | self.alpha_initializer = initializers.get(alpha_initializer) 52 | self.alpha_regularizer = regularizers.get(alpha_regularizer) 53 | self.alpha_constraint = constraints.get(alpha_constraint) 54 | self.beta_initializer = initializers.get(beta_initializer) 55 | self.beta_regularizer = regularizers.get(beta_regularizer) 56 | self.beta_constraint = constraints.get(beta_constraint) 57 | if shared_axes is None: 58 | self.shared_axes = None 59 | elif not isinstance(shared_axes, (list, tuple)): 60 | self.shared_axes = [shared_axes] 61 | else: 62 | self.shared_axes = list(shared_axes) 63 | 64 | def build(self, input_shape): 65 | input_shape = to_tuple(input_shape) 66 | param_shape = list(input_shape[1:]) 67 | self.param_broadcast = [False] * len(param_shape) 68 | if self.shared_axes is not None: 69 | for i in self.shared_axes: 70 | param_shape[i - 1] = 1 71 | self.param_broadcast[i - 1] = True 72 | 73 | param_shape = tuple(param_shape) 74 | # Initialised as ones to emulate the default ELU 75 | self.alpha = self.add_weight(shape=param_shape, 76 | name='alpha', 77 | initializer=self.alpha_initializer, 78 | regularizer=self.alpha_regularizer, 79 | constraint=self.alpha_constraint) 80 | self.beta = self.add_weight(shape=param_shape, 81 | name='beta', 82 | initializer=self.beta_initializer, 83 | regularizer=self.beta_regularizer, 84 | constraint=self.beta_constraint) 85 | 86 | # Set input spec 87 | axes = {} 88 | if self.shared_axes: 89 | for i in range(1, len(input_shape)): 90 | if i not in self.shared_axes: 91 | axes[i] = input_shape[i] 92 | self.input_spec = InputSpec(ndim=len(input_shape), axes=axes) 93 | self.built = True 94 | 95 | def call(self, x, mask=None): 96 | if K.backend() == 'theano': 97 | pos = K.relu(x) * (K.pattern_broadcast(self.alpha, self.param_broadcast) / 98 | K.pattern_broadcast(self.beta, self.param_broadcast)) 99 | neg = (K.pattern_broadcast(self.alpha, self.param_broadcast) * 100 | (K.exp((-K.relu(-x)) 101 | / K.pattern_broadcast(self.beta, self.param_broadcast)) - 1)) 102 | else: 103 | pos = K.relu(x) * self.alpha / self.beta 104 | neg = self.alpha * (K.exp((-K.relu(-x)) / self.beta) - 1) 105 | return neg + pos 106 | 107 | def get_config(self): 108 | config = { 109 | 'alpha_initializer': initializers.serialize(self.alpha_initializer), 110 | 'alpha_regularizer': regularizers.serialize(self.alpha_regularizer), 111 | 'alpha_constraint': constraints.serialize(self.alpha_constraint), 112 | 'beta_initializer': initializers.serialize(self.beta_initializer), 113 | 'beta_regularizer': regularizers.serialize(self.beta_regularizer), 114 | 'beta_constraint': constraints.serialize(self.beta_constraint), 115 | 'shared_axes': self.shared_axes 116 | } 117 | base_config = super(PELU, self).get_config() 118 | return dict(list(base_config.items()) + list(config.items())) 119 | 120 | def compute_output_shape(self, input_shape): 121 | return input_shape 122 | -------------------------------------------------------------------------------- /keras_contrib/layers/advanced_activations/sinerelu.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | from keras.layers import Layer 3 | 4 | 5 | class SineReLU(Layer): 6 | """Sine Rectified Linear Unit to generate oscilations. 7 | 8 | It allows an oscilation in the gradients when the weights are negative. 9 | The oscilation can be controlled with a parameter, which makes it be close 10 | or equal to zero. The functional is diferentiable at any point due to 11 | its derivative. 12 | For instance, at 0, the derivative of 'sin(0) - cos(0)' 13 | is 'cos(0) + sin(0)' which is 1. 14 | 15 | # Input shape 16 | Arbitrary. Use the keyword argument `input_shape` 17 | (tuple of integers, does not include the samples axis) 18 | when using this layer as the first layer in a model. 19 | 20 | # Output shape 21 | Same shape as the input. 22 | 23 | # Arguments 24 | epsilon: float. Hyper-parameter used to control the amplitude of the 25 | sinusoidal wave when weights are negative. 26 | The default value, 0.0025, since it works better for CNN layers and 27 | those are the most used layers nowadays. 28 | When using Dense Networks, try something around 0.006. 29 | 30 | # References: 31 | - [SineReLU: An Alternative to the ReLU Activation Function]( 32 | https://medium.com/@wilder.rodrigues/sinerelu-an-alternative-to-the-relu-activation-function-e46a6199997d). 33 | 34 | This function was 35 | first introduced at the Codemotion Amsterdam 2018 and then at 36 | the DevDays, in Vilnius, Lithuania. 37 | It has been extensively tested with Deep Nets, CNNs, 38 | LSTMs, Residual Nets and GANs, based 39 | on the MNIST, Kaggle Toxicity and IMDB datasets. 40 | 41 | # Performance: 42 | 43 | - Fashion MNIST 44 | * Mean of 6 runs per Activation Function 45 | * Fully Connection Network 46 | - SineReLU: loss mean -> 0.3522; accuracy mean -> 89.18; 47 | mean of std loss -> 0.08375204467435822 48 | - LeakyReLU: loss mean-> 0.3553; accuracy mean -> 88.98; 49 | mean of std loss -> 0.0831161868455245 50 | - ReLU: loss mean -> 0.3519; accuracy mean -> 88.84; 51 | mean of std loss -> 0.08358816501301362 52 | * Convolutional Neural Network 53 | - SineReLU: loss mean -> 0.2180; accuracy mean -> 92.49; 54 | mean of std loss -> 0.0781155784858847 55 | - LeakyReLU: loss mean -> 0.2205; accuracy mean -> 92.37; 56 | mean of std loss -> 0.09273670474788205 57 | - ReLU: loss mean -> 0.2144; accuracy mean -> 92.45; 58 | mean of std loss -> 0.09396114585977 59 | - MNIST 60 | * Mean of 6 runs per Activation Function 61 | * Fully Connection Network 62 | - SineReLU: loss mean -> 0.0623; accuracy mean -> 98.53; 63 | mean of std loss -> 0.06012015231824904 64 | - LeakyReLU: loss mean-> 0.0623; accuracy mean -> 98.50; 65 | mean of std loss -> 0.06052147632835356 66 | - ReLU: loss mean -> 0.0605; accuracy mean -> 98.49; 67 | mean of std loss -> 0.059599885665016096 68 | * Convolutional Neural Network 69 | - SineReLU: loss mean -> 0.0198; accuracy mean -> 99.51; 70 | mean of std loss -> 0.0425338329550847 71 | - LeakyReLU: loss mean -> 0.0216; accuracy mean -> 99.40; 72 | mean of std loss -> 0.04834468835196667 73 | - ReLU: loss mean -> 0.0185; accuracy mean -> 99.49; 74 | mean of std loss -> 0.05503719489690131 75 | 76 | # Jupyter Notebooks 77 | - https://github.com/ekholabs/DLinK/blob/master/notebooks/keras 78 | 79 | # Examples 80 | The Advanced Activation function SineReLU have to be imported from the 81 | keras_contrib.layers package. 82 | 83 | To see full source-code of this architecture and other examples, 84 | please follow this link: https://github.com/ekholabs/DLinK 85 | 86 | ```python 87 | model = Sequential() 88 | model.add(Dense(128, input_shape = (784,))) 89 | model.add(SineReLU()) 90 | model.add(Dropout(0.2)) 91 | 92 | model.add(Dense(256)) 93 | model.add(SineReLU()) 94 | model.add(Dropout(0.3)) 95 | 96 | model.add(Dense(1024)) 97 | model.add(SineReLU()) 98 | model.add(Dropout(0.5)) 99 | 100 | model.add(Dense(10, activation = 'softmax')) 101 | ``` 102 | """ 103 | 104 | def __init__(self, epsilon=0.0025, **kwargs): 105 | super(SineReLU, self).__init__(**kwargs) 106 | self.supports_masking = True 107 | self.epsilon = K.cast_to_floatx(epsilon) 108 | 109 | def call(self, Z): 110 | m = self.epsilon * (K.sin(Z) - K.cos(Z)) 111 | A = K.maximum(m, Z) 112 | return A 113 | 114 | def get_config(self): 115 | config = {'epsilon': float(self.epsilon)} 116 | base_config = super(SineReLU, self).get_config() 117 | return dict(list(base_config.items()) + list(config.items())) 118 | 119 | def compute_output_shape(self, input_shape): 120 | return input_shape 121 | -------------------------------------------------------------------------------- /keras_contrib/layers/advanced_activations/srelu.py: -------------------------------------------------------------------------------- 1 | from keras.layers import Layer, InputSpec 2 | from keras import initializers 3 | import keras.backend as K 4 | from keras_contrib.utils.test_utils import to_tuple 5 | 6 | 7 | class SReLU(Layer): 8 | """S-shaped Rectified Linear Unit. 9 | 10 | It follows: 11 | `f(x) = t^r + a^r(x - t^r) for x >= t^r`, 12 | `f(x) = x for t^r > x > t^l`, 13 | `f(x) = t^l + a^l(x - t^l) for x <= t^l`. 14 | 15 | # Input shape 16 | Arbitrary. Use the keyword argument `input_shape` 17 | (tuple of integers, does not include the samples axis) 18 | when using this layer as the first layer in a model. 19 | 20 | # Output shape 21 | Same shape as the input. 22 | 23 | # Arguments 24 | t_left_initializer: initializer function for the left part intercept 25 | a_left_initializer: initializer function for the left part slope 26 | t_right_initializer: initializer function for the right part intercept 27 | a_right_initializer: initializer function for the right part slope 28 | shared_axes: the axes along which to share learnable 29 | parameters for the activation function. 30 | For example, if the incoming feature maps 31 | are from a 2D convolution 32 | with output shape `(batch, height, width, channels)`, 33 | and you wish to share parameters across space 34 | so that each filter only has one set of parameters, 35 | set `shared_axes=[1, 2]`. 36 | 37 | # References 38 | - [Deep Learning with S-shaped Rectified Linear Activation Units]( 39 | http://arxiv.org/abs/1512.07030) 40 | """ 41 | 42 | def __init__(self, t_left_initializer='zeros', 43 | a_left_initializer=initializers.RandomUniform(minval=0, maxval=1), 44 | t_right_initializer=initializers.RandomUniform(minval=0, maxval=5), 45 | a_right_initializer='ones', 46 | shared_axes=None, 47 | **kwargs): 48 | super(SReLU, self).__init__(**kwargs) 49 | self.supports_masking = True 50 | self.t_left_initializer = initializers.get(t_left_initializer) 51 | self.a_left_initializer = initializers.get(a_left_initializer) 52 | self.t_right_initializer = initializers.get(t_right_initializer) 53 | self.a_right_initializer = initializers.get(a_right_initializer) 54 | if shared_axes is None: 55 | self.shared_axes = None 56 | elif not isinstance(shared_axes, (list, tuple)): 57 | self.shared_axes = [shared_axes] 58 | else: 59 | self.shared_axes = list(shared_axes) 60 | 61 | def build(self, input_shape): 62 | input_shape = to_tuple(input_shape) 63 | param_shape = list(input_shape[1:]) 64 | self.param_broadcast = [False] * len(param_shape) 65 | if self.shared_axes is not None: 66 | for i in self.shared_axes: 67 | param_shape[i - 1] = 1 68 | self.param_broadcast[i - 1] = True 69 | 70 | param_shape = tuple(param_shape) 71 | 72 | self.t_left = self.add_weight(shape=param_shape, 73 | name='t_left', 74 | initializer=self.t_left_initializer) 75 | 76 | self.a_left = self.add_weight(shape=param_shape, 77 | name='a_left', 78 | initializer=self.a_left_initializer) 79 | 80 | self.t_right = self.add_weight(shape=param_shape, 81 | name='t_right', 82 | initializer=self.t_right_initializer) 83 | 84 | self.a_right = self.add_weight(shape=param_shape, 85 | name='a_right', 86 | initializer=self.a_right_initializer) 87 | 88 | # Set input spec 89 | axes = {} 90 | if self.shared_axes: 91 | for i in range(1, len(input_shape)): 92 | if i not in self.shared_axes: 93 | axes[i] = input_shape[i] 94 | self.input_spec = InputSpec(ndim=len(input_shape), axes=axes) 95 | self.built = True 96 | 97 | def call(self, x, mask=None): 98 | # ensure the the right part is always to the right of the left 99 | t_right_actual = self.t_left + K.abs(self.t_right) 100 | 101 | if K.backend() == 'theano': 102 | t_left = K.pattern_broadcast(self.t_left, self.param_broadcast) 103 | a_left = K.pattern_broadcast(self.a_left, self.param_broadcast) 104 | a_right = K.pattern_broadcast(self.a_right, self.param_broadcast) 105 | t_right_actual = K.pattern_broadcast(t_right_actual, 106 | self.param_broadcast) 107 | else: 108 | t_left = self.t_left 109 | a_left = self.a_left 110 | a_right = self.a_right 111 | 112 | y_left_and_center = t_left + K.relu(x - t_left, 113 | a_left, 114 | t_right_actual - t_left) 115 | y_right = K.relu(x - t_right_actual) * a_right 116 | return y_left_and_center + y_right 117 | 118 | def get_config(self): 119 | config = { 120 | 't_left_initializer': self.t_left_initializer, 121 | 'a_left_initializer': self.a_left_initializer, 122 | 't_right_initializer': self.t_right_initializer, 123 | 'a_right_initializer': self.a_right_initializer, 124 | 'shared_axes': self.shared_axes 125 | } 126 | base_config = super(SReLU, self).get_config() 127 | return dict(list(base_config.items()) + list(config.items())) 128 | 129 | def compute_output_shape(self, input_shape): 130 | return input_shape 131 | -------------------------------------------------------------------------------- /keras_contrib/layers/advanced_activations/swish.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.layers import Layer 3 | 4 | 5 | class Swish(Layer): 6 | """ Swish (Ramachandranet al., 2017) 7 | 8 | # Input shape 9 | Arbitrary. Use the keyword argument `input_shape` 10 | (tuple of integers, does not include the samples axis) 11 | when using this layer as the first layer in a model. 12 | 13 | # Output shape 14 | Same shape as the input. 15 | 16 | # Arguments 17 | beta: float >= 0. Scaling factor 18 | if set to 1 and trainable set to False (default), 19 | Swish equals the SiLU activation (Elfwing et al., 2017) 20 | trainable: whether to learn the scaling factor during training or not 21 | 22 | # References 23 | - [Searching for Activation Functions](https://arxiv.org/abs/1710.05941) 24 | - [Sigmoid-weighted linear units for neural network function 25 | approximation in reinforcement learning](https://arxiv.org/abs/1702.03118) 26 | """ 27 | 28 | def __init__(self, beta=1.0, trainable=False, **kwargs): 29 | super(Swish, self).__init__(**kwargs) 30 | self.supports_masking = True 31 | self.beta = beta 32 | self.trainable = trainable 33 | 34 | def build(self, input_shape): 35 | self.scaling_factor = K.variable(self.beta, 36 | dtype=K.floatx(), 37 | name='scaling_factor') 38 | if self.trainable: 39 | self._trainable_weights.append(self.scaling_factor) 40 | super(Swish, self).build(input_shape) 41 | 42 | def call(self, inputs, mask=None): 43 | return inputs * K.sigmoid(self.scaling_factor * inputs) 44 | 45 | def get_config(self): 46 | config = {'beta': self.get_weights()[0] if self.trainable else self.beta, 47 | 'trainable': self.trainable} 48 | base_config = super(Swish, self).get_config() 49 | return dict(list(base_config.items()) + list(config.items())) 50 | 51 | def compute_output_shape(self, input_shape): 52 | return input_shape 53 | -------------------------------------------------------------------------------- /keras_contrib/layers/convolutional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/layers/convolutional/__init__.py -------------------------------------------------------------------------------- /keras_contrib/layers/convolutional/subpixelupscaling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | 4 | from keras.layers import Layer 5 | 6 | from keras_contrib import backend as KC 7 | from keras_contrib.utils.conv_utils import normalize_data_format 8 | 9 | 10 | class SubPixelUpscaling(Layer): 11 | """ Sub-pixel convolutional upscaling layer. 12 | 13 | This layer requires a Convolution2D prior to it, 14 | having output filters computed according to 15 | the formula : 16 | 17 | filters = k * (scale_factor * scale_factor) 18 | where k = a user defined number of filters (generally larger than 32) 19 | scale_factor = the upscaling factor (generally 2) 20 | 21 | This layer performs the depth to space operation on 22 | the convolution filters, and returns a 23 | tensor with the size as defined below. 24 | 25 | # Example : 26 | ```python 27 | # A standard subpixel upscaling block 28 | x = Convolution2D(256, 3, 3, padding='same', activation='relu')(...) 29 | u = SubPixelUpscaling(scale_factor=2)(x) 30 | 31 | # Optional 32 | x = Convolution2D(256, 3, 3, padding='same', activation='relu')(u) 33 | ``` 34 | 35 | In practice, it is useful to have a second convolution layer after the 36 | SubPixelUpscaling layer to speed up the learning process. 37 | 38 | However, if you are stacking multiple 39 | SubPixelUpscaling blocks, it may increase 40 | the number of parameters greatly, so the 41 | Convolution layer after SubPixelUpscaling 42 | layer can be removed. 43 | 44 | # Arguments 45 | scale_factor: Upscaling factor. 46 | data_format: Can be None, 'channels_first' or 'channels_last'. 47 | 48 | # Input shape 49 | 4D tensor with shape: 50 | `(samples, k * (scale_factor * scale_factor) channels, rows, cols)` 51 | if data_format='channels_first' 52 | or 4D tensor with shape: 53 | `(samples, rows, cols, k * (scale_factor * scale_factor) channels)` 54 | if data_format='channels_last'. 55 | 56 | # Output shape 57 | 4D tensor with shape: 58 | `(samples, k channels, rows * scale_factor, cols * scale_factor))` 59 | if data_format='channels_first' 60 | or 4D tensor with shape: 61 | `(samples, rows * scale_factor, cols * scale_factor, k channels)` 62 | if data_format='channels_last'. 63 | 64 | # References 65 | - [Real-Time Single Image and Video Super-Resolution Using an 66 | Efficient Sub-Pixel Convolutional Neural Network]( 67 | https://arxiv.org/abs/1609.05158) 68 | """ 69 | 70 | def __init__(self, scale_factor=2, data_format=None, **kwargs): 71 | super(SubPixelUpscaling, self).__init__(**kwargs) 72 | 73 | self.scale_factor = scale_factor 74 | self.data_format = normalize_data_format(data_format) 75 | 76 | def build(self, input_shape): 77 | pass 78 | 79 | def call(self, x, mask=None): 80 | y = KC.depth_to_space(x, self.scale_factor, self.data_format) 81 | return y 82 | 83 | def compute_output_shape(self, input_shape): 84 | if self.data_format == 'channels_first': 85 | b, k, r, c = input_shape 86 | new_k = k // (self.scale_factor ** 2) 87 | new_r = r * self.scale_factor 88 | new_c = c * self.scale_factor 89 | return b, new_k, new_r, new_c 90 | else: 91 | b, r, c, k = input_shape 92 | new_r = r * self.scale_factor 93 | new_c = c * self.scale_factor 94 | new_k = k // (self.scale_factor ** 2) 95 | return b, new_r, new_c, new_k 96 | 97 | def get_config(self): 98 | config = {'scale_factor': self.scale_factor, 99 | 'data_format': self.data_format} 100 | base_config = super(SubPixelUpscaling, self).get_config() 101 | return dict(list(base_config.items()) + list(config.items())) 102 | -------------------------------------------------------------------------------- /keras_contrib/layers/normalization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/layers/normalization/__init__.py -------------------------------------------------------------------------------- /keras_contrib/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from .dssim import DSSIMObjective 2 | from .jaccard import jaccard_distance 3 | from .crf_losses import crf_loss, crf_nll 4 | -------------------------------------------------------------------------------- /keras_contrib/losses/crf_losses.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.losses import categorical_crossentropy 3 | from keras.losses import sparse_categorical_crossentropy 4 | 5 | 6 | def crf_nll(y_true, y_pred): 7 | """The negative log-likelihood for linear chain Conditional Random Field (CRF). 8 | 9 | This loss function is only used when the `layers.CRF` layer 10 | is trained in the "join" mode. 11 | 12 | # Arguments 13 | y_true: tensor with true targets. 14 | y_pred: tensor with predicted targets. 15 | 16 | # Returns 17 | A scalar representing corresponding to the negative log-likelihood. 18 | 19 | # Raises 20 | TypeError: If CRF is not the last layer. 21 | 22 | # About GitHub 23 | If you open an issue or a pull request about CRF, please 24 | add `cc @lzfelix` to notify Luiz Felix. 25 | """ 26 | 27 | crf, idx = y_pred._keras_history[:2] 28 | if crf._outbound_nodes: 29 | raise TypeError('When learn_model="join", CRF must be the last layer.') 30 | if crf.sparse_target: 31 | y_true = K.one_hot(K.cast(y_true[:, :, 0], 'int32'), crf.units) 32 | X = crf._inbound_nodes[idx].input_tensors[0] 33 | mask = crf._inbound_nodes[idx].input_masks[0] 34 | nloglik = crf.get_negative_log_likelihood(y_true, X, mask) 35 | return nloglik 36 | 37 | 38 | def crf_loss(y_true, y_pred): 39 | """General CRF loss function depending on the learning mode. 40 | 41 | # Arguments 42 | y_true: tensor with true targets. 43 | y_pred: tensor with predicted targets. 44 | 45 | # Returns 46 | If the CRF layer is being trained in the join mode, returns the negative 47 | log-likelihood. Otherwise returns the categorical crossentropy implemented 48 | by the underlying Keras backend. 49 | 50 | # About GitHub 51 | If you open an issue or a pull request about CRF, please 52 | add `cc @lzfelix` to notify Luiz Felix. 53 | """ 54 | crf, idx = y_pred._keras_history[:2] 55 | if crf.learn_mode == 'join': 56 | return crf_nll(y_true, y_pred) 57 | else: 58 | if crf.sparse_target: 59 | return sparse_categorical_crossentropy(y_true, y_pred) 60 | else: 61 | return categorical_crossentropy(y_true, y_pred) 62 | -------------------------------------------------------------------------------- /keras_contrib/losses/dssim.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import keras_contrib.backend as KC 3 | from keras import backend as K 4 | 5 | 6 | class DSSIMObjective: 7 | """Difference of Structural Similarity (DSSIM loss function). 8 | Clipped between 0 and 0.5 9 | 10 | Note : You should add a regularization term like a l2 loss in addition to this one. 11 | Note : In theano, the `kernel_size` must be a factor of the output size. So 3 could 12 | not be the `kernel_size` for an output of 32. 13 | 14 | # Arguments 15 | k1: Parameter of the SSIM (default 0.01) 16 | k2: Parameter of the SSIM (default 0.03) 17 | kernel_size: Size of the sliding window (default 3) 18 | max_value: Max value of the output (default 1.0) 19 | """ 20 | 21 | def __init__(self, k1=0.01, k2=0.03, kernel_size=3, max_value=1.0): 22 | self.__name__ = 'DSSIMObjective' 23 | self.kernel_size = kernel_size 24 | self.k1 = k1 25 | self.k2 = k2 26 | self.max_value = max_value 27 | self.c1 = (self.k1 * self.max_value) ** 2 28 | self.c2 = (self.k2 * self.max_value) ** 2 29 | self.dim_ordering = K.image_data_format() 30 | self.backend = K.backend() 31 | 32 | def __int_shape(self, x): 33 | return K.int_shape(x) if self.backend == 'tensorflow' else K.shape(x) 34 | 35 | def __call__(self, y_true, y_pred): 36 | # There are additional parameters for this function 37 | # Note: some of the 'modes' for edge behavior do not yet have a 38 | # gradient definition in the Theano tree 39 | # and cannot be used for learning 40 | 41 | kernel = [self.kernel_size, self.kernel_size] 42 | y_true = K.reshape(y_true, [-1] + list(self.__int_shape(y_pred)[1:])) 43 | y_pred = K.reshape(y_pred, [-1] + list(self.__int_shape(y_pred)[1:])) 44 | 45 | patches_pred = KC.extract_image_patches(y_pred, kernel, kernel, 'valid', 46 | self.dim_ordering) 47 | patches_true = KC.extract_image_patches(y_true, kernel, kernel, 'valid', 48 | self.dim_ordering) 49 | 50 | # Reshape to get the var in the cells 51 | bs, w, h, c1, c2, c3 = self.__int_shape(patches_pred) 52 | patches_pred = K.reshape(patches_pred, [-1, w, h, c1 * c2 * c3]) 53 | patches_true = K.reshape(patches_true, [-1, w, h, c1 * c2 * c3]) 54 | # Get mean 55 | u_true = K.mean(patches_true, axis=-1) 56 | u_pred = K.mean(patches_pred, axis=-1) 57 | # Get variance 58 | var_true = K.var(patches_true, axis=-1) 59 | var_pred = K.var(patches_pred, axis=-1) 60 | # Get std dev 61 | covar_true_pred = K.mean(patches_true * patches_pred, axis=-1) - u_true * u_pred 62 | 63 | ssim = (2 * u_true * u_pred + self.c1) * (2 * covar_true_pred + self.c2) 64 | denom = ((K.square(u_true) 65 | + K.square(u_pred) 66 | + self.c1) * (var_pred + var_true + self.c2)) 67 | ssim /= denom # no need for clipping, c1 and c2 make the denom non-zero 68 | return K.mean((1.0 - ssim) / 2.0) 69 | -------------------------------------------------------------------------------- /keras_contrib/losses/jaccard.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | 4 | def jaccard_distance(y_true, y_pred, smooth=100): 5 | """Jaccard distance for semantic segmentation. 6 | 7 | Also known as the intersection-over-union loss. 8 | 9 | This loss is useful when you have unbalanced numbers of pixels within an image 10 | because it gives all classes equal weight. However, it is not the defacto 11 | standard for image segmentation. 12 | 13 | For example, assume you are trying to predict if 14 | each pixel is cat, dog, or background. 15 | You have 80% background pixels, 10% dog, and 10% cat. 16 | If the model predicts 100% background 17 | should it be be 80% right (as with categorical cross entropy) 18 | or 30% (with this loss)? 19 | 20 | The loss has been modified to have a smooth gradient as it converges on zero. 21 | This has been shifted so it converges on 0 and is smoothed to avoid exploding 22 | or disappearing gradient. 23 | 24 | Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|) 25 | = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|)) 26 | 27 | # Arguments 28 | y_true: The ground truth tensor. 29 | y_pred: The predicted tensor 30 | smooth: Smoothing factor. Default is 100. 31 | 32 | # Returns 33 | The Jaccard distance between the two tensors. 34 | 35 | # References 36 | - [What is a good evaluation measure for semantic segmentation?]( 37 | http://www.bmva.org/bmvc/2013/Papers/paper0032/paper0032.pdf) 38 | 39 | """ 40 | intersection = K.sum(K.abs(y_true * y_pred), axis=-1) 41 | sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1) 42 | jac = (intersection + smooth) / (sum_ - intersection + smooth) 43 | return (1 - jac) * smooth 44 | -------------------------------------------------------------------------------- /keras_contrib/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .crf_accuracies import crf_accuracy, crf_marginal_accuracy 2 | from .crf_accuracies import crf_viterbi_accuracy 3 | -------------------------------------------------------------------------------- /keras_contrib/metrics/crf_accuracies.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | 4 | def _get_accuracy(y_true, y_pred, mask, sparse_target=False): 5 | y_pred = K.argmax(y_pred, -1) 6 | if sparse_target: 7 | y_true = K.cast(y_true[:, :, 0], K.dtype(y_pred)) 8 | else: 9 | y_true = K.argmax(y_true, -1) 10 | judge = K.cast(K.equal(y_pred, y_true), K.floatx()) 11 | if mask is None: 12 | return K.mean(judge) 13 | else: 14 | mask = K.cast(mask, K.floatx()) 15 | return K.sum(judge * mask) / K.sum(mask) 16 | 17 | 18 | def crf_viterbi_accuracy(y_true, y_pred): 19 | '''Use Viterbi algorithm to get best path, and compute its accuracy. 20 | `y_pred` must be an output from CRF.''' 21 | crf, idx = y_pred._keras_history[:2] 22 | X = crf._inbound_nodes[idx].input_tensors[0] 23 | mask = crf._inbound_nodes[idx].input_masks[0] 24 | y_pred = crf.viterbi_decoding(X, mask) 25 | return _get_accuracy(y_true, y_pred, mask, crf.sparse_target) 26 | 27 | 28 | def crf_marginal_accuracy(y_true, y_pred): 29 | '''Use time-wise marginal argmax as prediction. 30 | `y_pred` must be an output from CRF with `learn_mode="marginal"`.''' 31 | crf, idx = y_pred._keras_history[:2] 32 | X = crf._inbound_nodes[idx].input_tensors[0] 33 | mask = crf._inbound_nodes[idx].input_masks[0] 34 | y_pred = crf.get_marginal_prob(X, mask) 35 | return _get_accuracy(y_true, y_pred, mask, crf.sparse_target) 36 | 37 | 38 | def crf_accuracy(y_true, y_pred): 39 | '''Ge default accuracy based on CRF `test_mode`.''' 40 | crf, idx = y_pred._keras_history[:2] 41 | if crf.test_mode == 'viterbi': 42 | return crf_viterbi_accuracy(y_true, y_pred) 43 | else: 44 | return crf_marginal_accuracy(y_true, y_pred) 45 | -------------------------------------------------------------------------------- /keras_contrib/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | from .ftml import FTML 2 | from .padam import Padam 3 | from .yogi import Yogi 4 | from .lars import LARS 5 | 6 | # aliases 7 | ftml = FTML 8 | lars = LARS 9 | -------------------------------------------------------------------------------- /keras_contrib/optimizers/ftml.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from keras.optimizers import Optimizer 3 | from keras import backend as K 4 | 5 | 6 | class FTML(Optimizer): 7 | """FTML optimizer. 8 | 9 | # Arguments 10 | lr: float >= 0. Learning rate. 11 | beta_1: float, 0 < beta < 1. Generally close to 0.5. 12 | beta_2: float, 0 < beta < 1. Generally close to 1. 13 | epsilon: float >= 0. Fuzz factor. 14 | decay: float >= 0. Learning rate decay over each update. 15 | 16 | # References 17 | - [FTML - Follow the Moving Leader in Deep Learning]( 18 | http://www.cse.ust.hk/~szhengac/papers/icml17.pdf) 19 | """ 20 | 21 | def __init__(self, lr=0.0025, beta_1=0.6, beta_2=0.999, 22 | epsilon=1e-8, decay=0., **kwargs): 23 | super(FTML, self).__init__(**kwargs) 24 | self.__dict__.update(locals()) 25 | self.iterations = K.variable(0) 26 | self.lr = K.variable(lr) 27 | self.beta_1 = K.variable(beta_1) 28 | self.beta_2 = K.variable(beta_2) 29 | self.decay = K.variable(decay) 30 | self.epsilon = epsilon 31 | self.inital_decay = decay 32 | 33 | def get_updates(self, loss, params): 34 | grads = self.get_gradients(loss, params) 35 | self.updates = [K.update_add(self.iterations, 1)] 36 | 37 | lr = self.lr 38 | if self.inital_decay > 0: 39 | lr *= (1. / (1. + self.decay * self.iterations)) 40 | 41 | t = self.iterations + 1 42 | 43 | lr_t = lr / (1. - K.pow(self.beta_1, t)) 44 | 45 | shapes = [K.int_shape(p) for p in params] 46 | zs = [K.zeros(shape) for shape in shapes] 47 | vs = [K.zeros(shape) for shape in shapes] 48 | ds = [K.zeros(shape) for shape in shapes] 49 | self.weights = [self.iterations] + zs + vs + ds 50 | 51 | for p, g, z, v, d in zip(params, grads, zs, vs, ds): 52 | v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g) 53 | d_t = (K.sqrt(v_t / (1. - K.pow(self.beta_2, t))) 54 | + self.epsilon) / lr_t 55 | sigma_t = d_t - self.beta_1 * d 56 | z_t = self.beta_1 * z + (1. - self.beta_1) * g - sigma_t * p 57 | 58 | p_t = - z_t / d_t 59 | 60 | self.updates.append(K.update(z, z_t)) 61 | self.updates.append(K.update(v, v_t)) 62 | self.updates.append(K.update(d, d_t)) 63 | 64 | new_p = p_t 65 | 66 | # Apply constraints. 67 | if getattr(p, 'constraint', None) is not None: 68 | new_p = p.constraint(new_p) 69 | 70 | self.updates.append(K.update(p, new_p)) 71 | return self.updates 72 | 73 | def get_config(self): 74 | config = {'lr': float(K.get_value(self.lr)), 75 | 'beta_1': float(K.get_value(self.beta_1)), 76 | 'beta_2': float(K.get_value(self.beta_2)), 77 | 'decay': float(K.get_value(self.decay)), 78 | 'epsilon': self.epsilon} 79 | base_config = super(FTML, self).get_config() 80 | return dict(list(base_config.items()) + list(config.items())) 81 | -------------------------------------------------------------------------------- /keras_contrib/optimizers/lars.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.optimizers import Optimizer 3 | 4 | 5 | class LARS(Optimizer): 6 | """Layer-wise Adaptive Rate Scaling for large batch training. 7 | Introduced by "Large Batch Training of Convolutional Networks" by Y. You, 8 | I. Gitman, and B. Ginsburg. (https://arxiv.org/abs/1708.03888) 9 | Implements the LARS learning rate scheme presented in the paper above. This 10 | optimizer is useful when scaling the batch size to up to 32K without 11 | significant performance degradation. It is recommended to use the optimizer 12 | in conjunction with: 13 | - Gradual learning rate warm-up 14 | - Linear learning rate scaling 15 | - Poly rule learning rate decay 16 | Note, LARS scaling is currently only enabled for dense tensors. 17 | 18 | Args: 19 | lr: A `Tensor` or floating point value. The base learning rate. 20 | momentum: A floating point value. Momentum hyperparameter. 21 | weight_decay: A floating point value. Weight decay hyperparameter. 22 | eeta: LARS coefficient as used in the paper. Dfault set to LARS 23 | coefficient from the paper. (eeta / weight_decay) determines the 24 | highest scaling factor in LARS. 25 | epsilon: Optional epsilon parameter to be set in models that have very 26 | small gradients. Default set to 0.0. 27 | nesterov: when set to True, nesterov momentum will be enabled 28 | """ 29 | 30 | def __init__(self, 31 | lr, 32 | momentum=0.9, 33 | weight_decay=0.0001, 34 | eeta=0.001, 35 | epsilon=0.0, 36 | nesterov=False, 37 | **kwargs): 38 | 39 | if momentum < 0.0: 40 | raise ValueError("momentum should be positive: %s" % momentum) 41 | if weight_decay < 0.0: 42 | raise ValueError("weight_decay is not positive: %s" % weight_decay) 43 | super(LARS, self).__init__(**kwargs) 44 | with K.name_scope(self.__class__.__name__): 45 | self.iterations = K.variable(0, dtype='int64', name='iterations') 46 | self.lr = K.variable(lr, name='lr') 47 | self.momentum = K.variable(momentum, name='momentum') 48 | self.weight_decay = K.variable(weight_decay, name='weight_decay') 49 | self.eeta = K.variable(eeta, name='eeta') 50 | self.epsilon = epsilon 51 | self.nesterov = nesterov 52 | 53 | def get_updates(self, loss, params): 54 | grads = self.get_gradients(loss, params) 55 | weights = self.get_weights() 56 | self.updates = [K.update_add(self.iterations, 1)] 57 | scaled_lr = self.lr 58 | w_norm = K.sqrt(K.sum([K.sum(K.square(weight)) 59 | for weight in weights])) 60 | g_norm = K.sqrt(K.sum([K.sum(K.square(grad)) 61 | for grad in grads])) 62 | scaled_lr = K.switch(K.greater(w_norm * g_norm, K.zeros([1])), 63 | K.expand_dims((self.eeta * w_norm / 64 | (g_norm + self.weight_decay * w_norm + 65 | self.epsilon)) * self.lr), 66 | K.ones([1]) * self.lr) 67 | if K.backend() == 'theano': 68 | scaled_lr = scaled_lr[0] # otherwise theano raise broadcasting error 69 | # momentum 70 | moments = [K.zeros(K.int_shape(param), dtype=K.dtype(param)) 71 | for param in params] 72 | self.weights = [self.iterations] + moments 73 | for param, grad, moment in zip(params, grads, moments): 74 | v0 = (moment * self.momentum) 75 | v1 = scaled_lr * grad # velocity 76 | veloc = v0 - v1 77 | self.updates.append(K.update(moment, veloc)) 78 | 79 | if self.nesterov: 80 | new_param = param + (veloc * self.momentum) - v1 81 | else: 82 | new_param = param + veloc 83 | 84 | # Apply constraints. 85 | if getattr(param, 'constraint', None) is not None: 86 | new_param = param.constraint(new_param) 87 | 88 | self.updates.append(K.update(param, new_param)) 89 | return self.updates 90 | 91 | def get_config(self): 92 | config = {'lr': float(K.get_value(self.lr)), 93 | 'momentum': float(K.get_value(self.momentum)), 94 | 'weight_decay': float(K.get_value(self.weight_decay)), 95 | 'epsilon': self.epsilon, 96 | 'eeta': float(K.get_value(self.eeta)), 97 | 'nesterov': self.nesterov} 98 | base_config = super(LARS, self).get_config() 99 | return dict(list(base_config.items()) + list(config.items())) 100 | -------------------------------------------------------------------------------- /keras_contrib/optimizers/padam.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.optimizers import Optimizer 3 | 4 | 5 | class Padam(Optimizer): 6 | """Partially adaptive momentum estimation optimizer. 7 | 8 | # Arguments 9 | lr: float >= 0. Learning rate. 10 | beta_1: float, 0 < beta < 1. Generally close to 1. 11 | beta_2: float, 0 < beta < 1. Generally close to 1. 12 | epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`. 13 | decay: float >= 0. Learning rate decay over each update. 14 | amsgrad: boolean. Whether to apply the AMSGrad variant of this 15 | algorithm from the paper "On the Convergence of Adam and 16 | Beyond". 17 | partial: float, 0 <= partial <= 0.5 . Parameter controlling partial 18 | momentum adaption. For `partial=0`, this optimizer behaves like SGD, 19 | for `partial=0.5` it behaves like AMSGrad. 20 | 21 | # References 22 | - [Closing the Generalization Gap of Adaptive Gradient Methods 23 | in Training Deep Neural Networks](https://arxiv.org/pdf/1806.06763.pdf) 24 | 25 | """ 26 | 27 | def __init__(self, lr=1e-1, beta_1=0.9, beta_2=0.999, 28 | epsilon=1e-8, decay=0., amsgrad=False, partial=1. / 8., **kwargs): 29 | if partial < 0 or partial > 0.5: 30 | raise ValueError( 31 | "Padam: 'partial' must be a positive float with a maximum " 32 | "value of `0.5`, since higher values will cause divergence " 33 | "during training." 34 | ) 35 | super(Padam, self).__init__(**kwargs) 36 | with K.name_scope(self.__class__.__name__): 37 | self.iterations = K.variable(0, dtype='int64', name='iterations') 38 | self.lr = K.variable(lr, name='lr') 39 | self.beta_1 = K.variable(beta_1, name='beta_1') 40 | self.beta_2 = K.variable(beta_2, name='beta_2') 41 | self.decay = K.variable(decay, name='decay') 42 | if epsilon is None: 43 | epsilon = K.epsilon() 44 | self.epsilon = epsilon 45 | self.partial = partial 46 | self.initial_decay = decay 47 | self.amsgrad = amsgrad 48 | 49 | def get_updates(self, loss, params): 50 | grads = self.get_gradients(loss, params) 51 | self.updates = [K.update_add(self.iterations, 1)] 52 | 53 | lr = self.lr 54 | if self.initial_decay > 0: 55 | lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, 56 | K.dtype(self.decay)))) 57 | 58 | t = K.cast(self.iterations, K.floatx()) + 1 59 | lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / 60 | (1. - K.pow(self.beta_1, t))) 61 | 62 | ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] 63 | vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] 64 | if self.amsgrad: 65 | vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] 66 | else: 67 | vhats = [K.zeros(1) for _ in params] 68 | self.weights = [self.iterations] + ms + vs + vhats 69 | 70 | for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats): 71 | m_t = (self.beta_1 * m) + (1. - self.beta_1) * g 72 | v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) 73 | if self.amsgrad: 74 | vhat_t = K.maximum(vhat, v_t) 75 | denom = (K.sqrt(vhat_t) + self.epsilon) 76 | self.updates.append(K.update(vhat, vhat_t)) 77 | else: 78 | denom = (K.sqrt(v_t) + self.epsilon) 79 | 80 | self.updates.append(K.update(m, m_t)) 81 | self.updates.append(K.update(v, v_t)) 82 | 83 | # Partial momentum adaption. 84 | new_p = p - (lr_t * (m_t / (denom ** (self.partial * 2)))) 85 | 86 | # Apply constraints. 87 | if getattr(p, 'constraint', None) is not None: 88 | new_p = p.constraint(new_p) 89 | 90 | self.updates.append(K.update(p, new_p)) 91 | return self.updates 92 | 93 | def get_config(self): 94 | config = {'lr': float(K.get_value(self.lr)), 95 | 'beta_1': float(K.get_value(self.beta_1)), 96 | 'beta_2': float(K.get_value(self.beta_2)), 97 | 'decay': float(K.get_value(self.decay)), 98 | 'epsilon': self.epsilon, 99 | 'amsgrad': self.amsgrad, 100 | 'partial': self.partial} 101 | base_config = super(Padam, self).get_config() 102 | return dict(list(base_config.items()) + list(config.items())) 103 | -------------------------------------------------------------------------------- /keras_contrib/optimizers/yogi.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.optimizers import Optimizer 3 | 4 | 5 | class Yogi(Optimizer): 6 | """Yogi optimizer. 7 | Yogi is a variation of Adam that controls the increase in effective 8 | learning rate, which (according to the paper) leads to even better 9 | performance than Adam with similar theoretical guarantees on convergence. 10 | Default parameters follow those provided in the original paper, Tab.1 11 | # Arguments 12 | lr: float >= 0. Learning rate. 13 | beta_1: float, 0 < beta < 1. Generally close to 1. 14 | beta_2: float, 0 < beta < 1. Generally close to 1. 15 | epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`. 16 | decay: float >= 0. Learning rate decay over each update. 17 | # References 18 | - [Adaptive Methods for Nonconvex Optimization]( 19 | https://papers.nips.cc/paper/8186-adaptive-methods-for-nonconvex-optimization) 20 | 21 | If you open an issue or a pull request about the Yogi optimizer, 22 | please add 'cc @MarcoAndreaBuchmann' to notify him. 23 | """ 24 | 25 | def __init__(self, lr=0.01, beta_1=0.9, beta_2=0.999, 26 | epsilon=1e-3, decay=0., **kwargs): 27 | super(Yogi, self).__init__(**kwargs) 28 | if beta_1 <= 0 or beta_1 >= 1: 29 | raise ValueError("beta_1 has to be in ]0, 1[") 30 | if beta_2 <= 0 or beta_2 >= 1: 31 | raise ValueError("beta_2 has to be in ]0, 1[") 32 | 33 | with K.name_scope(self.__class__.__name__): 34 | self.iterations = K.variable(0, dtype='int64', name='iterations') 35 | self.lr = K.variable(lr, name='lr') 36 | self.beta_1 = K.variable(beta_1, name='beta_1') 37 | self.beta_2 = K.variable(beta_2, name='beta_2') 38 | self.decay = K.variable(decay, name='decay') 39 | if epsilon is None: 40 | epsilon = K.epsilon() 41 | if epsilon <= 0: 42 | raise ValueError("epsilon has to be larger than 0") 43 | self.epsilon = epsilon 44 | self.initial_decay = decay 45 | 46 | def get_updates(self, loss, params): 47 | grads = self.get_gradients(loss, params) 48 | self.updates = [K.update_add(self.iterations, 1)] 49 | 50 | lr = self.lr 51 | if self.initial_decay > 0: 52 | lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, 53 | K.dtype(self.decay)))) 54 | 55 | t = K.cast(self.iterations, K.floatx()) + 1 56 | lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / 57 | (1. - K.pow(self.beta_1, t))) 58 | 59 | ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] 60 | vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] 61 | vhats = [K.zeros(1) for _ in params] 62 | self.weights = [self.iterations] + ms + vs + vhats 63 | 64 | for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats): 65 | g2 = K.square(g) 66 | m_t = (self.beta_1 * m) + (1. - self.beta_1) * g 67 | v_t = v - (1. - self.beta_2) * K.sign(v - g2) * g2 68 | p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) 69 | 70 | self.updates.append(K.update(m, m_t)) 71 | self.updates.append(K.update(v, v_t)) 72 | new_p = p_t 73 | 74 | # Apply constraints. 75 | if getattr(p, 'constraint', None) is not None: 76 | new_p = p.constraint(new_p) 77 | 78 | self.updates.append(K.update(p, new_p)) 79 | return self.updates 80 | 81 | def get_config(self): 82 | config = {'lr': float(K.get_value(self.lr)), 83 | 'beta_1': float(K.get_value(self.beta_1)), 84 | 'beta_2': float(K.get_value(self.beta_2)), 85 | 'decay': float(K.get_value(self.decay)), 86 | 'epsilon': self.epsilon} 87 | base_config = super(Yogi, self).get_config() 88 | return dict(list(base_config.items()) + list(config.items())) 89 | -------------------------------------------------------------------------------- /keras_contrib/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/preprocessing/__init__.py -------------------------------------------------------------------------------- /keras_contrib/regularizers/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import absolute_import 3 | -------------------------------------------------------------------------------- /keras_contrib/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/tests/__init__.py -------------------------------------------------------------------------------- /keras_contrib/tests/activations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras import backend as K 4 | 5 | 6 | def get_standard_values(): 7 | ''' 8 | These are just a set of floats used for testing the activation 9 | functions, and are useful in multiple tests. 10 | ''' 11 | return np.array([[0, 0.1, 0.5, 0.9, 1.0]], dtype=K.floatx()) 12 | 13 | 14 | def validate_activation(activation): 15 | activation(get_standard_values()) 16 | -------------------------------------------------------------------------------- /keras_contrib/tests/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras import backend as K 4 | 5 | all_metrics = [] 6 | all_sparse_metrics = [] 7 | 8 | 9 | def validate_metric(metric): 10 | y_a = K.variable(np.random.random((6, 7))) 11 | y_b = K.variable(np.random.random((6, 7))) 12 | output = metric(y_a, y_b) 13 | assert K.eval(output).shape == () 14 | -------------------------------------------------------------------------------- /keras_contrib/tests/optimizers.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | 4 | from keras_contrib.utils import test_utils 5 | from keras import optimizers 6 | from keras.models import Sequential 7 | from keras.layers import Dense, Activation 8 | from keras.utils import to_categorical 9 | 10 | 11 | def get_test_data(): 12 | np.random.seed(1337) 13 | (x_train, y_train), _ = test_utils.get_test_data(num_train=1000, 14 | num_test=200, 15 | input_shape=(10,), 16 | classification=True, 17 | num_classes=2) 18 | y_train = to_categorical(y_train) 19 | return x_train, y_train 20 | 21 | 22 | def get_model(input_dim, num_hidden, output_dim): 23 | model = Sequential() 24 | model.add(Dense(num_hidden, input_shape=(input_dim,))) 25 | model.add(Activation('relu')) 26 | model.add(Dense(output_dim)) 27 | model.add(Activation('softmax')) 28 | return model 29 | 30 | 31 | def _test_optimizer(optimizer, target=0.75): 32 | x_train, y_train = get_test_data() 33 | model = get_model(x_train.shape[1], 10, y_train.shape[1]) 34 | model.compile(loss='categorical_crossentropy', 35 | optimizer=optimizer, 36 | metrics=['accuracy']) 37 | history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0) 38 | assert history.history['acc'][-1] >= target 39 | config = optimizers.serialize(optimizer) 40 | custom_objects = {optimizer.__class__.__name__: optimizer.__class__} 41 | optim = optimizers.deserialize(config, custom_objects) 42 | new_config = optimizers.serialize(optim) 43 | assert config == new_config 44 | -------------------------------------------------------------------------------- /keras_contrib/tests/regularizers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.datasets import mnist 3 | from keras.layers import Activation 4 | from keras.layers import Dense 5 | from keras.models import Sequential 6 | from keras.utils import np_utils 7 | 8 | np.random.seed(1337) 9 | 10 | nb_classes = 10 11 | batch_size = 128 12 | nb_epoch = 5 13 | weighted_class = 9 14 | standard_weight = 1 15 | high_weight = 5 16 | max_train_samples = 5000 17 | max_test_samples = 1000 18 | 19 | 20 | def get_data(): 21 | # the data, shuffled and split between tran and test sets 22 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 23 | X_train = X_train.reshape(60000, 784)[:max_train_samples] 24 | X_test = X_test.reshape(10000, 784)[:max_test_samples] 25 | X_train = X_train.astype('float32') / 255 26 | X_test = X_test.astype('float32') / 255 27 | 28 | # convert class vectors to binary class matrices 29 | y_train = y_train[:max_train_samples] 30 | y_test = y_test[:max_test_samples] 31 | Y_train = np_utils.to_categorical(y_train, nb_classes) 32 | Y_test = np_utils.to_categorical(y_test, nb_classes) 33 | test_ids = np.where(y_test == np.array(weighted_class))[0] 34 | 35 | return (X_train, Y_train), (X_test, Y_test), test_ids 36 | 37 | 38 | def validate_regularizer(weight_reg=None, activity_reg=None): 39 | model = Sequential() 40 | model.add(Dense(50, input_shape=(784,))) 41 | model.add(Activation('relu')) 42 | model.add(Dense(10, W_regularizer=weight_reg, 43 | activity_regularizer=activity_reg)) 44 | model.add(Activation('softmax')) 45 | return model 46 | -------------------------------------------------------------------------------- /keras_contrib/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/utils/__init__.py -------------------------------------------------------------------------------- /keras_contrib/utils/conv_utils.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | 3 | 4 | def conv_output_length(input_length, filter_size, 5 | padding, stride, dilation=1): 6 | """Determines output length of a convolution given input length. 7 | 8 | Copy of the function of keras-team/keras because it's not in the public API 9 | So we can't use the function in keras-team/keras to test tf.keras 10 | 11 | # Arguments 12 | input_length: integer. 13 | filter_size: integer. 14 | padding: one of `"same"`, `"valid"`, `"full"`. 15 | stride: integer. 16 | dilation: dilation rate, integer. 17 | 18 | # Returns 19 | The output length (integer). 20 | """ 21 | if input_length is None: 22 | return None 23 | assert padding in {'same', 'valid', 'full', 'causal'} 24 | dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1) 25 | if padding == 'same': 26 | output_length = input_length 27 | elif padding == 'valid': 28 | output_length = input_length - dilated_filter_size + 1 29 | elif padding == 'causal': 30 | output_length = input_length 31 | elif padding == 'full': 32 | output_length = input_length + dilated_filter_size - 1 33 | return (output_length + stride - 1) // stride 34 | 35 | 36 | def normalize_data_format(value): 37 | """Checks that the value correspond to a valid data format. 38 | 39 | Copy of the function in keras-team/keras because it's not public API. 40 | 41 | # Arguments 42 | value: String or None. `'channels_first'` or `'channels_last'`. 43 | 44 | # Returns 45 | A string, either `'channels_first'` or `'channels_last'` 46 | 47 | # Example 48 | ```python 49 | >>> from keras import backend as K 50 | >>> K.normalize_data_format(None) 51 | 'channels_first' 52 | >>> K.normalize_data_format('channels_last') 53 | 'channels_last' 54 | ``` 55 | 56 | # Raises 57 | ValueError: if `value` or the global `data_format` invalid. 58 | """ 59 | if value is None: 60 | value = K.image_data_format() 61 | data_format = value.lower() 62 | if data_format not in {'channels_first', 'channels_last'}: 63 | raise ValueError('The `data_format` argument must be one of ' 64 | '"channels_first", "channels_last". Received: ' + 65 | str(value)) 66 | return data_format 67 | -------------------------------------------------------------------------------- /keras_contrib/utils/save_load_utils.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import h5py 4 | import keras.backend as K 5 | from keras import optimizers 6 | from keras.engine import saving 7 | 8 | 9 | def save_all_weights(model, filepath, include_optimizer=True): 10 | """ 11 | Save model weights and optimizer weights but not configuration to a HDF5 file. 12 | Functionally between `save` and `save_weights`. 13 | 14 | The HDF5 file contains: 15 | - the model's weights 16 | - the model's optimizer's state (if any) 17 | If you have a complicated model or set of models that do not serialize 18 | to JSON correctly, use this method. 19 | # Arguments 20 | model: Keras model instance to be saved. 21 | filepath: String, path where to save the model. 22 | include_optimizer: If True, save optimizer's state together. 23 | # Raises 24 | ImportError: if h5py is not available. 25 | """ 26 | if h5py is None: 27 | raise ImportError('`save_all_weights` requires h5py.') 28 | 29 | with h5py.File(filepath, 'w') as f: 30 | model_weights_group = f.create_group('model_weights') 31 | model_layers = model.layers 32 | saving.save_weights_to_hdf5_group(model_weights_group, model_layers) 33 | 34 | if include_optimizer and hasattr(model, 'optimizer') and model.optimizer: 35 | if isinstance(model.optimizer, optimizers.TFOptimizer): 36 | warnings.warn( 37 | 'TensorFlow optimizers do not ' 38 | 'make it possible to access ' 39 | 'optimizer attributes or optimizer state ' 40 | 'after instantiation. ' 41 | 'As a result, we cannot save the optimizer ' 42 | 'as part of the model save file.' 43 | 'You will have to compile your model again after loading it. ' 44 | 'Prefer using a Keras optimizer instead ' 45 | '(see keras.io/optimizers).') 46 | else: 47 | # Save optimizer weights. 48 | symbolic_weights = getattr(model.optimizer, 'weights') 49 | if symbolic_weights: 50 | optimizer_weights_group = f.create_group('optimizer_weights') 51 | weight_values = K.batch_get_value(symbolic_weights) 52 | weight_names = [] 53 | for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)): 54 | # Default values of symbolic_weights is /variable for theano 55 | if K.backend() == 'theano': 56 | if hasattr(w, 'name') and w.name != "/variable": 57 | name = str(w.name) 58 | else: 59 | name = 'param_' + str(i) 60 | else: 61 | if hasattr(w, 'name') and w.name: 62 | name = str(w.name) 63 | else: 64 | name = 'param_' + str(i) 65 | weight_names.append(name.encode('utf8')) 66 | optimizer_weights_group.attrs['weight_names'] = weight_names 67 | for name, val in zip(weight_names, weight_values): 68 | param_dset = optimizer_weights_group.create_dataset( 69 | name, 70 | val.shape, 71 | dtype=val.dtype) 72 | if not val.shape: 73 | # scalar 74 | param_dset[()] = val 75 | else: 76 | param_dset[:] = val 77 | 78 | 79 | def load_all_weights(model, filepath, include_optimizer=True): 80 | """Loads the weights of a model saved via `save_all_weights`. 81 | If model has been compiled, optionally load its optimizer's weights. 82 | # Arguments 83 | model: instantiated model with architecture matching the saved model. 84 | Compile the model beforehand if you want to load optimizer weights. 85 | filepath: String, path to the saved model. 86 | # Returns 87 | None. The model will have its weights updated. 88 | # Raises 89 | ImportError: if h5py is not available. 90 | ValueError: In case of an invalid savefile. 91 | """ 92 | if h5py is None: 93 | raise ImportError('`load_all_weights` requires h5py.') 94 | 95 | with h5py.File(filepath, mode='r') as f: 96 | # set weights 97 | saving.load_weights_from_hdf5_group(f['model_weights'], model.layers) 98 | # Set optimizer weights. 99 | if (include_optimizer 100 | and 'optimizer_weights' in f and hasattr(model, 'optimizer') 101 | and model.optimizer): 102 | optimizer_weights_group = f['optimizer_weights'] 103 | optimizer_weight_names = [n.decode('utf8') for n in 104 | optimizer_weights_group.attrs['weight_names']] 105 | optimizer_weight_values = [optimizer_weights_group[n] for n in 106 | optimizer_weight_names] 107 | model.optimizer.set_weights(optimizer_weight_values) 108 | -------------------------------------------------------------------------------- /keras_contrib/wrappers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/keras_contrib/wrappers/__init__.py -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # Configuration of py.test 2 | [pytest] 3 | addopts=-v 4 | -n 2 5 | --durations=10 6 | --cov-report term-missing 7 | 8 | # Do not run tests in the build folder 9 | norecursedirs= build 10 | 11 | # PEP-8 The following are ignored: 12 | # E402 module level import not at top of file - temporary measure to continue adding ros python packaged in sys.path 13 | # E731 do not assign a lambda expression, use a def 14 | 15 | pep8ignore=* E402 \ 16 | * E731 \ 17 | * W503 18 | 19 | pep8maxlinelength = 88 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools import find_packages 3 | import os 4 | 5 | 6 | if os.environ.get('USE_TF_KERAS', None) == '1': 7 | name = 'tf_keras_contrib' 8 | install_requires = [] 9 | else: 10 | name = 'keras_contrib' 11 | install_requires = ['keras'] 12 | 13 | setup(name=name, 14 | version='2.0.8', 15 | description='Keras Deep Learning for Python, Community Contributions', 16 | author='Fariz Rahman', 17 | author_email='farizrahman4u@gmail.com', 18 | url='https://github.com/farizrahman4u/keras-contrib', 19 | license='MIT', 20 | install_requires=install_requires, 21 | extras_require={ 22 | 'h5py': ['h5py'], 23 | 'visualize': ['pydot>=1.2.0'], 24 | 'tests': ['pytest', 25 | 'pytest-pep8', 26 | 'pytest-xdist', 27 | 'pytest-cov'], 28 | }, 29 | classifiers=[ 30 | 'Development Status :: 3 - Alpha', 31 | 'Intended Audience :: Developers', 32 | 'Intended Audience :: Education', 33 | 'Intended Audience :: Science/Research', 34 | 'License :: OSI Approved :: MIT License', 35 | 'Programming Language :: Python :: 2', 36 | 'Programming Language :: Python :: 2.7', 37 | 'Programming Language :: Python :: 3', 38 | 'Programming Language :: Python :: 3.6', 39 | 'Topic :: Software Development :: Libraries', 40 | 'Topic :: Software Development :: Libraries :: Python Modules' 41 | ], 42 | packages=find_packages()) 43 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from keras import backend as K 3 | 4 | 5 | @pytest.fixture(autouse=True) 6 | def clear_session_after_test(): 7 | """Test wrapper to clean up after TensorFlow and CNTK tests. 8 | 9 | This wrapper runs for all the tests in the keras test suite. 10 | """ 11 | yield 12 | if K.backend() == 'tensorflow' or K.backend() == 'cntk': 13 | K.clear_session() 14 | -------------------------------------------------------------------------------- /tests/keras_contrib/activations/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/tests/keras_contrib/activations/.gitkeep -------------------------------------------------------------------------------- /tests/keras_contrib/activations/test_squash.py: -------------------------------------------------------------------------------- 1 | from keras_contrib import activations 2 | import keras.backend as K 3 | import numpy as np 4 | from numpy.testing import assert_allclose 5 | 6 | 7 | def get_standard_values(): 8 | """A set of floats used for testing squash. 9 | """ 10 | return np.array([[0, 0.1, 0.5, 0.9, 1.0]], dtype=K.floatx()) 11 | 12 | 13 | def test_squash_valid(): 14 | """Test using a reference implementation of squash. 15 | """ 16 | def squash(x, axis=-1): 17 | s_squared_norm = np.sum(np.square(x), axis) + 1e-7 18 | scale = np.sqrt(s_squared_norm) / (0.5 + s_squared_norm) 19 | return scale * x 20 | 21 | x = K.placeholder(ndim=2) 22 | f = K.function([x], [activations.squash(x)]) 23 | test_values = get_standard_values() 24 | 25 | result = f([test_values])[0] 26 | expected = squash(test_values) 27 | assert_allclose(result, expected, rtol=1e-05) 28 | 29 | 30 | test_squash_valid() 31 | -------------------------------------------------------------------------------- /tests/keras_contrib/callbacks/cyclical_learning_rate_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | from keras_contrib import callbacks 4 | from keras.models import Sequential 5 | from keras.layers import Dense 6 | from numpy.testing import assert_allclose 7 | 8 | 9 | def build_model(): 10 | model = Sequential([ 11 | Dense(2, activation='relu', input_shape=(2,)), 12 | Dense(1, activation='sigmoid') 13 | ]) 14 | return model 15 | 16 | 17 | def cycle(i): 18 | return np.floor(1 + i / (2 * 2000)) 19 | 20 | 21 | def x(i): 22 | return np.abs(i / 2000. - 2 * cycle(i) + 1) 23 | 24 | 25 | def test_cyclic_lr_triangular_1(): 26 | X = np.random.rand(4000, 2) 27 | y = np.random.rand(4000).reshape(-1, 1) 28 | 29 | clr = callbacks.CyclicLR() 30 | 31 | model = build_model() 32 | model.compile( 33 | optimizer='sgd', 34 | loss='binary_crossentropy', 35 | metrics=['accuracy'] 36 | ) 37 | model.fit(X, y, batch_size=1, epochs=1, verbose=0, callbacks=[clr]) 38 | 39 | r = np.concatenate([ 40 | np.linspace(0.001, 0.006, num=2001)[1:], 41 | np.linspace(0.006, 0.001, num=2001)[1:] 42 | ]) 43 | 44 | assert_allclose(clr.history['lr'], r) 45 | 46 | 47 | def test_cyclic_lr_triangular_2(): 48 | X = np.random.rand(4000, 2) 49 | y = np.random.rand(4000).reshape(-1, 1) 50 | 51 | clr = callbacks.CyclicLR(mode='triangular2') 52 | 53 | model = build_model() 54 | model.compile( 55 | optimizer='sgd', 56 | loss='binary_crossentropy', 57 | metrics=['accuracy'] 58 | ) 59 | model.fit(X, y, batch_size=1, epochs=2, verbose=0, callbacks=[clr]) 60 | 61 | r = np.concatenate([ 62 | np.linspace(0.001, 0.006, num=2001)[1:], 63 | np.linspace(0.006, 0.001, num=2001)[1:], 64 | np.linspace(0.001, 0.0035, num=2001)[1:], 65 | np.linspace(0.0035, 0.001, num=2001)[1:], 66 | ]) 67 | 68 | assert_allclose(clr.history['lr'], r) 69 | 70 | 71 | def test_cyclic_lr_exp_range(): 72 | X = np.random.rand(4000, 2) 73 | y = np.random.rand(4000).reshape(-1, 1) 74 | 75 | clr = callbacks.CyclicLR(mode='exp_range', gamma=0.9996) 76 | 77 | model = build_model() 78 | model.compile( 79 | optimizer='sgd', 80 | loss='binary_crossentropy', 81 | metrics=['accuracy'] 82 | ) 83 | model.fit(X, y, batch_size=1, epochs=2, verbose=0, callbacks=[clr]) 84 | 85 | exp_range = [] 86 | 87 | def scale_fn(i): 88 | return 0.001 + (0.006 - 0.001) * np.maximum(0, (1 - x(i))) * (0.9996 ** i) 89 | 90 | for i in range(8000): 91 | exp_range.append(scale_fn(i + 1)) 92 | 93 | assert_allclose(clr.history['lr'], np.array(exp_range)) 94 | 95 | 96 | def test_cyclic_lr_custom_fn_test(): 97 | X = np.random.rand(4000, 2) 98 | y = np.random.rand(4000).reshape(-1, 1) 99 | 100 | def clr_fn(x): 101 | return 1 / (5 ** (x * 0.0001)) 102 | 103 | clr = callbacks.CyclicLR(scale_fn=clr_fn, scale_mode='iterations') 104 | 105 | model = build_model() 106 | model.compile( 107 | optimizer='sgd', 108 | loss='binary_crossentropy', 109 | metrics=['accuracy'] 110 | ) 111 | model.fit(X, y, batch_size=1, epochs=2, verbose=0, callbacks=[clr]) 112 | 113 | custom_range = [] 114 | 115 | def scale_fn(i): 116 | c = 0.006 - 0.001 117 | return 0.001 + c * np.maximum(0, (1 - x(i))) * 1 / (5 ** (i * 0.0001)) 118 | 119 | for i in range(8000): 120 | custom_range.append(scale_fn(i + 1)) 121 | 122 | assert_allclose(clr.history['lr'], np.array(custom_range)) 123 | 124 | 125 | if __name__ == '__main__': 126 | pytest.main([__file__]) 127 | -------------------------------------------------------------------------------- /tests/keras_contrib/callbacks/tensorboard_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | import os 4 | import shutil 5 | from keras.utils import to_categorical 6 | from keras.layers import Layer, Input, Dense, Dropout, BatchNormalization 7 | from keras_contrib.utils.test_utils import to_list, unpack_singleton 8 | from keras_contrib.utils.test_utils import get_test_data 9 | from keras import Model 10 | from keras import backend as K 11 | from keras_contrib.callbacks import TensorBoardGrouped 12 | 13 | input_dim = 2 14 | num_hidden = 4 15 | num_classes = 2 16 | batch_size = 5 17 | train_samples = 20 18 | test_samples = 20 19 | 20 | 21 | def data_generator(x, y, batch_size): 22 | x = to_list(x) 23 | y = to_list(y) 24 | max_batch_index = len(x[0]) // batch_size 25 | i = 0 26 | while 1: 27 | x_batch = [array[i * batch_size: (i + 1) * batch_size] for array in x] 28 | x_batch = unpack_singleton(x_batch) 29 | 30 | y_batch = [array[i * batch_size: (i + 1) * batch_size] for array in y] 31 | y_batch = unpack_singleton(y_batch) 32 | yield x_batch, y_batch 33 | i += 1 34 | i = i % max_batch_index 35 | 36 | 37 | # Changing the default arguments of get_test_data. 38 | def get_data_callbacks(num_train=train_samples, 39 | num_test=test_samples, 40 | input_shape=(input_dim,), 41 | classification=True, 42 | num_classes=num_classes): 43 | return get_test_data(num_train=num_train, 44 | num_test=num_test, 45 | input_shape=input_shape, 46 | classification=classification, 47 | num_classes=num_classes) 48 | 49 | 50 | def test_TensorBoard(tmpdir): 51 | np.random.seed(np.random.randint(1, 1e7)) 52 | filepath = str(tmpdir / 'logs') 53 | 54 | (X_train, y_train), (X_test, y_test) = get_data_callbacks() 55 | y_test = to_categorical(y_test) 56 | y_train = to_categorical(y_train) 57 | 58 | class DummyStatefulMetric(Layer): 59 | 60 | def __init__(self, name='dummy_stateful_metric', **kwargs): 61 | super(DummyStatefulMetric, self).__init__(name=name, **kwargs) 62 | self.stateful = True 63 | self.state = K.variable(value=0, dtype='int32') 64 | 65 | def reset_states(self): 66 | pass 67 | 68 | def __call__(self, y_true, y_pred): 69 | return self.state 70 | 71 | inp = Input((input_dim,)) 72 | hidden = Dense(num_hidden, activation='relu')(inp) 73 | hidden = Dropout(0.1)(hidden) 74 | hidden = BatchNormalization()(hidden) 75 | output = Dense(num_classes, activation='softmax')(hidden) 76 | model = Model(inputs=inp, outputs=output) 77 | model.compile(loss='categorical_crossentropy', 78 | optimizer='sgd', 79 | metrics=['accuracy', DummyStatefulMetric()]) 80 | 81 | # we must generate new callbacks for each test, as they aren't stateless 82 | def callbacks_factory(histogram_freq): 83 | return [TensorBoardGrouped(log_dir=filepath, 84 | histogram_freq=histogram_freq, 85 | write_images=True, write_grads=True, 86 | batch_size=5)] 87 | 88 | # fit without validation data 89 | model.fit(X_train, y_train, batch_size=batch_size, 90 | callbacks=callbacks_factory(histogram_freq=0), 91 | epochs=3) 92 | 93 | # fit with validation data and accuracy 94 | model.fit(X_train, y_train, batch_size=batch_size, 95 | validation_data=(X_test, y_test), 96 | callbacks=callbacks_factory(histogram_freq=0), epochs=2) 97 | 98 | # fit generator without validation data 99 | train_generator = data_generator(X_train, y_train, batch_size) 100 | model.fit_generator(train_generator, len(X_train), epochs=2, 101 | callbacks=callbacks_factory(histogram_freq=0)) 102 | 103 | # fit generator with validation data and accuracy 104 | train_generator = data_generator(X_train, y_train, batch_size) 105 | model.fit_generator(train_generator, len(X_train), epochs=2, 106 | validation_data=(X_test, y_test), 107 | callbacks=callbacks_factory(histogram_freq=1)) 108 | 109 | assert os.path.isdir(filepath) 110 | shutil.rmtree(filepath) 111 | assert not tmpdir.listdir() 112 | 113 | 114 | if __name__ == '__main__': 115 | pytest.main([__file__]) 116 | -------------------------------------------------------------------------------- /tests/keras_contrib/constraints_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | 4 | from keras import backend as K 5 | from keras_contrib import constraints 6 | 7 | 8 | test_values = [0.1, 0.5, 3, 8, 1e-7] 9 | np.random.seed(3537) 10 | example_array = np.random.random((100, 100)) * 100. - 50. 11 | example_array[0, 0] = 0. # 0 could possibly cause trouble 12 | 13 | 14 | def test_clip(): 15 | clip_instance = constraints.clip() 16 | clipped = clip_instance(K.variable(example_array)) 17 | assert(np.max(np.abs(K.eval(clipped))) <= K.cast_to_floatx(0.01)) 18 | clip_instance = constraints.clip(0.1) 19 | clipped = clip_instance(K.variable(example_array)) 20 | assert(np.max(np.abs(K.eval(clipped))) <= K.cast_to_floatx(0.1)) 21 | 22 | if __name__ == '__main__': 23 | pytest.main([__file__]) 24 | -------------------------------------------------------------------------------- /tests/keras_contrib/datasets/test_datasets.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import pytest 3 | import time 4 | import random 5 | from keras_contrib import datasets 6 | 7 | 8 | if __name__ == '__main__': 9 | pytest.main([__file__]) 10 | -------------------------------------------------------------------------------- /tests/keras_contrib/initializers_test.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras_contrib import backend as KC 3 | from keras_contrib import initializers 4 | import pytest 5 | import numpy as np 6 | 7 | 8 | # 2D tensor test fixture 9 | FC_SHAPE = (100, 100) 10 | 11 | # 4D convolution in th order. This shape has the same effective shape as 12 | # FC_SHAPE 13 | CONV_SHAPE = (25, 25, 2, 2) 14 | 15 | # The equivalent shape of both test fixtures 16 | SHAPE = (100, 100) 17 | 18 | 19 | def _runner(init, shape, target_mean=None, target_std=None, 20 | target_max=None, target_min=None, upper_bound=None, lower_bound=None): 21 | variable = init(shape) 22 | if not isinstance(variable, np.ndarray): 23 | output = K.get_value(variable) 24 | else: 25 | output = variable 26 | 27 | lim = 1e-2 28 | if target_std is not None: 29 | assert abs(output.std() - target_std) < lim 30 | if target_mean is not None: 31 | assert abs(output.mean() - target_mean) < lim 32 | if target_max is not None: 33 | assert abs(output.max() - target_max) < lim 34 | if target_min is not None: 35 | assert abs(output.min() - target_min) < lim 36 | if upper_bound is not None: 37 | assert output.max() < upper_bound 38 | if lower_bound is not None: 39 | assert output.min() > lower_bound 40 | 41 | 42 | ''' 43 | # Example : 44 | 45 | @pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV']) 46 | def test_uniform(tensor_shape): 47 | _runner(initializations.uniform, tensor_shape, target_mean=0., 48 | target_max=0.05, target_min=-0.05) 49 | 50 | ''' 51 | 52 | 53 | @pytest.mark.parametrize('tensor_shape', [FC_SHAPE, CONV_SHAPE], ids=['FC', 'CONV']) 54 | def test_cai(tensor_shape): 55 | # upper and lower bounds are proved in original paper 56 | _runner(initializers.ConvolutionAware(), tensor_shape, 57 | upper_bound=1, lower_bound=-1) 58 | 59 | if __name__ == '__main__': 60 | pytest.main([__file__]) 61 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/advanced_activations/test_pelu.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from keras_contrib.utils.test_utils import layer_test 3 | from keras_contrib.layers import PELU 4 | 5 | 6 | @pytest.mark.parametrize('kwargs', [{}, {'shared_axes': 1}]) 7 | def test_pelu(kwargs): 8 | layer_test(PELU, kwargs=kwargs, 9 | input_shape=(2, 3, 4)) 10 | 11 | 12 | if __name__ == '__main__': 13 | pytest.main([__file__]) 14 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/advanced_activations/test_sinerelu.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from keras_contrib.utils.test_utils import layer_test 3 | from keras_contrib.layers import SineReLU 4 | 5 | 6 | @pytest.mark.parametrize('epsilon', [0.0025, 0.0035, 0.0045]) 7 | def test_sine_relu(epsilon): 8 | layer_test(SineReLU, kwargs={'epsilon': epsilon}, input_shape=(2, 3, 4)) 9 | 10 | 11 | if __name__ == '__main__': 12 | pytest.main([__file__]) 13 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/advanced_activations/test_srelu.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from keras_contrib.utils.test_utils import layer_test 3 | from keras_contrib.layers import SReLU 4 | 5 | 6 | @pytest.mark.parametrize('kwargs', [{}, {'shared_axes': 1}]) 7 | def test_srelu(kwargs): 8 | layer_test(SReLU, kwargs=kwargs, input_shape=(2, 3, 4)) 9 | 10 | 11 | if __name__ == '__main__': 12 | pytest.main([__file__]) 13 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/advanced_activations/test_swish.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from keras_contrib.utils.test_utils import layer_test 3 | from keras_contrib.layers import Swish 4 | 5 | 6 | @pytest.mark.parametrize('trainable', [False, True]) 7 | def test_swish(trainable): 8 | layer_test(Swish, kwargs={'beta': 1.0, 'trainable': trainable}, 9 | input_shape=(2, 3, 4)) 10 | 11 | 12 | if __name__ == '__main__': 13 | pytest.main([__file__]) 14 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/convolutional/test_cosineconvolution2d.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | from keras import backend as K 4 | from keras.models import Sequential 5 | from numpy.testing import assert_allclose 6 | 7 | from keras_contrib.utils.test_utils import layer_test 8 | from keras_contrib.layers import CosineConvolution2D 9 | 10 | # TensorFlow does not support full convolution. 11 | if K.backend() == 'theano': 12 | _convolution_border_modes = ['valid', 'same'] 13 | data_format = 'channels_first' 14 | else: 15 | _convolution_border_modes = ['valid', 'same'] 16 | data_format = 'channels_last' 17 | 18 | 19 | @pytest.mark.parametrize('border_mode', _convolution_border_modes) 20 | @pytest.mark.parametrize('subsample', [(1, 1), (2, 2)]) 21 | @pytest.mark.parametrize('use_bias_mode', [True, False]) 22 | @pytest.mark.parametrize('use_regularizer', [True, False]) 23 | def test_cosineconvolution_2d(border_mode, 24 | subsample, 25 | use_bias_mode, 26 | use_regularizer): 27 | num_samples = 2 28 | num_filter = 2 29 | stack_size = 3 30 | num_row = 10 31 | num_col = 6 32 | 33 | if border_mode == 'same' and subsample != (1, 1): 34 | return 35 | 36 | kwargs = {'filters': num_filter, 37 | 'kernel_size': (3, 3), 38 | 'padding': border_mode, 39 | 'strides': subsample, 40 | 'use_bias': use_bias_mode, 41 | 'data_format': data_format} 42 | if use_regularizer: 43 | kwargs.update({'kernel_regularizer': 'l2', 44 | 'bias_regularizer': 'l2', 45 | 'activity_regularizer': 'l2'}) 46 | 47 | layer_test(CosineConvolution2D, 48 | kwargs=kwargs, 49 | input_shape=(num_samples, num_row, num_col, stack_size)) 50 | 51 | 52 | def test_cosineconvolution_2d_correctness(): 53 | if data_format == 'channels_first': 54 | X = np.random.randn(1, 3, 5, 5) 55 | input_dim = (3, 5, 5) 56 | W0 = X[:, :, ::-1, ::-1] 57 | elif data_format == 'channels_last': 58 | X = np.random.randn(1, 5, 5, 3) 59 | input_dim = (5, 5, 3) 60 | W0 = X[0, :, :, :, None] 61 | 62 | model = Sequential() 63 | model.add(CosineConvolution2D(1, (5, 5), use_bias=True, 64 | input_shape=input_dim, 65 | data_format=data_format)) 66 | model.compile(loss='mse', optimizer='rmsprop') 67 | W = model.get_weights() 68 | W[0] = W0 69 | W[1] = np.asarray([1.]) 70 | model.set_weights(W) 71 | out = model.predict(X) 72 | assert_allclose(out, np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5) 73 | 74 | model = Sequential() 75 | model.add(CosineConvolution2D(1, (5, 5), 76 | use_bias=False, 77 | input_shape=input_dim, 78 | data_format=data_format)) 79 | model.compile(loss='mse', optimizer='rmsprop') 80 | W = model.get_weights() 81 | W[0] = -2 * W0 82 | model.set_weights(W) 83 | out = model.predict(X) 84 | assert_allclose(out, -np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5) 85 | 86 | 87 | if __name__ == '__main__': 88 | pytest.main([__file__]) 89 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/convolutional/test_subpixelupscaling.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | from keras import backend as K 4 | 5 | from keras_contrib import backend as KC 6 | from keras_contrib.layers import SubPixelUpscaling 7 | from keras_contrib.utils.test_utils import layer_test 8 | 9 | # TensorFlow does not support full convolution. 10 | if K.backend() == 'theano': 11 | _convolution_border_modes = ['valid', 'same'] 12 | data_format = 'channels_first' 13 | else: 14 | _convolution_border_modes = ['valid', 'same'] 15 | data_format = 'channels_last' 16 | 17 | 18 | @pytest.mark.parametrize('scale_factor', [2, 3, 4]) 19 | def test_sub_pixel_upscaling(scale_factor): 20 | num_samples = 2 21 | num_row = 16 22 | num_col = 16 23 | input_dtype = K.floatx() 24 | 25 | nb_channels = 4 * (scale_factor ** 2) 26 | input_data = np.random.random((num_samples, nb_channels, num_row, num_col)) 27 | input_data = input_data.astype(input_dtype) 28 | 29 | if K.image_data_format() == 'channels_last': 30 | input_data = input_data.transpose((0, 2, 3, 1)) 31 | 32 | input_tensor = K.variable(input_data) 33 | expected_output = K.eval(KC.depth_to_space(input_tensor, 34 | scale=scale_factor)) 35 | 36 | layer_test(SubPixelUpscaling, 37 | kwargs={'scale_factor': scale_factor}, 38 | input_data=input_data, 39 | expected_output=expected_output, 40 | expected_output_dtype=K.floatx()) 41 | 42 | 43 | if __name__ == '__main__': 44 | pytest.main([__file__]) 45 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/test_capsule.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | from numpy.testing import assert_allclose 4 | 5 | from keras_contrib.utils.test_utils import layer_test 6 | from keras_contrib.utils.test_utils import is_tf_keras 7 | from keras import backend as K 8 | from keras_contrib.layers import capsule 9 | from keras.models import Sequential 10 | 11 | 12 | @pytest.mark.parametrize('num_capsule', [10, 20]) 13 | @pytest.mark.parametrize('dim_capsule', [10, 20]) 14 | @pytest.mark.parametrize('routings', [3, 4]) 15 | @pytest.mark.parametrize('share_weights', [True, False]) 16 | @pytest.mark.parametrize('activation', ['sigmoid', 'relu']) 17 | def test_capsule(num_capsule, 18 | dim_capsule, 19 | routings, 20 | share_weights, 21 | activation): 22 | 23 | # TODO: removed this once the issue #25546 in the Tensorflow repo is fixed. 24 | if is_tf_keras and not share_weights: 25 | return 26 | 27 | num_samples = 100 28 | num_rows = 256 29 | num_cols = 256 30 | 31 | kwargs = {'num_capsule': num_capsule, 32 | 'dim_capsule': dim_capsule, 33 | 'routings': routings, 34 | 'share_weights': share_weights, 35 | 'activation': activation} 36 | 37 | layer_test(capsule.Capsule, 38 | kwargs=kwargs, 39 | input_shape=(num_samples, num_rows, num_cols)) 40 | 41 | 42 | def test_capsule_correctness(): 43 | X = np.random.random((1, 1, 1)) 44 | 45 | model = Sequential() 46 | model.add(capsule.Capsule(1, 1, 1, True, activation='sigmoid')) 47 | 48 | model.compile(loss='mse', optimizer='rmsprop') 49 | init_out = model.predict(X) # mock predict call to initialize weights 50 | model.set_weights([np.zeros((1, 1, 1))]) 51 | out = model.predict(X) 52 | assert_allclose(out, np.zeros((1, 1, 1), dtype=K.floatx()) + 0.5, atol=1e-5) 53 | 54 | 55 | if __name__ == '__main__': 56 | pytest.main([__file__]) 57 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/test_core.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | 4 | from keras import regularizers 5 | from keras import constraints 6 | from keras.models import Sequential 7 | from keras import backend as K 8 | from keras_contrib.layers import core 9 | from keras_contrib.utils.test_utils import layer_test 10 | from numpy.testing import assert_allclose 11 | 12 | 13 | @pytest.mark.parametrize('input_shape', [(3, 2), 14 | (3, 4, 2), 15 | (None, None, 2), 16 | (3, 4, 5, 2)]) 17 | def test_cosinedense(input_shape): 18 | 19 | layer_test(core.CosineDense, 20 | kwargs={'units': 3}, 21 | input_shape=input_shape) 22 | 23 | 24 | def test_cosinedense_reg_constraint(): 25 | layer_test(core.CosineDense, 26 | kwargs={'units': 3, 27 | 'kernel_regularizer': regularizers.l2(0.01), 28 | 'bias_regularizer': regularizers.l1(0.01), 29 | 'activity_regularizer': regularizers.l2(0.01), 30 | 'kernel_constraint': constraints.MaxNorm(1), 31 | 'bias_constraint': constraints.MaxNorm(1)}, 32 | input_shape=(3, 2)) 33 | 34 | 35 | def test_cosinedense_correctness(): 36 | X = np.random.randn(1, 20) 37 | model = Sequential() 38 | model.add(core.CosineDense(1, use_bias=True, input_shape=(20,))) 39 | model.compile(loss='mse', optimizer='rmsprop') 40 | W = model.get_weights() 41 | W[0] = X.T 42 | W[1] = np.asarray([1.]) 43 | model.set_weights(W) 44 | out = model.predict(X) 45 | assert_allclose(out, np.ones((1, 1), dtype=K.floatx()), atol=1e-5) 46 | 47 | X = np.random.randn(1, 20) 48 | model = Sequential() 49 | model.add(core.CosineDense(1, use_bias=False, input_shape=(20,))) 50 | model.compile(loss='mse', optimizer='rmsprop') 51 | W = model.get_weights() 52 | W[0] = -2 * X.T 53 | model.set_weights(W) 54 | out = model.predict(X) 55 | assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5) 56 | 57 | 58 | if __name__ == '__main__': 59 | pytest.main([__file__]) 60 | -------------------------------------------------------------------------------- /tests/keras_contrib/layers/test_crf.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | import os 4 | from numpy.testing import assert_allclose 5 | 6 | from keras.layers import Embedding 7 | from keras.models import Sequential 8 | from keras.models import load_model 9 | from keras_contrib.losses import crf_loss 10 | from keras_contrib.metrics import crf_accuracy 11 | from keras_contrib.metrics import crf_marginal_accuracy 12 | from keras_contrib.metrics import crf_viterbi_accuracy 13 | from keras_contrib.layers import CRF 14 | from keras_contrib.utils.test_utils import is_tf_keras 15 | 16 | nb_samples, timesteps, embedding_dim, output_dim = 2, 10, 4, 5 17 | embedding_num = 12 18 | 19 | MODEL_PERSISTENCE_PATH = './test_saving_crf_model.h5' 20 | 21 | 22 | @pytest.mark.xfail(is_tf_keras, 23 | reason='TODO: fix it. Using K.tf which is bad.', 24 | strict=True) 25 | def test_CRF(): 26 | # data 27 | x = np.random.randint(1, embedding_num, nb_samples * timesteps) 28 | x = x.reshape((nb_samples, timesteps)) 29 | x[0, -4:] = 0 # right padding 30 | x[1, :5] = 0 # left padding 31 | y = np.random.randint(0, output_dim, nb_samples * timesteps) 32 | y = y.reshape((nb_samples, timesteps)) 33 | y_onehot = np.eye(output_dim)[y] 34 | y = np.expand_dims(y, 2) # .astype('float32') 35 | 36 | # test with no masking, onehot, fix length 37 | model = Sequential() 38 | model.add(Embedding(embedding_num, embedding_dim, input_length=timesteps)) 39 | crf = CRF(output_dim) 40 | model.add(crf) 41 | model.compile(optimizer='rmsprop', loss=crf_loss) 42 | model.fit(x, y_onehot, epochs=1, batch_size=10) 43 | model.save(MODEL_PERSISTENCE_PATH) 44 | load_model(MODEL_PERSISTENCE_PATH, 45 | custom_objects={'CRF': CRF, 46 | 'crf_loss': crf_loss, 47 | 'crf_viterbi_accuracy': crf_viterbi_accuracy}) 48 | 49 | # test with masking, sparse target, dynamic length; 50 | # test crf_viterbi_accuracy, crf_marginal_accuracy 51 | 52 | model = Sequential() 53 | model.add(Embedding(embedding_num, embedding_dim, mask_zero=True)) 54 | crf = CRF(output_dim, sparse_target=True) 55 | model.add(crf) 56 | model.compile(optimizer='rmsprop', loss=crf_loss, 57 | metrics=[crf_viterbi_accuracy, crf_marginal_accuracy]) 58 | model.fit(x, y, epochs=1, batch_size=10) 59 | 60 | # check mask 61 | y_pred = model.predict(x).argmax(-1) 62 | assert (y_pred[0, -4:] == 0).all() # right padding 63 | assert (y_pred[1, :5] == 0).all() # left padding 64 | 65 | # test viterbi_acc 66 | _, v_acc, _ = model.evaluate(x, y) 67 | np_acc = (y_pred[x > 0] == y[:, :, 0][x > 0]).astype('float32').mean() 68 | print(v_acc, np_acc) 69 | assert np.abs(v_acc - np_acc) < 1e-4 70 | 71 | # test config 72 | model.get_config() 73 | 74 | # test marginal learn mode, fix length 75 | 76 | model = Sequential() 77 | model.add(Embedding(embedding_num, embedding_dim, input_length=timesteps, 78 | mask_zero=True)) 79 | crf = CRF(output_dim, learn_mode='marginal', unroll=True) 80 | model.add(crf) 81 | model.compile(optimizer='rmsprop', loss=crf_loss) 82 | model.fit(x, y_onehot, epochs=1, batch_size=10) 83 | 84 | # check mask (marginal output) 85 | y_pred = model.predict(x) 86 | assert_allclose(y_pred[0, -4:], 1. / output_dim, atol=1e-6) 87 | assert_allclose(y_pred[1, :5], 1. / output_dim, atol=1e-6) 88 | 89 | # test marginal learn mode, but with Viterbi test_mode 90 | model = Sequential() 91 | model.add(Embedding(embedding_num, embedding_dim, input_length=timesteps, 92 | mask_zero=True)) 93 | crf = CRF(output_dim, learn_mode='marginal', test_mode='viterbi') 94 | model.add(crf) 95 | model.compile(optimizer='rmsprop', loss=crf_loss, metrics=[crf_accuracy]) 96 | model.fit(x, y_onehot, epochs=1, batch_size=10) 97 | 98 | y_pred = model.predict(x) 99 | 100 | # check y_pred is onehot vector (output from 'viterbi' test mode) 101 | assert_allclose(np.eye(output_dim)[y_pred.argmax(-1)], y_pred, atol=1e-6) 102 | 103 | try: 104 | os.remove(MODEL_PERSISTENCE_PATH) 105 | except OSError: 106 | pass 107 | 108 | 109 | if __name__ == '__main__': 110 | pytest.main([__file__]) 111 | -------------------------------------------------------------------------------- /tests/keras_contrib/losses/dssim_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | from keras_contrib.utils.test_utils import is_tf_keras 4 | from numpy.testing import assert_allclose 5 | from keras.layers import Conv2D 6 | from keras.models import Sequential 7 | from keras.optimizers import Adam 8 | 9 | from keras.losses import sparse_categorical_crossentropy 10 | from keras import backend as K 11 | from keras_contrib.losses import DSSIMObjective 12 | 13 | allobj = [] 14 | 15 | 16 | def test_objective_shapes_3d(): 17 | y_a = K.variable(np.random.random((5, 6, 7))) 18 | y_b = K.variable(np.random.random((5, 6, 7))) 19 | for obj in allobj: 20 | objective_output = obj(y_a, y_b) 21 | assert K.eval(objective_output).shape == (5, 6) 22 | 23 | 24 | def test_objective_shapes_2d(): 25 | y_a = K.variable(np.random.random((6, 7))) 26 | y_b = K.variable(np.random.random((6, 7))) 27 | for obj in allobj: 28 | objective_output = obj(y_a, y_b) 29 | assert K.eval(objective_output).shape == (6,) 30 | 31 | 32 | def test_cce_one_hot(): 33 | y_a = K.variable(np.random.randint(0, 7, (5, 6))) 34 | y_b = K.variable(np.random.random((5, 6, 7))) 35 | objective_output = sparse_categorical_crossentropy(y_a, y_b) 36 | assert K.eval(objective_output).shape == (5, 6) 37 | 38 | y_a = K.variable(np.random.randint(0, 7, (6,))) 39 | y_b = K.variable(np.random.random((6, 7))) 40 | assert K.eval(sparse_categorical_crossentropy(y_a, y_b)).shape == (6,) 41 | 42 | 43 | def test_DSSIM_channels_last(): 44 | prev_data = K.image_data_format() 45 | K.set_image_data_format('channels_last') 46 | for input_dim, kernel_size in zip([32, 33], [2, 3]): 47 | input_shape = [input_dim, input_dim, 3] 48 | X = np.random.random_sample(4 * input_dim * input_dim * 3) 49 | X = X.reshape([4] + input_shape) 50 | y = np.random.random_sample(4 * input_dim * input_dim * 3) 51 | y = y.reshape([4] + input_shape) 52 | 53 | model = Sequential() 54 | model.add(Conv2D(32, (3, 3), padding='same', input_shape=input_shape, 55 | activation='relu')) 56 | model.add(Conv2D(3, (3, 3), padding='same', input_shape=input_shape, 57 | activation='relu')) 58 | adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8) 59 | model.compile(loss=DSSIMObjective(kernel_size=kernel_size), 60 | metrics=['mse'], 61 | optimizer=adam) 62 | model.fit(X, y, batch_size=2, epochs=1, shuffle='batch') 63 | 64 | # Test same 65 | x1 = K.constant(X, 'float32') 66 | x2 = K.constant(X, 'float32') 67 | dssim = DSSIMObjective(kernel_size=kernel_size) 68 | assert_allclose(0.0, K.eval(dssim(x1, x2)), atol=1e-4) 69 | 70 | # Test opposite 71 | x1 = K.zeros([4] + input_shape) 72 | x2 = K.ones([4] + input_shape) 73 | dssim = DSSIMObjective(kernel_size=kernel_size) 74 | assert_allclose(0.5, K.eval(dssim(x1, x2)), atol=1e-4) 75 | 76 | K.set_image_data_format(prev_data) 77 | 78 | 79 | @pytest.mark.xfail(is_tf_keras, 80 | reason='TODO fix this.', 81 | strict=True) 82 | def test_DSSIM_channels_first(): 83 | prev_data = K.image_data_format() 84 | K.set_image_data_format('channels_first') 85 | for input_dim, kernel_size in zip([32, 33], [2, 3]): 86 | input_shape = [3, input_dim, input_dim] 87 | X = np.random.random_sample(4 * input_dim * input_dim * 3) 88 | X = X.reshape([4] + input_shape) 89 | y = np.random.random_sample(4 * input_dim * input_dim * 3) 90 | y = y.reshape([4] + input_shape) 91 | 92 | model = Sequential() 93 | model.add(Conv2D(32, (3, 3), padding='same', input_shape=input_shape, 94 | activation='relu')) 95 | model.add(Conv2D(3, (3, 3), padding='same', input_shape=input_shape, 96 | activation='relu')) 97 | adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8) 98 | model.compile(loss=DSSIMObjective(kernel_size=kernel_size), metrics=['mse'], 99 | optimizer=adam) 100 | model.fit(X, y, batch_size=2, epochs=1, shuffle='batch') 101 | 102 | # Test same 103 | x1 = K.constant(X, 'float32') 104 | x2 = K.constant(X, 'float32') 105 | dssim = DSSIMObjective(kernel_size=kernel_size) 106 | assert_allclose(0.0, K.eval(dssim(x1, x2)), atol=1e-4) 107 | 108 | # Test opposite 109 | x1 = K.zeros([4] + input_shape) 110 | x2 = K.ones([4] + input_shape) 111 | dssim = DSSIMObjective(kernel_size=kernel_size) 112 | assert_allclose(0.5, K.eval(dssim(x1, x2)), atol=1e-4) 113 | 114 | K.set_image_data_format(prev_data) 115 | 116 | 117 | if __name__ == '__main__': 118 | pytest.main([__file__]) 119 | -------------------------------------------------------------------------------- /tests/keras_contrib/losses/jaccard_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from keras_contrib.losses import jaccard_distance 4 | from keras_contrib.utils.test_utils import is_tf_keras 5 | from keras import backend as K 6 | import numpy as np 7 | 8 | 9 | @pytest.mark.xfail(is_tf_keras, 10 | reason='TODO fix this.', 11 | strict=True) 12 | def test_jaccard_distance(): 13 | # all_right, almost_right, half_right, all_wrong 14 | y_true = np.array([[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0], 15 | [0, 0, 1., 0.]]) 16 | y_pred = np.array([[0, 0, 1, 0], [0, 0, 0.9, 0], [0, 0, 0.1, 0], 17 | [1, 1, 0.1, 1.]]) 18 | 19 | r = jaccard_distance( 20 | K.variable(y_true), 21 | K.variable(y_pred), ) 22 | if K.is_keras_tensor(r): 23 | assert K.int_shape(r) == (4, ) 24 | 25 | all_right, almost_right, half_right, all_wrong = K.eval(r) 26 | assert all_right == 0, 'should converge on zero' 27 | assert all_right < almost_right 28 | assert almost_right < half_right 29 | assert half_right < all_wrong 30 | 31 | 32 | def test_jaccard_distance_shapes_3d(): 33 | y_a = K.variable(np.random.random((5, 6, 7))) 34 | y_b = K.variable(np.random.random((5, 6, 7))) 35 | objective_output = jaccard_distance(y_a, y_b) 36 | assert K.eval(objective_output).shape == (5, 6) 37 | 38 | 39 | def test_jaccard_distance_shapes_2d(): 40 | y_a = K.variable(np.random.random((6, 7))) 41 | y_b = K.variable(np.random.random((6, 7))) 42 | objective_output = jaccard_distance(y_a, y_b) 43 | assert K.eval(objective_output).shape == (6, ) 44 | -------------------------------------------------------------------------------- /tests/keras_contrib/metrics/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/tests/keras_contrib/metrics/.gitkeep -------------------------------------------------------------------------------- /tests/keras_contrib/optimizers/ftml_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import pytest 3 | from keras_contrib.utils.test_utils import is_tf_keras 4 | from keras_contrib.tests import optimizers 5 | from keras_contrib.optimizers import ftml 6 | 7 | 8 | @pytest.mark.xfail(is_tf_keras, 9 | reason='TODO fix this.', 10 | strict=True) 11 | def test_ftml(): 12 | optimizers._test_optimizer(ftml()) 13 | optimizers._test_optimizer(ftml(lr=0.003, beta_1=0.8, 14 | beta_2=0.9, epsilon=1e-5, 15 | decay=1e-3)) 16 | -------------------------------------------------------------------------------- /tests/keras_contrib/optimizers/lars_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from keras_contrib.tests import optimizers 4 | from keras_contrib.optimizers import lars 5 | from keras.models import Sequential 6 | from keras.layers import Dense 7 | 8 | 9 | def test_base_lars(): 10 | optimizers._test_optimizer(lars(0.01)) 11 | 12 | 13 | def test_nesterov_lars(): 14 | optimizers._test_optimizer(lars(0.01, nesterov=True)) 15 | -------------------------------------------------------------------------------- /tests/keras_contrib/optimizers/padam_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras_contrib.tests import optimizers 3 | from keras_contrib.optimizers import Padam 4 | 5 | 6 | def test_padam(): 7 | optimizers._test_optimizer(Padam()) 8 | optimizers._test_optimizer(Padam(decay=1e-3)) 9 | -------------------------------------------------------------------------------- /tests/keras_contrib/optimizers/yogi_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import pytest 3 | from keras_contrib.tests import optimizers 4 | from keras_contrib.optimizers import Yogi 5 | from keras_contrib.utils.test_utils import is_tf_keras 6 | 7 | 8 | def test_yogi(): 9 | optimizers._test_optimizer(Yogi()) 10 | optimizers._test_optimizer(Yogi(beta_1=0.9, beta_2=0.9)) 11 | optimizers._test_optimizer(Yogi(beta_1=0.9, beta_2=0.99)) 12 | optimizers._test_optimizer(Yogi(beta_1=0.9, beta_2=0.999)) 13 | 14 | 15 | @pytest.mark.skipif(is_tf_keras, 16 | reason='Sometimes fail. It is random.', 17 | strict=True) 18 | def test_yogi_change_lr(): 19 | optimizers._test_optimizer(Yogi(beta_1=0.9, beta_2=0.999, lr=0.001)) 20 | -------------------------------------------------------------------------------- /tests/keras_contrib/preprocessing/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/tests/keras_contrib/preprocessing/.gitkeep -------------------------------------------------------------------------------- /tests/keras_contrib/regularizers/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/tests/keras_contrib/regularizers/.gitkeep -------------------------------------------------------------------------------- /tests/keras_contrib/utils/save_load_utils_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | from keras import backend as K 4 | from keras.layers import Input, Dense 5 | from keras.models import Model 6 | from numpy.testing import assert_allclose 7 | 8 | from keras_contrib.utils.save_load_utils import save_all_weights, load_all_weights 9 | 10 | 11 | @pytest.mark.skipif(K.backend() != 'tensorflow', 12 | reason='save_all_weights and load_all_weights only ' 13 | 'supported on TensorFlow') 14 | def test_save_and_load_all_weights(): 15 | ''' 16 | Test save_all_weights and load_all_weights. 17 | Save and load optimizer and model weights but not configuration. 18 | ''' 19 | 20 | def make_model(): 21 | _x = Input((10,)) 22 | _y = Dense(10)(_x) 23 | _m = Model(_x, _y) 24 | _m.compile('adam', 'mean_squared_error') 25 | _m._make_train_function() 26 | return _m 27 | 28 | # make a model 29 | m1 = make_model() 30 | # set weights 31 | w1 = m1.layers[1].kernel # dense layer 32 | w1value = K.get_value(w1) 33 | w1value[0, 0:4] = [1, 3, 3, 7] 34 | K.set_value(w1, w1value) 35 | # set optimizer weights 36 | ow1 = m1.optimizer.weights[3] # momentum weights 37 | ow1value = K.get_value(ow1) 38 | ow1value[0, 0:3] = [4, 2, 0] 39 | K.set_value(ow1, ow1value) 40 | # save all weights 41 | save_all_weights(m1, 'model.h5') 42 | # new model 43 | m2 = make_model() 44 | # load all weights 45 | load_all_weights(m2, 'model.h5') 46 | # check weights 47 | assert_allclose(K.get_value(m2.layers[1].kernel)[0, 0:4], [1, 3, 3, 7]) 48 | # check optimizer weights 49 | assert_allclose(K.get_value(m2.optimizer.weights[3])[0, 0:3], [4, 2, 0]) 50 | os.remove('model.h5') 51 | 52 | 53 | if __name__ == '__main__': 54 | pytest.main([__file__]) 55 | -------------------------------------------------------------------------------- /tests/keras_contrib/wrappers/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keras-team/keras-contrib/3fc5ef709e061416f4bc8a92ca3750c824b5d2b0/tests/keras_contrib/wrappers/.gitkeep -------------------------------------------------------------------------------- /tests/tooling/test_codeowners.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from github import Github 4 | try: 5 | import pathlib 6 | except ImportError: 7 | import pathlib2 as pathlib 8 | 9 | path_to_keras_contrib = pathlib.Path(__file__).resolve().parents[2] 10 | path_to_codeowners = path_to_keras_contrib / 'CODEOWNERS' 11 | 12 | authenticated = True 13 | try: 14 | github_client = Github(os.environ['GITHUB_TOKEN']) 15 | except KeyError: 16 | try: 17 | github_client = Github(os.environ['GITHUB_USER'], 18 | os.environ['GITHUB_PASSWORD']) 19 | except KeyError: 20 | authenticated = False 21 | 22 | 23 | def parse_codeowners(): 24 | map_path_owner = [] 25 | for line in open(path_to_codeowners, 'r'): 26 | line = line.strip() 27 | if line.startswith('#') or line == '': 28 | continue 29 | x = line.split(' ') 30 | path = path_to_keras_contrib / x[0] 31 | owner = x[-1] 32 | map_path_owner.append((path, owner)) 33 | return map_path_owner 34 | 35 | 36 | def test_codeowners_file_exist(): 37 | for path, _ in parse_codeowners(): 38 | assert path.exists() 39 | 40 | 41 | @pytest.mark.skipif(not authenticated, 42 | reason='It should be possible to run the test without' 43 | 'authentication, but we might get our request refused' 44 | 'by github. To be deterministic, we\'ll disable it.') 45 | def test_codeowners_user_exist(): 46 | for _, user in parse_codeowners(): 47 | assert user[0] == '@' 48 | assert github_client.get_user(user[1:]) 49 | 50 | 51 | directories_to_test = [ 52 | 'examples', 53 | 'keras_contrib/activations', 54 | 'keras_contrib/applications', 55 | 'keras_contrib/callbacks', 56 | 'keras_contrib/constraints', 57 | 'keras_contrib/datasets', 58 | 'keras_contrib/initializers', 59 | 'keras_contrib/layers', 60 | 'keras_contrib/losses', 61 | 'keras_contrib/metrics', 62 | 'keras_contrib/optimizers', 63 | 'keras_contrib/preprocessing', 64 | 'keras_contrib/regularizers', 65 | 'keras_contrib/wrappers' 66 | ] 67 | directories_to_test = [path_to_keras_contrib / x for x in directories_to_test] 68 | 69 | # TODO: remove those files or find them owners. 70 | exclude = [ 71 | 'examples/cifar10_clr.py', 72 | 'examples/cifar10_densenet.py', 73 | 'examples/cifar10_nasnet.py', 74 | 'examples/cifar10_resnet.py', 75 | 'examples/cifar10_ror.py', 76 | 'examples/cifar10_wide_resnet.py', 77 | 'examples/conll2000_chunking_crf.py', 78 | 'examples/improved_wgan.py', 79 | 'examples/jaccard_loss.py', 80 | 'keras_contrib/callbacks/cyclical_learning_rate.py', 81 | 'keras_contrib/callbacks/dead_relu_detector.py', 82 | 'keras_contrib/applications/resnet.py', 83 | 'keras_contrib/constraints/clip.py', 84 | 'keras_contrib/datasets/coco.py', 85 | 'keras_contrib/datasets/conll2000.py', 86 | 'keras_contrib/datasets/pascal_voc.py', 87 | 'keras_contrib/initializers/convaware.py', 88 | 'keras_contrib/losses/crf_losses.py', 89 | 'keras_contrib/losses/dssim.py', 90 | 'keras_contrib/losses/jaccard.py', 91 | 'keras_contrib/layers/advanced_activations/pelu.py', 92 | 'keras_contrib/layers/advanced_activations/srelu.py', 93 | 'keras_contrib/layers/convolutional/cosineconvolution2d.py', 94 | 'keras_contrib/layers/core.py', 95 | 'keras_contrib/layers/crf.py', 96 | 'keras_contrib/layers/normalization/instancenormalization.py', 97 | 'keras_contrib/optimizers/ftml.py', 98 | 'keras_contrib/optimizers/lars.py', 99 | 'keras_contrib/metrics/crf_accuracies.py', 100 | ] 101 | exclude = [path_to_keras_contrib / x for x in exclude] 102 | 103 | 104 | @pytest.mark.parametrize('directory', directories_to_test) 105 | def test_all_files_have_owners(directory): 106 | files_with_owners = [x[0] for x in parse_codeowners()] 107 | for root, dirs, files in os.walk(directory): 108 | for name in files: 109 | file_path = pathlib.Path(root) / name 110 | if file_path.suffix != '.py': 111 | continue 112 | if file_path.name == '__init__.py': 113 | continue 114 | if file_path in exclude: 115 | continue 116 | assert file_path in files_with_owners 117 | 118 | 119 | if __name__ == '__main__': 120 | pytest.main([__file__]) 121 | -------------------------------------------------------------------------------- /tests/tooling/test_documentation.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import inspect 3 | import re 4 | import sys 5 | from itertools import compress 6 | 7 | import pytest 8 | 9 | modules = ['keras_contrib.layers', 10 | 'keras_contrib', 11 | 'keras_contrib.backend.tensorflow_backend', 12 | 'keras_contrib.wrappers', 13 | 'keras_contrib.utils', 14 | 'keras_contrib.callbacks', 15 | 'keras_contrib.activations', 16 | 'keras_contrib.losses', 17 | 'keras_contrib.optimizers'] 18 | accepted_name = ['from_config'] 19 | accepted_module = [] 20 | 21 | # Functions or classes with less than 'MIN_CODE_SIZE' lines can be ignored 22 | MIN_CODE_SIZE = 10 23 | 24 | 25 | def handle_class(name, member): 26 | if is_accepted(name, member): 27 | return 28 | 29 | if member.__doc__ is None and not member_too_small(member): 30 | raise ValueError("{} class doesn't have any documentation".format(name), 31 | member.__module__, inspect.getmodule(member).__file__) 32 | for n, met in inspect.getmembers(member): 33 | if inspect.ismethod(met): 34 | handle_method(n, met) 35 | 36 | 37 | def handle_function(name, member): 38 | if is_accepted(name, member) or member_too_small(member): 39 | # We don't need to check this one. 40 | return 41 | doc = member.__doc__ 42 | if doc is None: 43 | raise ValueError("{} function doesn't have any documentation".format(name), 44 | member.__module__, inspect.getmodule(member).__file__) 45 | 46 | args = list(inspect.signature(member).parameters.keys()) 47 | assert_args_presence(args, doc, member, name) 48 | assert_function_style(name, member, doc, args) 49 | assert_doc_style(name, member, doc) 50 | 51 | 52 | def assert_doc_style(name, member, doc): 53 | lines = doc.split("\n") 54 | first_line = lines[0] 55 | if len(first_line.strip()) == 0: 56 | raise ValueError( 57 | "{} the documentation should be on the first line.".format(name), 58 | member.__module__) 59 | if first_line.strip()[-1] != '.': 60 | raise ValueError("{} first line should end with a '.'".format(name), 61 | member.__module__) 62 | 63 | 64 | def assert_function_style(name, member, doc, args): 65 | code = inspect.getsource(member) 66 | has_return = re.findall(r"\s*return \S+", code, re.MULTILINE) 67 | if has_return and "# Returns" not in doc: 68 | innerfunction = [inspect.getsource(x) for x in member.__code__.co_consts if 69 | inspect.iscode(x)] 70 | return_in_sub = [ret for code_inner in innerfunction for ret in 71 | re.findall(r"\s*return \S+", code_inner, re.MULTILINE)] 72 | if len(return_in_sub) < len(has_return): 73 | raise ValueError("{} needs a '# Returns' section".format(name), 74 | member.__module__) 75 | 76 | has_raise = re.findall(r"^\s*raise \S+", code, re.MULTILINE) 77 | if has_raise and "# Raises" not in doc: 78 | innerfunction = [inspect.getsource(x) for x in member.__code__.co_consts if 79 | inspect.iscode(x)] 80 | raise_in_sub = [ret for code_inner in innerfunction for ret in 81 | re.findall(r"\s*raise \S+", code_inner, re.MULTILINE)] 82 | if len(raise_in_sub) < len(has_raise): 83 | raise ValueError("{} needs a '# Raises' section".format(name), 84 | member.__module__) 85 | 86 | if len(args) > 0 and "# Arguments" not in doc: 87 | raise ValueError("{} needs a '# Arguments' section".format(name), 88 | member.__module__) 89 | 90 | assert_blank_before(name, member, doc, ['# Arguments', '# Raises', '# Returns']) 91 | 92 | 93 | def assert_blank_before(name, member, doc, keywords): 94 | doc_lines = [x.strip() for x in doc.split('\n')] 95 | for keyword in keywords: 96 | if keyword in doc_lines: 97 | index = doc_lines.index(keyword) 98 | if doc_lines[index - 1] != '': 99 | raise ValueError( 100 | "{} '{}' should have a blank line above.".format(name, keyword), 101 | member.__module__) 102 | 103 | 104 | def is_accepted(name, member): 105 | if 'keras' not in str(member.__module__): 106 | return True 107 | return name in accepted_name or member.__module__ in accepted_module 108 | 109 | 110 | def member_too_small(member): 111 | code = inspect.getsource(member).split('\n') 112 | return len(code) < MIN_CODE_SIZE 113 | 114 | 115 | def assert_args_presence(args, doc, member, name): 116 | args_not_in_doc = [arg not in doc for arg in args] 117 | if any(args_not_in_doc): 118 | raise ValueError( 119 | "{} {} arguments are not present in documentation ".format(name, list( 120 | compress(args, args_not_in_doc))), member.__module__) 121 | words = doc.replace('*', '').split() 122 | # Check arguments styling 123 | styles = [arg + ":" not in words for arg in args] 124 | if any(styles): 125 | raise ValueError( 126 | "{} {} are not style properly 'argument': documentation".format( 127 | name, 128 | list(compress(args, styles))), 129 | member.__module__) 130 | 131 | # Check arguments order 132 | indexes = [words.index(arg + ":") for arg in args] 133 | if indexes != sorted(indexes): 134 | raise ValueError( 135 | "{} arguments order is different from the documentation".format(name), 136 | member.__module__) 137 | 138 | 139 | def handle_method(name, member): 140 | if name in accepted_name or member.__module__ in accepted_module: 141 | return 142 | handle_function(name, member) 143 | 144 | 145 | def handle_module(mod): 146 | for name, mem in inspect.getmembers(mod): 147 | if inspect.isclass(mem): 148 | handle_class(name, mem) 149 | elif inspect.isfunction(mem): 150 | handle_function(name, mem) 151 | elif 'keras' in name and inspect.ismodule(mem): 152 | # Only test keras' modules 153 | handle_module(mem) 154 | 155 | 156 | @pytest.mark.skipif(sys.version_info < (3, 3), reason="requires python3.3") 157 | @pytest.mark.parametrize('module', modules) 158 | def test_doc(module): 159 | mod = importlib.import_module(module) 160 | handle_module(mod) 161 | 162 | 163 | if __name__ == '__main__': 164 | pytest.main([__file__]) 165 | --------------------------------------------------------------------------------