├── .gitignore ├── .idea ├── dense_tensor.iml ├── encodings.xml ├── misc.xml ├── modules.xml └── workspace.xml ├── .travis.yml ├── LICENSE ├── README.md ├── dense_tensor ├── __init__.py ├── backend │ ├── __init__.py │ ├── tensorflow_backend.py │ └── theano_backend.py ├── dense_tensor.py ├── example_utils.py ├── tensor_factorization.py └── utils.py ├── examples ├── example_tensor.py ├── example_tensor_low_rank.py └── example_tensor_symmetric.py ├── pytest.ini ├── setup.py └── tests └── dense_tensor_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | /output 91 | -------------------------------------------------------------------------------- /.idea/dense_tensor.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 14 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 20 | 21 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 10 | 11 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 30 | 31 | 32 | 1478543705718 33 | 37 | 38 | 39 | 40 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: python 4 | matrix: 5 | include: 6 | - python: 2.7 7 | env: KERAS_BACKEND=theano TEST_MODE=PEP8 8 | - python: 2.7 9 | env: KERAS_BACKEND=theano LEGACY_KERAS=1 10 | - python: 3.5 11 | env: KERAS_BACKEND=theano LEGACY_KERAS=1 12 | - python: 2.7 13 | env: KERAS_BACKEND=theano 14 | - python: 3.5 15 | env: KERAS_BACKEND=theano 16 | - python: 2.7 17 | env: KERAS_BACKEND=tensorflow LEGACY_KERAS=1 18 | - python: 3.5 19 | env: KERAS_BACKEND=tensorflow LEGACY_KERAS=1 20 | - python: 2.7 21 | env: KERAS_BACKEND=tensorflow 22 | - python: 3.5 23 | env: KERAS_BACKEND=tensorflow 24 | install: 25 | # code below is taken from http://conda.pydata.org/docs/travis.html 26 | # We do this conditionally because it saves us some downloading if the 27 | # version is the same. 28 | - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then 29 | wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; 30 | else 31 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 32 | fi 33 | - bash miniconda.sh -b -p $HOME/miniconda 34 | - export PATH="$HOME/miniconda/bin:$PATH" 35 | - hash -r 36 | - conda config --set always_yes yes --set changeps1 no 37 | - conda update -q conda 38 | # Useful for debugging any issues with conda 39 | - conda info -a 40 | 41 | - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib pandas pytest h5py 42 | - source activate test-environment 43 | - pip install git+git://github.com/Theano/Theano.git 44 | - pip install pytest-pep8 45 | # install PIL for preprocessing tests 46 | - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then 47 | conda install pil; 48 | elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then 49 | conda install Pillow; 50 | fi 51 | 52 | - pip install -e .[tests] 53 | 54 | # install TensorFlow 55 | - pip install tensorflow 56 | - pip install tqdm 57 | - if [[ "$LEGACY_KERAS" == "1" ]]; then 58 | pip install keras==1.2.2; 59 | else 60 | pip install keras; 61 | fi 62 | 63 | # command to run tests 64 | script: 65 | # run keras backend init to initialize backend config 66 | - python -c "import keras.backend" 67 | # create dataset directory to avoid concurrent directory creation at runtime 68 | - mkdir ~/.keras/datasets 69 | # set up keras backend 70 | - sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json; 71 | - echo -e "Running tests with the following config:\n$(cat ~/.keras/keras.json)" 72 | - if [[ "$TEST_MODE" == "PEP8" ]]; then 73 | PYTHONPATH=$PWD:$PYTHONPATH py.test --pep8 -m pep8; 74 | else 75 | PYTHONPATH=$PWD:$PYTHONPATH py.test tests/; 76 | fi 77 | after_success: 78 | - coveralls 79 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 BenStriner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dense_tensor 2 | Dense Tensor Layer for Keras. Supports both Keras 1 and 2. Tensor networks/second order networks. 3 | 4 | Basically, this is like a quadratic layer if all other layers are linear. 5 | 6 | There is an additional weight matrix V. The layer output is `xVx+xW+b` instead of simply `xW+b`. 7 | 8 | See analysis by Socher. 9 | 10 | * http://nlp.stanford.edu/~socherr/SocherChenManningNg_NIPS2013.pdf 11 | * http://nlp.stanford.edu/~socherr/EMNLP2013_RNTN.pdf 12 | 13 | See also Fenglei Fan, Wenxiang Cong, Ge Wang (2017) A New Type of Neurons for Machine Learning. 14 | 15 | * https://arxiv.org/ftp/arxiv/papers/1704/1704.08362.pdf 16 | 17 | Normal Dense Layer: `f_i = a( W_ix^T + b_i)` 18 | 19 | Dense Tensor Layer: `f_i = a( xV_ix^T + W_ix^T + b_i)` 20 | 21 | `DenseTensor`: same usage as Keras `Dense` Layer 22 | 23 | ## Variations 24 | 25 | I provided several examples for different parameterizations of V, including a low-rank version of V, 26 | a symmetric V, and V restricted to positive-definite matrices. Please explore the examples and ask any questions. 27 | 28 | ### Simple parameterization 29 | 30 | ```python 31 | x = Input(input_dim) 32 | layer = DenseTensor(units=units) 33 | y = layer(x) 34 | ``` 35 | 36 | ### Low-rank parameterization 37 | 38 | ```python 39 | factorization = tensor_factorization_low_rank(q=10) 40 | layer = DenseTensor(units=units, factorization=factorization) 41 | ``` 42 | 43 | ## Comments 44 | 45 | Please feel free to add issues or pull requests. I'm always interested in any improvements or issues. 46 | 47 | ## Compatibility 48 | 49 | Travis tests a matrix including Theano, tensorflow, Python 2.7, Python 3.5, Keras 1 and Keras 2. 50 | Code should work on most configurations but please let me know if you run into issues. 51 | -------------------------------------------------------------------------------- /dense_tensor/__init__.py: -------------------------------------------------------------------------------- 1 | from .dense_tensor import DenseTensor 2 | from .tensor_factorization import simple_tensor_factorization 3 | from .tensor_factorization import tensor_factorization_low_rank 4 | from .tensor_factorization import tensor_factorization_symmetric 5 | -------------------------------------------------------------------------------- /dense_tensor/backend/__init__.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | 4 | def keras_backend(): 5 | if hasattr(K, 'backend'): 6 | return K.backend() 7 | else: 8 | return K._BACKEND 9 | 10 | 11 | if keras_backend() == 'theano': 12 | from .theano_backend import * 13 | elif keras_backend() == 'tensorflow': 14 | from .tensorflow_backend import * 15 | else: 16 | raise ValueError("Unknown backend: {}".format(keras_backend())) 17 | -------------------------------------------------------------------------------- /dense_tensor/backend/tensorflow_backend.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | import tensorflow as tf 3 | 4 | 5 | def eye(n, m): 6 | return tf.eye(n, m) 7 | 8 | 9 | def quadratic_batch(x, V): 10 | tmp1 = K.dot(x, V) # n,input_dim + units,input_dim,input_dim = n,units,input_dim 11 | xr = K.expand_dims(x, 2) # n, 1, input_dim 12 | tmp2 = K.permute_dimensions(tmp1, (0, 2, 1)) # n, input_dim, units 13 | tmp3 = K.batch_dot(xr, tmp2, axes=[[1], [1]]) # n,1,input_dim + n,input_dim,units = n,1,units 14 | tmp4 = tmp3[:, 0, :] 15 | return tmp4 16 | -------------------------------------------------------------------------------- /dense_tensor/backend/theano_backend.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | import theano.tensor as T 3 | 4 | 5 | def eye(n, m): 6 | return T.eye(n=n, m=m) 7 | 8 | 9 | def quadratic_batch(x, V): 10 | tmp1 = K.dot(x, V) # n,input_dim + units,input_dim,input_dim = n,units,input_dim 11 | tmp2 = K.batch_dot(x, tmp1, axes=[[1], [2]]) # n,input_dim + n,units,input_dim = n,units 12 | return tmp2 13 | -------------------------------------------------------------------------------- /dense_tensor/dense_tensor.py: -------------------------------------------------------------------------------- 1 | """ 2 | DenseTensor Layer 3 | Based on Dense Layer (https://github.com/fchollet/keras/blob/master/keras/layers/core.py) 4 | Calculates f_i = a( xV_ix^T + W_ix^T + b_i) 5 | """ 6 | 7 | from keras import activations, regularizers, constraints 8 | from keras import backend as K 9 | from keras.engine import InputSpec, Layer 10 | from .backend import quadratic_batch 11 | from .tensor_factorization import simple_tensor_factorization 12 | from .utils import add_weight, get_initializer, add_activity_regularizer 13 | 14 | 15 | class DenseTensor(Layer): 16 | '''Tensor layer: a = f(xVx^T + Wx + b) 17 | # Example 18 | ```python 19 | # as first layer in a sequential model: 20 | model = Sequential() 21 | model.add(DenseTensor(32, input_dim=16)) 22 | # now the model will take as input arrays of shape (*, 16) 23 | # and output arrays of shape (*, 32) 24 | # this is equivalent to the above: 25 | model = Sequential() 26 | model.add(DenseTensor(32, input_shape=(16,))) 27 | # after the first layer, you don't need to specify 28 | # the size of the input anymore: 29 | model.add(DenseTensor(32)) 30 | ``` 31 | # Arguments 32 | output_dim: int > 0. 33 | init: name of initialization function for the weights of the layer 34 | (see [initializations](../initializations.md)), 35 | or alternatively, Theano function to use for weights 36 | initialization. This parameter is only relevant 37 | if you don't pass a `weights` argument. 38 | activation: name of activation function to use 39 | (see [activations](../activations.md)), 40 | or alternatively, elementwise Theano function. 41 | If you don't specify anything, no activation is applied 42 | (ie. "linear" activation: a(x) = x). 43 | weights: list of Numpy arrays to set as initial weights. 44 | The list should have 2 elements, of shape `(input_dim, output_dim)` 45 | and (output_dim,) for weights and biases respectively. 46 | W_regularizer: instance of [WeightRegularizer](../regularizers.md) 47 | (eg. L1 or L2 regularization), applied to the main weights matrix. 48 | V_regularizer: instance of [WeightRegularizer](../regularizers.md) 49 | (eg. L1 or L2 regularization), applied to the V matrix (input_dim x output_dim x input_dim). 50 | b_regularizer: instance of [WeightRegularizer](../regularizers.md), 51 | applied to the bias. 52 | activity_regularizer: instance of [ActivityRegularizer](../regularizers.md), 53 | applied to the network output. 54 | W_constraint: instance of the [constraints](../constraints.md) module 55 | (eg. maxnorm, nonneg), applied to the main weights matrix. 56 | b_constraint: instance of the [constraints](../constraints.md) module, 57 | applied to the bias. 58 | bias: whether to include a bias (i.e. make the layer affine rather than linear). 59 | input_dim: dimensionality of the input (integer). 60 | This argument (or alternatively, the keyword argument `input_shape`) 61 | is required when using this layer as the first layer in a model. 62 | # Input shape 63 | 2D tensor with shape: `(nb_samples, input_dim)`. 64 | # Output shape 65 | 2D tensor with shape: `(nb_samples, output_dim)`. 66 | ''' 67 | 68 | def __init__(self, units, 69 | activation='linear', 70 | weights=None, 71 | kernel_initializer='glorot_uniform', 72 | kernel_regularizer=None, 73 | kernel_constraint=None, 74 | bias_initializer='uniform', 75 | bias_regularizer=None, 76 | bias_constraint=None, 77 | activity_regularizer=None, 78 | bias=True, 79 | input_dim=None, 80 | factorization=simple_tensor_factorization(), 81 | **kwargs): 82 | self.activation = activations.get(activation) 83 | self.units = units 84 | self.input_dim = input_dim 85 | self.factorization = factorization 86 | 87 | self.kernel_regularizer = regularizers.get(kernel_regularizer) 88 | self.bias_regularizer = regularizers.get(bias_regularizer) 89 | self.kernel_initializer = get_initializer(kernel_initializer) 90 | self.bias_initializer = get_initializer(bias_initializer) 91 | self.kernel_constraint = constraints.get(kernel_constraint) 92 | self.bias_constraint = constraints.get(bias_constraint) 93 | 94 | self.activity_regularizer = regularizers.get(activity_regularizer) 95 | 96 | self.bias = bias 97 | self.initial_weights = weights 98 | self.input_spec = [InputSpec(ndim=2)] 99 | 100 | if self.input_dim: 101 | kwargs['input_shape'] = (self.input_dim,) 102 | super(DenseTensor, self).__init__(**kwargs) 103 | 104 | def build(self, input_shape): 105 | assert len(input_shape) == 2 106 | input_dim = input_shape[1] 107 | self.input_spec = [InputSpec(dtype=K.floatx(), 108 | shape=(None, input_dim))] 109 | 110 | self.W = add_weight(layer=self, 111 | shape=(input_dim, self.units), 112 | name='{}_W'.format(self.name), 113 | initializer=self.kernel_initializer, 114 | regularizer=self.kernel_regularizer, 115 | constraint=self.kernel_constraint) 116 | self.V_weights, self.V = self.factorization(name='{}_V'.format(self.name), 117 | layer=self, 118 | input_dim=input_dim, 119 | units=self.units) 120 | if self.bias: 121 | self.b = add_weight(layer=self, 122 | shape=(self.units,), 123 | name='{}_b'.format(self.name), 124 | initializer=self.bias_initializer, 125 | regularizer=self.bias_regularizer, 126 | constraint=self.bias_constraint) 127 | 128 | add_activity_regularizer(layer=self) 129 | 130 | if self.initial_weights is not None: 131 | self.set_weights(self.initial_weights) 132 | del self.initial_weights 133 | 134 | def call(self, x, mask=None): 135 | output = K.dot(x, self.W) 136 | q = quadratic_batch(x, self.V) 137 | output += q 138 | if self.bias: 139 | output += self.b 140 | return self.activation(output) 141 | 142 | def get_output_shape_for(self, input_shape): 143 | return self.compute_output_shape(input_shape) 144 | 145 | def compute_output_shape(self, input_shape): 146 | assert input_shape and len(input_shape) == 2 147 | return input_shape[0], self.units 148 | 149 | def get_config(self): 150 | config = {'units': self.units, 151 | 'activation': self.activation.__name__, 152 | 'kernel_initializer': self.kernel_initializer.__name__, 153 | 'kernel_regularizer': self.kernel_regularizer.get_config() if self.kernel_regularizer else None, 154 | 'kernel_constraint': self.kernel_constraint.get_config() if self.kernel_constraint else None, 155 | 'bias_initializer': self.bias_initializer.__name__, 156 | 'bias_regularizer': self.bias_regularizer.get_config() if self.bias_regularizer else None, 157 | 'bias_constraint': self.bias_constraint.get_config() if self.bias_constraint else None, 158 | 'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None, 159 | 'bias': self.bias, 160 | 'input_dim': self.input_dim} 161 | base_config = super(DenseTensor, self).get_config() 162 | return dict(list(base_config.items()) + list(config.items())) 163 | -------------------------------------------------------------------------------- /dense_tensor/example_utils.py: -------------------------------------------------------------------------------- 1 | """"Example usage of DenseTensor layer on MNIST dataset (~0.2% train/2% test error with single layer). """ 2 | 3 | import os 4 | 5 | import numpy as np 6 | import pandas as pd 7 | from keras.datasets import mnist 8 | from keras.utils.np_utils import to_categorical 9 | 10 | from .utils import fit 11 | 12 | 13 | def mnist_data(): 14 | """Rescale and reshape MNIST data""" 15 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 16 | x_train = x_train.astype(np.float32) / 255. 17 | x_test = x_test.astype(np.float32) / 255. 18 | x_train = x_train.reshape((x_train.shape[0], -1)) 19 | x_test = x_test.reshape((x_test.shape[0], -1)) 20 | return x_train, y_train, x_test, y_test 21 | 22 | 23 | def experiment(path, model, epochs=100): 24 | if not os.path.exists(path): 25 | os.makedirs(path) 26 | print("Training %s" % path) 27 | model.summary() 28 | csvpath = os.path.join(path, "history.csv") 29 | modelpath = os.path.join(path, "model.h5") 30 | if os.path.exists(csvpath): 31 | print("Already exists: %s" % csvpath) 32 | return 33 | x_train, y_train, x_test, y_test = mnist_data() 34 | 35 | batch_size = 32 36 | k = 10 37 | history = fit(model, x_train, to_categorical(y_train, k), 38 | epochs=epochs, 39 | batch_size=batch_size, 40 | validation_data=(x_test, to_categorical(y_test, k))) 41 | model.save_weights(modelpath) 42 | df = pd.DataFrame(history.history) 43 | df.to_csv(csvpath) 44 | -------------------------------------------------------------------------------- /dense_tensor/tensor_factorization.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | from .backend import eye 4 | from .utils import add_weight 5 | 6 | """ 7 | Factorizations of inner tensor. Each factorization should return a tuple of parameters and the tensor. 8 | """ 9 | 10 | 11 | def simple_tensor_factorization(tensor_initializer='uniform', 12 | tensor_regularizer=None, 13 | tensor_constraint=None): 14 | def fun(layer, units, input_dim, name): 15 | V = add_weight(layer=layer, 16 | initializer=tensor_initializer, 17 | regularizer=tensor_regularizer, 18 | constraint=tensor_constraint, 19 | shape=(units, input_dim, input_dim), 20 | name=name) 21 | return [V], V 22 | 23 | return fun 24 | 25 | 26 | def tensor_factorization_low_rank(q, 27 | tensor_initializer='uniform', 28 | tensor_regularizer=None, 29 | tensor_constraint=None): 30 | def fun(layer, units, input_dim, name): 31 | qs = [add_weight(layer=layer, 32 | initializer=tensor_initializer, 33 | regularizer=tensor_regularizer, 34 | constraint=tensor_constraint, 35 | shape=(units, q, input_dim), 36 | name="{}_Q{}".format(name, i)) for i in range(2)] 37 | V = K.batch_dot(qs[0], qs[1], axes=[[1], [1]]) # p,m,q + p,q,m = p,m,m 38 | return qs, V 39 | 40 | return fun 41 | 42 | 43 | def tensor_factorization_symmetric(q, 44 | alpha=1e-7, 45 | beta=1.0, 46 | tensor_initializer='uniform', 47 | tensor_regularizer=None, 48 | tensor_constraint=None): 49 | """ 50 | :param q: rank of inner parameter 51 | :param alpha: scale of eye to add. 0=pos/neg semidefinite, >0=pos/neg definite 52 | :param beta: multiplier of tensor. 1=positive,-1=negative 53 | """ 54 | 55 | def fun(layer, units, input_dim, name): 56 | Q = add_weight(layer=layer, 57 | initializer=tensor_initializer, 58 | regularizer=tensor_regularizer, 59 | constraint=tensor_constraint, 60 | shape=(units, q, input_dim), 61 | name=name) # units, input_dim, q 62 | tmp = K.batch_dot(Q, Q, axes=[[1], [1]]) # p,m,q + p,m,q = p,m,m 63 | V = beta * ((eye(input_dim, input_dim) * alpha) + tmp) # m,p,p 64 | return [q], V 65 | 66 | return fun 67 | -------------------------------------------------------------------------------- /dense_tensor/utils.py: -------------------------------------------------------------------------------- 1 | # Utils for Keras 1/2 compatibility 2 | 3 | import keras 4 | 5 | keras_2 = int(keras.__version__.split(".")[0]) > 1 # Keras > 1 6 | 7 | 8 | def add_activity_regularizer(layer): 9 | if layer.activity_regularizer and not keras_2: 10 | layer.activity_regularizer.set_layer(layer) 11 | if not hasattr(layer, 'regularizers'): 12 | layer.regularizers = [] 13 | layer.regularizers.append(layer.activity_regularizer) 14 | 15 | 16 | def l1l2(l1_weight=0, l2_weight=0): 17 | if keras_2: 18 | from keras.regularizers import L1L2 19 | return L1L2(l1_weight, l2_weight) 20 | else: 21 | from keras.regularizers import l1l2 22 | return l1l2(l1_weight, l2_weight) 23 | 24 | 25 | def get_initializer(initializer): 26 | if keras_2: 27 | from keras import initializers 28 | return initializers.get(initializer) 29 | else: 30 | from keras import initializations 31 | return initializations.get(initializer) 32 | 33 | 34 | def fit(model, x, y, epochs=100, **kwargs): 35 | if keras_2: 36 | return model.fit(x, y, epochs=epochs, **kwargs) 37 | else: 38 | return model.fit(x, y, nb_epoch=epochs, **kwargs) 39 | 40 | 41 | def add_weight(layer, 42 | shape, 43 | name, 44 | initializer='random_uniform', 45 | regularizer=None, 46 | constraint=None): 47 | initializer = get_initializer(initializer) 48 | if keras_2: 49 | return layer.add_weight(initializer=initializer, 50 | shape=shape, 51 | name=name, 52 | regularizer=regularizer, 53 | constraint=constraint) 54 | else: 55 | # create weight 56 | w = initializer(shape, name=name) 57 | # add to trainable_weights 58 | if not hasattr(layer, 'trainable_weights'): 59 | layer.trainable_weights = [] 60 | layer.trainable_weights.append(w) 61 | # add to regularizers 62 | if regularizer: 63 | if not hasattr(layer, 'regularizers'): 64 | layer.regularizers = [] 65 | regularizer.set_param(w) 66 | layer.regularizers.append(regularizer) 67 | return w 68 | -------------------------------------------------------------------------------- /examples/example_tensor.py: -------------------------------------------------------------------------------- 1 | """"Example usage of DenseTensor layer on MNIST dataset (~0.2% train/2% test error with single layer). """ 2 | from keras.layers import Input 3 | from keras.models import Model 4 | from keras.optimizers import Adam 5 | 6 | from dense_tensor import DenseTensor, simple_tensor_factorization 7 | from dense_tensor.example_utils import experiment 8 | from dense_tensor.utils import l1l2 9 | 10 | 11 | def tensor_model(input_dim=28 * 28, output_dim=10, reg=lambda: l1l2(1e-6, 1e-6)): 12 | """ 13 | One layer of a DenseTensor 14 | """ 15 | _x = Input(shape=(input_dim,)) 16 | factorization = simple_tensor_factorization(tensor_regularizer=reg()) 17 | y = DenseTensor(units=output_dim, 18 | activation='softmax', 19 | kernel_regularizer=reg(), 20 | factorization=factorization) 21 | _y = y(_x) 22 | m = Model(_x, _y) 23 | m.compile(Adam(1e-3, decay=1e-4), loss='categorical_crossentropy', metrics=["accuracy"]) 24 | return m 25 | 26 | 27 | if __name__ == "__main__": 28 | path = "output/dense_tensor" 29 | model = tensor_model() 30 | experiment(path, model) 31 | -------------------------------------------------------------------------------- /examples/example_tensor_low_rank.py: -------------------------------------------------------------------------------- 1 | """"Example usage of DenseTensor layer on MNIST dataset (~0.2% train/2% test error with single layer). """ 2 | 3 | from keras.layers import Input 4 | from keras.models import Model 5 | from keras.optimizers import Adam 6 | from dense_tensor import DenseTensor, tensor_factorization_low_rank 7 | 8 | from dense_tensor.utils import l1l2 9 | from dense_tensor.example_utils import experiment 10 | 11 | 12 | def tensor_model_low_rank(input_dim=28 * 28, output_dim=10, reg=lambda: l1l2(1e-6, 1e-6)): 13 | """ 14 | One layer of a DenseTensor low rank 15 | """ 16 | _x = Input(shape=(input_dim,)) 17 | factorization = tensor_factorization_low_rank(q=5, tensor_regularizer=reg()) 18 | y = DenseTensor(units=output_dim, 19 | activation='softmax', 20 | kernel_regularizer=reg(), 21 | factorization=factorization) 22 | _y = y(_x) 23 | m = Model(_x, _y) 24 | m.compile(Adam(1e-3, decay=1e-4), loss='categorical_crossentropy', metrics=["accuracy"]) 25 | return m 26 | 27 | 28 | if __name__ == "__main__": 29 | path = "output/dense_tensor_low_rank" 30 | model = tensor_model_low_rank() 31 | experiment(path, model) 32 | -------------------------------------------------------------------------------- /examples/example_tensor_symmetric.py: -------------------------------------------------------------------------------- 1 | """"Example usage of DenseTensor layer on MNIST dataset (~0.2% train/2% test error with single layer). """ 2 | 3 | from keras.layers import Input 4 | from keras.models import Model 5 | from keras.optimizers import Adam 6 | 7 | from dense_tensor import DenseTensor, tensor_factorization_symmetric 8 | from dense_tensor.example_utils import experiment 9 | from dense_tensor.utils import l1l2 10 | 11 | 12 | def tensor_model_symmetric(input_dim=28 * 28, output_dim=10, reg=lambda: l1l2(1e-6, 1e-6)): 13 | """ 14 | One layer of a DenseTensor low rank 15 | """ 16 | _x = Input(shape=(input_dim,)) 17 | factorization = tensor_factorization_symmetric(q=5, tensor_regularizer=reg()) 18 | y = DenseTensor(units=output_dim, 19 | activation='softmax', 20 | kernel_regularizer=reg(), 21 | factorization=factorization) 22 | _y = y(_x) 23 | m = Model(_x, _y) 24 | m.compile(Adam(1e-3, decay=1e-4), loss='categorical_crossentropy', metrics=["accuracy"]) 25 | return m 26 | 27 | 28 | if __name__ == "__main__": 29 | path = "output/dense_tensor_symmetric" 30 | model = tensor_model_symmetric() 31 | experiment(path, model) 32 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # Configuration of py.test 2 | [pytest] 3 | addopts=-v 4 | --durations=10 5 | 6 | python_functions = *_test 7 | # Do not run tests in the build folder 8 | norecursedirs= build 9 | 10 | # PEP-8 The following are ignored: 11 | # E501 line too long (82 > 79 characters) 12 | # E402 module level import not at top of file - temporary measure to continue adding ros python packaged in sys.path 13 | # E731 do not assign a lambda expression, use a def 14 | 15 | pep8ignore=* E501 \ 16 | * E402 \ 17 | * E731 \ 18 | 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | version = '0.0.2' 4 | 5 | setup(name='dense-tensor', 6 | version=version, 7 | description='Neural tensor layer for Keras', 8 | url='https://github.com/bstriner/dense_tensor', 9 | author='Ben Striner', 10 | author_email='bstriner@gmail.com', 11 | packages=find_packages(), 12 | install_requires=['Keras'], 13 | keywords=['keras', 'tensor', 'neural tensor network'], 14 | license='MIT', 15 | classifiers=[ 16 | # Indicate who your project is intended for 17 | 'Intended Audience :: Developers', 18 | # Pick your license as you wish (should match "license" above) 19 | 'License :: OSI Approved :: MIT License', 20 | 21 | # Specify the Python versions you support here. In particular, ensure 22 | # that you indicate whether you support Python 2, Python 3 or both. 23 | 'Programming Language :: Python :: 2', 24 | 'Programming Language :: Python :: 3' 25 | ]) 26 | -------------------------------------------------------------------------------- /tests/dense_tensor_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from keras.layers import Input 3 | from keras.models import Model 4 | from keras.optimizers import Adam 5 | from keras.utils.np_utils import to_categorical 6 | 7 | from dense_tensor import DenseTensor 8 | from dense_tensor import simple_tensor_factorization 9 | from dense_tensor import tensor_factorization_low_rank 10 | from dense_tensor import tensor_factorization_symmetric 11 | from dense_tensor.example_utils import mnist_data 12 | from dense_tensor.utils import l1l2, fit 13 | 14 | 15 | # Models 16 | 17 | def tensor_model(input_dim=28 * 28, output_dim=10, reg=lambda: l1l2(1e-6, 1e-6)): 18 | """ 19 | One layer of a DenseTensor 20 | """ 21 | _x = Input(shape=(input_dim,)) 22 | factorization = simple_tensor_factorization(tensor_regularizer=reg()) 23 | y = DenseTensor(units=output_dim, 24 | activation='softmax', 25 | kernel_regularizer=reg(), 26 | factorization=factorization) 27 | _y = y(_x) 28 | m = Model(_x, _y) 29 | m.compile(Adam(1e-3, decay=1e-4), loss='categorical_crossentropy', metrics=["accuracy"]) 30 | return m 31 | 32 | 33 | def tensor_model_low_rank(input_dim=28 * 28, output_dim=10, reg=lambda: l1l2(1e-6, 1e-6)): 34 | """ 35 | One layer of a DenseTensor low rank 36 | """ 37 | _x = Input(shape=(input_dim,)) 38 | factorization = tensor_factorization_low_rank(q=10, tensor_regularizer=reg()) 39 | y = DenseTensor(units=output_dim, 40 | activation='softmax', 41 | kernel_regularizer=reg(), 42 | factorization=factorization) 43 | _y = y(_x) 44 | m = Model(_x, _y) 45 | m.compile(Adam(1e-3, decay=1e-4), loss='categorical_crossentropy', metrics=["accuracy"]) 46 | return m 47 | 48 | 49 | def tensor_model_symmetric(input_dim=28 * 28, output_dim=10, reg=lambda: l1l2(1e-6, 1e-6)): 50 | """ 51 | One layer of a DenseTensor low rank 52 | """ 53 | _x = Input(shape=(input_dim,)) 54 | factorization = tensor_factorization_symmetric(q=10, tensor_regularizer=reg()) 55 | y = DenseTensor(units=output_dim, 56 | activation='softmax', 57 | kernel_regularizer=reg(), 58 | factorization=factorization) 59 | _y = y(_x) 60 | m = Model(_x, _y) 61 | m.compile(Adam(1e-3, decay=1e-4), loss='categorical_crossentropy', metrics=["accuracy"]) 62 | return m 63 | 64 | 65 | @pytest.mark.parametrize("model_function_name", ["tensor_model", "tensor_model_low_rank", "tensor_model_symmetric"]) 66 | def dense_tensor_test(model_function_name): 67 | batch_size = 32 68 | batches = 20 69 | k = 10 70 | epochs = 1 71 | model_function = globals()[model_function_name] 72 | model = model_function() 73 | x_train, y_train, x_test, y_test = mnist_data() 74 | x_train = x_train[:batches * batch_size, ...] 75 | y_train = y_train[:batches * batch_size, ...] 76 | x = x_train 77 | y = to_categorical(y_train, k) 78 | fit(model, x, y, 79 | epochs=epochs, 80 | batch_size=batch_size) 81 | 82 | 83 | if __name__ == "__main__": 84 | pytest.main([__file__]) 85 | --------------------------------------------------------------------------------