├── .gitignore ├── LICENSE ├── README.md ├── examples └── test_gpu.py ├── setup.py └── tensorbayes ├── __init__.py ├── distributions.py ├── layers ├── __init__.py ├── normalization.py ├── sample.py └── simple.py ├── nbutils.py ├── nputils.py ├── tbutils.py ├── tfutils.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Project 2 | *data 3 | .hide 4 | 5 | # emacs 6 | .tramp_history 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *,cover 53 | .hypothesis/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # IPython Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # dotenv 86 | .env 87 | 88 | # virtualenv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Rui Shu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tensorbayes: Deep learning library for deep generative models in Tensorflow 2 | 3 | Tensorbayes is a light-weight wrapper around Tensorflow that provides access to 4 | functions useful for deep generative models. To a first-order approximation, 5 | Tensorbayes is a Tensorflow analog to [Parmesan](https://github.com/casperkaae/parmesan). 6 | 7 | # Installation 8 | Tensorbayes requires the following dependencies: 9 | 1. numpy 10 | 2. tensorflow >= 0.12 11 | 12 | To install, simply run: 13 | ``` 14 | pip install tensorbayes 15 | ``` 16 | -------------------------------------------------------------------------------- /examples/test_gpu.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic wall-clock test for a generic convolutional neural network 3 | 4 | Tesla K40c: 5 | Elapsed wall-clock time: 58.0986320972 6 | Average time per iter: 0.0580986320972 7 | 8 | GeForce GTX 1080 Ti: 9 | Elapsed wall-clock time: 41.549612999 10 | Average time per iter: 0.041549612999 11 | 12 | GeForce GTX TITAN X: 13 | Elapsed wall-clock time: 61.8042290211 14 | Average time per iter: 0.0618042290211 15 | """ 16 | 17 | import numpy as np 18 | import tensorflow as tf 19 | import tensorbayes as tb 20 | from tensorflow.contrib.framework import arg_scope 21 | from tensorbayes.layers import conv2d, dense 22 | from tensorflow.python.ops.nn_ops import softmax_cross_entropy_with_logits_v2 as softmax_xent 23 | import time 24 | import os 25 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 26 | 27 | def classifier(x, phase, reuse=None): 28 | with tf.variable_scope('class', reuse=reuse): 29 | with arg_scope([conv2d, dense], bn=True, phase=phase, activation=tf.nn.relu): 30 | for i in range(4): 31 | x = conv2d(x, 64 + 64 * i, 3, 2) 32 | x = conv2d(x, 64 + 64 * i, 3, 1) 33 | 34 | x = dense(x, 500) 35 | x = dense(x, 10, activation=None) 36 | 37 | return x 38 | 39 | def build_graph(): 40 | T = tb.TensorDict(dict( 41 | sess = tf.Session(config=tb.growth_config()), 42 | x = tb.nn.placeholder((None, 32, 32, 3)), 43 | y = tb.nn.placeholder((None, 10)), 44 | )) 45 | 46 | y = classifier(T.x, phase=True) 47 | loss = tf.reduce_mean(softmax_xent(labels=T.y, logits=y)) 48 | train_main = tf.train.AdamOptimizer().minimize(loss) 49 | T.main_ops = [train_main, loss] 50 | T.sess.run(tf.global_variables_initializer()) 51 | return T 52 | 53 | def train(T): 54 | for i in range(1000): 55 | x = np.random.randn(100, 32, 32, 3) 56 | y = np.tile(np.eye(10)[0], (100, 1)) 57 | _, loss = T.sess.run(T.main_ops, feed_dict={T.x: x, T.y: y}) 58 | tb.utils.progbar(i, 1000, '{} / {:.2f}'.format(i, loss)) 59 | 60 | 61 | if __name__ == '__main__': 62 | T = build_graph() 63 | t = time.time() 64 | train(T) 65 | t = time.time() - t 66 | print('Elapsed wall-clock time: {}'.format(t)) 67 | print('Average time per iter: {}'.format(t / 1000)) 68 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup file for tensorbayes 2 | 3 | For easy installation and uninstallation, do the following. 4 | MANUAL INSTALL: 5 | python setup.py install --record files.txt 6 | UNINSTALL: 7 | cat files.txt | xargs rm -r 8 | """ 9 | 10 | from setuptools import setup, find_packages 11 | import os 12 | 13 | setup( 14 | name="tensorbayes", 15 | version="0.4.0", 16 | author="Rui Shu", 17 | author_email="ruishu@stanford.edu", 18 | url="http://www.github.com/RuiShu/tensorbayes", 19 | download_url="https://github.com/RuiShu/tensorbayes/archive/0.4.0.tar.gz", 20 | license="MIT", 21 | description="Deep Variational Inference in TensorFlow", 22 | install_requires = ['numpy'], 23 | extras_require={ 24 | 'notebook': ['jupyter'] 25 | }, 26 | packages=find_packages() 27 | ) 28 | -------------------------------------------------------------------------------- /tensorbayes/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from . import layers 3 | from . import layers as nn 4 | from . import utils 5 | from . import nputils 6 | from . import tbutils 7 | from . import tfutils 8 | from . import distributions 9 | from .utils import FileWriter 10 | from .tfutils import growth_config, function, TensorDict 11 | 12 | if 'ipykernel' in sys.argv[0]: 13 | from . import nbutils 14 | 15 | __version__ = '0.4.0' 16 | -------------------------------------------------------------------------------- /tensorbayes/distributions.py: -------------------------------------------------------------------------------- 1 | """ Assumes softplus activations for gaussian 2 | """ 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | def log_bernoulli(x, logits, eps=0.0, axis=-1): 7 | return log_bernoulli_with_logits(x, logits, eps, axis) 8 | 9 | def log_bernoulli_with_logits(x, logits, eps=0.0, axis=-1): 10 | if eps > 0.0: 11 | max_val = np.log(1.0 - eps) - np.log(eps) 12 | logits = tf.clip_by_value(logits, -max_val, max_val, 13 | name='clipped_logit') 14 | return -tf.reduce_sum( 15 | tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=x), axis) 16 | 17 | def log_normal(x, mu, var, eps=0.0, axis=-1): 18 | if eps > 0.0: 19 | var = tf.add(var, eps, name='clipped_var') 20 | return -0.5 * tf.reduce_sum( 21 | tf.log(2 * np.pi) + tf.log(var) + tf.square(x - mu) / var, axis) 22 | 23 | def kl_normal(qm, qv, pm, pv, eps=0.0, axis=-1): 24 | if eps > 0.0: 25 | qv = tf.add(qv, eps, name='clipped_var1') 26 | pv = tf.add(qv, eps, name='clipped_var2') 27 | 28 | return 0.5 * tf.reduce_sum(tf.log(pv) - tf.log(qv) + qv / pv + 29 | tf.square(qm - pm) / pv - 1, axis=-1) 30 | -------------------------------------------------------------------------------- /tensorbayes/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from .simple import * 2 | from .sample import * 3 | from .normalization import * 4 | -------------------------------------------------------------------------------- /tensorbayes/layers/normalization.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.contrib.framework import add_arg_scope, arg_scope 3 | 4 | def _assign_moving_average(orig_val, new_val, momentum, name): 5 | with tf.name_scope(name): 6 | scaled_diff = (1 - momentum) * (new_val - orig_val) 7 | return tf.assign_add(orig_val, scaled_diff) 8 | 9 | @add_arg_scope 10 | def batch_norm(x, 11 | phase, 12 | shift=True, 13 | scale=True, 14 | momentum=0.99, 15 | eps=1e-3, 16 | internal_update=False, 17 | scope=None, 18 | reuse=None): 19 | 20 | C = x._shape_as_list()[-1] 21 | ndim = len(x.shape) 22 | var_shape = [1] * (ndim - 1) + [C] 23 | 24 | with tf.variable_scope(scope, 'batch_norm', reuse=reuse): 25 | def training(): 26 | m, v = tf.nn.moments(x, list(range(ndim - 1)), keep_dims=True) 27 | update_m = _assign_moving_average(moving_m, m, momentum, 'update_mean') 28 | update_v = _assign_moving_average(moving_v, v, momentum, 'update_var') 29 | tf.add_to_collection('update_ops', update_m) 30 | tf.add_to_collection('update_ops', update_v) 31 | 32 | if internal_update: 33 | with tf.control_dependencies([update_m, update_v]): 34 | output = (x - m) * tf.rsqrt(v + eps) 35 | else: 36 | output = (x - m) * tf.rsqrt(v + eps) 37 | return output 38 | 39 | def testing(): 40 | m, v = moving_m, moving_v 41 | output = (x - m) * tf.rsqrt(v + eps) 42 | return output 43 | 44 | # Get mean and variance, normalize input 45 | moving_m = tf.get_variable('mean', var_shape, initializer=tf.zeros_initializer, trainable=False) 46 | moving_v = tf.get_variable('var', var_shape, initializer=tf.ones_initializer, trainable=False) 47 | 48 | if isinstance(phase, bool): 49 | output = training() if phase else testing() 50 | else: 51 | output = tf.cond(phase, training, testing) 52 | 53 | if scale: 54 | output *= tf.get_variable('gamma', var_shape, initializer=tf.ones_initializer) 55 | 56 | if shift: 57 | output += tf.get_variable('beta', var_shape, initializer=tf.zeros_initializer) 58 | 59 | return output 60 | 61 | @add_arg_scope 62 | def instance_norm(x, 63 | shift=True, 64 | scale=True, 65 | eps=1e-3, 66 | scope=None, 67 | reuse=None): 68 | 69 | # Expect a 4-D Tensor 70 | C = x._shape_as_list()[-1] 71 | 72 | with tf.variable_scope(scope, 'instance_norm', reuse=reuse): 73 | # Get mean and variance, normalize input 74 | m, v = tf.nn.moments(x, [1, 2], keep_dims=True) 75 | output = (x - m) * tf.rsqrt(v + eps) 76 | 77 | if scale: 78 | output *= tf.get_variable('gamma', C, initializer=tf.ones_initializer) 79 | 80 | if shift: 81 | output += tf.get_variable('beta', C, initializer=tf.zeros_initializer) 82 | 83 | return output 84 | 85 | @add_arg_scope 86 | def context_shift(x, 87 | context, 88 | shift=True, 89 | scale=True, 90 | scope=None, 91 | reuse=None): 92 | 93 | B = context._shape_as_list()[-1] 94 | C = x._shape_as_list()[-1] 95 | ndim = len(x.shape) 96 | var_shape = [B] + [1] * (ndim - 2) + [C] 97 | 98 | with tf.variable_scope(scope, 'context_shift', reuse=reuse): 99 | output = x 100 | 101 | if scale: 102 | gamma = tf.get_variable('gamma', var_shape, initializer=tf.ones_initializer) 103 | output *= tf.tensordot(context, gamma, 1) 104 | 105 | if shift: 106 | beta = tf.get_variable('beta', var_shape, initializer=tf.zeros_initializer) 107 | output += tf.tensordot(context, beta, 1) 108 | 109 | output.set_shape(x.get_shape()) 110 | 111 | return output 112 | 113 | @add_arg_scope 114 | def lookup_shift(x, 115 | context, 116 | shift=True, 117 | scale=True, 118 | scope=None, 119 | reuse=None): 120 | 121 | B = context._shape_as_list()[-1] 122 | C = x._shape_as_list()[-1] 123 | ndim = len(x.shape) 124 | var_shape = [B] + [1] * (ndim - 2) + [C] 125 | 126 | with tf.variable_scope(scope, 'lookup_shift', reuse=reuse): 127 | output = x 128 | ids = tf.argmax(context, -1) 129 | 130 | if scale: 131 | gamma = tf.get_variable('gamma', var_shape, initializer=tf.ones_initializer) 132 | output *= tf.nn.embedding_lookup(gamma, ids) 133 | 134 | if shift: 135 | beta = tf.get_variable('beta', var_shape, initializer=tf.zeros_initializer) 136 | output += tf.nn.embedding_lookup(beta, ids) 137 | 138 | return output 139 | -------------------------------------------------------------------------------- /tensorbayes/layers/sample.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | def gaussian_sample(mean, var, scope=None): 4 | with tf.variable_scope(scope, 'gaussian_sample'): 5 | sample = tf.random_normal(tf.shape(mean), mean, tf.sqrt(var)) 6 | sample.set_shape(mean.get_shape()) 7 | return sample 8 | 9 | def bernoulli_sample(mean, scope=None): 10 | with tf.variable_scope(scope, 'bernoulli_sample'): 11 | sample = tf.cast( 12 | tf.greater(mean, tf.random_uniform(tf.shape(mean), 0, 1)), 13 | tf.float32) 14 | sample.set_shape(mean.get_shape()) 15 | return sample 16 | 17 | def duplicate(x, n_iw=1, n_mc=1, scope=None): 18 | """ Duplication function adds samples according to n_iw and n_mc. 19 | 20 | This function is specifically for importance weighting and monte carlo 21 | sampling. 22 | """ 23 | with tf.variable_scope(scope, 'duplicate'): 24 | sample_shape = x._shape_as_list()[1:] 25 | y = tf.reshape(x, [1, 1, -1] + sample_shape) 26 | multiplier = tf.stack([n_iw, n_mc, 1] + [1] * len(sample_shape)) 27 | y = tf.tile(y, multiplier) 28 | y = tf.reshape(y, [-1] + sample_shape) 29 | return y 30 | -------------------------------------------------------------------------------- /tensorbayes/layers/simple.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.contrib.framework import add_arg_scope 3 | from .normalization import * 4 | from tensorflow.contrib.layers import variance_scaling_initializer 5 | import numpy as np 6 | 7 | def constant(value, dtype='float32', name=None): 8 | return tf.constant(value, dtype, name=name) 9 | 10 | def placeholder(shape, dtype='float32', name=None): 11 | return tf.placeholder(dtype, shape, name=name) 12 | 13 | @add_arg_scope 14 | def dense(x, 15 | num_outputs, 16 | scope=None, 17 | activation=None, 18 | reuse=None, 19 | bn=False, 20 | post_bn=False, 21 | phase=None): 22 | 23 | with tf.variable_scope(scope, 'dense', reuse=reuse): 24 | # convert x to 2-D tensor 25 | dim = np.prod(x._shape_as_list()[1:]) 26 | x = tf.reshape(x, [-1, dim]) 27 | weights_shape = (x.get_shape().dims[-1], num_outputs) 28 | 29 | # dense layer 30 | weights = tf.get_variable('weights', weights_shape, 31 | initializer=variance_scaling_initializer()) 32 | biases = tf.get_variable('biases', [num_outputs], 33 | initializer=tf.zeros_initializer) 34 | output = tf.matmul(x, weights) + biases 35 | if bn: output = batch_norm(output, phase, scope='bn') 36 | if activation: output = activation(output) 37 | if post_bn: output = batch_norm(output, phase, scope='post_bn') 38 | 39 | return output 40 | 41 | @add_arg_scope 42 | def conv2d(x, 43 | num_outputs, 44 | kernel_size, 45 | strides, 46 | padding='SAME', 47 | activation=None, 48 | bn=False, 49 | post_bn=False, 50 | phase=None, 51 | scope=None, 52 | reuse=None): 53 | # Convert int to list 54 | kernel_size = [kernel_size] * 2 if isinstance(kernel_size, int) else kernel_size 55 | strides = [strides] * 2 if isinstance(strides, int) else strides 56 | 57 | # Convert list to valid list 58 | kernel_size = list(kernel_size) + [x.get_shape().dims[-1], num_outputs] 59 | strides = [1] + list(strides) + [1] 60 | 61 | # Conv operation 62 | with tf.variable_scope(scope, 'conv2d', reuse=reuse): 63 | kernel = tf.get_variable('weights', kernel_size, 64 | initializer=variance_scaling_initializer()) 65 | biases = tf.get_variable('biases', [num_outputs], 66 | initializer=tf.zeros_initializer) 67 | output = tf.nn.conv2d(x, kernel, strides, padding, name='conv2d') 68 | output += biases 69 | if bn: output = batch_norm(output, phase, scope='bn') 70 | if activation: output = activation(output) 71 | if post_bn: output = batch_norm(output, phase, scope='post_bn') 72 | 73 | return output 74 | 75 | @add_arg_scope 76 | def conv2d_transpose(x, 77 | num_outputs, 78 | kernel_size, 79 | strides, 80 | padding='SAME', 81 | output_shape=None, 82 | output_like=None, 83 | activation=None, 84 | bn=False, 85 | post_bn=False, 86 | phase=None, 87 | scope=None, 88 | reuse=None): 89 | # Convert int to list 90 | kernel_size = [kernel_size] * 2 if isinstance(kernel_size, int) else kernel_size 91 | strides = [strides] * 2 if isinstance(strides, int) else strides 92 | 93 | # Convert list to valid list 94 | kernel_size = list(kernel_size) + [num_outputs, x.get_shape().dims[-1]] 95 | strides = [1] + list(strides) + [1] 96 | 97 | # Get output shape both as tensor obj and as list 98 | if output_shape: 99 | bs = tf.shape(x)[0] 100 | _output_shape = tf.stack([bs] + output_shape[1:]) 101 | elif output_like: 102 | _output_shape = tf.shape(output_like) 103 | output_shape = output_like.get_shape() 104 | else: 105 | assert padding == 'SAME', "Shape inference only applicable with padding is SAME" 106 | bs, h, w, c = x._shape_as_list() 107 | bs_tf = tf.shape(x)[0] 108 | _output_shape = tf.stack([bs_tf, strides[1] * h, strides[2] * w, num_outputs]) 109 | output_shape = [bs, strides[1] * h, strides[2] * w, num_outputs] 110 | 111 | # Transposed conv operation 112 | with tf.variable_scope(scope, 'conv2d', reuse=reuse): 113 | kernel = tf.get_variable('weights', kernel_size, 114 | initializer=variance_scaling_initializer()) 115 | biases = tf.get_variable('biases', [num_outputs], 116 | initializer=tf.zeros_initializer) 117 | output = tf.nn.conv2d_transpose(x, kernel, _output_shape, strides, 118 | padding, name='conv2d_transpose') 119 | output += biases 120 | output.set_shape(output_shape) 121 | if bn: output = batch_norm(output, phase, scope='bn') 122 | if activation: output = activation(output) 123 | if post_bn: output = batch_norm(output, phase, scope='post_bn') 124 | 125 | return output 126 | 127 | @add_arg_scope 128 | def upsample(x, 129 | strides, 130 | scope=None): 131 | # Convert int to list 132 | strides = [strides] * 2 if isinstance(strides, int) else strides 133 | shape = x._shape_as_list()[1:3] 134 | h, w = strides[0] * shape[0], strides[1] * shape[1] 135 | 136 | with tf.variable_scope(scope, 'upsample'): 137 | output = tf.image.resize_nearest_neighbor(x, [h, w]) 138 | 139 | return output 140 | 141 | @add_arg_scope 142 | def max_pool(x, 143 | kernel_size, 144 | strides, 145 | padding='SAME', 146 | scope=None): 147 | # Convert int to list 148 | kernel_size = [kernel_size] * 2 if isinstance(kernel_size, int) else kernel_size 149 | strides = [strides] * 2 if isinstance(strides, int) else strides 150 | 151 | # Convert list to valid list 152 | kernel_size = [1] + list(kernel_size) + [1] 153 | strides = [1] + list(strides) + [1] 154 | 155 | with tf.variable_scope(scope, 'max_pool'): 156 | output = tf.nn.max_pool(x, kernel_size, strides, padding=padding) 157 | 158 | return output 159 | 160 | @add_arg_scope 161 | def avg_pool(x, 162 | kernel_size=None, 163 | strides=None, 164 | padding='SAME', 165 | global_pool=False, 166 | scope=None): 167 | 168 | if global_pool: 169 | return tf.reduce_mean(x, axis=[1, 2]) 170 | 171 | # Convert int to list 172 | kernel_size = [kernel_size] * 2 if isinstance(kernel_size, int) else kernel_size 173 | strides = [strides] * 2 if isinstance(strides, int) else strides 174 | 175 | # Convert list to valid list 176 | kernel_size = [1] + list(kernel_size) + [1] 177 | strides = [1] + list(strides) + [1] 178 | 179 | with tf.variable_scope(scope, 'avg_pool'): 180 | output = tf.nn.avg_pool(x, kernel_size, strides, padding=padding) 181 | 182 | return output 183 | 184 | @add_arg_scope 185 | def gaussian_update(zm1, zv1, 186 | zm2, zv2, 187 | scope=None, 188 | eps=0.0): 189 | with tf.variable_scope(scope, 'gaussian_update'): 190 | with tf.name_scope('variance'): 191 | if eps > 0.0: 192 | """It is not clear to me yet whether this will cause our loss 193 | function to be severely biased 194 | """ 195 | raise Exception("Adding eps noise deprecated at the moment " 196 | "for gaussian update fn") 197 | zv1 = tf.add(zv1, eps, name='clip_var1') 198 | zv2 = tf.add(zv2, eps, name='clip_var2') 199 | zp1 = 1.0/zv1 200 | zp2 = 1.0/zv2 201 | zv = 1.0/(zp1 + zp2) 202 | with tf.name_scope('mean'): 203 | zm = (zm1 * zp1 + zm2 * zp2) * zv 204 | return zm, zv 205 | 206 | @add_arg_scope 207 | def concat_img_vec(img, 208 | vec, 209 | scope=None): 210 | H, W = img._shape_as_list()[1:-1] 211 | V = vec._shape_as_list()[-1] 212 | 213 | with tf.variable_scope(scope, 'concat_img_vec'): 214 | # Depth-wise concatenation of vec to img 215 | # Replicate vec via broadcasting 216 | broadcast = tf.ones([1, H, W, 1]) 217 | vec = tf.reshape(vec, [-1, 1, 1, V]) 218 | vec = broadcast * vec 219 | output = tf.concat([img, vec], axis=-1) 220 | 221 | return output 222 | -------------------------------------------------------------------------------- /tensorbayes/nbutils.py: -------------------------------------------------------------------------------- 1 | """ Code taken from 2 | https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/deepdream/deepdream.ipynb 3 | Placed here for convenience only. 4 | """ 5 | from IPython.display import clear_output, Image, display, HTML 6 | import tensorflow as tf 7 | import numpy as np 8 | 9 | def strip_consts(graph_def, max_const_size=32): 10 | """Strip large constant values from graph_def.""" 11 | strip_def = tf.GraphDef() 12 | for n0 in graph_def.node: 13 | n = strip_def.node.add() 14 | n.MergeFrom(n0) 15 | if n.op == 'Const': 16 | tensor = n.attr['value'].tensor 17 | size = len(tensor.tensor_content) 18 | if size > max_const_size: 19 | tensor.tensor_content = ""%size 20 | return strip_def 21 | 22 | def show_graph(graph_def, max_const_size=32): 23 | """Visualize TensorFlow graph.""" 24 | if hasattr(graph_def, 'as_graph_def'): 25 | graph_def = graph_def.as_graph_def() 26 | strip_def = strip_consts(graph_def, max_const_size=max_const_size) 27 | code = """ 28 | 33 | 34 |
35 | 36 |
37 | """.format(data=repr(str(strip_def)), id='graph'+str(np.random.rand())) 38 | 39 | iframe = """ 40 | 41 | """.format(code.replace('"', '"')) 42 | display(HTML(iframe)) 43 | 44 | def show_default_graph(): 45 | show_graph(tf.get_default_graph().as_graph_def()) 46 | -------------------------------------------------------------------------------- /tensorbayes/nputils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def log_sum_exp(x, axis=-1): 4 | a = x.max(axis=axis, keepdims=True) 5 | out = a + np.log(np.sum(np.exp(x - a), axis=axis, keepdims=True)) 6 | return np.squeeze(out, axis=axis) 7 | 8 | def kl_normal(qm, qv, pm, pv): 9 | return 0.5 * np.sum(np.log(pv) - np.log(qv) + qv/pv + 10 | np.square(qm - pm) / pv - 1, axis=-1) 11 | 12 | def convert_to_ssl(x, y, n_labels, n_classes, complement=False): 13 | if y.shape[-1] == n_classes: 14 | y_sparse = y.argmax(1) 15 | else: 16 | y_sparse = y 17 | x_label, y_label = [], [] 18 | if complement: 19 | x_comp, y_comp = [], [] 20 | for i in xrange(n_classes): 21 | idx = y_sparse == i 22 | x_cand, y_cand = x[idx], y[idx] 23 | idx = np.random.choice(len(x_cand), n_labels/n_classes, replace=False) 24 | x_select, y_select = x_cand[idx], y_cand[idx] 25 | x_label += [x_select] 26 | y_label += [y_select] 27 | if complement: 28 | x_select, y_select = np.delete(x_cand, idx, 0), np.delete(y_cand, idx, 0) 29 | x_comp += [x_select] 30 | y_comp += [y_select] 31 | x_label = np.concatenate(x_label, axis=0) 32 | y_label = np.concatenate(y_label, axis=0) 33 | if complement: 34 | x_comp = np.concatenate(x_comp, axis=0) 35 | y_comp = np.concatenate(y_comp, axis=0) 36 | return x_label, y_label, x_comp, y_comp 37 | else: 38 | return x_label, y_label, x, y 39 | 40 | def conv_shape(x, k, s, p, ceil=True): 41 | if p == 'SAME': 42 | output = float(x) / float(s) 43 | elif p == 'VALID': 44 | output = float(x - k + 1) / float(s) 45 | else: 46 | raise Exception('Unknown padding type') 47 | if ceil: 48 | return int(np.ceil(output)) 49 | else: 50 | assert output.is_integer(), 'Does not satisfy conv int requirement' 51 | return int(output) 52 | 53 | def conv_shape_list(x, ksp_list, ceil=True): 54 | x_list = [x] 55 | for k, s, p in ksp_list: 56 | x_list.append(conv_shape(x_list[-1], k, s, p, ceil)) 57 | return x_list 58 | 59 | def split(arr, size): 60 | for i in range(0, len(arr), size): 61 | yield arr[i:i + size] 62 | 63 | class FixedSeed: 64 | def __init__(self, seed): 65 | self.seed = seed 66 | self.state = None 67 | 68 | def __enter__(self): 69 | self.state = np.random.get_state() 70 | np.random.seed(self.seed) 71 | 72 | def __exit__(self, exc_type, exc_value, traceback): 73 | np.random.set_state(self.state) 74 | -------------------------------------------------------------------------------- /tensorbayes/tbutils.py: -------------------------------------------------------------------------------- 1 | """Deprecated. Used for backwards-compatibility. Code now in tfutils.py""" 2 | from .tfutils import * 3 | -------------------------------------------------------------------------------- /tensorbayes/tfutils.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python.ops.nn_ops import softmax_cross_entropy_with_logits_v2 as softmax_xent 3 | 4 | def accuracy(x, y, scope=None): 5 | with tf.variable_scope(scope, 'acc') as sc: 6 | x = tf.argmax(x, 1) 7 | y = tf.argmax(y, 1) 8 | _, acc = tf.metrics.accuracy(x, y) 9 | acc_init = tf.variables_initializer(tf.get_collection('local_variables', sc.name)) 10 | return acc, acc_init 11 | 12 | def reduce_sum_sq(x, axis=None, keepdims=False, name=None): 13 | with tf.name_scope(name): 14 | return tf.reduce_sum(tf.square(x), axis=axis, keepdims=keepdims) 15 | 16 | def reduce_l2_loss(x, axis=None, keepdims=False, name=None): 17 | with tf.name_scope(name): 18 | return reduce_sum_sq(x, axis=axis, keepdims=keepdims) / 2 19 | 20 | def log_sum_exp(x, axis=1, keepdims=False): 21 | a = tf.reduce_max(x, axis, keepdims=True) 22 | out = a + tf.log(tf.reduce_sum(tf.exp(x - a), axis, keepdims=True)) 23 | if keepdims: 24 | return out 25 | else: 26 | if type(axis) is list: 27 | return tf.squeeze(out, axis) 28 | else: 29 | return tf.squeeze(out, [axis]) 30 | 31 | def softmax_cross_entropy_with_two_logits(logits=None, labels=None): 32 | return softmax_xent(labels=tf.nn.softmax(labels), logits=logits) 33 | 34 | def clip_gradients(optimizer, loss, max_clip=0.9, max_norm=4): 35 | grads_and_vars = optimizer.compute_gradients(loss) 36 | # Filter for non-None grads 37 | grads_and_vars = [gv for gv in grads_and_vars if gv[0] is not None] 38 | grads = [g for g, _ in grads_and_vars] 39 | grads, global_grad_norm = tf.clip_by_global_norm(grads, max_norm) 40 | clipped_grads_and_vars = [] 41 | for i in xrange(len(grads_and_vars)): 42 | g = tf.clip_by_value(grads[i], -max_clip, max_clip) 43 | v = grads_and_vars[i][1] 44 | clipped_grads_and_vars += [(g, v)] 45 | return clipped_grads_and_vars, global_grad_norm 46 | 47 | class Function(object): 48 | def __init__(self, sess, inputs, outputs): 49 | self.inputs = inputs 50 | self.outputs = outputs 51 | self.sess = sess 52 | 53 | def __call__(self, *args): 54 | feeds = {} 55 | for (i, arg) in enumerate(args): 56 | feeds[self.inputs[i]] = arg 57 | return self.sess.run(self.outputs, feeds) 58 | 59 | def function(sess, inputs, outputs): 60 | return Function(sess, inputs, outputs) 61 | 62 | class TensorDict(object): 63 | def __init__(self, d={}): 64 | self.__dict__ = dict(d) 65 | 66 | def __iter__(self): 67 | return iter(self.__dict__) 68 | 69 | def __contains__(self, key): 70 | return key in self.__dict__ 71 | 72 | def __getitem__(self, key): 73 | return self.__dict__[key] 74 | 75 | def __repr__(self): 76 | return repr(self.__dict__) 77 | 78 | def growth_config(*args, **kwargs): 79 | config = tf.ConfigProto(*args, **kwargs) 80 | config.gpu_options.allow_growth = True 81 | return config 82 | 83 | def get_getter(ema): 84 | def ema_getter(getter, name, *args, **kwargs): 85 | var = getter(name, *args, **kwargs) 86 | ema_var = ema.average(var) 87 | return ema_var if ema_var else var 88 | return ema_getter 89 | 90 | def make_value_summary(tag, val): 91 | summary = tf.Summary.Value(tag=tag, simple_value=val) 92 | return tf.Summary(value=[summary]) 93 | -------------------------------------------------------------------------------- /tensorbayes/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | 5 | def progbar(i, iter_per_epoch, message='', bar_length=50, display=True): 6 | j = (i % iter_per_epoch) + 1 7 | end_epoch = j == iter_per_epoch 8 | if display: 9 | perc = int(100. * j / iter_per_epoch) 10 | prog = ''.join(['='] * (bar_length * perc // 100)) 11 | template = "\r[{:" + str(bar_length) + "s}] {:3d}%. {:s}" 12 | string = template.format(prog, perc, message) 13 | sys.stdout.write(string) 14 | sys.stdout.flush() 15 | if end_epoch: 16 | sys.stdout.write('\r{:100s}\r'.format('')) 17 | sys.stdout.flush() 18 | return end_epoch, (i + 1) // iter_per_epoch 19 | 20 | class FileWriter(object): 21 | def __init__(self, log_file, args=None, 22 | overwrite=False, pipe_to_sys=True): 23 | self.written = False 24 | self.log_file = log_file 25 | self.pipe = pipe_to_sys 26 | self.args = args 27 | # non-tensorflow values to be stored 28 | self.names = [] 29 | self.formats = [] 30 | # tf tensor values 31 | self.tensor_names = [] 32 | self.tensors = [] 33 | self.tensor_formats = [] 34 | # check file existence, then create file 35 | if os.path.exists(self.log_file) and not overwrite: 36 | raise Exception("Overwriting existing log directory is " 37 | "not allowed unless overwrite=True") 38 | log_dir = os.path.dirname(self.log_file) 39 | if not os.path.exists(log_dir): 40 | os.makedirs(log_dir) 41 | self.f = open(self.log_file, 'w', 0) 42 | if args is not None: 43 | # create file 44 | # write args 45 | v_dict = vars(self.args) 46 | string = '# ArgParse Values:' 47 | self._write(string) 48 | for k in v_dict: 49 | string = '# {:s}: {:s}'.format(str(k), str(v_dict[k])) 50 | self._write(string) 51 | 52 | @staticmethod 53 | def list_args(args): 54 | v_dict = vars(args) 55 | print('# ArgParse Values:') 56 | for k in v_dict: 57 | print('# {:s}: {:s}'.format(str(k), str(v_dict[k]))) 58 | 59 | def initialize(self): 60 | # create header name 61 | self.header = ','.join(self.tensor_names + self.names) 62 | self._write(self.header) 63 | 64 | def add_var(self, name, var_format, tensor=None): 65 | if tensor is None: 66 | self.names += [name] 67 | self.formats += [var_format] 68 | else: 69 | self.tensor_names += [name] 70 | self.tensors += [tensor] 71 | self.tensor_formats += [var_format] 72 | 73 | def write(self, tensor_values=[], values=[]): 74 | values = tensor_values + values 75 | string = ','.join(self.tensor_formats + self.formats).format(*values) 76 | self._write(string, is_summary=True, pipe=True) 77 | 78 | def _write(self, string, is_summary=False, pipe=False): 79 | self.f.write(string + '\n') 80 | self.f.flush() 81 | if self.pipe and pipe: 82 | if is_summary: 83 | print(self.header) 84 | print(string) 85 | 86 | # Retaining for backwards compatibility 87 | from .tfutils import TensorDict 88 | --------------------------------------------------------------------------------