├── LICENSE ├── README.md ├── deploy.sh ├── examples ├── mnist.ipynb ├── mnist.py ├── rnn.py ├── shared │ ├── __init__.py │ ├── cifar_utils.py │ ├── convert_py_to_ipynb.py │ ├── data_loader.py │ ├── gan.py │ ├── ops.py │ ├── resize_image_patch.py │ ├── util.py │ └── variational_autoencoder.py ├── track.py └── vae-mnist.py ├── hyperchamber ├── __init__.py ├── config.py ├── io │ └── __init__.py └── selector.py ├── setup.cfg ├── setup.py └── tests ├── acceptance_io.py ├── test_hyperchamber.py └── test_permute.py /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 255BITS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hyperchamber 2 | Random search your hyper parameters. 3 | 4 | # Changelog 5 | 6 | ## 0.2.x 7 | 8 | * feature: access Config variables with '.' notation 9 | 10 | ## 0.2 11 | 12 | * local save/load 13 | 14 | ## 0.1 15 | 16 | * initial pip release 17 | 18 | You set a list of options that define your hyperparams: 19 | ```python 20 | import hyperchamber as hc 21 | 22 | hc.set('learning_rate', [0.1, 0.2, 0.5]) 23 | config = hc.random_config() # => { 'learning_rate' : 0.2 } 24 | ``` 25 | 26 | ## Examples 27 | 28 | * logistic regression classifier on MNIST [code](examples/track.py) 29 | 30 | Based on a simple tensorflow example. We find the best learning rate from a small set of options. 31 | 32 | * Finding a better network architecture for MNIST [code](examples/mnist.py) 33 | 34 | Uses hyperparameter tuning to find the best performing MNIST fully connected deep network configuration. 35 | 36 | Our search space of options here is now 720 options. Note we only have 2 variables. This search space expands exponentially with new options to search. 37 | 38 | 39 | # Installation 40 | 41 | ## Developer mode 42 | 43 | ``` 44 | python setup.py develop 45 | ``` 46 | 47 | # API 48 | 49 | ```python 50 | import hyperchamber as hc 51 | ``` 52 | ```python 53 | hc.set(name, values) 54 | ``` 55 | 56 | Sets a hyperparameter to values. 57 | 58 | * If values is an array, config[name] will be set to one element in that array. 59 | * If values is a scalar, config[name] will always be set to that scalar 60 | 61 | ```python 62 | hc.configs(n) 63 | ``` 64 | Returns up to n configs of the form {name:value} for each hyperparameter. 65 | 66 | 67 | ```python 68 | hc.save(config, filename) 69 | ``` 70 | Saves the config to a file. 71 | 72 | ```python 73 | hc.load(filename) 74 | ``` 75 | Load a configuration from file 76 | 77 | ```python 78 | hc.load_or_create_config(filename, config) 79 | ``` 80 | Load a configuration from file if that file exists. Otherwise save `config` to that file. `config` is assumed to be a Dictionary. 81 | 82 | 83 | 84 | ```python 85 | hc.record(filename, config) 86 | ``` 87 | Store the cost of a config's training results. 88 | 89 | 90 | ```python 91 | hc.top(sort_by) 92 | ``` 93 | 94 | Return the top results across all recorded results 95 | 96 | Example: 97 | 98 | ```python 99 | def by_cost(x): 100 | config, result =x 101 | return result['cost'] 102 | for config, result in hc.top(by_cost): 103 | print(config, result) 104 | ``` 105 | 106 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python3 setup.py sdist upload 3 | -------------------------------------------------------------------------------- /examples/mnist.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import hyperchamber as hc\n", 12 | "from shared.ops import *\n", 13 | "\n", 14 | "import os\n", 15 | "import time\n", 16 | "import numpy as np\n", 17 | "import tensorflow as tf\n", 18 | "\n", 19 | "from tensorflow.python.framework import ops\n", 20 | "from tensorflow.examples.tutorials.mnist import input_data\n", 21 | "\n", 22 | "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n", 23 | "\n", 24 | "learning_rates = [1, 0.75, 0.5, 0.25]\n", 25 | "hc.set(\"learning_rate\", learning_rates)\n", 26 | "hidden_layers = [ [], [26], [128], [16, 32] ]\n", 27 | "hc.permute.set(\"hidden_layer\", hidden_layers)\n", 28 | "\n", 29 | "hc.set(\"batch_size\", 128)\n", 30 | "\n", 31 | "X_DIMS=[28,28]\n", 32 | "Y_DIMS=10\n", 33 | "\n", 34 | "#def validate(value):\n", 35 | "# return value != value #NaN\n", 36 | "\n", 37 | "#hc.evolve.evolve(\"learn_rate\", 0.2, validate)\n", 38 | "\n", 39 | "def hidden_layers(config, x):\n", 40 | " output = tf.reshape(x, [config[\"batch_size\"], X_DIMS[0]*X_DIMS[1]])\n", 41 | " for i, layer in enumerate(config['hidden_layer']):\n", 42 | " output = linear(output, layer, scope=\"l\"+str(i))\n", 43 | " output = tf.nn.tanh(output)\n", 44 | " return output\n", 45 | "\n", 46 | "def output_layer(config, x):\n", 47 | " return linear(x, Y_DIMS)\n", 48 | "\n", 49 | "def create(config):\n", 50 | " batch_size = config[\"batch_size\"]\n", 51 | " x = tf.placeholder(tf.float32, [batch_size, X_DIMS[0], X_DIMS[1], 1], name=\"x\")\n", 52 | " y = tf.placeholder(tf.float32, [batch_size, Y_DIMS], name=\"y\")\n", 53 | "\n", 54 | " hidden = hidden_layers(config, x)\n", 55 | " output = output_layer(config, hidden)\n", 56 | "\n", 57 | " loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(output, y), name=\"loss\")\n", 58 | "\n", 59 | " output = tf.nn.softmax(output)\n", 60 | " correct_prediction = tf.equal(tf.argmax(output,1), tf.argmax(y,1))\n", 61 | " accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", 62 | "\n", 63 | " variables = tf.trainable_variables()\n", 64 | "\n", 65 | " optimizer = tf.train.GradientDescentOptimizer(config['learning_rate']).minimize(loss)\n", 66 | "\n", 67 | "\n", 68 | " set_tensor(\"x\", x)\n", 69 | " set_tensor(\"y\", y)\n", 70 | " set_tensor(\"loss\", loss)\n", 71 | " set_tensor(\"optimizer\", optimizer)\n", 72 | " set_tensor(\"accuracy\", accuracy)\n", 73 | " \n", 74 | "def train(sess, config, x_input, y_labels):\n", 75 | " x = get_tensor(\"x\")\n", 76 | " y = get_tensor(\"y\")\n", 77 | " cost = get_tensor(\"loss\")\n", 78 | " optimizer = get_tensor(\"optimizer\")\n", 79 | " accuracy = get_tensor(\"accuracy\")\n", 80 | "\n", 81 | " _, accuracy, cost = sess.run([optimizer, accuracy, cost], feed_dict={x:x_input, y:y_labels})\n", 82 | "\n", 83 | "\n", 84 | " #hc.cost(config, cost)\n", 85 | " #print(\"Accuracy %.2f Cost %.2f\" % (accuracy, cost))\n", 86 | "\n", 87 | "def test(sess, config, x_input, y_labels):\n", 88 | " x = get_tensor(\"x\")\n", 89 | " y = get_tensor(\"y\")\n", 90 | " cost = get_tensor(\"loss\")\n", 91 | " accuracy = get_tensor(\"accuracy\")\n", 92 | "\n", 93 | " accuracy, cost = sess.run([accuracy, cost], feed_dict={x:x_input, y:y_labels})\n", 94 | "\n", 95 | "\n", 96 | " print(\"Accuracy %.2f Cost %.2f\" % (accuracy, cost))\n", 97 | " return accuracy, cost\n", 98 | "\n", 99 | "\n", 100 | "def epoch(sess, config):\n", 101 | " batch_size = config[\"batch_size\"]\n", 102 | " n_samples = mnist.train.num_examples\n", 103 | " total_batch = int(n_samples / batch_size)\n", 104 | " for i in range(total_batch):\n", 105 | " x, y = mnist.train.next_batch(batch_size)\n", 106 | " x=np.reshape(x, [batch_size, X_DIMS[0], X_DIMS[1], 1])\n", 107 | " train(sess, config, x, y)\n", 108 | "\n", 109 | "def test_config(sess, config):\n", 110 | " batch_size = config[\"batch_size\"]\n", 111 | " n_samples = mnist.test.num_examples\n", 112 | " total_batch = int(n_samples / batch_size)\n", 113 | " accuracies = []\n", 114 | " costs = []\n", 115 | " for i in range(total_batch):\n", 116 | " x, y = mnist.test.next_batch(batch_size)\n", 117 | " x=np.reshape(x, [batch_size, X_DIMS[0], X_DIMS[1], 1])\n", 118 | " accuracy, cost = test(sess, config, x, y)\n", 119 | " accuracies.append(accuracy)\n", 120 | " costs.append(cost)\n", 121 | " return accuracies, costs\n", 122 | "\n", 123 | "\n", 124 | "for config in hc.configs(100):\n", 125 | " print(\"Testing configuration\", config)\n", 126 | " sess = tf.Session()\n", 127 | " graph = create(config)\n", 128 | " init = tf.initialize_all_variables()\n", 129 | " sess.run(init)\n", 130 | " for i in range(10):\n", 131 | " epoch(sess, config)\n", 132 | " accuracies, costs = test_config(sess, config)\n", 133 | " accuracy, cost = np.mean(accuracies), np.mean(costs)\n", 134 | " results = {\n", 135 | " 'accuracy':accuracy,\n", 136 | " 'cost':cost\n", 137 | " }\n", 138 | " hc.record(config, results)\n", 139 | " ops.reset_default_graph()\n", 140 | " sess.close()\n", 141 | "\n", 142 | "\n", 143 | "def by_accuracy(x):\n", 144 | " config,result = x\n", 145 | " return 1-result['accuracy']\n", 146 | "\n", 147 | "for config, result in hc.top(by_accuracy):\n", 148 | " print(\"RESULTS\")\n", 149 | " print(config, result)\n", 150 | " \n", 151 | "\n", 152 | "\n", 153 | " #print(\"Done testing. Final cost was:\", hc.cost())\n", 154 | "\n", 155 | "print(\"Done\")\n", 156 | "\n", 157 | "#for gold, silver, bronze in hc.top_configs(3):" 158 | ] 159 | } 160 | ], 161 | "metadata": {}, 162 | "nbformat": 4, 163 | "nbformat_minor": 0 164 | } 165 | -------------------------------------------------------------------------------- /examples/mnist.py: -------------------------------------------------------------------------------- 1 | import hyperchamber as hc 2 | from shared.ops import * 3 | from shared.util import * 4 | 5 | import os 6 | import time 7 | import numpy as np 8 | import tensorflow as tf 9 | 10 | from tensorflow.python.framework import ops 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 14 | 15 | learning_rates = list(np.linspace(0.0001, 1, num=30)) 16 | hc.set("learning_rate", learning_rates) 17 | hidden_layers = [ [], [26], [128], [16, 32], [32,16,8], [16,8,8,4], [64,64]] 18 | hc.set("hidden_layer", hidden_layers) 19 | hc.set("activation", [tf.nn.elu, tf.nn.relu, tf.nn.relu6, tf.tanh, tf.sigmoid, lrelu]); 20 | 21 | hc.set("batch_size", 128) 22 | 23 | X_DIMS=[28,28] 24 | Y_DIMS=10 25 | 26 | def hidden_layers(config, x): 27 | output = tf.reshape(x, [config["batch_size"], X_DIMS[0]*X_DIMS[1]]) 28 | for i, layer in enumerate(config['hidden_layer']): 29 | output = linear(output, layer, scope="l"+str(i)) 30 | output = config['activation'](output) 31 | return output 32 | 33 | def output_layer(config, x): 34 | return linear(x, Y_DIMS) 35 | 36 | def create(config): 37 | batch_size = config["batch_size"] 38 | x = tf.placeholder(tf.float32, [batch_size, X_DIMS[0], X_DIMS[1], 1], name="x") 39 | y = tf.placeholder(tf.float32, [batch_size, Y_DIMS], name="y") 40 | 41 | hidden = hidden_layers(config, x) 42 | output = output_layer(config, hidden) 43 | 44 | loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(output, y), name="loss") 45 | 46 | output = tf.nn.softmax(output) 47 | correct_prediction = tf.equal(tf.argmax(output,1), tf.argmax(y,1)) 48 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 49 | 50 | variables = tf.trainable_variables() 51 | 52 | optimizer = tf.train.GradientDescentOptimizer(config['learning_rate']).minimize(loss) 53 | 54 | 55 | set_tensor("x", x) 56 | set_tensor("y", y) 57 | set_tensor("loss", loss) 58 | set_tensor("optimizer", optimizer) 59 | set_tensor("accuracy", accuracy) 60 | 61 | def train(sess, config, x_input, y_labels): 62 | x = get_tensor("x") 63 | y = get_tensor("y") 64 | cost = get_tensor("loss") 65 | optimizer = get_tensor("optimizer") 66 | accuracy = get_tensor("accuracy") 67 | 68 | _, accuracy, cost = sess.run([optimizer, accuracy, cost], feed_dict={x:x_input, y:y_labels}) 69 | 70 | 71 | #hc.cost(config, cost) 72 | #print("Accuracy %.2f Cost %.2f" % (accuracy, cost)) 73 | 74 | def test(sess, config, x_input, y_labels): 75 | x = get_tensor("x") 76 | y = get_tensor("y") 77 | cost = get_tensor("loss") 78 | accuracy = get_tensor("accuracy") 79 | 80 | accuracy, cost = sess.run([accuracy, cost], feed_dict={x:x_input, y:y_labels}) 81 | 82 | 83 | print("Accuracy %.2f Cost %.2f" % (accuracy, cost)) 84 | return accuracy, cost 85 | 86 | 87 | def epoch(sess, config): 88 | batch_size = config["batch_size"] 89 | n_samples = mnist.train.num_examples 90 | total_batch = int(n_samples / batch_size) 91 | for i in range(total_batch): 92 | x, y = mnist.train.next_batch(batch_size) 93 | x=np.reshape(x, [batch_size, X_DIMS[0], X_DIMS[1], 1]) 94 | train(sess, config, x, y) 95 | 96 | def test_config(sess, config): 97 | batch_size = config["batch_size"] 98 | n_samples = mnist.test.num_examples 99 | total_batch = int(n_samples / batch_size) 100 | accuracies = [] 101 | costs = [] 102 | for i in range(total_batch): 103 | x, y = mnist.test.next_batch(batch_size) 104 | x=np.reshape(x, [batch_size, X_DIMS[0], X_DIMS[1], 1]) 105 | accuracy, cost = test(sess, config, x, y) 106 | accuracies.append(accuracy) 107 | costs.append(cost) 108 | return accuracies, costs 109 | 110 | 111 | print("Searching randomly with %d possible configurations." % hc.count_configs()) 112 | for i in range(100): 113 | config = hc.random_config() 114 | print("Testing configuration", config) 115 | sess = tf.Session() 116 | graph = create(config) 117 | init = tf.initialize_all_variables() 118 | sess.run(init) 119 | for i in range(10): 120 | epoch(sess, config) 121 | accuracies, costs = test_config(sess, config) 122 | accuracy, cost = np.mean(accuracies), np.mean(costs) 123 | results = { 124 | 'accuracy':accuracy, 125 | 'cost':cost 126 | } 127 | hc.record(config, results) 128 | ops.reset_default_graph() 129 | sess.close() 130 | 131 | 132 | def by_accuracy(x): 133 | config,result = x 134 | return 1-result['accuracy'] 135 | 136 | for config, result in hc.top(by_accuracy): 137 | print("RESULTS") 138 | print(config, result) 139 | 140 | 141 | -------------------------------------------------------------------------------- /examples/rnn.py: -------------------------------------------------------------------------------- 1 | #inspired by http://people.idsia.ch/~juergen/icann2009bayer.pdf 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | import hyperchamber as hc 6 | from tensorflow.models.rnn import rnn_cell, rnn 7 | 8 | 9 | hc.set("learning_rate", 10 | [0.01]) 11 | hc.set("batch_size", 12 | [128]) 13 | hc.set('rnn_size', 14 | [4]) 15 | 16 | TRAIN_STEPS=100 17 | TESTS = TRAIN_STEPS/10 18 | 19 | def create_rnn(config, x, scope='rnn'): 20 | with tf.variable_scope(scope): 21 | memory=config['rnn_size'] 22 | cell = rnn_cell.BasicLSTMCell(memory) 23 | state = cell.zero_state(batch_size=config['batch_size'], dtype=tf.float32) 24 | x, state = rnn.rnn(cell, [tf.cast(x,tf.float32)], initial_state=state, dtype=tf.float32) 25 | x = x[-1] 26 | #w = tf.get_variable('w', [hc.get('rnn_size'),4]) 27 | #b = tf.get_variable('b', [4]) 28 | #x = tf.nn.xw_plus_b(x, w, b) 29 | x=tf.sign(x) 30 | return x, state 31 | 32 | # Each step of the graph we have: 33 | # x is [BATCH_SIZE, 4] where the data is an one hot binary vector of the form: 34 | # [start_token end_token a b] 35 | # 36 | # y is [BATCH_SIZE, 4] is a binary vector of the chance each character is correct 37 | # 38 | def create_graph(config, x, y): 39 | output, state = create_rnn(config, x) 40 | y = tf.cast(y, tf.float32) 41 | return tf.reduce_sum(tf.reduce_mean(output*tf.log(y))) 42 | 43 | def parallel_run(sess, train_step, costs): 44 | return [sess.run(train_step, cost) for cost in costs] 45 | 46 | def create_optimizer(cost,learning_rate): 47 | optimizer = tf.train.AdamOptimizer(learning_rate) 48 | grad_clip = 5. 49 | tvars = tf.trainable_variables() 50 | grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), grad_clip) 51 | train_step = optimizer.apply_gradients(zip(grads, tvars)) 52 | return train_step 53 | 54 | 55 | def run(): 56 | #x_input = get_an_bn(1) 57 | #y_input = get_an_bn_grammar(1) 58 | 59 | print("Training RNNs") 60 | configs = hc.configs(1, offset=0) 61 | sess = tf.Session() 62 | # TODO: Create subgraphs on config 63 | graph_costs=[] 64 | optimizers=[] 65 | for config in configs: 66 | x = tf.placeholder("bool", [config['batch_size'], 4]) 67 | y = tf.placeholder("bool", [config['batch_size'], 4]) 68 | x_input = [[True,False,False,False],[False,True,False,False]] 69 | y_input = [[True,True,False,False],[True,False,True,False]] 70 | graph_cost = create_graph(config, x, y) 71 | optimizer = create_optimizer(graph_cost,config['learning_rate']) 72 | graph_costs.append(graph_cost) 73 | optimizers.append(optimizer) 74 | 75 | for i in range(TRAIN_STEPS): 76 | _, costs = parallel_run(sess, optimizers, graph_costs) 77 | hc.cost(configs, costs) 78 | 79 | for config in hc.top_configs(3): 80 | print(config) 81 | 82 | def encode(chars): 83 | def vectorize(c): 84 | if(c=="S"): 85 | return [True,False,False,False] 86 | if(c=="T"): 87 | return [False,True,False,False] 88 | if(c=="a"): 89 | return [False,False,True,False] 90 | if(c=="b"): 91 | return [False,False,False,True] 92 | return [vectorize(c) for c in chars] 93 | 94 | 95 | def get_an_bn(n): 96 | return encode("Sa"*n+"b"*n+"T") 97 | def get_an_bn_an(n): 98 | return encode("Sa"*n+"b"*n+"a"*n+"T") 99 | 100 | # returns a vector of a**nb**n grammars given a set of one-hot vectors x, denoting the next element in a generated sequence 101 | # Each value is returned as an array of possible next inputs. 102 | # for example: 103 | # get_an_bn_grammar(ST) # T 104 | # get_an_bn_grammar(SabT) # T/a-b-T 105 | # get_an_bn_grammar(SaaabbbT) # T/a-a/b-a/b-b-b-b-T 106 | # 107 | # where S and T designate start and end symbols 108 | def get_an_bn_grammar(x): 109 | return "" 110 | 111 | # same as above but with extra 'a's 112 | def get_an_bn_an_grammar(x): 113 | return "" 114 | 115 | run() 116 | -------------------------------------------------------------------------------- /examples/shared/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HyperGAN/hyperchamber/f9b92f5518f1b23e6873e7da63bc31aac3819aae/examples/shared/__init__.py -------------------------------------------------------------------------------- /examples/shared/cifar_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tarfile 4 | import os 5 | import sys 6 | 7 | from six.moves import xrange 8 | from six.moves import urllib 9 | import matplotlib 10 | import matplotlib.pyplot as plt 11 | 12 | from scipy.misc import imsave 13 | 14 | from tensorflow.models.image.cifar10 import cifar10_input 15 | 16 | 17 | def inputs(eval_data, data_dir, batch_size): 18 | """Construct input for CIFAR evaluation using the Reader ops. 19 | Args: 20 | eval_data: bool, indicating if one should use the train or eval data set. 21 | data_dir: Path to the CIFAR-10 data directory. 22 | batch_size: Number of images per batch. 23 | Returns: 24 | images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. 25 | labels: Labels. 1D tensor of [batch_size] size. 26 | """ 27 | if not eval_data: 28 | filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i) 29 | for i in xrange(1, 6)] 30 | num_examples_per_epoch = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN 31 | else: 32 | filenames = [os.path.join(data_dir, 'test_batch.bin')] 33 | num_examples_per_epoch = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL 34 | 35 | for f in filenames: 36 | if not tf.gfile.Exists(f): 37 | raise ValueError('Failed to find file: ' + f) 38 | 39 | # Create a queue that produces the filenames to read. 40 | filename_queue = tf.train.string_input_producer(filenames) 41 | 42 | # Read examples from files in the filename queue. 43 | read_input = cifar10_input.read_cifar10(filename_queue) 44 | reshaped_image = tf.cast(read_input.uint8image, tf.float32) 45 | 46 | IMAGE_SIZE=32 47 | height = IMAGE_SIZE 48 | width = IMAGE_SIZE 49 | 50 | # Image processing for evaluation. 51 | # Crop the central [height, width] of the image. 52 | resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, 53 | width, height) 54 | 55 | # Subtract off the mean and divide by the variance of the pixels. 56 | #float_image = tf.image.per_image_whitening(resized_image) 57 | float_image = resized_image / 127.5 - 1. 58 | 59 | # Ensure that the random shuffling has good mixing properties. 60 | min_fraction_of_examples_in_queue = 0.4 61 | min_queue_examples = int(num_examples_per_epoch * 62 | min_fraction_of_examples_in_queue) 63 | 64 | # Generate a batch of images and labels by building up a queue of examples. 65 | return cifar10_input._generate_image_and_label_batch(float_image, read_input.label, 66 | min_queue_examples, batch_size, 67 | shuffle=False) 68 | 69 | 70 | 71 | DATA_URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' 72 | 73 | def maybe_download_and_extract(): 74 | """Download and extract the tarball from Alex's website.""" 75 | dest_directory = "/tmp/cifar" 76 | if not os.path.exists(dest_directory): 77 | os.makedirs(dest_directory) 78 | filename = DATA_URL.split('/')[-1] 79 | filepath = os.path.join(dest_directory, filename) 80 | if not os.path.exists(filepath): 81 | def _progress(count, block_size, total_size): 82 | sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename, 83 | float(count * block_size) / float(total_size) * 100.0)) 84 | sys.stdout.flush() 85 | filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) 86 | print() 87 | statinfo = os.stat(filepath) 88 | print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') 89 | tarfile.open(filepath, 'r:gz').extractall(dest_directory) 90 | 91 | def plot(config, image, file): 92 | """ Plot a single CIFAR image.""" 93 | image = np.squeeze(image) 94 | print(file, image.shape) 95 | imsave(file, image) 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /examples/shared/convert_py_to_ipynb.py: -------------------------------------------------------------------------------- 1 | from nbformat import v3, v4 2 | 3 | with open("mnist-io.py") as fpin: 4 | text = fpin.read() 5 | 6 | nbook = v3.reads_py(text) 7 | nbook = v4.upgrade(nbook) # Upgrade v3 to v4 8 | 9 | jsonform = v4.writes(nbook) + "\n" 10 | with open("mnist.ipynb", "w") as fpout: 11 | fpout.write(jsonform) 12 | -------------------------------------------------------------------------------- /examples/shared/data_loader.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import tensorflow as tf 3 | import shared.resize_image_patch 4 | 5 | def build_labels(dirs): 6 | next_id=0 7 | labels = {} 8 | for dir in dirs: 9 | labels[dir.split('/')[-1]]=next_id 10 | next_id+=1 11 | return labels,next_id 12 | def labelled_image_tensors_from_directory(directory, batch_size, channels=3, format='jpg', width=64, height=64, crop=True): 13 | filenames = glob.glob(directory+"/**/*."+format) 14 | labels,total_labels = build_labels(sorted(glob.glob(directory+"/*"))) 15 | num_examples_per_epoch = 40000 16 | 17 | # Create a queue that produces the filenames to read. 18 | classes = [labels[f.split('/')[-2]] for f in filenames] 19 | 20 | filenames = tf.convert_to_tensor(filenames, dtype=tf.string) 21 | classes = tf.convert_to_tensor(classes, dtype=tf.int32) 22 | print("[0]", filenames[0], classes[0]) 23 | 24 | input_queue = tf.train.slice_input_producer([filenames, classes], shuffle=True) 25 | 26 | # Read examples from files in the filename queue. 27 | value = tf.read_file(input_queue[0]) 28 | if(format == 'jpg'): 29 | img = tf.image.decode_jpeg(value, channels=channels) 30 | elif(format == 'png'): 31 | img = tf.image.decode_png(value, channels=channels) 32 | else: 33 | print("Failed to load format", format) 34 | #img = tf.zeros([64,64,3]) 35 | reshaped_image = tf.cast(img, tf.float32) 36 | tf.Tensor.set_shape(img, [None, None, None]) 37 | print('img', img) 38 | label = input_queue[1] 39 | 40 | # Image processing for evaluation. 41 | # Crop the central [height, width] of the image. 42 | if(crop): 43 | resized_image = shared.resize_image_patch.resize_image_with_crop_or_pad(reshaped_image, 44 | width, height, dynamic_shape=True) 45 | else: 46 | resized_image = reshaped_image 47 | 48 | #resized_image = reshaped_image 49 | tf.Tensor.set_shape(resized_image, [width,height,channels]) 50 | print(resized_image) 51 | #resized_image = tf.image.convert_image_dtype(resized_image, tf.float32) 52 | # Subtract off the mean and divide by the variance of the pixels. 53 | #float_image = tf.image.per_image_whitening(resized_image) 54 | float_image = resized_image / 127.5 - 1. 55 | 56 | # Ensure that the random shuffling has good mixing properties. 57 | min_fraction_of_examples_in_queue = 0.4 58 | min_queue_examples = int(num_examples_per_epoch * 59 | min_fraction_of_examples_in_queue) 60 | 61 | # Generate a batch of images and labels by building up a queue of examples. 62 | x,y= _get_data(float_image, label, min_queue_examples, batch_size) 63 | 64 | 65 | 66 | return x, y,total_labels, num_examples_per_epoch 67 | 68 | def _get_data(image, label, min_queue_examples, batch_size): 69 | num_preprocess_threads = 8 70 | print(image, label) 71 | images, label_batch = tf.train.shuffle_batch( 72 | [image, label], 73 | batch_size=batch_size, 74 | num_threads=num_preprocess_threads, 75 | capacity= 10000, 76 | min_after_dequeue=100) 77 | return images, tf.reshape(label_batch, [batch_size]) 78 | 79 | -------------------------------------------------------------------------------- /examples/shared/gan.py: -------------------------------------------------------------------------------- 1 | 2 | from shared.ops import * 3 | from shared.util import * 4 | import tensorflow as tf 5 | def generator(config, y,z, reuse=False): 6 | x_dims = config['x_dims'] 7 | with(tf.variable_scope("generator", reuse=reuse)): 8 | output_shape = x_dims[0]*x_dims[1]*config['channels'] 9 | z_proj_dims = int(config['conv_g_layers'][0])*2 10 | z_dims = int(z.get_shape()[1]) 11 | print("z_proj_dims", z_proj_dims, z_dims, config['y_dims']) 12 | noise_dims = z_proj_dims-z_dims-config['y_dims'] 13 | print(noise_dims) 14 | if(noise_dims < 0): 15 | result = tf.concat(1, [y, z]) 16 | result = linear(result, z_proj_dims, 'g_noise_proj') 17 | if(config['g_batch_norm']): 18 | result = batch_norm(config['batch_size'], name='g_noise_bn')(result) 19 | result = config['g_activation'](result) 20 | else: 21 | noise = tf.random_uniform([config['batch_size'], noise_dims],-1, 1) 22 | 23 | if(config['g_project'] == 'noise'): 24 | result = tf.concat(1, [y, z, noise]) 25 | elif(config['g_project'] == 'zeros'): 26 | result = tf.concat(1, [y, z]) 27 | #result = z 28 | result = tf.pad(result, [[0, 0],[noise_dims//2, noise_dims//2]]) 29 | else: 30 | result = tf.concat(1, [y, z]) 31 | #result = z 32 | result = linear(result, z_proj_dims, 'g_input_proj') 33 | 34 | def build_layers(result, z_proj_dims, offset): 35 | if config['conv_g_layers']: 36 | if(z_proj_dims % 16 == 0 and config['x_dims'][0]%4 == 0): 37 | result = tf.reshape(result, [config['batch_size'], 4,4,z_proj_dims//16]) 38 | elif(z_proj_dims % 36 == 0 and config['x_dims'][0]%6 == 0): 39 | result = tf.reshape(result, [config['batch_size'], 6,6,z_proj_dims//36]) 40 | else: 41 | print('z_proj dims not divisible by 4 or 8') 42 | #result = tf.nn.dropout(result, 0.7) 43 | for i, layer in enumerate(config['conv_g_layers']): 44 | j=int(result.get_shape()[1]*2) 45 | k=int(result.get_shape()[2]*2) 46 | stride=2 47 | if(j > x_dims[0]): 48 | j = x_dims[0] 49 | k = x_dims[1] 50 | stride=1 51 | output = [config['batch_size'], j,k,int(layer)] 52 | result = deconv2d(result, output, name="g_conv_"+str(i+offset), k_w=config['conv_size'], k_h=config['conv_size'], d_h=stride, d_w=stride) 53 | if(len(config['conv_g_layers']) == i+1): 54 | print("Skipping last layer") 55 | else: 56 | print("Adding nonlinear") 57 | if(config['g_batch_norm']): 58 | result = batch_norm(config['batch_size'], name='g_conv_bn_'+str(i+offset))(result) 59 | result = config['g_activation'](result) 60 | return result 61 | result = build_layers(result, z_proj_dims, 0) 62 | 63 | if(config['g_last_layer']): 64 | result = config['g_last_layer'](result) 65 | return result 66 | 67 | def discriminator(config, x, z,g,gz, reuse=False): 68 | x_dims = config['x_dims'] 69 | if(reuse): 70 | tf.get_variable_scope().reuse_variables() 71 | batch_size = config['batch_size']*2 72 | single_batch_size = config['batch_size'] 73 | x = tf.concat(0, [x,g]) 74 | z = tf.concat(0, [z,gz]) 75 | x = tf.reshape(x, [batch_size, -1, config['channels']]) 76 | if(config['d_add_noise']): 77 | x += tf.random_normal(x.get_shape(), mean=0, stddev=0.1) 78 | 79 | channels = (config['channels']+1) 80 | 81 | if(config['d_project'] == 'zeros'): 82 | noise_dims = int(x.get_shape()[1])-int(z.get_shape()[1]) 83 | z = tf.pad(z, [[0, 0],[noise_dims//2, noise_dims//2]]) 84 | z = tf.reshape(z, [batch_size, int(x.get_shape()[1]), 1]) 85 | print("CONCAT", x.get_shape(), z.get_shape()) 86 | result = tf.concat(2, [x,z]) 87 | else: 88 | x = tf.reshape(x, [batch_size, -1]) 89 | result = tf.concat(1, [z,x]) 90 | result = linear(result, x_dims[0]*x_dims[1]*channels, scope='d_z') 91 | result = config['d_activation'](result) 92 | 93 | if config['conv_d_layers']: 94 | result = tf.reshape(result, [batch_size, x_dims[0],x_dims[1],channels]) 95 | for i, layer in enumerate(config['conv_d_layers']): 96 | filter = config['d_conv_size'] 97 | stride = 2 98 | if(filter > result.get_shape()[1]): 99 | filter = int(result.get_shape()[1]) 100 | stride = 1 101 | result = conv2d(result, layer, name='d_conv'+str(i), k_w=filter, k_h=filter, d_h=stride, d_w=stride) 102 | if(config['d_batch_norm']): 103 | result = batch_norm(batch_size, name='d_conv_bn_'+str(i))(result) 104 | result = config['d_activation'](result) 105 | result = tf.reshape(x, [batch_size, -1]) 106 | 107 | def get_minibatch_features(h): 108 | n_kernels = int(config['d_kernels']) 109 | dim_per_kernel = int(config['d_kernel_dims']) 110 | x = linear(h, n_kernels * dim_per_kernel, scope="d_h") 111 | activation = tf.reshape(x, (batch_size, n_kernels, dim_per_kernel)) 112 | 113 | big = np.zeros((batch_size, batch_size), dtype='float32') 114 | big += np.eye(batch_size) 115 | big = tf.expand_dims(big, 1) 116 | 117 | abs_dif = tf.reduce_sum(tf.abs(tf.expand_dims(activation,3) - tf.expand_dims(tf.transpose(activation, [1, 2, 0]), 0)), 2) 118 | mask = 1. - big 119 | masked = tf.exp(-abs_dif) * mask 120 | def half(tens, second): 121 | m, n, _ = tens.get_shape() 122 | m = int(m) 123 | n = int(n) 124 | return tf.slice(tens, [0, 0, second * single_batch_size], [m, n, single_batch_size]) 125 | # TODO: speedup by allocating the denominator directly instead of constructing it by sum 126 | # (current version makes it easier to play with the mask and not need to rederive 127 | # the denominator) 128 | f1 = tf.reduce_sum(half(masked, 0), 2) / tf.reduce_sum(half(mask, 0)) 129 | f2 = tf.reduce_sum(half(masked, 1), 2) / tf.reduce_sum(half(mask, 1)) 130 | 131 | return [f1, f2] 132 | minis = get_minibatch_features(result) 133 | g_proj = tf.concat(1, [result]+minis) 134 | 135 | #result = tf.nn.dropout(result, 0.7) 136 | if(config['d_linear_layer']): 137 | result = linear(result, config['d_linear_layers'], scope="d_linear_layer") 138 | result = config['d_activation'](result) 139 | 140 | last_layer = result 141 | result = linear(result, config['y_dims']+1, scope="d_proj") 142 | 143 | 144 | def build_logits(class_logits, num_classes): 145 | 146 | generated_class_logits = tf.squeeze(tf.slice(class_logits, [0, num_classes - 1], [batch_size, 1])) 147 | positive_class_logits = tf.slice(class_logits, [0, 0], [batch_size, num_classes - 1]) 148 | 149 | """ 150 | # make these a separate matmul with weights initialized to 0, attached only to generated_class_logits, or things explode 151 | generated_class_logits = tf.squeeze(generated_class_logits) + tf.squeeze(linear(diff_feat, 1, stddev=0., scope="d_indivi_logits_from_diff_feat")) 152 | assert len(generated_class_logits.get_shape()) == 1 153 | # re-assemble the logits after incrementing the generated class logits 154 | class_logits = tf.concat(1, [positive_class_logits, tf.expand_dims(generated_class_logits, 1)]) 155 | """ 156 | 157 | mx = tf.reduce_max(positive_class_logits, 1, keep_dims=True) 158 | safe_pos_class_logits = positive_class_logits - mx 159 | 160 | gan_logits = tf.log(tf.reduce_sum(tf.exp(safe_pos_class_logits), 1)) + tf.squeeze(mx) - generated_class_logits 161 | assert len(gan_logits.get_shape()) == 1 162 | 163 | return class_logits, gan_logits 164 | num_classes = config['y_dims'] +1 165 | class_logits, gan_logits = build_logits(result, num_classes) 166 | print("Class logits gan logits", class_logits, gan_logits) 167 | return [tf.slice(class_logits, [0, 0], [single_batch_size, num_classes]), 168 | tf.slice(gan_logits, [0], [single_batch_size]), 169 | tf.slice(class_logits, [single_batch_size, 0], [single_batch_size, num_classes]), 170 | tf.slice(gan_logits, [single_batch_size], [single_batch_size]), 171 | last_layer] 172 | 173 | 174 | def build_conv_tower(result, layers, filter, batch_size, batch_norm_enabled, name, activation): 175 | for i, layer in enumerate(layers): 176 | print('-!-', result, tf.reshape(result, [batch_size, -1])) 177 | print(layer) 178 | stride = 2 179 | if filter > result.get_shape()[2]: 180 | filter = int(result.get_shape()[2]) 181 | stride = 1 182 | result = conv2d(result, layer, name=name+str(i), k_w=filter, k_h=filter, d_h=stride, d_w=stride) 183 | if(batch_norm_enabled): 184 | result = batch_norm(batch_size, name=name+'_bn_'+str(i))(result) 185 | if(len(layers) == i+1): 186 | print("Skipping last layer") 187 | else: 188 | print("Adding nonlinear") 189 | result = activation(result) 190 | print(tf.reshape(result, [batch_size, -1])) 191 | result = tf.reshape(result, [batch_size, -1]) 192 | return result 193 | 194 | 195 | def approximate_z(config, x, y): 196 | x_dims = config['x_dims'] 197 | transfer_fct = config['transfer_fct'] 198 | x = tf.reshape(x, [config["batch_size"], -1,config['channels']]) 199 | noise_dims = int(x.get_shape()[1])-int(y.get_shape()[1]) 200 | n_hidden_recog_1 = int(config['n_hidden_recog_1']) 201 | n_hidden_recog_2 = int(config['n_hidden_recog_2']) 202 | n_z = int(config['z_dim']) 203 | channels = (config['channels']+1) 204 | if(config['e_project'] == 'zeros'): 205 | noise_dims = int(x.get_shape()[1])-config['y_dims'] 206 | #y = tf.pad(y, [[0, 0],[noise_dims//2, noise_dims//2]]) 207 | noise = tf.zeros([config['batch_size'], noise_dims]) 208 | result = tf.concat(1, [tf.reshape(x, [config['batch_size'], -1]), y, noise]) 209 | #y = tf.reshape(y, [config['batch_size'], int(x.get_shape()[1]), 1]) 210 | #result = tf.concat(2, [x,y]) 211 | print(result) 212 | result = tf.reshape(result, [config["batch_size"], x_dims[0],x_dims[1],channels]) 213 | 214 | if config['g_encode_layers']: 215 | result = build_conv_tower(result, config['g_encode_layers'], config['e_conv_size'], config['batch_size'], config['d_batch_norm'], 'g_vae_', transfer_fct) 216 | 217 | result = transfer_fct(result) 218 | 219 | b_out_mean= tf.get_variable('g_b_out_mean', initializer=tf.zeros([n_z], dtype=tf.float32)) 220 | out_mean= tf.get_variable('g_out_mean', [result.get_shape()[1], n_z], initializer=tf.contrib.layers.xavier_initializer()) 221 | mu = tf.add(tf.matmul(result, out_mean),b_out_mean) 222 | 223 | out_log_sigma=tf.get_variable('g_out_log_sigma', [result.get_shape()[1], n_z], initializer=tf.contrib.layers.xavier_initializer()) 224 | b_out_log_sigma= tf.get_variable('g_b_out_log_sigma', initializer=tf.zeros([n_z], dtype=tf.float32)) 225 | sigma = tf.add(tf.matmul(result, out_log_sigma),b_out_log_sigma) 226 | 227 | e_projected_z=tf.get_variable('g_encoded_z', [result.get_shape()[1], n_z], initializer=tf.contrib.layers.xavier_initializer()) 228 | b_encoded_z= tf.get_variable('g_b_encoded_z', initializer=tf.zeros([n_z], dtype=tf.float32)) 229 | e_z = tf.add(tf.matmul(result, e_projected_z),b_encoded_z) 230 | 231 | n_z = int(config["z_dim"]) 232 | eps = tf.random_normal((config['batch_size'], n_z), 0, 1, 233 | dtype=tf.float32) 234 | 235 | z = tf.add(mu, tf.mul(tf.sqrt(tf.exp(sigma)), eps)) 236 | z = batch_norm(config['batch_size'], name='g_e_z_bn')(z) 237 | 238 | e_z = batch_norm(config['batch_size'], name='g_e_ez_bn')(e_z) 239 | 240 | if(config['e_last_layer']): 241 | z = config['e_last_layer'](z) 242 | e_z = config['e_last_layer'](e_z) 243 | return e_z, z, mu, sigma 244 | def sigmoid_kl_with_logits(logits, targets): 245 | print(targets) 246 | # broadcasts the same target value across the whole batch 247 | # this is implemented so awkwardly because tensorflow lacks an x log x op 248 | assert isinstance(targets, float) 249 | if targets in [0., 1.]: 250 | entropy = 0. 251 | else: 252 | entropy = - targets * np.log(targets) - (1. - targets) * np.log(1. - targets) 253 | return tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.ones_like(logits) * targets) - entropy 254 | 255 | 256 | def create(config, x,y): 257 | batch_size = config["batch_size"] 258 | print(y) 259 | 260 | #x = x/tf.reduce_max(tf.abs(x), 0) 261 | encoded_z, z, z_mu, z_sigma = approximate_z(config, x, y) 262 | 263 | print("Build generator") 264 | g = generator(config, y, z) 265 | encoded = generator(config, y, encoded_z, reuse=True) 266 | print("shape of g,x", g.get_shape(), x.get_shape()) 267 | print("shape of z,encoded_z", z.get_shape(), encoded_z.get_shape()) 268 | d_real, d_real_sig, d_fake, d_fake_sig, d_last_layer = discriminator(config,x, encoded_z, g, z, reuse=False) 269 | 270 | latent_loss = -config['latent_lambda'] * tf.reduce_mean(1 + z_sigma 271 | - tf.square(z_mu) 272 | - tf.exp(z_sigma), 1) 273 | np_fake = np.array([0]*config['y_dims']+[1]) 274 | print('np_fake', np_fake) 275 | fake_symbol = tf.tile(tf.constant(np_fake, dtype=tf.float32), [config['batch_size']]) 276 | fake_symbol = tf.reshape(fake_symbol, [config['batch_size'],config['y_dims']+1]) 277 | 278 | real_symbols = tf.concat(1, [y, tf.zeros([config['batch_size'], 1])]) 279 | #real_symbols = y 280 | 281 | 282 | if(config['loss'] == 'softmax'): 283 | d_fake_loss = tf.nn.softmax_cross_entropy_with_logits(d_fake, fake_symbol) 284 | d_real_loss = tf.nn.softmax_cross_entropy_with_logits(d_real, real_symbols) 285 | 286 | g_loss= tf.nn.softmax_cross_entropy_with_logits(d_fake, real_symbols) 287 | 288 | else: 289 | zeros = tf.zeros_like(d_fake_sig) 290 | ones = tf.zeros_like(d_real_sig) 291 | 292 | #d_fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(d_fake, zeros) 293 | #d_real_loss = tf.nn.sigmoid_cross_entropy_with_logits(d_real, ones) 294 | 295 | generator_target_prob = config['g_target_prob'] 296 | d_label_smooth = config['d_label_smooth'] 297 | d_fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(d_fake_sig, zeros) 298 | #d_real_loss = tf.nn.sigmoid_cross_entropy_with_logits(d_real_sig, ones) 299 | d_real_loss = sigmoid_kl_with_logits(d_real_sig, 1.-d_label_smooth) 300 | d_class_loss = tf.nn.softmax_cross_entropy_with_logits(d_real,real_symbols) 301 | d_fake_class_loss = tf.nn.softmax_cross_entropy_with_logits(d_fake,fake_symbol) 302 | 303 | g_loss= sigmoid_kl_with_logits(d_fake_sig, generator_target_prob) 304 | g_class_loss = tf.nn.softmax_cross_entropy_with_logits(d_fake, real_symbols) 305 | 306 | #g_loss_encoder = tf.nn.sigmoid_cross_entropy_with_logits(d_real, zeros) 307 | #TINY = 1e-12 308 | #d_real = tf.nn.sigmoid(d_real) 309 | #d_fake = tf.nn.sigmoid(d_fake) 310 | #d_fake_loss = -tf.log(1-d_fake+TINY) 311 | #d_real_loss = -tf.log(d_real+TINY) 312 | #g_loss_softmax = -tf.log(1-d_real+TINY) 313 | #g_loss_encoder = -tf.log(d_fake+TINY) 314 | if(config['latent_loss']): 315 | g_loss = tf.reduce_mean(g_loss)+tf.reduce_mean(latent_loss)+tf.reduce_mean(g_class_loss) 316 | else: 317 | g_loss = tf.reduce_mean(g_loss)+tf.reduce_mean(g_class_loss) 318 | d_loss = tf.reduce_mean(d_fake_loss) + tf.reduce_mean(d_real_loss) + \ 319 | tf.reduce_mean(d_class_loss)+tf.reduce_mean(d_fake_class_loss) 320 | print('d_loss', d_loss.get_shape()) 321 | 322 | 323 | 324 | if config['regularize']: 325 | ws = None 326 | with tf.variable_scope("generator"): 327 | with tf.variable_scope("g_conv_0"): 328 | tf.get_variable_scope().reuse_variables() 329 | ws = tf.get_variable('w') 330 | tf.get_variable_scope().reuse_variables() 331 | b = tf.get_variable('biases') 332 | lam = config['regularize_lambda'] 333 | g_loss += lam*tf.nn.l2_loss(ws)+lam*tf.nn.l2_loss(b) 334 | 335 | 336 | mse_loss = tf.reduce_max(tf.square(x-encoded)) 337 | if config['mse_loss']: 338 | mse_lam = config['mse_lambda'] 339 | g_loss += mse_lam * mse_loss 340 | 341 | g_vars = [var for var in tf.trainable_variables() if 'g_' in var.name] 342 | d_vars = [var for var in tf.trainable_variables() if 'd_' in var.name] 343 | 344 | print(config); 345 | print('vars', [v.name for v in tf.trainable_variables()]) 346 | g_optimizer = tf.train.AdamOptimizer(np.float32(config['g_learning_rate'])).minimize(g_loss, var_list=g_vars) 347 | d_optimizer = tf.train.AdamOptimizer(np.float32(config['d_learning_rate'])).minimize(d_loss, var_list=d_vars) 348 | 349 | mse_optimizer = tf.train.AdamOptimizer(np.float32(config['g_learning_rate'])).minimize(mse_loss, var_list=tf.trainable_variables()) 350 | 351 | set_tensor("x", x) 352 | set_tensor("y", y) 353 | set_tensor("g_loss", g_loss) 354 | set_tensor("d_loss", d_loss) 355 | set_tensor("g_optimizer", g_optimizer) 356 | set_tensor("d_optimizer", d_optimizer) 357 | set_tensor("mse_optimizer", mse_optimizer) 358 | set_tensor("g", g) 359 | set_tensor("encoded", encoded) 360 | set_tensor("encoder_mse", mse_loss) 361 | set_tensor("d_fake", tf.reduce_mean(d_fake)) 362 | set_tensor("d_real", tf.reduce_mean(d_real)) 363 | set_tensor("d_fake_loss", tf.reduce_mean(d_fake_loss)) 364 | set_tensor("d_real_loss", tf.reduce_mean(d_real_loss)) 365 | set_tensor("d_class_loss", tf.reduce_mean(d_real_loss)) 366 | set_tensor("g_class_loss", tf.reduce_mean(g_class_loss)) 367 | set_tensor("d_loss", tf.reduce_mean(d_real_loss)) 368 | 369 | def train(sess, config): 370 | x = get_tensor('x') 371 | g = get_tensor('g') 372 | g_loss = get_tensor("g_loss") 373 | d_loss = get_tensor("d_loss") 374 | d_fake_loss = get_tensor('d_fake_loss') 375 | d_real_loss = get_tensor('d_real_loss') 376 | g_optimizer = get_tensor("g_optimizer") 377 | d_optimizer = get_tensor("d_optimizer") 378 | d_class_loss = get_tensor("d_class_loss") 379 | g_class_loss = get_tensor("g_class_loss") 380 | mse_optimizer = get_tensor("mse_optimizer") 381 | encoder_mse = get_tensor("encoder_mse") 382 | _, d_cost = sess.run([d_optimizer, d_loss]) 383 | _, g_cost, x, g,e_loss,d_fake,d_real, d_class, g_class = sess.run([g_optimizer, g_loss, x, g, encoder_mse,d_fake_loss, d_real_loss, d_class_loss, g_class_loss]) 384 | #_ = sess.run([mse_optimizer]) 385 | 386 | print("g cost %.2f d cost %.2f encoder %.2f d_fake %.6f d_real %.2f d_class %.2f g_class %.2f" % (g_cost, d_cost,e_loss, d_fake, d_real, d_class, g_class)) 387 | print("X mean %.2f max %.2f min %.2f" % (np.mean(x), np.max(x), np.min(x))) 388 | print("G mean %.2f max %.2f min %.2f" % (np.mean(g), np.max(g), np.min(g))) 389 | 390 | return d_cost, g_cost 391 | 392 | def test(sess, config): 393 | x = get_tensor("x") 394 | y = get_tensor("y") 395 | d_fake = get_tensor("d_fake") 396 | d_real = get_tensor("d_real") 397 | g_loss = get_tensor("g_loss") 398 | encoder_mse = get_tensor("encoder_mse") 399 | 400 | g_cost, d_fake_cost, d_real_cost, e_cost = sess.run([g_loss, d_fake, d_real, encoder_mse]) 401 | 402 | 403 | #hc.event(costs, sample_image = sample[0]) 404 | 405 | #print("test g_loss %.2f d_fake %.2f d_loss %.2f" % (g_cost, d_fake_cost, d_real_cost)) 406 | return g_cost,d_fake_cost, d_real_cost, e_cost 407 | 408 | 409 | -------------------------------------------------------------------------------- /examples/shared/ops.py: -------------------------------------------------------------------------------- 1 | import math 2 | from contextlib import contextmanager 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | from tensorflow.python.framework import ops 7 | 8 | rng = np.random.RandomState([2016, 6, 1]) 9 | 10 | class batch_norm(object): 11 | """Code modification of http://stackoverflow.com/a/33950177 12 | 13 | """ 14 | def __init__(self, batch_size, epsilon=1e-5, momentum = 0.1, name="batch_norm", half=None): 15 | assert half is None 16 | del momentum # unused 17 | with tf.variable_scope(name) as scope: 18 | self.epsilon = epsilon 19 | self.batch_size = batch_size 20 | 21 | self.name=name 22 | 23 | def __call__(self, x, train=True): 24 | del train # unused 25 | 26 | shape = x.get_shape().as_list() 27 | 28 | needs_reshape = len(shape) != 4 29 | if needs_reshape: 30 | orig_shape = shape 31 | if len(shape) == 2: 32 | x = tf.reshape(x, [shape[0], 1, 1, shape[1]]) 33 | elif len(shape) == 1: 34 | x = tf.reshape(x, [shape[0], 1, 1, 1]) 35 | else: 36 | assert False, shape 37 | shape = x.get_shape().as_list() 38 | 39 | with tf.variable_scope(self.name) as scope: 40 | self.gamma = tf.get_variable("gamma", [shape[-1]], 41 | initializer=tf.random_normal_initializer(1., 0.02)) 42 | self.beta = tf.get_variable("beta", [shape[-1]], 43 | initializer=tf.constant_initializer(0.)) 44 | 45 | self.mean, self.variance = tf.nn.moments(x, [0, 1, 2]) 46 | 47 | out = tf.nn.batch_norm_with_global_normalization( 48 | x, self.mean, self.variance, self.beta, self.gamma, self.epsilon, 49 | scale_after_normalization=True) 50 | if needs_reshape: 51 | out = tf.reshape(out, orig_shape) 52 | return out 53 | 54 | TRAIN_MODE = True 55 | class conv_batch_norm(object): 56 | """Code modification of http://stackoverflow.com/a/33950177""" 57 | def __init__(self, name="batch_norm", epsilon=1e-5, momentum=0.1, 58 | in_dim=None): 59 | with tf.variable_scope(name) as scope: 60 | self.epsilon = epsilon 61 | self.momentum = momentum 62 | self.name = name 63 | self.in_dim = in_dim 64 | global TRAIN_MODE 65 | self.train = TRAIN_MODE 66 | self.ema = tf.train.ExponentialMovingAverage(decay=0.9) 67 | print("initing %s in train: %s" % (scope.name, self.train)) 68 | 69 | def __call__(self, x): 70 | shape = x.get_shape() 71 | shp = self.in_dim or shape[-1] 72 | with tf.variable_scope(self.name) as scope: 73 | self.gamma = tf.get_variable("gamma", [shp], 74 | initializer=tf.random_normal_initializer(1., 0.02)) 75 | self.beta = tf.get_variable("beta", [shp], 76 | initializer=tf.constant_initializer(0.)) 77 | 78 | self.mean, self.variance = tf.nn.moments(x, [0, 1, 2]) 79 | self.mean.set_shape((shp,)) 80 | self.variance.set_shape((shp,)) 81 | self.ema_apply_op = self.ema.apply([self.mean, self.variance]) 82 | 83 | if self.train: 84 | # with tf.control_dependencies([self.ema_apply_op]): 85 | normalized_x = tf.nn.batch_norm_with_global_normalization( 86 | x, self.mean, self.variance, self.beta, self.gamma, self.epsilon, 87 | scale_after_normalization=True) 88 | else: 89 | normalized_x = tf.nn.batch_norm_with_global_normalization( 90 | x, self.ema.average(self.mean), self.ema.average(self.variance), self.beta, 91 | self.gamma, self.epsilon, 92 | scale_after_normalization=True) 93 | return normalized_x 94 | 95 | class fc_batch_norm(conv_batch_norm): 96 | def __call__(self, fc_x): 97 | ori_shape = fc_x.get_shape().as_list() 98 | if ori_shape[0] is None: 99 | ori_shape[0] = -1 100 | new_shape = [ori_shape[0], 1, 1, ori_shape[1]] 101 | x = tf.reshape(fc_x, new_shape) 102 | normalized_x = super(fc_batch_norm, self).__call__(x) 103 | return tf.reshape(normalized_x, ori_shape) 104 | 105 | 106 | def conv_cond_concat(x, y): 107 | """Concatenate conditioning vector on feature map axis.""" 108 | x_shapes = x.get_shape() 109 | y_shapes = y.get_shape() 110 | return tf.concat(3, [x, y*tf.ones([x_shapes[0], x_shapes[1], x_shapes[2], y_shapes[3]])]) 111 | 112 | def conv2d(input_, output_dim, 113 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 114 | name="conv2d"): 115 | with tf.variable_scope(name): 116 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 117 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 118 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 119 | 120 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 121 | conv = tf.nn.bias_add(conv, biases) 122 | 123 | return conv 124 | 125 | def deconv2d(input_, output_shape, 126 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 127 | name="deconv2d", with_w=False, 128 | init_bias=0.): 129 | with tf.variable_scope(name): 130 | # filter : [height, width, output_channels, in_channels] 131 | w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], 132 | initializer=tf.random_normal_initializer(stddev=stddev)) 133 | 134 | try: 135 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, 136 | strides=[1, d_h, d_w, 1]) 137 | 138 | # Support for versions of TensorFlow before 0.7.0 139 | except AttributeError: 140 | deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape, 141 | strides=[1, d_h, d_w, 1]) 142 | 143 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(init_bias)) 144 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 145 | 146 | if with_w: 147 | return deconv, w, biases 148 | else: 149 | return deconv 150 | 151 | def special_deconv2d(input_, output_shape, 152 | k_h=6, k_w=6, d_h=2, d_w=2, stddev=0.02, 153 | name="deconv2d", with_w=False, 154 | init_bias=0.): 155 | # designed to reduce padding and stride artifacts in the generator 156 | 157 | # If the following fail, it is hard to avoid grid pattern artifacts 158 | assert k_h % d_h == 0 159 | assert k_w % d_w == 0 160 | 161 | with tf.variable_scope(name): 162 | # filter : [height, width, output_channels, in_channels] 163 | w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], 164 | initializer=tf.random_normal_initializer(stddev=stddev)) 165 | 166 | def check_shape(h_size, im_size, stride): 167 | if h_size != (im_size + stride - 1) // stride: 168 | print( "Need h_size == (im_size + stride - 1) // stride") 169 | print( "h_size: ", h_size) 170 | print( "im_size: ", im_size) 171 | print( "stride: ", stride) 172 | print( "(im_size + stride - 1) / float(stride): ", (im_size + stride - 1) / float(stride)) 173 | raise ValueError() 174 | 175 | check_shape(int(input_.get_shape()[1]), output_shape[1] + k_h, d_h) 176 | check_shape(int(input_.get_shape()[2]), output_shape[2] + k_w, d_w) 177 | 178 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=[output_shape[0], 179 | output_shape[1] + k_h, output_shape[2] + k_w, output_shape[3]], 180 | strides=[1, d_h, d_w, 1]) 181 | deconv = tf.slice(deconv, [0, k_h // 2, k_w // 2, 0], output_shape) 182 | 183 | 184 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(init_bias)) 185 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 186 | 187 | if with_w: 188 | return deconv, w, biases 189 | else: 190 | return deconv 191 | 192 | def lrelu(x, leak=0.2, name="lrelu"): 193 | with tf.variable_scope(name): 194 | f1 = 0.5 * (1 + leak) 195 | f2 = 0.5 * (1 - leak) 196 | return f1 * x + f2 * abs(x) 197 | 198 | def sin_and_cos(x, name="ignored"): 199 | return tf.concat(len(x.get_shape()) - 1, [tf.sin(x), tf.cos(x)]) 200 | 201 | def maxout(x, k = 2): 202 | shape = [int(e) for e in x.get_shape()] 203 | ax = len(shape) 204 | ch = shape[-1] 205 | print('x',x,ch,k) 206 | assert ch % k == 0 207 | shape[-1] = ch // k 208 | shape.append(k) 209 | x = tf.reshape(x, shape) 210 | return tf.reduce_max(x, ax) 211 | 212 | def offset_maxout(x, k = 2): 213 | shape = [int(e) for e in x.get_shape()] 214 | ax = len(shape) 215 | ch = shape[-1] 216 | assert ch % k == 0 217 | shape[-1] = ch // k 218 | shape.append(k) 219 | x = tf.reshape(x, shape) 220 | ofs = rng.randn(1000, k).max(axis=1).mean() 221 | return tf.reduce_max(x, ax) - ofs 222 | 223 | def lrelu_sq(x): 224 | """ 225 | Concatenates lrelu and square 226 | """ 227 | dim = len(x.get_shape()) - 1 228 | return tf.concat(dim, [lrelu(x), tf.minimum(tf.abs(x), tf.square(x))]) 229 | 230 | def linear(input_, output_size, scope=None, mean=0., stddev=0.02, bias_start=0.0, with_w=False): 231 | shape = input_.get_shape().as_list() 232 | 233 | with tf.variable_scope(scope or "Linear"): 234 | matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, 235 | tf.random_normal_initializer(mean=mean, stddev=stddev)) 236 | bias = tf.get_variable("bias", [output_size], 237 | initializer=tf.constant_initializer(bias_start)) 238 | if with_w: 239 | # import ipdb; ipdb.set_trace() 240 | return tf.matmul(input_, matrix) + bias, matrix, bias 241 | else: 242 | return tf.matmul(input_, matrix) + bias 243 | 244 | @contextmanager 245 | def variables_on_cpu(): 246 | old_fn = tf.get_variable 247 | def new_fn(*args, **kwargs): 248 | with tf.device("/cpu:0"): 249 | return old_fn(*args, **kwargs) 250 | tf.get_variable = new_fn 251 | yield 252 | tf.get_variable = old_fn 253 | 254 | @contextmanager 255 | def variables_on_gpu0(): 256 | old_fn = tf.get_variable 257 | def new_fn(*args, **kwargs): 258 | with tf.device("/gpu:0"): 259 | return old_fn(*args, **kwargs) 260 | tf.get_variable = new_fn 261 | yield 262 | tf.get_variable = old_fn 263 | 264 | def avg_grads(tower_grads): 265 | """Calculate the average gradient for each shared variable across all towers. 266 | 267 | Note that this function provides a synchronization point across all towers. 268 | 269 | Args: 270 | tower_grads: List of lists of (gradient, variable) tuples. The outer list 271 | is over individual gradients. The inner list is over the gradient 272 | calculation for each tower. 273 | Returns: 274 | List of pairs of (gradient, variable) where the gradient has been averaged 275 | across all towers. 276 | """ 277 | average_grads = [] 278 | for grad_and_vars in zip(*tower_grads): 279 | # Note that each grad_and_vars looks like the following: 280 | # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) 281 | grads = [] 282 | for g, _ in grad_and_vars: 283 | # Add 0 dimension to the gradients to represent the tower. 284 | expanded_g = tf.expand_dims(g, 0) 285 | 286 | # Append on a 'tower' dimension which we will average over below. 287 | grads.append(expanded_g) 288 | 289 | # Average over the 'tower' dimension. 290 | grad = tf.concat(0, grads) 291 | grad = tf.reduce_mean(grad, 0) 292 | 293 | # Keep in mind that the Variables are redundant because they are shared 294 | # across towers. So .. we will just return the first tower's pointer to 295 | # the Variable. 296 | v = grad_and_vars[0][1] 297 | grad_and_var = (grad, v) 298 | average_grads.append(grad_and_var) 299 | return average_grads 300 | 301 | class batch_norm_second_half(object): 302 | """Code modification of http://stackoverflow.com/a/33950177 303 | 304 | """ 305 | def __init__(self, epsilon=1e-5, name="batch_norm"): 306 | with tf.variable_scope(name) as scope: 307 | self.epsilon = epsilon 308 | 309 | self.name=name 310 | 311 | def __call__(self, x): 312 | 313 | shape = x.get_shape().as_list() 314 | 315 | needs_reshape = len(shape) != 4 316 | if needs_reshape: 317 | orig_shape = shape 318 | if len(shape) == 2: 319 | x = tf.reshape(x, [shape[0], 1, 1, shape[1]]) 320 | elif len(shape) == 1: 321 | x = tf.reshape(x, [shape[0], 1, 1, 1]) 322 | else: 323 | assert False, shape 324 | shape = x.get_shape().as_list() 325 | 326 | with tf.variable_scope(self.name) as scope: 327 | self.gamma = tf.get_variable("gamma", [shape[-1]], 328 | initializer=tf.random_normal_initializer(1., 0.02)) 329 | self.beta = tf.get_variable("beta", [shape[-1]], 330 | initializer=tf.constant_initializer(0.)) 331 | 332 | second_half = tf.slice(x, [shape[0] // 2, 0, 0, 0], 333 | [shape[0] // 2, shape[1], shape[2], shape[3]]) 334 | 335 | self.mean, self.variance = tf.nn.moments(second_half, [0, 1, 2]) 336 | 337 | out = tf.nn.batch_norm_with_global_normalization( 338 | x, self.mean, self.variance, self.beta, self.gamma, self.epsilon, 339 | scale_after_normalization=True) 340 | if needs_reshape: 341 | out = tf.reshape(out, orig_shape) 342 | return out 343 | 344 | class batch_norm_first_half(object): 345 | """Code modification of http://stackoverflow.com/a/33950177 346 | 347 | """ 348 | def __init__(self, epsilon=1e-5, name="batch_norm"): 349 | with tf.variable_scope(name) as scope: 350 | self.epsilon = epsilon 351 | 352 | self.name=name 353 | 354 | def __call__(self, x): 355 | 356 | shape = x.get_shape().as_list() 357 | 358 | needs_reshape = len(shape) != 4 359 | if needs_reshape: 360 | orig_shape = shape 361 | if len(shape) == 2: 362 | x = tf.reshape(x, [shape[0], 1, 1, shape[1]]) 363 | elif len(shape) == 1: 364 | x = tf.reshape(x, [shape[0], 1, 1, 1]) 365 | else: 366 | assert False, shape 367 | shape = x.get_shape().as_list() 368 | 369 | with tf.variable_scope(self.name) as scope: 370 | self.gamma = tf.get_variable("gamma", [shape[-1]], 371 | initializer=tf.random_normal_initializer(1., 0.02)) 372 | self.beta = tf.get_variable("beta", [shape[-1]], 373 | initializer=tf.constant_initializer(0.)) 374 | 375 | first_half = tf.slice(x, [0, 0, 0, 0], 376 | [shape[0] // 2, shape[1], shape[2], shape[3]]) 377 | 378 | self.mean, self.variance = tf.nn.moments(first_half, [0, 1, 2]) 379 | 380 | out = tf.nn.batch_norm_with_global_normalization( 381 | x, self.mean, self.variance, self.beta, self.gamma, self.epsilon, 382 | scale_after_normalization=True) 383 | if needs_reshape: 384 | out = tf.reshape(out, orig_shape) 385 | return out 386 | 387 | def decayer(x, name="decayer"): 388 | with tf.variable_scope(name): 389 | scale = tf.get_variable("scale", [1], initializer=tf.constant_initializer(1.)) 390 | decay_scale = tf.get_variable("decay_scale", [1], initializer=tf.constant_initializer(1.)) 391 | relu = tf.nn.relu(x) 392 | return scale * relu / (1. + tf.abs(decay_scale) * tf.square(decay_scale)) 393 | 394 | def decayer2(x, name="decayer"): 395 | with tf.variable_scope(name): 396 | scale = tf.get_variable("scale", [int(x.get_shape()[-1])], initializer=tf.constant_initializer(1.)) 397 | decay_scale = tf.get_variable("decay_scale", [int(x.get_shape()[-1])], initializer=tf.constant_initializer(1.)) 398 | relu = tf.nn.relu(x) 399 | return scale * relu / (1. + tf.abs(decay_scale) * tf.square(decay_scale)) 400 | 401 | class batch_norm_cross(object): 402 | def __init__(self, epsilon=1e-5, name="batch_norm"): 403 | with tf.variable_scope(name) as scope: 404 | self.epsilon = epsilon 405 | self.name=name 406 | 407 | def __call__(self, x): 408 | 409 | shape = x.get_shape().as_list() 410 | 411 | needs_reshape = len(shape) != 4 412 | if needs_reshape: 413 | orig_shape = shape 414 | if len(shape) == 2: 415 | x = tf.reshape(x, [shape[0], 1, 1, shape[1]]) 416 | elif len(shape) == 1: 417 | x = tf.reshape(x, [shape[0], 1, 1, 1]) 418 | else: 419 | assert False, shape 420 | shape = x.get_shape().as_list() 421 | 422 | with tf.variable_scope(self.name) as scope: 423 | self.gamma0 = tf.get_variable("gamma0", [shape[-1] // 2], 424 | initializer=tf.random_normal_initializer(1., 0.02)) 425 | self.beta0 = tf.get_variable("beta0", [shape[-1] // 2], 426 | initializer=tf.constant_initializer(0.)) 427 | self.gamma1 = tf.get_variable("gamma1", [shape[-1] // 2], 428 | initializer=tf.random_normal_initializer(1., 0.02)) 429 | self.beta1 = tf.get_variable("beta1", [shape[-1] // 2], 430 | initializer=tf.constant_initializer(0.)) 431 | 432 | ch0 = tf.slice(x, [0, 0, 0, 0], 433 | [shape[0], shape[1], shape[2], shape[3] // 2]) 434 | ch1 = tf.slice(x, [0, 0, 0, shape[3] // 2], 435 | [shape[0], shape[1], shape[2], shape[3] // 2]) 436 | 437 | ch0b0 = tf.slice(ch0, [0, 0, 0, 0], 438 | [shape[0] // 2, shape[1], shape[2], shape[3] // 2]) 439 | 440 | ch1b1 = tf.slice(ch1, [shape[0] // 2, 0, 0, 0], 441 | [shape[0] // 2, shape[1], shape[2], shape[3] // 2]) 442 | 443 | 444 | ch0_mean, ch0_variance = tf.nn.moments(ch0b0, [0, 1, 2]) 445 | ch1_mean, ch1_variance = tf.nn.moments(ch1b1, [0, 1, 2]) 446 | 447 | ch0 = tf.nn.batch_norm_with_global_normalization( 448 | ch0, ch0_mean, ch0_variance, self.beta0, self.gamma0, self.epsilon, 449 | scale_after_normalization=True) 450 | 451 | ch1 = tf.nn.batch_norm_with_global_normalization( 452 | ch1, ch1_mean, ch1_variance, self.beta1, self.gamma1, self.epsilon, 453 | scale_after_normalization=True) 454 | 455 | out = tf.concat(3, [ch0, ch1]) 456 | 457 | if needs_reshape: 458 | out = tf.reshape(out, orig_shape) 459 | 460 | return out 461 | 462 | def constrained_conv2d(input_, output_dim, 463 | k_h=6, k_w=6, d_h=2, d_w=2, stddev=0.02, 464 | name="conv2d"): 465 | assert k_h % d_h == 0 466 | assert k_w % d_w == 0 467 | # constrained to have stride be a factor of kernel width 468 | # this is intended to reduce convolution artifacts 469 | with tf.variable_scope(name): 470 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 471 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 472 | 473 | # This is meant to reduce boundary artifacts 474 | padded = tf.pad(input_, [[0, 0], 475 | [k_h-1, 0], 476 | [k_w-1, 0], 477 | [0, 0]]) 478 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 479 | 480 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 481 | conv = tf.nn.bias_add(conv, biases) 482 | 483 | return conv 484 | 485 | def masked_relu(x, name="ignored"): 486 | shape = [int(e) for e in x.get_shape()] 487 | prefix = [0] * (len(shape) - 1) 488 | most = shape[:-1] 489 | assert shape[-1] % 2 == 0 490 | half = shape[-1] // 2 491 | first_half = tf.slice(x, prefix + [0], most + [half]) 492 | second_half = tf.slice(x, prefix + [half], most + [half]) 493 | return tf.nn.relu(first_half) * tf.nn.sigmoid(second_half) 494 | -------------------------------------------------------------------------------- /examples/shared/resize_image_patch.py: -------------------------------------------------------------------------------- 1 | 2 | # coding: utf-8 3 | 4 | # In[1]: 5 | 6 | from tensorflow.python.ops import image_ops 7 | from tensorflow.python.ops import math_ops 8 | from tensorflow.python.ops import array_ops 9 | from tensorflow.python import ops 10 | 11 | 12 | # In[2]: 13 | 14 | def crop_to_bounding_box(image, offset_height, offset_width, target_height, 15 | target_width, dynamic_shape=False): 16 | """Crops an image to a specified bounding box. 17 | 18 | This op cuts a rectangular part out of `image`. The top-left corner of the 19 | returned image is at `offset_height, offset_width` in `image`, and its 20 | lower-right corner is at 21 | `offset_height + target_height, offset_width + target_width`. 22 | 23 | Args: 24 | image: 3-D tensor with shape `[height, width, channels]` 25 | offset_height: Vertical coordinate of the top-left corner of the result in 26 | the input. 27 | offset_width: Horizontal coordinate of the top-left corner of the result in 28 | the input. 29 | target_height: Height of the result. 30 | target_width: Width of the result. 31 | dynamic_shape: Whether the input image has undertermined shape. If set to 32 | `True`, shape information will be retrieved at run time. Default to 33 | `False`. 34 | 35 | Returns: 36 | 3-D tensor of image with shape `[target_height, target_width, channels]` 37 | 38 | Raises: 39 | ValueError: If the shape of `image` is incompatible with the `offset_*` or 40 | `target_*` arguments, and `dynamic_shape` is set to `False`. 41 | """ 42 | image = ops.convert_to_tensor(image, name='image') 43 | _Check3DImage(image, require_static=(not dynamic_shape)) 44 | height, width, _ = _ImageDimensions(image, dynamic_shape=dynamic_shape) 45 | 46 | if not dynamic_shape: 47 | if offset_width < 0: 48 | raise ValueError('offset_width must be >= 0.') 49 | if offset_height < 0: 50 | raise ValueError('offset_height must be >= 0.') 51 | 52 | if width < (target_width + offset_width): 53 | raise ValueError('width must be >= target + offset.') 54 | if height < (target_height + offset_height): 55 | raise ValueError('height must be >= target + offset.') 56 | 57 | cropped = array_ops.slice(image, 58 | array_ops.pack([offset_height, offset_width, 0]), 59 | array_ops.pack([target_height, target_width, -1])) 60 | 61 | return cropped 62 | 63 | 64 | # In[3]: 65 | 66 | def pad_to_bounding_box(image, offset_height, offset_width, target_height, 67 | target_width, dynamic_shape=False): 68 | """Pad `image` with zeros to the specified `height` and `width`. 69 | 70 | Adds `offset_height` rows of zeros on top, `offset_width` columns of 71 | zeros on the left, and then pads the image on the bottom and right 72 | with zeros until it has dimensions `target_height`, `target_width`. 73 | 74 | This op does nothing if `offset_*` is zero and the image already has size 75 | `target_height` by `target_width`. 76 | 77 | Args: 78 | image: 3-D tensor with shape `[height, width, channels]` 79 | offset_height: Number of rows of zeros to add on top. 80 | offset_width: Number of columns of zeros to add on the left. 81 | target_height: Height of output image. 82 | target_width: Width of output image. 83 | dynamic_shape: Whether the input image has undertermined shape. If set to 84 | `True`, shape information will be retrieved at run time. Default to 85 | `False`. 86 | 87 | Returns: 88 | 3-D tensor of shape `[target_height, target_width, channels]` 89 | Raises: 90 | ValueError: If the shape of `image` is incompatible with the `offset_*` or 91 | `target_*` arguments, and `dynamic_shape` is set to `False`. 92 | """ 93 | image = ops.convert_to_tensor(image, name='image') 94 | _Check3DImage(image, require_static=(not dynamic_shape)) 95 | height, width, depth = _ImageDimensions(image, dynamic_shape=dynamic_shape) 96 | 97 | after_padding_width = target_width - offset_width - width 98 | after_padding_height = target_height - offset_height - height 99 | 100 | if not dynamic_shape: 101 | if target_width < width: 102 | raise ValueError('target_width must be >= width') 103 | if target_height < height: 104 | raise ValueError('target_height must be >= height') 105 | 106 | if after_padding_width < 0: 107 | raise ValueError('target_width not possible given ' 108 | 'offset_width and image width') 109 | if after_padding_height < 0: 110 | raise ValueError('target_height not possible given ' 111 | 'offset_height and image height') 112 | 113 | # Do not pad on the depth dimensions. 114 | if (dynamic_shape or offset_width or offset_height or 115 | after_padding_width or after_padding_height): 116 | paddings = array_ops.reshape( 117 | array_ops.pack([offset_height, after_padding_height, 118 | offset_width, after_padding_width, 119 | 0, 0]), 120 | [3, 2]) 121 | padded = array_ops.pad(image, paddings) 122 | if not dynamic_shape: 123 | padded.set_shape([target_height, target_width, depth]) 124 | else: 125 | padded = image 126 | 127 | return padded 128 | 129 | 130 | # In[4]: 131 | 132 | def resize_image_with_crop_or_pad(image, target_height, target_width, 133 | dynamic_shape=False): 134 | """Crops and/or pads an image to a target width and height. 135 | 136 | Resizes an image to a target width and height by either centrally 137 | cropping the image or padding it evenly with zeros. 138 | 139 | If `width` or `height` is greater than the specified `target_width` or 140 | `target_height` respectively, this op centrally crops along that dimension. 141 | If `width` or `height` is smaller than the specified `target_width` or 142 | `target_height` respectively, this op centrally pads with 0 along that 143 | dimension. 144 | 145 | Args: 146 | image: 3-D tensor of shape [height, width, channels] 147 | target_height: Target height. 148 | target_width: Target width. 149 | dynamic_shape: Whether the input image has undertermined shape. If set to 150 | `True`, shape information will be retrieved at run time. Default to 151 | `False`. 152 | 153 | Raises: 154 | ValueError: if `target_height` or `target_width` are zero or negative. 155 | 156 | Returns: 157 | Cropped and/or padded image of shape 158 | `[target_height, target_width, channels]` 159 | """ 160 | image = ops.convert_to_tensor(image, name='image') 161 | _Check3DImage(image, require_static=(not dynamic_shape)) 162 | original_height, original_width, _ = _ImageDimensions(image, dynamic_shape=dynamic_shape) 163 | 164 | if target_width <= 0: 165 | raise ValueError('target_width must be > 0.') 166 | if target_height <= 0: 167 | raise ValueError('target_height must be > 0.') 168 | 169 | if dynamic_shape: 170 | max_ = math_ops.maximum 171 | min_ = math_ops.minimum 172 | else: 173 | max_ = max 174 | min_ = min 175 | 176 | width_diff = target_width - original_width 177 | offset_crop_width = max_(-width_diff // 2, 0) 178 | offset_pad_width = max_(width_diff // 2, 0) 179 | 180 | height_diff = target_height - original_height 181 | offset_crop_height = max_(-height_diff // 2, 0) 182 | offset_pad_height = max_(height_diff // 2, 0) 183 | 184 | # Maybe crop if needed. 185 | cropped = crop_to_bounding_box(image, offset_crop_height, offset_crop_width, 186 | min_(target_height, original_height), 187 | min_(target_width, original_width), 188 | dynamic_shape=dynamic_shape) 189 | 190 | # Maybe pad if needed. 191 | resized = pad_to_bounding_box(cropped, offset_pad_height, offset_pad_width, 192 | target_height, target_width, 193 | dynamic_shape=dynamic_shape) 194 | 195 | if resized.get_shape().ndims is None: 196 | raise ValueError('resized contains no shape.') 197 | if not resized.get_shape()[0].is_compatible_with(target_height): 198 | raise ValueError('resized height is not correct.') 199 | if not resized.get_shape()[1].is_compatible_with(target_width): 200 | raise ValueError('resized width is not correct.') 201 | return resized 202 | 203 | 204 | # In[5]: 205 | 206 | def _ImageDimensions(images, dynamic_shape=False): 207 | """Returns the dimensions of an image tensor. 208 | Args: 209 | images: 4-D Tensor of shape [batch, height, width, channels] 210 | dynamic_shape: Whether the input image has undertermined shape. If set to 211 | `True`, shape information will be retrieved at run time. Default to 212 | `False`. 213 | 214 | Returns: 215 | list of integers [batch, height, width, channels] 216 | """ 217 | # A simple abstraction to provide names for each dimension. This abstraction 218 | # should make it simpler to switch dimensions in the future (e.g. if we ever 219 | # want to switch height and width.) 220 | if dynamic_shape: 221 | return array_ops.unpack(array_ops.shape(images)) 222 | else: 223 | return images.get_shape().as_list() 224 | 225 | 226 | # In[6]: 227 | 228 | def _Check3DImage(image, require_static=True): 229 | """Assert that we are working with properly shaped image. 230 | Args: 231 | image: 3-D Tensor of shape [height, width, channels] 232 | require_static: If `True`, requires that all dimensions of `image` are 233 | known and non-zero. 234 | 235 | Raises: 236 | ValueError: if image.shape is not a [3] vector. 237 | """ 238 | try: 239 | image_shape = image.get_shape().with_rank(3) 240 | except ValueError: 241 | raise ValueError('\'image\' must be three-dimensional.') 242 | if require_static and not image_shape.is_fully_defined(): 243 | raise ValueError('\'image\' must be fully defined.') 244 | if any(x == 0 for x in image_shape): 245 | raise ValueError('all dims of \'image.shape\' must be > 0: %s' % 246 | image_shape) 247 | 248 | 249 | # In[7]: 250 | 251 | image_ops.resize_image_with_crop_or_pad = resize_image_with_crop_or_pad 252 | #image_ops.crop_to_bounding_box = crop_to_bounding_box 253 | #image_ops.pad_to_bounding_box = pad_to_bounding_box 254 | #image_ops._ImageDimensions = _ImageDimensions 255 | #image_ops._Check3DImage = _Check3DImage 256 | 257 | -------------------------------------------------------------------------------- /examples/shared/util.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | tensors = {} 3 | def set_tensor(name, tensor): 4 | tensors[name]=tensor 5 | 6 | def get_tensor(name, graph=tf.get_default_graph(), isOperation=False): 7 | return tensors[name]# || graph.as_graph_element(name) 8 | 9 | 10 | -------------------------------------------------------------------------------- /examples/shared/variational_autoencoder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | # From https://jmetzen.github.io/2015-11-27/vae.html 7 | 8 | def xavier_init(fan_in, fan_out, constant=1): 9 | """ Xavier initialization of network weights""" 10 | # https://stackoverflow.com/questions/33640581/how-to-do-xavier-initialization-on-tensorflow 11 | low = -constant*np.sqrt(6.0/(fan_in + fan_out)) 12 | high = constant*np.sqrt(6.0/(fan_in + fan_out)) 13 | return tf.random_uniform((fan_in, fan_out), 14 | minval=low, maxval=high, 15 | dtype=tf.float32) 16 | 17 | class VariationalAutoencoder(object): 18 | """ Variation Autoencoder (VAE) with an sklearn-like interface implemented using TensorFlow. 19 | 20 | This implementation uses probabilistic encoders and decoders using Gaussian 21 | distributions and realized by multi-layer perceptrons. The VAE can be learned 22 | end-to-end. 23 | 24 | See "Auto-Encoding Variational Bayes" by Kingma and Welling for more details. 25 | """ 26 | def __init__(self, network_architecture, transfer_fct=tf.nn.softplus, 27 | learning_rate=0.001, batch_size=100): 28 | self.network_architecture = network_architecture 29 | self.transfer_fct = transfer_fct 30 | self.learning_rate = learning_rate 31 | self.batch_size = batch_size 32 | 33 | # tf Graph input 34 | self.x = tf.placeholder(tf.float32, [None, network_architecture["n_input"]]) 35 | 36 | # Create autoencoder network 37 | self._create_network() 38 | # Define loss function based variational upper-bound and 39 | # corresponding optimizer 40 | self._create_loss_optimizer() 41 | 42 | # Initializing the tensor flow variables 43 | init = tf.initialize_all_variables() 44 | 45 | # Launch the session 46 | self.sess = tf.InteractiveSession() 47 | self.sess.run(init) 48 | 49 | def _create_network(self): 50 | # Initialize autoencode network weights and biases 51 | network_weights = self._initialize_weights(**self.network_architecture) 52 | 53 | # Use recognition network to determine mean and 54 | # (log) variance of Gaussian distribution in latent 55 | # space 56 | self.z_mean, self.z_log_sigma_sq = \ 57 | self._recognition_network(network_weights["weights_recog"], 58 | network_weights["biases_recog"]) 59 | 60 | # Draw one sample z from Gaussian distribution 61 | n_z = self.network_architecture["n_z"] 62 | eps = tf.random_normal((self.batch_size, n_z), 0, 1, 63 | dtype=tf.float32) 64 | # z = mu + sigma*epsilon 65 | self.z = tf.add(self.z_mean, 66 | tf.mul(tf.sqrt(tf.exp(self.z_log_sigma_sq)), eps)) 67 | 68 | # Use generator to determine mean of 69 | # Bernoulli distribution of reconstructed input 70 | self.x_reconstr_mean = \ 71 | self._generator_network(network_weights["weights_gener"], 72 | network_weights["biases_gener"]) 73 | 74 | def _initialize_weights(self, n_hidden_recog_1, n_hidden_recog_2, 75 | n_hidden_gener_1, n_hidden_gener_2, 76 | n_input, n_z, **extra): 77 | all_weights = dict() 78 | all_weights['weights_recog'] = { 79 | 'h1': tf.Variable(xavier_init(n_input, n_hidden_recog_1)), 80 | 'h2': tf.Variable(xavier_init(n_hidden_recog_1, n_hidden_recog_2)), 81 | 'out_mean': tf.Variable(xavier_init(n_hidden_recog_2, n_z)), 82 | 'out_log_sigma': tf.Variable(xavier_init(n_hidden_recog_2, n_z))} 83 | all_weights['biases_recog'] = { 84 | 'b1': tf.Variable(tf.zeros([n_hidden_recog_1], dtype=tf.float32)), 85 | 'b2': tf.Variable(tf.zeros([n_hidden_recog_2], dtype=tf.float32)), 86 | 'out_mean': tf.Variable(tf.zeros([n_z], dtype=tf.float32)), 87 | 'out_log_sigma': tf.Variable(tf.zeros([n_z], dtype=tf.float32))} 88 | all_weights['weights_gener'] = { 89 | 'h1': tf.Variable(xavier_init(n_z, n_hidden_gener_1)), 90 | 'h2': tf.Variable(xavier_init(n_hidden_gener_1, n_hidden_gener_2)), 91 | 'out_mean': tf.Variable(xavier_init(n_hidden_gener_2, n_input)), 92 | 'out_log_sigma': tf.Variable(xavier_init(n_hidden_gener_2, n_input))} 93 | all_weights['biases_gener'] = { 94 | 'b1': tf.Variable(tf.zeros([n_hidden_gener_1], dtype=tf.float32)), 95 | 'b2': tf.Variable(tf.zeros([n_hidden_gener_2], dtype=tf.float32)), 96 | 'out_mean': tf.Variable(tf.zeros([n_input], dtype=tf.float32)), 97 | 'out_log_sigma': tf.Variable(tf.zeros([n_input], dtype=tf.float32))} 98 | return all_weights 99 | 100 | def _recognition_network(self, weights, biases): 101 | # Generate probabilistic encoder (recognition network), which 102 | # maps inputs onto a normal distribution in latent space. 103 | # The transformation is parametrized and can be learned. 104 | layer_1 = self.transfer_fct(tf.add(tf.matmul(self.x, weights['h1']), 105 | biases['b1'])) 106 | layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1, weights['h2']), 107 | biases['b2'])) 108 | z_mean = tf.add(tf.matmul(layer_2, weights['out_mean']), 109 | biases['out_mean']) 110 | z_log_sigma_sq = \ 111 | tf.add(tf.matmul(layer_2, weights['out_log_sigma']), 112 | biases['out_log_sigma']) 113 | return (z_mean, z_log_sigma_sq) 114 | 115 | def _generator_network(self, weights, biases): 116 | # Generate probabilistic decoder (decoder network), which 117 | # maps points in latent space onto a Bernoulli distribution in data space. 118 | # The transformation is parametrized and can be learned. 119 | layer_1 = self.transfer_fct(tf.add(tf.matmul(self.z, weights['h1']), 120 | biases['b1'])) 121 | layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1, weights['h2']), 122 | biases['b2'])) 123 | x_reconstr_mean = \ 124 | tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['out_mean']), 125 | biases['out_mean'])) 126 | return x_reconstr_mean 127 | 128 | def _create_loss_optimizer(self): 129 | # The loss is composed of two terms: 130 | # 1.) The reconstruction loss (the negative log probability 131 | # of the input under the reconstructed Bernoulli distribution 132 | # induced by the decoder in the data space). 133 | # This can be interpreted as the number of "nats" required 134 | # for reconstructing the input when the activation in latent 135 | # is given. 136 | # Adding 1e-10 to avoid evaluatio of log(0.0) 137 | reconstr_loss = \ 138 | -tf.reduce_sum(self.x * tf.log(1e-10 + self.x_reconstr_mean) 139 | + (1-self.x) * tf.log(1e-10 + 1 - self.x_reconstr_mean), 140 | 1) 141 | # 2.) The latent loss, which is defined as the Kullback Leibler divergence 142 | ## between the distribution in latent space induced by the encoder on 143 | # the data and some prior. This acts as a kind of regularizer. 144 | # This can be interpreted as the number of "nats" required 145 | # for transmitting the the latent space distribution given 146 | # the prior. 147 | latent_loss = -0.5 * tf.reduce_sum(1 + self.z_log_sigma_sq 148 | - tf.square(self.z_mean) 149 | - tf.exp(self.z_log_sigma_sq), 1) 150 | self.cost = tf.reduce_mean(reconstr_loss + latent_loss) # average over batch 151 | # Use ADAM optimizer 152 | self.optimizer = \ 153 | tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.cost) 154 | 155 | def partial_fit(self, X): 156 | """Train model based on mini-batch of input data. 157 | 158 | Return cost of mini-batch. 159 | """ 160 | opt, cost = self.sess.run((self.optimizer, self.cost), 161 | feed_dict={self.x: X}) 162 | return cost 163 | 164 | def transform(self, X): 165 | """Transform data by mapping it into the latent space.""" 166 | # Note: This maps to mean of distribution, we could alternatively 167 | # sample from Gaussian distribution 168 | return self.sess.run(self.z_mean, feed_dict={self.x: X}) 169 | 170 | def generate(self, z_mu=None): 171 | """ Generate data by sampling from latent space. 172 | 173 | If z_mu is not None, data for this point in latent space is 174 | generated. Otherwise, z_mu is drawn from prior in latent 175 | space. 176 | """ 177 | if z_mu is None: 178 | z_mu = np.random.normal(size=self.network_architecture["n_z"]) 179 | # Note: This maps to mean of distribution, we could alternatively 180 | # sample from Gaussian distribution 181 | return self.sess.run(self.x_reconstr_mean, 182 | feed_dict={self.z: z_mu}) 183 | 184 | def reconstruct(self, X): 185 | """ Use VAE to reconstruct given data. """ 186 | return self.sess.run(self.x_reconstr_mean, 187 | feed_dict={self.x: X}) 188 | -------------------------------------------------------------------------------- /examples/track.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This example is integrated with Aymeric's wonderful logistic regression example. 3 | 4 | --- 5 | 6 | A logistic regression learning algorithm example using TensorFlow library. 7 | This example is using the MNIST database of handwritten digits 8 | (http://yann.lecun.com/exdb/mnist/) 9 | Author: Aymeric Damien 10 | Project: https://github.com/aymericdamien/TensorFlow-Examples/ 11 | ''' 12 | 13 | import tensorflow as tf 14 | import hyperchamber as hc 15 | 16 | # Import MINST data 17 | from tensorflow.examples.tutorials.mnist import input_data 18 | mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) 19 | 20 | # Parameters 21 | # These are what hyperchamber stores and permutes through for you. 22 | hc.set('learning_rate', [0.01, 0.02, 0.03]) 23 | config = hc.random_config() # => {learning_rate: 0.01 or 0.02 or 0.03} 24 | 25 | training_epochs = 25 26 | batch_size = 100 27 | display_step = 1 28 | 29 | # tf Graph Input 30 | x = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 31 | y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes 32 | 33 | # Set model weights 34 | W = tf.Variable(tf.zeros([784, 10])) 35 | b = tf.Variable(tf.zeros([10])) 36 | 37 | 38 | 39 | # Construct model 40 | pred = tf.nn.softmax(tf.matmul(x, W) + b) # Softmax 41 | 42 | # Minimize error using cross entropy 43 | cost = tf.reduce_mean(-tf.reduce_sum(y*tf.log(pred), reduction_indices=1)) 44 | # Gradient Descent 45 | optimizer = tf.train.GradientDescentOptimizer(config['learning_rate']).minimize(cost) 46 | 47 | # Initializing the variables 48 | init = tf.initialize_all_variables() 49 | 50 | # Launch the graph 51 | with tf.Session() as sess: 52 | sess.run(init) 53 | 54 | # Training cycle 55 | for epoch in range(training_epochs): 56 | avg_cost = 0. 57 | total_batch = int(mnist.train.num_examples/batch_size) 58 | # Loop over all batches 59 | for i in range(total_batch): 60 | batch_xs, batch_ys = mnist.train.next_batch(batch_size) 61 | # Run optimization op (backprop) and cost op (to get loss value) 62 | _, c = sess.run([optimizer, cost], feed_dict={x: batch_xs, 63 | y: batch_ys}) 64 | # Compute average loss 65 | avg_cost += c / total_batch 66 | # Display logs per epoch step 67 | if (epoch+1) % display_step == 0: 68 | print("Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost)) 69 | 70 | print("Optimization Finished!") 71 | 72 | # Test model 73 | correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) 74 | # Calculate accuracy 75 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 76 | print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels})) 77 | print("Used learning rate:", config['learning_rate']) 78 | -------------------------------------------------------------------------------- /examples/vae-mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | from shared.variational_autoencoder import * 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | from scipy.misc import imsave 9 | 10 | import hyperchamber as hc 11 | 12 | hc.set("model", "255bits/vae-mnist") 13 | 14 | mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 15 | n_samples = mnist.train.num_examples 16 | 17 | # only works for n_z=2 18 | def visualize(config, vae): 19 | if(config['n_z'] != 2): 20 | print("Skipping visuals since n_z is not 2") 21 | return 22 | nx = ny = 20 23 | x_values = np.linspace(-3, 3, nx) 24 | y_values = np.linspace(-3, 3, ny) 25 | 26 | canvas = np.empty((28*ny, 28*nx)) 27 | for i, yi in enumerate(x_values): 28 | for j, xi in enumerate(y_values): 29 | z_mu = np.array([[xi, yi]]) 30 | x_mean = vae.generate(np.tile(z_mu, [config['batch_size'], 1])) 31 | canvas[(nx-i-1)*28:(nx-i)*28, j*28:(j+1)*28] = x_mean[0].reshape(28, 28) 32 | 33 | plt.figure(figsize=(8, 10)) 34 | Xi, Yi = np.meshgrid(x_values, y_values) 35 | plt.imshow(canvas, origin="upper") 36 | plt.tight_layout() 37 | 38 | img = "samples/2d-visualization.png" 39 | plt.savefig(img) 40 | hc.io.sample(config, [{"label": "2d visualization", "image": img}]) 41 | 42 | def train(config, vae, learning_rate=0.001, 43 | training_epochs=10, display_step=5): 44 | 45 | batch_size=config['batch_size'] 46 | # Training cycle 47 | for epoch in range(training_epochs): 48 | avg_cost = 0. 49 | total_batch = int(n_samples / batch_size) 50 | # Loop over all batches 51 | for i in range(total_batch): 52 | batch_xs, _ = mnist.train.next_batch(batch_size) 53 | 54 | # Fit training using batch data 55 | cost = vae.partial_fit(batch_xs) 56 | # Compute average loss 57 | avg_cost += cost / n_samples * batch_size 58 | 59 | # Display logs per epoch step 60 | if epoch % display_step == 0: 61 | print("Epoch:", '%04d' % (epoch+1), \ 62 | "cost=", "{:.9f}".format(avg_cost)) 63 | return avg_cost 64 | 65 | def sample(config, vae): 66 | x_sample = mnist.test.next_batch(100)[0] 67 | x_reconstruct = vae.reconstruct(x_sample) 68 | 69 | plt.figure(figsize=(8, 12)) 70 | for i in range(5): 71 | 72 | plt.subplot(5, 2, 2*i + 1) 73 | plt.imshow(x_sample[i].reshape(28, 28), vmin=0, vmax=1) 74 | plt.title("Test input") 75 | plt.colorbar() 76 | plt.subplot(5, 2, 2*i + 2) 77 | plt.imshow(x_reconstruct[i].reshape(28, 28), vmin=0, vmax=1) 78 | plt.title("Reconstruction") 79 | plt.colorbar() 80 | plt.tight_layout() 81 | img = "samples/reconstruction.png" 82 | plt.savefig(img) 83 | hc.io.sample(config, [{"label": "Reconstruction", "image": img}]) 84 | 85 | hc.permute.set("learning_rate", list(np.linspace(0.0001, 0.003, num=10, dtype=np.float32))) 86 | hc.permute.set("n_hidden_recog_1", list(np.linspace(100, 1000, num=10, dtype=np.int32))) 87 | hc.permute.set("n_hidden_recog_2", list(np.linspace(100, 1000, num=10, dtype=np.int32))) 88 | hc.permute.set("n_hidden_gener_1", list(np.linspace(100, 1000, num=10, dtype=np.int32))) 89 | hc.permute.set("n_hidden_gener_2", list(np.linspace(100, 1000, num=10, dtype=np.int32))) 90 | 91 | hc.set("n_input", 784) # MNIST data input (img shape: 28*28) 92 | hc.permute.set("n_z", [1,2,4,8,16,32,64,128]) # dimensionality of latent space 93 | hc.set('batch_size', 100) 94 | hc.permute.set("transfer_fct", [tf.tanh, tf.nn.elu, tf.nn.relu, tf.nn.relu6, tf.nn.softplus, tf.nn.softsign]); 95 | 96 | hc.set("epochs", 10) 97 | 98 | print("hypersearch space", hc.count_configs()) 99 | for config in hc.configs(10000): 100 | print("Config", config) 101 | vae = VariationalAutoencoder(config, 102 | learning_rate=config['learning_rate'], 103 | transfer_fct=config['transfer_fct']) 104 | cost = 1000 105 | for i in range(config['epochs']): 106 | cost = train(config, vae, training_epochs=1) 107 | sample(config, vae) 108 | hc.io.measure(config, {"rank": cost}) 109 | visualize(config, vae) 110 | -------------------------------------------------------------------------------- /hyperchamber/__init__.py: -------------------------------------------------------------------------------- 1 | import hyperchamber.io as io 2 | 3 | from hyperchamber.selector import * 4 | 5 | import importlib 6 | 7 | 8 | default_selector = Selector() 9 | 10 | def set(key, value): 11 | """Sets a hyperparameter. Can be used to set an array of hyperparameters.""" 12 | global default_selector 13 | return default_selector.set(key, value) 14 | 15 | def count_configs(): 16 | global default_selector 17 | return default_selector.count_configs() 18 | 19 | def get_config_value(k, i): 20 | """Gets the ith config value for k. e.g. get_config_value('x', 1)""" 21 | global default_selector 22 | return default_selector.get_config_value(k, i) 23 | 24 | def configs(max_configs=1, offset=None, serial=False, create_uuid=True): 25 | """Generate max configs, each one a dictionary. e.g. [{'x': 1}] 26 | 27 | Will also add a config UUID, useful for tracking configs. 28 | You can turn this off by passing create_uuid=False. 29 | """ 30 | global default_selector 31 | return default_selector.configs(max_configs, offset, serial, create_uuid) 32 | 33 | def config_at(i): 34 | """Gets the ith config""" 35 | global default_selector 36 | return default_selector.config_at(i) 37 | 38 | def random_config(): 39 | global default_selector 40 | return default_selector.random_config() 41 | 42 | def reset(): 43 | """Reset the hyperchamber variables""" 44 | global default_selector 45 | return default_selector.reset() 46 | 47 | def top(sort_by): 48 | """Get the best results according to your custom sort method.""" 49 | global default_selector 50 | return default_selector.top(sort_by) 51 | 52 | def record(config, result): 53 | """Record the results of a config.""" 54 | global default_selector 55 | return default_selector.record(config, result) 56 | 57 | def load(filename): 58 | """Loads a config from disk""" 59 | global default_selector 60 | return default_selector.load(filename) 61 | 62 | def load_or_create_config(filename, config=None): 63 | """Loads a config from disk. Defaults to a random config if none is specified""" 64 | global default_selector 65 | return default_selector.load_or_create_config(filename, config) 66 | 67 | def save(filename, config): 68 | """Loads a config from disk""" 69 | global default_selector 70 | return default_selector.save(filename, config) 71 | 72 | 73 | # This looks up a function by name. 74 | def get_function(name): 75 | if not isinstance(name, str): 76 | return name 77 | namespaced_method = name.split(":")[1] 78 | method = namespaced_method.split(".")[-1] 79 | namespace = ".".join(namespaced_method.split(".")[0:-1]) 80 | return getattr(importlib.import_module(namespace),method) 81 | 82 | # Take a config and replace any string starting with 'function:' with a function lookup. 83 | def lookup_functions(config): 84 | for key, value in config.items(): 85 | if(isinstance(value, str) and value.startswith("function:")): 86 | config[key]=get_function(value) 87 | if(isinstance(value, list) and len(value) > 0 and isinstance(value[0],str) and value[0].startswith("function:")): 88 | config[key]=[get_function(v) for v in value] 89 | 90 | return config 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /hyperchamber/config.py: -------------------------------------------------------------------------------- 1 | # reference http://stackoverflow.com/questions/2352181/how-to-use-a-dot-to-access-members-of-dictionary 2 | class Config(dict): 3 | def __init__(self, *args, **kwargs): 4 | super(Config, self).__init__(*args, **kwargs) 5 | for arg in args: 6 | if isinstance(arg, dict): 7 | for k, v in arg.items(): 8 | self[k] = v 9 | 10 | if kwargs: 11 | for k, v in kwargs.items(): 12 | self[k] = v 13 | 14 | def __getattr__(self, attr): 15 | if attr in self: 16 | return self.get(attr) 17 | return None 18 | 19 | def __setattr__(self, key, value): 20 | self.__setitem__(key, value) 21 | 22 | def __setitem__(self, key, value): 23 | super(Config, self).__setitem__(key, value) 24 | self.__dict__.update({key: value}) 25 | 26 | def __delattr__(self, item): 27 | self.__delitem__(item) 28 | 29 | def __delitem__(self, key): 30 | super(Config, self).__delitem__(key) 31 | del self.__dict__[key] 32 | 33 | def __getstate__(self): 34 | return dict(self) 35 | 36 | def __setstate__(self, d): 37 | self.__dict__ = d 38 | -------------------------------------------------------------------------------- /hyperchamber/io/__init__.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import sys 4 | import os 5 | import uuid 6 | 7 | from json import JSONEncoder 8 | 9 | class MissingHCKeyException(Exception): 10 | pass 11 | 12 | # for function serialization 13 | class HCEncoder(JSONEncoder): 14 | def default(self, o): 15 | if(hasattr(o, '__call__')): # is function 16 | return "function:" +o.__module__+"."+o.__name__ 17 | else: 18 | try: 19 | return o.__dict__ 20 | except AttributeError: 21 | try: 22 | return str(o) 23 | except AttributeError: 24 | return super(o) 25 | 26 | 27 | def get_api_path(end): 28 | return "https://hyperchamber.255bits.com/api/v1/"+end 29 | 30 | def get_headers(no_content_type=False): 31 | if("HC_API_KEY" not in os.environ): 32 | raise MissingHCKeyException("hyperchamber.io api key needed. export HC_API_KEY='...'"); 33 | apikey = os.environ["HC_API_KEY"] 34 | if(no_content_type): 35 | return { 36 | 'apikey': apikey 37 | } 38 | return { 39 | 'Content-Type': 'application/json', 40 | 'apikey': apikey 41 | } 42 | 43 | def sample(config, samples): 44 | """Upload a series of samples. Each sample has keys 'image' and 'label'. 45 | Images are ignored if the rate limit is hit.""" 46 | url = get_api_path('sample.json') 47 | multiple_files = [] 48 | images = [s['image'] for s in samples] 49 | labels = [s['label'] for s in samples] 50 | for image in images: 51 | multiple_files.append(('images', (image, open(image, 'rb'), 'image/png'))) 52 | headers=get_headers(no_content_type=True) 53 | headers["config"]= json.dumps(config, cls=HCEncoder) 54 | headers["labels"]= json.dumps(labels) 55 | 56 | try: 57 | r = requests.post(url, files=multiple_files, headers=headers, timeout=30) 58 | return r.text 59 | except requests.exceptions.RequestException: 60 | e = sys.exc_info()[0] 61 | print("Error while calling hyperchamber - ", e) 62 | return None 63 | 64 | def measure(config, result, max_retries=10): 65 | """Records results on hyperchamber.io. Used when you are done testing a config.""" 66 | url = get_api_path('measurement.json') 67 | data = {'config': config, 'result': result} 68 | retries = 0 69 | while(retries < max_retries): 70 | try: 71 | r = requests.post(url, data=json.dumps(data, cls=HCEncoder), headers=get_headers(), timeout=30) 72 | return r.text 73 | except requests.exceptions.RequestException: 74 | e = sys.exc_info()[0] 75 | print("Error while calling hyperchamber - retrying ", e) 76 | retries += 1 77 | 78 | 79 | def load_config(id): 80 | url = get_api_path('config/'+id+'.json') 81 | r = requests.get(url, headers=get_headers()) 82 | config = json.loads(r.text) 83 | if(not config): 84 | config = {} 85 | config['parent_uuid']=id 86 | config["uuid"]=uuid.uuid4().hex 87 | return config 88 | -------------------------------------------------------------------------------- /hyperchamber/selector.py: -------------------------------------------------------------------------------- 1 | from json import JSONEncoder 2 | import random 3 | import os 4 | import uuid 5 | import json 6 | 7 | from .config import Config 8 | 9 | 10 | # for function serialization 11 | class HCEncoder(JSONEncoder): 12 | def default(self, o): 13 | if(hasattr(o, '__call__')): # is function 14 | return "function:" +o.__module__+"."+o.__name__ 15 | else: 16 | try: 17 | return o.__dict__ 18 | except AttributeError: 19 | try: 20 | return str(o) 21 | except AttributeError: 22 | return super(o) 23 | 24 | 25 | class Selector: 26 | def __init__(self, initialStore = {}): 27 | self.store = initialStore 28 | self.results = [] 29 | def set(self, key, value): 30 | """Sets a hyperparameter. Can be used to set an array of hyperparameters.""" 31 | self.store[key]=value 32 | return self.store 33 | 34 | def count_configs(self): 35 | count = 1 36 | 37 | for key in self.store: 38 | value = self.store[key] 39 | if(isinstance(value,list)): 40 | count *= len(value) 41 | 42 | return count 43 | 44 | def get_config_value(self, k, i): 45 | """Gets the ith config value for k. e.g. get_config_value('x', 1)""" 46 | if(not isinstance(self.store[k], list)): 47 | return self.store[k] 48 | else: 49 | return self.store[k][i] 50 | 51 | def configs(self, max_configs=1, offset=None, serial=False, create_uuid=True): 52 | """Generate max configs, each one a dictionary. e.g. [{'x': 1}] 53 | 54 | Will also add a config UUID, useful for tracking configs. 55 | You can turn this off by passing create_uuid=False. 56 | """ 57 | if len(self.store)==0: 58 | return [] 59 | 60 | configs = [] 61 | 62 | if(offset is None): 63 | offset = max(0, random.randint(0, self.count_configs())) 64 | for i in range(max_configs): 65 | # get an element to index over 66 | 67 | config = self.config_at(offset) 68 | if(create_uuid): 69 | config["uuid"]=uuid.uuid4().hex 70 | configs.append(config) 71 | if(serial): 72 | offset+=1 73 | else: 74 | offset = max(0, random.randint(0, self.count_configs())) 75 | return configs 76 | 77 | def config_at(self, i): 78 | """Gets the ith config""" 79 | selections = {} 80 | for key in self.store: 81 | value = self.store[key] 82 | if isinstance(value, list): 83 | selected = i % len(value) 84 | i = i // len(value) 85 | selections[key]= value[selected] 86 | else: 87 | selections[key]= value 88 | 89 | return Config(selections) 90 | 91 | def random_config(self): 92 | offset = max(0, random.randint(0, self.count_configs())) 93 | return self.config_at(offset) 94 | 95 | def reset(self): 96 | """Reset the hyperchamber variables""" 97 | self.store = {} 98 | self.results = [] 99 | return 100 | 101 | def top(self, sort_by): 102 | """Get the best results according to your custom sort method.""" 103 | sort = sorted(self.results, key=sort_by) 104 | return sort 105 | 106 | def record(self, config, result): 107 | """Record the results of a config.""" 108 | self.results.append((config, result)) 109 | 110 | def load(self, filename, load_toml=False): 111 | """Loads a config from disk""" 112 | content = open(filename) 113 | if load_toml: 114 | import toml 115 | return Config(toml.load(content)) 116 | else: 117 | return Config(json.load(content)) 118 | 119 | def load_or_create_config(self, filename, config=None): 120 | """Loads a config from disk. Defaults to a random config if none is specified""" 121 | os.makedirs(os.path.dirname(os.path.expanduser(filename)), exist_ok=True) 122 | if os.path.exists(filename): 123 | return self.load(filename) 124 | 125 | if(config == None): 126 | config = self.random_config() 127 | 128 | self.save(filename, config) 129 | return config 130 | 131 | def save(self, filename, config): 132 | """Loads a config from disk""" 133 | return open(os.path.expanduser(filename), 'w').write(json.dumps(config, cls=HCEncoder, sort_keys=True, indent=2, separators=(',', ': '))) 134 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='hyperchamber', 5 | version='0.3.1', 6 | description='Tune and optimize your hyperparameters', 7 | author='255bits', 8 | author_email="support@255bits.com", 9 | packages=['hyperchamber', 'hyperchamber.io'], 10 | install_requires=['requests >= 2.4.2'], 11 | url='https://github.com/255bits/hyperchamber', 12 | license='MIT', 13 | classifiers=[ 14 | ], 15 | ) 16 | -------------------------------------------------------------------------------- /tests/acceptance_io.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import tensorflow as tf 3 | import numpy as np 4 | 5 | import hyperchamber as hc 6 | import matplotlib.pyplot as plt 7 | 8 | def test_graph(config, filename, n): 9 | plt.rcdefaults() 10 | 11 | people = ('Tom', 'Dick', 'Harry', 'Slim', 'Jim') 12 | y_pos = np.arange(len(people)) 13 | performance = 3 + 10 * np.random.rand(len(people)) 14 | error = np.random.rand(len(people)) 15 | 16 | plt.barh(y_pos, performance, xerr=error, align='center', alpha=0.4) 17 | plt.yticks(y_pos, people) 18 | plt.xlabel("Step "+str(n)) 19 | plt.title('Are samples measurements working?') 20 | 21 | plt.savefig(filename) 22 | 23 | hc.set("model", "255bits/acceptance_test") 24 | hc.set("version", "0.0.1") 25 | hc.set("test", "acceptance_io") 26 | 27 | for config in hc.configs(1): 28 | filenames = [ "/tmp/acceptance_io_"+str(i)+".png" for i in range(10) ] 29 | graphs = [test_graph(config, filename, i) for i, filename in enumerate(filenames)] 30 | hc.io.sample(config, [{'image':f, 'label':f} for f in filenames]) 31 | 32 | hc.io.measure(config, {'ranking': 1}) 33 | print("Stored config", config) 34 | -------------------------------------------------------------------------------- /tests/test_hyperchamber.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import tensorflow as tf 3 | import numpy as np 4 | 5 | import hyperchamber as hc 6 | 7 | 8 | class hyperchamber_test(unittest.TestCase): 9 | def test_set(self): 10 | hc.reset() 11 | hc.set('x', [1]) 12 | self.assertEqual(hc.configs(1, offset=0, serial=True,create_uuid=False), [{'x':1}]) 13 | 14 | def test_set2(self): 15 | hc.reset() 16 | hc.set('x', [1,2]) 17 | self.assertEqual(hc.configs(2, offset=0, serial=True,create_uuid=False), [{'x':1},{'x':2}]) 18 | 19 | def test_Config_accessor(self): 20 | hc.reset() 21 | hc.set('x', [1]) 22 | config = hc.configs(1, offset=0, serial=True,create_uuid=False)[0] 23 | self.assertEqual(config.x, 1) 24 | self.assertEqual(config.y, None) 25 | 26 | def test_createUUID(self): 27 | hc.reset() 28 | hc.set('x', [1]) 29 | self.assertTrue(len(hc.configs(1)[0]["uuid"]) > 1) 30 | 31 | 32 | def test_pagination(self): 33 | hc.reset() 34 | hc.set('x', [1,2]) 35 | self.assertEqual(hc.configs(1, create_uuid=False, serial=True,offset=0), [{'x':1}]) 36 | self.assertEqual(hc.configs(1, create_uuid=False, serial=True,offset=1), [{'x':2}]) 37 | self.assertEqual(hc.configs(1, create_uuid=False, serial=True,offset=2), [{'x':1}]) 38 | 39 | def test_constant_set(self): 40 | hc.reset() 41 | hc.set('x', 1) 42 | hc.set('y', [2,3]) 43 | self.assertEqual(hc.configs(1, create_uuid=False,serial=True, offset=0), [{'x':1, 'y':2}]) 44 | print("--") 45 | self.assertEqual(hc.configs(1, create_uuid=False, serial=True,offset=1), [{'x':1, 'y':3}]) 46 | self.assertEqual(hc.configs(1, create_uuid=False,serial=True, offset=2), [{'x':1, 'y':2}]) 47 | 48 | def test_set2_2vars(self): 49 | hc.reset() 50 | hc.set('x', [1]) 51 | hc.set('y', [3,4]) 52 | self.assertEqual(hc.configs(2, create_uuid=False, serial=True,offset=0), [{'x':1,'y':3},{'x':1,'y':4}]) 53 | 54 | def test_configs(self): 55 | hc.reset() 56 | self.assertEqual(hc.configs(create_uuid=False), []) 57 | 58 | 59 | def test_record(self): 60 | hc.reset() 61 | def do_nothing(x): 62 | return 0 63 | self.assertEqual(hc.top(sort_by=do_nothing), []) 64 | config = {'a':1} 65 | result = {'b':2} 66 | hc.record(config, result) 67 | self.assertEqual(hc.top(sort_by=do_nothing)[0], (config, result)) 68 | 69 | def test_reset(self): 70 | hc.reset() 71 | self.assertEqual(hc.configs(), []) 72 | self.assertEqual(hc.count_configs(), 1) 73 | 74 | def test_store_size(self): 75 | hc.reset() 76 | hc.set('x', [1,2]) 77 | self.assertEqual(hc.count_configs(), 2) 78 | 79 | def test_top(self): 80 | hc.reset() 81 | for i in range(10): 82 | config = {"i": i} 83 | result = {"cost": 10-i} 84 | hc.record(config, result) 85 | 86 | def by_cost(x): 87 | config,result = x 88 | return result['cost'] 89 | 90 | self.assertEqual(hc.top(sort_by=by_cost)[0], ({'i': 9}, {'cost': 1})) 91 | 92 | 93 | if __name__ == '__main__': 94 | unittest.main() 95 | -------------------------------------------------------------------------------- /tests/test_permute.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import tensorflow as tf 3 | import numpy as np 4 | 5 | import hyperchamber as hc 6 | #import hyperchamber.permute 7 | 8 | 9 | class permute_test(unittest.TestCase): 10 | def test_one_permute(self): 11 | hc.reset() 12 | hc.set('x', 1) 13 | hc.set('y', [1]) 14 | self.assertEqual(hc.count_configs(), 1) 15 | self.assertEqual(hc.configs(1, serial=True,offset=0,create_uuid=False), [{'x':1, 'y': 1}]) 16 | 17 | def test_many_permute(self): 18 | hc.reset() 19 | hc.set('x', 1) 20 | hc.set('y', [1, 2, 2]) 21 | self.assertEqual(hc.count_configs(), 3) 22 | self.assertEqual(hc.configs(1, serial=True,offset=0,create_uuid=False), [{'x':1, 'y': 1}]) 23 | 24 | def test_many_to_many(self): 25 | hc.reset() 26 | hc.set('x', [1,2,3]) 27 | hc.set('y', [1, 2, 3]) 28 | self.assertEqual(hc.count_configs(), 9) 29 | configs = hc.configs(3, serial=True,create_uuid=False,offset=0) 30 | self.assertNotEqual(configs[0], configs[1]) 31 | self.assertNotEqual(configs[1], configs[2]) 32 | 33 | def test_many_to_many_nested(self): 34 | hc.reset() 35 | hc.set('x', [1, 2, 3]) 36 | hc.set('y', [1, 2, 3]) 37 | hc.set('z', [1, 2, 3]) 38 | self.assertEqual(hc.count_configs(), 27) 39 | self.assertEqual(hc.configs(1, serial=True,offset=0,create_uuid=False), [{'x':1, 'y': 1, 'z': 1}]) 40 | 41 | # TODO: These results are not in a guaranteed order since store is a dictionary. 42 | # These tests dont pass reliable 43 | #self.assertEqual(hc.configs(1,offset=1), [{'x':1, 'y': 1, 'z': 2}]) 44 | #self.assertEqual(hc.configs(1,offset=5), [{'x':1, 'y': 1, 'z': 2}]) 45 | 46 | def test_only_permutation(self): 47 | hc.reset() 48 | for i in range(20): 49 | hc.set('a'+str(i), [1, 2, 3]) 50 | self.assertEqual(hc.count_configs(), 3**20) 51 | self.assertEqual(hc.configs(1, serial=True, offset=0)[0]['a1'], 1) 52 | self.assertEqual(hc.configs(1, serial=True, offset=0)[0]['a2'], 1) 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | --------------------------------------------------------------------------------