├── .gitignore ├── LICENSE ├── README.md ├── deepcde ├── __init__.py ├── bases │ ├── __init__.py │ ├── cosine.py │ ├── tensor.py │ └── wavelet.py ├── deepcde_pytorch.py ├── deepcde_tensorflow.py ├── tests.py └── utils.py ├── model_examples ├── alexnet_pytorch.md └── alexnet_tf.md └── tutorial └── DeepCDE-pytorch-tutorial-teddy.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | deepcde/__pycache__/* 3 | deepcde/deepcde.py 4 | examples/* 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Niccolo Dalmasso 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | DeepCDE: Neural Networks Conditional Density Estimation 2 | === 3 | 4 | Tensorflow and PyTorch code Conditional density estimation via deep neural network using CDE loss. 5 | 6 | Authors: Taylor Pospisil, Nic Dalmasso 7 | 8 | 9 | Citation 10 | === 11 | ```text 12 | @article{dalmasso2020cdetools, 13 | author = {{Dalmasso}, N. and {Pospisil}, T. and {Lee}, A.~B. and {Izbicki}, R. and 14 | {Freeman}, P.~E. and {Malz}, A.~I.}, 15 | title = "{Conditional density estimation tools in python and R with applications to photometric redshifts and likelihood-free cosmological inference}", 16 | journal = {Astronomy and Computing}, 17 | year = 2020, 18 | month = jan, 19 | volume = {30}, 20 | eid = {100362}, 21 | pages = {100362}, 22 | doi = {10.1016/j.ascom.2019.100362} 23 | } 24 | ``` -------------------------------------------------------------------------------- /deepcde/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /deepcde/bases/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /deepcde/bases/cosine.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class CosineBasis(object): 4 | def __init__(self, n_basis): 5 | self.n_basis = n_basis 6 | 7 | def evaluate(self, z_test): 8 | n_obs = z_test.shape[0] 9 | basis = np.empty((n_obs, self.n_basis)) 10 | 11 | z_test = z_test.flatten() 12 | 13 | basis[:, 0] = 1.0 14 | for col in range(1, self.n_basis): 15 | basis[:, col] = np.sqrt(2) * np.cos(np.pi * col * z_test) 16 | return basis 17 | -------------------------------------------------------------------------------- /deepcde/bases/tensor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class TensorBasis(object): 4 | """ 5 | Combines single-dimensional basis functions \phi_{d}(z) to form 6 | orthogonal tensor basis $\phi(z_{1}, \dots, z_{D}) = \prod_{d} 7 | \phi_{d}(z_{d})$. 8 | """ 9 | def __init__(self, base, n_basis): 10 | self.base = base 11 | self.n_basis = n_basis 12 | 13 | def evaluate(self, z_test): 14 | """Evaluates tensor basis.""" 15 | 16 | n_obs, n_dim = z_test.shape 17 | assert len(self.n_basis) == n_dim 18 | 19 | basis = np.ones((n_obs, np.prod(self.n_basis))) 20 | period = 1 21 | for dim in range(n_dim): 22 | 23 | sub_basis = self.base(self.n_basis[dim]).evaluate(z_test[:, dim]) 24 | col = 0 25 | for _ in range(np.prod(self.n_basis) // (self.n_basis[dim] * period)): 26 | for sub_col in range(self.n_basis[dim]): 27 | for _ in range(period): 28 | basis[:, col] *= sub_basis[:, sub_col] 29 | col += 1 30 | period *= self.n_basis[dim] 31 | return basis 32 | -------------------------------------------------------------------------------- /deepcde/bases/wavelet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pywt 3 | 4 | class WaveletBasis(object): 5 | def __init__(self, n_basis, family, n_aux=15): 6 | self.n_basis = n_basis 7 | 8 | ## Generate wavelet on fine grid 9 | wav = pywt.DiscreteContinuousWavelet(family) 10 | _, self.wavelet, self.x_grid = wav.wavefun(n_aux) 11 | self.N = max(self.x_grid) 12 | 13 | def _wave_fun(self, val): 14 | if val < 0: 15 | return 0.0 16 | if val > self.N: 17 | return 0.0 18 | return self.wavelet[np.argmin(abs(val - self.x_grid))] 19 | 20 | def evaluate(self, z_test): 21 | n_obs = z_test.shape[0] 22 | basis = np.empty((n_obs, self.n_basis)) 23 | basis[:,0] = 1.0 24 | 25 | z_test = z_test.flatten() 26 | 27 | ## Update wavelet for closest point 28 | j, k = 0, 0 29 | for col in range(1, self.n_basis): 30 | for row, t in enumerate(z_test): 31 | basis[row, col] = 2 ** (j / 2) * self._wave_fun(2 ** j * t - k) 32 | k += 1 33 | if k == 2 ** j: 34 | j += 1 35 | k = 0 36 | return basis 37 | 38 | class HaarBasis(WaveletBasis): 39 | def __init__(self, n_basis): 40 | super(HaarBasis, self).__init__(n_basis, "db1") 41 | -------------------------------------------------------------------------------- /deepcde/deepcde_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import numpy as np 3 | from .utils import normalize, remove_bumps 4 | 5 | 6 | ### DEFINE CUSTOM LAYER 7 | 8 | class cde_layer(nn.Module): 9 | 10 | def __init__(self, D_in, n_basis): 11 | super(cde_layer, self).__init__() 12 | self.linear = nn.Linear(D_in, n_basis-1) 13 | 14 | def forward(self, x): 15 | y_pred = self.linear(x) 16 | return y_pred 17 | 18 | 19 | ### DEFINE CUSTOM LOSS 20 | 21 | class cde_loss(nn.Module): 22 | 23 | def __init__(self): 24 | super(cde_loss, self).__init__() 25 | 26 | def forward(self, beta, z_basis, shrink_factor=1.0): 27 | complexity_terms = (beta**2).sum(dim=1) + 1.0 28 | fit_terms = (beta * z_basis.expand_as(beta)).sum(dim=1) + 1.0 29 | loss = (complexity_terms - 2 * fit_terms).float().mean() / shrink_factor 30 | return loss 31 | 32 | class cde_nll_loss(nn.Module): 33 | 34 | def __init__(self): 35 | super(cde_nll_loss, self).__init__() 36 | 37 | def forward(self, beta, z_basis, shrink_factor=1.0): 38 | fit_terms = (beta * z_basis.view(-1, 1).expand_as(beta)).sum(dim=1) + 1.0 39 | loss = ((- 1 * fit_terms) / shrink_factor).float().mean() 40 | return loss 41 | 42 | 43 | class approx_cde_loss(nn.Module): 44 | 45 | def __init__(self): 46 | super(approx_cde_loss, self).__init__() 47 | 48 | def forward(self, beta, shrink_factor=1.0): 49 | complexity_terms = (beta**2).sum(dim=1) + 1.0 50 | loss = (-1*complexity_terms).mean() / shrink_factor 51 | return loss 52 | 53 | 54 | #### DEFINE PREDICTION FUNCTION 55 | 56 | def cde_predict(model_output, z_min, z_max, z_grid, 57 | basis, delta=None, bin_size=0.01): 58 | 59 | n_obs = model_output.shape[0] 60 | beta = np.hstack((np.ones((n_obs, 1)), model_output)) 61 | z_grid_basis = basis.evaluate(z_grid)[:, :basis.n_basis] 62 | cdes = np.matmul(beta, z_grid_basis.T) 63 | if delta is not None: 64 | remove_bumps(cdes, delta=delta, bin_size=bin_size) 65 | normalize(cdes) 66 | cdes /= np.prod(z_max - z_min) 67 | return cdes 68 | 69 | -------------------------------------------------------------------------------- /deepcde/deepcde_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from .utils import normalize 3 | 4 | ### LOSS FUNCTIONS 5 | 6 | def cde_loss(beta, z_basis, shrink_factor=1.0, name="cde_loss"): 7 | with tf.name_scope(name): 8 | complexity_terms = tf.reduce_sum(tf.square(beta), axis=1) + 1.0 9 | fit_terms = tf.reduce_sum(tf.multiply(beta, z_basis), axis=1) + 1.0 10 | loss = tf.reduce_mean(complexity_terms - 2 * fit_terms) / shrink_factor 11 | return loss 12 | 13 | def nll_loss(beta, z_basis, shrink_factor=1.0, name="nll_loss"): 14 | with tf.name_scope(name): 15 | fit_terms = tf.reduce_sum(tf.multiply(beta, z_basis), axis=1) + 1.0 16 | loss = tf.reduce_mean(-fit_terms) / shrink_factor 17 | return loss 18 | 19 | def approx_cde_loss(beta, shrink_factor=1.0, name="approx cde_loss"): 20 | with tf.name_scope(name): 21 | complexity_terms = tf.reduce_sum(tf.square(beta), axis=1) + 1.0 22 | loss = tf.reduce_mean(-1 * complexity_terms) / shrink_factor 23 | return loss 24 | 25 | 26 | ### EXTRA LAYER (before softmax) 27 | 28 | def cde_layer(inputs, weight_sd, marginal_beta=None, name="cde_layer"): 29 | with tf.name_scope(name): 30 | W_shape = [int(inputs.shape[1]), len(marginal_beta) - 1] 31 | W = tf.Variable(tf.random_normal(W_shape, stddev=weight_sd), name="W") 32 | b = tf.get_variable("b", initializer=marginal_beta[1:]) if marginal_beta is not None else tf.get_variable("b") 33 | beta = tf.matmul(inputs, W) + b 34 | # tf.summary.histogram("weights", W) 35 | # tf.summary.histogram("biases", b) 36 | # tf.summary.histogram("betas", beta) 37 | return beta 38 | 39 | 40 | ### PREDICTION FUNCTION 41 | 42 | def cde_predict(sess, beta, z_min, z_max, z_grid, basis, input_dict, 43 | delta=None, bin_size=0.01): 44 | beta = sess.run(beta, feed_dict=input_dict) 45 | n_obs = beta.shape[0] 46 | beta = np.hstack((np.ones((n_obs, 1)), beta)) 47 | z_grid_basis = basis.evaluate(z_grid)[:, basis.n_basis] 48 | cdes = np.matmul(beta, z_grid_basis.T) 49 | if delta: 50 | remove_bumps(cdes, delta=delta, bin_size=bin_size) 51 | normalize(cdes) 52 | cdes /= np.prod(z_max - z_min) 53 | return cdes 54 | -------------------------------------------------------------------------------- /deepcde/tests.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import unittest2 as unittest 3 | import torch 4 | 5 | from utils import box_transform, normalize 6 | from deepcde_pytorch import cde_loss 7 | 8 | 9 | class MyTest(unittest.TestCase): 10 | 11 | def test__box_transform(self): 12 | example_array = np.arange(0, 1, 0.01).round(2) 13 | 14 | self.assertRaises(ValueError, box_transform, example_array, 0.1, 1) 15 | self.assertRaises(ValueError, box_transform, example_array, 0, 0.8) 16 | self.assertRaises(ValueError, box_transform, example_array, 0.1, 0.8) 17 | 18 | self.assertTrue((box_transform(np.arange(0, 10, 0.1), 0, 10).round(2) == example_array).all()) 19 | self.assertTrue((box_transform(np.arange(0, 100, 1), 0, 100).round(2) == example_array).all()) 20 | self.assertTrue((box_transform(np.arange(-50, 50, 1), -50, 50).round(2) == example_array).all()) 21 | 22 | 23 | def test__normalize(self): 24 | example_range = np.arange(0, 1, 0.01) 25 | example_density = np.random.normal(loc=0, scale=0.1, size=example_range.shape) 26 | idx_negative = (example_density <= 0) 27 | epsilon = 0.05 28 | 29 | # Normalization in place 30 | normalize(example_density) 31 | 32 | self.assertTrue((1- epsilon) < np.trapz(example_density, example_range) < (1 + epsilon)) 33 | self.assertTrue(np.sum(example_density[idx_negative]) == 0) 34 | 35 | 36 | def test__loss_pytorch(self): 37 | beta = np.repeat(a=np.arange(0, 10, 1).reshape(-1, 1), axis=1, repeats=10) 38 | z_basis = np.arange(0, 10, 1).reshape(-1, 1) 39 | 40 | ## Numpy computation 41 | fit_terms = (beta * z_basis).sum(axis=1) + 1 42 | comp_terms = (beta**2).sum(axis=1) + 1 43 | loss_val = np.average((comp_terms - 2*fit_terms)) 44 | 45 | ## Pytorch Computation 46 | beta_t = torch.from_numpy(beta) 47 | z_basis_t = torch.from_numpy(z_basis) 48 | criterion = cde_loss() 49 | 50 | self.assertEqual(loss_val, criterion(beta_t, z_basis_t, 1).item()) 51 | 52 | 53 | if __name__ == '__main__': 54 | unittest.main() -------------------------------------------------------------------------------- /deepcde/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def normalize(cde_estimates, tol=1e-6, max_iter=10000): 5 | """Normalizes conditional density estimates to be non-negative and 6 | integrate to one. 7 | 8 | Assumes densities are evaluated on the unit grid. 9 | 10 | :param cde_estimates: a numpy array or matrix of conditional density estimates. 11 | :param tol: float, the tolerance to accept for abs(area - 1). 12 | :param max_iter: int, the maximal number of search iterations. 13 | :returns: the normalized conditional density estimates. 14 | :rtype: numpy array or matrix. 15 | 16 | """ 17 | if cde_estimates.ndim == 1: 18 | _normalize(cde_estimates, tol, max_iter) 19 | else: 20 | np.apply_along_axis(_normalize, 1, cde_estimates, tol=tol, 21 | max_iter=max_iter) 22 | 23 | def _normalize(density, tol=1e-6, max_iter=500): 24 | """Normalizes a density estimate to be non-negative and integrate to 25 | one. 26 | 27 | Assumes density is evaluated on the unit grid. 28 | 29 | :param density: a numpy array of density estimates. 30 | :param z_grid: an array, the grid points at the density is estimated. 31 | :param tol: float, the tolerance to accept for abs(area - 1). 32 | :param max_iter: int, the maximal number of search iterations. 33 | :returns: the normalized density estimate. 34 | :rtype: numpy array. 35 | 36 | """ 37 | 38 | if max_iter <= 0: 39 | raise ValueError("max_iter needs to be positive. Currently %s" % (max_iter)) 40 | 41 | hi = np.max(density) 42 | lo = 0.0 43 | 44 | area = np.mean(np.maximum(density, 0.0)) 45 | if area == 0.0: 46 | # replace with uniform if all negative density 47 | density[:] = 1.0 48 | elif area < 1: 49 | density /= area 50 | density[density < 0.0] = 0.0 51 | return 52 | 53 | for _ in range(max_iter): 54 | mid = (hi + lo) / 2 55 | area = np.mean(np.maximum(density - mid, 0.0)) 56 | if abs(1.0 - area) <= tol: 57 | break 58 | if area < 1.0: 59 | hi = mid 60 | else: 61 | lo = mid 62 | 63 | # update in place 64 | density -= mid 65 | density[density < 0.0] = 0.0 66 | 67 | 68 | def box_transform(z, z_min, z_max): 69 | """ 70 | Standardazing between 0 and 1 by subtracting the minimum and dividing by the range. 71 | :param z: a numpy array 72 | :param z_min: minimum of the numpy array/minimum to be used for standardization 73 | (has to be smaller or equal to the minimum of the numpy array) 74 | :param z_max: maximum of the numpy array/minimum to be used for standardization 75 | (has to be larger or equal to the minimum of the numpy array) 76 | :return: a normalized numpy array with same shape as input z 77 | """ 78 | if np.min(z) < z_min or np.max(z) > z_max: 79 | raise ValueError('Passed minimum and maximum need to be outside the array range.' 80 | 'Current min: %s, max: %s, while array range: (%s,%s)' % ( 81 | z_min, z_max, np.min(z), np.max(z) 82 | )) 83 | 84 | return (z - z_min) / (z_max - z_min) 85 | 86 | 87 | def remove_bumps(cde_estimates, delta, bin_size=0.01): 88 | """Removes bumps in conditional density estimates 89 | 90 | :param cde_estimates: a numpy array or matrix of conditional density estimates. 91 | :param delta: float, the threshold for bump removal 92 | :param bin_size: float, size of the bin for density evaluation 93 | :returns: the conditional density estimates with bumps removed 94 | :rtype: numpy array or matrix 95 | 96 | """ 97 | if cde_estimates.ndim == 1: 98 | _remove_bumps(cde_estimates, delta=delta, bin_size=bin_size) 99 | else: 100 | np.apply_along_axis(_remove_bumps, 1, cde_estimates, 101 | delta=delta, bin_size=bin_size) 102 | 103 | def _remove_bumps(density, delta, bin_size=0.01): 104 | """Removes bumps in conditional density estimates. 105 | 106 | :param density: a numpy array of conditional density estimate. 107 | :param delta: float, the threshold for bump removal. 108 | :param bin_size: float, size of the bin for density evaluation. 109 | :returns: the conditional density estimate with bumps removed. 110 | :rtype: numpy array. 111 | 112 | """ 113 | area = 0.0 114 | left_idx = 0 115 | for right_idx, val in enumerate(density): 116 | if val <= 0.0: 117 | if area < delta: 118 | density[left_idx:(right_idx + 1)] = 0.0 119 | left_idx = right_idx + 1 120 | area = 0.0 121 | else: 122 | area += val * bin_size 123 | if area < delta: # final check at end 124 | density[left_idx:] = 0.0 -------------------------------------------------------------------------------- /model_examples/alexnet_pytorch.md: -------------------------------------------------------------------------------- 1 | 2 | ### Alexnet - Pytorch 3 | Pytorch implementation of AlexNet for the task of estimating the probability distribution of correct orientations of an image. 4 | The input to the model consists of (277, 277) colored images with 3 channels (i.e. color bands). 5 | The target is a continuous variable $y \in (0,2\pi)$ for image orientation. 6 | 7 | Please note that the attached implementations do not include code for generating training and testing sets. 8 | ```python 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | import math 13 | from deepcde.deepcde_pytorch import cde_layer, cde_loss, cde_predict 14 | from deepcde.utils import box_transform 15 | from deepcde.bases.cosine import CosineBasis 16 | 17 | # PRE-PROCESSING ############################################################################# 18 | # Create basis (in this case 31 cosine basis) 19 | n_basis = 31 20 | basis = CosineBasis(n_basis) 21 | 22 | # ... Creation of training and testing set ... 23 | 24 | # Evaluate the y_train over the basis 25 | y_train = box_transform(y_train, 0, 2*math.pi) # transform to a variable between 0 and 1 26 | y_basis = basis.evaluate(y_train) # evaluate basis 27 | y_basis = y_basis.astype(np.float32) 28 | 29 | # ALEXNET DEFINITION ######################################################################### 30 | # `basis_size` is the number of basis (in this case 31). 31 | # `marginal_beta` is the initial value for the bias of the cde layer, if available 32 | class AlexNetCDE(nn.Module): 33 | def __init__(self, basis_size, marginal_beta=None): 34 | super(AlexNetCDE, self).__init__() 35 | self.conv1 = nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2) 36 | self.conv2 = nn.Conv2d(64, 192, kernel_size=5, padding=2) 37 | self.conv3 = nn.Conv2d(192, 384, kernel_size=3, padding=1) 38 | self.conv4 = nn.Conv2d(384, 256, kernel_size=3, padding=1) 39 | self.conv5 = nn.Conv2d(256, 256, kernel_size=3, padding=1) 40 | 41 | self.fc1 = nn.Linear(256 * 6 * 6, 4096) 42 | self.fc2 = nn.Linear(4096, 4096) 43 | self.cde = cde_layer(4096, basis_size - 1) 44 | if marginal_beta: 45 | self.cde.bias.data = torch.from_numpy(marginal_beta[1:]).type(torch.FloatTensor) 46 | 47 | def forward(self, x): 48 | x = F.max_pool2d(F.relu(self.conv1(x)), kernel_size=3, stride=2) 49 | x = F.max_pool2d(F.relu(self.conv2(x)), kernel_size=3, stride=2) 50 | x = F.relu(self.conv3(x)) 51 | x = F.relu(self.conv4(x)) 52 | x = F.max_pool2d(F.relu(self.conv5(x)), kernel_size=3, stride=2) 53 | x = F.dropout(x.view(x.size(0), 256 * 6 * 6), training=self.training) 54 | x = F.dropout(F.relu(self.fc1(x)), training=self.training) 55 | x = F.dropout(F.relu(self.fc2(x)), training=self.training) 56 | beta = self.cde(x) 57 | return beta 58 | 59 | # Definition of model and loss function (examples) 60 | model = AlexNetCDE(basis_size=n_basis) 61 | loss = cde_loss() 62 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 63 | optimizer = optim.SGD(model.parameters(), lr=1e-4, momentum=0.5) 64 | 65 | # TRAINING ################################################################################# 66 | # ... Creation of `training_set_loader` and `testing_set_loader` object ... 67 | for epoch in range(n_epochs): 68 | model.train() 69 | for batch_idx, (x_batch, y_basis_batch) in enumerate(training_set_loader): 70 | x_batch, y_basis_batch = x_batch.to(device), y_basis_batch.to(device) 71 | optimizer.zero_grad() 72 | beta_batch = model(x_batch) 73 | loss = loss(beta_batch, y_basis_batch) 74 | loss.backward() 75 | optimizer.step() 76 | 77 | # ... Evaluation of testing set ... 78 | 79 | # PREDICTION ############################################################################## 80 | # ... Selection of `x_test` to get conditional density estimate of ... 81 | y_grid = np.linspace(0, 1, 1000) # Creating a grid over the density range 82 | beta_prediction = model(x_test) 83 | cdes = cde_predict(beta_prediction, 0, 1, y_grid, basis, n_basis) 84 | predicted_cdes = cdes * 2 * math.pi # Re-normalize 85 | ``` -------------------------------------------------------------------------------- /model_examples/alexnet_tf.md: -------------------------------------------------------------------------------- 1 | ### Alexnet - Tensorflow 2 | Tensorflow implementation of AlexNet for the task of estimating the probability distribution of correct orientations of an image. 3 | The input to the model consists of (277, 277) colored images with 3 channels (i.e. color bands). 4 | The target is a continuous variable $y \in (0,2\pi)$ for image orientation. 5 | 6 | Please note that the attached implementations do not include code for generating training and testing sets. 7 | 8 | 9 | ```python 10 | import numpy as np 11 | import tensorflow as tf 12 | import math 13 | from deepcde.deepcde_tensorflow import cde_loss, cde_predict 14 | from deepcde.utils import box_transform 15 | from deepcde.bases.cosine import CosineBasis 16 | 17 | # PRE-PROCESSING ############################################################################# 18 | # Create basis (in this case 31 cosine basis) 19 | n_basis = 31 20 | basis = CosineBasis(n_basis) 21 | 22 | # ... Creation of training and testing set ... 23 | 24 | # Evaluate the y_train over the basis 25 | y_train = box_transform(y_train, 0, 2*math.pi) # transform to a variable between 0 and 1 26 | y_basis = basis.evaluate(y_train) # evaluate basis 27 | y_basis = y_basis.astype(np.float32) 28 | 29 | # Features and basis are inserted in a dictionary of this form 30 | features = { 31 | 'x': tf.FixedLenFeature([20 * 5 * 1], tf.float32), 32 | 'y': tf.FixedLenFeature([1], tf.float32), 33 | 'y_basis': tf.FixedLenFeature([n_basis - 1], tf.float32) 34 | } 35 | # ... Generation of TensorFlow training and testing data ... 36 | 37 | # ALEXNET DEFINITION ######################################################################### 38 | weight_sd = 0.01 # sd parameter for initialization of weights 39 | marginal_beta = None # Initialization parameter for bias in CDE layer 40 | 41 | def model_function(features, y_basis, mode, 42 | weight_sd=weight_sd, marginal_beta=marginal_beta): 43 | # Input Layer 44 | input_layer = tf.reshape(features, shape=[-1, 277, 277, 3]) 45 | 46 | # Convolutional Layer 1 47 | conv1 = tf.layers.conv2d( 48 | strides=[1, 4, 4, 1], 49 | inputs=input_layer, 50 | filters=64, 51 | kernel_size=[1, 11, 11, 1], 52 | padding="same", 53 | activation=tf.nn.relu) 54 | pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[3, 3], strides=2) 55 | 56 | # Convolutional Layer 2 57 | conv2 = tf.layers.conv2d( 58 | inputs=pool1, 59 | filters=192, 60 | kernel_size=[5, 5], 61 | strides=[1, 1, 1, 1], 62 | padding="same", 63 | activation=tf.nn.relu) 64 | pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[3, 3], strides=2) 65 | 66 | # Convolutional Layers 3, 4 and 5 67 | conv3 = tf.layers.conv2d( 68 | inputs=pool2, 69 | filters=384, 70 | kernel_size=[3, 3], 71 | strides=[1, 1, 1, 1], 72 | padding="same", 73 | activation=tf.nn.relu) 74 | 75 | conv4 = tf.layers.conv2d( 76 | inputs=conv3, 77 | filters=256, 78 | kernel_size=[3, 3], 79 | strides=[1, 1, 1, 1], 80 | padding="same", 81 | activation=tf.nn.relu) 82 | 83 | conv5 = tf.layers.conv2d( 84 | inputs=conv4, 85 | filters=256, 86 | kernel_size=[3, 3], 87 | strides=[1, 1, 1, 1], 88 | padding="same", 89 | activation=tf.nn.relu) 90 | pool5 = tf.layers.max_pooling2d(inputs=conv5, pool_size=[3, 3], strides=2) 91 | pool5_flat = tf.reshape(pool5, [-1, 256 * 6 * 6]) 92 | 93 | # Dense Layers 94 | dense_1 = tf.layers.dense(inputs=pool5_flat, units=4096, activation=tf.nn.relu) 95 | dropout_1 = tf.layers.dropout(inputs=dense_1, rate=0.5, 96 | training=mode == tf.estimator.ModeKeys.TRAIN) 97 | 98 | dense_2 = tf.layers.dense(inputs=dropout_1, units=4096, activation=tf.nn.relu) 99 | dropout_2 = tf.layers.dropout(inputs=dense_2, rate=0.5, 100 | training=mode == tf.estimator.ModeKeys.TRAIN) 101 | 102 | # CDE Layer 103 | beta = cde_layer(dropout_2, weight_sd, marginal_beta) 104 | 105 | # Loss Computation 106 | loss = cde_loss(beta, y_basis) 107 | metrics = { 108 | 'cde_loss': tf.metrics.mean(cde_loss(beta, y_basis)) 109 | } 110 | 111 | # Training and Evaluation Steps 112 | if mode == tf.estimator.ModeKeys.EVAL: 113 | return tf.estimator.EstimatorSpec(mode, 114 | loss=loss, 115 | eval_metric_ops=metrics) 116 | 117 | elif mode == tf.estimator.ModeKeys.TRAIN: 118 | # Get train operator, using Adam for instance 119 | train_op = tf.train.AdamOptimizer().minimize( 120 | loss, global_step=tf.train.get_or_create_global_step()) 121 | return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) 122 | 123 | else: 124 | raise NotImplementedError('Unknown mode {}'.format(mode)) 125 | 126 | # TRAINING ################################################################################# 127 | cfg = tf.estimator.RunConfig(save_checkpoints_secs=120) 128 | estimator = tf.estimator.Estimator(model_fn, model_dir, cfg) 129 | # ... Creation of `train_set` and `test_set` objects ... 130 | # ... Inclusion of all extra parameters like learning rate, momentum, etc. ... 131 | tf.estimator.train_and_evaluate(estimator, train_set, test_set) 132 | 133 | 134 | # PREDICTION ############################################################################## 135 | # ... Selection of `x_test` to get conditional density estimate of ... 136 | y_grid = np.linspace(0, 1, 1000) 137 | beta = tf.placeholder(tf.float32, [1, n_basis-1]) 138 | with tf.Session() as sess: 139 | # ... Resume model from checkpoint ... 140 | cdes = cde_predict(sess, beta, 0, 1, y_grid, basis, n_basis, 141 | input_dict={'features': x_test}) 142 | cdes = cdes * 2 * math.pi # Re-normalize 143 | ``` --------------------------------------------------------------------------------