├── LICENSE ├── README.md ├── alad ├── arrhythmia_utilities.py ├── cifar10_utilities.py ├── kdd_utilities.py ├── run.py └── svhn_utilities.py ├── anogan ├── arrhythmia_utilities.py ├── cifar10_utilities.py ├── kdd_utilities.py ├── run.py └── svhn_utilities.py ├── dagmm ├── arrhythmia_utilities.py ├── gmm_utils.py ├── kdd_utilities.py └── run.py ├── data ├── arrhythmia.mat ├── arrhythmia.py ├── cifar10.py ├── kdd.py ├── kddcup.data_10_percent_corrected └── svhn.py ├── dsebm ├── arrhythmia_utilities.py ├── cifar10_utilities.py ├── kdd_utilities.py ├── run.py └── svhn_utilities.py ├── main.py ├── requirements.txt ├── toy_experiments ├── .ipynb_checkpoints │ └── ALAD - toy example-checkpoint.ipynb ├── ALAD - toy example.ipynb └── utils │ ├── __init__.py │ ├── data_gmm.py │ ├── data_utils.py │ ├── rng.py │ └── utils.py └── utils ├── adapt_data.py ├── constants.py ├── evaluations.py ├── plot_reconstructions.py └── sn.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Houssam Zenati, Manon Romain, Chuan Sheng Foo, Bruno Lecouat, Vijay Chandrasekhar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Adversarially-Learned-Anomaly-Detection 2 | ALAD (Proceedings of IEEE ICDM 2018) official code 3 | 4 | The code for the paper ["Adversarially Learned Anomaly Detection" (authors: Houssam Zenati*, Manon Romain*, Chuan Sheng Foo*, Bruno Lecouat, Vijay Ramaseshan Chandrasekhar)](https://arxiv.org/abs/1812.02288) is now open source! 5 | 6 | Please reach us via emails or via github issues for any enquiries! 7 | 8 | Please cite our work if you find it useful for your research and work. 9 | ``` 10 | @article{Zenati2018AdversariallyLA, 11 | title={Adversarially Learned Anomaly Detection}, 12 | author={Houssam Zenati and Manon Romain and Chuan Sheng Foo and Bruno Lecouat and Vijay R. Chandrasekhar}, 13 | journal={2018 IEEE International Conference on Data Mining (ICDM)}, 14 | year={2018}, 15 | pages={727-736} 16 | } 17 | ``` 18 | 19 | ## Prerequisites. 20 | To run the code, follow those steps: 21 | 22 | Download the project code: 23 | 24 | ``` 25 | git clone https://github.com/houssamzenati/Adversarially-Learned-Anomaly-Detection.git 26 | ``` 27 | Install requirements (in the cloned repository): 28 | 29 | ``` 30 | pip3 install -r requirements.txt 31 | ``` 32 | 33 | 34 | ## Doing anomaly detection. 35 | 36 | Running the code with different options 37 | 38 | ``` 39 | python3 main.py run --nb_epochs= --label=<0, 1, 2, 3, 4, 5, 6, 7, 8, 9> --m=<'cross-e','fm'> --d= --rd= etc. 40 | ``` 41 | Please refer to the argument parser in main.py for more details. 42 | 43 | When using alad, please use it with --sn, --enable_early_stop and --enable_dzz. (Different options are provided to enable the ablation study). 44 | Important: we also provide implementations of DSEBM and DAGMM methods as open source work, the results reported in our paper for those methods, however, are derived from the DAGMM paper. 45 | -------------------------------------------------------------------------------- /alad/arrhythmia_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Arrhythmia ALAD architecture. 4 | 5 | Generator (decoder), encoder and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | from utils import sn 10 | 11 | learning_rate = 1e-5 12 | batch_size = 32 13 | latent_dim = 64 14 | init_kernel = tf.contrib.layers.xavier_initializer() 15 | 16 | def leakyReLu(x, alpha=0.2, name=None): 17 | if name: 18 | with tf.variable_scope(name): 19 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 20 | else: 21 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 22 | 23 | def encoder(x_inp, is_training=False, getter=None, reuse=False, 24 | do_spectral_norm=False): 25 | """ Encoder architecture in tensorflow 26 | 27 | Maps the data into the latent space 28 | 29 | Args: 30 | x_inp (tensor): input data for the encoder. 31 | is_training (bool): for batch norms and dropouts 32 | getter: for exponential moving average during inference 33 | reuse (bool): sharing variables or not 34 | 35 | Returns: 36 | net (tensor): last activation layer of the encoder 37 | 38 | """ 39 | with tf.variable_scope('encoder', reuse=reuse, custom_getter=getter): 40 | name_net = 'layer_1' 41 | with tf.variable_scope(name_net): 42 | net = tf.layers.dense(x_inp, 43 | units=256, 44 | kernel_initializer=init_kernel, 45 | name='fc') 46 | net = leakyReLu(net) 47 | 48 | name_net = 'layer_2' 49 | with tf.variable_scope(name_net): 50 | net = tf.layers.dense(net, 51 | units=128, 52 | kernel_initializer=init_kernel, 53 | name='fc') 54 | net = leakyReLu(net) 55 | 56 | name_net = 'layer_3' 57 | with tf.variable_scope(name_net): 58 | net = tf.layers.dense(net, 59 | units=latent_dim, 60 | kernel_initializer=init_kernel, 61 | name='fc') 62 | return net 63 | 64 | def decoder(z_inp, is_training=False, getter=None, reuse=False): 65 | """ Generator architecture in tensorflow 66 | 67 | Generates data from the latent space 68 | 69 | Args: 70 | z_inp (tensor): input variable in the latent space 71 | is_training (bool): for batch norms and dropouts 72 | getter: for exponential moving average during inference 73 | reuse (bool): sharing variables or not 74 | 75 | Returns: 76 | net (tensor): last activation layer of the generator 77 | 78 | """ 79 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 80 | name_net = 'layer_1' 81 | with tf.variable_scope(name_net): 82 | net = tf.layers.dense(z_inp, 83 | units=128, 84 | kernel_initializer=init_kernel, 85 | name='fc') 86 | net = tf.nn.relu(net) 87 | 88 | name_net = 'layer_2' 89 | with tf.variable_scope(name_net): 90 | net = tf.layers.dense(net, 91 | units=256, 92 | kernel_initializer=init_kernel, 93 | name='fc') 94 | net = tf.nn.relu(net) 95 | 96 | name_net = 'layer_3' 97 | with tf.variable_scope(name_net): 98 | net = tf.layers.dense(net, 99 | units=274, 100 | kernel_initializer=init_kernel, 101 | name='fc') 102 | return net 103 | 104 | def discriminator_xz(x_inp, z_inp, is_training=False, getter=None, reuse=False, 105 | do_spectral_norm=False): 106 | """ Discriminator architecture in tensorflow 107 | 108 | Discriminates between pairs (E(x), x) and (z, G(z)) 109 | 110 | Args: 111 | x_inp (tensor): input data for the discriminator. 112 | z_inp (tensor): input variable in the latent space 113 | is_training (bool): for batch norms and dropouts 114 | getter: for exponential moving average during inference 115 | reuse (bool): sharing variables or not 116 | 117 | Returns: 118 | logits (tensor): last activation layer of the discriminator (shape 1) 119 | intermediate_layer (tensor): intermediate layer for feature matching 120 | 121 | """ 122 | with tf.variable_scope('discriminator_xz', reuse=reuse, custom_getter=getter): 123 | # D(x) 124 | name_x = 'x_layer_1' 125 | with tf.variable_scope(name_x): 126 | x = tf.layers.dense(x_inp, 127 | units=128, 128 | kernel_initializer=init_kernel, 129 | name='fc') 130 | x = tf.layers.batch_normalization(x, 131 | training=is_training, 132 | name='batch_normalization') 133 | x = leakyReLu(x) 134 | 135 | # D(z) 136 | name_z = 'z_layer_1' 137 | with tf.variable_scope(name_z): 138 | z = tf.layers.dense(z_inp, 128, kernel_initializer=init_kernel) 139 | z = leakyReLu(z) 140 | z = tf.layers.dropout(z, rate=0.5, name='dropout', training=is_training) 141 | 142 | 143 | # D(x,z) 144 | y = tf.concat([x, z], axis=1) 145 | 146 | name_y = 'y_layer_1' 147 | with tf.variable_scope(name_y): 148 | y = tf.layers.dense(y, 149 | 256, 150 | kernel_initializer=init_kernel) 151 | y = leakyReLu(y) 152 | y = tf.layers.dropout(y, rate=0.5, name='dropout', training=is_training) 153 | 154 | 155 | intermediate_layer = y 156 | 157 | name_y = 'y_layer_2' 158 | with tf.variable_scope(name_y): 159 | logits = tf.layers.dense(y, 160 | 1, 161 | kernel_initializer=init_kernel) 162 | 163 | return logits, intermediate_layer 164 | 165 | def discriminator_xx(x, rec_x, is_training=False,getter=None, reuse=False, 166 | do_spectral_norm=False): 167 | """ Discriminator architecture in tensorflow 168 | 169 | Discriminates between (x,x) and (x,rec_x) 170 | 171 | Args: 172 | x (tensor): input from the data space 173 | rec_x (tensor): reconstructed data 174 | is_training (bool): for batch norms and dropouts 175 | getter: for exponential moving average during inference 176 | reuse (bool): sharing variables or not 177 | 178 | Returns: 179 | logits (tensor): last activation layer of the discriminator 180 | intermediate_layer (tensor): intermediate layer for feature matching 181 | 182 | """ 183 | with tf.variable_scope('discriminator_xx', reuse=reuse, custom_getter=getter): 184 | net = tf.concat([x, rec_x], axis=1) 185 | 186 | name_net = 'layer_1' 187 | with tf.variable_scope(name_net): 188 | net = tf.layers.dense(net, 189 | units=256, 190 | kernel_initializer=init_kernel, 191 | name='fc') 192 | net = leakyReLu(net) 193 | net = tf.layers.dropout(net, rate=0.2, name='dropout', training=is_training) 194 | 195 | name_net = 'layer_2' 196 | with tf.variable_scope(name_net): 197 | net = tf.layers.dense(net, 198 | units=128, 199 | kernel_initializer=init_kernel, 200 | name='fc') 201 | net = leakyReLu(net) 202 | net = tf.layers.dropout(net, rate=0.2, name='dropout', training=is_training) 203 | 204 | intermediate_layer = net 205 | 206 | name_net = 'layer_3' 207 | with tf.variable_scope(name_net): 208 | logits = tf.layers.dense(net, 209 | units=1, 210 | kernel_initializer=init_kernel, 211 | name='fc') 212 | 213 | return logits, intermediate_layer 214 | 215 | def discriminator_zz(z, rec_z, is_training=False, getter=None, reuse=False, 216 | do_spectral_norm=False): 217 | """ Discriminator architecture in tensorflow 218 | 219 | Discriminates between (z,z) and (z,rec_z) 220 | 221 | Args: 222 | z (tensor): input from the latent space 223 | rec_z (tensor): reconstructed data 224 | is_training (bool): for batch norms and dropouts 225 | getter: for exponential moving average during inference 226 | reuse (bool): sharing variables or not 227 | 228 | Returns: 229 | logits (tensor): last activation layer of the discriminator 230 | intermediate_layer (tensor): intermediate layer for feature matching 231 | 232 | """ 233 | with tf.variable_scope('discriminator_zz', reuse=reuse, custom_getter=getter): 234 | 235 | net = tf.concat([z, rec_z], axis=-1) 236 | name_net = 'layer_1' 237 | with tf.variable_scope(name_net): 238 | net = tf.layers.dense(net, 239 | units=64, 240 | kernel_initializer=init_kernel, 241 | name='fc') 242 | 243 | net = leakyReLu(net, 0.2, name='conv1/leaky_relu') 244 | net = tf.layers.dropout(net, rate=0.2, name='dropout', 245 | training=is_training) 246 | 247 | name_net = 'layer_2' 248 | with tf.variable_scope(name_net): 249 | net = tf.layers.dense(net, 250 | units=32, 251 | kernel_initializer=init_kernel, 252 | name='fc') 253 | 254 | net = leakyReLu(net, 0.2, name='conv1/leaky_relu') 255 | net = tf.layers.dropout(net, rate=0.2, name='dropout', 256 | training=is_training) 257 | intermediate_layer = net 258 | 259 | name_net = 'layer_3' 260 | with tf.variable_scope(name_net): 261 | logits = tf.layers.dense(net, 262 | units=1, 263 | kernel_initializer=init_kernel, 264 | name='fc') 265 | 266 | return logits, intermediate_layer -------------------------------------------------------------------------------- /alad/cifar10_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | CIFAR10 ALAD architecture. 4 | 5 | Generator (decoder), encoder and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | from utils import sn 10 | 11 | learning_rate = 0.0002 12 | batch_size = 32 13 | latent_dim = 100 14 | init_kernel = tf.random_normal_initializer(mean=0.0, stddev=0.01) 15 | 16 | def leakyReLu(x, alpha=0.2, name=None): 17 | if name: 18 | with tf.variable_scope(name): 19 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 20 | else: 21 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 22 | 23 | def encoder(x_inp, is_training=False, getter=None, reuse=False, 24 | do_spectral_norm=True): 25 | """ Encoder architecture in tensorflow 26 | 27 | Maps the data into the latent space 28 | 29 | Args: 30 | x_inp (tensor): input data for the encoder. 31 | is_training (bool): for batch norms and dropouts 32 | getter: for exponential moving average during inference 33 | reuse (bool): sharing variables or not 34 | 35 | Returns: 36 | net (tensor): last activation layer of the encoder 37 | 38 | """ 39 | layers = sn if do_spectral_norm else tf.layers 40 | 41 | with tf.variable_scope('encoder', reuse=reuse, custom_getter=getter): 42 | x_inp = tf.reshape(x_inp, [-1, 32, 32, 3]) 43 | 44 | name_net = 'layer_1' 45 | with tf.variable_scope(name_net): 46 | net = layers.conv2d(x_inp, 47 | 128, 48 | kernel_size=4, 49 | padding='SAME', 50 | strides=2, 51 | kernel_initializer=init_kernel, 52 | name='conv') 53 | net = tf.layers.batch_normalization(net, 54 | training=is_training) 55 | net = leakyReLu(net, name='leaky_relu') 56 | 57 | name_net = 'layer_2' 58 | with tf.variable_scope(name_net): 59 | net = layers.conv2d(net, 60 | 256, 61 | kernel_size=4, 62 | padding='SAME', 63 | strides=2, 64 | kernel_initializer=init_kernel, 65 | name='conv') 66 | net = tf.layers.batch_normalization(net, 67 | training=is_training) 68 | net = leakyReLu(net, name='leaky_relu') 69 | 70 | name_net = 'layer_3' 71 | with tf.variable_scope(name_net): 72 | net = layers.conv2d(net, 73 | 512, 74 | kernel_size=4, 75 | padding='SAME', 76 | strides=2, 77 | kernel_initializer=init_kernel, 78 | name='conv') 79 | net = tf.layers.batch_normalization(net, 80 | training=is_training) 81 | net = leakyReLu(net, name='leaky_relu') 82 | 83 | name_net = 'layer_4' 84 | with tf.variable_scope(name_net): 85 | net = tf.layers.conv2d(net, 86 | latent_dim, 87 | kernel_size=4, 88 | strides=1, 89 | padding='VALID', 90 | kernel_initializer=init_kernel, 91 | name='conv') 92 | net = tf.squeeze(net, [1, 2]) 93 | 94 | return net 95 | 96 | def decoder(z_inp, is_training=False, getter=None, reuse=False): 97 | """ Generator architecture in tensorflow 98 | 99 | Generates data from the latent space 100 | 101 | Args: 102 | z_inp (tensor): input variable in the latent space 103 | is_training (bool): for batch norms and dropouts 104 | getter: for exponential moving average during inference 105 | reuse (bool): sharing variables or not 106 | 107 | Returns: 108 | net (tensor): last activation layer of the generator 109 | 110 | """ 111 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 112 | net = tf.reshape(z_inp, [-1, 1, 1, latent_dim]) 113 | name_net = 'layer_1' 114 | with tf.variable_scope(name_net): 115 | net = tf.layers.conv2d_transpose(net, 116 | filters=512, 117 | kernel_size=4, 118 | strides=2, 119 | padding='VALID', 120 | kernel_initializer=init_kernel, 121 | name='tconv1') 122 | 123 | net = tf.layers.batch_normalization(net, 124 | training=is_training, 125 | name='tconv1/batch_normalization') 126 | 127 | net = tf.nn.relu(net, name='tconv1/relu') 128 | 129 | name_net = 'layer_2' 130 | with tf.variable_scope(name_net): 131 | net = tf.layers.conv2d_transpose(net, 132 | filters=256, 133 | kernel_size=4, 134 | strides=2, 135 | padding='SAME', 136 | kernel_initializer=init_kernel, 137 | name='tconv2') 138 | 139 | net = tf.layers.batch_normalization(net, 140 | training=is_training, 141 | name='tconv2/batch_normalization') 142 | net = tf.nn.relu(net, name='tconv2/relu') 143 | 144 | name_net = 'layer_3' 145 | with tf.variable_scope(name_net): 146 | net = tf.layers.conv2d_transpose(net, 147 | filters=128, 148 | kernel_size=4, 149 | strides=2, 150 | padding='SAME', 151 | kernel_initializer=init_kernel, 152 | name='tconv3') 153 | 154 | net = tf.layers.batch_normalization(net, 155 | training=is_training, 156 | name='tconv3/batch_normalization') 157 | net = tf.nn.relu(net, name='tconv3/relu') 158 | 159 | name_net = 'layer_4' 160 | with tf.variable_scope(name_net): 161 | net = tf.layers.conv2d_transpose(net, 162 | filters=3, 163 | kernel_size=4, 164 | strides=2, 165 | padding='SAME', 166 | kernel_initializer=init_kernel, 167 | name='tconv4') 168 | 169 | net = tf.tanh(net, name='tconv4/tanh') 170 | 171 | return net 172 | 173 | 174 | def discriminator_xz(x_inp, z_inp, is_training=False, getter=None, reuse=False, 175 | do_spectral_norm=True): 176 | """ Discriminator architecture in tensorflow 177 | 178 | Discriminates between pairs (E(x), x) and (z, G(z)) 179 | 180 | Args: 181 | x_inp (tensor): input data for the discriminator. 182 | z_inp (tensor): input variable in the latent space 183 | is_training (bool): for batch norms and dropouts 184 | getter: for exponential moving average during inference 185 | reuse (bool): sharing variables or not 186 | 187 | Returns: 188 | logits (tensor): last activation layer of the discriminator (shape 1) 189 | intermediate_layer (tensor): intermediate layer for feature matching 190 | 191 | """ 192 | layers = sn if do_spectral_norm else tf.layers 193 | 194 | with tf.variable_scope('discriminator_xz', reuse=reuse, custom_getter=getter): 195 | name_net = 'x_layer_1' 196 | with tf.variable_scope(name_net): 197 | x = layers.conv2d(x_inp, 198 | filters=128, 199 | kernel_size=4, 200 | strides=2, 201 | padding='SAME', 202 | kernel_initializer=init_kernel, 203 | name='conv1') 204 | 205 | x = leakyReLu(x, 0.2, name='conv1/leaky_relu') 206 | 207 | name_net = 'x_layer_2' 208 | with tf.variable_scope(name_net): 209 | x = layers.conv2d(x, 210 | filters=256, 211 | kernel_size=4, 212 | strides=2, 213 | padding='SAME', 214 | kernel_initializer=init_kernel, 215 | name='conv2') 216 | 217 | x = tf.layers.batch_normalization(x, 218 | training=is_training, 219 | name='conv2/batch_normalization') 220 | 221 | x = leakyReLu(x, 0.2, name='conv2/leaky_relu') 222 | 223 | name_net = 'x_layer_3' 224 | with tf.variable_scope(name_net): 225 | x = layers.conv2d(x, 226 | filters=512, 227 | kernel_size=4, 228 | strides=2, 229 | padding='SAME', 230 | kernel_initializer=init_kernel, 231 | name='conv3') 232 | 233 | x = tf.layers.batch_normalization(x, 234 | training=is_training, 235 | name='conv3/batch_normalization') 236 | 237 | x = leakyReLu(x, 0.2, name='conv3/leaky_relu') 238 | 239 | x = tf.reshape(x, [-1,1,1,512*4*4]) 240 | 241 | z = tf.reshape(z_inp, [-1, 1, 1, latent_dim]) 242 | 243 | name_net = 'z_layer_1' 244 | with tf.variable_scope(name_net): 245 | z = layers.conv2d(z, 246 | filters=512, 247 | kernel_size=1, 248 | strides=1, 249 | padding='SAME', 250 | kernel_initializer=init_kernel, 251 | name='conv') 252 | z = leakyReLu(z) 253 | z = tf.layers.dropout(z, rate=0.2, training=is_training, 254 | name='dropout') 255 | 256 | name_net = 'z_layer_2' 257 | with tf.variable_scope(name_net): 258 | z = layers.conv2d(z, 259 | filters=512, 260 | kernel_size=1, 261 | strides=1, 262 | padding='SAME', 263 | kernel_initializer=init_kernel, 264 | name='conv') 265 | z = leakyReLu(z) 266 | z = tf.layers.dropout(z, rate=0.2, training=is_training, 267 | name='dropout') 268 | 269 | y = tf.concat([x, z], axis=-1) 270 | 271 | name_net = 'y_layer_1' 272 | with tf.variable_scope(name_net): 273 | y = layers.conv2d(y, 274 | filters=1024, 275 | kernel_size=1, 276 | strides=1, 277 | padding='SAME', 278 | kernel_initializer=init_kernel, 279 | name='conv') 280 | y = leakyReLu(y) 281 | y = tf.layers.dropout(y, rate=0.2, training=is_training, 282 | name='dropout') 283 | 284 | intermediate_layer = y 285 | 286 | name_net = 'y_layer_2' 287 | with tf.variable_scope(name_net): 288 | y = tf.layers.conv2d(y, 289 | filters=1, 290 | kernel_size=1, 291 | strides=1, 292 | padding='SAME', 293 | kernel_initializer=init_kernel, 294 | name='conv') 295 | 296 | logits = tf.squeeze(y) 297 | 298 | return logits, intermediate_layer 299 | 300 | def discriminator_xx(x, rec_x, is_training=False, getter=None, reuse=False, 301 | do_spectral_norm=True): 302 | """ Discriminator architecture in tensorflow 303 | 304 | Discriminates between (x,x) and (x,rec_x) 305 | 306 | Args: 307 | x (tensor): input from the data space 308 | rec_x (tensor): reconstructed data 309 | is_training (bool): for batch norms and dropouts 310 | getter: for exponential moving average during inference 311 | reuse (bool): sharing variables or not 312 | 313 | Returns: 314 | logits (tensor): last activation layer of the discriminator 315 | intermediate_layer (tensor): intermediate layer for feature matching 316 | 317 | """ 318 | layers = sn if do_spectral_norm else tf.layers 319 | 320 | with tf.variable_scope('discriminator_xx', reuse=reuse, custom_getter=getter): 321 | 322 | net = tf.concat([x, rec_x], axis=1) 323 | 324 | name_net = 'layer_1' 325 | with tf.variable_scope(name_net): 326 | net = layers.conv2d(net, 327 | filters=64, 328 | kernel_size=5, 329 | strides=2, 330 | padding='SAME', 331 | kernel_initializer=init_kernel, 332 | name='conv1') 333 | 334 | net = leakyReLu(net, 0.2, name='conv1/leaky_relu') 335 | 336 | net = tf.layers.dropout(net, rate=0.2, training=is_training, 337 | name='dropout') 338 | with tf.variable_scope(name_net, reuse=True): 339 | weights = tf.get_variable('conv1/kernel') 340 | 341 | name_net = 'layer_2' 342 | with tf.variable_scope(name_net): 343 | net = layers.conv2d(net, 344 | filters=128, 345 | kernel_size=5, 346 | strides=2, 347 | padding='SAME', 348 | kernel_initializer=init_kernel, 349 | name='conv2') 350 | 351 | net = leakyReLu(net, 0.2, name='conv2/leaky_relu') 352 | 353 | net = tf.layers.dropout(net, rate=0.2, training=is_training, 354 | name='dropout') 355 | 356 | net = tf.contrib.layers.flatten(net) 357 | 358 | intermediate_layer = net 359 | name_net = 'layer_3' 360 | with tf.variable_scope(name_net): 361 | net = tf.layers.dense(net, 362 | units=1, 363 | kernel_initializer=init_kernel, 364 | name='fc') 365 | 366 | logits = tf.squeeze(net) 367 | 368 | return logits, intermediate_layer 369 | 370 | def discriminator_zz(z, rec_z, is_training=False, getter=None, reuse=False, 371 | do_spectral_norm=True): 372 | """ Discriminator architecture in tensorflow 373 | 374 | Discriminates between (z,z) and (z,rec_z) 375 | 376 | Args: 377 | z (tensor): input from the latent space 378 | rec_z (tensor): reconstructed data 379 | is_training (bool): for batch norms and dropouts 380 | getter: for exponential moving average during inference 381 | reuse (bool): sharing variables or not 382 | 383 | Returns: 384 | logits (tensor): last activation layer of the discriminator 385 | intermediate_layer (tensor): intermediate layer for feature matching 386 | 387 | """ 388 | layers = sn if do_spectral_norm else tf.layers 389 | 390 | with tf.variable_scope('discriminator_zz', reuse=reuse, 391 | custom_getter=getter): 392 | 393 | y = tf.concat([z, rec_z], axis=-1) 394 | 395 | name_net = 'y_layer_1' 396 | with tf.variable_scope(name_net): 397 | y = layers.dense(y, units=64, kernel_initializer=init_kernel, 398 | name='fc') 399 | 400 | y = leakyReLu(y) 401 | y = tf.layers.dropout(y, rate=0.2, training=is_training, 402 | name='dropout') 403 | 404 | name_net = 'y_layer_2' 405 | with tf.variable_scope(name_net): 406 | y = layers.dense(y, 407 | units=32, 408 | kernel_initializer=init_kernel, 409 | name='fc') 410 | 411 | y = leakyReLu(y) 412 | y = tf.layers.dropout(y, rate=0.2, training=is_training, 413 | name='dropout') 414 | 415 | intermediate_layer = y 416 | 417 | name_net = 'y_layer_3' 418 | with tf.variable_scope(name_net): 419 | y = tf.layers.dense(y, 420 | units=1, 421 | kernel_initializer=init_kernel, 422 | name='fc') 423 | 424 | logits = tf.squeeze(y) 425 | 426 | return logits, intermediate_layer 427 | -------------------------------------------------------------------------------- /alad/kdd_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | KDD ALAD architecture. 4 | 5 | Generator (decoder), encoder and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | from utils import sn 10 | 11 | learning_rate = 1e-5 12 | batch_size = 50 13 | latent_dim = 32 14 | init_kernel = tf.contrib.layers.xavier_initializer() 15 | 16 | def leakyReLu(x, alpha=0.2, name=None): 17 | if name: 18 | with tf.variable_scope(name): 19 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 20 | else: 21 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 22 | 23 | def encoder(x_inp, is_training=False, getter=None, reuse=False, 24 | do_spectral_norm=False): 25 | """ Encoder architecture in tensorflow 26 | 27 | Maps the data into the latent space 28 | 29 | Args: 30 | x_inp (tensor): input data for the encoder. 31 | is_training (bool): for batch norms and dropouts 32 | getter: for exponential moving average during inference 33 | reuse (bool): sharing variables or not 34 | 35 | Returns: 36 | net (tensor): last activation layer of the encoder 37 | 38 | """ 39 | with tf.variable_scope('encoder', reuse=reuse, custom_getter=getter): 40 | 41 | name_net = 'layer_1' 42 | with tf.variable_scope(name_net): 43 | net = tf.layers.dense(x_inp, 44 | units=64, 45 | kernel_initializer=init_kernel, 46 | name='fc') 47 | net = leakyReLu(net) 48 | 49 | 50 | name_net = 'layer_2' 51 | with tf.variable_scope(name_net): 52 | net = tf.layers.dense(net, 53 | units=latent_dim, 54 | kernel_initializer=init_kernel, 55 | name='fc') 56 | return net 57 | 58 | def decoder(z_inp, is_training=False, getter=None, reuse=False): 59 | """ Generator architecture in tensorflow 60 | 61 | Generates data from the latent space 62 | 63 | Args: 64 | z_inp (tensor): input variable in the latent space 65 | is_training (bool): for batch norms and dropouts 66 | getter: for exponential moving average during inference 67 | reuse (bool): sharing variables or not 68 | 69 | Returns: 70 | net (tensor): last activation layer of the generator 71 | 72 | """ 73 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 74 | name_net = 'layer_1' 75 | with tf.variable_scope(name_net): 76 | net = tf.layers.dense(z_inp, 77 | units=64, 78 | kernel_initializer=init_kernel, 79 | name='fc') 80 | net = tf.nn.relu(net) 81 | 82 | name_net = 'layer_2' 83 | with tf.variable_scope(name_net): 84 | net = tf.layers.dense(net, 85 | units=128, 86 | kernel_initializer=init_kernel, 87 | name='fc') 88 | net = tf.nn.relu(net) 89 | 90 | name_net = 'layer_3' 91 | with tf.variable_scope(name_net): 92 | net = tf.layers.dense(net, 93 | units=121, 94 | kernel_initializer=init_kernel, 95 | name='fc') 96 | 97 | return net 98 | 99 | def discriminator_xz(x_inp, z_inp, is_training=False, getter=None, reuse=False, 100 | do_spectral_norm=False): 101 | """ Discriminator architecture in tensorflow 102 | 103 | Discriminates between pairs (E(x), x) and (z, G(z)) 104 | 105 | Args: 106 | x_inp (tensor): input data for the discriminator. 107 | z_inp (tensor): input variable in the latent space 108 | is_training (bool): for batch norms and dropouts 109 | getter: for exponential moving average during inference 110 | reuse (bool): sharing variables or not 111 | 112 | Returns: 113 | logits (tensor): last activation layer of the discriminator (shape 1) 114 | intermediate_layer (tensor): intermediate layer for feature matching 115 | 116 | """ 117 | with tf.variable_scope('discriminator_xz', reuse=reuse, custom_getter=getter): 118 | # D(x) 119 | name_x = 'x_layer_1' 120 | with tf.variable_scope(name_x): 121 | x = tf.layers.dense(x_inp, 122 | units=128, 123 | kernel_initializer=init_kernel, 124 | name='fc') 125 | x = tf.layers.batch_normalization(x, 126 | training=is_training, 127 | name='batch_normalization') 128 | x = leakyReLu(x) 129 | 130 | # D(z) 131 | name_z = 'z_layer_1' 132 | with tf.variable_scope(name_z): 133 | z = tf.layers.dense(z_inp, 128, kernel_initializer=init_kernel) 134 | z = leakyReLu(z) 135 | z = tf.layers.dropout(z, rate=0.5, name='dropout', training=is_training) 136 | 137 | # D(x,z) 138 | y = tf.concat([x, z], axis=1) 139 | 140 | name_y = 'y_layer_1' 141 | with tf.variable_scope(name_y): 142 | y = tf.layers.dense(y, 143 | 128, 144 | kernel_initializer=init_kernel) 145 | y = leakyReLu(y) 146 | y = tf.layers.dropout(y, rate=0.5, name='dropout', training=is_training) 147 | 148 | intermediate_layer = y 149 | 150 | name_y = 'y_layer_2' 151 | with tf.variable_scope(name_y): 152 | logits = tf.layers.dense(y, 153 | 1, 154 | kernel_initializer=init_kernel) 155 | 156 | return logits, intermediate_layer 157 | 158 | def discriminator_xx(x, rec_x, is_training=False,getter=None, reuse=False, 159 | do_spectral_norm=False): 160 | """ Discriminator architecture in tensorflow 161 | 162 | Discriminates between (x,x) and (x,rec_x) 163 | 164 | Args: 165 | x (tensor): input from the data space 166 | rec_x (tensor): reconstructed data 167 | is_training (bool): for batch norms and dropouts 168 | getter: for exponential moving average during inference 169 | reuse (bool): sharing variables or not 170 | 171 | Returns: 172 | logits (tensor): last activation layer of the discriminator 173 | intermediate_layer (tensor): intermediate layer for feature matching 174 | 175 | """ 176 | with tf.variable_scope('discriminator_xx', reuse=reuse, custom_getter=getter): 177 | net = tf.concat([x, rec_x], axis=1) 178 | 179 | name_net = 'layer_1' 180 | with tf.variable_scope(name_net): 181 | net = tf.layers.dense(net, 182 | units=128, 183 | kernel_initializer=init_kernel, 184 | name='fc') 185 | net = leakyReLu(net) 186 | net = tf.layers.dropout(net, rate=0.2, name='dropout', training=is_training) 187 | 188 | intermediate_layer = net 189 | 190 | name_net = 'layer_2' 191 | with tf.variable_scope(name_net): 192 | logits = tf.layers.dense(net, 193 | units=1, 194 | kernel_initializer=init_kernel, 195 | name='fc') 196 | 197 | return logits, intermediate_layer 198 | 199 | def discriminator_zz(z, rec_z, is_training=False, getter=None, reuse=False, 200 | do_spectral_norm=False): 201 | """ Discriminator architecture in tensorflow 202 | 203 | Discriminates between (z,z) and (z,rec_z) 204 | 205 | Args: 206 | z (tensor): input from the latent space 207 | rec_z (tensor): reconstructed data 208 | is_training (bool): for batch norms and dropouts 209 | getter: for exponential moving average during inference 210 | reuse (bool): sharing variables or not 211 | 212 | Returns: 213 | logits (tensor): last activation layer of the discriminator 214 | intermediate_layer (tensor): intermediate layer for feature matching 215 | 216 | """ 217 | with tf.variable_scope('discriminator_zz', reuse=reuse, custom_getter=getter): 218 | 219 | net = tf.concat([z, rec_z], axis=-1) 220 | 221 | name_net = 'layer_1' 222 | with tf.variable_scope(name_net): 223 | net = tf.layers.dense(net, 224 | units=32, 225 | kernel_initializer=init_kernel, 226 | name='fc') 227 | net = leakyReLu(net, 0.2, name='conv2/leaky_relu') 228 | net = tf.layers.dropout(net, rate=0.2, name='dropout', 229 | training=is_training) 230 | 231 | intermediate_layer = net 232 | 233 | name_net = 'layer_2' 234 | with tf.variable_scope(name_net): 235 | logits = tf.layers.dense(net, 236 | units=1, 237 | kernel_initializer=init_kernel, 238 | name='fc') 239 | 240 | return logits, intermediate_layer 241 | -------------------------------------------------------------------------------- /alad/svhn_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | CIFAR10 ALAD architecture. 4 | 5 | Generator (decoder), encoder and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | from utils import sn 10 | 11 | learning_rate = 0.0002 12 | batch_size = 32 13 | latent_dim = 100 14 | init_kernel = tf.random_normal_initializer(mean=0.0, stddev=0.01) 15 | 16 | def leakyReLu(x, alpha=0.2, name=None): 17 | if name: 18 | with tf.variable_scope(name): 19 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 20 | else: 21 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 22 | 23 | def encoder(x_inp, is_training=False, getter=None, reuse=False, 24 | do_spectral_norm=True): 25 | """ Encoder architecture in tensorflow 26 | 27 | Maps the data into the latent space 28 | 29 | Args: 30 | x_inp (tensor): input data for the encoder. 31 | is_training (bool): for batch norms and dropouts 32 | getter: for exponential moving average during inference 33 | reuse (bool): sharing variables or not 34 | 35 | Returns: 36 | net (tensor): last activation layer of the encoder 37 | 38 | """ 39 | layers = sn if do_spectral_norm else tf.layers 40 | 41 | with tf.variable_scope('encoder', reuse=reuse, custom_getter=getter): 42 | x_inp = tf.reshape(x_inp, [-1, 32, 32, 3]) 43 | 44 | name_net = 'layer_1' 45 | with tf.variable_scope(name_net): 46 | net = layers.conv2d(x_inp, 47 | 128, 48 | kernel_size=4, 49 | padding='SAME', 50 | strides=2, 51 | kernel_initializer=init_kernel, 52 | name='conv') 53 | net = tf.layers.batch_normalization(net, 54 | training=is_training) 55 | net = leakyReLu(net, name='leaky_relu') 56 | 57 | name_net = 'layer_2' 58 | with tf.variable_scope(name_net): 59 | net = layers.conv2d(net, 60 | 256, 61 | kernel_size=4, 62 | padding='SAME', 63 | strides=2, 64 | kernel_initializer=init_kernel, 65 | name='conv') 66 | net = tf.layers.batch_normalization(net, 67 | training=is_training) 68 | net = leakyReLu(net, name='leaky_relu') 69 | 70 | name_net = 'layer_3' 71 | with tf.variable_scope(name_net): 72 | net = layers.conv2d(net, 73 | 512, 74 | kernel_size=4, 75 | padding='SAME', 76 | strides=2, 77 | kernel_initializer=init_kernel, 78 | name='conv') 79 | net = tf.layers.batch_normalization(net, 80 | training=is_training) 81 | net = leakyReLu(net, name='leaky_relu') 82 | 83 | name_net = 'layer_4' 84 | with tf.variable_scope(name_net): 85 | net = tf.layers.conv2d(net, 86 | latent_dim, 87 | kernel_size=4, 88 | strides=1, 89 | padding='VALID', 90 | kernel_initializer=init_kernel, 91 | name='conv') 92 | net = tf.squeeze(net, [1, 2]) 93 | 94 | return net 95 | 96 | def decoder(z_inp, is_training=False, getter=None, reuse=False): 97 | """ Generator architecture in tensorflow 98 | 99 | Generates data from the latent space 100 | 101 | Args: 102 | z_inp (tensor): input variable in the latent space 103 | is_training (bool): for batch norms and dropouts 104 | getter: for exponential moving average during inference 105 | reuse (bool): sharing variables or not 106 | 107 | Returns: 108 | net (tensor): last activation layer of the generator 109 | 110 | """ 111 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 112 | net = tf.reshape(z_inp, [-1, 1, 1, latent_dim]) 113 | name_net = 'layer_1' 114 | with tf.variable_scope(name_net): 115 | net = tf.layers.conv2d_transpose(net, 116 | filters=512, 117 | kernel_size=4, 118 | strides=2, 119 | padding='VALID', 120 | kernel_initializer=init_kernel, 121 | name='tconv1') 122 | 123 | net = tf.layers.batch_normalization(net, 124 | training=is_training, 125 | name='tconv1/batch_normalization') 126 | 127 | net = tf.nn.relu(net, name='tconv1/relu') 128 | 129 | name_net = 'layer_2' 130 | with tf.variable_scope(name_net): 131 | net = tf.layers.conv2d_transpose(net, 132 | filters=256, 133 | kernel_size=4, 134 | strides=2, 135 | padding='SAME', 136 | kernel_initializer=init_kernel, 137 | name='tconv2') 138 | 139 | net = tf.layers.batch_normalization(net, 140 | training=is_training, 141 | name='tconv2/batch_normalization') 142 | net = tf.nn.relu(net, name='tconv2/relu') 143 | 144 | name_net = 'layer_3' 145 | with tf.variable_scope(name_net): 146 | net = tf.layers.conv2d_transpose(net, 147 | filters=128, 148 | kernel_size=4, 149 | strides=2, 150 | padding='SAME', 151 | kernel_initializer=init_kernel, 152 | name='tconv3') 153 | 154 | net = tf.layers.batch_normalization(net, 155 | training=is_training, 156 | name='tconv3/batch_normalization') 157 | net = tf.nn.relu(net, name='tconv3/relu') 158 | 159 | name_net = 'layer_4' 160 | with tf.variable_scope(name_net): 161 | net = tf.layers.conv2d_transpose(net, 162 | filters=3, 163 | kernel_size=4, 164 | strides=2, 165 | padding='SAME', 166 | kernel_initializer=init_kernel, 167 | name='tconv4') 168 | 169 | net = tf.tanh(net, name='tconv4/tanh') 170 | 171 | return net 172 | 173 | 174 | def discriminator_xz(x_inp, z_inp, is_training=False, getter=None, reuse=False, 175 | do_spectral_norm=True): 176 | """ Discriminator architecture in tensorflow 177 | 178 | Discriminates between pairs (E(x), x) and (z, G(z)) 179 | 180 | Args: 181 | x_inp (tensor): input data for the discriminator. 182 | z_inp (tensor): input variable in the latent space 183 | is_training (bool): for batch norms and dropouts 184 | getter: for exponential moving average during inference 185 | reuse (bool): sharing variables or not 186 | 187 | Returns: 188 | logits (tensor): last activation layer of the discriminator (shape 1) 189 | intermediate_layer (tensor): intermediate layer for feature matching 190 | 191 | """ 192 | layers = sn if do_spectral_norm else tf.layers 193 | 194 | with tf.variable_scope('discriminator_xz', reuse=reuse, custom_getter=getter): 195 | name_net = 'x_layer_1' 196 | with tf.variable_scope(name_net): 197 | x = layers.conv2d(x_inp, 198 | filters=128, 199 | kernel_size=4, 200 | strides=2, 201 | padding='SAME', 202 | kernel_initializer=init_kernel, 203 | name='conv1') 204 | 205 | x = leakyReLu(x, 0.2, name='conv1/leaky_relu') 206 | 207 | name_net = 'x_layer_2' 208 | with tf.variable_scope(name_net): 209 | x = layers.conv2d(x, 210 | filters=256, 211 | kernel_size=4, 212 | strides=2, 213 | padding='SAME', 214 | kernel_initializer=init_kernel, 215 | name='conv2') 216 | 217 | x = tf.layers.batch_normalization(x, 218 | training=is_training, 219 | name='conv2/batch_normalization') 220 | 221 | x = leakyReLu(x, 0.2, name='conv2/leaky_relu') 222 | 223 | name_net = 'x_layer_3' 224 | with tf.variable_scope(name_net): 225 | x = layers.conv2d(x, 226 | filters=512, 227 | kernel_size=4, 228 | strides=2, 229 | padding='SAME', 230 | kernel_initializer=init_kernel, 231 | name='conv3') 232 | 233 | x = tf.layers.batch_normalization(x, 234 | training=is_training, 235 | name='conv3/batch_normalization') 236 | 237 | x = leakyReLu(x, 0.2, name='conv3/leaky_relu') 238 | 239 | x = tf.reshape(x, [-1,1,1,512*4*4]) 240 | 241 | z = tf.reshape(z_inp, [-1, 1, 1, latent_dim]) 242 | 243 | name_net = 'z_layer_1' 244 | with tf.variable_scope(name_net): 245 | z = layers.conv2d(z, 246 | filters=512, 247 | kernel_size=1, 248 | strides=1, 249 | padding='SAME', 250 | kernel_initializer=init_kernel, 251 | name='conv') 252 | z = leakyReLu(z) 253 | z = tf.layers.dropout(z, rate=0.2, training=is_training, 254 | name='dropout') 255 | 256 | name_net = 'z_layer_2' 257 | with tf.variable_scope(name_net): 258 | z = layers.conv2d(z, 259 | filters=512, 260 | kernel_size=1, 261 | strides=1, 262 | padding='SAME', 263 | kernel_initializer=init_kernel, 264 | name='conv') 265 | z = leakyReLu(z) 266 | z = tf.layers.dropout(z, rate=0.2, training=is_training, 267 | name='dropout') 268 | 269 | y = tf.concat([x, z], axis=-1) 270 | 271 | name_net = 'y_layer_1' 272 | with tf.variable_scope(name_net): 273 | y = layers.conv2d(y, 274 | filters=1024, 275 | kernel_size=1, 276 | strides=1, 277 | padding='SAME', 278 | kernel_initializer=init_kernel, 279 | name='conv') 280 | y = leakyReLu(y) 281 | y = tf.layers.dropout(y, rate=0.2, training=is_training, 282 | name='dropout') 283 | 284 | intermediate_layer = y 285 | 286 | name_net = 'y_layer_2' 287 | with tf.variable_scope(name_net): 288 | y = tf.layers.conv2d(y, 289 | filters=1, 290 | kernel_size=1, 291 | strides=1, 292 | padding='SAME', 293 | kernel_initializer=init_kernel, 294 | name='conv') 295 | 296 | logits = tf.squeeze(y) 297 | 298 | return logits, intermediate_layer 299 | 300 | def discriminator_xx(x, rec_x, is_training=False, getter=None, reuse=False, 301 | do_spectral_norm=True): 302 | """ Discriminator architecture in tensorflow 303 | 304 | Discriminates between (x,x) and (x,rec_x) 305 | 306 | Args: 307 | x (tensor): input from the data space 308 | rec_x (tensor): reconstructed data 309 | is_training (bool): for batch norms and dropouts 310 | getter: for exponential moving average during inference 311 | reuse (bool): sharing variables or not 312 | 313 | Returns: 314 | logits (tensor): last activation layer of the discriminator 315 | intermediate_layer (tensor): intermediate layer for feature matching 316 | 317 | """ 318 | layers = sn if do_spectral_norm else tf.layers 319 | 320 | with tf.variable_scope('discriminator_xx', reuse=reuse, custom_getter=getter): 321 | 322 | net = tf.concat([x, rec_x], axis=1) 323 | 324 | name_net = 'layer_1' 325 | with tf.variable_scope(name_net): 326 | net = layers.conv2d(net, 327 | filters=64, 328 | kernel_size=5, 329 | strides=2, 330 | padding='SAME', 331 | kernel_initializer=init_kernel, 332 | name='conv1') 333 | 334 | net = leakyReLu(net, 0.2, name='conv1/leaky_relu') 335 | 336 | net = tf.layers.dropout(net, rate=0.2, training=is_training, 337 | name='dropout') 338 | with tf.variable_scope(name_net, reuse=True): 339 | weights = tf.get_variable('conv1/kernel') 340 | 341 | name_net = 'layer_2' 342 | with tf.variable_scope(name_net): 343 | net = layers.conv2d(net, 344 | filters=128, 345 | kernel_size=5, 346 | strides=2, 347 | padding='SAME', 348 | kernel_initializer=init_kernel, 349 | name='conv2') 350 | net = leakyReLu(net, 0.2, name='conv2/leaky_relu') 351 | 352 | net = tf.layers.dropout(net, rate=0.2, training=is_training, 353 | name='dropout') 354 | 355 | net = tf.contrib.layers.flatten(net) 356 | 357 | intermediate_layer = net 358 | name_net = 'layer_3' 359 | with tf.variable_scope(name_net): 360 | net = tf.layers.dense(net, 361 | units=1, 362 | kernel_initializer=init_kernel, 363 | name='fc') 364 | 365 | logits = tf.squeeze(net) 366 | 367 | return logits, intermediate_layer 368 | 369 | def discriminator_zz(z, rec_z, is_training=False, getter=None, reuse=False, 370 | do_spectral_norm=True): 371 | """ Discriminator architecture in tensorflow 372 | 373 | Discriminates between (z,z) and (z,rec_z) 374 | 375 | Args: 376 | z (tensor): input from the latent space 377 | rec_z (tensor): reconstructed data 378 | is_training (bool): for batch norms and dropouts 379 | getter: for exponential moving average during inference 380 | reuse (bool): sharing variables or not 381 | 382 | Returns: 383 | logits (tensor): last activation layer of the discriminator 384 | intermediate_layer (tensor): intermediate layer for feature matching 385 | 386 | """ 387 | layers = sn if do_spectral_norm else tf.layers 388 | 389 | with tf.variable_scope('discriminator_zz', reuse=reuse, 390 | custom_getter=getter): 391 | 392 | y = tf.concat([z, rec_z], axis=-1) 393 | 394 | name_net = 'y_layer_1' 395 | with tf.variable_scope(name_net): 396 | y = layers.dense(y, units=64, kernel_initializer=init_kernel, 397 | name='fc') 398 | 399 | y = leakyReLu(y) 400 | y = tf.layers.dropout(y, rate=0.2, training=is_training, 401 | name='dropout') 402 | 403 | name_net = 'y_layer_2' 404 | with tf.variable_scope(name_net): 405 | y = layers.dense(y, 406 | units=32, 407 | kernel_initializer=init_kernel, 408 | name='fc') 409 | 410 | y = leakyReLu(y) 411 | y = tf.layers.dropout(y, rate=0.2, training=is_training, 412 | name='dropout') 413 | 414 | intermediate_layer = y 415 | 416 | name_net = 'y_layer_3' 417 | with tf.variable_scope(name_net): 418 | y = tf.layers.dense(y, 419 | units=1, 420 | kernel_initializer=init_kernel, 421 | name='fc') 422 | 423 | logits = tf.squeeze(y) 424 | 425 | return logits, intermediate_layer 426 | -------------------------------------------------------------------------------- /anogan/arrhythmia_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Arrhythmia GAN architecture. 4 | 5 | Generator (decoder), encoder and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | from utils import sn 10 | 11 | learning_rate = 1e-5 12 | batch_size = 32 13 | latent_dim = 64 14 | init_kernel = tf.contrib.layers.xavier_initializer() 15 | 16 | def leakyReLu(x, alpha=0.2, name=None): 17 | if name: 18 | with tf.variable_scope(name): 19 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 20 | else: 21 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 22 | 23 | def generator(z_inp, is_training=False, getter=None, reuse=False): 24 | """ Generator architecture in tensorflow 25 | 26 | Generates data from the latent space 27 | 28 | Args: 29 | z_inp (tensor): input variable in the latent space 30 | is_training (bool): for batch norms and dropouts 31 | getter: for exponential moving average during inference 32 | reuse (bool): sharing variables or not 33 | 34 | Returns: 35 | net (tensor): last activation layer of the generator 36 | 37 | """ 38 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 39 | name_net = 'layer_1' 40 | with tf.variable_scope(name_net): 41 | net = tf.layers.dense(z_inp, 42 | units=128, 43 | kernel_initializer=init_kernel, 44 | name='fc') 45 | net = tf.nn.relu(net) 46 | 47 | name_net = 'layer_2' 48 | with tf.variable_scope(name_net): 49 | net = tf.layers.dense(net, 50 | units=256, 51 | kernel_initializer=init_kernel, 52 | name='fc') 53 | net = tf.nn.relu(net) 54 | 55 | name_net = 'layer_3' 56 | with tf.variable_scope(name_net): 57 | net = tf.layers.dense(net, 58 | units=274, 59 | kernel_initializer=init_kernel, 60 | name='fc') 61 | 62 | return net 63 | 64 | def discriminator(x_inp, is_training=False, getter=None, reuse=False, 65 | do_spectral_norm=False): 66 | """ Discriminator architecture in tensorflow 67 | 68 | Discriminates between pairs (E(x), x) and (z, G(z)) 69 | 70 | Args: 71 | x_inp (tensor): input data for the discriminator. 72 | is_training (bool): for batch norms and dropouts 73 | getter: for exponential moving average during inference 74 | reuse (bool): sharing variables or not 75 | 76 | Returns: 77 | logits (tensor): last activation layer of the discriminator (shape 1) 78 | intermediate_layer (tensor): intermediate layer for feature matching 79 | 80 | """ 81 | with tf.variable_scope('discriminator', reuse=reuse, custom_getter=getter): 82 | 83 | name_net = 'layer_1' 84 | with tf.variable_scope(name_net): 85 | net = tf.layers.dense(x_inp, 86 | units=256, 87 | kernel_initializer=init_kernel, 88 | name='fc') 89 | net = leakyReLu(net) 90 | net = tf.layers.dropout(net, rate=0.2, name='dropout', training=is_training) 91 | 92 | name_net = 'layer_2' 93 | with tf.variable_scope(name_net): 94 | net = tf.layers.dense(net, 95 | 128, 96 | kernel_initializer=init_kernel) 97 | net = leakyReLu(net) 98 | net = tf.layers.dropout(net, rate=0.5, name='dropout', training=is_training) 99 | 100 | 101 | intermediate_layer = net 102 | 103 | name_net = 'layer_3' 104 | with tf.variable_scope(name_net): 105 | logits = tf.layers.dense(net, 106 | 1, 107 | kernel_initializer=init_kernel) 108 | 109 | return logits, intermediate_layer 110 | -------------------------------------------------------------------------------- /anogan/cifar10_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | CIFAR10 DCGAN architecture. 4 | 5 | Generator and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | 10 | learning_rate = 0.0002 11 | batch_size = 32 12 | latent_dim = 100 13 | init_kernel = tf.random_normal_initializer(mean=0.0, stddev=0.01) 14 | 15 | def leakyReLu(x, alpha=0.2, name=None): 16 | if name: 17 | with tf.variable_scope(name): 18 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 19 | else: 20 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 21 | 22 | def generator(z_inp, is_training=False, getter=None, reuse=False): 23 | """ Generator architecture in tensorflow 24 | 25 | Generates data from the latent space 26 | 27 | Args: 28 | z_inp (tensor): input variable in the latent space 29 | is_training (bool): for batch norms and dropouts 30 | getter: for exponential moving average during inference 31 | reuse (bool): sharing variables or not 32 | 33 | Returns: 34 | net (tensor): last activation layer of the generator 35 | 36 | """ 37 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 38 | net = tf.reshape(z_inp, [-1, 1, 1, latent_dim]) 39 | 40 | name_net = 'layer_1' 41 | with tf.variable_scope(name_net): 42 | net = tf.layers.conv2d_transpose(net, 43 | filters=512, 44 | kernel_size=4, 45 | strides=2, 46 | padding='VALID', 47 | kernel_initializer=init_kernel, 48 | name='tconv1') 49 | 50 | net = tf.layers.batch_normalization(net, 51 | training=is_training, 52 | name='tconv1/batch_normalization') 53 | 54 | net = tf.nn.relu(net, name='tconv1/relu') 55 | 56 | name_net = 'layer_2' 57 | with tf.variable_scope(name_net): 58 | net = tf.layers.conv2d_transpose(net, 59 | filters=256, 60 | kernel_size=4, 61 | strides=2, 62 | padding='SAME', 63 | kernel_initializer=init_kernel, 64 | name='tconv2') 65 | 66 | net = tf.layers.batch_normalization(net, 67 | training=is_training, 68 | name='tconv2/batch_normalization') 69 | net = tf.nn.relu(net, name='tconv2/relu') 70 | 71 | name_net = 'layer_3' 72 | with tf.variable_scope(name_net): 73 | net = tf.layers.conv2d_transpose(net, 74 | filters=128, 75 | kernel_size=4, 76 | strides=2, 77 | padding='SAME', 78 | kernel_initializer=init_kernel, 79 | name='tconv3') 80 | 81 | net = tf.layers.batch_normalization(net, 82 | training=is_training, 83 | name='tconv3/batch_normalization') 84 | net = tf.nn.relu(net, name='tconv3/relu') 85 | 86 | name_net = 'layer_4' 87 | with tf.variable_scope(name_net): 88 | net = tf.layers.conv2d_transpose(net, 89 | filters=3, 90 | kernel_size=4, 91 | strides=2, 92 | padding='SAME', 93 | kernel_initializer=init_kernel, 94 | name='tconv4') 95 | 96 | net = tf.tanh(net, name='tconv4/tanh') 97 | 98 | return net 99 | 100 | def discriminator(x_inp, is_training=False, getter=None, reuse=False): 101 | """ Discriminator architecture in tensorflow 102 | 103 | Discriminates between real data and generated data 104 | 105 | Args: 106 | x_inp (tensor): input data for the discriminator. 107 | is_training (bool): for batch norms and dropouts 108 | getter: for exponential moving average during inference 109 | reuse (bool): sharing variables or not 110 | 111 | Returns: 112 | logits (tensor): last activation layer of the discriminator (shape 1) 113 | intermediate_layer (tensor): intermediate layer for feature matching 114 | 115 | """ 116 | with tf.variable_scope('discriminator', reuse=reuse, custom_getter=getter): 117 | name_net = 'layer_1' 118 | with tf.variable_scope(name_net): 119 | net = tf.layers.conv2d(x_inp, 120 | filters=128, 121 | kernel_size=4, 122 | strides=2, 123 | padding='SAME', 124 | kernel_initializer=init_kernel, 125 | name='conv1') 126 | 127 | net = leakyReLu(net, 0.2, name='conv1/leaky_relu') 128 | 129 | name_net = 'x_layer_2' 130 | with tf.variable_scope(name_net): 131 | net = tf.layers.conv2d(net, 132 | filters=256, 133 | kernel_size=4, 134 | strides=2, 135 | padding='SAME', 136 | kernel_initializer=init_kernel, 137 | name='conv2') 138 | 139 | net = tf.layers.batch_normalization(net, 140 | training=is_training, 141 | name='conv2/batch_normalization') 142 | 143 | net = leakyReLu(net, 0.2, name='conv2/leaky_relu') 144 | 145 | name_net = 'layer_3' 146 | with tf.variable_scope(name_net): 147 | net = tf.layers.conv2d(net, 148 | filters=512, 149 | kernel_size=4, 150 | strides=2, 151 | padding='SAME', 152 | kernel_initializer=init_kernel, 153 | name='conv3') 154 | 155 | net = tf.layers.batch_normalization(net, 156 | training=is_training, 157 | name='conv3/batch_normalization') 158 | 159 | net = leakyReLu(net, 0.2, name='conv3/leaky_relu') 160 | 161 | net = tf.contrib.layers.flatten(net) 162 | 163 | intermediate_layer = net 164 | 165 | name_net = 'layer_4' 166 | with tf.variable_scope(name_net): 167 | net = tf.layers.dense(net, 168 | units=1, 169 | kernel_initializer=init_kernel, 170 | name='fc') 171 | 172 | logits = tf.squeeze(net) 173 | 174 | return logits, intermediate_layer -------------------------------------------------------------------------------- /anogan/kdd_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | KDD GAN architecture. 4 | 5 | Generator (decoder), encoder and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | from utils import sn 10 | 11 | learning_rate = 1e-5 12 | batch_size = 50 13 | latent_dim = 32 14 | init_kernel = tf.contrib.layers.xavier_initializer() 15 | 16 | def leakyReLu(x, alpha=0.2, name=None): 17 | if name: 18 | with tf.variable_scope(name): 19 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 20 | else: 21 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 22 | 23 | def generator(z_inp, is_training=False, getter=None, reuse=False): 24 | """ Generator architecture in tensorflow 25 | 26 | Generates data from the latent space 27 | 28 | Args: 29 | z_inp (tensor): input variable in the latent space 30 | is_training (bool): for batch norms and dropouts 31 | getter: for exponential moving average during inference 32 | reuse (bool): sharing variables or not 33 | 34 | Returns: 35 | net (tensor): last activation layer of the generator 36 | 37 | """ 38 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 39 | 40 | name_net = 'layer_1' 41 | with tf.variable_scope(name_net): 42 | net = tf.layers.dense(z_inp, 43 | units=64, 44 | kernel_initializer=init_kernel, 45 | name='fc') 46 | net = tf.nn.relu(net, name='relu') 47 | 48 | name_net = 'layer_2' 49 | with tf.variable_scope(name_net): 50 | net = tf.layers.dense(net, 51 | units=128, 52 | kernel_initializer=init_kernel, 53 | name='fc') 54 | net = tf.nn.relu(net, name='relu') 55 | 56 | name_net = 'layer_3' 57 | with tf.variable_scope(name_net): 58 | net = tf.layers.dense(net, 59 | units=121, 60 | kernel_initializer=init_kernel, 61 | name='fc') 62 | 63 | return net 64 | 65 | def discriminator(x_inp, is_training=False, getter=None, reuse=False): 66 | """ Discriminator architecture in tensorflow 67 | 68 | Discriminates between real data and generated data 69 | 70 | Args: 71 | x_inp (tensor): input data for the discriminator. 72 | is_training (bool): for batch norms and dropouts 73 | getter: for exponential moving average during inference 74 | reuse (bool): sharing variables or not 75 | 76 | Returns: 77 | logits (tensor): last activation layer of the discriminator (shape 1) 78 | intermediate_layer (tensor): intermediate layer for feature matching 79 | 80 | """ 81 | with tf.variable_scope('discriminator', reuse=reuse, custom_getter=getter): 82 | 83 | name_net = 'layer_1' 84 | with tf.variable_scope(name_net): 85 | net = tf.layers.dense(x_inp, 86 | units=256, 87 | kernel_initializer=init_kernel, 88 | name='fc') 89 | net = leakyReLu(net) 90 | net = tf.layers.dropout(net, rate=0.2, name='dropout', 91 | training=is_training) 92 | 93 | name_net = 'layer_2' 94 | with tf.variable_scope(name_net): 95 | net = tf.layers.dense(net, 96 | units=128, 97 | kernel_initializer=init_kernel, 98 | name='fc') 99 | net = leakyReLu(net) 100 | net = tf.layers.dropout(net, rate=0.2, name='dropout', 101 | training=is_training) 102 | 103 | name_net = 'layer_3' 104 | with tf.variable_scope(name_net): 105 | net = tf.layers.dense(net, 106 | units=128, 107 | kernel_initializer=init_kernel, 108 | name='fc') 109 | net = leakyReLu(net) 110 | net = tf.layers.dropout(net, 111 | rate=0.2, 112 | name='dropout', 113 | training=is_training) 114 | intermediate_layer = net 115 | 116 | name_net = 'layer_4' 117 | with tf.variable_scope(name_net): 118 | net = tf.layers.dense(net, 119 | units=1, 120 | kernel_initializer=init_kernel, 121 | name='fc') 122 | 123 | net = tf.squeeze(net) 124 | 125 | return net, intermediate_layer -------------------------------------------------------------------------------- /anogan/run.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tensorflow as tf 4 | import logging 5 | import importlib 6 | import sys 7 | import os 8 | from utils.adapt_data import batch_fill 9 | from utils.evaluations import save_results, heatmap 10 | from utils.constants import IMAGES_DATASETS 11 | 12 | FREQ_PRINT = 200 # print frequency image tensorboard [20] 13 | FREQ_EV = 1 14 | PATIENCE = 20 15 | STEPS_NUMBER = 500 16 | 17 | def get_getter(ema): # to update neural net with moving avg variables, suitable for ss learning cf Saliman 18 | def ema_getter(getter, name, *args, **kwargs): 19 | var = getter(name, *args, **kwargs) 20 | ema_var = ema.average(var) 21 | return ema_var if ema_var else var 22 | return ema_getter 23 | 24 | def display_parameters(batch_size, starting_lr, ema_decay, 25 | weight, degree, label): 26 | """See parameters 27 | """ 28 | print('Batch size: ', batch_size) 29 | print('Starting learning rate: ', starting_lr) 30 | print('EMA Decay: ', ema_decay) 31 | print('Weight: ', weight) 32 | print('Degree for L norms: ', degree) 33 | print('Anomalous label: ', label) 34 | 35 | def display_progression_epoch(j, id_max): 36 | """See epoch progression 37 | """ 38 | batch_progression = int((j / id_max) * 100) 39 | sys.stdout.write(str(batch_progression) + ' % epoch' + chr(13)) 40 | _ = sys.stdout.flush 41 | 42 | def create_logdir(dataset, weight, label, rd): 43 | """ Directory to save training logs, weights, biases, etc.""" 44 | return "train_logs/{}/anogan/label{}/weight{}/rd{}".format( 45 | dataset, label, weight, rd) 46 | 47 | def train_and_test(dataset, nb_epochs, weight, degree, random_seed, label, 48 | enable_sm, score_method, enable_early_stop): 49 | 50 | """ Runs the AnoGAN on the specified dataset 51 | 52 | Note: 53 | Saves summaries on tensorboard. To display them, please use cmd line 54 | tensorboard --logdir=model.training_logdir() --port=number 55 | Args: 56 | dataset (str): name of the dataset 57 | nb_epochs (int): number of epochs 58 | weight (float): weight in the inverting loss function 59 | degree (int): degree of the norm in the feature matching 60 | random_seed (int): trying different seeds for averaging the results 61 | label (int): label which is normal for image experiments 62 | enable_sm (bool): allow TF summaries for monitoring the training 63 | score_method (str): which metric to use for the ablation study 64 | enable_early_stop (bool): allow early stopping for determining the number of epochs 65 | """ 66 | 67 | config = tf.ConfigProto() 68 | config.gpu_options.allow_growth = True 69 | 70 | logger = logging.getLogger("AnoGAN.run.{}.{}".format(dataset, label)) 71 | 72 | # Import model and data 73 | network = importlib.import_module('anogan.{}_utilities'.format(dataset)) 74 | data = importlib.import_module("data.{}".format(dataset)) 75 | 76 | # Parameters 77 | starting_lr = network.learning_rate 78 | batch_size = network.batch_size 79 | latent_dim = network.latent_dim 80 | ema_decay = 0.999 81 | 82 | global_step = tf.Variable(0, name='global_step', trainable=False) 83 | # Placeholders 84 | x_pl = tf.placeholder(tf.float32, shape=data.get_shape_input(), name="input") 85 | is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl') 86 | learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl") 87 | 88 | # Data 89 | logger.info('Data loading...') 90 | trainx, trainy = data.get_train(label) 91 | if enable_early_stop: 92 | validx, validy = data.get_valid(label) 93 | nr_batches_valid = int(validx.shape[0] / batch_size) 94 | trainx_copy = trainx.copy() 95 | testx, testy = data.get_test(label) 96 | 97 | rng = np.random.RandomState(random_seed) 98 | nr_batches_train = int(trainx.shape[0] / batch_size) 99 | nr_batches_test = int(testx.shape[0] / batch_size) 100 | 101 | logger.info('Building graph...') 102 | logger.warn("The GAN is training with the following parameters:") 103 | display_parameters(batch_size, starting_lr, ema_decay, weight, degree, label) 104 | 105 | gen = network.generator 106 | dis = network.discriminator 107 | 108 | # Sample noise from random normal distribution 109 | random_z = tf.random_normal([batch_size, latent_dim], mean=0.0, 110 | stddev=1.0, name='random_z') 111 | # Generate images with generator 112 | x_gen = gen(random_z, is_training=is_training_pl) 113 | 114 | real_d, inter_layer_real = dis(x_pl, is_training=is_training_pl) 115 | fake_d, inter_layer_fake = dis(x_gen, is_training=is_training_pl, 116 | reuse=True) 117 | 118 | with tf.name_scope('loss_functions'): 119 | # Calculate seperate losses for discriminator with real and fake images 120 | real_discriminator_loss = tf.losses.sigmoid_cross_entropy( 121 | tf.ones_like(real_d), real_d, 122 | scope='real_discriminator_loss') 123 | fake_discriminator_loss = tf.losses.sigmoid_cross_entropy( 124 | tf.zeros_like(fake_d), fake_d, 125 | scope='fake_discriminator_loss') 126 | # Add discriminator losses 127 | loss_discriminator = real_discriminator_loss + fake_discriminator_loss 128 | # Calculate loss for generator by flipping label on discriminator output 129 | loss_generator = tf.losses.sigmoid_cross_entropy( 130 | tf.ones_like(fake_d), fake_d, scope='generator_loss') 131 | 132 | with tf.name_scope('optimizers'): 133 | # control op dependencies for batch norm and trainable variables 134 | dvars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 135 | scope='discriminator') 136 | gvars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 137 | scope='generator') 138 | 139 | update_ops_gen = tf.get_collection(tf.GraphKeys.UPDATE_OPS, 140 | scope='generator') 141 | update_ops_dis = tf.get_collection(tf.GraphKeys.UPDATE_OPS, 142 | scope='discriminator') 143 | 144 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, 145 | beta1=0.5) 146 | 147 | with tf.control_dependencies(update_ops_gen): 148 | gen_op = optimizer.minimize(loss_generator, var_list=gvars, 149 | global_step=global_step) 150 | 151 | with tf.control_dependencies(update_ops_dis): 152 | dis_op = optimizer.minimize(loss_discriminator, var_list=dvars) 153 | 154 | # Exponential Moving Average for inference 155 | def train_op_with_ema_dependency(vars, op): 156 | ema = tf.train.ExponentialMovingAverage(decay=ema_decay) 157 | maintain_averages_op = ema.apply(vars) 158 | with tf.control_dependencies([op]): 159 | train_op = tf.group(maintain_averages_op) 160 | return train_op, ema 161 | 162 | train_gen_op, gen_ema = train_op_with_ema_dependency(gvars, gen_op) 163 | train_dis_op, dis_ema = train_op_with_ema_dependency(dvars, dis_op) 164 | 165 | ### Testing ### 166 | with tf.variable_scope("latent_variable"): 167 | z_optim = tf.get_variable(name='z_optim', 168 | shape= [batch_size, latent_dim], 169 | initializer=tf.truncated_normal_initializer()) 170 | reinit_z = z_optim.initializer 171 | 172 | # EMA 173 | x_gen_ema = gen(random_z, is_training=is_training_pl, 174 | getter=get_getter(gen_ema), reuse=True) 175 | rec_x_ema = gen(z_optim, is_training=is_training_pl, 176 | getter=get_getter(gen_ema), reuse=True) 177 | # Pass real and fake images into discriminator separately 178 | real_d_ema, inter_layer_real_ema = dis(x_pl, 179 | is_training=is_training_pl, 180 | getter=get_getter(gen_ema), 181 | reuse=True) 182 | fake_d_ema, inter_layer_fake_ema = dis(rec_x_ema, 183 | is_training=is_training_pl, 184 | getter=get_getter(gen_ema), 185 | reuse=True) 186 | 187 | with tf.name_scope('Testing'): 188 | with tf.variable_scope('Reconstruction_loss'): 189 | delta = x_pl - rec_x_ema 190 | delta_flat = tf.contrib.layers.flatten(delta) 191 | reconstruction_score = tf.norm(delta_flat, ord=degree, axis=1, 192 | keep_dims=False, name='epsilon') 193 | 194 | with tf.variable_scope('Discriminator_scores'): 195 | 196 | if score_method == 'cross-e': 197 | dis_score = tf.nn.sigmoid_cross_entropy_with_logits( 198 | labels=tf.ones_like(fake_d_ema),logits=fake_d_ema) 199 | 200 | else: 201 | fm = inter_layer_real_ema - inter_layer_fake_ema 202 | fm = tf.contrib.layers.flatten(fm) 203 | dis_score = tf.norm(fm, ord=degree, axis=1, 204 | keep_dims=False, name='d_loss') 205 | 206 | dis_score = tf.squeeze(dis_score) 207 | 208 | with tf.variable_scope('Score'): 209 | loss_invert = weight * reconstruction_score \ 210 | + (1 - weight) * dis_score 211 | 212 | rec_error_valid = tf.reduce_mean(loss_invert) 213 | 214 | with tf.variable_scope("Test_learning_rate"): 215 | step_lr = tf.Variable(0, trainable=False) 216 | learning_rate_invert = 0.001 217 | reinit_lr = tf.variables_initializer( 218 | tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 219 | scope="Test_learning_rate")) 220 | 221 | with tf.name_scope('Test_optimizer'): 222 | invert_op = tf.train.AdamOptimizer(learning_rate_invert).\ 223 | minimize(loss_invert,global_step=step_lr, var_list=[z_optim], 224 | name='optimizer') 225 | reinit_optim = tf.variables_initializer( 226 | tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 227 | scope='Test_optimizer')) 228 | 229 | reinit_test_graph_op = [reinit_z, reinit_lr, reinit_optim] 230 | 231 | with tf.name_scope("Scores"): 232 | list_scores = loss_invert 233 | 234 | if enable_sm: 235 | with tf.name_scope('training_summary'): 236 | with tf.name_scope('dis_summary'): 237 | tf.summary.scalar('real_discriminator_loss', 238 | real_discriminator_loss, ['dis']) 239 | tf.summary.scalar('fake_discriminator_loss', 240 | fake_discriminator_loss, ['dis']) 241 | tf.summary.scalar('discriminator_loss', loss_discriminator, ['dis']) 242 | 243 | with tf.name_scope('gen_summary'): 244 | tf.summary.scalar('loss_generator', loss_generator, ['gen']) 245 | 246 | with tf.name_scope('img_summary'): 247 | heatmap_pl_latent = tf.placeholder(tf.float32, 248 | shape=(1, 480, 640, 3), 249 | name="heatmap_pl_latent") 250 | sum_op_latent = tf.summary.image('heatmap_latent', 251 | heatmap_pl_latent) 252 | 253 | with tf.name_scope('validation_summary'): 254 | tf.summary.scalar('valid', rec_error_valid, ['v']) 255 | 256 | if dataset in IMAGES_DATASETS: 257 | with tf.name_scope('image_summary'): 258 | tf.summary.image('reconstruct', x_gen, 8, ['image']) 259 | tf.summary.image('input_images', x_pl, 8, ['image']) 260 | 261 | else: 262 | heatmap_pl_rec = tf.placeholder(tf.float32, shape=(1, 480, 640, 3), 263 | name="heatmap_pl_rec") 264 | with tf.name_scope('image_summary'): 265 | tf.summary.image('heatmap_rec', heatmap_pl_rec, 1, ['image']) 266 | 267 | sum_op_dis = tf.summary.merge_all('dis') 268 | sum_op_gen = tf.summary.merge_all('gen') 269 | sum_op = tf.summary.merge([sum_op_dis, sum_op_gen]) 270 | sum_op_im = tf.summary.merge_all('image') 271 | sum_op_valid = tf.summary.merge_all('v') 272 | 273 | logdir = create_logdir(dataset, weight, label, random_seed) 274 | 275 | sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=None, 276 | save_model_secs=None) 277 | 278 | logger.info('Start training...') 279 | with sv.managed_session(config=config) as sess: 280 | 281 | logger.info('Initialization done') 282 | 283 | writer = tf.summary.FileWriter(logdir, sess.graph) 284 | 285 | train_batch = 0 286 | epoch = 0 287 | best_valid_loss = 0 288 | 289 | while not sv.should_stop() and epoch < nb_epochs: 290 | 291 | lr = starting_lr 292 | begin = time.time() 293 | 294 | trainx = trainx[rng.permutation(trainx.shape[0])] # shuffling unl dataset 295 | trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])] 296 | 297 | train_loss_dis, train_loss_gen = [0, 0] 298 | # training 299 | for t in range(nr_batches_train): 300 | display_progression_epoch(t, nr_batches_train) 301 | 302 | # construct randomly permuted minibatches 303 | ran_from = t * batch_size 304 | ran_to = (t + 1) * batch_size 305 | 306 | # train discriminator 307 | feed_dict = {x_pl: trainx[ran_from:ran_to], 308 | is_training_pl:True, 309 | learning_rate:lr} 310 | _, ld, step = sess.run([train_dis_op, loss_discriminator, global_step], feed_dict=feed_dict) 311 | train_loss_dis += ld 312 | 313 | # train generator 314 | feed_dict = {x_pl: trainx_copy[ran_from:ran_to], 315 | is_training_pl:True, 316 | learning_rate:lr} 317 | _, lg = sess.run([train_gen_op, loss_generator], feed_dict=feed_dict) 318 | train_loss_gen += lg 319 | 320 | if enable_sm: 321 | sm = sess.run(sum_op, feed_dict=feed_dict) 322 | writer.add_summary(sm, step) 323 | 324 | if t % FREQ_PRINT == 0: # inspect reconstruction 325 | # t = np.random.randint(0,400) 326 | # ran_from = t 327 | # ran_to = t + batch_size 328 | # sm = sess.run(sum_op_im, feed_dict={x_pl: trainx[ran_from:ran_to],is_training_pl: False}) 329 | # writer.add_summary(sm, train_batch) 330 | 331 | # data = sess.run(z_gen, feed_dict={ 332 | # x_pl: trainx[ran_from:ran_to], 333 | # is_training_pl: False}) 334 | # data = np.expand_dims(heatmap(data), axis=0) 335 | # sml = sess.run(sum_op_latent, feed_dict={ 336 | # heatmap_pl_latent: data, 337 | # is_training_pl: False}) 338 | # 339 | # writer.add_summary(sml, train_batch) 340 | 341 | if dataset in IMAGES_DATASETS: 342 | sm = sess.run(sum_op_im, feed_dict={ 343 | x_pl: trainx[ran_from:ran_to], 344 | is_training_pl: False}) 345 | # 346 | # else: 347 | # data = sess.run(z_gen, feed_dict={ 348 | # x_pl: trainx[ran_from:ran_to], 349 | # z_pl: np.random.normal( 350 | # size=[batch_size, latent_dim]), 351 | # is_training_pl: False}) 352 | # data = np.expand_dims(heatmap(data), axis=0) 353 | # sm = sess.run(sum_op_im, feed_dict={ 354 | # heatmap_pl_rec: data, 355 | # is_training_pl: False}) 356 | writer.add_summary(sm, step)#train_batch) 357 | train_batch += 1 358 | 359 | train_loss_gen /= nr_batches_train 360 | train_loss_dis /= nr_batches_train 361 | 362 | logger.info('Epoch terminated') 363 | print("Epoch %d | time = %ds | loss gen = %.4f | loss dis = %.4f " 364 | % (epoch, time.time() - begin, train_loss_gen, train_loss_dis)) 365 | 366 | ##EARLY STOPPING 367 | if (epoch + 1) % FREQ_EV == 0 and enable_early_stop: 368 | logger.info('Validation...') 369 | 370 | inds = rng.permutation(validx.shape[0]) 371 | validx = validx[inds] # shuffling dataset 372 | validy = validy[inds] # shuffling dataset 373 | 374 | valid_loss = 0 375 | 376 | # Create scores 377 | for t in range(nr_batches_valid): 378 | # construct randomly permuted minibatches 379 | display_progression_epoch(t, nr_batches_valid) 380 | ran_from = t * batch_size 381 | ran_to = (t + 1) * batch_size 382 | 383 | feed_dict = {x_pl: validx[ran_from:ran_to], 384 | is_training_pl: False} 385 | for _ in range(STEPS_NUMBER): 386 | _ = sess.run(invert_op, feed_dict=feed_dict) 387 | vl = sess.run(rec_error_valid, 388 | feed_dict=feed_dict) 389 | valid_loss += vl 390 | sess.run(reinit_test_graph_op) 391 | 392 | valid_loss /= nr_batches_valid 393 | sess.run(reinit_test_graph_op) 394 | 395 | if enable_sm: 396 | sm = sess.run(sum_op_valid, feed_dict=feed_dict) 397 | writer.add_summary(sm, step) # train_batch) 398 | 399 | logger.info( 400 | 'Validation: valid loss {:.4f}'.format(valid_loss)) 401 | 402 | if valid_loss < best_valid_loss or epoch == FREQ_EV - 1: 403 | 404 | best_valid_loss = valid_loss 405 | logger.info("Best model - valid loss = {:.4f} - saving...".format( 406 | best_valid_loss)) 407 | sv.saver.save(sess, logdir + '/model.ckpt', 408 | global_step=step) 409 | nb_without_improvements = 0 410 | else: 411 | nb_without_improvements += FREQ_EV 412 | 413 | if nb_without_improvements > PATIENCE: 414 | sv.request_stop() 415 | logger.warning( 416 | "Early stopping at epoch {} with weights from epoch {}".format( 417 | epoch, epoch - nb_without_improvements)) 418 | 419 | epoch += 1 420 | 421 | logger.warn('Testing evaluation...') 422 | step = sess.run(global_step) 423 | sv.saver.save(sess, logdir + '/model.ckpt', global_step=step) 424 | 425 | rect_x, rec_error, latent, scores = [], [], [], [] 426 | inference_time = [] 427 | 428 | # Create scores 429 | for t in range(nr_batches_test): 430 | # construct randomly permuted minibatches 431 | display_progression_epoch(t, nr_batches_test) 432 | ran_from = t * batch_size 433 | ran_to = (t + 1) * batch_size 434 | begin_val_batch = time.time() 435 | 436 | feed_dict = {x_pl: testx[ran_from:ran_to], 437 | is_training_pl:False} 438 | 439 | for _ in range(STEPS_NUMBER): 440 | _ = sess.run(invert_op, feed_dict=feed_dict) 441 | 442 | brect_x, brec_error, bscores, blatent = sess.run([rec_x_ema, reconstruction_score, loss_invert, z_optim], feed_dict=feed_dict) 443 | rect_x.append(brect_x) 444 | rec_error.append(brec_error) 445 | scores.append(bscores) 446 | latent.append(blatent) 447 | sess.run(reinit_test_graph_op) 448 | 449 | inference_time.append(time.time() - begin_val_batch) 450 | 451 | logger.info('Testing : mean inference time is %.4f' % ( 452 | np.mean(inference_time))) 453 | 454 | if testx.shape[0] % batch_size != 0: 455 | batch, size = batch_fill(testx, batch_size) 456 | feed_dict = {x_pl: batch, 457 | is_training_pl: False} 458 | for _ in range(STEPS_NUMBER): 459 | _ = sess.run(invert_op, feed_dict=feed_dict) 460 | brect_x, brec_error, bscores, blatent = sess.run([rec_x_ema, reconstruction_score, loss_invert, z_optim], feed_dict=feed_dict) 461 | rect_x.append(brect_x[:size]) 462 | rec_error.append(brec_error[:size]) 463 | scores.append(bscores[:size]) 464 | latent.append(blatent[:size]) 465 | sess.run(reinit_test_graph_op) 466 | 467 | rect_x = np.concatenate(rect_x, axis=0) 468 | rec_error = np.concatenate(rec_error, axis=0) 469 | scores = np.concatenate(scores, axis=0) 470 | latent = np.concatenate(latent, axis=0) 471 | save_results(scores, testy, 'anogan', dataset, score_method, 472 | weight, label, random_seed) 473 | 474 | def run(args): 475 | """ Runs the training process""" 476 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) 477 | with tf.Graph().as_default(): 478 | # Set the graph level seed 479 | tf.set_random_seed(args.rd) 480 | train_and_test(args.dataset, args.nb_epochs, args.w, args.d, args.rd, args.label, 481 | args.enable_sm, args.m, args.enable_early_stop) 482 | -------------------------------------------------------------------------------- /anogan/svhn_utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | SVHN DCGAN architecture. 4 | 5 | Generator and discriminator. 6 | 7 | """ 8 | import tensorflow as tf 9 | 10 | learning_rate = 0.0002 11 | batch_size = 32 12 | latent_dim = 100 13 | init_kernel = tf.random_normal_initializer(mean=0.0, stddev=0.01) 14 | 15 | def leakyReLu(x, alpha=0.2, name=None): 16 | if name: 17 | with tf.variable_scope(name): 18 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 19 | else: 20 | return tf.nn.relu(x) - (alpha * tf.nn.relu(-x)) 21 | 22 | def generator(z_inp, is_training=False, getter=None, reuse=False): 23 | """ Generator architecture in tensorflow 24 | 25 | Generates data from the latent space 26 | 27 | Args: 28 | z_inp (tensor): input variable in the latent space 29 | is_training (bool): for batch norms and dropouts 30 | getter: for exponential moving average during inference 31 | reuse (bool): sharing variables or not 32 | 33 | Returns: 34 | net (tensor): last activation layer of the generator 35 | 36 | """ 37 | with tf.variable_scope('generator', reuse=reuse, custom_getter=getter): 38 | net = tf.reshape(z_inp, [-1, 1, 1, latent_dim]) 39 | 40 | name_net = 'layer_1' 41 | with tf.variable_scope(name_net): 42 | net = tf.layers.conv2d_transpose(net, 43 | filters=512, 44 | kernel_size=4, 45 | strides=2, 46 | padding='VALID', 47 | kernel_initializer=init_kernel, 48 | name='tconv1') 49 | 50 | net = tf.layers.batch_normalization(net, 51 | training=is_training, 52 | name='tconv1/batch_normalization') 53 | 54 | net = tf.nn.relu(net, name='tconv1/relu') 55 | 56 | name_net = 'layer_2' 57 | with tf.variable_scope(name_net): 58 | net = tf.layers.conv2d_transpose(net, 59 | filters=256, 60 | kernel_size=4, 61 | strides=2, 62 | padding='SAME', 63 | kernel_initializer=init_kernel, 64 | name='tconv2') 65 | 66 | net = tf.layers.batch_normalization(net, 67 | training=is_training, 68 | name='tconv2/batch_normalization') 69 | net = tf.nn.relu(net, name='tconv2/relu') 70 | 71 | name_net = 'layer_3' 72 | with tf.variable_scope(name_net): 73 | net = tf.layers.conv2d_transpose(net, 74 | filters=128, 75 | kernel_size=4, 76 | strides=2, 77 | padding='SAME', 78 | kernel_initializer=init_kernel, 79 | name='tconv3') 80 | 81 | net = tf.layers.batch_normalization(net, 82 | training=is_training, 83 | name='tconv3/batch_normalization') 84 | net = tf.nn.relu(net, name='tconv3/relu') 85 | 86 | name_net = 'layer_4' 87 | with tf.variable_scope(name_net): 88 | net = tf.layers.conv2d_transpose(net, 89 | filters=3, 90 | kernel_size=4, 91 | strides=2, 92 | padding='SAME', 93 | kernel_initializer=init_kernel, 94 | name='tconv4') 95 | 96 | net = tf.tanh(net, name='tconv4/tanh') 97 | 98 | return net 99 | 100 | def discriminator(x_inp, is_training=False, getter=None, reuse=False): 101 | """ Discriminator architecture in tensorflow 102 | 103 | Discriminates between real data and generated data 104 | 105 | Args: 106 | x_inp (tensor): input data for the discriminator. 107 | is_training (bool): for batch norms and dropouts 108 | getter: for exponential moving average during inference 109 | reuse (bool): sharing variables or not 110 | 111 | Returns: 112 | logits (tensor): last activation layer of the discriminator (shape 1) 113 | intermediate_layer (tensor): intermediate layer for feature matching 114 | 115 | """ 116 | with tf.variable_scope('discriminator', reuse=reuse, custom_getter=getter): 117 | name_net = 'layer_1' 118 | with tf.variable_scope(name_net): 119 | net = tf.layers.conv2d(x_inp, 120 | filters=128, 121 | kernel_size=4, 122 | strides=2, 123 | padding='SAME', 124 | kernel_initializer=init_kernel, 125 | name='conv1') 126 | 127 | net = leakyReLu(net, 0.2, name='conv1/leaky_relu') 128 | 129 | name_net = 'x_layer_2' 130 | with tf.variable_scope(name_net): 131 | net = tf.layers.conv2d(net, 132 | filters=256, 133 | kernel_size=4, 134 | strides=2, 135 | padding='SAME', 136 | kernel_initializer=init_kernel, 137 | name='conv2') 138 | 139 | net = tf.layers.batch_normalization(net, 140 | training=is_training, 141 | name='conv2/batch_normalization') 142 | 143 | net = leakyReLu(net, 0.2, name='conv2/leaky_relu') 144 | 145 | name_net = 'layer_3' 146 | with tf.variable_scope(name_net): 147 | net = tf.layers.conv2d(net, 148 | filters=512, 149 | kernel_size=4, 150 | strides=2, 151 | padding='SAME', 152 | kernel_initializer=init_kernel, 153 | name='conv3') 154 | 155 | net = tf.layers.batch_normalization(net, 156 | training=is_training, 157 | name='conv3/batch_normalization') 158 | 159 | net = leakyReLu(net, 0.2, name='conv3/leaky_relu') 160 | 161 | net = tf.contrib.layers.flatten(net) 162 | 163 | intermediate_layer = net 164 | 165 | name_net = 'layer_4' 166 | with tf.variable_scope(name_net): 167 | net = tf.layers.dense(net, 168 | units=1, 169 | kernel_initializer=init_kernel, 170 | name='fc') 171 | 172 | logits = tf.squeeze(net) 173 | 174 | return logits, intermediate_layer -------------------------------------------------------------------------------- /dagmm/arrhythmia_utilities.py: -------------------------------------------------------------------------------- 1 | """Arrhythmia architecture. 2 | 3 | Autoencoder and estimation network. 4 | eature extraction. 5 | 6 | 7 | Taken from Section 4.3 of Zong, Bo et al. “Deep Autoencoding Gaussian 8 | Mixture Model for Unsupervised Anomaly Detection.” (2018). 9 | """ 10 | 11 | import tensorflow as tf 12 | 13 | init_kernel = tf.contrib.layers.xavier_initializer() 14 | 15 | params = { 16 | 'is_image': False, 17 | 'learning_rate': 0.0001, 18 | 'batch_size': 128, 19 | 'latent_dim': 2, 20 | 'K': 2, 21 | 'n_epochs': 200, 22 | "l1":0.1, 23 | "l2":0.005 24 | } 25 | 26 | 27 | def encoder(x_inp, is_training=False, reuse=False): 28 | """ Encoder architecture in tensorflow 29 | 30 | Maps the data into the latent space 31 | 32 | Note: 33 | Provides histogram and distribution tensorflow summaries 34 | 35 | Args: 36 | x_inp (tensor): input data for the encoder. 37 | reuse (bool): sharing variables or not 38 | 39 | Returns: 40 | (tensor): last activation layer of the encoder 41 | 42 | """ 43 | 44 | with tf.variable_scope('encoder', reuse=reuse): 45 | 46 | net = x_inp 47 | name_net = 'layer_1' 48 | with tf.variable_scope(name_net): 49 | net = tf.layers.dense(net, 50 | units=10, 51 | activation=tf.nn.tanh, 52 | kernel_initializer=init_kernel, 53 | name='fc') 54 | name_net = 'layer_2' 55 | with tf.variable_scope(name_net): 56 | net = tf.layers.dense(net, 57 | units=2, 58 | kernel_initializer=init_kernel, 59 | name='fc') 60 | 61 | return net 62 | 63 | def decoder(z_inp, n_features, is_training=False, getter=None, reuse=False): 64 | """ Decoder architecture in tensorflow 65 | 66 | Generates data from the latent space 67 | 68 | Note: 69 | Provides histogram and distribution tensorflow summaries 70 | 71 | Args: 72 | z_inp (tensor): variable in the latent space 73 | reuse (bool): sharing variables or not 74 | 75 | Returns: 76 | (tensor): last activation layer of the generator 77 | 78 | """ 79 | with tf.variable_scope('decoder', reuse=reuse): 80 | 81 | name_net = 'layer_1' 82 | with tf.variable_scope(name_net): 83 | net = tf.layers.dense(z_inp, 84 | units=10, 85 | activation=tf.nn.tanh, 86 | kernel_initializer=init_kernel, 87 | name='fc') 88 | 89 | name_net = 'layer_2' 90 | #there actually are 121 features in kdd 91 | with tf.variable_scope(name_net): 92 | net = tf.layers.dense(net, 93 | units=n_features, 94 | kernel_initializer=init_kernel, 95 | name='fc') 96 | 97 | return net 98 | 99 | def feature_extractor(x, x_r): 100 | """ 101 | Computes the reconstruction features for the autoencoder 102 | 103 | Args: 104 | - x : [N, 121] input data 105 | - x_r : same shape - reconstructed thanks to the autoencoder 106 | 107 | Returns: 108 | - f : chosen features 109 | here relative Euclidean distance and cosine similarity 110 | """ 111 | dist = tf.norm(x-x_r, keepdims=True, axis=1)/tf.norm(x, keepdims=True, axis=1) 112 | n1 = tf.nn.l2_normalize(x, 1) 113 | n2 = tf.nn.l2_normalize(x_r, 1) 114 | cosine_similarity = tf.reduce_sum(tf.multiply(n1, n2), axis=1, keepdims=True) 115 | tf.summary.scalar("dist", tf.reduce_mean(dist), ["loss"]) 116 | tf.summary.scalar("cosine", tf.reduce_mean(cosine_similarity), ["loss"]) 117 | return tf.concat([dist, cosine_similarity], axis=-1) 118 | 119 | def estimator(z_inp, K, is_training=False, getter=None, reuse=False): 120 | """ Estimation network architecture in tensorflow 121 | 122 | Computes the probability of x represented by z to be in the training data 123 | 124 | Note: 125 | Provides histogram and distribution tensorflow summaries 126 | 127 | Args: 128 | z_inp (tensor): variable in the latent space + reconstruction features 129 | reuse (bool): sharing variables or not 130 | 131 | Returns: 132 | logits (tensor): last activation layer of the estimation network (shape 1) 133 | 134 | """ 135 | init_kernel = tf.contrib.layers.xavier_initializer() 136 | with tf.variable_scope('estimator', reuse=reuse, custom_getter=getter): 137 | name_layer = 'layer_1' 138 | with tf.variable_scope(name_layer): 139 | net = tf.layers.dense(z_inp, 140 | units=10, 141 | activation=tf.nn.tanh, 142 | kernel_initializer=init_kernel, 143 | name='fc') 144 | net = tf.layers.dropout(net, rate=0.5, name='dropout', training=is_training) 145 | 146 | name_layer = 'layer_2' 147 | with tf.variable_scope(name_layer): 148 | net = tf.layers.dense(net, 149 | units=K, 150 | kernel_initializer=init_kernel, 151 | name='fc') 152 | logits = tf.nn.softmax(net) 153 | 154 | return logits 155 | -------------------------------------------------------------------------------- /dagmm/gmm_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Tue Apr 10 14:07:15 2018 5 | 6 | @author: Manon Romain 7 | """ 8 | import numpy as np 9 | import tensorflow as tf 10 | import math as m 11 | 12 | tfd = tf.contrib.distributions 13 | 14 | def tricky_divide(x, y): 15 | return tf.transpose(tf.transpose(x)/y) 16 | 17 | def tricky_multiply(x, y): 18 | return tf.transpose(tf.transpose(x)*y) 19 | 20 | 21 | def compute_energy_and_penalty(z, gammas, is_training): 22 | """ 23 | Computes the energy and penalty of the GMM as described 24 | in Zong, Bo et al. “Deep Autoencoding Gaussian Mixture Model 25 | for Unsupervised Anomaly Detection.” (2018). 26 | 27 | K: number of mixtures in the GMM 28 | N: number of samples 29 | M: number of features of z 30 | 31 | Args: 32 | - z : [N, M] - the reconstruction features concatenated 33 | with the latent representation of x 34 | 35 | - gamma : [N, K] - density of probability - output of the 36 | estimation network 37 | 38 | Returns: 39 | 40 | - E(z): Tensor of shape [batch_size] - energy of the 41 | GMM computed by the networks 42 | 43 | - P(sigma): Scalar - Penalty appled to small values in diagonal entries of 44 | covariance matrices 45 | 46 | Note: could probably be simplified 47 | 48 | """ 49 | #shapes 50 | K = gammas.get_shape()[1] 51 | M = z.get_shape()[1] 52 | with tf.variable_scope('gmm_parameters'): 53 | phis = tf.get_variable('phis', shape=[K], initializer=tf.ones_initializer(), dtype=tf.float32, trainable=False) 54 | mus = tf.get_variable('mus', shape=[K, M], initializer=tf.ones_initializer(), dtype=tf.float32, trainable=False) 55 | 56 | init_sigmas = 0.5 * np.expand_dims(np.identity(M), axis=0) 57 | init_sigmas = np.tile(init_sigmas, [2, 1, 1]) 58 | init_sigmas = tf.constant_initializer(init_sigmas) 59 | sigmas = tf.get_variable('sigmas', shape=[K, M, M], initializer=init_sigmas, dtype=tf.float32, trainable=False) 60 | 61 | sums = tf.reduce_sum(gammas, axis=0) 62 | sums_exp_dims = tf.expand_dims(sums, axis=-1) 63 | 64 | phis_ = tf.reduce_mean(gammas, axis=0) 65 | mus_ = tf.matmul(gammas, z, transpose_a=True) / sums_exp_dims 66 | 67 | def assign_training_phis_mus(): 68 | with tf.control_dependencies([phis.assign(phis_), mus.assign(mus_)]): 69 | return [tf.identity(phis), tf.identity(mus)] 70 | 71 | phis, mus = tf.cond(is_training, assign_training_phis_mus, lambda: [phis, mus]) 72 | phis_exp_dims = tf.expand_dims(phis, axis=0) 73 | phis_exp_dims = tf.expand_dims(phis_exp_dims, axis=-1) 74 | phis_exp_dims = tf.expand_dims(phis_exp_dims, axis=-1) 75 | 76 | zs_exp_dims = tf.expand_dims(z, 1) 77 | zs_exp_dims = tf.expand_dims(zs_exp_dims, -1) 78 | mus_exp_dims = tf.expand_dims(mus, 0) 79 | mus_exp_dims = tf.expand_dims(mus_exp_dims, -1) 80 | 81 | zs_minus_mus = zs_exp_dims - mus_exp_dims 82 | 83 | sigmas_ = tf.matmul(zs_minus_mus, zs_minus_mus, transpose_b=True) 84 | broadcast_gammas = tf.expand_dims(gammas, axis=-1) 85 | broadcast_gammas = tf.expand_dims(broadcast_gammas, axis=-1) 86 | sigmas_ = broadcast_gammas * sigmas_ 87 | sigmas_ = tf.reduce_sum(sigmas_, axis=0) 88 | sigmas_ = sigmas_ / tf.expand_dims(sums_exp_dims, axis=-1) 89 | sigmas_ = add_noise(sigmas_) 90 | 91 | def assign_training_sigmas(): 92 | with tf.control_dependencies([sigmas.assign(sigmas_)]): 93 | return tf.identity(sigmas) 94 | 95 | sigmas = tf.cond(is_training, assign_training_sigmas, lambda: sigmas) 96 | 97 | inversed_sigmas = tf.expand_dims(tf.matrix_inverse(sigmas), axis=0) 98 | inversed_sigmas = tf.tile(inversed_sigmas, [tf.shape(zs_minus_mus)[0], 1, 1, 1]) 99 | energy = tf.matmul(zs_minus_mus, inversed_sigmas, transpose_a=True) 100 | energy = tf.matmul(energy, zs_minus_mus) 101 | energy = tf.squeeze(phis_exp_dims * tf.exp(-0.5 * energy), axis=[2, 3]) 102 | energy_divided_by = tf.expand_dims(tf.sqrt(2.0 * m.pi * tf.matrix_determinant(sigmas)), axis=0) + 1e-12 103 | energy = tf.reduce_sum(energy / energy_divided_by, axis=1) + 1e-12 104 | energy = -1.0 * tf.log(energy) 105 | 106 | penalty = 1.0 / tf.matrix_diag_part(sigmas) 107 | penalty = tf.reduce_sum(penalty) 108 | return energy, penalty 109 | 110 | def add_noise(mat, stdev=0.001): 111 | """ 112 | :param mat: should be of shape(k, d, d) 113 | :param stdev: the standard deviation of noise 114 | :return: a matrix with little noises 115 | """ 116 | with tf.name_scope('gaussian_noise'): 117 | dims = mat.get_shape().as_list()[1] 118 | noise = stdev + tf.random_normal([dims], 0, stdev * 1e-1) 119 | noise = tf.diag(noise) 120 | noise = tf.expand_dims(noise, axis=0) 121 | noise = tf.tile(noise, (mat.get_shape()[0], 1, 1)) 122 | return mat + noise 123 | 124 | -------------------------------------------------------------------------------- /dagmm/kdd_utilities.py: -------------------------------------------------------------------------------- 1 | """KDD architecture. 2 | 3 | Autoencoder and estimation network. 4 | eature extraction. 5 | 6 | 7 | Taken from Section 4.3 of Zong, Bo et al. “Deep Autoencoding Gaussian 8 | Mixture Model for Unsupervised Anomaly Detection.” (2018). 9 | """ 10 | 11 | import tensorflow as tf 12 | 13 | init_kernel = tf.contrib.layers.xavier_initializer() 14 | 15 | params = { 16 | 'is_image': False, 17 | 'learning_rate': 0.0001, 18 | 'batch_size': 1024, 19 | 'latent_dim': 1, 20 | 'K': 4, 21 | 'n_epochs': 200, 22 | "l1":0.1, 23 | "l2":0.005 24 | } 25 | 26 | 27 | def encoder(x_inp, is_training=False, reuse=False): 28 | """ Encoder architecture in tensorflow 29 | 30 | Maps the data into the latent space 31 | 32 | Note: 33 | Provides histogram and distribution tensorflow summaries 34 | 35 | Args: 36 | x_inp (tensor): input data for the encoder. 37 | reuse (bool): sharing variables or not 38 | 39 | Returns: 40 | (tensor): last activation layer of the encoder 41 | 42 | """ 43 | 44 | with tf.variable_scope('encoder', reuse=reuse): 45 | 46 | name_net = 'layer_1' 47 | with tf.variable_scope(name_net): 48 | net = tf.layers.dense(x_inp, 49 | units=60, 50 | activation=tf.nn.tanh, 51 | kernel_initializer=init_kernel, 52 | name='fc') 53 | name_net = 'layer_2' 54 | with tf.variable_scope(name_net): 55 | net = tf.layers.dense(net, 56 | units=30, 57 | activation=tf.nn.tanh, 58 | kernel_initializer=init_kernel, 59 | name='fc') 60 | 61 | name_net = 'layer_3' 62 | with tf.variable_scope(name_net): 63 | net = tf.layers.dense(net, 64 | units=10, 65 | activation=tf.nn.tanh, 66 | kernel_initializer=init_kernel, 67 | name='fc') 68 | name_net = 'layer_4' 69 | with tf.variable_scope(name_net): 70 | net = tf.layers.dense(net, 71 | units=1, 72 | kernel_initializer=init_kernel, 73 | name='fc') 74 | 75 | return net 76 | 77 | def decoder(z_inp, n_features, is_training=False, getter=None, reuse=False): 78 | """ Decoder architecture in tensorflow 79 | 80 | Generates data from the latent space 81 | 82 | Note: 83 | Provides histogram and distribution tensorflow summaries 84 | 85 | Args: 86 | z_inp (tensor): variable in the latent space 87 | reuse (bool): sharing variables or not 88 | 89 | Returns: 90 | (tensor): last activation layer of the generator 91 | 92 | """ 93 | with tf.variable_scope('decoder', reuse=reuse): 94 | 95 | name_net = 'layer_1' 96 | with tf.variable_scope(name_net): 97 | net = tf.layers.dense(z_inp, 98 | units=10, 99 | activation=tf.nn.tanh, 100 | kernel_initializer=init_kernel, 101 | name='fc') 102 | 103 | name_net = 'layer_2' 104 | with tf.variable_scope(name_net): 105 | net = tf.layers.dense(net, 106 | units=30, 107 | activation=tf.nn.tanh, 108 | kernel_initializer=init_kernel, 109 | name='fc') 110 | 111 | name_net = 'layer_3' 112 | with tf.variable_scope(name_net): 113 | net = tf.layers.dense(net, 114 | units=60, 115 | activation=tf.nn.tanh, 116 | kernel_initializer=init_kernel, 117 | name='fc') 118 | 119 | name_net = 'layer_4' 120 | #there actually are 121 features in kdd 121 | with tf.variable_scope(name_net): 122 | net = tf.layers.dense(net, 123 | units=n_features, 124 | kernel_initializer=init_kernel, 125 | name='fc') 126 | 127 | return net 128 | 129 | def feature_extractor(x, x_r): 130 | """ 131 | Computes the reconstruction features for the autoencoder 132 | 133 | Args: 134 | - x : [N, 121] input data 135 | - x_r : same shape - reconstructed thanks to the autoencoder 136 | 137 | Returns: 138 | - f : chosen features 139 | here relative Euclidean distance and cosine similarity 140 | """ 141 | dist = tf.norm(x-x_r, keepdims=True, axis=1)/tf.norm(x, keepdims=True, axis=1) 142 | cosine_dist = tf.losses.cosine_distance(tf.nn.l2_normalize(x, 1), tf.nn.l2_normalize(x_r, 1), axis=1, reduction=tf.losses.Reduction.NONE) 143 | tf.summary.scalar("dist", tf.reduce_mean(dist), ["loss"]) 144 | tf.summary.scalar("cosine", tf.reduce_mean(1-cosine_dist), ["loss"]) 145 | # tf.summary.scalar("cosine_dist", tf.reduce_mean(cosine_dist), ["loss"]) 146 | return tf.concat([dist, 1-cosine_dist], axis=-1) 147 | 148 | def estimator(z_inp, K, is_training=False, getter=None, reuse=False): 149 | """ Estimation network architecture in tensorflow 150 | 151 | Computes the probability of x represented by z to be in the training data 152 | 153 | Note: 154 | Provides histogram and distribution tensorflow summaries 155 | 156 | Args: 157 | z_inp (tensor): variable in the latent space + reconstruction features 158 | reuse (bool): sharing variables or not 159 | 160 | Returns: 161 | logits (tensor): last activation layer of the estimation network (shape 1) 162 | 163 | """ 164 | init_kernel = tf.contrib.layers.xavier_initializer() 165 | with tf.variable_scope('estimator', reuse=reuse, custom_getter=getter): 166 | name_layer = 'layer_1' 167 | with tf.variable_scope(name_layer): 168 | net = tf.layers.dense(z_inp, 169 | units=10, 170 | activation=tf.nn.tanh, 171 | kernel_initializer=init_kernel, 172 | name='fc') 173 | net = tf.layers.dropout(net, rate=0.5, name='dropout', training=is_training) 174 | 175 | name_layer = 'layer_2' 176 | with tf.variable_scope(name_layer): 177 | net = tf.layers.dense(net, 178 | units=K, 179 | kernel_initializer=init_kernel, 180 | name='fc') 181 | logits = tf.nn.softmax(net) 182 | 183 | return logits 184 | -------------------------------------------------------------------------------- /dagmm/run.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import logging 4 | import importlib 5 | import numpy as np 6 | import tensorflow as tf 7 | import os 8 | 9 | 10 | from utils.adapt_data import batch_fill 11 | from utils.evaluations import save_results 12 | import dagmm.gmm_utils as gmm 13 | 14 | RANDOM_SEED = 13 15 | FREQ_PRINT = 5000 # print frequency image tensorboard [20] 16 | METHOD = "inception" 17 | def display_parameters(batch_size, starting_lr, 18 | l1, l2, label): 19 | '''See parameters 20 | ''' 21 | print('Batch size: ', batch_size) 22 | print('Starting learning rate: ', starting_lr) 23 | print('Weights loss - l1:', l1, '; l2:', l2) 24 | print('Anomalous label: ', label) 25 | 26 | def display_progression_epoch(j, id_max): 27 | '''See epoch progression 28 | ''' 29 | batch_progression = int((j / id_max) * 100) 30 | sys.stdout.write(str(batch_progression) + ' % epoch' + chr(13)) 31 | _ = sys.stdout.flush 32 | 33 | def create_logdir(dataset, K, l1, l2, label, rd): 34 | """ Directory to save training logs, weights, biases, etc.""" 35 | return "dagmm/train_logs/{}/K{}/{}_{}/{}/{}".format(dataset, K, l1, l2, 36 | label, rd) 37 | 38 | def reconstruction_error(x, x_rec): 39 | return tf.norm(x-x_rec, axis=1) 40 | 41 | def train_and_test(dataset, nb_epochs, K, l1, l2, label, 42 | random_seed): 43 | 44 | """ Runs the DAGMM on the specified dataset 45 | 46 | Note: 47 | Saves summaries on tensorboard. To display them, please use cmd line 48 | tensorboard --logdir=model.training_logdir() --port=number 49 | Args: 50 | nb_epochs (int): number of epochs 51 | weight (float, optional): weight for the anomaly score composition 52 | anomalous_label (int): int in range 0 to 10, is the class/digit 53 | which is considered outlier 54 | """ 55 | logger = logging.getLogger("DAGMM.train.{}.{}".format(dataset,label)) 56 | 57 | # Import model and data 58 | model = importlib.import_module('dagmm.{}_utilities'.format(dataset)) 59 | data = importlib.import_module("data.{}".format(dataset)) 60 | 61 | # Parameters 62 | starting_lr = model.params["learning_rate"] 63 | batch_size = model.params["batch_size"] 64 | if l1==-1: l1 = model.params["l1"] 65 | if l2==-1: l2 = model.params["l2"] 66 | if K==-1: K = model.params["K"] 67 | 68 | # Placeholders 69 | 70 | x_pl = tf.placeholder(tf.float32, data.get_shape_input()) 71 | is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl') 72 | learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl") 73 | 74 | logger.info('Building training graph...') 75 | 76 | logger.warning("The DAGMM is training with the following parameters:") 77 | display_parameters(batch_size, starting_lr, l1, l2, 78 | label) 79 | 80 | 81 | global_step = tf.Variable(0, name='global_step', trainable=False) 82 | 83 | enc = model.encoder 84 | dec = model.decoder 85 | feat_ex = model.feature_extractor 86 | est = model.estimator 87 | 88 | #feature extraction for images 89 | if model.params["is_image"] and not METHOD=="pca": 90 | x_features = image_features.extract_features(x_pl) 91 | else: 92 | x_features = x_pl 93 | n_features = x_features.shape[1] 94 | 95 | with tf.variable_scope('encoder_model'): 96 | z_c = enc(x_features, is_training=is_training_pl) 97 | 98 | with tf.variable_scope('decoder_model'): 99 | x_rec = dec(z_c, n_features, is_training=is_training_pl) 100 | 101 | with tf.variable_scope('feature_extractor_model'): 102 | x_flat = tf.layers.flatten(x_features) 103 | x_rec_flat = tf.layers.flatten(x_rec) 104 | z_r = feat_ex(x_flat, x_rec_flat) 105 | 106 | z = tf.concat([z_c, z_r], axis=1) 107 | 108 | with tf.variable_scope('estimator_model'): 109 | gamma = est(z, K, is_training=is_training_pl) 110 | 111 | with tf.variable_scope('gmm'): 112 | energy, penalty = gmm.compute_energy_and_penalty(z, gamma, is_training_pl) 113 | 114 | with tf.name_scope('loss_functions'): 115 | # reconstruction error 116 | rec_error = reconstruction_error(x_flat, x_rec_flat) 117 | loss_rec = tf.reduce_mean(rec_error) 118 | 119 | # probabilities to observe 120 | loss_energy = tf.reduce_mean(energy) 121 | 122 | # full loss 123 | full_loss = loss_rec + l1*loss_energy + l2*penalty 124 | 125 | 126 | 127 | with tf.name_scope('optimizer'): 128 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, 129 | beta1=0.5, name='dis_optimizer') 130 | 131 | train_op = optimizer.minimize(full_loss, global_step=global_step) 132 | 133 | with tf.name_scope('predictions'): 134 | # Highest 20% are anomalous 135 | if dataset=="kdd": 136 | per = tf.contrib.distributions.percentile(energy, 80) 137 | else: 138 | per = tf.contrib.distributions.percentile(energy, 80) 139 | y_pred = tf.greater_equal(energy, per) 140 | 141 | with tf.name_scope('summary'): 142 | with tf.name_scope('loss_summary'): 143 | tf.summary.scalar('loss_rec', loss_rec, ['loss']) 144 | tf.summary.scalar('mean_energy', loss_energy, ['loss']) 145 | tf.summary.scalar('penalty', penalty, ['loss']) 146 | tf.summary.scalar('full_loss', full_loss, ['loss']) 147 | 148 | sum_op_loss = tf.summary.merge_all('loss') 149 | 150 | # Data 151 | logger.info('Data loading...') 152 | 153 | trainx, trainy = data.get_train(label) 154 | trainx_copy = trainx.copy() 155 | testx, testy = data.get_test(label) 156 | 157 | if model.params["is_image"] and METHOD=="pca": 158 | logger.info('PCA...') 159 | trainx = trainx.reshape([trainx.shape[0], -1]) 160 | testx = testx.reshape([testx.shape[0], -1]) 161 | trainx, testx = image_features.pca(trainx, testx, 20) 162 | logger.info('Done') 163 | 164 | rng = np.random.RandomState(RANDOM_SEED) 165 | nr_batches_train = int(trainx.shape[0] / batch_size) 166 | nr_batches_test = int(testx.shape[0] / batch_size) 167 | 168 | 169 | logdir = create_logdir(dataset, K, l1, l2, label, random_seed) 170 | 171 | sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=None, 172 | save_model_secs=10) 173 | 174 | logger.info('Start training...') 175 | with sv.managed_session() as sess: 176 | 177 | logger.info('Initialization done') 178 | writer = tf.summary.FileWriter(logdir, sess.graph) 179 | train_batch = 0 180 | epoch = 0 181 | 182 | while not sv.should_stop() and epoch < nb_epochs: 183 | 184 | lr = starting_lr 185 | begin = time.time() 186 | 187 | # construct randomly permuted minibatches 188 | trainx = trainx[rng.permutation(trainx.shape[0])] # shuffling dataset 189 | trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])] 190 | train_loss_rec, train_loss = [0, 0] 191 | 192 | # training 193 | for t in range(nr_batches_train): 194 | 195 | display_progression_epoch(t, nr_batches_train) 196 | ran_from = t * batch_size 197 | ran_to = ran_from + batch_size 198 | feed_dict = {x_pl: trainx[ran_from:ran_to], 199 | is_training_pl: True, 200 | learning_rate:lr} 201 | 202 | _, lrec, loss, sm, step = sess.run([train_op, 203 | loss_rec, 204 | full_loss, 205 | sum_op_loss, 206 | global_step], 207 | feed_dict=feed_dict) 208 | train_loss_rec += lrec 209 | train_loss += loss 210 | writer.add_summary(sm, step)#train_batch) 211 | 212 | if np.isnan(loss): 213 | logger.info("Loss is nan - Stopping") 214 | break 215 | 216 | train_batch += 1 217 | 218 | if np.isnan(loss): 219 | logger.info("Loss is nan - Stopping") 220 | break 221 | 222 | train_loss_rec /= nr_batches_train 223 | train_loss /= nr_batches_train 224 | 225 | logger.info('Epoch terminated') 226 | print("Epoch %d | time = %ds | loss rec = %.4f " 227 | "| loss = %.4f" 228 | % (epoch, time.time() - begin, train_loss_rec, 229 | train_loss)) 230 | 231 | epoch += 1 232 | 233 | 234 | logger.warning('Testing evaluation...') 235 | 236 | inds = rng.permutation(testx.shape[0]) 237 | 238 | ##TESTING PER BATCHS 239 | inference_time = [] 240 | scores = [] 241 | for t in range(nr_batches_test+1): 242 | ran_from = t * batch_size 243 | ran_to = min(ran_from + batch_size, testx.shape[0]) 244 | feed_dict = {x_pl: testx[ran_from:ran_to], 245 | is_training_pl: False} 246 | begin_val = time.time() 247 | if l1>0: 248 | scoresb, step = sess.run([energy, global_step], feed_dict=feed_dict) 249 | else: 250 | scoresb, step = sess.run([rec_error, global_step], feed_dict=feed_dict) 251 | scores.append(scoresb) 252 | inference_time.append(time.time() - begin_val) 253 | scores = np.concatenate(scores, axis = 0) 254 | 255 | logger.warning('Testing : inference time is %.4f' % ( 256 | np.mean(inference_time))) 257 | 258 | #scores = np.array(scores) 259 | save_results(scores, testy, 'dagmm/K{}'.format(K), dataset, None, str(l1)+"_"+str(l2), label, 260 | random_seed, step) 261 | 262 | 263 | def run(args): 264 | """ Runs the training process""" 265 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) 266 | with tf.Graph().as_default(): 267 | # Set the graph level seed 268 | tf.set_random_seed(args.rd) 269 | train_and_test(args.dataset, args.nb_epochs, args.K, args.l1, args.l2, 270 | args.label, args.rd) 271 | -------------------------------------------------------------------------------- /data/arrhythmia.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/houssamzenati/Adversarially-Learned-Anomaly-Detection/1f1c3109c957bdfab23d684638124282beee7894/data/arrhythmia.mat -------------------------------------------------------------------------------- /data/arrhythmia.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import numpy as np 3 | import pandas as pd 4 | import scipy.io 5 | from sklearn.model_selection import train_test_split 6 | from sklearn.preprocessing import MinMaxScaler 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | def get_train(label=0, scale=False, *args): 11 | """Get training dataset for Thyroid dataset""" 12 | return _get_adapted_dataset("train", scale) 13 | 14 | def get_test(label=0, scale=False, *args): 15 | """Get testing dataset for Thyroid dataset""" 16 | return _get_adapted_dataset("test", scale) 17 | 18 | def get_valid(label=0, scale=False, *args): 19 | """Get validation dataset for Thyroid dataset""" 20 | return None 21 | 22 | def get_shape_input(): 23 | """Get shape of the dataset for Thyroid dataset""" 24 | return (None, 274) 25 | 26 | def get_shape_input_flatten(): 27 | """Get shape of the dataset for Thyroid dataset""" 28 | return (None, 274) 29 | 30 | def get_shape_label(): 31 | """Get shape of the labels in Thyroid dataset""" 32 | return (None,) 33 | 34 | def get_anomalous_proportion(): 35 | return 0.15 36 | 37 | def _get_dataset(scale): 38 | """ Gets the basic dataset 39 | Returns : 40 | dataset (dict): containing the data 41 | dataset['x_train'] (np.array): training images shape 42 | (?, 120) 43 | dataset['y_train'] (np.array): training labels shape 44 | (?,) 45 | dataset['x_test'] (np.array): testing images shape 46 | (?, 120) 47 | dataset['y_test'] (np.array): testing labels shape 48 | (?,) 49 | """ 50 | data = scipy.io.loadmat("data/arrhythmia.mat") 51 | 52 | full_x_data = data["X"] 53 | full_y_data = data['y'] 54 | 55 | x_train, x_test, \ 56 | y_train, y_test = train_test_split(full_x_data, 57 | full_y_data, 58 | test_size=0.5, 59 | random_state=42) 60 | 61 | y_train = y_train.flatten().astype(int) 62 | y_test = y_test.flatten().astype(int) 63 | 64 | if scale: 65 | print("Scaling dataset") 66 | scaler = MinMaxScaler() 67 | scaler.fit(x_train) 68 | x_train = scaler.transform(x_train) 69 | x_test = scaler.transform(x_test) 70 | 71 | 72 | dataset = {} 73 | dataset['x_train'] = x_train.astype(np.float32) 74 | dataset['y_train'] = y_train.astype(np.float32) 75 | dataset['x_test'] = x_test.astype(np.float32) 76 | dataset['y_test'] = y_test.astype(np.float32) 77 | 78 | return dataset 79 | 80 | def _get_adapted_dataset(split, scale): 81 | """ Gets the adapted dataset for the experiments 82 | 83 | Args : 84 | split (str): train or test 85 | Returns : 86 | (tuple): images and labels 87 | """ 88 | # print("_get_adapted",scale) 89 | dataset = _get_dataset(scale) 90 | key_img = 'x_' + split 91 | key_lbl = 'y_' + split 92 | 93 | print("Size of split", split, ":", dataset[key_lbl].shape[0]) 94 | 95 | return (dataset[key_img], dataset[key_lbl]) 96 | 97 | def _to_xy(df, target): 98 | """Converts a Pandas dataframe to the x,y inputs that TensorFlow needs""" 99 | result = [] 100 | for x in df.columns: 101 | if x != target: 102 | result.append(x) 103 | dummies = df[target] 104 | return df.as_matrix(result).astype(np.float32), dummies.as_matrix().astype(np.float32) 105 | 106 | -------------------------------------------------------------------------------- /data/cifar10.py: -------------------------------------------------------------------------------- 1 | # CIFAR10 Downloader 2 | 3 | import logging 4 | import pickle 5 | import math 6 | import os 7 | import errno 8 | import tarfile 9 | import shutil 10 | import numpy as np 11 | import urllib3 12 | from sklearn.model_selection import train_test_split 13 | from utils.adapt_data import adapt_labels_outlier_task 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | def get_train(label=-1, centered=True, normalize=True): 18 | return _get_adapted_dataset("train", label, centered, normalize) 19 | 20 | def get_test(label=-1, centered=True, normalize=True): 21 | return _get_adapted_dataset("test", label, centered, normalize) 22 | 23 | def get_valid(label=-1, centered=True, normalize=True): 24 | return _get_adapted_dataset("valid", label, centered, normalize) 25 | 26 | def get_shape_input(): 27 | return (None, 32, 32, 3) 28 | 29 | def get_shape_input_flatten(): 30 | return (None, 32*32*3) 31 | 32 | def get_shape_label(): 33 | return (None,) 34 | 35 | def num_classes(): 36 | return 10 37 | 38 | def get_anomalous_proportion(): 39 | return 0.9 40 | 41 | def _unpickle_file(filename): 42 | logger.debug("Loading pickle file: {}".format(filename)) 43 | 44 | with open(filename, mode='rb') as file: 45 | data = pickle.load(file, encoding='bytes') 46 | 47 | # Reorder the data 48 | img = data[b'data'] 49 | img = img.reshape([-1, 3, 32, 32]) 50 | img = img.transpose([0, 2, 3, 1]) 51 | # Load labels 52 | lbl = np.array(data[b'labels']) 53 | 54 | return img, lbl 55 | 56 | def _get_dataset(split, centered=False, normalize=False): 57 | ''' 58 | Gets the adapted dataset for the experiments 59 | Args : 60 | split (str): train or test 61 | normalize (bool): (Default=True) normalize data 62 | centered (bool): (Default=False) data centered to [-1, 1] 63 | Returns : 64 | (tuple): images and labels 65 | ''' 66 | path = "data" 67 | dirname = "cifar-10-batches-py" 68 | data_url = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 69 | 70 | if not os.path.exists(os.path.join(path, dirname)): 71 | # Extract or download data 72 | try: 73 | os.makedirs(path) 74 | except OSError as exception: 75 | if exception.errno != errno.EEXIST: 76 | raise 77 | 78 | file_path = os.path.join(path, data_url.split('/')[-1]) 79 | if not os.path.exists(file_path): 80 | # Download 81 | logger.warn("Downloading {}".format(data_url)) 82 | with urllib3.PoolManager().request('GET', data_url, preload_content=False) as r, \ 83 | open(file_path, 'wb') as w: 84 | shutil.copyfileobj(r, w) 85 | 86 | logger.warn("Unpacking {}".format(file_path)) 87 | # Unpack data 88 | tarfile.open(name=file_path, mode="r:gz").extractall(path) 89 | 90 | # Import the data 91 | if split == 'test': 92 | filenames = ["test_batch"] 93 | # elif split == 'valid': 94 | # filenames = ["data_batch_5"] 95 | else: 96 | filenames = ["data_batch_{}".format(i) for i in range(1, 6)] 97 | 98 | imgs = [] 99 | lbls = [] 100 | for f in filenames: 101 | img, lbl = _unpickle_file(os.path.join(path, dirname, f)) 102 | imgs.append(img) 103 | lbls.append(lbl) 104 | 105 | # Now we flatten the arrays 106 | imgs = np.concatenate(imgs) 107 | lbls = np.concatenate(lbls) 108 | 109 | # Convert images to [0..1] range 110 | if normalize: 111 | imgs = imgs.astype(np.float32)/255.0 112 | if centered: 113 | imgs = imgs.astype(np.float32)*2. - 1. 114 | return imgs.astype(np.float32), lbls 115 | 116 | def _get_adapted_dataset(split, label=None, centered=False, normalize=False): 117 | """ 118 | Gets the adapted dataset for the experiments 119 | Args : 120 | split (str): train or test 121 | mode (str): inlier or outlier 122 | label (int): int in range 0 to 10, is the class/digit 123 | which is considered inlier or outlier 124 | rho (float): proportion of anomalous classes INLIER 125 | MODE ONLY 126 | centered (bool): (Default=False) data centered to [-1, 1] 127 | Returns : 128 | (tuple): images and labels 129 | """ 130 | dataset = {} 131 | dataset['x_train'], dataset['y_train'] = _get_dataset('train', 132 | centered=centered, 133 | normalize=normalize) 134 | dataset['x_test'], dataset['y_test'] = _get_dataset('test', centered=centered, 135 | normalize=normalize) 136 | 137 | full_x_data = np.concatenate([dataset['x_train'], dataset['x_test']], axis=0) 138 | full_y_data = np.concatenate([dataset['y_train'], dataset['y_test']], axis=0) 139 | 140 | full_y_data[full_y_data == 10] = 0 141 | 142 | dataset['x_train'], dataset['x_test'], \ 143 | dataset['y_train'], dataset['y_test'] = train_test_split(full_x_data, 144 | full_y_data, 145 | test_size=0.2, 146 | random_state=42) 147 | 148 | dataset['x_train'], dataset['x_valid'], \ 149 | dataset['y_train'], dataset['y_valid'] = train_test_split(dataset['x_train'], 150 | dataset['y_train'], 151 | test_size=0.25, 152 | random_state=42) 153 | 154 | 155 | key_img = 'x_' + split 156 | key_lbl = 'y_' + split 157 | 158 | if label != -1: 159 | 160 | if split in ['train', 'valid']: 161 | 162 | inliers = dataset[key_img][dataset[key_lbl] == label], \ 163 | dataset[key_lbl][dataset[key_lbl] == label] 164 | outliers = dataset[key_img][dataset[key_lbl] != label], \ 165 | dataset[key_lbl][dataset[key_lbl] != label] 166 | 167 | dataset[key_img], dataset[key_lbl] = inliers 168 | 169 | dataset[key_lbl] = adapt_labels_outlier_task(dataset[key_lbl], 170 | label) 171 | return (dataset[key_img], dataset[key_lbl]) 172 | else: 173 | dataset[key_lbl] = adapt_labels_outlier_task(dataset[key_lbl], 174 | label) 175 | 176 | return (dataset[key_img], dataset[key_lbl]) 177 | -------------------------------------------------------------------------------- /data/kdd.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import numpy as np 3 | import pandas as pd 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.preprocessing import MinMaxScaler 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | def get_train(label=0, scale=False, *args): 10 | """Get training dataset for KDD 10 percent""" 11 | return _get_adapted_dataset("train", scale) 12 | 13 | def get_test(label=0, scale=False, *args): 14 | """Get testing dataset for KDD 10 percent""" 15 | return _get_adapted_dataset("test", scale) 16 | 17 | def get_valid(label=0, scale=False, *args): 18 | """Get validation dataset for KDD 10 percent""" 19 | return None 20 | 21 | def get_shape_input(): 22 | """Get shape of the dataset for KDD 10 percent""" 23 | return (None, 121) 24 | 25 | def get_shape_label(): 26 | """Get shape of the labels in KDD 10 percent""" 27 | return (None,) 28 | 29 | def get_anomalous_proportion(): 30 | return 0.2 31 | 32 | 33 | def _get_dataset(scale): 34 | """ Gets the basic dataset 35 | Returns : 36 | dataset (dict): containing the data 37 | dataset['x_train'] (np.array): training images shape 38 | (?, 120) 39 | dataset['y_train'] (np.array): training labels shape 40 | (?,) 41 | dataset['x_test'] (np.array): testing images shape 42 | (?, 120) 43 | dataset['y_test'] (np.array): testing labels shape 44 | (?,) 45 | """ 46 | col_names = _col_names() 47 | df = pd.read_csv("data/kddcup.data_10_percent_corrected", header=None, names=col_names) 48 | text_l = ['protocol_type', 'service', 'flag', 'land', 'logged_in', 'is_host_login', 'is_guest_login'] 49 | 50 | for name in text_l: 51 | _encode_text_dummy(df, name) 52 | 53 | labels = df['label'].copy() 54 | labels[labels != 'normal.'] = 0 55 | labels[labels == 'normal.'] = 1 56 | 57 | df['label'] = labels 58 | 59 | df_train = df.sample(frac=0.5, random_state=42) 60 | df_test = df.loc[~df.index.isin(df_train.index)] 61 | df_valid = df_train.sample(frac=0.1, random_state=42) 62 | # df_train = df_train.loc[~df_train.index.isin(df_valid.index)] 63 | 64 | x_train, y_train = _to_xy(df_train, target='label') 65 | x_valid, y_valid = _to_xy(df_valid, target='label') 66 | x_test, y_test = _to_xy(df_test, target='label') 67 | 68 | y_train = y_train.flatten().astype(int) 69 | y_valid = y_valid.flatten().astype(int) 70 | y_test = y_test.flatten().astype(int) 71 | x_train = x_train[y_train != 1] 72 | y_train = y_train[y_train != 1] 73 | x_valid = x_valid[y_valid != 1] 74 | y_valid = y_valid[y_valid != 1] 75 | 76 | if scale: 77 | print("Scaling KDD dataset") 78 | scaler = MinMaxScaler() 79 | scaler.fit(x_train) 80 | x_train = scaler.transform(x_train) 81 | x_valid = scaler.transform(x_valid) 82 | x_test = scaler.transform(x_test) 83 | 84 | dataset = {} 85 | dataset['x_train'] = x_train.astype(np.float32) 86 | dataset['y_train'] = y_train.astype(np.float32) 87 | dataset['x_valid'] = x_valid.astype(np.float32) 88 | dataset['y_valid'] = y_valid.astype(np.float32) 89 | dataset['x_test'] = x_test.astype(np.float32) 90 | dataset['y_test'] = y_test.astype(np.float32) 91 | 92 | return dataset 93 | 94 | def _get_adapted_dataset(split, scale): 95 | """ Gets the adapted dataset for the experiments 96 | 97 | Args : 98 | split (str): train or test 99 | Returns : 100 | (tuple): images and labels 101 | """ 102 | # print("_get_adapted",scale) 103 | dataset = _get_dataset(scale) 104 | key_img = 'x_' + split 105 | key_lbl = 'y_' + split 106 | 107 | if split == 'test': 108 | dataset[key_img], dataset[key_lbl] = _adapt_ratio(dataset[key_img], 109 | dataset[key_lbl]) 110 | 111 | return (dataset[key_img], dataset[key_lbl]) 112 | 113 | def _encode_text_dummy(df, name): 114 | """Encode text values to dummy variables(i.e. [1,0,0],[0,1,0],[0,0,1] 115 | for red,green,blue) 116 | """ 117 | dummies = pd.get_dummies(df.loc[:,name]) 118 | for x in dummies.columns: 119 | dummy_name = "{}-{}".format(name, x) 120 | df.loc[:, dummy_name] = dummies[x] 121 | df.drop(name, axis=1, inplace=True) 122 | 123 | def _to_xy(df, target): 124 | """Converts a Pandas dataframe to the x,y inputs that TensorFlow needs""" 125 | result = [] 126 | for x in df.columns: 127 | if x != target: 128 | result.append(x) 129 | dummies = df[target] 130 | return df.as_matrix(result).astype(np.float32), dummies.as_matrix().flatten().astype(int) 131 | 132 | def _col_names(): 133 | """Column names of the dataframe""" 134 | return ["duration","protocol_type","service","flag","src_bytes", 135 | "dst_bytes","land","wrong_fragment","urgent","hot","num_failed_logins", 136 | "logged_in","num_compromised","root_shell","su_attempted","num_root", 137 | "num_file_creations","num_shells","num_access_files","num_outbound_cmds", 138 | "is_host_login","is_guest_login","count","srv_count","serror_rate", 139 | "srv_serror_rate","rerror_rate","srv_rerror_rate","same_srv_rate", 140 | "diff_srv_rate","srv_diff_host_rate","dst_host_count","dst_host_srv_count", 141 | "dst_host_same_srv_rate","dst_host_diff_srv_rate","dst_host_same_src_port_rate", 142 | "dst_host_srv_diff_host_rate","dst_host_serror_rate","dst_host_srv_serror_rate", 143 | "dst_host_rerror_rate","dst_host_srv_rerror_rate","label"] 144 | 145 | def _adapt_ratio(x, y, rho=0.2): 146 | """Adapt the ratio of normal/anomalous data""" 147 | 148 | # Normal data: label =0, anomalous data: label =1 149 | 150 | rng = np.random.RandomState(42) # seed shuffling 151 | 152 | inliersx = x[y == 0] 153 | inliersy = y[y == 0] 154 | outliersx = x[y == 1] 155 | outliersy = y[y == 1] 156 | 157 | size_outliers = outliersx.shape[0] 158 | inds = rng.permutation(size_outliers) 159 | outliersx, outliersy = outliersx[inds], outliersy[inds] 160 | 161 | size_x = inliersx.shape[0] 162 | out_size_x = int(size_x*rho/(1-rho)) 163 | 164 | out_sample_x = outliersx[:out_size_x] 165 | out_sample_y = outliersy[:out_size_x] 166 | 167 | x_adapted = np.concatenate((inliersx,out_sample_x), axis=0) 168 | y_adapted = np.concatenate((inliersy,out_sample_y), axis=0) 169 | 170 | size_x = x_adapted.shape[0] 171 | inds = rng.permutation(size_x) 172 | x_adapted, y_adapted = x_adapted[inds], y_adapted[inds] 173 | 174 | return x_adapted, y_adapted 175 | -------------------------------------------------------------------------------- /data/svhn.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | from six.moves import urllib 4 | from scipy.io import loadmat 5 | import logging 6 | from utils.adapt_data import adapt_labels_outlier_task 7 | import numpy as np 8 | from sklearn.model_selection import train_test_split 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def get_train(label=-1, centered=True, normalize=True): 14 | return _get_adapted_dataset("train", label, centered, normalize) 15 | 16 | def get_test(label=-1, centered=True, normalize=True): 17 | return _get_adapted_dataset("test", label, centered, normalize) 18 | 19 | def get_valid(label=-1, centered=True, normalize=True): 20 | return _get_adapted_dataset("valid", label, centered, normalize) 21 | 22 | def get_shape_input(): 23 | return (None, 32, 32, 3) 24 | 25 | def get_shape_input_flatten(): 26 | return (None, 3072) 27 | 28 | def get_anomalous_proportion(): 29 | return 0.9 30 | 31 | def _get_adapted_dataset(split, label, centered, normalize): 32 | """ 33 | Gets the adapted dataset for the experiments 34 | Args : 35 | split (str): train or test 36 | mode (str): inlier or outlier 37 | label (int): int in range 0 to 10, is the class/digit 38 | which is considered inlier or outlier 39 | centered (bool): (Default=False) data centered to [-1, 1] 40 | Returns : 41 | (tuple): images and labels 42 | :type label: object 43 | """ 44 | dataset = {} 45 | dataset['x_train'], dataset['y_train'] = load(subset='train') 46 | dataset['x_test'], dataset['y_test'] = load(subset='test') 47 | 48 | def rescale(mat): 49 | return np.transpose(mat, (3, 0, 1, 2)) 50 | 51 | dataset['x_train'] = rescale(dataset['x_train']) 52 | dataset['x_test'] = rescale(dataset['x_test']) 53 | 54 | if normalize: 55 | dataset['x_train'] = dataset['x_train'].astype(np.float32) / 255.0 56 | dataset['x_test'] = dataset['x_test'].astype(np.float32) / 255.0 57 | if centered: 58 | dataset['x_train'] = dataset['x_train'].astype(np.float32) * 2. - 1. 59 | dataset['x_test'] = dataset['x_test'].astype(np.float32) * 2. - 1. 60 | 61 | full_x_data = np.concatenate([dataset['x_train'], dataset['x_test']], axis=0) 62 | full_y_data = np.concatenate([dataset['y_train'], dataset['y_test']], axis=0) 63 | 64 | # 65 | dataset['x_train'], dataset['x_test'], \ 66 | dataset['y_train'], dataset['y_test'] = train_test_split(full_x_data, 67 | full_y_data, 68 | test_size=0.2, 69 | random_state=42) 70 | 71 | dataset['x_train'], dataset['x_valid'], \ 72 | dataset['y_train'], dataset['y_valid'] = train_test_split(dataset['x_train'], 73 | dataset['y_train'], 74 | test_size=0.25, 75 | random_state=42) 76 | 77 | key_img = 'x_' + split 78 | key_lbl = 'y_' + split 79 | 80 | if label != -1: 81 | 82 | if split in ['train', 'valid']: 83 | 84 | inliers = dataset[key_img][dataset[key_lbl] == label], \ 85 | dataset[key_lbl][dataset[key_lbl] == label] 86 | outliers = dataset[key_img][dataset[key_lbl] != label], \ 87 | dataset[key_lbl][dataset[key_lbl] != label] 88 | 89 | dataset[key_img], dataset[key_lbl] = inliers 90 | 91 | dataset[key_lbl] = adapt_labels_outlier_task(dataset[key_lbl], 92 | label) 93 | else: 94 | dataset[key_lbl] = adapt_labels_outlier_task(dataset[key_lbl], 95 | label) 96 | 97 | return (dataset[key_img], dataset[key_lbl]) 98 | 99 | def maybe_download(data_dir): 100 | new_data_dir = os.path.join(data_dir, 'svhn') 101 | if not os.path.exists(new_data_dir): 102 | os.makedirs(new_data_dir) 103 | def _progress(count, block_size, total_size): 104 | sys.stdout.write('\r>> Downloading %.1f%%' % (float(count * block_size) / float(total_size) * 100.0)) 105 | sys.stdout.flush() 106 | filepath, _ = urllib.request.urlretrieve('http://ufldl.stanford.edu/housenumbers/train_32x32.mat', new_data_dir+'/train_32x32.mat', _progress) 107 | filepath, _ = urllib.request.urlretrieve('http://ufldl.stanford.edu/housenumbers/test_32x32.mat', new_data_dir+'/test_32x32.mat', _progress) 108 | 109 | def load(data_dir='./data/svhn', subset='train'): 110 | maybe_download(data_dir) 111 | if subset=='train': 112 | train_data = loadmat(os.path.join(data_dir, 'svhn') + '/train_32x32.mat') 113 | trainx = train_data['X'] 114 | trainy = train_data['y'].flatten() 115 | trainy[trainy==10] = 0 116 | return trainx, trainy 117 | elif subset=='test': 118 | test_data = loadmat(os.path.join(data_dir, 'svhn') + '/test_32x32.mat') 119 | testx = test_data['X'] 120 | testy = test_data['y'].flatten() 121 | testy[testy==10] = 0 122 | return testx, testy 123 | else: 124 | raise NotImplementedError('subset should be either train or test') 125 | -------------------------------------------------------------------------------- /dsebm/arrhythmia_utilities.py: -------------------------------------------------------------------------------- 1 | # TensorFlow implementation of a DCGAN model for arrhythmia 2 | 3 | import tensorflow as tf 4 | 5 | learning_rate = 0.0001 6 | batch_size = 26 7 | init_kernel = tf.contrib.layers.xavier_initializer() 8 | 9 | def UnPooling2x2ZeroFilled(x): 10 | # https://github.com/tensorflow/tensorflow/issues/2169 11 | out = tf.concat([x, tf.zeros_like(x)], 3) 12 | out = tf.concat([out, tf.zeros_like(out)], 2) 13 | 14 | sh = x.get_shape().as_list() 15 | if None not in sh[1:]: 16 | out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]] 17 | return tf.reshape(out, out_size) 18 | else: 19 | shv = tf.shape(x) 20 | ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]])) 21 | return ret 22 | 23 | 24 | def network(x_inp, is_training=False, getter=None, reuse=False): 25 | """ Network architecture in tensorflow 26 | 27 | Discriminates between real data and generated data 28 | 29 | Note: 30 | Provides histogram and distribution tensorflow summaries 31 | 32 | Args: 33 | x_inp (tensor): input data for the encoder. 34 | reuse (bool): sharing variables or not 35 | 36 | Returns: 37 | logits (tensor): last activation layer of the discriminator (shape 1) 38 | intermediate_layer (tensor): intermediate layer for feature matching 39 | 40 | """ 41 | with tf.variable_scope('network', reuse=reuse, custom_getter=getter): 42 | kernel_dense = tf.get_variable('kernel_dense', [274, 10], initializer=init_kernel) 43 | bias_dense = tf.get_variable('bias_dense', [10]) 44 | kernel_dense2 = tf.get_variable('kernel_dense2', [10, 2], initializer=init_kernel) 45 | bias_dense2 = tf.get_variable('bias_dense2', [2]) 46 | bias_inv_dense2 = tf.get_variable('bias_inv_dense2', [10]) 47 | bias_inv_dense = tf.get_variable('bias_inv_dense', [274]) 48 | 49 | 50 | x = tf.nn.softplus(tf.matmul(x_inp, kernel_dense) + bias_dense) 51 | 52 | x = tf.nn.softplus(tf.matmul(x, kernel_dense2) + bias_dense2) 53 | 54 | ###INVERSE LAYERS 55 | 56 | x = tf.nn.softplus(tf.matmul(x, tf.transpose(kernel_dense2)) + bias_inv_dense2) 57 | 58 | x = tf.nn.softplus(tf.matmul(x, tf.transpose(kernel_dense)) + bias_inv_dense) 59 | 60 | return x 61 | 62 | 63 | -------------------------------------------------------------------------------- /dsebm/cifar10_utilities.py: -------------------------------------------------------------------------------- 1 | # TensorFlow implementation of a DSEBM model for CIFAR10 2 | 3 | import tensorflow as tf 4 | 5 | init_kernel = tf.contrib.layers.xavier_initializer() 6 | image_size = 32 7 | 8 | learning_rate = 0.003 9 | batch_size = 32 10 | kernel_conv_size = 3 11 | filters_conv = 64 12 | filters_fc = 128 13 | strides_conv = 2 14 | 15 | def UnPooling2x2ZeroFilled(x): 16 | # https://github.com/tensorflow/tensorflow/issues/2169 17 | out = tf.concat([x, tf.zeros_like(x)], 3) 18 | out = tf.concat([out, tf.zeros_like(out)], 2) 19 | 20 | sh = x.get_shape().as_list() 21 | if None not in sh[1:]: 22 | out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]] 23 | return tf.reshape(out, out_size) 24 | else: 25 | shv = tf.shape(x) 26 | ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]])) 27 | return ret 28 | 29 | 30 | def network(x_inp, is_training=False, getter=None, reuse=False): 31 | """ Network architecture in tensorflow 32 | 33 | Discriminates between real data and generated data 34 | 35 | Note: 36 | Provides histogram and distribution tensorflow summaries 37 | 38 | Args: 39 | x_inp (tensor): input data for the encoder. 40 | reuse (bool): sharing variables or not 41 | 42 | Returns: 43 | logits (tensor): last activation layer of the discriminator (shape 1) 44 | intermediate_layer (tensor): intermediate layer for feature matching 45 | 46 | """ 47 | with tf.variable_scope('network', reuse=reuse, custom_getter=getter): 48 | kernel_conv = tf.get_variable('kernel_conv', [kernel_conv_size, kernel_conv_size, 3, filters_conv], initializer=init_kernel) 49 | conv_output_size = int(image_size*image_size/4/strides_conv/strides_conv*filters_conv) 50 | kernel_dense = tf.get_variable('kernel_dense', [conv_output_size, filters_fc], initializer=init_kernel) 51 | bias_dense = tf.get_variable('bias_dense', [filters_fc]) 52 | bias_inv_dense = tf.get_variable('bias_inv_dense', [conv_output_size]) 53 | 54 | x = tf.nn.conv2d(x_inp, 55 | kernel_conv, 56 | [1, strides_conv, strides_conv, 1], 57 | 'SAME') 58 | x = tf.nn.softplus(x) 59 | x = tf.nn.pool(x, (2, 2), "MAX", "SAME", strides=(2, 2)) 60 | 61 | x = tf.contrib.layers.flatten(x) 62 | x = tf.nn.softplus(tf.matmul(x, kernel_dense) + bias_dense) 63 | 64 | ###INVERSE LAYERS 65 | 66 | x = tf.nn.softplus(tf.matmul(x, tf.transpose(kernel_dense)) + bias_inv_dense) 67 | new_image_size = int(image_size/2/strides_conv) 68 | x = tf.reshape(x, [-1, new_image_size, new_image_size, filters_conv]) 69 | 70 | x = UnPooling2x2ZeroFilled(x) 71 | x = tf.nn.conv2d_transpose(x, 72 | kernel_conv, 73 | tf.shape(x_inp), 74 | [1, strides_conv, strides_conv, 1], 75 | 'SAME') 76 | 77 | x = tf.nn.softplus(x, name='softplus') 78 | 79 | 80 | return x 81 | 82 | 83 | -------------------------------------------------------------------------------- /dsebm/kdd_utilities.py: -------------------------------------------------------------------------------- 1 | # TensorFlow implementation of a DCGAN model for MNIST 2 | 3 | import tensorflow as tf 4 | learning_rate = 0.0001 5 | batch_size = 100 6 | init_kernel = tf.contrib.layers.xavier_initializer() 7 | 8 | def UnPooling2x2ZeroFilled(x): 9 | # https://github.com/tensorflow/tensorflow/issues/2169 10 | out = tf.concat([x, tf.zeros_like(x)], 3) 11 | out = tf.concat([out, tf.zeros_like(out)], 2) 12 | 13 | sh = x.get_shape().as_list() 14 | if None not in sh[1:]: 15 | out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]] 16 | return tf.reshape(out, out_size) 17 | else: 18 | shv = tf.shape(x) 19 | ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]])) 20 | return ret 21 | 22 | 23 | def network(x_inp, is_training=False, getter=None, reuse=False): 24 | """ Network architecture in tensorflow 25 | 26 | Discriminates between real data and generated data 27 | 28 | Note: 29 | Provides histogram and distribution tensorflow summaries 30 | 31 | Args: 32 | x_inp (tensor): input data for the encoder. 33 | reuse (bool): sharing variables or not 34 | 35 | Returns: 36 | logits (tensor): last activation layer of the discriminator (shape 1) 37 | intermediate_layer (tensor): intermediate layer for feature matching 38 | 39 | """ 40 | with tf.variable_scope('network', reuse=reuse, custom_getter=getter): 41 | kernel_dense = tf.get_variable('kernel_dense', [121, 128], initializer=init_kernel) 42 | bias_dense = tf.get_variable('bias_dense', [128]) 43 | kernel_dense2 = tf.get_variable('kernel_dense2', [128, 512], initializer=init_kernel) 44 | bias_dense2 = tf.get_variable('bias_dense2', [512]) 45 | bias_inv_dense2 = tf.get_variable('bias_inv_dense2', [128]) 46 | bias_inv_dense = tf.get_variable('bias_inv_dense', [121]) 47 | 48 | 49 | x = tf.nn.softplus(tf.matmul(x_inp, kernel_dense) + bias_dense) 50 | 51 | x = tf.nn.softplus(tf.matmul(x, kernel_dense2) + bias_dense2) 52 | 53 | ###INVERSE LAYERS 54 | 55 | x = tf.nn.softplus(tf.matmul(x, tf.transpose(kernel_dense2)) + bias_inv_dense2) 56 | 57 | x = tf.nn.softplus(tf.matmul(x, tf.transpose(kernel_dense)) + bias_inv_dense) 58 | 59 | return x 60 | 61 | 62 | -------------------------------------------------------------------------------- /dsebm/run.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tensorflow as tf 4 | import logging 5 | import importlib 6 | import sys 7 | import os 8 | from utils.adapt_data import batch_fill 9 | from utils.evaluations import save_grid_plot, save_results 10 | from utils.constants import IMAGES_DATASETS 11 | 12 | RANDOM_SEED = 13 13 | FREQ_PRINT = 50 # print frequency image tensorboard [20] 14 | FREQ_EV = 1 15 | STRIP_EV = 5 16 | FREQ_SNAP = 1000 17 | BATCH_EV = 50 18 | ENABLE_EV = False 19 | PATIENCE = 25 20 | 21 | def display_parameters(batch_size, starting_lr, label): 22 | """See parameters 23 | """ 24 | print('Batch size: ', batch_size) 25 | print('Starting learning rate: ', starting_lr) 26 | print('Anomalous label: ', label) 27 | 28 | def display_progression_epoch(j, id_max): 29 | """See epoch progression 30 | """ 31 | batch_progression = int((j / id_max) * 100) 32 | sys.stdout.write(str(batch_progression) + ' % epoch' + chr(13)) 33 | _ = sys.stdout.flush 34 | 35 | def create_logdir(dataset, label, rd): 36 | """ Directory to save training logs, weights, biases, etc.""" 37 | return "dsebm/train_logs/{}/label{}/rd{}".format( 38 | dataset,label, rd) 39 | 40 | 41 | def train_and_test(dataset, nb_epochs, random_seed, label): 42 | 43 | """ Runs DSEBM on available datasets 44 | 45 | Note: 46 | Saves summaries on tensorboard. To display them, please use cmd line 47 | tensorboard --logdir=model.training_logdir() --port=number 48 | Args: 49 | dataset (string): dataset to run the model on 50 | nb_epochs (int): number of epochs 51 | random_seed (int): trying different seeds for averaging the results 52 | label (int): label which is normal for image experiments 53 | anomaly_type (string): "novelty" for 100% normal samples in the training set 54 | "outlier" for a contamined training set 55 | anomaly_proportion (float): if "outlier", anomaly proportion in the training set 56 | """ 57 | logger = logging.getLogger("DSEBM.run.{}.{}".format( 58 | dataset, label)) 59 | 60 | # Import model and data 61 | network = importlib.import_module('dsebm.{}_utilities'.format(dataset)) 62 | data = importlib.import_module("data.{}".format(dataset)) 63 | 64 | # Parameters 65 | starting_lr = network.learning_rate 66 | batch_size = network.batch_size 67 | 68 | # Placeholders 69 | x_pl = tf.placeholder(tf.float32, shape=data.get_shape_input(), 70 | name="input") 71 | is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl') 72 | learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl") 73 | 74 | #test 75 | y_true = tf.placeholder(tf.int32, shape=[None], name="y_true") 76 | 77 | logger.info('Building training graph...') 78 | logger.warn("The DSEBM is training with the following parameters:") 79 | display_parameters(batch_size, starting_lr, label) 80 | 81 | net = network.network 82 | 83 | global_step = tf.train.get_or_create_global_step() 84 | 85 | noise = tf.random_normal(shape=tf.shape(x_pl), mean=0.0, stddev=1., 86 | dtype=tf.float32) 87 | x_noise = x_pl + noise 88 | 89 | with tf.variable_scope('network'): 90 | b_prime_shape = list(data.get_shape_input()) 91 | b_prime_shape[0] = batch_size 92 | b_prime = tf.get_variable(name='b_prime', shape=b_prime_shape)#tf.shape(x_pl)) 93 | net_out = net(x_pl, is_training=is_training_pl) 94 | net_out_noise = net(x_noise, is_training=is_training_pl, reuse=True) 95 | 96 | with tf.name_scope('energies'): 97 | energy = 0.5 * tf.reduce_sum(tf.square(x_pl - b_prime)) \ 98 | - tf.reduce_sum(net_out) 99 | 100 | energy_noise = 0.5 * tf.reduce_sum(tf.square(x_noise - b_prime)) \ 101 | - tf.reduce_sum(net_out_noise) 102 | 103 | with tf.name_scope('reconstructions'): 104 | # reconstruction 105 | grad = tf.gradients(energy, x_pl) 106 | fx = x_pl - tf.gradients(energy, x_pl) 107 | fx = tf.squeeze(fx, axis=0) 108 | fx_noise = x_noise - tf.gradients(energy_noise, x_noise) 109 | 110 | with tf.name_scope("loss_function"): 111 | # DSEBM for images 112 | if len(data.get_shape_input())==4: 113 | loss = tf.reduce_mean(tf.reduce_sum(tf.square(x_pl - fx_noise), 114 | axis=[1,2,3])) 115 | # DSEBM for tabular data 116 | else: 117 | loss = tf.reduce_mean(tf.square(x_pl - fx_noise)) 118 | 119 | with tf.name_scope('optimizers'): 120 | # control op dependencies for batch norm and trainable variables 121 | tvars = tf.trainable_variables() 122 | netvars = [var for var in tvars if 'network' in var.name] 123 | 124 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 125 | update_ops_net = [x for x in update_ops if ('network' in x.name)] 126 | 127 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, 128 | name='optimizer') 129 | 130 | with tf.control_dependencies(update_ops_net): 131 | train_op = optimizer.minimize(loss, var_list=netvars, global_step=global_step) 132 | 133 | with tf.variable_scope('Scores'): 134 | 135 | with tf.name_scope('Energy_score'): 136 | flat = tf.layers.flatten(x_pl - b_prime) 137 | if len(data.get_shape_input())==4: 138 | list_scores_energy = 0.5 * tf.reduce_sum(tf.square(flat), axis=1) \ 139 | - tf.reduce_sum(net_out, axis=[1, 2, 3]) 140 | else: 141 | list_scores_energy = 0.5 * tf.reduce_sum(tf.square(flat), axis=1) \ 142 | - tf.reduce_sum(net_out, axis=1) 143 | with tf.name_scope('Reconstruction_score'): 144 | delta = x_pl - fx 145 | delta_flat = tf.layers.flatten(delta) 146 | list_scores_reconstruction = tf.norm(delta_flat, ord=2, axis=1, 147 | keepdims=False, 148 | name='reconstruction') 149 | 150 | # with tf.name_scope('predictions'): 151 | # # Highest 20% are anomalous 152 | # if dataset=="kdd": 153 | # per = tf.contrib.distributions.percentile(list_scores_energy, 80) 154 | # else: 155 | # per = tf.contrib.distributions.percentile(list_scores_energy, 95) 156 | # y_pred = tf.greater_equal(list_scores_energy, per) 157 | # 158 | # #y_test_true = tf.cast(y_test_true, tf.float32) 159 | # cm = tf.confusion_matrix(y_true, y_pred, num_classes=2) 160 | # recall = cm[1,1]/(cm[1,0]+cm[1,1]) 161 | # precision = cm[1,1]/(cm[0,1]+cm[1,1]) 162 | # f1 = 2*precision*recall/(precision + recall) 163 | 164 | with tf.name_scope('training_summary'): 165 | 166 | tf.summary.scalar('score_matching_loss', loss, ['net']) 167 | tf.summary.scalar('energy', energy, ['net']) 168 | 169 | if dataset in IMAGES_DATASETS: 170 | with tf.name_scope('image_summary'): 171 | tf.summary.image('reconstruct', fx, 6, ['image']) 172 | tf.summary.image('input_images', x_pl, 6, ['image']) 173 | sum_op_im = tf.summary.merge_all('image') 174 | 175 | sum_op_net = tf.summary.merge_all('net') 176 | 177 | logdir = create_logdir(dataset, label, random_seed) 178 | 179 | sv = tf.train.Supervisor(logdir=logdir+"/train", save_summaries_secs=None, 180 | save_model_secs=None) 181 | 182 | # Data 183 | logger.info('Data loading...') 184 | trainx, trainy = data.get_train(label) 185 | trainx_copy = trainx.copy() 186 | if dataset in IMAGES_DATASETS: validx, validy = data.get_valid(label) 187 | testx, testy = data.get_test(label) 188 | 189 | rng = np.random.RandomState(RANDOM_SEED) 190 | nr_batches_train = int(trainx.shape[0] / batch_size) 191 | if dataset in IMAGES_DATASETS: nr_batches_valid = int(validx.shape[0] / batch_size) 192 | nr_batches_test = int(testx.shape[0] / batch_size) 193 | 194 | logger.info("Train: {} samples in {} batches".format(trainx.shape[0], nr_batches_train)) 195 | if dataset in IMAGES_DATASETS: logger.info("Valid: {} samples in {} batches".format(validx.shape[0], nr_batches_valid)) 196 | logger.info("Test: {} samples in {} batches".format(testx.shape[0], nr_batches_test)) 197 | 198 | logger.info('Start training...') 199 | with sv.managed_session() as sess: 200 | logger.info('Initialization done') 201 | 202 | train_writer = tf.summary.FileWriter(logdir+"/train", sess.graph) 203 | valid_writer = tf.summary.FileWriter(logdir+"/valid", sess.graph) 204 | 205 | train_batch = 0 206 | epoch = 0 207 | best_valid_loss = 0 208 | train_losses = [0]*STRIP_EV 209 | 210 | while not sv.should_stop() and epoch < nb_epochs: 211 | lr = starting_lr 212 | 213 | begin = time.time() 214 | trainx = trainx[rng.permutation(trainx.shape[0])] # shuffling unl dataset 215 | trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])] 216 | 217 | losses, energies = [0, 0] 218 | # training 219 | for t in range(nr_batches_train): 220 | display_progression_epoch(t, nr_batches_train) 221 | 222 | # construct randomly permuted minibatches 223 | ran_from = t * batch_size 224 | ran_to = (t + 1) * batch_size 225 | 226 | # train the net 227 | feed_dict = {x_pl: trainx[ran_from:ran_to], 228 | is_training_pl:True, 229 | learning_rate:lr} 230 | _, ld, en, sm, step = sess.run([train_op, loss, energy, sum_op_net, global_step], feed_dict=feed_dict) 231 | losses += ld 232 | energies += en 233 | train_writer.add_summary(sm, step) 234 | 235 | if t % FREQ_PRINT == 0 and dataset in IMAGES_DATASETS: # inspect reconstruction 236 | t= np.random.randint(0,40) 237 | ran_from = t 238 | ran_to = t + batch_size 239 | sm = sess.run(sum_op_im, feed_dict={x_pl: trainx[ran_from:ran_to],is_training_pl: False}) 240 | train_writer.add_summary(sm, step) 241 | 242 | train_batch += 1 243 | 244 | losses /= nr_batches_train 245 | energies /= nr_batches_train 246 | # Remembering loss for early stopping 247 | train_losses[epoch%STRIP_EV] = losses 248 | 249 | logger.info('Epoch terminated') 250 | print("Epoch %d | time = %ds | loss = %.4f | energy = %.4f " 251 | % (epoch, time.time() - begin, losses, energies)) 252 | 253 | if (epoch + 1) % FREQ_SNAP == 0 and dataset in IMAGES_DATASETS: 254 | 255 | print("Take a snap of the reconstructions...") 256 | x = trainx[:batch_size] 257 | feed_dict = {x_pl: x, 258 | is_training_pl: False} 259 | 260 | rect_x = sess.run(fx, feed_dict=feed_dict) 261 | nama_e_wa = "dsebm/reconstructions/{}/{}/" \ 262 | "{}_epoch{}".format(dataset, 263 | label, 264 | random_seed, epoch) 265 | nb_imgs = 50 266 | save_grid_plot(x[:nb_imgs], rect_x[:nb_imgs], nama_e_wa, nb_imgs) 267 | 268 | if (epoch + 1) % FREQ_EV == 0 and dataset in IMAGES_DATASETS: 269 | logger.info("Validation") 270 | inds = rng.permutation(validx.shape[0]) 271 | validx = validx[inds] # shuffling dataset 272 | validy = validy[inds] # shuffling dataset 273 | valid_loss = 0 274 | for t in range(nr_batches_valid): 275 | display_progression_epoch(t, nr_batches_valid) 276 | 277 | # construct randomly permuted minibatches 278 | ran_from = t * batch_size 279 | ran_to = (t + 1) * batch_size 280 | 281 | # train the net 282 | feed_dict = {x_pl: validx[ran_from:ran_to], 283 | y_true: validy[ran_from:ran_to], 284 | is_training_pl:False} 285 | 286 | vl, sm, step = sess.run([loss, sum_op_net, global_step], feed_dict=feed_dict) 287 | valid_writer.add_summary(sm, step+t)#train_batch) 288 | valid_loss += vl 289 | 290 | valid_loss /= nr_batches_valid 291 | 292 | 293 | 294 | # train the net 295 | 296 | logger.info("Validation loss at step "+str(step)+":"+str(valid_loss)) 297 | ##EARLY STOPPING 298 | #UPDATE WEIGHTS 299 | if valid_loss PATIENCE: 309 | sv.request_stop() 310 | logger.warning( 311 | "Early stopping at epoch {} with weights from epoch {}".format( 312 | epoch, epoch - nb_without_improvements)) 313 | epoch += 1 314 | 315 | logger.warn('Testing evaluation...') 316 | 317 | step = sess.run(global_step) 318 | scores_e = [] 319 | scores_r = [] 320 | inference_time = [] 321 | 322 | # Create scores 323 | for t in range(nr_batches_test): 324 | # construct randomly permuted minibatches 325 | ran_from = t * batch_size 326 | ran_to = (t + 1) * batch_size 327 | begin_val_batch = time.time() 328 | 329 | feed_dict = {x_pl: testx[ran_from:ran_to], 330 | is_training_pl:False} 331 | 332 | scores_e += sess.run(list_scores_energy, 333 | feed_dict=feed_dict).tolist() 334 | 335 | scores_r += sess.run(list_scores_reconstruction, 336 | feed_dict=feed_dict).tolist() 337 | inference_time.append(time.time() - begin_val_batch) 338 | 339 | logger.info('Testing : mean inference time is %.4f' % ( 340 | np.mean(inference_time))) 341 | 342 | if testx.shape[0] % batch_size != 0: 343 | batch, size = batch_fill(testx, batch_size) 344 | feed_dict = {x_pl: batch, 345 | is_training_pl: False} 346 | batch_score_e = sess.run(list_scores_energy, 347 | feed_dict=feed_dict).tolist() 348 | batch_score_r = sess.run(list_scores_reconstruction, 349 | feed_dict=feed_dict).tolist() 350 | scores_e += batch_score_e[:size] 351 | scores_r += batch_score_r[:size] 352 | 353 | save_results(scores_e, testy, 'dsebm', dataset, 'energy', "test", label, 354 | random_seed, step) 355 | save_results(scores_r, testy, 'dsebm', dataset, 'reconstruction', "test", 356 | label, random_seed, step) 357 | 358 | def run(args): 359 | """ Runs the training process""" 360 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) 361 | with tf.Graph().as_default(): 362 | # Set the graph level seed 363 | tf.set_random_seed(args.rd) 364 | train_and_test(args.dataset, args.nb_epochs, args.rd, args.label) 365 | -------------------------------------------------------------------------------- /dsebm/svhn_utilities.py: -------------------------------------------------------------------------------- 1 | # TensorFlow implementation of a DCGAN model for SVHN 2 | 3 | import tensorflow as tf 4 | 5 | init_kernel = tf.contrib.layers.xavier_initializer() 6 | image_size = 32 7 | 8 | learning_rate = 0.003 9 | batch_size = 32 10 | kernel_conv_size = 3 11 | filters_conv = 64 12 | filters_fc = 128 13 | strides_conv = 2 14 | 15 | def UnPooling2x2ZeroFilled(x): 16 | # https://github.com/tensorflow/tensorflow/issues/2169 17 | out = tf.concat([x, tf.zeros_like(x)], 3) 18 | out = tf.concat([out, tf.zeros_like(out)], 2) 19 | 20 | sh = x.get_shape().as_list() 21 | if None not in sh[1:]: 22 | out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]] 23 | return tf.reshape(out, out_size) 24 | else: 25 | shv = tf.shape(x) 26 | ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]])) 27 | return ret 28 | 29 | 30 | def network(x_inp, is_training=False, getter=None, reuse=False): 31 | """ Network architecture in tensorflow 32 | 33 | Discriminates between real data and generated data 34 | 35 | Note: 36 | Provides histogram and distribution tensorflow summaries 37 | 38 | Args: 39 | x_inp (tensor): input data for the encoder. 40 | reuse (bool): sharing variables or not 41 | 42 | Returns: 43 | logits (tensor): last activation layer of the discriminator (shape 1) 44 | intermediate_layer (tensor): intermediate layer for feature matching 45 | 46 | """ 47 | with tf.variable_scope('network', reuse=reuse, custom_getter=getter): 48 | kernel_conv = tf.get_variable('kernel_conv', [kernel_conv_size, kernel_conv_size, 3, filters_conv], initializer=init_kernel) 49 | conv_output_size = int(image_size*image_size/4/strides_conv/strides_conv*filters_conv) 50 | kernel_dense = tf.get_variable('kernel_dense', [conv_output_size, filters_fc], initializer=init_kernel) 51 | bias_dense = tf.get_variable('bias_dense', [filters_fc]) 52 | bias_inv_dense = tf.get_variable('bias_inv_dense', [conv_output_size]) 53 | 54 | x = tf.nn.conv2d(x_inp, 55 | kernel_conv, 56 | [1, strides_conv, strides_conv, 1], 57 | 'SAME') 58 | x = tf.nn.softplus(x) 59 | x = tf.nn.pool(x, (2, 2), "MAX", "SAME", strides=(2, 2)) 60 | 61 | x = tf.contrib.layers.flatten(x) 62 | x = tf.nn.softplus(tf.matmul(x, kernel_dense) + bias_dense) 63 | 64 | ###INVERSE LAYERS 65 | 66 | x = tf.nn.softplus(tf.matmul(x, tf.transpose(kernel_dense)) + bias_inv_dense) 67 | new_image_size = int(image_size/2/strides_conv) 68 | x = tf.reshape(x, [-1, new_image_size, new_image_size, filters_conv]) 69 | 70 | x = UnPooling2x2ZeroFilled(x) 71 | x = tf.nn.conv2d_transpose(x, 72 | kernel_conv, 73 | tf.shape(x_inp), 74 | [1, strides_conv, strides_conv, 1], 75 | 'SAME') 76 | 77 | x = tf.nn.softplus(x, name='softplus') 78 | 79 | return x 80 | 81 | 82 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #! python3 2 | 3 | import argparse 4 | import importlib 5 | import logging 6 | import os 7 | import shutil 8 | import urllib3 9 | import zipfile 10 | # import data 11 | 12 | # Logging 13 | console = logging.StreamHandler() 14 | console.setLevel(logging.INFO) 15 | console.setFormatter(logging.Formatter('[%(asctime)s %(levelname)-3s @%(name)s] %(message)s', datefmt='%H:%M:%S')) 16 | logging.basicConfig(level=logging.DEBUG, handlers=[console]) 17 | logging.getLogger("tensorflow").setLevel(logging.WARNING) 18 | logger = logging.getLogger("AnomalyDetection") 19 | 20 | 21 | def run(args): 22 | print(""" 23 | ______ _____ _____ ____ 24 | |_ `.|_ _| / ___ `. .' '. 25 | | | `. \ | | |_/___) | | .--. | 26 | | | | | | | _ .'____.' | | | | 27 | _| |_.' /_| |__/ | / /_____ _| `--' | 28 | |______.'|________| |_______|(_)'.____.' 29 | 30 | """) 31 | 32 | has_effect = False 33 | 34 | if args.model and args.dataset and args.split: 35 | try: 36 | 37 | mod_name = "{}.{}".format(args.model, args.split) 38 | 39 | logger.info("Running script at {}".format(mod_name)) 40 | 41 | mod = importlib.import_module(mod_name) 42 | 43 | mod.run(args) 44 | 45 | except Exception as e: 46 | logger.exception(e) 47 | logger.error("Uhoh, the script halted with an error.") 48 | else: 49 | if not has_effect: 50 | logger.error("Script halted without any effect. To run code, use command:\npython3 main.py {train, test}") 51 | 52 | def path(d): 53 | try: 54 | assert os.path.isdir(d) 55 | return d 56 | except Exception as e: 57 | raise argparse.ArgumentTypeError("Example {} cannot be located.".format(d)) 58 | 59 | if __name__ == "__main__": 60 | 61 | parser = argparse.ArgumentParser(description='Run examples from the DL 2.0 Anomaly Detector.') 62 | parser.add_argument('model', nargs="?", type=path, help='the folder name of the example you want to run e.g gan or bigan') 63 | parser.add_argument('dataset', nargs="?", choices=['kdd','cifar10', 'svhn', 'arrhythmia'], help='the name of the dataset you want to run the experiments on') 64 | 65 | parser.add_argument('split', nargs="?", choices=['run']) 66 | parser.add_argument('--nb_epochs', nargs="?", type=int, default=0, help='number of epochs you want to train the dataset on') 67 | parser.add_argument('--gpu', nargs="?", type=int, default=0, help='which gpu to use') 68 | #anomaly 69 | parser.add_argument('--label', nargs="?", type=int, default=0, help='anomalous label for the experiment') 70 | parser.add_argument('--m', nargs="?", default='fm', choices=['cross-e', 'fm'], 71 | help='mode/method for discriminator loss') 72 | parser.add_argument('--w', nargs="?", type=float, default=0.1, help='weight for AnoGAN') 73 | parser.add_argument('--d', nargs="?", type=int, default=1, help='degree for the L norm') 74 | parser.add_argument('--rd', nargs="?", type=int, default=42, help='random_seed') 75 | parser.add_argument('--enable_sm', action='store_true', help='enable TF summaries') 76 | parser.add_argument('--enable_dzz', action='store_true', help='enable dzz discriminator') 77 | parser.add_argument('--enable_early_stop', action='store_true', help='enable early_stopping') 78 | parser.add_argument('--sn', action='store_true', help='enable spectral_norm') 79 | # args for dagmm 80 | parser.add_argument('--K', nargs="?", type=float, default=-1, help='number of mixtures in GMM') 81 | parser.add_argument('--l1', nargs="?", type=float, default=-1, help='weight of the energy in DAGMM') 82 | parser.add_argument('--l2', nargs="?", type=float, default=-1, help='weight of the penalty of diag term in DAGMM') 83 | 84 | run(parser.parse_args()) 85 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | keras 2 | numpy 3 | urllib3 4 | matplotlib 5 | seaborn 6 | scikit-learn 7 | opencv-python 8 | gensim 9 | pillow 10 | regex 11 | tensorflow-gpu==1.6 12 | -------------------------------------------------------------------------------- /toy_experiments/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/houssamzenati/Adversarially-Learned-Anomaly-Detection/1f1c3109c957bdfab23d684638124282beee7894/toy_experiments/utils/__init__.py -------------------------------------------------------------------------------- /toy_experiments/utils/data_gmm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Created on Sun Mar 19 13:22:00 2017 5 | 6 | @author: cli 7 | """ 8 | 9 | import numpy as np 10 | import numpy.random as npr 11 | from scipy.stats import multivariate_normal 12 | from matplotlib import pyplot as plt 13 | import matplotlib.cm as cm 14 | from functools import reduce 15 | 16 | class sample_GMM(): 17 | """ Toy dataset containing points sampled from a gaussian mixture distribution. 18 | 19 | The dataset contains 3 sources: 20 | * samples 21 | * label 22 | * densities 23 | 24 | """ 25 | def __init__(self, num_examples, means=None, variances=None, priors=None, 26 | **kwargs): 27 | rng = kwargs.pop('rng', None) 28 | if rng is None: 29 | seed = kwargs.pop('seed', 0) 30 | rng = np.random.RandomState(seed) 31 | 32 | gaussian_mixture = GMM_distribution(means=means, 33 | variances=variances, 34 | priors=priors, 35 | rng=rng) 36 | self.means = gaussian_mixture.means 37 | self.variances = gaussian_mixture.variances 38 | self.priors = gaussian_mixture.priors 39 | 40 | features, labels = gaussian_mixture.sample(nsamples=num_examples) 41 | densities = gaussian_mixture.pdf(x=features) 42 | 43 | data ={'samples': features, 'label': labels, 'density': densities} 44 | 45 | self.data = data 46 | 47 | class GMM_distribution(object): 48 | """ Gaussian Mixture Distribution 49 | 50 | Parameters 51 | ---------- 52 | means : tuple of ndarray. 53 | Specifies the means for the gaussian components. 54 | variances : tuple of ndarray. 55 | Specifies the variances for the gaussian components. 56 | priors : tuple of ndarray 57 | Specifies the prior distribution of the components. 58 | 59 | """ 60 | 61 | def __init__(self, means=None, variances=None, priors=None, rng=None, seed=None): 62 | 63 | if means is None: 64 | means = map(lambda x: 10.0 * np.array(x), [[0, 0], 65 | [1, 1], 66 | [-1, -1], 67 | [1, -1], 68 | [-1, 1]]) 69 | # Number of components 70 | self.ncomponents = len(means) 71 | self.dim = means[0].shape[0] 72 | self.means = means 73 | # If prior is not specified let prior be flat. 74 | if priors is None: 75 | priors = [1.0/self.ncomponents for _ in range(self.ncomponents)] 76 | self.priors = priors 77 | # If variances are not specified let variances be identity 78 | if variances is None: 79 | variances = [np.eye(self.dim) for _ in range(self.ncomponents)] 80 | self.variances = variances 81 | 82 | assert len(means) == len(variances), "Mean variances mismatch" 83 | assert len(variances) == len(priors), "prior mismatch" 84 | 85 | if rng is None: 86 | rng = npr.RandomState(seed=seed) 87 | self.rng = rng 88 | 89 | def _sample_prior(self, nsamples): 90 | return self.rng.choice(a=self.ncomponents, 91 | size=(nsamples, ), 92 | replace=True, 93 | p=self.priors) 94 | 95 | def sample(self, nsamples): 96 | # Sampling priors 97 | samples = [] 98 | fathers = self._sample_prior(nsamples=nsamples).tolist() 99 | for father in fathers: 100 | samples.append(self._sample_gaussian(self.means[father], 101 | self.variances[father])) 102 | return np.array(samples), np.array(fathers) 103 | 104 | def _sample_gaussian(self, mean, variance): 105 | # sampling unit gaussians 106 | epsilons = self.rng.normal(size=(self.dim, )) 107 | 108 | return mean + np.linalg.cholesky(variance).dot(epsilons) 109 | 110 | def _gaussian_pdf(self, x, mean, variance): 111 | return multivariate_normal.pdf(x, mean=mean, cov=variance) 112 | 113 | def pdf(self, x): 114 | "Evaluates the the probability density function at the given point x" 115 | pdfs = map(lambda m, v, p: p * self._gaussian_pdf(x, m, v), 116 | self.means, self.variances, self.priors) 117 | return reduce(lambda x, y: x + y, pdfs, 0.0) 118 | 119 | def plot_GMM(dataset, save_path): 120 | figure, axes = plt.subplots(nrows=1, ncols=1, figsize=(4.5, 4.5)) 121 | ax = axes 122 | ax.set_aspect('equal') 123 | ax.set_xlim(-3, 3); ax.set_ylim(-3.5, 3.5) 124 | # ax.set_xlim([-6, 6]) 125 | # ax.set_ylim([-6, 6]) 126 | # ax.set_xticks([-6, -4, -2, 0, 2, 4, 6]) 127 | # ax.set_yticks([-6, -4, -2, 0, 2, 4, 6]) 128 | # ax.set_xlabel('$x_1$') 129 | # ax.set_ylabel('$x_2$') 130 | ax.axis('on') 131 | 132 | ax.set_title('GMM') 133 | 134 | x = dataset.data['samples'] 135 | targets = dataset.data['label'] 136 | 137 | axes.scatter(x[:, 0], x[:, 1], marker='.', c=cm.Set1(targets.astype(float)/2.0/2.0) , alpha=0.3) 138 | plt.tight_layout() 139 | plt.savefig(save_path, transparent=True, bbox_inches='tight') 140 | 141 | if __name__ == '__main__': 142 | means = map(lambda x: np.array(x), [[0, 0], 143 | [2, 2], 144 | [-1, -1], 145 | [1, -1], 146 | [-1, 1]]) 147 | std = 0.1 148 | variances = [np.eye(2) * std for _ in means] 149 | 150 | priors = [1.0/len(means) for _ in means] 151 | 152 | gaussian_mixture = GaussianMixtureDistribution(means=means, 153 | variances=variances, 154 | priors=priors) 155 | dataset = GaussianMixture(1000, means, variances, priors, sources=('features', )) 156 | save_path = './gmm_data.pdf' 157 | draw_GMM(dataset, save_path) 158 | -------------------------------------------------------------------------------- /toy_experiments/utils/data_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn import utils as skutils 3 | 4 | from utils.rng import np_rng, py_rng 5 | 6 | 7 | class ToyDataset(): 8 | def __init__(self, samples, target): 9 | self.data = {"samples":samples, "label":target} 10 | 11 | def center_crop(x, ph, pw=None): 12 | if pw is None: 13 | pw = ph 14 | h, w = x.shape[:2] 15 | j = int(round((h - ph)/2.)) 16 | i = int(round((w - pw)/2.)) 17 | return x[j:j+ph, i:i+pw] 18 | 19 | def patch(x, ph, pw=None): 20 | if pw is None: 21 | pw = ph 22 | h, w = x.shape[:2] 23 | j = py_rng.randint(0, h-ph) 24 | i = py_rng.randint(0, w-pw) 25 | x = x[j:j+ph, i:i+pw] 26 | return x 27 | 28 | def list_shuffle(*data): 29 | idxs = np_rng.permutation(np.arange(len(data[0]))) 30 | if len(data) == 1: 31 | return [data[0][idx] for idx in idxs] 32 | else: 33 | return [[d[idx] for idx in idxs] for d in data] 34 | 35 | def tensor_shuffle(data1, data2): 36 | print(data1) 37 | idxs = np_rng.permutation(np.arange(data1.shape[0])) 38 | return data1[idxs], data2[idxs] 39 | 40 | def shuffle(*arrays, **options): 41 | if isinstance(arrays[0][0], str): 42 | return list_shuffle(*arrays) 43 | else: 44 | return skutils.shuffle(*arrays, random_state=np_rng) 45 | 46 | def OneHot(X, n=None, negative_class=0.): 47 | X = np.asarray(X).flatten() 48 | if n is None: 49 | n = np.max(X) + 1 50 | Xoh = np.ones((len(X), n)) * negative_class 51 | Xoh[np.arange(len(X)), X] = 1. 52 | return Xoh 53 | 54 | def iter_data(*data, **kwargs): 55 | size = kwargs.get('size', 128) 56 | try: 57 | n = len(data[0]) 58 | except: 59 | n = data[0].shape[0] 60 | batches = int(n / size) 61 | if n % size != 0: 62 | batches += 1 63 | 64 | for b in range(batches): 65 | start = b * size 66 | end = (b + 1) * size 67 | if end > n: 68 | end = n 69 | if len(data) == 1: 70 | yield data[0][start:end] 71 | else: 72 | yield tuple([d[start:end] for d in data]) 73 | -------------------------------------------------------------------------------- /toy_experiments/utils/rng.py: -------------------------------------------------------------------------------- 1 | from numpy.random import RandomState 2 | from random import Random 3 | 4 | seed = 2 5 | 6 | py_rng = Random(seed) 7 | np_rng = RandomState(seed) 8 | 9 | def set_seed(n): 10 | global seed, py_rng, np_rng 11 | 12 | seed = n 13 | py_rng = Random(seed) 14 | np_rng = RandomState(seed) 15 | -------------------------------------------------------------------------------- /toy_experiments/utils/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import OrderedDict 3 | 4 | 5 | 6 | def zipp(params, tparams): 7 | """ 8 | When we reload the model. Needed for the GPU stuff. 9 | """ 10 | for kk, vv in params.iteritems(): 11 | tparams[kk].set_value(vv) 12 | 13 | def unzip(zipped): 14 | """ 15 | When we pickle the model. Needed for the GPU stuff. 16 | """ 17 | new_params = OrderedDict() 18 | for kk, vv in zipped.iteritems(): 19 | new_params[kk] = vv.get_value() 20 | return new_params 21 | 22 | 23 | def get_minibatches_idx(n, minibatch_size, shuffle=True): 24 | idx_list = np.arange(n, dtype="int32") 25 | 26 | if shuffle: 27 | np.random.shuffle(idx_list) 28 | 29 | minibatches = [] 30 | minibatch_start = 0 31 | for i in range(n // minibatch_size): 32 | minibatches.append(idx_list[minibatch_start: 33 | minibatch_start + minibatch_size]) 34 | minibatch_start += minibatch_size 35 | 36 | if (minibatch_start != n): 37 | # Make a minibatch out of what is left 38 | minibatches.append(idx_list[minibatch_start:]) 39 | 40 | return zip(range(len(minibatches)), minibatches) 41 | 42 | def _p(pp, name): 43 | return '%s_%s' % (pp, name) 44 | 45 | def dropout(X, trng, p=0.): 46 | if p != 0: 47 | retain_prob = 1 - p 48 | X = X / retain_prob * trng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX) 49 | return X 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /utils/adapt_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def batch_fill(testx, batch_size): 4 | """ Quick and dirty hack for filling smaller batch 5 | 6 | :param testx: 7 | :param batch_size: 8 | :return: 9 | """ 10 | nr_batches_test = int(testx.shape[0] / batch_size) 11 | ran_from = nr_batches_test * batch_size 12 | ran_to = (nr_batches_test + 1) * batch_size 13 | size = testx[ran_from:ran_to].shape[0] 14 | new_shape = [batch_size - size]+list(testx.shape[1:]) 15 | fill = np.ones(new_shape) 16 | return np.concatenate([testx[ran_from:ran_to], fill], axis=0), size 17 | 18 | def adapt_labels_outlier_task(true_labels, label): 19 | """Adapt labels to anomaly detection context 20 | 21 | Args : 22 | true_labels (list): list of ints 23 | label (int): label which is considered inlier 24 | Returns : 25 | true_labels (list): list of labels, 1 for anomalous and 0 for normal 26 | """ 27 | if label == 1: 28 | (true_labels[true_labels == label], true_labels[true_labels != label]) = (1, 0) 29 | true_labels = [1] * true_labels.shape[0] - true_labels 30 | else: 31 | (true_labels[true_labels != label], true_labels[true_labels == label]) = (1, 0) 32 | return true_labels 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /utils/constants.py: -------------------------------------------------------------------------------- 1 | IMAGES_DATASETS = ['cifar10', 'svhn'] -------------------------------------------------------------------------------- /utils/evaluations.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import numpy as np 4 | import matplotlib 5 | # Force matplotlib to not use any Xwindows backend. 6 | matplotlib.use('Agg') 7 | import matplotlib.pyplot as plt 8 | from matplotlib.offsetbox import OffsetImage, AnnotationBbox 9 | from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas 10 | from sklearn import manifold 11 | from sklearn.metrics import roc_curve, auc, precision_recall_fscore_support 12 | import matplotlib.gridspec as gridspec 13 | import seaborn as sns 14 | import pandas as pd 15 | import time 16 | import matplotlib.cm as cm 17 | from utils.constants import IMAGES_DATASETS 18 | # import cv2 19 | sns.set(color_codes=True) 20 | 21 | 22 | def do_roc(scores, true_labels, file_name='', directory='', plot=True): 23 | """ Does the ROC curve 24 | 25 | Args: 26 | scores (list): list of scores from the decision function 27 | true_labels (list): list of labels associated to the scores 28 | file_name (str): name of the ROC curve 29 | directory (str): directory to save the jpg file 30 | plot (bool): plots the ROC curve or not 31 | Returns: 32 | roc_auc (float): area under the under the ROC curve 33 | thresholds (list): list of thresholds for the ROC 34 | """ 35 | fpr, tpr, _ = roc_curve(true_labels, scores) 36 | roc_auc = auc(fpr, tpr) # compute area under the curve 37 | if plot: 38 | plt.figure() 39 | plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % (roc_auc)) 40 | plt.plot([0, 1], [0, 1], 'k--') 41 | plt.xlim([0.0, 1.0]) 42 | plt.ylim([0.0, 1.05]) 43 | plt.xlabel('False Positive Rate') 44 | plt.ylabel('True Positive Rate') 45 | plt.title('Receiver operating characteristic') 46 | plt.legend(loc="lower right") 47 | 48 | plt.savefig(directory + file_name + 'roc.png') 49 | plt.close() 50 | 51 | return roc_auc 52 | 53 | 54 | def do_cumdist(scores, file_name='', directory='', plot=True): 55 | N = len(scores) 56 | X2 = np.sort(scores) 57 | F2 = np.array(range(N))/float(N) 58 | if plot: 59 | plt.figure() 60 | plt.xlabel("Anomaly score") 61 | plt.ylabel("Percentage") 62 | plt.title("Cumulative distribution function of the anomaly score") 63 | plt.plot(X2, F2) 64 | if not os.path.exists(directory): 65 | os.makedirs(directory) 66 | plt.savefig(directory + file_name + 'cum_dist.png') 67 | 68 | def get_percentile(scores, dataset): 69 | if dataset == 'kdd': 70 | # Highest 20% are anomalous 71 | per = np.percentile(scores, 80) 72 | elif dataset == "arrhythmia": 73 | # Highest 15% are anomalous 74 | per = np.percentile(scores, 85) 75 | else: 76 | c = 90 77 | per = np.percentile(scores, 100 - c) 78 | return per 79 | 80 | 81 | 82 | def do_hist(scores, true_labels, directory, dataset, random_seed, display=False): 83 | plt.figure() 84 | idx_inliers = (true_labels == 0) 85 | idx_outliers = (true_labels == 1) 86 | hrange = (min(scores), max(scores)) 87 | plt.hist(scores[idx_inliers], 50, facecolor=(0, 1, 0, 0.5), 88 | label="Normal samples", density=True, range=hrange) 89 | plt.hist(scores[idx_outliers], 50, facecolor=(1, 0, 0, 0.5), 90 | label="Anomalous samples", density=True, range=hrange) 91 | plt.title("Distribution of the anomaly score") 92 | plt.legend() 93 | if display: 94 | plt.show() 95 | else: 96 | plt.savefig(directory + 'histogram_{}_{}.png'.format(random_seed, dataset), 97 | transparent=True, bbox_inches='tight') 98 | plt.close() 99 | 100 | 101 | def do_hists(scores, true_labels, directory, dataset, random_seed, display=False): 102 | plt.figure() 103 | n_samples = len(scores) 104 | n_labels = np.max(true_labels) 105 | hrange = (min(scores), max(scores)) 106 | for l in range(n_labels): 107 | idx = (true_labels == l) 108 | plt.hist(scores[idx_inliers], 50, facecolor=(0, 1, 0, 0.5), 109 | label="{}".format(l), density=True, range=hrange) 110 | plt.title("Distribution of the anomaly score") 111 | plt.legend() 112 | if display: 113 | plt.show() 114 | else: 115 | plt.savefig(directory + 'hists_{}_{}.png'.format(random_seed, dataset), 116 | transparent=True, bbox_inches='tight') 117 | plt.close() 118 | 119 | def predict(scores, threshold): 120 | return scores>=threshold 121 | 122 | def make_meshgrid(x_min,x_max,y_min,y_max, h=.02): 123 | """Create a mesh of points to plot in 124 | 125 | Parameters 126 | ---------- 127 | x: data to base x-axis meshgrid on 128 | y: data to base y-axis meshgrid on 129 | h: stepsize for meshgrid, optional 130 | 131 | Returns 132 | ------- 133 | xx, yy : ndarray 134 | """ 135 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), 136 | np.arange(y_min, y_max, h)) 137 | return xx, yy 138 | 139 | def save_grid_plot(samples, samples_rec, name_model, dataset, nb_images=50, 140 | grid_width=10): 141 | 142 | args = name_model.split('/')[:-1] 143 | directory = os.path.join(*args) 144 | if not os.path.exists(directory): 145 | os.makedirs(directory) 146 | samples = (samples + 1) / 2 147 | samples_rec = (samples_rec + 1) / 2 148 | 149 | figsize = (32, 32) 150 | plt.figure(figsize=figsize) 151 | gs = gridspec.GridSpec(grid_width, grid_width) 152 | gs.update(wspace=0.05, hspace=0.05) 153 | list_samples = [] 154 | for x, x_rec in zip(np.split(samples, nb_images // grid_width), 155 | np.split(samples_rec, nb_images // grid_width)): 156 | list_samples += np.split(x, grid_width) + np.split(x_rec, grid_width) 157 | list_samples = [np.squeeze(sample) for sample in list_samples] 158 | for i, sample in enumerate(list_samples): 159 | if i>=nb_images*2: 160 | break 161 | ax = plt.subplot(gs[i]) 162 | if dataset == 'mnist': 163 | plt.imshow(sample, cmap=cm.gray) 164 | else: 165 | plt.imshow(sample) 166 | plt.axis('off') 167 | ax.set_xticklabels([]) 168 | ax.set_yticklabels([]) 169 | ax.set_aspect('equal') 170 | plt.savefig('{}.png'.format(name_model)) 171 | 172 | 173 | def save_results(scores, true_labels, model, dataset, method, weight, label, 174 | random_seed, step=-1): 175 | 176 | directory = 'results/{}/{}/{}/w{}/'.format(model, 177 | dataset, 178 | method, 179 | weight) 180 | if not os.path.exists(directory): 181 | os.makedirs(directory) 182 | 183 | if dataset in IMAGES_DATASETS: 184 | file_name = "{}_step{}_rd{}".format(label, step, random_seed) 185 | fname = directory + "{}.csv".format(label) 186 | else: 187 | file_name = "{}_step{}_rd{}".format(dataset, step, random_seed) 188 | fname = directory + "results.csv" 189 | 190 | 191 | scores = np.array(scores) 192 | 193 | roc_auc = do_roc(scores, true_labels, file_name=file_name, 194 | directory=directory) 195 | 196 | do_cumdist(scores, file_name=file_name, directory=directory) 197 | 198 | do_hist(scores, true_labels, directory, dataset, random_seed) 199 | if np.max(true_labels)>1: 200 | do_hists(scores, true_labels, directory, dataset, random_seed) 201 | 202 | per = get_percentile(scores, dataset) 203 | y_pred = (scores>=per) 204 | 205 | precision, recall, f1, _ = precision_recall_fscore_support(true_labels.astype(int), 206 | y_pred.astype(int), 207 | average='binary') 208 | if dataset in IMAGES_DATASETS: 209 | print("Testing at step %i, method %s: AUROC = %.4f" 210 | % (step, method, roc_auc)) 211 | else: 212 | print("Testing at step %i, method %s: Prec = %.4f | Rec = %.4f | F1 = %.4f" 213 | % (step, method, precision, recall, f1)) 214 | 215 | results = [model, dataset, method, weight, label, 216 | step, roc_auc, precision, recall, f1, random_seed, time.ctime()] 217 | save_results_csv("results/results.csv", results, header=0) 218 | 219 | results = [step, roc_auc, precision, recall, f1, random_seed] 220 | save_results_csv(fname, results, header=2) 221 | 222 | def heatmap(data, name=None, save=False): 223 | 224 | fig = plt.figure() 225 | ax = sns.heatmap(data, cmap="YlGnBu") 226 | fig.add_subplot(ax) 227 | fig.canvas.draw() 228 | data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') 229 | data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) 230 | if save: 231 | args = name.split('/')[:-1] 232 | directory = os.path.join(*args) 233 | if not os.path.exists(directory): 234 | os.makedirs(directory) 235 | plt.savefig('{}.png'.format(name)) 236 | return data 237 | 238 | 239 | def save_results_csv(fname, results, header=0): 240 | """Saves results in csv file 241 | Args: 242 | fname (str): name of the file 243 | results (list): list of prec, rec, F1, rds 244 | """ 245 | 246 | new_rows = [] 247 | if not os.path.isfile(fname): 248 | args = fname.split('/')[:-1] 249 | directory = os.path.join(*args) 250 | if not os.path.exists(directory): 251 | os.makedirs(directory) 252 | 253 | with open(fname, 'wt') as f: 254 | writer = csv.writer(f) 255 | if header == 0: 256 | writer.writerows( 257 | [['Model', 'Dataset', 'Method', 'Weight', 'Label', 258 | 'Step', 'AUROC', 'Precision', 'Recall', 259 | 'F1 score', 'Random Seed', 'Date']]) 260 | if header == 1: 261 | writer.writerows( 262 | [['Precision', 'Recall', 'F1 score', 'Random Seed']]) 263 | elif header ==2: 264 | writer.writerows( 265 | [['Step', 'AUROC', 'Precision', 'Recall', 266 | 'F1 score', 'Random Seed']]) 267 | 268 | elif header == 5: 269 | writer.writerows( 270 | [['Model', 'Dataset', 'Method', 'Weight', 'Label', 271 | 'Step', 'Scores']]) 272 | 273 | with open(fname, 'at') as f: 274 | # Overwrite the old file with the modified rows 275 | writer = csv.writer(f) 276 | new_rows.append(results) # add the modified rows 277 | writer.writerows(new_rows) -------------------------------------------------------------------------------- /utils/plot_reconstructions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from utils.evaluations import save_results_csv 3 | import time 4 | import os 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | def plot_hist_dis_reconstructions(): 9 | walk_dir = 'scores' 10 | label = [] 11 | for root, _, files in os.walk(walk_dir): 12 | for filename in files: 13 | file_path = os.path.join(root, filename) 14 | 15 | try: 16 | array = np.vstack( 17 | [array, np.genfromtxt(file_path, delimiter=',')]) 18 | except NameError: 19 | array = np.genfromtxt(file_path, delimiter=',') 20 | 21 | label += [file_path.split('/')[-3] + '-' + file_path.split('/')[-2]] 22 | 23 | hrange = (np.min(array), np.max(array)) 24 | 25 | for i in range(array.shape[0]): 26 | plt.figure() 27 | plt.hist(array[i, :], 50, 28 | label=label[i], density=True, range=hrange) 29 | plt.title(label[i]) 30 | plt.savefig(walk_dir + '/' + label[i] + '.png') 31 | plt.close() 32 | 33 | if __name__ == "__main__": 34 | plot_hist_dis_reconstructions() -------------------------------------------------------------------------------- /utils/sn.py: -------------------------------------------------------------------------------- 1 | ### Credits https://github.com/taki0112/Spectral_Normalization-Tensorflow 2 | 3 | import tensorflow as tf 4 | 5 | def conv2d(inputs, filters, kernel_size, strides=1, padding='valid', 6 | use_bias=True, kernel_initializer=None, 7 | bias_initializer=tf.zeros_initializer(), kernel_regularizer=None, 8 | name=None,reuse=None): 9 | 10 | with tf.variable_scope(name, reuse=reuse): 11 | w = tf.get_variable("kernel", shape=[kernel_size, kernel_size, inputs.get_shape()[-1], filters], initializer=kernel_initializer, 12 | regularizer=kernel_regularizer) 13 | bias = tf.get_variable("bias", [filters], initializer=bias_initializer) 14 | x = tf.nn.conv2d(input=inputs, filter=spectral_norm(w), 15 | strides=[1, strides, strides, 1], padding=padding) 16 | if use_bias : 17 | x = tf.nn.bias_add(x, bias) 18 | 19 | return x 20 | 21 | def dense(inputs, units, use_bias=True, kernel_initializer=None, 22 | bias_initializer=tf.zeros_initializer(), kernel_regularizer=None, 23 | name=None,reuse=None): 24 | 25 | with tf.variable_scope(name, reuse=reuse): 26 | inputs = tf.contrib.layers.flatten(inputs) 27 | shape = inputs.get_shape().as_list() 28 | channels = shape[-1] 29 | 30 | w = tf.get_variable("kernel", [channels, units], tf.float32, 31 | initializer=kernel_initializer, regularizer=kernel_regularizer) 32 | if use_bias : 33 | bias = tf.get_variable("bias", [units], 34 | initializer=bias_initializer) 35 | 36 | x = tf.matmul(inputs, spectral_norm(w)) + bias 37 | else : 38 | x = tf.matmul(inputs, spectral_norm(w)) 39 | 40 | return x 41 | 42 | def spectral_norm(w, iteration=1, eps=1e-12): 43 | w_shape = w.shape.as_list() 44 | w = tf.reshape(w, [-1, w_shape[-1]]) 45 | 46 | u = tf.get_variable("u", [1, w_shape[-1]], initializer=tf.truncated_normal_initializer(), trainable=False) 47 | 48 | u_hat = u 49 | v_hat = None 50 | for i in range(iteration): 51 | """ 52 | power iteration 53 | Usually iteration = 1 will be enough 54 | """ 55 | v_ = tf.matmul(u_hat, tf.transpose(w)) 56 | v_hat = v_ / (tf.reduce_sum(v_ ** 2) ** 0.5 + eps) 57 | 58 | u_ = tf.matmul(v_hat, w) 59 | u_hat = u_ / (tf.reduce_sum(u_ ** 2) ** 0.5 + eps) 60 | 61 | sigma = tf.matmul(tf.matmul(v_hat, w), tf.transpose(u_hat)) 62 | w_norm = w / sigma 63 | 64 | with tf.control_dependencies([u.assign(u_hat)]): 65 | w_norm = tf.reshape(w_norm, w_shape) 66 | 67 | return w_norm 68 | --------------------------------------------------------------------------------