├── images ├── img.txt ├── inf_cond3-1.jpg ├── classes_camera-1.jpg ├── train_percept_1-1.jpg └── train_percept_1.pdf ├── README.md ├── Latent_Search.py └── TrainVAE.py /images/img.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /images/inf_cond3-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinshul/WBC-Classification-UDA/HEAD/images/inf_cond3-1.jpg -------------------------------------------------------------------------------- /images/classes_camera-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinshul/WBC-Classification-UDA/HEAD/images/classes_camera-1.jpg -------------------------------------------------------------------------------- /images/train_percept_1-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinshul/WBC-Classification-UDA/HEAD/images/train_percept_1-1.jpg -------------------------------------------------------------------------------- /images/train_percept_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinshul/WBC-Classification-UDA/HEAD/images/train_percept_1.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WBC-Classification-UDA 2 | Peripheral Blood Smear White Blood Cells Classification using Unsupervised Domain Adaptation (No target images used during training unlike SOTA UDA). 3 | 4 | 5 | 6 | ![](images/classes_camera-1.jpg) 7 | ![](images/train_percept_1-1.jpg) 8 | 9 | - tensorflow = 1.14.0 10 | - python = 3.6 or higher 11 | - keras = 2.2.5 12 | - Nvidia cuda = 10.1 13 | 14 | ## Usage 15 | Train a Variational Auto-Encoder using Source domain (code in the file TrainVAE.py) 16 | ``` 17 | python TrainVAE.py 18 | ``` 19 | 20 | Perform Unsupervised Domain Adaptation on the Target domain (code in the file Latent_Search.py) 21 | ``` 22 | python Latent_Search.py 23 | ``` 24 | -------------------------------------------------------------------------------- /Latent_Search.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | import os 6 | import keras 7 | from keras import backend as K 8 | from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D 9 | from keras.preprocessing import image 10 | from keras.applications.vgg19 import preprocess_input 11 | from keras.models import Model, Sequential 12 | import numpy as np 13 | import matplotlib 14 | import pandas as pd 15 | import cv2 16 | from keras.utils import to_categorical 17 | from keras.layers import Dense, GlobalAveragePooling2D 18 | import tensorflow as tf 19 | from keras.callbacks import ModelCheckpoint 20 | from cleverhans.attacks import FastGradientMethod 21 | from cleverhans.loss import CrossEntropy 22 | from cleverhans.train import train 23 | from cleverhans.utils import AccuracyReport 24 | from cleverhans.utils_keras import cnn_model 25 | from cleverhans.utils_keras import KerasModelWrapper 26 | from cleverhans.utils_tf import model_eval 27 | from cleverhans.utils_tf import model_argmax 28 | import functools 29 | import tensorflow as tf 30 | from cleverhans import initializers 31 | from cleverhans.model import Model 32 | #from cleverhans.picklable_model import MLP, Conv2D, ReLU, Flatten, Linear 33 | from cleverhans.picklable_model import Softmax 34 | import math 35 | import logging 36 | from tensorflow.python.platform import flags 37 | from cleverhans.dataset import MNIST 38 | from cleverhans.utils import AccuracyReport, set_log_level 39 | from cleverhans.augmentation import random_horizontal_flip, random_shift 40 | from cleverhans.dataset import CIFAR10 41 | from cleverhans.model_zoo.all_convolutional import ModelAllConvolutional 42 | from keras.models import load_model 43 | 44 | from pdb import set_trace as trace 45 | 46 | from shutil import copyfile 47 | import imageio 48 | 49 | import tensorflow as tf 50 | from tensorflow.contrib.layers.python.layers import batch_norm 51 | 52 | import os, csv, keras, math, logging, functools, cv2, sys 53 | #from keras.applications.vgg19 import VGG19, preprocess_input 54 | from keras.preprocessing import image 55 | from keras.models import Model, Sequential 56 | from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, GlobalAveragePooling2D, ZeroPadding2D, Convolution2D, MaxPooling2D 57 | import numpy as np 58 | import pandas as pd 59 | from keras.utils import to_categorical 60 | #from sklearn.preprocessing import OneHotEncoder 61 | import tensorflow as tf 62 | from keras.callbacks import ModelCheckpoint 63 | from cleverhans.attacks import FastGradientMethod 64 | from cleverhans.loss import CrossEntropy 65 | from cleverhans.train import train 66 | from cleverhans.utils import AccuracyReport, set_log_level 67 | from cleverhans.utils_keras import cnn_model 68 | from cleverhans.utils_keras import KerasModelWrapper 69 | from cleverhans.utils_tf import model_eval, model_argmax 70 | from cleverhans import initializers 71 | from cleverhans.model import Model 72 | from tensorflow.python.platform import flags 73 | #from cleverhans.model_zoo.all_convolutional import ModelAllConvolutional 74 | #from vgg import VGG16 75 | #from vgg19 import VGG19 76 | from keras.datasets import cifar10 77 | #from sklearn.utils import class_weight 78 | import ssl 79 | ssl._create_default_https_context = ssl._create_unverified_context 80 | 81 | 82 | 83 | 84 | classes = ['nrbc', 'notawbc', 'giant platelet', 'platelet clump', 'basophil', 85 | 'neutrophil', 'eosinophil', 'lymphocyte', 'monocyte', 'ig', 'atypical-blast'] 86 | #os.environ["CUDA_VISIBLE_DEVICES"]="0" 87 | #os.environ["TF_CPP_MIN_LOG_LEVEL"]="2" 88 | def toOneHot(a): 89 | b = np.zeros((a.shape[0], 11)) 90 | for i in range(a.shape[0]): 91 | for j in range(11): 92 | if a[i] == classes[j]: 93 | b[i][j] = 1 94 | return b 95 | 96 | def del_all_flags(FLAGS): 97 | flags_dict = FLAGS._flags() 98 | keys_list = [keys for keys in flags_dict] 99 | for keys in keys_list: 100 | FLAGS.__delattr__(keys) 101 | 102 | 103 | 104 | 105 | 106 | def lrelu(x , alpha = 0.2 , name="LeakyReLU"): 107 | return tf.maximum(x , alpha*x) 108 | 109 | def conv2d(input_, output_dim, 110 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 111 | name="conv2d"): 112 | 113 | with tf.variable_scope(name): 114 | 115 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 116 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 117 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 118 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 119 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 120 | 121 | return conv 122 | 123 | def de_conv(input_, output_shape, 124 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 125 | name="deconv2d", with_w=False): 126 | 127 | with tf.variable_scope(name): 128 | # filter : [height, width, output_channels, in_channels] 129 | w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], 130 | initializer=tf.random_normal_initializer(stddev=stddev)) 131 | 132 | try: 133 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, 134 | strides=[1, d_h, d_w, 1]) 135 | 136 | # Support for verisons of TensorFlow before 0.7.0 137 | except AttributeError: 138 | 139 | deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape, 140 | strides=[1, d_h, d_w, 1]) 141 | 142 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 143 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 144 | 145 | if with_w: 146 | 147 | return deconv, w, biases 148 | 149 | else: 150 | 151 | return deconv 152 | 153 | def fully_connect(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False): 154 | shape = input_.get_shape().as_list() 155 | with tf.variable_scope(scope or "Linear"): 156 | 157 | matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, 158 | tf.random_normal_initializer(stddev=stddev)) 159 | bias = tf.get_variable("bias", [output_size], 160 | initializer=tf.constant_initializer(bias_start)) 161 | 162 | if with_w: 163 | return tf.matmul(input_, matrix) + bias, matrix, bias 164 | else: 165 | 166 | return tf.matmul(input_, matrix) + bias 167 | 168 | def conv_cond_concat(x, y): 169 | """Concatenate conditioning vector on feature map axis.""" 170 | x_shapes = x.get_shape() 171 | y_shapes = y.get_shape() 172 | 173 | return tf.concat(3 , [x , y*tf.ones([x_shapes[0], x_shapes[1], x_shapes[2] , y_shapes[3]])]) 174 | 175 | def batch_normal(input , scope="scope" , reuse=False): 176 | return batch_norm(input , epsilon=1e-5, decay=0.9 , scale=True, scope=scope , reuse=reuse , updates_collections=None) 177 | 178 | def instance_norm(x): 179 | 180 | epsilon = 1e-9 181 | mean, var = tf.nn.moments(x, [1, 2], keep_dims=True) 182 | return tf.div(tf.subtract(x, mean), tf.sqrt(tf.add(var, epsilon))) 183 | 184 | def residual(x, output_dims, kernel, strides, name_1, name_2): 185 | 186 | with tf.variable_scope('residual') as scope: 187 | 188 | conv1 = conv2d(x, output_dims, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_1) 189 | conv2 = conv2d(tf.nn.relu(conv1), output_dims, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_2) 190 | resi = x + conv2 191 | 192 | return resi 193 | 194 | def deresidual(x, output_shape, kernel, strides, name_1, name_2): 195 | 196 | with tf.variable_scope('residual_un') as scope: 197 | 198 | deconv1 = de_conv(x, output_shape=output_shape, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_1) 199 | deconv2 = de_conv(tf.nn.relu(deconv1), output_shape=output_shape, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_2) 200 | resi = x + deconv2 201 | 202 | return resi 203 | import os 204 | import errno 205 | import numpy as np 206 | import scipy 207 | import scipy.misc 208 | from keras.models import Model 209 | 210 | 211 | def mkdir_p(path): 212 | try: 213 | os.makedirs(path) 214 | except OSError as exc: # Python >2.5 215 | if exc.errno == errno.EEXIST and os.path.isdir(path): 216 | pass 217 | else: 218 | raise 219 | 220 | def get_image(image_path, image_size, is_crop=True, resize_w=64, is_grayscale=False): 221 | return transform(imread(image_path, is_grayscale), image_size, is_crop, resize_w) 222 | 223 | 224 | def transform(image, npx=64, is_crop=False, resize_w=64): 225 | # npx : # of pixels width/height of image 226 | if is_crop: 227 | cropped_image = center_crop(image, npx, resize_w=resize_w) 228 | else: 229 | cropped_image = image 230 | cropped_image = scipy.misc.imresize(cropped_image, 231 | [resize_w, resize_w]) 232 | return np.array(cropped_image) / 127.5 - 1 233 | 234 | def center_crop(x, crop_h , crop_w=None, resize_w=64): 235 | 236 | if crop_w is None: 237 | crop_w = crop_h 238 | h, w = x.shape[:2] 239 | j = int(round((h - crop_h)/2.)) 240 | i = int(round((w - crop_w)/2.)) 241 | return scipy.misc.imresize(x[j:j+crop_h, i:i+crop_w], 242 | [resize_w, resize_w]) 243 | 244 | 245 | def save_images(images, size, image_path): 246 | return imsave(inverse_transform(images), size, image_path) 247 | 248 | def imread(path, is_grayscale=False): 249 | if (is_grayscale): 250 | return scipy.misc.imread(path, flatten=True).astype(np.float) 251 | else: 252 | return scipy.misc.imread(path).astype(np.float) 253 | 254 | 255 | def imsave(images, size, path): 256 | return scipy.misc.imsave(path, merge(images, size)) 257 | 258 | def merge(images, size): 259 | h, w = images.shape[1], images.shape[2] 260 | size1 = np.int(h * size[0]) 261 | size2 = np.int(w * size[1]) 262 | img = np.zeros((size1,size2, 3)) 263 | for idx, image in enumerate(images): 264 | i = idx % size[1] 265 | j = idx // size[1] 266 | img[np.int(j * h):np.int(j * h + h), np.int(i * w): np.int(i * w + w), :] = image 267 | 268 | return img 269 | 270 | 271 | 272 | def inverse_transform(image): 273 | return ((image + 1) * 127.5).astype(np.uint8) 274 | 275 | 276 | 277 | import tensorflow as tf 278 | 279 | from cleverhans import initializers 280 | from cleverhans.serial import NoRefModel 281 | 282 | 283 | 284 | 285 | 286 | import numpy as np 287 | import scipy 288 | from tensorflow.python.framework.ops import convert_to_tensor 289 | import os 290 | TINY = 1e-8 291 | d_scale_factor = 0.25 292 | g_scale_factor = 1 - 0.75/2 293 | import csv 294 | import PIL 295 | 296 | def getAcc(pred, next_y_images): 297 | acc = np.zeros([11]) 298 | Tc = np.ones([11]) 299 | for i in range(len(pred)): 300 | Tc[np.argmax(next_y_images[i])] = Tc[np.argmax(next_y_images[i])] + 1 301 | if (np.argmax(next_y_images[i]) == np.argmax(pred[i])): 302 | acc[np.argmax(next_y_images[i])] = acc[np.argmax(next_y_images[i])] + 1 303 | print(100*np.sum(acc)/(np.sum(Tc)-11)) 304 | return 100*acc/Tc 305 | class ModelAllConvolutional(NoRefModel): 306 | """ 307 | A simple model that uses only convolution and downsampling---no batch norm or other techniques that can complicate 308 | adversarial training. 309 | """ 310 | def __init__(self, scope, nb_classes, nb_filters, input_shape, **kwargs): 311 | del kwargs 312 | NoRefModel.__init__(self, scope, nb_classes, locals()) 313 | self.nb_filters = nb_filters 314 | self.input_shape = input_shape 315 | 316 | # Do a dummy run of fprop to create the variables from the start 317 | self.fprop(tf.placeholder(tf.float32, [32] + input_shape)) 318 | # Put a reference to the params in self so that the params get pickled 319 | self.params = self.get_params() 320 | 321 | def fprop(self, x, **kwargs): 322 | del kwargs 323 | conv_args = dict( 324 | activation=tf.nn.leaky_relu, 325 | kernel_initializer=initializers.HeReLuNormalInitializer, 326 | kernel_size=3, 327 | padding='same') 328 | y = x 329 | 330 | with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): 331 | log_resolution = int(round( 332 | math.log(self.input_shape[0]) / math.log(2))) 333 | for scale in range(log_resolution - 2): 334 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 335 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 336 | y = tf.layers.average_pooling2d(y, 2, 2) 337 | y = tf.layers.conv2d(y, self.nb_classes, **conv_args) 338 | logits = tf.reduce_mean(y, [1, 2]) 339 | return {self.O_LOGITS: logits, 340 | self.O_PROBS: tf.nn.softmax(logits=logits)} 341 | 342 | 343 | class ModelAllConvolutional1(NoRefModel): 344 | """ 345 | A simple model that uses only convolution and downsampling---no batch norm or other techniques that can complicate 346 | adversarial training. 347 | """ 348 | def __init__(self, scope, nb_classes, nb_filters, input_shape, **kwargs): 349 | del kwargs 350 | NoRefModel.__init__(self, scope, nb_classes, locals()) 351 | self.nb_filters = nb_filters 352 | self.input_shape = input_shape 353 | 354 | # Do a dummy run of fprop to create the variables from the start 355 | self.fprop(tf.placeholder(tf.float32, [32] + input_shape)) 356 | # Put a reference to the params in self so that the params get pickled 357 | self.params = self.get_params() 358 | 359 | def fprop(self, x, **kwargs): 360 | del kwargs 361 | conv_args = dict( 362 | activation=tf.nn.leaky_relu, 363 | kernel_initializer=initializers.HeReLuNormalInitializer, 364 | kernel_size=3, 365 | padding='same') 366 | y = x 367 | 368 | with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): 369 | log_resolution = int(round( 370 | math.log(self.input_shape[0]) / math.log(2))) 371 | for scale in range(log_resolution - 4): 372 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 373 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 374 | y = tf.layers.average_pooling2d(y, 2, 2) 375 | conv = y 376 | scale = log_resolution - 4 377 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 378 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 379 | y = tf.layers.average_pooling2d(y, 2, 2) 380 | 381 | 382 | scale = log_resolution - 3 383 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 384 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 385 | y = tf.layers.average_pooling2d(y, 2, 2) 386 | y = tf.layers.conv2d(y, self.nb_classes, **conv_args) 387 | 388 | logits = tf.reduce_mean(y, [1, 2]) 389 | return {self.O_LOGITS: conv, 390 | self.O_PROBS: tf.nn.softmax(logits=logits)} 391 | 392 | 393 | class vaegan(object): 394 | 395 | #build model 396 | def __init__(self, batch_size, max_iters, repeat, model_path, latent_dim, sample_path, log_dir, learnrate_init): 397 | 398 | self.batch_size = batch_size 399 | self.max_iters = max_iters 400 | self.repeat_num = repeat 401 | self.saved_model_path = model_path 402 | 403 | self.latent_dim = latent_dim 404 | self.sample_path = sample_path 405 | self.log_dir = log_dir 406 | self.learn_rate_init = learnrate_init 407 | 408 | self.log_vars = [] 409 | 410 | self.channel = 3 411 | self.output_size = 128 412 | 413 | self.x_input = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, 3]) 414 | self.x_true = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, self.channel]) 415 | 416 | 417 | 418 | self.labels = tf.placeholder(tf.float32, [self.batch_size, 11]) 419 | 420 | 421 | self.ep1 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 422 | self.zp1 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 423 | 424 | self.ep2 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 425 | self.zp2 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 426 | self.keep_prob = tf.placeholder_with_default(1.0, shape=()) 427 | self.z_custom = tf.placeholder(tf.float32, [self.batch_size, self.latent_dim]) 428 | 429 | print('Data Loading Begins') 430 | 431 | y_train=[] 432 | x_train1=[] 433 | for dirs in os.listdir('/home/manu_kohli/wbc/cam3/trainset/'): 434 | for files in os.listdir('/home/manu_kohli/wbc/cam3/trainset/'+dirs): 435 | y_train.append(int(dirs)) 436 | x_train1.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam3/trainset/'+dirs+'/'+files))) 437 | 438 | #x_train1 =np.asarray(x_train1)/255.0 439 | 440 | cam3_train_data=[] 441 | cam3_train_label=[] 442 | 443 | l=list(range(0,len(y_train))) 444 | l=np.asarray(l) 445 | np.random.shuffle(l) 446 | for i in l: 447 | cam3_train_data.append(x_train1[i]) 448 | cam3_train_label.append(y_train[i]) 449 | 450 | x_train1=cam3_train_data 451 | y_train=cam3_train_label 452 | 453 | x_train1 = np.asarray(x_train1)/127.5 454 | x_train1 =x_train1 - 1. 455 | y_train = np.asarray(y_train) 456 | #y_train = toOneHot(y_train) 457 | y_train= to_categorical(y_train, num_classes=11) 458 | # x_train1 = np.load( '/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/X_Train.npy').astype('float32') 459 | # y_train = np.load( '/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/Y_Train.npy') 460 | # x_train1_1 = np.load('/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/X_Test.npy').astype('float32') 461 | # y_train_1 = np.load('/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/Y_Test.npy') 462 | 463 | # x_train1_2 = np.load( '/home/vinay/projects/Sigtuple/CameraInvariance/Cam3Classifier/Data_Augmentation/X_Train_extra.npy').astype('float32') 464 | # y_train_2 = np.load( '/home/vinay/projects/Sigtuple/CameraInvariance/Cam3Classifier/Data_Augmentation/Y_Train_extra.npy') 465 | 466 | # x_train1 = np.append(x_train1, x_train1_2,axis =0) 467 | # y_train = np.append(y_train, y_train_2,axis =0) 468 | 469 | 470 | # x_train1 = np.concatenate((x_train1, x_train1_1), axis=0) 471 | # y_train = np.concatenate((y_train, y_train_1), axis=0) 472 | 473 | 474 | x_test1_cam3 = [] 475 | y_test_cam3 = [] 476 | 477 | for dirs in os.listdir('/home/manu_kohli/wbc/cam3/testset/'): 478 | for files in os.listdir('/home/manu_kohli/wbc/cam3/testset/'+dirs): 479 | y_test_cam3.append(int(dirs)) 480 | x_test1_cam3.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam3/testset/'+dirs+'/'+files))) 481 | 482 | cam3_test_data=[] 483 | cam3_test_label=[] 484 | 485 | l=list(range(0,len(y_test_cam3))) 486 | l=np.asarray(l) 487 | np.random.shuffle(l) 488 | for i in l: 489 | cam3_test_data.append(x_test1_cam3[i]) 490 | cam3_test_label.append(y_test_cam3[i]) 491 | 492 | x_test1_cam3 = cam3_test_data 493 | y_test_cam3 = cam3_test_label 494 | 495 | y_test_cam3= to_categorical(y_test_cam3, num_classes=11) 496 | #y_test_cam3 = toOneHot(np.asarray(y_test_cam3)) 497 | #x_test1_cam3=np.asarray(x_test1_cam3)/255.0 498 | x_test1_cam3 = np.asarray(x_test1_cam3)/127.5 499 | x_test1_cam3 =x_test1_cam3 - 1. 500 | 501 | x_test1=[] 502 | y_test =[] 503 | 504 | for dirs in os.listdir('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'): 505 | for files in os.listdir('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'+dirs): 506 | y_test.append(int(dirs)) 507 | x_test1.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'+dirs+'/'+files))) 508 | 509 | # x_test1 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam2_images.npy').astype('float32')/255 510 | # y_test = np.load('/home/vinay/projects/Sigtuple/CreateData/cam2_labels.npy') 511 | 512 | cam2_data=[] 513 | cam2_label=[] 514 | 515 | l=list(range(0,len(y_test))) 516 | l=np.asarray(l) 517 | np.random.shuffle(l) 518 | for i in l: 519 | cam2_data.append(x_test1[i]) 520 | cam2_label.append(y_test[i]) 521 | 522 | x_test1 = cam2_data 523 | y_test = cam2_label 524 | 525 | y_test= to_categorical(y_test, num_classes=11) 526 | #y_test = toOneHot(np.asarray(y_test)) 527 | # x_test1=np.asarray(x_test1)/255.0 528 | x_test1 = np.asarray(x_test1)/127.5 529 | x_test1 =x_test1 - 1. 530 | 531 | # x_test1_cam3 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam3_images.npy').astype('float32')/255 532 | # y_test_cam3 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam3_labels.npy') 533 | # y_test_cam3 = toOneHot(y_test_cam3) 534 | 535 | #print(x_train1.shape, y_train.shape) 536 | #print(x_test1.shape, y_test.shape) 537 | #x_train = np.zeros([x_train1.shape[0], self.output_size,self.output_size,self.channel]) 538 | #x_test = np.zeros([x_test1.shape[0], self.output_size,self.output_size,self.channel]) 539 | #x_test_cam3 = np.zeros([x_test1_cam3.shape[0], self.output_size,self.output_size,self.channel]) 540 | 541 | # x_train[:,:,:,0] = x_train1[:,:,:,2] 542 | # x_train[:,:,:,1] = x_train1[:,:,:,1] 543 | # x_train[:,:,:,2] = x_train1[:,:,:,0] 544 | 545 | # x_test[:,:,:,0] = x_test1[:,:,:,2] 546 | # x_test[:,:,:,1] = x_test1[:,:,:,1] 547 | # x_test[:,:,:,2] = x_test1[:,:,:,0] 548 | 549 | # x_test_cam3[:,:,:,0] = x_test1_cam3[:,:,:,2] 550 | # x_test_cam3[:,:,:,1] = x_test1_cam3[:,:,:,1] 551 | # x_test_cam3[:,:,:,2] = x_test1_cam3[:,:,:,0] 552 | 553 | x_train = np.float32(x_train1).reshape([-1,self.output_size,self.output_size,self.channel]) 554 | x_test = np.float32(x_test1).reshape([-1,self.output_size,self.output_size,self.channel]) 555 | x_test_cam3 = np.float32(x_test1_cam3).reshape([-1,self.output_size,self.output_size,self.channel]) 556 | 557 | print(x_train.shape, y_train.shape) 558 | print(x_test.shape, y_test.shape) 559 | print(x_test_cam3.shape, y_test_cam3.shape) 560 | print(np.amin(x_train), np.amin( x_test ), np.amin(x_test_cam3)) 561 | print(np.amax(x_train), np.amax( x_test ), np.amax(x_test_cam3)) 562 | 563 | 564 | TrainDataSize = x_train.shape[0] 565 | TestDataSize = x_test.shape[0] 566 | self.TrainDataSize = TrainDataSize 567 | self.TestDataSize = TestDataSize 568 | self.TestDataSize_cam3 = x_test_cam3.shape[0] 569 | 570 | 571 | self.X_Real_Test = x_test 572 | self.X_Real_Train = x_train 573 | self.X_Real_Test_cam3 = x_test_cam3 574 | self.Y_train = y_train 575 | self.Y_test = y_test 576 | self.Y_test_cam3 = y_test_cam3 577 | 578 | 579 | # self.X_Real_Train = self.X_Real_Train*2 - 1 580 | # self.X_Real_Test = self.X_Real_Test*2 - 1 581 | # self.X_Real_Test_cam3 = self.X_Real_Test_cam3*2 - 1 582 | 583 | print('Max', np.max(self.X_Real_Train)) 584 | print('Min', np.min(self.X_Real_Train)) 585 | 586 | print('Data Loading Completed') 587 | 588 | 589 | 590 | def build_model_vaegan(self): 591 | 592 | self.z1_mean, self.z1_sigm = self.Encode1(self.x_input) 593 | self.z1_x = tf.add( self.z1_mean, tf.sqrt(tf.exp(self.z1_sigm))*self.ep1) 594 | self.x_input_sobel = tf.image.sobel_edges(self.x_input) 595 | self.x_input_sobel = tf.reshape(self.x_input_sobel, [64,128,128,6]) 596 | self.x_out = self.generate1(self.x_input_sobel, self.z1_x, reuse=False) 597 | 598 | self.x_filt2 = self.generate1(self.x_input_sobel, self.z1_mean, reuse=True) 599 | 600 | self.model_classifier_logits = ModelAllConvolutional('model1', 11, 64, input_shape=[self.output_size,self.output_size,self.channel]) 601 | self.model_classifier_percept = ModelAllConvolutional1('model2', 11, 64, input_shape=[self.output_size,self.output_size,self.channel]) 602 | self.logits_x_true = self.model_classifier_logits.get_logits((self.x_true+1)*0.5) 603 | self.percept_x_true = self.model_classifier_percept.get_logits((self.x_true+1)*0.5) 604 | #self.pred_x_true = tf.nn.softmax(self.logits_x_true) 605 | self.pred_x_true = self.model_classifier_percept.get_probs((self.x_true+1)*0.5) 606 | 607 | 608 | self.logits_x_out = self.model_classifier_logits.get_logits((self.x_out+1)*0.5) 609 | self.percept_x_out = self.model_classifier_percept.get_logits((self.x_out+1)*0.5) 610 | self.pred_x_out = tf.nn.softmax(self.logits_x_out) 611 | 612 | 613 | self.logits_x_filt2 = self.model_classifier_logits.get_logits((self.x_filt2+1)*0.5) 614 | self.pred_x_filt2 = tf.nn.softmax(self.logits_x_filt2) 615 | 616 | 617 | 618 | self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels)) 619 | self.cl_loss_x_out = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_out , labels = self.labels)) 620 | self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels)) 621 | 622 | 623 | 624 | self.kl1_loss = self.KL_loss(self.z1_mean, self.z1_sigm)/(self.latent_dim*self.batch_size) 625 | 626 | 627 | self.Loss_vae1_pixel = tf.reduce_mean(tf.square(tf.subtract(self.x_out, self.x_true))) + tf.reduce_mean(tf.abs(tf.subtract(self.x_out, self.x_true))) 628 | self.Loss_vae1_percept = tf.reduce_mean(tf.square(tf.subtract(self.percept_x_out, self.percept_x_true))) 629 | self.Loss_vae1_logits = tf.reduce_mean(tf.square(tf.subtract(self.logits_x_out, self.logits_x_true))) 630 | 631 | 632 | 633 | #For encode 634 | self.encode1_loss = 1*self.kl1_loss + 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept 635 | 636 | #for Gen 637 | self.G1_loss = 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept 638 | 639 | 640 | self.x_custom = self.generate1(self.x_input_sobel,self.z_custom, reuse=True) 641 | self.logits_x_custom = self.model_classifier_logits.get_logits((self.x_custom+1)*0.5) 642 | self.percept_x_custom = self.model_classifier_percept.get_logits((self.x_custom+1)*0.5) 643 | self.pred_x_custom = tf.nn.softmax(self.logits_x_custom) 644 | 645 | 646 | self.x_custom1 = 0.2989*self.x_custom[:,:,:,0] + 0.5870*self.x_custom[:,:,:,1] + 0.1140*self.x_custom[:,:,:,2] 647 | self.x_input1 = 0.2989*self.x_input[:,:,:,0] + 0.5870*self.x_input[:,:,:,1] + 0.1140*self.x_input[:,:,:,2] 648 | 649 | 650 | self.Loss_vae1_custom_pixel = tf.reduce_mean(tf.square(tf.subtract(self.x_custom1, self.x_input1))) 651 | #self.Loss_vae1_custom_pixel = tf.reduce_mean(tf.square(tf.subtract(self.x_custom, self.x_input))) 652 | # self.Loss_vae1_custom_percept = tf.reduce_mean(tf.square(tf.subtract(self.percept_x_custom, self.percept_x_true))) 653 | 654 | self.x_custom_sobel = tf.image.sobel_edges(self.x_custom) 655 | self.x_input_sobel = tf.image.sobel_edges(self.x_input) 656 | 657 | self.Loss_vae1_custom_edges = tf.reduce_mean(tf.square(tf.subtract(self.x_input_sobel, self.x_custom_sobel))) 658 | 659 | # self.Loss_vae1_custom = self.Loss_vae1_custom_pixel 660 | self.TV_Loss = tf.reduce_mean(tf.image.total_variation(self.x_custom)) 661 | self.Loss_vae1_custom = (1 - tf.image.ssim((self.x_custom1+1)*0.5, (self.x_input1+1)*0.5,max_val=1.0)) 662 | 663 | #self.Grad = tf.gradients(self.Loss_vae1_custom, self.z_custom) 664 | self.Grad = tf.gradients([self.Loss_vae1_custom, self.z_custom) 665 | 666 | t_vars = tf.trainable_variables() 667 | 668 | self.log_vars.append(("encode1_loss", self.encode1_loss)) 669 | self.log_vars.append(("generator1_loss", self.G1_loss)) 670 | 671 | 672 | 673 | self.g1_vars = [var for var in t_vars if 'VAE_gen1' in var.name] 674 | self.e1_vars = [var for var in t_vars if 'VAE_e1_' in var.name] 675 | 676 | 677 | self.saver = tf.train.Saver() 678 | for k, v in self.log_vars: 679 | tf.summary.scalar(k, v) 680 | 681 | print('Model is Built') 682 | 683 | 684 | 685 | 686 | 687 | #do train 688 | def train(self): 689 | 690 | global_step = tf.Variable(0, trainable=False) 691 | add_global = global_step.assign_add(1) 692 | new_learning_rate = tf.train.exponential_decay(self.learn_rate_init, global_step=global_step, decay_steps=10000, 693 | decay_rate=0.98) 694 | 695 | 696 | 697 | 698 | #for G1 699 | trainer_G1 = tf.train.RMSPropOptimizer(learning_rate=new_learning_rate) 700 | gradients_G1 = trainer_G1.compute_gradients(self.G1_loss, var_list=self.g1_vars) 701 | opti_G1 = trainer_G1.apply_gradients(gradients_G1) 702 | 703 | 704 | 705 | #for E1 706 | trainer_E1 = tf.train.RMSPropOptimizer(learning_rate=new_learning_rate) 707 | gradients_E1 = trainer_E1.compute_gradients(self.encode1_loss, var_list=self.e1_vars) 708 | opti_E1 = trainer_E1.apply_gradients(gradients_E1) 709 | 710 | #for ascent 711 | # trainer_ascent = tf.train.AdamOptimizer(learning_rate=new_learning_rate) 712 | # gradients_ascent = trainer_ascent.compute_gradients(self.Loss_vae1_custom, self.z_custom) 713 | # opti_ascent = trainer_ascent.apply_gradients(gradients_ascent) 714 | #self.Grad = tf.gradients(self.Loss_vae1_custom, self.z_custom) 715 | 716 | 717 | 718 | 719 | init = tf.global_variables_initializer() 720 | config = tf.ConfigProto() 721 | config.gpu_options.allow_growth = True 722 | with tf.Session(config=config) as sess: 723 | 724 | ckpt = tf.train.get_checkpoint_state('/home/manu_kohli/vae_classifier_weights/Classifier/checkpoint') 725 | #ckpt = tf.train.get_checkpoint_state('/home/manu_kohli/vae_classifier_weights/Classifier/re-trained_classifier/') 726 | ckpt_path = ckpt.model_checkpoint_path 727 | sess.run(init) 728 | self.saver.restore(sess , self.saved_model_path) 729 | print(tf.trainable_variables(),'tf.trainable_variables()') 730 | saver = tf.train.Saver([var for var in tf.trainable_variables() if var.name.startswith('model1')]) 731 | print(ckpt_path) 732 | #saver.restore(sess, ckpt_path) 733 | #self.saver.save(sess , self.saved_model_path) 734 | 735 | print('Creating a Replica of s1 onto s2') 736 | s1_vars1 = [var.name for var in tf.trainable_variables() if 'model1' in var.name] 737 | s2_vars1 = [var for var in tf.trainable_variables() if 'model2' in var.name] 738 | dictionary = {} 739 | for i in range(len(s2_vars1)): 740 | dictionary[s1_vars1[i][0:-2]] = s2_vars1[i] 741 | saver_new = tf.train.Saver(var_list=dictionary) 742 | #saver_new.restore(sess, self.saved_model_path) 743 | 744 | 745 | # self.saver.save(sess , self.saved_model_path) 746 | 747 | print('******************') 748 | print(' ') 749 | print(' ') 750 | print('Plain VAE Training Begins') 751 | print(' ') 752 | print(' ') 753 | print('******************') 754 | 755 | ############################################################################################################# 756 | beta = 0.5 757 | idx=np.random.choice(self.TestDataSize, 1024) 758 | sampled_test=self.X_Real_Test[idx] 759 | sampled_test_lbls=self.Y_test[idx] 760 | #Preddiction = np.zeros([self.TestDataSize,11]) 761 | Preddiction = np.zeros([len(sampled_test),11]) 762 | for i in range(np.int(len(sampled_test)/self.batch_size)): 763 | #for i in range(np.int(sel/self.batch_size)): 764 | next_x_images = sampled_test[i*self.batch_size:(i+1)*self.batch_size] 765 | next_y_labels = sampled_test_lbls[i*self.batch_size:(i+1)*self.batch_size] 766 | z_grad_ascent = np.random.normal(0,1,size = (64,64)) 767 | z_grad_ascent = np.reshape(z_grad_ascent,(64,64)) 768 | V = np.zeros([64,64]) 769 | for j in range(0,1000): 770 | #z_grad_ascent = z_grad_ascent - gamma*V 771 | gard1, loss,loss_edges = sess.run([self.Grad,self.Loss_vae1_custom, self.TV_Loss], feed_dict={self.z_custom: z_grad_ascent+beta*V,self.x_input: next_x_images, self.x_true:next_x_images,self.keep_prob:1}) 772 | #z_grad_ascent = z_grad_ascent + gamma*V 773 | gard1 = np.asarray(gard1) 774 | gard1 = np.reshape(gard1, (64,64)) 775 | print(np.sum(np.abs(gard1))) 776 | V = V*beta - 100*gard1 777 | z_grad_ascent = z_grad_ascent + V 778 | print(np.sum(np.abs(gard1))) 779 | pred = sess.run(self.pred_x_custom, feed_dict={self.z_custom: z_grad_ascent,self.x_input: next_x_images, self.x_true:next_x_images,self.keep_prob:1}) 780 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 781 | #self.x_custom = self.generate1(self.x_input_sobel,self.z_custom, reuse=True) 782 | gen_images = sess.run(self.x_custom, feed_dict={self.z_custom: z_grad_ascent,self.x_input: next_x_images,self.keep_prob:1}) 783 | name = 'gen_images/x_gen__' + str(i) + '_.npy' 784 | name1 = 'gen_images/y_gen__' + str(i) + '_.npy' 785 | np.save(name,gen_images) 786 | np.save(name1,next_y_labels) 787 | 788 | name = 'real_images/x_gen__' + str(i) + '_.npy' 789 | name1 = 'real_images/y_gen__' + str(i) + '_.npy' 790 | np.save(name,next_x_images) 791 | np.save(name1,next_y_labels) 792 | 793 | #print('Full Filtered Real Cam2 Example Acc = ',i ,getAcc(Preddiction, self.Y_test)) 794 | print('Full Filtered Real Cam2 Example Acc = ',i ,getAcc(Preddiction, sampled_test_lbls)) 795 | 796 | ############################################################################################################# 797 | 798 | 799 | # OutSize = 96 800 | # step = 0 801 | # batchNum = 0 802 | # gamma = 0.5 803 | # if (1): 804 | # if (1): 805 | # Preddiction = np.zeros([6400,11]) 806 | # for i in range(10): 807 | # next_x_images = self.X_Real_Test[(12+i)*self.batch_size:(12+i+1)*self.batch_size] 808 | # V = np.zeros([64,64]) 809 | # z_grad_ascent = np.random.normal(0,1,size = (64,64)) 810 | # z_grad_ascent = np.reshape(z_grad_ascent,(64,64)) 811 | # #z_grad_ascent = sess.run(self.z1_mean, feed_dict={self.x_input: next_x_images,self.keep_prob:1}) 812 | # #z_grad_ascent = sess.run(self.z1_mean, feed_dict={self.z_custom: z_grad_ascent,self.x_input: next_x_images, self.x_true:next_x_images,self.keep_prob:1}) 813 | 814 | # for j in range(5): 815 | # for k in range(100): 816 | # z_grad_ascent = z_grad_ascent - gamma*V 817 | # gard1, loss,loss_edges = sess.run([self.Grad,self.Loss_vae1_custom, self.TV_Loss], feed_dict={self.z_custom: z_grad_ascent,self.x_input: next_x_images, self.x_true:next_x_images,self.keep_prob:1}) 818 | # z_grad_ascent = z_grad_ascent + gamma*V 819 | # gard1 = np.asarray(gard1) 820 | # gard1 = np.reshape(gard1, (64,64)) 821 | # print(np.sum(np.abs(gard1)), loss, loss_edges) 822 | # V = V*gamma + 100*gard1 823 | # z_grad_ascent = z_grad_ascent - V 824 | # print(np.sum(np.abs(gard1)), loss) 825 | # pred = sess.run(self.pred_x_custom, feed_dict={self.z_custom: z_grad_ascent,self.x_input: next_x_images, self.x_true:next_x_images,self.keep_prob:1}) 826 | # Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 827 | # #self.x_custom = self.generate1(self.x_input_sobel,self.z_custom, reuse=True) 828 | # gen_images = sess.run(self.x_custom, feed_dict={self.z_custom: z_grad_ascent,self.x_input: next_x_images,self.keep_prob:1}) 829 | # name = 'gen_images/x_gen__' + str(i) + '_.npy' 830 | # np.save(name,gen_images) 831 | 832 | # print('Full Filtered Real Cam2 Example Acc = ',i ,getAcc(Preddiction[0:(i+1)*64], self.Y_test[12*64:(12+i+1)*64])) 833 | 834 | 835 | 836 | 837 | 838 | def generate1(self, edge, z_var, reuse=False): 839 | 840 | with tf.variable_scope('generator1') as scope: 841 | 842 | if reuse == True: 843 | scope.reuse_variables() 844 | 845 | d1 = lrelu(fully_connect(z_var , output_size=64*4*4, scope='VAE_gen1_fully1')) 846 | d2 = lrelu(fully_connect(d1 , output_size=128*4*4, scope='VAE_gen1_fully2')) 847 | d3 = tf.reshape(d2, [self.batch_size, 4, 4, 128]) 848 | d4 = lrelu(de_conv(d3, output_shape=[self.batch_size, 8, 8, 128], k_h=3, k_w=3,name='VAE_gen1_deconv1')) 849 | d5 = lrelu(de_conv(d4, output_shape=[self.batch_size, 16, 16, 128], k_h=3, k_w=3,name='VAE_gen1_deconv2')) 850 | d6 = lrelu(de_conv(d5, output_shape=[self.batch_size, 32, 32, 128], k_h=3, k_w=3,name='VAE_gen1_deconv3')) 851 | d7 = lrelu(de_conv(d6, output_shape=[self.batch_size, 64, 64, 128], k_h=3, k_w=3,name='VAE_gen1_deconv4')) 852 | d8 = de_conv(d7, output_shape=[self.batch_size, 128, 128, 3] , k_h=3, k_w=3, name='VAE_gen1_deconv5') 853 | d9 = tf.nn.tanh(d8) 854 | d10 = tf.concat([d9, edge], 3) 855 | conv1 = lrelu(conv2d(d10, output_dim=128, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c1')) 856 | conv2 = lrelu(conv2d(conv1, output_dim=128, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c2')) 857 | conv3 = conv2d(conv2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c3') 858 | 859 | 860 | return tf.nn.tanh(conv3) 861 | 862 | 863 | 864 | 865 | def Encode1(self, x, reuse=False): 866 | 867 | with tf.variable_scope('encode1') as scope: 868 | 869 | if reuse == True: 870 | scope.reuse_variables() 871 | conv1 = lrelu(conv2d(x, output_dim=128, k_h=3, k_w=3, name='VAE_e1_c1')) 872 | conv2 = lrelu(conv2d(conv1, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c2')) 873 | conv3 = lrelu(conv2d(conv2, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c3')) 874 | conv4 = lrelu(conv2d(conv3, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c4')) 875 | conv5 = lrelu(conv2d(conv4, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c5')) 876 | conv6 = tf.reshape(conv5, [self.batch_size, 128 * 4 * 4]) 877 | fc1 = lrelu(fully_connect(conv6, output_size= 64*4*4, scope='VAE_e1_f1')) 878 | z_mean = fully_connect(fc1, output_size=self.latent_dim, scope='VAE_e1_f2') 879 | z_sigma = fully_connect(fc1, output_size=self.latent_dim, scope='VAE_e1_f3') 880 | return z_mean, z_sigma 881 | 882 | 883 | def KL_loss(self, mu, log_var): 884 | return -0.5 * tf.reduce_sum(1 + log_var - tf.pow(mu, 2) - tf.exp(log_var)) 885 | 886 | def sample_z(self, mu, log_var): 887 | eps = tf.random_normal(shape=tf.shape(mu)) 888 | return mu + tf.exp(log_var / 2) * eps 889 | 890 | 891 | def NLLNormal(self, pred, target): 892 | 893 | c = -0.5 * tf.log(2 * np.pi) 894 | multiplier = 1.0 / (2.0 * 1) 895 | tmp = tf.square(pred - target) 896 | tmp *= -multiplier 897 | tmp += c 898 | 899 | return tmp 900 | 901 | 902 | 903 | 904 | 905 | 906 | 907 | flags = tf.app.flags 908 | 909 | flags.DEFINE_integer("batch_size" , 64, "batch size") 910 | flags.DEFINE_integer("max_iters" , 10000, "the maxmization epoch") 911 | flags.DEFINE_integer("latent_dim" , 64, "the dim of latent code") 912 | flags.DEFINE_float("learn_rate_init" , 0.0001, "the init of learn rate") 913 | flags.DEFINE_integer("repeat", 10000, "the numbers of repeat for your datasets") 914 | flags.DEFINE_string("path", '/home/?/data/', "for example, '/home/jack/data/' is the directory of your celebA data") 915 | flags.DEFINE_integer("op", 0, "Training or Test") 916 | 917 | FLAGS = flags.FLAGS 918 | FLAGS.op = 0 919 | if (1): 920 | path123 = '.' 921 | root_log_dir = path123 + "/log_dir" 922 | #vaegan_checkpoint_dir = '/home/manu_kohli/vae_classifier_weights/VAE/itr_model/model.cpkt-43600' 923 | vaegan_checkpoint_dir = '/home/manu_kohli/vae_classifier_weights/VAE/itr_model_1/model.cpkt-5400' 924 | sample_path = path123 + "/sample" 925 | 926 | 927 | model_path = vaegan_checkpoint_dir 928 | 929 | batch_size = FLAGS.batch_size 930 | max_iters = FLAGS.max_iters 931 | latent_dim = FLAGS.latent_dim 932 | data_repeat = FLAGS.repeat 933 | 934 | learn_rate_init = FLAGS.learn_rate_init 935 | 936 | vaeGan = vaegan(batch_size= batch_size, max_iters= max_iters, repeat = data_repeat, 937 | model_path= model_path, latent_dim= latent_dim, 938 | sample_path= sample_path , log_dir= root_log_dir , learnrate_init= learn_rate_init) 939 | 940 | vaeGan.build_model_vaegan() 941 | vaeGan.train() 942 | 943 | 944 | 945 | 946 | 947 | 948 | 949 | 950 | 951 | 952 | 953 | -------------------------------------------------------------------------------- /TrainVAE.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | import os 6 | import keras 7 | from keras import backend as K 8 | from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D 9 | from keras.preprocessing import image 10 | from keras.applications.vgg19 import preprocess_input 11 | from keras.models import Model, Sequential 12 | import numpy as np 13 | import matplotlib 14 | import pandas as pd 15 | import cv2 16 | from keras.utils import to_categorical 17 | from keras.layers import Dense, GlobalAveragePooling2D 18 | import tensorflow as tf 19 | from keras.callbacks import ModelCheckpoint 20 | from cleverhans.attacks import FastGradientMethod 21 | from cleverhans.loss import CrossEntropy 22 | from cleverhans.train import train 23 | from cleverhans.utils import AccuracyReport 24 | from cleverhans.utils_keras import cnn_model 25 | from cleverhans.utils_keras import KerasModelWrapper 26 | from cleverhans.utils_tf import model_eval 27 | from cleverhans.utils_tf import model_argmax 28 | import functools 29 | import tensorflow as tf 30 | from cleverhans import initializers 31 | from cleverhans.model import Model 32 | #from cleverhans.picklable_model import MLP, Conv2D, ReLU, Flatten, Linear 33 | from cleverhans.picklable_model import Softmax 34 | import math 35 | import logging 36 | from tensorflow.python.platform import flags 37 | from cleverhans.dataset import MNIST 38 | from cleverhans.utils import AccuracyReport, set_log_level 39 | from cleverhans.augmentation import random_horizontal_flip, random_shift 40 | from cleverhans.dataset import CIFAR10 41 | from cleverhans.model_zoo.all_convolutional import ModelAllConvolutional 42 | from keras.models import load_model 43 | 44 | from pdb import set_trace as trace 45 | 46 | from shutil import copyfile 47 | import imageio 48 | 49 | import tensorflow as tf 50 | from tensorflow.contrib.layers.python.layers import batch_norm 51 | 52 | import os, csv, keras, math, logging, functools, cv2, sys 53 | #from keras.applications.vgg19 import VGG19, preprocess_input 54 | from keras.preprocessing import image 55 | from keras.models import Model, Sequential 56 | from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, GlobalAveragePooling2D, ZeroPadding2D, Convolution2D, MaxPooling2D 57 | import numpy as np 58 | import pandas as pd 59 | from keras.utils import to_categorical 60 | #from sklearn.preprocessing import OneHotEncoder 61 | import tensorflow as tf 62 | from keras.callbacks import ModelCheckpoint 63 | from cleverhans.attacks import FastGradientMethod 64 | from cleverhans.loss import CrossEntropy 65 | from cleverhans.train import train 66 | from cleverhans.utils import AccuracyReport, set_log_level 67 | from cleverhans.utils_keras import cnn_model 68 | from cleverhans.utils_keras import KerasModelWrapper 69 | from cleverhans.utils_tf import model_eval, model_argmax 70 | from cleverhans import initializers 71 | from cleverhans.model import Model 72 | from tensorflow.python.platform import flags 73 | #from cleverhans.model_zoo.all_convolutional import ModelAllConvolutional 74 | #from vgg import VGG16 75 | #from vgg19 import VGG19 76 | from keras.datasets import cifar10 77 | #from sklearn.utils import class_weight 78 | import ssl 79 | ssl._create_default_https_context = ssl._create_unverified_context 80 | 81 | 82 | 83 | 84 | classes = ['nrbc', 'notawbc', 'giant platelet', 'platelet clump', 'basophil', 85 | 'neutrophil', 'eosinophil', 'lymphocyte', 'monocyte', 'ig', 'atypical-blast'] 86 | #os.environ["CUDA_VISIBLE_DEVICES"]="0" 87 | #os.environ["TF_CPP_MIN_LOG_LEVEL"]="2" 88 | def toOneHot(a): 89 | b = np.zeros((a.shape[0], 11)) 90 | for i in range(a.shape[0]): 91 | for j in range(11): 92 | if a[i] == classes[j]: 93 | b[i][j] = 1 94 | return b 95 | 96 | def del_all_flags(FLAGS): 97 | flags_dict = FLAGS._flags() 98 | keys_list = [keys for keys in flags_dict] 99 | for keys in keys_list: 100 | FLAGS.__delattr__(keys) 101 | 102 | 103 | 104 | 105 | 106 | def lrelu(x , alpha = 0.2 , name="LeakyReLU"): 107 | return tf.maximum(x , alpha*x) 108 | 109 | def conv2d(input_, output_dim, 110 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 111 | name="conv2d"): 112 | 113 | with tf.variable_scope(name): 114 | 115 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 116 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 117 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 118 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 119 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 120 | 121 | return conv 122 | 123 | def de_conv(input_, output_shape, 124 | k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, 125 | name="deconv2d", with_w=False): 126 | 127 | with tf.variable_scope(name): 128 | # filter : [height, width, output_channels, in_channels] 129 | w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], 130 | initializer=tf.random_normal_initializer(stddev=stddev)) 131 | 132 | try: 133 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, 134 | strides=[1, d_h, d_w, 1]) 135 | 136 | # Support for verisons of TensorFlow before 0.7.0 137 | except AttributeError: 138 | 139 | deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape, 140 | strides=[1, d_h, d_w, 1]) 141 | 142 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 143 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 144 | 145 | if with_w: 146 | 147 | return deconv, w, biases 148 | 149 | else: 150 | 151 | return deconv 152 | 153 | def fully_connect(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False): 154 | shape = input_.get_shape().as_list() 155 | with tf.variable_scope(scope or "Linear"): 156 | 157 | matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, 158 | tf.random_normal_initializer(stddev=stddev)) 159 | bias = tf.get_variable("bias", [output_size], 160 | initializer=tf.constant_initializer(bias_start)) 161 | 162 | if with_w: 163 | return tf.matmul(input_, matrix) + bias, matrix, bias 164 | else: 165 | 166 | return tf.matmul(input_, matrix) + bias 167 | 168 | def conv_cond_concat(x, y): 169 | """Concatenate conditioning vector on feature map axis.""" 170 | x_shapes = x.get_shape() 171 | y_shapes = y.get_shape() 172 | 173 | return tf.concat(3 , [x , y*tf.ones([x_shapes[0], x_shapes[1], x_shapes[2] , y_shapes[3]])]) 174 | 175 | def batch_normal(input , scope="scope" , reuse=False): 176 | return batch_norm(input , epsilon=1e-5, decay=0.9 , scale=True, scope=scope , reuse=reuse , updates_collections=None) 177 | 178 | def instance_norm(x): 179 | 180 | epsilon = 1e-9 181 | mean, var = tf.nn.moments(x, [1, 2], keep_dims=True) 182 | return tf.div(tf.subtract(x, mean), tf.sqrt(tf.add(var, epsilon))) 183 | 184 | # def residual(x, output_dims, kernel, strides, name_1, name_2): 185 | 186 | # with tf.variable_scope('residual') as scope: 187 | 188 | # conv1 = conv2d(x, output_dims, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_1) 189 | # conv2 = conv2d(tf.nn.relu(conv1), output_dims, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_2) 190 | # resi = x + conv2 191 | 192 | # return resi 193 | 194 | # def deresidual(x, output_shape, kernel, strides, name_1, name_2): 195 | 196 | # with tf.variable_scope('residual_un') as scope: 197 | 198 | # deconv1 = de_conv(x, output_shape=output_shape, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_1) 199 | # deconv2 = de_conv(tf.nn.relu(deconv1), output_shape=output_shape, k_h=kernel, k_w=kernel, d_h=strides, d_w=strides, name=name_2) 200 | # resi = x + deconv2 201 | 202 | # return resi 203 | import os 204 | import errno 205 | import numpy as np 206 | import scipy 207 | import scipy.misc 208 | from keras.models import Model 209 | 210 | 211 | def mkdir_p(path): 212 | try: 213 | os.makedirs(path) 214 | except OSError as exc: # Python >2.5 215 | if exc.errno == errno.EEXIST and os.path.isdir(path): 216 | pass 217 | else: 218 | raise 219 | 220 | def get_image(image_path, image_size, is_crop=True, resize_w=64, is_grayscale=False): 221 | return transform(imread(image_path, is_grayscale), image_size, is_crop, resize_w) 222 | 223 | 224 | # def transform(image, npx=64, is_crop=False, resize_w=64): 225 | # # npx : # of pixels width/height of image 226 | # if is_crop: 227 | # cropped_image = center_crop(image, npx, resize_w=resize_w) 228 | # else: 229 | # cropped_image = image 230 | # cropped_image = scipy.misc.imresize(cropped_image, 231 | # [resize_w, resize_w]) 232 | # return np.array(cropped_image) / 127.5 - 1 233 | 234 | def center_crop(x, crop_h , crop_w=None, resize_w=64): 235 | 236 | if crop_w is None: 237 | crop_w = crop_h 238 | h, w = x.shape[:2] 239 | j = int(round((h - crop_h)/2.)) 240 | i = int(round((w - crop_w)/2.)) 241 | return scipy.misc.imresize(x[j:j+crop_h, i:i+crop_w], 242 | [resize_w, resize_w]) 243 | 244 | 245 | def save_images(images, size, image_path): 246 | return imsave(inverse_transform(images), size, image_path) 247 | 248 | def imread(path, is_grayscale=False): 249 | if (is_grayscale): 250 | return scipy.misc.imread(path, flatten=True).astype(np.float) 251 | else: 252 | return scipy.misc.imread(path).astype(np.float) 253 | 254 | 255 | def imsave(images, size, path): 256 | return scipy.misc.imsave(path, merge(images, size)) 257 | 258 | def merge(images, size): 259 | h, w = images.shape[1], images.shape[2] 260 | size1 = np.int(h * size[0]) 261 | size2 = np.int(w * size[1]) 262 | img = np.zeros((size1,size2, 3)) 263 | for idx, image in enumerate(images): 264 | i = idx % size[1] 265 | j = idx // size[1] 266 | img[np.int(j * h):np.int(j * h + h), np.int(i * w): np.int(i * w + w), :] = image 267 | 268 | return img 269 | 270 | 271 | 272 | # def inverse_transform(image): 273 | # return ((image + 1) * 127.5).astype(np.uint8) 274 | 275 | 276 | 277 | import tensorflow as tf 278 | 279 | from cleverhans import initializers 280 | from cleverhans.serial import NoRefModel 281 | 282 | 283 | 284 | from keras.utils.np_utils import to_categorical 285 | import PIL 286 | import numpy as np 287 | import scipy 288 | from tensorflow.python.framework.ops import convert_to_tensor 289 | import os 290 | TINY = 1e-8 291 | d_scale_factor = 0.25 292 | g_scale_factor = 1 - 0.75/2 293 | import csv 294 | 295 | def getAcc(pred, next_y_images): 296 | acc = np.zeros([11]) 297 | Tc = np.ones([11]) 298 | for i in range(len(pred)): 299 | Tc[np.argmax(next_y_images[i])] = Tc[np.argmax(next_y_images[i])] + 1 300 | if (np.argmax(next_y_images[i]) == np.argmax(pred[i])): 301 | acc[np.argmax(next_y_images[i])] = acc[np.argmax(next_y_images[i])] + 1 302 | print(100*np.sum(acc)/(np.sum(Tc)-11)) 303 | return 100*acc/Tc,100*np.sum(acc)/(np.sum(Tc)-11) 304 | class ModelAllConvolutional(NoRefModel): 305 | """ 306 | A simple model that uses only convolution and downsampling---no batch norm or other techniques that can complicate 307 | adversarial training. 308 | """ 309 | def __init__(self, scope, nb_classes, nb_filters, input_shape, **kwargs): 310 | del kwargs 311 | NoRefModel.__init__(self, scope, nb_classes, locals()) 312 | self.nb_filters = nb_filters 313 | self.input_shape = input_shape 314 | 315 | # Do a dummy run of fprop to create the variables from the start 316 | self.fprop(tf.placeholder(tf.float32, [32] + input_shape)) 317 | # Put a reference to the params in self so that the params get pickled 318 | self.params = self.get_params() 319 | 320 | def fprop(self, x, **kwargs): 321 | del kwargs 322 | conv_args = dict( 323 | activation=tf.nn.leaky_relu, 324 | kernel_initializer=initializers.HeReLuNormalInitializer, 325 | kernel_size=3, 326 | padding='same') 327 | y = x 328 | 329 | with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): 330 | log_resolution = int(round( 331 | math.log(self.input_shape[0]) / math.log(2))) 332 | for scale in range(log_resolution - 2): 333 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 334 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 335 | y = tf.layers.average_pooling2d(y, 2, 2) 336 | y = tf.layers.conv2d(y, self.nb_classes, **conv_args) 337 | logits = tf.reduce_mean(y, [1, 2]) 338 | return {self.O_LOGITS: logits, 339 | self.O_PROBS: tf.nn.softmax(logits=logits)} 340 | 341 | 342 | class ModelAllConvolutional1(NoRefModel): 343 | """ 344 | A simple model that uses only convolution and downsampling---no batch norm or other techniques that can complicate 345 | adversarial training. 346 | """ 347 | def __init__(self, scope, nb_classes, nb_filters, input_shape, **kwargs): 348 | del kwargs 349 | NoRefModel.__init__(self, scope, nb_classes, locals()) 350 | self.nb_filters = nb_filters 351 | self.input_shape = input_shape 352 | 353 | # Do a dummy run of fprop to create the variables from the start 354 | self.fprop(tf.placeholder(tf.float32, [32] + input_shape)) 355 | # Put a reference to the params in self so that the params get pickled 356 | self.params = self.get_params() 357 | 358 | def fprop(self, x, **kwargs): 359 | del kwargs 360 | conv_args = dict( 361 | activation=tf.nn.leaky_relu, 362 | kernel_initializer=initializers.HeReLuNormalInitializer, 363 | kernel_size=3, 364 | padding='same') 365 | y = x 366 | 367 | with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE): 368 | log_resolution = int(round( 369 | math.log(self.input_shape[0]) / math.log(2))) 370 | for scale in range(log_resolution - 4): 371 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 372 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 373 | y = tf.layers.average_pooling2d(y, 2, 2) 374 | conv = y 375 | scale = log_resolution - 4 376 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 377 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 378 | y = tf.layers.average_pooling2d(y, 2, 2) 379 | 380 | 381 | scale = log_resolution - 3 382 | y = tf.layers.conv2d(y, self.nb_filters << scale, **conv_args) 383 | y = tf.layers.conv2d(y, self.nb_filters << (scale + 1), **conv_args) 384 | y = tf.layers.average_pooling2d(y, 2, 2) 385 | y = tf.layers.conv2d(y, self.nb_classes, **conv_args) 386 | 387 | logits = tf.reduce_mean(y, [1, 2]) 388 | return {self.O_LOGITS: conv, 389 | self.O_PROBS: tf.nn.softmax(logits=logits)} 390 | 391 | 392 | 393 | 394 | 395 | 396 | class vaegan(object): 397 | 398 | #build model 399 | def __init__(self, batch_size, max_iters, repeat, model_path, latent_dim, sample_path, log_dir, learnrate_init): 400 | 401 | self.batch_size = batch_size 402 | self.max_iters = max_iters 403 | self.repeat_num = repeat 404 | self.saved_model_path = model_path 405 | 406 | self.latent_dim = latent_dim 407 | self.sample_path = sample_path 408 | self.log_dir = log_dir 409 | self.learn_rate_init = learnrate_init 410 | 411 | self.log_vars = [] 412 | 413 | self.channel = 3 414 | self.output_size = 128 415 | 416 | self.x_input = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, 3]) 417 | self.x_true = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, self.channel]) 418 | 419 | 420 | 421 | self.labels = tf.placeholder(tf.float32, [self.batch_size, 11]) 422 | 423 | 424 | self.ep1 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 425 | self.zp1 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 426 | 427 | self.ep2 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 428 | self.zp2 = tf.random_normal(shape=[self.batch_size, self.latent_dim]) 429 | self.keep_prob = tf.placeholder_with_default(1.0, shape=()) 430 | 431 | print('Data Loading Begins') 432 | 433 | y_train=[] 434 | x_train1=[] 435 | for dirs in os.listdir('/home/manu_kohli/wbc/cam3/trainset/'): 436 | for files in os.listdir('/home/manu_kohli/wbc/cam3/trainset/'+dirs): 437 | y_train.append(int(dirs)) 438 | x_train1.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam3/trainset/'+dirs+'/'+files))) 439 | 440 | #x_train1 =np.asarray(x_train1)/255.0 441 | 442 | cam3_train_data=[] 443 | cam3_train_label=[] 444 | 445 | l=list(range(0,len(y_train))) 446 | l=np.asarray(l) 447 | np.random.shuffle(l) 448 | for i in l: 449 | cam3_train_data.append(x_train1[i]) 450 | cam3_train_label.append(y_train[i]) 451 | 452 | x_train1=cam3_train_data 453 | y_train=cam3_train_label 454 | 455 | x_train1 = np.asarray(x_train1)/127.5 456 | x_train1 =x_train1 - 1. 457 | y_train = np.asarray(y_train) 458 | #y_train = toOneHot(y_train) 459 | y_train= to_categorical(y_train, num_classes=11) 460 | # x_train1 = np.load( '/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/X_Train.npy').astype('float32') 461 | # y_train = np.load( '/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/Y_Train.npy') 462 | # x_train1_1 = np.load('/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/X_Test.npy').astype('float32') 463 | # y_train_1 = np.load('/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/Y_Test.npy') 464 | 465 | # x_train1_2 = np.load( '/home/vinay/projects/Sigtuple/CameraInvariance/Cam3Classifier/Data_Augmentation/X_Train_extra.npy').astype('float32') 466 | # y_train_2 = np.load( '/home/vinay/projects/Sigtuple/CameraInvariance/Cam3Classifier/Data_Augmentation/Y_Train_extra.npy') 467 | 468 | # x_train1 = np.append(x_train1, x_train1_2,axis =0) 469 | # y_train = np.append(y_train, y_train_2,axis =0) 470 | 471 | 472 | # x_train1 = np.concatenate((x_train1, x_train1_1), axis=0) 473 | # y_train = np.concatenate((y_train, y_train_1), axis=0) 474 | 475 | 476 | x_test1_cam3 = [] 477 | y_test_cam3 = [] 478 | 479 | for dirs in os.listdir('/home/manu_kohli/wbc/cam3/testset/'): 480 | for files in os.listdir('/home/manu_kohli/wbc/cam3/testset/'+dirs): 481 | y_test_cam3.append(int(dirs)) 482 | x_test1_cam3.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam3/testset/'+dirs+'/'+files))) 483 | 484 | cam3_test_data=[] 485 | cam3_test_label=[] 486 | 487 | l=list(range(0,len(y_test_cam3))) 488 | l=np.asarray(l) 489 | np.random.shuffle(l) 490 | for i in l: 491 | cam3_test_data.append(x_test1_cam3[i]) 492 | cam3_test_label.append(y_test_cam3[i]) 493 | 494 | x_test1_cam3 = cam3_test_data 495 | y_test_cam3 = cam3_test_label 496 | 497 | y_test_cam3= to_categorical(y_test_cam3, num_classes=11) 498 | #y_test_cam3 = toOneHot(np.asarray(y_test_cam3)) 499 | #x_test1_cam3=np.asarray(x_test1_cam3)/255.0 500 | x_test1_cam3 = np.asarray(x_test1_cam3)/127.5 501 | x_test1_cam3 =x_test1_cam3 - 1. 502 | 503 | x_test1=[] 504 | y_test =[] 505 | 506 | for dirs in os.listdir('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'): 507 | for files in os.listdir('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'+dirs): 508 | y_test.append(int(dirs)) 509 | x_test1.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'+dirs+'/'+files))) 510 | 511 | # x_test1 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam2_images.npy').astype('float32')/255 512 | # y_test = np.load('/home/vinay/projects/Sigtuple/CreateData/cam2_labels.npy') 513 | 514 | cam2_data=[] 515 | cam2_label=[] 516 | 517 | l=list(range(0,len(y_test))) 518 | l=np.asarray(l) 519 | np.random.shuffle(l) 520 | for i in l: 521 | cam2_data.append(x_test1[i]) 522 | cam2_label.append(y_test[i]) 523 | 524 | x_test1 = cam2_data 525 | y_test = cam2_label 526 | 527 | y_test= to_categorical(y_test, num_classes=11) 528 | #y_test = toOneHot(np.asarray(y_test)) 529 | # x_test1=np.asarray(x_test1)/255.0 530 | x_test1 = np.asarray(x_test1)/127.5 531 | x_test1 =x_test1 - 1. 532 | 533 | # x_test1_cam3 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam3_images.npy').astype('float32')/255 534 | # y_test_cam3 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam3_labels.npy') 535 | # y_test_cam3 = toOneHot(y_test_cam3) 536 | 537 | #print(x_train1.shape, y_train.shape) 538 | #print(x_test1.shape, y_test.shape) 539 | #x_train = np.zeros([x_train1.shape[0], self.output_size,self.output_size,self.channel]) 540 | #x_test = np.zeros([x_test1.shape[0], self.output_size,self.output_size,self.channel]) 541 | #x_test_cam3 = np.zeros([x_test1_cam3.shape[0], self.output_size,self.output_size,self.channel]) 542 | 543 | # x_train[:,:,:,0] = x_train1[:,:,:,2] 544 | # x_train[:,:,:,1] = x_train1[:,:,:,1] 545 | # x_train[:,:,:,2] = x_train1[:,:,:,0] 546 | 547 | # x_test[:,:,:,0] = x_test1[:,:,:,2] 548 | # x_test[:,:,:,1] = x_test1[:,:,:,1] 549 | # x_test[:,:,:,2] = x_test1[:,:,:,0] 550 | 551 | # x_test_cam3[:,:,:,0] = x_test1_cam3[:,:,:,2] 552 | # x_test_cam3[:,:,:,1] = x_test1_cam3[:,:,:,1] 553 | # x_test_cam3[:,:,:,2] = x_test1_cam3[:,:,:,0] 554 | 555 | x_train = np.float32(x_train1).reshape([-1,self.output_size,self.output_size,self.channel]) 556 | x_test = np.float32(x_test1).reshape([-1,self.output_size,self.output_size,self.channel]) 557 | x_test_cam3 = np.float32(x_test1_cam3).reshape([-1,self.output_size,self.output_size,self.channel]) 558 | 559 | print(x_train.shape, y_train.shape) 560 | print(x_test.shape, y_test.shape) 561 | print(x_test_cam3.shape, y_test_cam3.shape) 562 | print(np.amin(x_train), np.amin( x_test ), np.amin(x_test_cam3)) 563 | print(np.amax(x_train), np.amax( x_test ), np.amax(x_test_cam3)) 564 | 565 | 566 | TrainDataSize = x_train.shape[0] 567 | TestDataSize = x_test.shape[0] 568 | self.TrainDataSize = TrainDataSize 569 | self.TestDataSize = TestDataSize 570 | self.TestDataSize_cam3 = x_test_cam3.shape[0] 571 | 572 | 573 | self.X_Real_Test = x_test 574 | self.X_Real_Train = x_train 575 | self.X_Real_Test_cam3 = x_test_cam3 576 | self.Y_train = y_train 577 | self.Y_test = y_test 578 | self.Y_test_cam3 = y_test_cam3 579 | 580 | 581 | # self.X_Real_Train = self.X_Real_Train*2 - 1 582 | # self.X_Real_Test = self.X_Real_Test*2 - 1 583 | # self.X_Real_Test_cam3 = self.X_Real_Test_cam3*2 - 1 584 | 585 | print('Max', np.max(self.X_Real_Train)) 586 | print('Min', np.min(self.X_Real_Train)) 587 | 588 | print('Data Loading Completed') 589 | 590 | 591 | 592 | 593 | 594 | def build_model_vaegan(self): 595 | 596 | self.z1_mean, self.z1_sigm = self.Encode1(self.x_input) 597 | self.z1_x = tf.add( self.z1_mean, tf.sqrt(tf.exp(self.z1_sigm))*self.ep1) 598 | self.x_input_sobel = tf.image.sobel_edges(self.x_input) 599 | self.x_input_sobel = tf.reshape(self.x_input_sobel, [64,128,128,6]) 600 | self.x_out = self.generate1(self.x_input_sobel, self.z1_x, reuse=False) 601 | 602 | self.x_filt2 = self.generate1(self.x_input_sobel, self.z1_mean, reuse=True) 603 | 604 | self.model_classifier_logits = ModelAllConvolutional('model1', 11, 64, input_shape=[self.output_size,self.output_size,self.channel]) 605 | self.model_classifier_percept = ModelAllConvolutional1('model2', 11, 64, input_shape=[self.output_size,self.output_size,self.channel]) 606 | #tanh o/p -1 to 1 607 | self.logits_x_true = self.model_classifier_logits.get_logits((self.x_true+1)*0.5) 608 | self.percept_x_true = self.model_classifier_percept.get_logits((self.x_true+1)*0.5) 609 | #self.pred_x_true = tf.nn.softmax(self.logits_x_true) 610 | self.pred_x_true = self.model_classifier_percept.get_probs((self.x_true+1)*0.5) 611 | 612 | 613 | self.logits_x_out = self.model_classifier_logits.get_logits((self.x_out+1)*0.5) 614 | self.percept_x_out = self.model_classifier_percept.get_logits((self.x_out+1)*0.5) 615 | self.pred_x_out = tf.nn.softmax(self.logits_x_out) 616 | 617 | 618 | self.logits_x_filt2 = self.model_classifier_logits.get_logits((self.x_filt2+1)*0.5) 619 | self.pred_x_filt2 = tf.nn.softmax(self.logits_x_filt2) 620 | 621 | 622 | 623 | self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels)) 624 | self.cl_loss_x_out = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_out , labels = self.labels)) 625 | self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels)) 626 | 627 | 628 | 629 | self.kl1_loss = self.KL_loss(self.z1_mean, self.z1_sigm)/(self.latent_dim*self.batch_size) 630 | 631 | 632 | self.Loss_vae1_pixel = tf.reduce_mean(tf.square(tf.subtract(self.x_out, self.x_true))) + tf.reduce_mean(tf.abs(tf.subtract(self.x_out, self.x_true))) 633 | self.Loss_vae1_percept = tf.reduce_mean(tf.square(tf.subtract(self.percept_x_out, self.percept_x_true))) 634 | self.Loss_vae1_logits = tf.reduce_mean(tf.square(tf.subtract(self.logits_x_out, self.logits_x_true))) 635 | 636 | 637 | 638 | #For encode 639 | self.encode1_loss = 1*self.kl1_loss + 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept 640 | 641 | #for Gen 642 | self.G1_loss = 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept 643 | 644 | 645 | t_vars = tf.trainable_variables() 646 | 647 | self.log_vars.append(("encode1_loss", self.encode1_loss)) 648 | self.log_vars.append(("generator1_loss", self.G1_loss)) 649 | 650 | 651 | 652 | self.g1_vars = [var for var in t_vars if 'VAE_gen1' in var.name] 653 | self.e1_vars = [var for var in t_vars if 'VAE_e1_' in var.name] 654 | 655 | 656 | self.saver = tf.train.Saver() 657 | for k, v in self.log_vars: 658 | tf.summary.scalar(k, v) 659 | 660 | print('Model is Built') 661 | 662 | 663 | 664 | 665 | 666 | #do train 667 | def train(self): 668 | 669 | global_step = tf.Variable(0, trainable=False) 670 | add_global = global_step.assign_add(1) 671 | new_learning_rate = tf.train.exponential_decay(self.learn_rate_init, global_step=global_step, decay_steps=10000, 672 | decay_rate=0.98) 673 | 674 | 675 | 676 | 677 | #for G1 678 | trainer_G1 = tf.train.RMSPropOptimizer(learning_rate=new_learning_rate) 679 | #trainer_G1 = tf.train.RMSPropOptimizer(learning_rate=self.learn_rate_init) 680 | #trainer_G1 = tf.train.AdamOptimizer(learning_rate=new_learning_rate) 681 | gradients_G1 = trainer_G1.compute_gradients(self.G1_loss, var_list=self.g1_vars) 682 | opti_G1 = trainer_G1.apply_gradients(gradients_G1) 683 | 684 | 685 | 686 | #for E1 687 | trainer_E1 = tf.train.RMSPropOptimizer(learning_rate=new_learning_rate) 688 | #trainer_E1 = tf.train.RMSPropOptimizer(learning_rate=self.learn_rate_init) 689 | #trainer_E1 = tf.train.AdamOptimizer(learning_rate=new_learning_rate) 690 | gradients_E1 = trainer_E1.compute_gradients(self.encode1_loss, var_list=self.e1_vars) 691 | opti_E1 = trainer_E1.apply_gradients(gradients_E1) 692 | 693 | 694 | 695 | 696 | init = tf.global_variables_initializer() 697 | config = tf.ConfigProto() 698 | config.gpu_options.allow_growth = True 699 | with tf.Session(config=config) as sess: 700 | 701 | #changed restoring of weights. 702 | ckpt = tf.train.get_checkpoint_state('/home/manu_kohli/vae_classifier_weights/Classifier/checkpoint') 703 | ckpt_path = ckpt.model_checkpoint_path 704 | sess.run(init) 705 | self.saver.restore(sess , self.saved_model_path) 706 | #print(tf.trainable_variables(),'tf.trainable_variables()') 707 | #saver = tf.train.Saver([var for var in tf.trainable_variables() if var.name.startswith('model1')]) 708 | #print(ckpt_path) 709 | #saver.restore(sess, ckpt_path) 710 | 711 | ##self.saver.save(sess , self.saved_model_path) 712 | 713 | print('Creating a Replica of s1 onto s2') 714 | s1_vars1 = [var.name for var in tf.trainable_variables() if 'model1' in var.name] 715 | s2_vars1 = [var for var in tf.trainable_variables() if 'model2' in var.name] 716 | dictionary = {} 717 | for i in range(len(s2_vars1)): 718 | dictionary[s1_vars1[i][0:-2]] = s2_vars1[i] 719 | saver_new = tf.train.Saver(var_list=dictionary) 720 | #saver_new.restore(sess, ckpt_path) 721 | 722 | 723 | ##self.saver.save(sess , ckpt.model_checkpoint_path) 724 | 725 | 726 | print('******************') 727 | print(' ') 728 | print(' ') 729 | print('Plain VAE Training Begins') 730 | print(' ') 731 | print(' ') 732 | print('******************') 733 | 734 | step = 0 735 | g_acc=87.0 736 | batchNum = 0 737 | step=0 738 | while step <= 100000: 739 | next_x_images = self.X_Real_Train[batchNum*self.batch_size:(batchNum+1)*self.batch_size] 740 | next_y_images = self.Y_train[batchNum*self.batch_size:(batchNum+1)*self.batch_size] 741 | batchNum = batchNum +1 742 | #print(batchNum*self.batch_size) 743 | if(((batchNum+1)%170)==0): 744 | idx = np.random.permutation(len(self.X_Real_Train)) 745 | self.X_Real_Train,self.Y_train = self.X_Real_Train[idx], self.Y_train[idx] 746 | batchNum = 0 747 | print('data exhausted') 748 | #print(idx) 749 | #print(self.X_Real_Train.shape, self.Y_train.shape) 750 | #print(batchNum) 751 | #print(next_y_images) 752 | fd ={self.keep_prob:1, self.x_input: next_x_images, self.x_true: next_x_images, self.labels: next_y_images} 753 | sess.run(opti_E1, feed_dict=fd) 754 | sess.run(opti_G1, feed_dict=fd) 755 | 756 | 757 | 758 | new_learn_rate = sess.run(new_learning_rate) 759 | 760 | if new_learn_rate > 0.00005: 761 | sess.run(add_global) 762 | 763 | 764 | if np.mod(step , 100) == 0 and step != 0: 765 | # for iter in range(200): 766 | # print('step', step) 767 | #print('model saved: ', self.saved_model_path) 768 | #self.saver.save(sess , self.saved_model_path, global_step=step) 769 | 770 | print('lr:', new_learn_rate) 771 | k1, e1, l11, l12, l13, cl, g1 = sess.run([self.kl1_loss , self.encode1_loss,self.Loss_vae1_pixel,self.Loss_vae1_percept, self.Loss_vae1_logits,self.cl_loss_x_out,self.G1_loss],feed_dict=fd) 772 | print('E1_loss_KL_Loss: ',k1) 773 | print('E1_loss_Total: ', e1) 774 | 775 | print('G1_loss_MSE: ', l11, 10*l11) 776 | print('G1_loss_Percept: ', l12, 0*l12) 777 | print('G1_loss_Logits: ', l13, 0*l13) 778 | print('G1_loss_CL: ', cl, 1*cl) 779 | print('G1_loss_Total: ', g1) 780 | 781 | Preddiction = np.zeros([self.TestDataSize_cam3,11]) 782 | for i in range(np.int(self.TestDataSize_cam3/self.batch_size)): 783 | next_x_images = self.X_Real_Test_cam3[i*self.batch_size:(i+1)*self.batch_size] 784 | pred = sess.run(self.pred_x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1}) 785 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 786 | x_filt = sess.run(self.x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1}) 787 | x_filt_percept = sess.run(self.percept_x_out, feed_dict={self.x_input: next_x_images, self.keep_prob:1}) 788 | print('shape:', x_filt_percept.shape) 789 | if (step == 100): 790 | np.save('Data/x_cam3_test.npy',next_x_images) 791 | name = 'Data/x_filt__' + str(step) + '_.npy' 792 | np.save(name,x_filt) 793 | # print('Full Filtered Real Train Example Acc = ',getAcc(Preddiction[0:150*64], self.Y_test_cam3[0:150*64])) 794 | # print('Full Filtered Real Test Example Acc = ',getAcc(Preddiction[150*64:], self.Y_test_cam3[150*64:])) 795 | accs,l_acc = getAcc(Preddiction, self.Y_test_cam3) 796 | print('Full Filtered Real Test Example Acc = ',accs,l_acc) 797 | if(l_acc>g_acc): 798 | print('model saved: ', '/home/manu_kohli/vae_classifier_weights/VAE/itr_model_2/model.cpkt') 799 | self.saver.save(sess , '/home/manu_kohli/vae_classifier_weights/VAE/itr_model_2/model.cpkt', global_step=step) 800 | g_acc= l_acc 801 | 802 | Preddiction = np.zeros([self.TrainDataSize,11]) 803 | for i in range(np.int(self.TrainDataSize/self.batch_size)): 804 | next_x_images = self.X_Real_Train[i*self.batch_size:(i+1)*self.batch_size] 805 | pred = sess.run(self.pred_x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1}) 806 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 807 | print('Full Filtered Real Train Example Acc = ',getAcc(Preddiction, self.Y_train)) 808 | if (step == 100): 809 | np.save('Data/x_cam3_train.npy',next_x_images) 810 | 811 | Preddiction = np.zeros([self.TestDataSize,11]) 812 | for i in range(np.int(self.TestDataSize/self.batch_size)): 813 | next_x_images = self.X_Real_Test[i*self.batch_size:(i+1)*self.batch_size] 814 | pred = sess.run(self.pred_x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1}) 815 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 816 | 817 | print('Full Filtered Real Cam2 Example Acc = ',getAcc(Preddiction, self.Y_test)) 818 | if (step == 100): 819 | np.save('Data/x_cam2.npy',next_x_images) 820 | 821 | Preddiction = np.zeros([self.TestDataSize,11]) 822 | for i in range(np.int(self.TestDataSize/self.batch_size)): 823 | next_x_images = self.X_Real_Test[i*self.batch_size:(i+1)*self.batch_size] 824 | pred = sess.run(self.pred_x_true, feed_dict={self.x_true: next_x_images, self.keep_prob:1}) 825 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 826 | print('Full Real Cam2 Example Acc = ',getAcc(Preddiction, self.Y_test)) 827 | 828 | Preddiction = np.zeros([self.TestDataSize_cam3,11]) 829 | for i in range(np.int(self.TestDataSize_cam3/self.batch_size)): 830 | next_x_images = self.X_Real_Test_cam3[i*self.batch_size:(i+1)*self.batch_size] 831 | pred = sess.run(self.pred_x_true, feed_dict={self.x_true: next_x_images, self.keep_prob:1}) 832 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 833 | 834 | print('Full Real Test Example Acc = ',getAcc(Preddiction, self.Y_test_cam3)) 835 | 836 | Preddiction = np.zeros([self.TrainDataSize,11]) 837 | for i in range(np.int(self.TrainDataSize/self.batch_size)): 838 | next_x_images = self.X_Real_Train[i*self.batch_size:(i+1)*self.batch_size] 839 | pred = sess.run(self.pred_x_true, feed_dict={self.x_true: next_x_images, self.keep_prob:1}) 840 | Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11]) 841 | 842 | print('Full Real Train Example Acc = ',getAcc(Preddiction, self.Y_train)) 843 | 844 | # print('Full Filtered Real Train Example Acc = ',getAcc(Preddiction[0:150*64], self.Y_test_cam3[0:150*64])) 845 | # print('Full Filtered Real Test Example Acc = ',getAcc(Preddiction[150*64:], self.Y_test_cam3[150*64:])) 846 | 847 | 848 | step += 1 849 | 850 | def generate1(self, edge, z_var, reuse=False): 851 | 852 | with tf.variable_scope('generator1') as scope: 853 | 854 | if reuse == True: 855 | scope.reuse_variables() 856 | 857 | d1 = lrelu(fully_connect(z_var , output_size=64*4*4, scope='VAE_gen1_fully1')) 858 | d2 = lrelu(fully_connect(d1 , output_size=128*4*4, scope='VAE_gen1_fully2')) 859 | d3 = tf.reshape(d2, [self.batch_size, 4, 4, 128]) 860 | d4 = lrelu(de_conv(d3, output_shape=[self.batch_size, 8, 8, 128], k_h=3, k_w=3,name='VAE_gen1_deconv1')) 861 | d5 = lrelu(de_conv(d4, output_shape=[self.batch_size, 16, 16, 128], k_h=3, k_w=3,name='VAE_gen1_deconv2')) 862 | d6 = lrelu(de_conv(d5, output_shape=[self.batch_size, 32, 32, 128], k_h=3, k_w=3,name='VAE_gen1_deconv3')) 863 | d7 = lrelu(de_conv(d6, output_shape=[self.batch_size, 64, 64, 128], k_h=3, k_w=3,name='VAE_gen1_deconv4')) 864 | d8 = de_conv(d7, output_shape=[self.batch_size, 128, 128, 3] , k_h=3, k_w=3, name='VAE_gen1_deconv5') 865 | d9 = tf.nn.tanh(d8) 866 | d10 = tf.concat([d9, edge], 3) 867 | conv1 = lrelu(conv2d(d10, output_dim=128, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c1')) 868 | conv2 = lrelu(conv2d(conv1, output_dim=128, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c2')) 869 | conv3 = conv2d(conv2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c3') 870 | 871 | 872 | return tf.nn.tanh(conv3) 873 | 874 | 875 | 876 | def Encode1(self, x, reuse=False): 877 | 878 | with tf.variable_scope('encode1') as scope: 879 | 880 | if reuse == True: 881 | scope.reuse_variables() 882 | conv1 = lrelu(conv2d(x, output_dim=128, k_h=3, k_w=3, name='VAE_e1_c1')) 883 | conv2 = lrelu(conv2d(conv1, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c2')) 884 | conv3 = lrelu(conv2d(conv2, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c3')) 885 | conv4 = lrelu(conv2d(conv3, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c4')) 886 | conv5 = lrelu(conv2d(conv4, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c5')) 887 | conv6 = tf.reshape(conv5, [self.batch_size, 128 * 4 * 4]) 888 | fc1 = lrelu(fully_connect(conv6, output_size= 64*4*4, scope='VAE_e1_f1')) 889 | z_mean = fully_connect(fc1, output_size=self.latent_dim, scope='VAE_e1_f2') 890 | z_sigma = fully_connect(fc1, output_size=self.latent_dim, scope='VAE_e1_f3') 891 | return z_mean, z_sigma 892 | 893 | 894 | def KL_loss(self, mu, log_var): 895 | return -0.5 * tf.reduce_sum(1 + log_var - tf.pow(mu, 2) - tf.exp(log_var)) 896 | 897 | def sample_z(self, mu, log_var): 898 | eps = tf.random_normal(shape=tf.shape(mu)) 899 | return mu + tf.exp(log_var / 2) * eps 900 | 901 | 902 | def NLLNormal(self, pred, target): 903 | 904 | c = -0.5 * tf.log(2 * np.pi) 905 | multiplier = 1.0 / (2.0 * 1) 906 | tmp = tf.square(pred - target) 907 | tmp *= -multiplier 908 | tmp += c 909 | 910 | return tmp 911 | 912 | 913 | flags = tf.app.flags 914 | 915 | flags.DEFINE_integer("batch_size" , 64, "batch size") 916 | flags.DEFINE_integer("max_iters" , 10000, "the maxmization epoch") 917 | flags.DEFINE_integer("latent_dim" , 64, "the dim of latent code") 918 | flags.DEFINE_float("learn_rate_init" , 0.0001, "the init of learn rate") 919 | flags.DEFINE_integer("repeat", 10000, "the numbers of repeat for your datasets") 920 | flags.DEFINE_string("path", '/home/?/data/', "for example, '/home/jack/data/' is the directory of your celebA data") 921 | flags.DEFINE_integer("op", 0, "Training or Test") 922 | 923 | FLAGS = flags.FLAGS 924 | FLAGS.op = 0 925 | if (1): 926 | path123 = '.' 927 | root_log_dir = path123 + "/log_dir" 928 | vaegan_checkpoint_dir = "/home/manu_kohli/vae_classifier_weights/VAE/itr_model_1/model.cpkt-5400" 929 | sample_path = path123 + "/sample" 930 | 931 | 932 | model_path = vaegan_checkpoint_dir 933 | 934 | batch_size = FLAGS.batch_size 935 | max_iters = FLAGS.max_iters 936 | latent_dim = FLAGS.latent_dim 937 | data_repeat = FLAGS.repeat 938 | 939 | learn_rate_init = FLAGS.learn_rate_init 940 | #learn_rate_init= 9e-5 941 | vaeGan = vaegan(batch_size= batch_size, max_iters= max_iters, repeat = data_repeat, 942 | model_path= model_path, latent_dim= latent_dim, 943 | sample_path= sample_path , log_dir= root_log_dir , learnrate_init= learn_rate_init) 944 | 945 | vaeGan.build_model_vaegan() 946 | vaeGan.train() 947 | 948 | 949 | --------------------------------------------------------------------------------