├── CycleGAN ├── __init__.py ├── cycle_ops.py ├── discriminator.py ├── export_graph.py ├── gen_image.py ├── generator.py ├── model.py ├── reader.py ├── train_cyclegan.py └── utils.py ├── WGAN.py ├── emotion ├── emotion_data.py ├── emotion_evaluate.py ├── emotion_model.py ├── emotion_tfr.py ├── emotion_train.py └── manifold.py ├── evaluate_one_image.py ├── face_detect ├── FaceDetection.py └── haarcascade_frontalface_alt.xml ├── ops.py ├── train_wgan.py └── utils.py /CycleGAN/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午4:25 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : cycle_ops.py.py 6 | # @Software : PyCharm -------------------------------------------------------------------------------- /CycleGAN/cycle_ops.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午4:25 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : cycle_ops.py.py 6 | # @Software : PyCharm 7 | 8 | import tensorflow as tf 9 | 10 | ## Layers: follow the naming convention used in the original paper 11 | ### Generator layers 12 | def c7s1_k(input, k, reuse=False, norm='instance', activation='relu', is_training=True, name='c7s1_k'): 13 | """ A 7x7 Convolution-BatchNorm-ReLU layer with k filters and stride 1 14 | Args: 15 | input: 4D tensor 16 | k: integer, number of filters (output depth) 17 | norm: 'instance' or 'batch' or None 18 | activation: 'relu' or 'tanh' 19 | name: string, e.g. 'c7sk-32' 20 | is_training: boolean or BoolTensor 21 | name: string 22 | reuse: boolean 23 | Returns: 24 | 4D tensor 25 | """ 26 | with tf.variable_scope(name, reuse=reuse): 27 | weights = _weights("weights", 28 | shape=[7, 7, input.get_shape()[3], k]) 29 | 30 | padded = tf.pad(input, [[0,0],[3,3],[3,3],[0,0]], 'REFLECT') 31 | conv = tf.nn.conv2d(padded, weights, 32 | strides=[1, 1, 1, 1], padding='VALID') 33 | 34 | normalized = _norm(conv, is_training, norm) 35 | 36 | if activation == 'relu': 37 | output = tf.nn.relu(normalized) 38 | if activation == 'tanh': 39 | output = tf.nn.tanh(normalized) 40 | return output 41 | 42 | def dk(input, k, reuse=False, norm='instance', is_training=True, name=None): 43 | """ A 3x3 Convolution-BatchNorm-ReLU layer with k filters and stride 2 44 | Args: 45 | input: 4D tensor 46 | k: integer, number of filters (output depth) 47 | norm: 'instance' or 'batch' or None 48 | is_training: boolean or BoolTensor 49 | name: string 50 | reuse: boolean 51 | name: string, e.g. 'd64' 52 | Returns: 53 | 4D tensor 54 | """ 55 | with tf.variable_scope(name, reuse=reuse): 56 | weights = _weights("weights", 57 | shape=[3, 3, input.get_shape()[3], k]) 58 | 59 | conv = tf.nn.conv2d(input, weights, 60 | strides=[1, 2, 2, 1], padding='SAME') 61 | normalized = _norm(conv, is_training, norm) 62 | output = tf.nn.relu(normalized) 63 | return output 64 | 65 | def Rk(input, k, reuse=False, norm='instance', is_training=True, name=None): 66 | """ A residual block that contains two 3x3 convolutional layers 67 | with the same number of filters on both layer 68 | Args: 69 | input: 4D Tensor 70 | k: integer, number of filters (output depth) 71 | reuse: boolean 72 | name: string 73 | Returns: 74 | 4D tensor (same shape as input) 75 | """ 76 | with tf.variable_scope(name, reuse=reuse): 77 | with tf.variable_scope('layer1', reuse=reuse): 78 | weights1 = _weights("weights1", 79 | shape=[3, 3, input.get_shape()[3], k]) 80 | padded1 = tf.pad(input, [[0,0],[1,1],[1,1],[0,0]], 'REFLECT') 81 | conv1 = tf.nn.conv2d(padded1, weights1, 82 | strides=[1, 1, 1, 1], padding='VALID') 83 | normalized1 = _norm(conv1, is_training, norm) 84 | relu1 = tf.nn.relu(normalized1) 85 | 86 | with tf.variable_scope('layer2', reuse=reuse): 87 | weights2 = _weights("weights2", 88 | shape=[3, 3, relu1.get_shape()[3], k]) 89 | 90 | padded2 = tf.pad(relu1, [[0,0],[1,1],[1,1],[0,0]], 'REFLECT') 91 | conv2 = tf.nn.conv2d(padded2, weights2, 92 | strides=[1, 1, 1, 1], padding='VALID') 93 | normalized2 = _norm(conv2, is_training, norm) 94 | output = input+normalized2 95 | return output 96 | 97 | def n_res_blocks(input, reuse, norm='instance', is_training=True, n=6): 98 | depth = input.get_shape()[3] 99 | for i in range(1,n+1): 100 | output = Rk(input, depth, reuse, norm, is_training, 'R{}_{}'.format(depth, i)) 101 | input = output 102 | return output 103 | 104 | def uk(input, k, reuse=False, norm='instance', is_training=True, name=None, output_size=None): 105 | """ A 3x3 fractional-strided-Convolution-BatchNorm-ReLU layer 106 | with k filters, stride 1/2 107 | Args: 108 | input: 4D tensor 109 | k: integer, number of filters (output depth) 110 | norm: 'instance' or 'batch' or None 111 | is_training: boolean or BoolTensor 112 | reuse: boolean 113 | name: string, e.g. 'c7sk-32' 114 | output_size: integer, desired output size of layer 115 | Returns: 116 | 4D tensor 117 | """ 118 | with tf.variable_scope(name, reuse=reuse): 119 | input_shape = input.get_shape().as_list() 120 | 121 | weights = _weights("weights", 122 | shape=[3, 3, k, input_shape[3]]) 123 | 124 | if not output_size: 125 | output_size = input_shape[1]*2 126 | output_shape = [input_shape[0], output_size, output_size, k] 127 | fsconv = tf.nn.conv2d_transpose(input, weights, 128 | output_shape=output_shape, 129 | strides=[1, 2, 2, 1], padding='SAME') 130 | normalized = _norm(fsconv, is_training, norm) 131 | output = tf.nn.relu(normalized) 132 | return output 133 | 134 | ### Discriminator layers 135 | def Ck(input, k, slope=0.2, stride=2, reuse=False, norm='instance', is_training=True, name=None): 136 | """ A 4x4 Convolution-BatchNorm-LeakyReLU layer with k filters and stride 2 137 | Args: 138 | input: 4D tensor 139 | k: integer, number of filters (output depth) 140 | slope: LeakyReLU's slope 141 | stride: integer 142 | norm: 'instance' or 'batch' or None 143 | is_training: boolean or BoolTensor 144 | reuse: boolean 145 | name: string, e.g. 'C64' 146 | Returns: 147 | 4D tensor 148 | """ 149 | with tf.variable_scope(name, reuse=reuse): 150 | weights = _weights("weights", 151 | shape=[4, 4, input.get_shape()[3], k]) 152 | 153 | conv = tf.nn.conv2d(input, weights, 154 | strides=[1, stride, stride, 1], padding='SAME') 155 | 156 | normalized = _norm(conv, is_training, norm) 157 | output = _leaky_relu(normalized, slope) 158 | return output 159 | 160 | def last_conv(input, reuse=False, use_sigmoid=False, name=None): 161 | """ Last convolutional layer of discriminator network 162 | (1 filter with size 4x4, stride 1) 163 | Args: 164 | input: 4D tensor 165 | reuse: boolean 166 | use_sigmoid: boolean (False if use lsgan) 167 | name: string, e.g. 'C64' 168 | """ 169 | with tf.variable_scope(name, reuse=reuse): 170 | weights = _weights("weights", 171 | shape=[4, 4, input.get_shape()[3], 1]) 172 | biases = _biases("biases", [1]) 173 | 174 | conv = tf.nn.conv2d(input, weights, 175 | strides=[1, 1, 1, 1], padding='SAME') 176 | output = conv + biases 177 | if use_sigmoid: 178 | output = tf.sigmoid(output) 179 | return output 180 | 181 | ### Helpers 182 | def _weights(name, shape, mean=0.0, stddev=0.02): 183 | """ Helper to create an initialized Variable 184 | Args: 185 | name: name of the variable 186 | shape: list of ints 187 | mean: mean of a Gaussian 188 | stddev: standard deviation of a Gaussian 189 | Returns: 190 | A trainable variable 191 | """ 192 | var = tf.get_variable( 193 | name, shape, 194 | initializer=tf.random_normal_initializer( 195 | mean=mean, stddev=stddev, dtype=tf.float32)) 196 | return var 197 | 198 | def _biases(name, shape, constant=0.0): 199 | """ Helper to create an initialized Bias with constant 200 | """ 201 | return tf.get_variable(name, shape, 202 | initializer=tf.constant_initializer(constant)) 203 | 204 | def _leaky_relu(input, slope): 205 | return tf.maximum(slope*input, input) 206 | 207 | def _norm(input, is_training, norm='instance'): 208 | """ Use Instance Normalization or Batch Normalization or None 209 | """ 210 | if norm == 'instance': 211 | return _instance_norm(input) 212 | elif norm == 'batch': 213 | return _batch_norm(input, is_training) 214 | else: 215 | return input 216 | 217 | def _batch_norm(input, is_training): 218 | """ Batch Normalization 219 | """ 220 | with tf.variable_scope("batch_norm"): 221 | return tf.contrib.layers.batch_norm(input, 222 | decay=0.9, 223 | scale=True, 224 | updates_collections=None, 225 | is_training=is_training) 226 | 227 | def _instance_norm(input): 228 | """ Instance Normalization 229 | """ 230 | with tf.variable_scope("instance_norm"): 231 | depth = input.get_shape()[3] 232 | scale = _weights("scale", [depth], mean=1.0) 233 | offset = _biases("offset", [depth]) 234 | mean, variance = tf.nn.moments(input, axes=[1,2], keep_dims=True) 235 | epsilon = 1e-5 236 | inv = tf.rsqrt(variance + epsilon) 237 | normalized = (input-mean)*inv 238 | return scale*normalized + offset 239 | 240 | def safe_log(x, eps=1e-12): 241 | return tf.log(x + eps) 242 | -------------------------------------------------------------------------------- /CycleGAN/discriminator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午4:27 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : discriminator.py 6 | # @Software : PyCharm 7 | 8 | import tensorflow as tf 9 | import CycleGAN.cycle_ops as ops 10 | 11 | class Discriminator: 12 | def __init__(self, name, is_training, norm='instance', use_sigmoid=False): 13 | self.name = name 14 | self.is_training = is_training 15 | self.norm = norm 16 | self.reuse = False 17 | self.use_sigmoid = use_sigmoid 18 | 19 | def __call__(self, input): 20 | """ 21 | Args: 22 | input: batch_size x image_size x image_size x 3 23 | Returns: 24 | output: 4D tensor batch_size x out_size x out_size x 1 (default 1x5x5x1) 25 | filled with 0.9 if real, 0.0 if fake 26 | """ 27 | with tf.variable_scope(self.name): 28 | # convolution layers 29 | C64 = ops.Ck(input, 64, reuse=self.reuse, norm=None, 30 | is_training=self.is_training, name='C64') # (?, w/2, h/2, 64) 31 | C128 = ops.Ck(C64, 128, reuse=self.reuse, norm=self.norm, 32 | is_training=self.is_training, name='C128') # (?, w/4, h/4, 128) 33 | C256 = ops.Ck(C128, 256, reuse=self.reuse, norm=self.norm, 34 | is_training=self.is_training, name='C256') # (?, w/8, h/8, 256) 35 | C512 = ops.Ck(C256, 512,reuse=self.reuse, norm=self.norm, 36 | is_training=self.is_training, name='C512') # (?, w/16, h/16, 512) 37 | 38 | # apply a convolution to produce a 1 dimensional output (1 channel?) 39 | # use_sigmoid = False if use_lsgan = True 40 | output = ops.last_conv(C512, reuse=self.reuse, 41 | use_sigmoid=self.use_sigmoid, name='output') # (?, w/16, h/16, 1) 42 | 43 | self.reuse = True 44 | self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) 45 | 46 | return output 47 | -------------------------------------------------------------------------------- /CycleGAN/export_graph.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/19 上午10:19 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : export_graph.py.py 6 | # @Software : PyCharm 7 | 8 | """ Freeze variables and convert 2 generator networks to 2 GraphDef files. 9 | This makes file size smaller and can be used for inference in production. 10 | An example of command-line usage is: 11 | python export_graph.py --checkpoint_dir checkpoints/20171018-1457 \ 12 | --XtoY_model apple2orange.pb \ 13 | --YtoX_model orange2apple.pb \ 14 | --image_size 48 15 | """ 16 | 17 | import sys 18 | import tensorflow as tf 19 | import os 20 | from tensorflow.python.tools.freeze_graph import freeze_graph 21 | from CycleGAN.model import CycleGAN as CG 22 | import CycleGAN.utils 23 | 24 | FLAGS = tf.flags.FLAGS 25 | 26 | tf.flags.DEFINE_string('checkpoint_dir', 'checkpoints/20171018-1457/', 'checkpoints directory path') 27 | tf.flags.DEFINE_string('XtoY_model', 'face2emotion.pb', 'XtoY model name, default: apple2orange.pb') 28 | tf.flags.DEFINE_string('YtoX_model', 'emotion2face.pb', 'YtoX model name, default: orange2apple.pb') 29 | tf.flags.DEFINE_integer('image_size', '48', 'image size, default: 256') 30 | tf.flags.DEFINE_integer('ngf', 64, 31 | 'number of gen filters in first conv layer, default: 64') 32 | tf.flags.DEFINE_string('norm', 'instance', 33 | '[instance, batch] use instance norm or batch norm, default: instance') 34 | 35 | def export_graph(model_name, XtoY=True): 36 | graph = tf.Graph() 37 | 38 | with graph.as_default(): 39 | # cycle_gan = CycleGAN(ngf=FLAGS.ngf, norm=FLAGS.norm, image_size=FLAGS.image_size) 40 | cycle_gan = CG( 41 | X_train_file='', 42 | Y_train_file='', 43 | dataset_name='face_emotion', 44 | batch_size=1) 45 | 46 | input_image = tf.placeholder(tf.float32, shape=[FLAGS.image_size, FLAGS.image_size, 1], name='input_image') 47 | cycle_gan.model() 48 | 49 | if XtoY: 50 | output_image = cycle_gan.G.sample(tf.expand_dims(input_image, 0)) 51 | else: 52 | output_image = cycle_gan.F.sample(tf.expand_dims(input_image, 0)) 53 | 54 | print(output_image) 55 | output_image = tf.identity(output_image, name='output_image') 56 | restore_saver = tf.train.Saver() 57 | # export_saver = tf.train.Saver() 58 | 59 | with tf.Session(graph=graph) as sess: 60 | sess.run(tf.global_variables_initializer()) 61 | latest_ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) 62 | restore_saver.restore(sess, latest_ckpt) 63 | output_graph_def = tf.graph_util.convert_variables_to_constants( 64 | sess, graph.as_graph_def(), [output_image.op.name]) 65 | 66 | tf.train.write_graph(output_graph_def, 'pretrained', model_name, as_text=False) 67 | 68 | def main(unused_argv): 69 | print('Export XtoY model...') 70 | export_graph(FLAGS.XtoY_model, XtoY=True) 71 | # print('Export YtoX model...') 72 | # export_graph(FLAGS.YtoX_model, XtoY=False) 73 | 74 | if __name__ == '__main__': 75 | tf.app.run() 76 | -------------------------------------------------------------------------------- /CycleGAN/gen_image.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/19 上午10:31 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : gen_image.py 6 | # @Software : PyCharm 7 | 8 | """Translate an image to another image 9 | An example of command-line usage is: 10 | python export_graph.py --model pretrained/face2emotion.pb \ 11 | --input input_sample.jpg \ 12 | --output output_sample.jpg \ 13 | --image_size 48 14 | """ 15 | import sys 16 | import tensorflow as tf 17 | import numpy as np 18 | import os 19 | from CycleGAN.model import CycleGAN as CG 20 | import CycleGAN.utils as utils 21 | from scipy import misc 22 | from emotion.emotion_data import read_train_sets 23 | 24 | FLAGS = tf.flags.FLAGS 25 | 26 | # tf.flags.DEFINE_string('model', 'pretrained/face2emotion.pb', 'model path (.pb)') 27 | tf.flags.DEFINE_string('input', 'image_0001.jpg', 'input image path (.jpg)') 28 | tf.flags.DEFINE_string('inputdir', '/Users/zhuxinyue/ML/face_emotion/', 'input image dir') 29 | # tf.flags.DEFINE_string('output', 'output_sample.jpg', 'output image path (.jpg)') 30 | tf.flags.DEFINE_integer('image_size', '48', 'image size, default: 256') 31 | os.mkdir('/Users/zhuxinyue/ML/gen_CG_surprise72w/') 32 | tf.flags.DEFINE_string('checkpoint_dir', 'checkpoints/surprise-1026/', 'checkpoints directory path') 33 | tf.flags.DEFINE_string('result_dir', '/Users/zhuxinyue/ML/gen_CG_surprise72w/', 'checkpoints directory path') 34 | tf.flags.DEFINE_integer('ngf', 64, 35 | 'number of gen filters in first conv layer, default: 64') 36 | tf.flags.DEFINE_string('norm', 'instance', 37 | '[instance, batch] use instance norm or batch norm, default: instance') 38 | 39 | import matplotlib.pyplot as plt 40 | def display_image(image, gray=False): 41 | dis_image = image.astype(np.uint8) 42 | plt.figure() 43 | 44 | if gray: 45 | plt.imshow(dis_image, cmap='gray') 46 | else: 47 | plt.imshow(dis_image) 48 | 49 | 50 | def restore_gen(): 51 | '''test single image''' 52 | # image = misc.imread(FLAGS.input, mode='L') 53 | # print(image.shape) 54 | # image = misc.imresize(image, [FLAGS.image_size, FLAGS.image_size]) 55 | # print(image.shape) 56 | # image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, -1]) 57 | # print(image.shape) 58 | # image = image / 255 59 | 60 | with tf.Graph().as_default(): 61 | with tf.Session() as sess: 62 | 63 | init_op = tf.group(tf.global_variables_initializer(), 64 | tf.local_variables_initializer()) 65 | sess.run(init_op) 66 | 67 | cycle_gan = CG( 68 | X_train_file='', 69 | Y_train_file='', 70 | dataset_name='face_emotion', 71 | batch_size=1) 72 | 73 | input_image = tf.placeholder(tf.float32, shape=[1, FLAGS.image_size, FLAGS.image_size, 1], name='input_image') 74 | cycle_gan.model() 75 | 76 | saver = tf.train.Saver() 77 | print("Reading checkpoints...") 78 | ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) 79 | if ckpt and ckpt.model_checkpoint_path: 80 | global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] 81 | saver.restore(sess, ckpt.model_checkpoint_path) 82 | print("successful loading,global step is %s" % global_step) 83 | else: 84 | print("no checkpoint file founded") 85 | return 86 | 87 | # _, face_images = load_caltech101(FLAGS.inputdir, FLAGS.image_size) 88 | classes = ['neutral'] 89 | face_images, lables, _, _, _, _ = read_train_sets(FLAGS.inputdir, classes, 0, 5000) 90 | print(face_images.shape) 91 | gen_num = face_images.shape[0] 92 | print(gen_num) 93 | 94 | for i in range(gen_num): 95 | print('Generating...', i) 96 | face_image = np.reshape(face_images[i], [1, FLAGS.image_size, FLAGS.image_size, 1]) 97 | save_path = FLAGS.result_dir + 'genimg_' + str(i) + '.jpg' 98 | 99 | gen_image = cycle_gan.G.__call__(input_image) 100 | 101 | output = sess.run(gen_image, feed_dict={input_image: face_image}) 102 | # print(output) 103 | output = np.reshape(output, [FLAGS.image_size, FLAGS.image_size]) 104 | output = output * 255 105 | print(output.shape) 106 | # display_image(output, gray=True) 107 | # plt.show() 108 | 109 | misc.toimage(output).save(save_path) 110 | 111 | # display fake image 112 | # display_image(gen_image) 113 | # plt.show() 114 | # sys.exit() 115 | sess.close() 116 | 117 | def main(unused_argv): 118 | restore_gen() 119 | 120 | if __name__ == '__main__': 121 | tf.app.run() 122 | -------------------------------------------------------------------------------- /CycleGAN/generator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午4:24 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : generator.py 6 | # @Software : PyCharm 7 | 8 | import tensorflow as tf 9 | import CycleGAN.cycle_ops as ops 10 | import CycleGAN.utils as utils 11 | 12 | class Generator: 13 | def __init__(self, name, is_training, ngf=64, norm='instance', image_size=128): 14 | self.name = name 15 | self.reuse = False 16 | self.ngf = ngf 17 | self.norm = norm 18 | self.is_training = is_training 19 | self.image_size = image_size 20 | 21 | def __call__(self, input): 22 | """ 23 | Args: 24 | input: batch_size x width x height x 3 25 | Returns: 26 | output: same size as input 27 | """ 28 | with tf.variable_scope(self.name): 29 | # conv layers 30 | c7s1_32 = ops.c7s1_k(input, self.ngf, is_training=self.is_training, norm=self.norm, 31 | reuse=self.reuse, name='c7s1_32') # (?, w, h, 32) 32 | d64 = ops.dk(c7s1_32, 2*self.ngf, is_training=self.is_training, norm=self.norm, 33 | reuse=self.reuse, name='d64') # (?, w/2, h/2, 64) 34 | d128 = ops.dk(d64, 4*self.ngf, is_training=self.is_training, norm=self.norm, 35 | reuse=self.reuse, name='d128') 36 | # (?, w/4, h/4, 128) 37 | 38 | if self.image_size <= 128: 39 | # use 6 residual blocks for 128x128 images 40 | res_output = ops.n_res_blocks(d128, reuse=self.reuse, n=6) # (?, w/4, h/4, 128) 41 | else: 42 | # 9 blocks for higher resolution 43 | res_output = ops.n_res_blocks(d128, reuse=self.reuse, n=9) # (?, w/4, h/4, 128) 44 | 45 | # fractional-strided convolution 46 | u64 = ops.uk(res_output, 2*self.ngf, is_training=self.is_training, norm=self.norm, 47 | reuse=self.reuse, name='u64') # (?, w/2, h/2, 64) 48 | u32 = ops.uk(u64, self.ngf, is_training=self.is_training, norm=self.norm, 49 | reuse=self.reuse, name='u32', output_size=self.image_size) # (?, w, h, 32) 50 | 51 | # conv layer 52 | # Note: the paper said that ReLU and _norm were used 53 | # but actually tanh was used and no _norm here 54 | output = ops.c7s1_k(u32, 1, norm=None, 55 | activation='tanh', reuse=self.reuse, name='output') # (?, w, h, 1) 56 | # set reuse=True for next call 57 | self.reuse = True 58 | self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) 59 | 60 | return output 61 | 62 | 63 | def sample(self, input): 64 | image = utils.batch_convert2int(self.__call__(input)) 65 | image = tf.image.encode_jpeg(tf.squeeze(image, [0])) 66 | return image -------------------------------------------------------------------------------- /CycleGAN/model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午2:47 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : CycleGAN.py 6 | # @Software : PyCharm 7 | 8 | ''' 9 | CycleGAN codes partly come from https://github.com/vanhuyz/CycleGAN-TensorFlow 10 | ''' 11 | 12 | from __future__ import absolute_import 13 | from __future__ import division 14 | from __future__ import print_function 15 | 16 | import os 17 | import sys 18 | import time 19 | import tensorflow as tf 20 | import numpy as np 21 | 22 | import CycleGAN.cycle_ops as ops 23 | from utils import * 24 | from emotion.emotion_data import read_train_sets 25 | from CycleGAN.discriminator import Discriminator 26 | from CycleGAN.generator import Generator 27 | from CycleGAN.reader import Reader 28 | 29 | 30 | REAL_LABEL = 0.9 31 | 32 | class CycleGAN(object): 33 | def __init__(self, X_train_file, Y_train_file, dataset_name, batch_size): 34 | self.dataset_name = dataset_name 35 | self.batch_size = batch_size 36 | self.model_name = "cycleWGAN" # name for checkpoint 37 | self.is_training = tf.placeholder_with_default(True, shape=[], name='is_training') 38 | self.norm = 'instance' 39 | self.X_train_file = '' 40 | self.Y_train_file = '' 41 | 42 | if dataset_name == 'face_emotion': 43 | self.X_train_file = X_train_file 44 | self.Y_train_file = Y_train_file 45 | dataset_dir = '/Users/zhuxinyue/ML/' + dataset_name + '/' 46 | # parameters 47 | self.image_size = 48 48 | 49 | self.c_dim = 1 # dimension of channels? 50 | self.ngf = 64 51 | 52 | # CycleGAN parameter 53 | self.disc_iters = 5 # The number of critic iterations for one-step of generator 54 | 55 | # train 56 | self.learning_rate = 0.0002 57 | self.lambda1 = 10.0, 58 | self.lambda2 = 10.0, 59 | self.beta1 = 0.5 60 | 61 | else: 62 | if dataset_name == 'generate': 63 | self.image_size = 48 64 | self.output_height = 48 65 | self.output_width = 48 66 | 67 | self.c_dim = 1 # dimension of channels? 68 | 69 | # WGAN parameter 70 | self.disc_iters = 5 # The number of critic iterations for one-step of generator 71 | 72 | # train 73 | self.learning_rate = 0.0002 74 | self.beta1 = 0.5 75 | 76 | else: 77 | raise NotImplementedError 78 | 79 | # Generator X --> Y 80 | self.G = Generator('G', self.is_training, ngf=self.ngf, norm=self.norm, image_size=self.image_size) 81 | # Discriminator Y 82 | self.D_Y = Discriminator('D_Y', 83 | self.is_training, norm=self.norm, use_sigmoid=False) 84 | # Generator Y --> X 85 | self.F = Generator('F', self.is_training, norm=self.norm, image_size=self.image_size) 86 | # Discriminator X 87 | self.D_X = Discriminator('D_X', 88 | self.is_training, norm=self.norm, use_sigmoid=False) 89 | 90 | self.fake_x = tf.placeholder(tf.float32, 91 | shape=[batch_size, self.image_size, self.image_size, 1]) 92 | self.fake_y = tf.placeholder(tf.float32, 93 | shape=[batch_size, self.image_size, self.image_size, 1]) 94 | 95 | 96 | def model(self): 97 | X_reader = Reader(self.X_train_file, name='X', 98 | image_size=self.image_size, batch_size=self.batch_size) 99 | Y_reader = Reader(self.Y_train_file, name='Y', 100 | image_size=self.image_size, batch_size=self.batch_size) 101 | 102 | x = X_reader.feed() 103 | y = Y_reader.feed() 104 | 105 | """ Loss Function """ 106 | 107 | cycle_loss = self.cycle_consistency_loss(self.G, self.F, x, y) 108 | print(cycle_loss) 109 | 110 | # X -> Y 111 | fake_y = self.G(x) # __call__(input) 112 | G_gan_loss = self.generator_loss(self.D_Y, fake_y, use_lsgan=True) 113 | print(G_gan_loss) 114 | G_loss = G_gan_loss + cycle_loss 115 | D_Y_loss = self.discriminator_loss(self.D_Y, y, self.fake_y, use_lsgan=True) 116 | 117 | # Y -> X 118 | fake_x = self.F(y) 119 | F_gan_loss = self.generator_loss(self.D_X, fake_x, use_lsgan=True) 120 | F_loss = F_gan_loss + cycle_loss 121 | D_X_loss = self.discriminator_loss(self.D_X, x, self.fake_x, use_lsgan=True) 122 | 123 | # summary 124 | tf.summary.histogram('D_Y/true', self.D_Y(y)) 125 | tf.summary.histogram('D_Y/fake', self.D_Y(self.G(x))) 126 | tf.summary.histogram('D_X/true', self.D_X(x)) 127 | tf.summary.histogram('D_X/fake', self.D_X(self.F(y))) 128 | 129 | tf.summary.scalar('loss/G', G_gan_loss) 130 | tf.summary.scalar('loss/D_Y', D_Y_loss) 131 | tf.summary.scalar('loss/F', F_gan_loss) 132 | tf.summary.scalar('loss/D_X', D_X_loss) 133 | tf.summary.scalar('loss/cycle', cycle_loss) 134 | 135 | return G_loss, D_Y_loss, F_loss, D_X_loss, fake_y, fake_x 136 | 137 | def optimize(self, G_loss, D_Y_loss, F_loss, D_X_loss): 138 | def make_optimizer(loss, variables, name='Adam'): 139 | """ Adam optimizer with learning rate 0.0002 for the first 100k steps (~100 epochs) 140 | and a linearly decaying rate that goes to zero over the next 100k steps 141 | """ 142 | global_step = tf.Variable(0, trainable=False) 143 | starter_learning_rate = self.learning_rate 144 | end_learning_rate = 0.0 145 | start_decay_step = 100000 146 | decay_steps = 100000 147 | beta1 = self.beta1 148 | learning_rate = ( 149 | tf.where( 150 | tf.greater_equal(global_step, start_decay_step), 151 | tf.train.polynomial_decay(starter_learning_rate, global_step - start_decay_step, 152 | decay_steps, end_learning_rate, 153 | power=1.0), 154 | starter_learning_rate 155 | ) 156 | 157 | ) 158 | tf.summary.scalar('learning_rate/{}'.format(name), learning_rate) 159 | 160 | learning_step = ( 161 | tf.train.AdamOptimizer(learning_rate, beta1=beta1, name=name) 162 | .minimize(loss, global_step=global_step, var_list=variables) 163 | ) 164 | return learning_step 165 | 166 | G_optimizer = make_optimizer(G_loss, self.G.variables, name='Adam_G') 167 | D_Y_optimizer = make_optimizer(D_Y_loss, self.D_Y.variables, name='Adam_D_Y') 168 | F_optimizer = make_optimizer(F_loss, self.F.variables, name='Adam_F') 169 | D_X_optimizer = make_optimizer(D_X_loss, self.D_X.variables, name='Adam_D_X') 170 | 171 | with tf.control_dependencies([G_optimizer, D_Y_optimizer, F_optimizer, D_X_optimizer]): 172 | return tf.no_op(name='optimizers') 173 | 174 | '''Loss Function''' 175 | def discriminator_loss(self, D, y, fake_y, use_lsgan=True): 176 | """ Note: default: D(y).shape == (batch_size,5,5,1), 177 | fake_buffer_size=50, batch_size=1 178 | Args: 179 | G: generator object 180 | D: discriminator object 181 | y: 4D tensor (batch_size, image_size, image_size, 3) 182 | Returns: 183 | loss: scalar 184 | """ 185 | if use_lsgan: 186 | # use mean squared error 187 | error_real = tf.reduce_mean(tf.squared_difference(D(y), REAL_LABEL)) 188 | error_fake = tf.reduce_mean(tf.square(D(fake_y))) 189 | else: 190 | # use cross entropy 191 | error_real = -tf.reduce_mean(ops.safe_log(D(y))) 192 | error_fake = -tf.reduce_mean(ops.safe_log(1 - D(fake_y))) 193 | loss = (error_real + error_fake) / 2 194 | return loss 195 | 196 | def generator_loss(self, D, fake_y, use_lsgan=True): 197 | """ fool discriminator into believing that G(x) is real 198 | """ 199 | if use_lsgan: 200 | # use mean squared error 201 | loss = tf.reduce_mean(tf.squared_difference(D(fake_y), REAL_LABEL)) 202 | else: 203 | # heuristic, non-saturating loss 204 | loss = -tf.reduce_mean(ops.safe_log(D(fake_y))) / 2 205 | return loss 206 | 207 | def cycle_consistency_loss(self, G, F, x, y): 208 | """ cycle consistency loss (L1 norm) 209 | """ 210 | forward_loss = tf.reduce_mean(tf.abs(F(G(x)) - x)) 211 | backward_loss = tf.reduce_mean(tf.abs(G(F(y)) - y)) 212 | loss = self.lambda1 * forward_loss + self.lambda2 * backward_loss 213 | loss = tf.reshape(loss, []) 214 | return loss -------------------------------------------------------------------------------- /CycleGAN/reader.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import tensorflow as tf 4 | # import CycleGAN.utils as utils 5 | os.environ['TF_CPP_MIN_LOG_LEVEL']='2' 6 | 7 | class Reader(): 8 | def __init__(self, tfrecords_file, image_size=48, 9 | min_queue_examples=1000, batch_size=1, num_threads=8, name=''): 10 | """ 11 | Args: 12 | tfrecords_file: string, tfrecords file path 13 | min_queue_examples: integer, minimum number of samples to retain in the queue that provides of batches of examples 14 | batch_size: integer, number of images per batch 15 | num_threads: integer, number of preprocess threads 16 | """ 17 | self.tfrecords_file = tfrecords_file 18 | self.image_size = image_size 19 | self.min_queue_examples = min_queue_examples 20 | self.batch_size = batch_size 21 | self.num_threads = num_threads 22 | self.reader = tf.TFRecordReader() 23 | self.name = name 24 | 25 | def feed(self): 26 | """ 27 | Returns: 28 | images: 4D tensor [batch_size, image_width, image_height, image_depth] 29 | """ 30 | with tf.name_scope(self.name): 31 | filename_queue = tf.train.string_input_producer([self.tfrecords_file]) 32 | # reader = tf.TFRecordReader() 33 | 34 | _, serialized_example = self.reader.read(filename_queue) 35 | features = tf.parse_single_example( 36 | serialized_example, 37 | features={ 38 | 'img_raw': tf.FixedLenFeature([], tf.string) 39 | }) 40 | 41 | image = tf.decode_raw(features['img_raw'], tf.float32) 42 | image = tf.reshape(image, [self.image_size, self.image_size, 1]) 43 | print(image.shape) 44 | 45 | images = tf.train.shuffle_batch( 46 | [image], batch_size=self.batch_size, num_threads=self.num_threads, 47 | capacity=self.min_queue_examples + 3*self.batch_size, 48 | min_after_dequeue=self.min_queue_examples 49 | ) 50 | 51 | tf.summary.image('input', images) 52 | return images 53 | 54 | def test_reader(): 55 | TRAIN_FILE_1 = '/Users/zhuxinyue/ML/tfrecords/emotion.tfrecords' 56 | TRAIN_FILE_2 = '/Users/zhuxinyue/ML/tfrecords/faces.tfrecords' 57 | 58 | with tf.Graph().as_default(): 59 | reader1 = Reader(TRAIN_FILE_1, batch_size=2) 60 | reader2 = Reader(TRAIN_FILE_2, batch_size=2) 61 | images_op1 = reader1.feed() 62 | images_op2 = reader2.feed() 63 | 64 | sess = tf.Session() 65 | init = tf.global_variables_initializer() 66 | sess.run(init) 67 | 68 | coord = tf.train.Coordinator() 69 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 70 | 71 | try: 72 | step = 0 73 | while not coord.should_stop(): 74 | batch_images1, batch_images2 = sess.run([images_op1, images_op2]) 75 | print("image1 shape: {}".format(batch_images1.shape)) 76 | print("image1 shape: {}".format(batch_images2.shape)) 77 | print("="*10) 78 | step += 1 79 | sys.exit() 80 | except KeyboardInterrupt: 81 | print('Interrupted') 82 | coord.request_stop() 83 | except Exception as e: 84 | coord.request_stop(e) 85 | finally: 86 | # When done, ask the threads to stop. 87 | coord.request_stop() 88 | coord.join(threads) 89 | 90 | # if __name__ == '__main__': 91 | # test_reader() 92 | -------------------------------------------------------------------------------- /CycleGAN/train_cyclegan.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午4:48 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : train_cyclegan.py 6 | # @Software : PyCharm 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import sys 13 | import tensorflow as tf 14 | from CycleGAN.model import CycleGAN 15 | from CycleGAN.reader import Reader 16 | from datetime import datetime 17 | import os 18 | import logging 19 | from CycleGAN.utils import ImagePool 20 | 21 | FLAGS = tf.flags.FLAGS 22 | 23 | tf.flags.DEFINE_integer('batch_size', 1, 'batch size, default: 1') 24 | tf.flags.DEFINE_integer('image_size', 48, 'image size, default: 256') 25 | tf.flags.DEFINE_bool('use_lsgan', True, 26 | 'use lsgan (mean squared error) or cross entropy loss, default: True') 27 | tf.flags.DEFINE_string('norm', 'instance', 28 | '[instance, batch] use instance norm or batch norm, default: instance') 29 | tf.flags.DEFINE_integer('lambda1', 10.0, 30 | 'weight for forward cycle loss (X->Y->X), default: 10.0') 31 | tf.flags.DEFINE_integer('lambda2', 10.0, 32 | 'weight for backward cycle loss (Y->X->Y), default: 10.0') 33 | tf.flags.DEFINE_float('learning_rate', 2e-4, 34 | 'initial learning rate for Adam, default: 0.0002') 35 | tf.flags.DEFINE_float('beta1', 0.5, 36 | 'momentum term of Adam, default: 0.5') 37 | tf.flags.DEFINE_float('pool_size', 30, 38 | 'size of image buffer that stores previously generated images, default: 50') 39 | tf.flags.DEFINE_integer('ngf', 64, 40 | 'number of gen filters in first conv layer, default: 64') 41 | 42 | tf.flags.DEFINE_string('X', '/Users/zhuxinyue/ML/tfrecords/neutral.tfrecords', 43 | 'X tfrecords file for training, default: data/tfrecords/apple.tfrecords') 44 | tf.flags.DEFINE_string('Y', '/Users/zhuxinyue/ML/tfrecords/disgust.tfrecords', 45 | 'Y tfrecords file for training, default: data/tfrecords/orange.tfrecords') 46 | tf.flags.DEFINE_string('load_model', '20171021-1558', 47 | 'folder of saved model that you wish to continue training (e.g. 20170602-1936), default: None') 48 | 49 | def train(): 50 | if FLAGS.load_model is not None: 51 | checkpoints_dir = "checkpoints/" + FLAGS.load_model 52 | else: 53 | current_time = datetime.now().strftime("%Y%m%d-%H%M") 54 | checkpoints_dir = "checkpoints/{}".format(current_time) 55 | try: 56 | os.makedirs(checkpoints_dir) 57 | except os.error: 58 | pass 59 | 60 | graph = tf.Graph() 61 | with graph.as_default(): 62 | cycle_gan = CycleGAN( 63 | X_train_file=FLAGS.X, 64 | Y_train_file=FLAGS.Y, 65 | dataset_name='face_emotion', 66 | batch_size=FLAGS.batch_size) 67 | 68 | G_loss, D_Y_loss, F_loss, D_X_loss, fake_y, fake_x = cycle_gan.model() 69 | optimizers = cycle_gan.optimize(G_loss, D_Y_loss, F_loss, D_X_loss) 70 | 71 | summary_op = tf.summary.merge_all() 72 | train_writer = tf.summary.FileWriter(checkpoints_dir, graph) 73 | saver = tf.train.Saver() 74 | 75 | with tf.Session(graph=graph) as sess: 76 | if FLAGS.load_model is not None: 77 | checkpoint = tf.train.get_checkpoint_state(checkpoints_dir) 78 | meta_graph_path = checkpoint.model_checkpoint_path + ".meta" 79 | restore = tf.train.import_meta_graph(meta_graph_path) 80 | restore.restore(sess, tf.train.latest_checkpoint(checkpoints_dir)) 81 | step = int(meta_graph_path.split("-")[2].split(".")[0]) 82 | else: 83 | sess.run(tf.global_variables_initializer()) 84 | step = 0 85 | 86 | coord = tf.train.Coordinator() 87 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 88 | 89 | try: 90 | fake_Y_pool = ImagePool(FLAGS.pool_size) 91 | fake_X_pool = ImagePool(FLAGS.pool_size) 92 | 93 | while not coord.should_stop(): 94 | # get previously generated images 95 | fake_y_val, fake_x_val = sess.run([fake_y, fake_x]) 96 | 97 | # train 98 | _, G_loss_val, D_Y_loss_val, F_loss_val, D_X_loss_val, summary = ( 99 | sess.run( 100 | [optimizers, G_loss, D_Y_loss, F_loss, D_X_loss, summary_op], 101 | feed_dict={cycle_gan.fake_y: fake_Y_pool.query(fake_y_val), 102 | cycle_gan.fake_x: fake_X_pool.query(fake_x_val)} 103 | ) 104 | ) 105 | 106 | train_writer.add_summary(summary, step) 107 | train_writer.flush() 108 | 109 | if step % 100 == 0: 110 | logging.info('-----------Step %d:-------------' % step) 111 | logging.info(' G_loss : {}'.format(G_loss_val)) 112 | logging.info(' D_Y_loss : {}'.format(D_Y_loss_val)) 113 | logging.info(' F_loss : {}'.format(F_loss_val)) 114 | logging.info(' D_X_loss : {}'.format(D_X_loss_val)) 115 | 116 | if step % 10000 == 0: 117 | save_path = saver.save(sess, checkpoints_dir + "/model.ckpt", global_step=step) 118 | logging.info("Model saved in file: %s" % save_path) 119 | 120 | step += 1 121 | 122 | except KeyboardInterrupt: 123 | logging.info('Interrupted') 124 | coord.request_stop() 125 | except Exception as e: 126 | coord.request_stop(e) 127 | finally: 128 | save_path = saver.save(sess, checkpoints_dir + "/model.ckpt", global_step=step) 129 | logging.info("Model saved in file: %s" % save_path) 130 | # When done, ask the threads to stop. 131 | coord.request_stop() 132 | coord.join(threads) 133 | 134 | def main(unused_argv): 135 | train() 136 | 137 | if __name__ == '__main__': 138 | logging.basicConfig(level=logging.INFO) 139 | tf.app.run() -------------------------------------------------------------------------------- /CycleGAN/utils.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import random 3 | 4 | def convert2int(image): 5 | """ Transfrom from float tensor ([-1.,1.]) to int image ([0,255]) 6 | """ 7 | return tf.image.convert_image_dtype((image+1.0)/2.0, tf.uint8) 8 | 9 | def convert2float(image): 10 | """ Transfrom from int image ([0,255]) to float tensor ([-1.,1.]) 11 | """ 12 | image = tf.image.convert_image_dtype(image, dtype=tf.float32) 13 | return (image/127.5) - 1.0 14 | 15 | def batch_convert2int(images): 16 | """ 17 | Args: 18 | images: 4D float tensor (batch_size, image_size, image_size, depth) 19 | Returns: 20 | 4D int tensor 21 | """ 22 | return tf.map_fn(convert2int, images, dtype=tf.uint8) 23 | 24 | def batch_convert2float(images): 25 | """ 26 | Args: 27 | images: 4D int tensor (batch_size, image_size, image_size, depth) 28 | Returns: 29 | 4D float tensor 30 | """ 31 | return tf.map_fn(convert2float, images, dtype=tf.float32) 32 | 33 | class ImagePool: 34 | """ History of generated images 35 | Same logic as https://github.com/junyanz/CycleGAN/blob/master/util/image_pool.lua 36 | """ 37 | def __init__(self, pool_size): 38 | self.pool_size = pool_size 39 | self.images = [] 40 | 41 | def query(self, image): 42 | if self.pool_size == 0: 43 | return image 44 | 45 | if len(self.images) < self.pool_size: 46 | self.images.append(image) 47 | return image 48 | else: 49 | p = random.random() 50 | if p > 0.5: 51 | # use old image 52 | random_id = random.randrange(0, self.pool_size) 53 | tmp = self.images[random_id].copy() 54 | self.images[random_id] = image.copy() 55 | return tmp 56 | else: 57 | return image 58 | 59 | -------------------------------------------------------------------------------- /WGAN.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | from __future__ import division 3 | import os 4 | import time 5 | import tensorflow as tf 6 | import numpy as np 7 | 8 | from ops import * 9 | from utils import * 10 | from emotion.emotion_data import read_train_sets 11 | 12 | class WGAN(object): 13 | def __init__(self, classes, sess, epoch, batch_size, z_dim, dataset_name, augmentation, aug_ratio, checkpoint_dir, result_dir, log_dir): 14 | self.classes = classes 15 | self.sess = sess 16 | self.dataset_name = dataset_name 17 | self.augmentation = augmentation 18 | self.aug_ratio = aug_ratio 19 | self.checkpoint_dir = checkpoint_dir 20 | self.result_dir = result_dir 21 | self.log_dir = log_dir 22 | self.epoch = epoch 23 | self.batch_size = batch_size 24 | self.model_name = "WGAN" # name for checkpoint 25 | 26 | if dataset_name == 'Caltech101': 27 | dataset_dir = '/Users/zhuxinyue/ML/' + dataset_name + '/' + classes + '/' 28 | # parameters 29 | self.input_height = 64 30 | self.input_width = 64 31 | self.output_height = 64 32 | self.output_width = 64 33 | 34 | self.z_dim = z_dim # dimension of noise-vector 35 | self.c_dim = 3 # dimension of channels? 36 | 37 | # WGAN parameter 38 | self.disc_iters = 5 # The number of critic iterations for one-step of generator 39 | 40 | # train 41 | self.learning_rate = 0.0002 42 | self.beta1 = 0.5 43 | 44 | # test 45 | self.sample_num = 64 # number of generated images to be saved 46 | 47 | # load Caltech101 48 | self.dataX_uint8, self.data_X, self.data_y = load_caltech101(dataset_dir, self.input_height) 49 | 50 | # Augmentation 51 | if augmentation == True: 52 | self.dataX_aug = data_augmentation(ratio=self.aug_ratio, images=self.dataX_uint8) 53 | self.data_X = np.concatenate((self.data_X, self.dataX_aug), axis=0) 54 | 55 | # get number of batches for a single epoch 56 | self.num_batches = len(self.data_X) // self.batch_size 57 | else: 58 | if dataset_name == 'CartoonFaces': 59 | dataset_dir = '/Users/zhuxinyue/Documents/' + dataset_name + '/' 60 | # parameters 61 | self.input_height = 64 62 | self.input_width = 64 63 | self.output_height = 64 64 | self.output_width = 64 65 | 66 | self.z_dim = z_dim # dimension of noise-vector 67 | self.c_dim = 3 # dimension of channels? 68 | 69 | # WGAN parameter 70 | self.disc_iters = 5 # The number of critic iterations for one-step of generator 71 | 72 | # train 73 | self.learning_rate = 0.0002 74 | self.beta1 = 0.5 75 | 76 | # test 77 | self.sample_num = 64 # number of generated images to be saved 78 | 79 | # load Cartoon 80 | _, self.data_X, _ = load_caltech101(dataset_dir, self.input_height) 81 | 82 | # Augmentation 83 | if augmentation == True: 84 | self.dataX_aug = data_augmentation(ratio=self.aug_ratio, images=self.dataX_uint8) 85 | self.data_X = np.concatenate((self.data_X, self.dataX_aug), axis=0) 86 | 87 | # get number of batches for a single epoch 88 | self.num_batches = len(self.data_X) // self.batch_size 89 | else: 90 | if dataset_name == 'generate': 91 | self.input_height = 64 92 | self.input_width = 64 93 | self.output_height = 64 94 | self.output_width = 64 95 | 96 | self.z_dim = z_dim # dimension of noise-vector 97 | self.c_dim = 3 # dimension of channels? 98 | 99 | # WGAN parameter 100 | self.disc_iters = 5 # The number of critic iterations for one-step of generator 101 | 102 | # train 103 | self.learning_rate = 0.0002 104 | self.beta1 = 0.5 105 | 106 | else: 107 | if dataset_name == 'face_emotion': 108 | dataset_dir = '/Users/zhuxinyue/ML/' + dataset_name + '/' 109 | # parameters 110 | self.input_height = 48 111 | self.input_width = 48 112 | self.output_height = 48 113 | self.output_width = 48 114 | 115 | self.z_dim = z_dim # dimension of noise-vector 116 | self.c_dim = 1 # dimension of channels? 117 | 118 | # WGAN parameter 119 | self.disc_iters = 5 # The number of critic iterations for one-step of generator 120 | 121 | # train 122 | self.learning_rate = 0.0002 123 | self.beta1 = 0.5 124 | 125 | # test 126 | self.sample_num = 64 # number of generated images to be saved 127 | 128 | # load face_emotion 129 | self.data_X, self.data_y, _, _, _, _ = read_train_sets(dataset_dir, self.classes, 0, 6000) 130 | 131 | # Augmentation 132 | if augmentation == True: 133 | self.dataX_aug = data_augmentation(ratio=self.aug_ratio, images=self.dataX_uint8) 134 | self.data_X = np.concatenate((self.data_X, self.dataX_aug), axis=0) 135 | 136 | # get number of batches for a single epoch 137 | self.num_batches = len(self.data_X) // self.batch_size 138 | 139 | else: 140 | raise NotImplementedError 141 | 142 | def discriminator(self, x, is_training=True, reuse=False): 143 | # Network Architecture is exactly same as in infoGAN (https://arxiv.org/abs/1606.03657) 144 | # Architecture : (64)4c2s-(128)4c2s_BL-FC1024_BL-FC1_S 145 | with tf.variable_scope("discriminator", reuse=reuse): 146 | 147 | net = lrelu(conv2d(x, 32, 4, 4, 2, 2, name='d_conv1')) 148 | net = lrelu(bn(conv2d(net, 64, 4, 4, 2, 2, name='d_conv2'), is_training=is_training, scope='d_bn2')) 149 | net = lrelu(bn(conv2d(net, 128, 4, 4, 2, 2, name='d_conv3'), is_training=is_training, scope='d_bn3')) 150 | net = lrelu(bn(conv2d(net, 256, 4, 4, 2, 2, name='d_conv4'), is_training=is_training, scope='d_bn4')) 151 | # net = lrelu(bn(conv2d(net, 512, 4, 4, 2, 2, name='d_conv5'), is_training=is_training, scope='d_bn5')) 152 | print(net) 153 | # net = tf.reshape(net, [self.batch_size, -1]) 154 | # net = lrelu(bn(linear(net, 1024, scope='d_fc5'), is_training=is_training, scope='d_bn5')) 155 | # out_logit = linear(net, 1, scope='d_fc6') 156 | # out = tf.nn.sigmoid(out_logit) 157 | logit = fc(net, 1) 158 | print('discriminator out', logit.shape) 159 | return logit 160 | 161 | def generator(self, z, is_training=True, reuse=False): 162 | # Network Architecture is exactly same as in infoGAN (https://arxiv.org/abs/1606.03657) 163 | # Architecture : FC1024_BR-FC7x7x128_BR-(64)4dc2s_BR-(1)4dc2s_S 164 | with tf.variable_scope("generator", reuse=reuse): 165 | net = tf.nn.relu(bn(fc(z, 3*3*512, scope='g_fc1'), is_training=is_training, scope='g_bn1')) 166 | # net = tf.nn.relu(bn(linear(net, 128 * 8 * 8, scope='g_fc2'), is_training=is_training, scope='g_bn2')) 167 | net = tf.reshape(net, [self.batch_size, 3, 3, 512]) 168 | 169 | # deconv 170 | # net = tf.nn.relu( 171 | # bn(deconv2d(net, [self.batch_size, 8, 8, 128], 4, 4, 2, 2, name='g_dc3'), is_training=is_training, 172 | # scope='g_bn3')) 173 | # net = tf.nn.relu( 174 | # bn(deconv2d(net, [self.batch_size, 16, 16, 64], 4, 4, 2, 2, name='g_dc4'), is_training=is_training, 175 | # scope='g_bn4')) 176 | # net = tf.nn.relu( 177 | # bn(deconv2d(net, [self.batch_size, 32, 32, 32], 4, 4, 2, 2, name='g_dc5'), is_training=is_training, 178 | # scope='g_bn5')) 179 | # out = tf.tanh(deconv2d(net, [self.batch_size, 64, 64, 3], 4, 4, 2, 2, name='g_dc6')) 180 | 181 | # resize_conv 182 | net = tf.nn.relu( 183 | bn(resize_conv(net, 6, 6, 256, name='g_rc1'), is_training=is_training, 184 | scope='g_bn3')) 185 | net = tf.nn.relu( 186 | bn(resize_conv(net, 12, 12, 128, name='g_rc2'), is_training=is_training, 187 | scope='g_bn4')) 188 | net = tf.nn.relu( 189 | bn(resize_conv(net, 24, 24, 64, name='g_rc3'), is_training=is_training, 190 | scope='g_bn5')) 191 | # net = tf.nn.relu( 192 | # bn(resize_conv(net, 64, 64, 32, name='g_rc4'), is_training=is_training, 193 | # scope='g_bn6')) 194 | 195 | # out = tf.nn.sigmoid(deconv2d(net, [self.batch_size, 128, 128, 3], 4, 4, 2, 2, name='g_dc6')) 196 | out = tf.tanh(resize_conv(net, 48, 48, 1, name='g_rc5')) 197 | print('generator out', out.shape) 198 | 199 | return out 200 | 201 | def build_model(self): 202 | # some parameters 203 | image_dims = [self.input_height, self.input_width, self.c_dim] 204 | bs = self.batch_size 205 | 206 | """ Graph Input """ 207 | # images 208 | self.inputs = tf.placeholder(tf.float32, [bs] + image_dims, name='real_images') 209 | 210 | # noises 211 | self.z = tf.placeholder(tf.float32, [bs, self.z_dim], name='z') 212 | 213 | """ Loss Function """ 214 | 215 | # output of D for real images 216 | # D_real, D_real_logits, _ = self.discriminator(self.inputs, is_training=True, reuse=False) 217 | # 218 | # # output of D for fake images 219 | # G = self.generator(self.z, is_training=True, reuse=False) 220 | # D_fake, D_fake_logits, _ = self.discriminator(G, is_training=True, reuse=True) 221 | # 222 | # # get loss for discriminator 223 | # d_loss_real = - tf.reduce_mean(D_real) 224 | # d_loss_fake = tf.reduce_mean(D_fake) 225 | # 226 | # self.d_loss = d_loss_real + d_loss_fake 227 | # 228 | # # get loss for generator 229 | # self.g_loss = - d_loss_fake 230 | 231 | # generate 232 | fake = self.generator(self.z, is_training=True, reuse=False) 233 | 234 | # discriminate 235 | r_logit = self.discriminator(self.inputs, is_training=True, reuse=False) 236 | f_logit = self.discriminator(fake, reuse=True) 237 | 238 | # losses 239 | wd = tf.reduce_mean(r_logit) - tf.reduce_mean(f_logit) 240 | self.d_loss = -wd 241 | self.g_loss= -tf.reduce_mean(f_logit) 242 | 243 | """ Training """ 244 | # divide trainable variables into a group for D and a group for G 245 | t_vars = tf.trainable_variables() 246 | d_vars = [var for var in t_vars if 'd_' in var.name] 247 | g_vars = [var for var in t_vars if 'g_' in var.name] 248 | 249 | # optimizers 250 | with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)): 251 | # self.d_optim = tf.train.AdamOptimizer(self.learning_rate, beta1=self.beta1) \ 252 | # .minimize(self.d_loss, var_list=d_vars) 253 | # self.g_optim = tf.train.AdamOptimizer(self.learning_rate*5, beta1=self.beta1) \ 254 | # .minimize(self.g_loss, var_list=g_vars) 255 | 256 | optimizer = tf.train.RMSPropOptimizer(self.learning_rate) 257 | # self.d_optim = optimizer.minimize(self.d_loss, var_list=d_vars) 258 | # 259 | # self.g_optim = optimizer.minimize(self.g_loss, var_list=g_vars) 260 | 261 | # compute gradient and vars 262 | self.d_grads_and_vars = optimizer.compute_gradients(self.d_loss, var_list=d_vars) 263 | self.g_grads_and_vars = optimizer.compute_gradients(self.g_loss, var_list=g_vars) 264 | 265 | # training ops 266 | self.d_optim = optimizer.apply_gradients(self.d_grads_and_vars) 267 | self.g_optim = optimizer.apply_gradients(self.g_grads_and_vars) 268 | 269 | 270 | # weight clipping 271 | self.clip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in d_vars] 272 | 273 | """" Testing """ 274 | # for test 275 | self.fake_images = self.generator(self.z, is_training=False, reuse=True) 276 | 277 | """ Summary """ 278 | # d_loss_real_sum = tf.summary.scalar("d_loss_real", d_loss_real) 279 | # d_loss_fake_sum = tf.summary.scalar("d_loss_fake", d_loss_fake) 280 | # d_loss_sum = tf.summary.scalar("d_loss", self.d_loss) 281 | # g_loss_sum = tf.summary.scalar("g_loss", self.g_loss) 282 | 283 | 284 | # final summary operations 285 | # self.g_sum = tf.summary.merge([d_loss_fake_sum, g_loss_sum]) 286 | # self.d_sum = tf.summary.merge([d_loss_real_sum, d_loss_sum]) 287 | 288 | self.d_sum = tf.summary.scalar("wd", wd) 289 | self.g_sum = tf.summary.scalar("g_loss", self.g_loss) 290 | 291 | for var in tf.trainable_variables(): 292 | self.var_sum = tf.summary.histogram(var.op.name + '/values', var) 293 | 294 | for grad, var in self.d_grads_and_vars + self.g_grads_and_vars: 295 | self.grad_sum = tf.summary.histogram(var.op.name + "/gradients", grad) 296 | 297 | def train(self): 298 | 299 | # initialize all variables 300 | tf.global_variables_initializer().run() 301 | 302 | # graph inputs for visualize training results 303 | self.sample_z = np.random.uniform(-1, 1, size=(self.batch_size , self.z_dim)) 304 | 305 | # saver to save model 306 | self.saver = tf.train.Saver() 307 | 308 | # summary writer 309 | self.summary_op = tf.summary.merge_all() 310 | self.writer = tf.summary.FileWriter(self.log_dir + '/' + self.model_name, self.sess.graph) 311 | 312 | # restore check-point if it exits 313 | could_load, checkpoint_counter = self.load(self.checkpoint_dir) 314 | if could_load: 315 | start_epoch = (int)(checkpoint_counter / self.num_batches) 316 | start_batch_id = checkpoint_counter - start_epoch * self.num_batches 317 | counter = checkpoint_counter 318 | print(counter) 319 | print(" [*] Load SUCCESS") 320 | else: 321 | start_epoch = 0 322 | start_batch_id = 0 323 | counter = 1 324 | print(" [!] Load failed...") 325 | 326 | 327 | # loop for epoch 328 | start_time = time.time() 329 | for epoch in range(start_epoch, self.epoch): 330 | 331 | # get batch data 332 | for idx in range(start_batch_id, self.num_batches): 333 | batch_images = self.data_X[idx*self.batch_size:(idx+1)*self.batch_size] 334 | batch_z = np.random.uniform(-1, 1, [self.batch_size, self.z_dim]).astype(np.float32) 335 | 336 | # update D network 337 | # _, _, summary_str, d_loss = self.sess.run([self.d_optim, self.clip_D, self.d_sum, self.d_loss], 338 | # feed_dict={self.inputs: batch_images, self.z: batch_z}) 339 | _, _, summary_str, d_loss = self.sess.run([self.d_optim, self.clip_D, self.summary_op, self.d_loss], 340 | feed_dict={self.inputs: batch_images, self.z: batch_z}) 341 | self.writer.add_summary(summary_str, counter) 342 | 343 | # update G network 344 | if (counter) % self.disc_iters == 0: 345 | _, summary_str, g_loss = self.sess.run([self.g_optim, self.summary_op, self.g_loss], 346 | feed_dict={self.inputs: batch_images, self.z: batch_z}) 347 | self.writer.add_summary(summary_str, counter) 348 | 349 | # display training status 350 | counter += 1 351 | print("Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" \ 352 | % (epoch, idx, self.num_batches, time.time() - start_time, d_loss, g_loss)) 353 | 354 | # save training results for every 300 steps 355 | if np.mod(counter, 300) == 0: 356 | samples = self.sess.run(self.fake_images, 357 | feed_dict={self.z: self.sample_z}) 358 | tot_num_samples = min(self.sample_num, self.batch_size) 359 | manifold_h = int(np.floor(np.sqrt(tot_num_samples))) 360 | manifold_w = int(np.floor(np.sqrt(tot_num_samples))) 361 | save_images(samples[:manifold_h * manifold_w, :, :, :], [manifold_h, manifold_w], 362 | check_folder(self.result_dir + '/' + self.model_dir) + '/' + self.model_name + '_train_{:02d}_{:04d}.jpg'.format( 363 | epoch, idx)) 364 | 365 | # After an epoch, start_batch_id is set to zero 366 | # non-zero value is only for the first epoch after loading pre-trained model 367 | start_batch_id = 0 368 | 369 | # save model 370 | self.save(self.checkpoint_dir, counter) 371 | 372 | # show temporal results 373 | # self.visualize_results(epoch) 374 | 375 | # save model for final step 376 | self.save(self.checkpoint_dir, counter) 377 | 378 | def visualize_results(self, epoch): 379 | tot_num_samples = min(self.sample_num, self.batch_size) 380 | image_frame_dim = int(np.floor(np.sqrt(tot_num_samples))) 381 | 382 | """ random condition, random noise """ 383 | 384 | z_sample = np.random.uniform(-1, 1, size=(self.batch_size, self.z_dim)) 385 | 386 | samples = self.sess.run(self.fake_images, feed_dict={self.z: z_sample}) 387 | 388 | save_images(samples[:image_frame_dim * image_frame_dim, :, :, :], [image_frame_dim, image_frame_dim], 389 | check_folder(self.result_dir + '/' + self.model_dir) + '/' + self.model_name + '_epoch%03d' % epoch + '_test_all_classes.png') 390 | 391 | @property 392 | def model_dir(self): 393 | return "{}_{}_{}_{}".format( 394 | self.model_name, self.dataset_name, 395 | self.batch_size, self.z_dim) 396 | 397 | def save(self, checkpoint_dir, step): 398 | checkpoint_dir = os.path.join(checkpoint_dir, self.model_dir, self.model_name) 399 | 400 | if not os.path.exists(checkpoint_dir): 401 | os.makedirs(checkpoint_dir) 402 | 403 | self.saver.save(self.sess,os.path.join(checkpoint_dir, self.model_name+'.model'), global_step=step) 404 | 405 | def load(self, checkpoint_dir): 406 | import re 407 | print(" [*] Reading checkpoints...") 408 | checkpoint_dir = os.path.join(checkpoint_dir, self.model_dir, self.model_name) 409 | 410 | ckpt = tf.train.get_checkpoint_state(checkpoint_dir) 411 | if ckpt and ckpt.model_checkpoint_path: 412 | ckpt_name = os.path.basename(ckpt.model_checkpoint_path) 413 | self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name)) 414 | counter = int(next(re.finditer("(\d+)(?!.*\d)",ckpt_name)).group(0)) 415 | print(" [*] Success to read {}".format(ckpt_name)) 416 | return True, counter 417 | else: 418 | print(" [*] Failed to find a checkpoint") 419 | return False, 0 -------------------------------------------------------------------------------- /emotion/emotion_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/16 下午4:58 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : emotion_data.py 6 | # @Software : PyCharm 7 | 8 | ''' 9 | Emotion dataset comes from https://github.com/sjchoi86/img_dataset 10 | ''' 11 | 12 | from __future__ import absolute_import 13 | from __future__ import division 14 | from __future__ import print_function 15 | 16 | import os 17 | import sys 18 | import glob 19 | import numpy as np 20 | # import cv2 21 | import matplotlib.pyplot as plt 22 | from scipy import misc 23 | from sklearn.utils import shuffle 24 | import tensorflow as tf 25 | from collections import Counter 26 | 27 | def display_image(image): 28 | dis_image = image.astype(np.uint8) 29 | plt.figure() 30 | 31 | plt.imshow(dis_image) 32 | 33 | 34 | def load_caltech101(data_path, image_size): 35 | images = [] 36 | labels = [] 37 | files = os.listdir(data_path) 38 | 39 | for fl in files: 40 | file = data_path + fl 41 | image = misc.imread(file, mode='L') 42 | image = misc.imresize(image, [image_size, image_size]) 43 | 44 | '''test image''' 45 | # print(image.shape) 46 | # display_image(image, True) 47 | # plt.show() 48 | # sys.exit() 49 | 50 | image = np.reshape(image, [image_size, image_size, -1]) 51 | # print(image.shape) 52 | images.append(image) 53 | 54 | # label = 1.0 55 | # labels.append(label) 56 | 57 | images_uint = np.array(images, dtype=np.uint8) 58 | 59 | images = np.array(images, dtype=np.float32) 60 | # labels = np.array(labels, dtype=np.int64) 61 | 62 | print('Loding training data completed') 63 | print('images_shape:', images.shape) 64 | # print('labels_shape:', labels.shape) 65 | 66 | return images_uint, images/255 67 | 68 | def load_data(data_set, data_path, classes, bound1, bound2): 69 | images = [] 70 | test_images = [] 71 | test_cls = [] 72 | labels = [] 73 | cls = [] 74 | 75 | print('Reading training images...') 76 | 77 | for obj in classes: 78 | index = classes.index(obj) 79 | print('Loading {} files (index:{})'.format(obj, index)) 80 | 81 | path = os.path.join(data_path, obj, '*g') 82 | files = glob.glob(path) 83 | 84 | for fl in files: 85 | # print(fl) 86 | obj_num = int((fl.split('/')[-1]).split('.')[0]) 87 | # throw away test data 88 | if obj_num < bound1 and data_set == 'train': 89 | image = misc.imread(fl) 90 | # '''when doing embedding''' 91 | # image = misc.imresize(image, [8, 8]) 92 | # image = np.reshape(image, [8, 8, 1]) 93 | image = np.reshape(image, [48, 48, 1]) 94 | images.append(image) 95 | 96 | label = np.zeros(len(classes), dtype=np.float32) 97 | label[index] = 1.0 98 | labels.append(label) 99 | 100 | cls.append(obj) 101 | else: 102 | if obj_num < bound2 and data_set == 'test': 103 | image = misc.imread(fl) 104 | image = np.reshape(image, [48, 48, 1]) 105 | test_images.append(image) 106 | test_cls.append(obj) 107 | 108 | if data_set == 'train': 109 | images = np.array(images, dtype=np.float32) 110 | labels = np.array(labels, dtype=np.int64) 111 | cls = np.array(cls) 112 | 113 | print('Loding training data completed') 114 | print('images_shape:', images.shape) 115 | print('labels_shape:', labels.shape) 116 | 117 | # Calculate data numbers in each class 118 | cls_num = Counter(cls) 119 | print(cls_num) 120 | return images, labels, cls 121 | 122 | if data_set == 'test': 123 | # Normalization 124 | test_images = np.array(test_images, dtype=np.uint8) 125 | test_images = test_images.astype(np.float32) 126 | test_images = test_images / 255 127 | print('Loading test data completed') 128 | 129 | return test_images, test_cls 130 | 131 | def read_train_sets(data_path, classes, validation_size, bound_train): 132 | 133 | images, labels, cls = load_data('train', data_path, classes, bound_train, 0) 134 | images, labels, cls = shuffle(images, labels, cls) # shuffle the data 135 | 136 | if isinstance(validation_size, float): 137 | validation_size = int(validation_size * images.shape[0]) 138 | 139 | if validation_size != 0: 140 | val_images = images[:validation_size] 141 | val_images = val_images / 255 142 | val_labels = labels[:validation_size] 143 | val_cls = cls[:validation_size] 144 | else: 145 | val_images, val_labels, val_cls = [None, None, None] 146 | train_images = images[validation_size:] 147 | train_images = train_images / 255 148 | train_labels = labels[validation_size:] 149 | train_cls = cls[validation_size:] 150 | 151 | return train_images, train_labels, train_cls, val_images, val_labels, val_cls 152 | 153 | 154 | def read_test_set(data_path, classes, bound_test): 155 | images, cls = load_data('test', data_path, classes, 0, bound_test) 156 | return images, cls 157 | 158 | def load_gan_image(genpath, classes, gan_class): 159 | images = [] 160 | labels = [] 161 | clss = [] 162 | files = os.listdir(genpath) 163 | index = classes.index(gan_class) 164 | cls = gan_class 165 | 166 | for fl in files: 167 | file = genpath + fl 168 | if file.split('.')[-1] != 'jpg': 169 | continue 170 | image = misc.imread(file, mode='L') 171 | image = np.reshape(image, [48, 48, 1]) 172 | images.append(image) 173 | 174 | label = np.zeros(len(classes), dtype=np.float32) 175 | label[index] = 1.0 176 | labels.append(label) 177 | 178 | clss.append(cls) 179 | # images_uint = np.array(images, dtype=np.uint8) 180 | 181 | images = np.array(images, dtype=np.float32) 182 | labels = np.array(labels, dtype=np.int64) 183 | clss = np.array(clss) 184 | 185 | print('Loding training data completed') 186 | print('images_shape:', images.shape) 187 | print('labels_shape:', labels.shape) 188 | cls_num = Counter(clss) 189 | print(cls_num) 190 | 191 | return images / 255, labels, clss 192 | 193 | def inputs(image, label, cls, batch_size, capacity, min_after_dequeue): 194 | ''' 195 | Input a batch of data 196 | ''' 197 | with tf.name_scope('input'): 198 | input_queue = tf.train.slice_input_producer([image, label, cls]) 199 | image = input_queue[0] 200 | label = input_queue[1] 201 | cls = input_queue[2] 202 | # Shuffle the examples and collect them into batch_size batches. 203 | # (Internally uses a RandomShuffleQueue.) 204 | # We run this in two threads to avoid being a bottleneck. 205 | images, labels, clss= tf.train.shuffle_batch( 206 | [image, label, cls], batch_size=batch_size, num_threads=2, 207 | capacity=capacity, 208 | # Ensures a minimum amount of shuffling of examples. 209 | min_after_dequeue=min_after_dequeue) 210 | 211 | return images, labels, clss -------------------------------------------------------------------------------- /emotion/emotion_evaluate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 上午10:06 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : emotion_evaluate.py 6 | # @Software : PyCharm 7 | 8 | import sys 9 | 10 | import tensorflow as tf 11 | import numpy as np 12 | from collections import Counter 13 | 14 | from PIL import Image 15 | import matplotlib.pyplot as plt 16 | import emotion.emotion_model as model 17 | import os 18 | import emotion.emotion_data as data 19 | 20 | data_path = '/Users/zhuxinyue/ML/face_emotion/' 21 | logs_train_dir = '../emotion/logs/gan_dands/' 22 | logs_var_dir = '../emotion/logs/gan_dands/' 23 | 24 | BATCH_SIZE = 1 25 | N_CLASSES = 7 26 | image_size = 48 27 | classes = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'] 28 | 29 | def evaluate_one_image(test_image, test_cls, logs_train_dir): 30 | # display image 31 | # data.display_image(test_image) 32 | 33 | with tf.Graph().as_default(): 34 | x = tf.placeholder(tf.float32, shape=[BATCH_SIZE, image_size, image_size, 1]) 35 | 36 | test_image = np.reshape(test_image, [BATCH_SIZE, image_size, image_size, 1]) 37 | 38 | logit, _ = model.inference(test_image, BATCH_SIZE, N_CLASSES) 39 | logit = tf.nn.softmax(logit) 40 | 41 | saver = tf.train.Saver() 42 | 43 | with tf.Session() as sess: 44 | print("Reading checkpoints...") 45 | ckpt = tf.train.get_checkpoint_state(logs_train_dir) 46 | if ckpt and ckpt.model_checkpoint_path: 47 | global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] 48 | saver.restore(sess, ckpt.model_checkpoint_path) 49 | print("successful loading,global step is %s" % global_step) 50 | else: 51 | print("no checkpoint file founded") 52 | return 53 | prediction = sess.run(logit, feed_dict={x: test_image}) 54 | # print(prediction) 55 | max_index = np.argmax(prediction) 56 | pred_index = classes[max_index] 57 | 58 | print('The true label is:', test_cls) 59 | print('The predict label is:', pred_index) 60 | # plt.show() 61 | return pred_index, max_index 62 | 63 | 64 | def main(argv=None): # pylint: disable=unused-argument 65 | right_pred = 0 66 | right_pred_cls = np.zeros(shape=[N_CLASSES], dtype=np.int64) 67 | cls_accuracy = [] 68 | 69 | test_images, test_clss = data.read_test_set(data_path, classes, 2000) 70 | test_num = test_images.shape[0] 71 | print('Number of test data:', test_num) 72 | 73 | # Calculate data numbers in each class 74 | cls_num = Counter(test_clss) 75 | print('cla_num:', cls_num) 76 | 77 | for num in range(test_num): 78 | pred_index, max_index = evaluate_one_image(test_images[num], test_clss[num], logs_train_dir) 79 | # print(test_clss[num]) 80 | # print(pred_index) 81 | if pred_index == test_clss[num]: 82 | right_pred += 1 83 | right_pred_cls[max_index] += 1 84 | # print(right_pred_cls) 85 | 86 | accuracy = right_pred / test_num 87 | print('Final accuracy:', accuracy) 88 | 89 | for i in classes: 90 | index = classes.index(i) 91 | cls_accuracy.append(right_pred_cls[index] / cls_num[i]) 92 | print(i, cls_accuracy[index], right_pred_cls[index], cls_num[i]) 93 | print(cls_accuracy) 94 | if __name__ == '__main__': 95 | tf.app.run() -------------------------------------------------------------------------------- /emotion/emotion_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/9/17 下午8:20 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : model.py.py 6 | # @Software : PyCharm 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import tensorflow as tf 13 | from ops import * 14 | 15 | def inference(images, batch_size, n_classes): 16 | ''' 17 | 18 | :param images:image batch, 4D tensor 19 | :param batch_size: 20 | :param n_classes: 50 21 | :return: 22 | output tensor with the computed logits, float, [batch_size,n_classes] 23 | ''' 24 | 25 | # conv1, shape = [kernel size, kernel size, channels, kernel numbers] 26 | with tf.variable_scope('conv1') as scope: 27 | weights = tf.get_variable('weights', 28 | shape=[3, 3, 1, 64], 29 | dtype=tf.float32, 30 | initializer=tf.truncated_normal_initializer( 31 | stddev=5e-2, dtype=tf.float32)) 32 | biases = tf.get_variable('biases', 33 | shape=[64], 34 | dtype=tf.float32, 35 | initializer=tf.constant_initializer(0.1)) 36 | conv = tf.nn.conv2d(images, weights, strides=[1, 1, 1, 1], padding='SAME') 37 | pre_activation = tf.nn.bias_add(conv, biases) 38 | conv1 = tf.nn.relu(pre_activation, name=scope.name) 39 | 40 | # pooling1 and norm1 41 | with tf.variable_scope('pooling1') as scope: 42 | pooling1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], 43 | padding='SAME', name='pooling1') 44 | norm1 = tf.nn.lrn(pooling1, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, 45 | beta=0.75, name='norm1') 46 | 47 | # conv2 48 | with tf.variable_scope('conv2') as scope: 49 | weights = tf.get_variable('weights', 50 | shape=[3, 3, 64, 128], 51 | dtype=tf.float32, 52 | initializer=tf.truncated_normal_initializer( 53 | stddev=5e-2, dtype=tf.float32)) 54 | biases = tf.get_variable('biases', 55 | shape=[128], 56 | dtype=tf.float32, 57 | initializer=tf.constant_initializer(0.1)) 58 | conv = tf.nn.conv2d(norm1, weights, strides=[1, 1, 1, 1], padding='SAME') 59 | pre_activation = tf.nn.bias_add(conv, biases) 60 | conv2 = tf.nn.relu(pre_activation, name='conv2') 61 | 62 | # pooling2 and norm2 63 | with tf.variable_scope('pooling2') as scope: 64 | norm2 = tf.nn.lrn(conv2, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, 65 | beta=0.75, name='norm2') 66 | pooling2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], 67 | padding='SAME', name='pooling2') 68 | 69 | # fc1 70 | with tf.variable_scope('fc1') as scope: 71 | reshape = tf.reshape(pooling2, shape=[batch_size, -1]) 72 | dim = reshape.get_shape()[1].value 73 | weights = tf.get_variable('weights', 74 | shape=[dim, 256], 75 | dtype=tf.float32, 76 | initializer=tf.truncated_normal_initializer(stddev=0.005, 77 | dtype=tf.float32)) 78 | biases = tf.get_variable('biases', 79 | shape=[256], 80 | dtype=tf.float32, 81 | initializer=tf.constant_initializer(0.1)) 82 | fc1 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name) 83 | 84 | # fc2 85 | with tf.variable_scope('fc2') as scope: 86 | weights = tf.get_variable('weights', 87 | shape=[256, 256], 88 | dtype=tf.float32, 89 | initializer=tf.truncated_normal_initializer(stddev=0.005, 90 | dtype=tf.float32)) 91 | biases = tf.get_variable('biases', 92 | shape=[256], 93 | dtype=tf.float32, 94 | initializer=tf.constant_initializer(0.1)) 95 | fc2 = tf.nn.relu(tf.matmul(fc1, weights) + biases, name='fc2') 96 | 97 | # softmax 98 | with tf.variable_scope('softmax_linear') as scope: 99 | weights = tf.get_variable('softmax_linear', 100 | shape=[256, n_classes], 101 | dtype=tf.float32, 102 | initializer=tf.truncated_normal_initializer(stddev=0.005, 103 | dtype=tf.float32)) 104 | biases = tf.get_variable('biases', 105 | shape=[n_classes], 106 | dtype=tf.float32, 107 | initializer=tf.constant_initializer(0.1)) 108 | softmax_linear = tf.add(tf.matmul(fc2, weights), biases, name='softmax_linear') 109 | 110 | # epsilon = tf.constant(0.001, shape=[n_classes]) 111 | # 112 | # softmax_linear += epsilon 113 | return softmax_linear, fc2 114 | 115 | 116 | def losses(logits,labels): 117 | ''' 118 | compute losses from logits and labels 119 | :param logits: 120 | :param labels: 121 | :return: loss tensor of float type 122 | ''' 123 | labels = tf.cast(labels,tf.float32) 124 | with tf.variable_scope('loss') as scope: 125 | cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, 126 | labels=labels, 127 | name='xentropy_per_example') 128 | 129 | loss = tf.reduce_mean(cross_entropy,name='loss') 130 | tf.summary.scalar(scope.name+'/loss',loss) 131 | return loss 132 | 133 | def training(loss,learning_rate): 134 | ''' 135 | training ops 136 | :param loss: 137 | :param learning_rate: 138 | :return: train_op: the op for training 139 | ''' 140 | with tf.name_scope('optimizer'): 141 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) 142 | global_step = tf.Variable(0,name='global_step',trainable=False) 143 | train_op = optimizer.minimize(loss,global_step=global_step) 144 | return train_op 145 | 146 | def evaluation(logits,labels): 147 | with tf.variable_scope('accuracy') as scope: 148 | correct = tf.equal(tf.argmax(logits,1),tf.argmax(labels,1)) 149 | accuracy = tf.reduce_mean(tf.cast(correct,tf.float32)) 150 | tf.summary.scalar(scope.name+'/accuracy',accuracy) 151 | return accuracy -------------------------------------------------------------------------------- /emotion/emotion_tfr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/17 下午7:58 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : emotion_tfr.py 6 | # @Software : PyCharm 7 | 8 | import sys 9 | import tensorflow as tf 10 | import numpy as np 11 | import emotion.emotion_data as data 12 | from utils import load_caltech101 13 | import matplotlib.pyplot as plt 14 | 15 | tfrecords_filename_disgust = '/Users/zhuxinyue/ML/tfrecords/disgust.tfrecords' 16 | tfrecords_filename_sad = '/Users/zhuxinyue/ML/tfrecords/sad.tfrecords' 17 | tfrecords_filename_angry = '/Users/zhuxinyue/ML/tfrecords/angry.tfrecords' 18 | tfrecords_filename_neutral = '/Users/zhuxinyue/ML/tfrecords/neutral.tfrecords' 19 | tfrecords_filename_fear = '/Users/zhuxinyue/ML/tfrecords/fear.tfrecords' 20 | tfrecords_filename_happy = '/Users/zhuxinyue/ML/tfrecords/happy.tfrecords' 21 | tfrecords_filename_surprise = '/Users/zhuxinyue/ML/tfrecords/surprise.tfrecords' 22 | data_path_emotion = '/Users/zhuxinyue/ML/face_emotion/' 23 | data_path_faces = '/Users/zhuxinyue/ML/Caltech101/Faces_easy/' 24 | 25 | def _int64_feature(value): 26 | """Wrapper for inserting int64 features into Example proto.""" 27 | if not isinstance(value, list): 28 | value = [value] 29 | return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) 30 | 31 | 32 | def _bytes_feature(value): 33 | """Wrapper for inserting bytes features into Example proto.""" 34 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) 35 | 36 | 37 | def conver_to_tfrecord(set, tfrecords_filename, data_path, classes): 38 | writer = tf.python_io.TFRecordWriter(tfrecords_filename) 39 | if set == 'train': 40 | mode = tfrecords_filename.split('/')[-1].split('.')[0] 41 | print(mode) 42 | if mode == 'disgust': 43 | images, lables, _, _, _, _ = data.read_train_sets(data_path, classes, 0, 40000) 44 | # print(images.shape) 45 | else: 46 | if mode == 'neutral': 47 | images, lables, _, _, _, _ = data.read_train_sets(data_path, classes, 0, 10000) 48 | # print(images.shape) 49 | else: 50 | if mode == 'sad' or mode == 'angry': 51 | images, lables, _, _, _, _ = data.read_train_sets(data_path, classes, 0, 10000) 52 | else: 53 | if mode == 'fear': 54 | images, lables, _, _, _, _ = data.read_train_sets(data_path, classes, 0, 12000) 55 | else: 56 | if mode == 'happy': 57 | images, lables, _, _, _, _ = data.read_train_sets(data_path, classes, 0, 7000) 58 | else: 59 | if mode == 'surprise': 60 | images, lables, _, _, _, _ = data.read_train_sets(data_path, classes, 0, 14000) 61 | else: 62 | images, labels = [None, None] 63 | else: 64 | if set == 'test': 65 | images, labels = data.read_test_set(data_path, classes, 3000) 66 | else: 67 | images, labels = [None, None] 68 | 69 | set_num = images.shape[0] 70 | 71 | for i in range(set_num): 72 | if i % 10 == 0: 73 | print('tfrecording...', i) 74 | img_raw = images[i].tostring() 75 | # label_raw = labels[i] 76 | 77 | # write to tfRecord 78 | example = tf.train.Example(features=tf.train.Features(feature={ 79 | 'img_raw': _bytes_feature(img_raw), 80 | # 'ann_raw': _int64_feature(label_raw) 81 | })) 82 | writer.write(example.SerializeToString()) 83 | writer.close() 84 | 85 | def main(unused_argv): 86 | print("Convert emotion data to tfrecords...") 87 | print(data_path_faces.split('/')[-2]) 88 | # classes_emotion = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'] 89 | classes_emotion = ['surprise'] 90 | conver_to_tfrecord('train', tfrecords_filename_surprise, data_path_emotion, classes_emotion) 91 | print('Load successfully') 92 | 93 | print("Convert face data to tfrecords...") 94 | # classes_faces = ['Faces_easy'] 95 | # conver_to_tfrecord('train', tfrecords_filename_faces, data_path_faces, classes_faces) 96 | classes_emotion = ['neutral'] 97 | conver_to_tfrecord('train', tfrecords_filename_neutral, data_path_emotion, classes_emotion) 98 | 99 | if __name__ == '__main__': 100 | tf.app.run() 101 | -------------------------------------------------------------------------------- /emotion/emotion_train.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/9/17 下午8:31 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : train.py 6 | # @Software : PyCharm 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import os 13 | os.environ['TF_CPP_MIN_LOG_LEVEL']='2' 14 | import sys 15 | import tensorflow as tf 16 | import numpy as np 17 | from emotion import emotion_data as data 18 | import emotion.emotion_model as model 19 | 20 | data_path = '/Users/zhuxinyue/ML/face_emotion/' 21 | logs_train_dir = 'logs/gan_dands/' 22 | logs_val_dir = 'logs/gan_dands/' 23 | gen_path = ['/Users/zhuxinyue/ML/gen_CG/angry/', 24 | '/Users/zhuxinyue/ML/gen_CG/disgust/', 25 | '/Users/zhuxinyue/ML/gen_CG/fear/', 26 | '/Users/zhuxinyue/ML/gen_CG/happy/', 27 | '/Users/zhuxinyue/ML/gen_CG/sad/', 28 | '/Users/zhuxinyue/ML/gen_CG/surprise/'] 29 | 30 | IMG_H = 48 31 | IMG_W = 48 32 | 33 | classes = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'] 34 | N_CLASSES = 7 35 | 36 | BATCH_SIZE = 32 37 | CAPACITY = 1000 + 3*BATCH_SIZE 38 | MIN_AFTER_DEQUEUE = 500 39 | MAX_STEP = 20000 40 | learning_rate = 0.001 41 | 42 | def run_training(): 43 | # load data 44 | # tra_img, tra_label, tra_cls, val_img, val_label, val_cls = data.read_train_sets( 45 | # data_path, classes, validation_size=0.1, bound_train=6000) 46 | 47 | # angry_img, angry_label, angry_cls = data.load_gan_image(gen_path[0], classes, 'angry') 48 | disgust_img, disgust_label, disgust_cls = data.load_gan_image(gen_path[1], classes, 'disgust') 49 | # fear_img, fear_label, fear_cls = data.load_gan_image(gen_path[2], classes, 'fear') 50 | # happy_img, happy_label, happy_cls = data.load_gan_image(gen_path[3], classes, 'happy') 51 | sad_img, sad_label, sad_cls = data.load_gan_image(gen_path[4], classes, 'sad') 52 | # surprise_img, surprise_label, surprise_cls = data.load_gan_image(gen_path[5], classes, 'surprise') 53 | 54 | 55 | tra_img, tra_label, tra_cls, val_img, val_label, val_cls = data.read_train_sets( 56 | data_path, classes, validation_size=0.1, bound_train=6000) 57 | 58 | concat_images = np.concatenate([tra_img, disgust_img, sad_img], 59 | axis=0) 60 | concat_labels = np.concatenate([tra_label, disgust_label, sad_label], axis=0) 61 | concat_clss = np.concatenate([tra_cls, disgust_cls, sad_cls], 62 | axis=0) 63 | 64 | # print(gan_label[0]) 65 | # concat_images = np.concatenate([tra_img, gan_img]) 66 | # concat_lables = np.concatenate([tra_label, gan_label]) 67 | # concat_clss = np.concatenate([tra_cls, gen_cls]) 68 | print(concat_images.shape) 69 | print(concat_labels.shape) 70 | print(concat_labels[1000]) 71 | 72 | # train_images, train_labels, train_clss= data.inputs(tra_img, tra_label, tra_cls, 73 | # BATCH_SIZE, CAPACITY, MIN_AFTER_DEQUEUE) 74 | train_images, train_labels, train_clss = data.inputs(concat_images, concat_labels, concat_clss, 75 | BATCH_SIZE, CAPACITY, MIN_AFTER_DEQUEUE) 76 | val_images, val_labels, val_clss= data.inputs(val_img, val_label, val_cls, 77 | BATCH_SIZE, CAPACITY, MIN_AFTER_DEQUEUE) 78 | print(train_images.shape) 79 | print(train_labels.shape) 80 | 81 | train_logits = model.inference(train_images, BATCH_SIZE, N_CLASSES) 82 | train_loss = model.losses(train_logits, train_labels) 83 | train_op = model.training(train_loss, learning_rate) 84 | train_eval = model.evaluation(train_logits, train_labels) 85 | 86 | with tf.Session() as sess: 87 | saver = tf.train.Saver() 88 | sess.run(tf.global_variables_initializer()) 89 | coord = tf.train.Coordinator() 90 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 91 | summary_op = tf.summary.merge_all() 92 | train_writer = tf.summary.FileWriter(logs_train_dir,sess.graph) 93 | val_writer = tf.summary.FileWriter(logs_val_dir,sess.graph) 94 | 95 | x_image = tf.placeholder(tf.float32, shape=[BATCH_SIZE, IMG_H, IMG_W, 1]) 96 | y_true = tf.placeholder(tf.float32, shape=[BATCH_SIZE, N_CLASSES]) 97 | # y_true_cls = tf.argmax(y_true, dimension=1) 98 | 99 | print("Reading checkpoints...") 100 | ckpt = tf.train.get_checkpoint_state(logs_train_dir) 101 | if ckpt and ckpt.model_checkpoint_path: 102 | global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] 103 | saver.restore(sess, ckpt.model_checkpoint_path) 104 | print("successful loading,global step is %s" % global_step) 105 | else: 106 | print("no checkpoint file founded") 107 | 108 | try: 109 | for step in np.arange(MAX_STEP): 110 | if coord.should_stop(): 111 | break 112 | # test values of label and logits in a batch: 113 | tra_images, tra_labels, tra_clss = sess.run([train_images, train_labels, train_clss]) 114 | _, tra_loss, tra_eval = sess.run([train_op, train_loss, train_eval], 115 | feed_dict={x_image: tra_images, 116 | y_true: tra_labels}) 117 | # print('logits', tra_logits) 118 | # print('ce', tra_ce) 119 | # print(tra_loss) 120 | # sys.exit() 121 | if step % 10 == 0: 122 | print('Step %d, train loss=%.4f, train accuracy=%.2f%%' % (step, tra_loss, tra_eval * 100)) 123 | summary_str = sess.run(summary_op) 124 | train_writer.add_summary(summary_str, step) 125 | if step % 100 == 0 or (step + 1) == MAX_STEP: 126 | vali_images, vali_labels, vali_clss = sess.run([val_images, val_labels, val_clss]) 127 | val_loss, val_acc = sess.run([train_loss, train_eval], 128 | feed_dict={x_image: vali_images, 129 | y_true: vali_labels}) 130 | print('** Step %d, val loss = %.4f, val accuracy = %.2f%% **' % (step, val_loss, val_acc * 100.0)) 131 | summary_str = sess.run(summary_op) 132 | val_writer.add_summary(summary_str, step) 133 | if step % 500 == 0 or (step + 1) == MAX_STEP: 134 | checkpoint_path = os.path.join(logs_train_dir, 'model.ckpt') 135 | saver.save(sess, checkpoint_path, global_step=step) 136 | except tf.errors.OutOfRangeError: 137 | print('Done training') 138 | finally: 139 | coord.request_stop() 140 | 141 | coord.join(threads) 142 | sess.close() 143 | 144 | 145 | def main(argv=None): # pylint: disable=unused-argument 146 | run_training() 147 | 148 | 149 | if __name__ == '__main__': 150 | tf.app.run() -------------------------------------------------------------------------------- /emotion/manifold.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/10/30 下午3:52 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : manifold.py 6 | # @Software : PyCharm 7 | 8 | from time import time 9 | import sys 10 | import numpy as np 11 | import matplotlib.pyplot as plt 12 | from sklearn import(manifold, datasets, decomposition, ensemble, 13 | discriminant_analysis, random_projection) 14 | from emotion.emotion_data import read_train_sets 15 | import tensorflow as tf 16 | import emotion.emotion_model as model 17 | import emotion.emotion_data as data 18 | 19 | data_path = '/Users/zhuxinyue/ML/face_emotion/' 20 | gen_path1 = '/Users/zhuxinyue/ML/gen_CG/disgust/' 21 | gen_path2 = '/Users/zhuxinyue/ML/gen_CG/sad/' 22 | classes = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'] 23 | # classes = ['angry', 'happy', ' fear', 'surprise'] 24 | 25 | # 26 | images, labels, _, _, _, _ = read_train_sets(data_path, classes, 0, 6000) 27 | disgust_img, disgust_label, _ = data.load_gan_image(gen_path1, classes, 'disgust') 28 | # sad_img, sad_label, _ = data.load_gan_image(gen_path2, classes, 'sad') 29 | 30 | concat_images = np.concatenate([images, disgust_img], 31 | axis=0) 32 | concat_labels = np.concatenate([labels, disgust_label], axis=0) 33 | 34 | 35 | print(concat_images.shape) 36 | # images = np.reshape(images, [-1, 48, 48]) 37 | X = np.reshape(concat_images, [-1, 48*48*1]) 38 | y = np.argmax(concat_labels, axis=1) 39 | n_samples, n_featues = X.shape 40 | n_neighbors = 30 41 | 42 | # digits = datasets.load_digits(n_class=6) 43 | # X = digits.data 44 | # y = digits.target 45 | # images = digits.images 46 | # print(X.shape) 47 | # print(y.shape) 48 | 49 | print(images[0].shape) 50 | 51 | 52 | 53 | #---------------------------------------------------------------------- 54 | # read the fc2 layer as input map 55 | logs_train_dir = '../emotion/logs/gan_train/' 56 | with tf.Graph().as_default(): 57 | x = tf.placeholder(tf.float32, shape=[n_samples, 48, 48, 1]) 58 | _, map = model.inference(x, n_samples, 7) 59 | print(map.shape) 60 | saver = tf.train.Saver() 61 | 62 | with tf.Session() as sess: 63 | print("Reading checkpoints...") 64 | ckpt = tf.train.get_checkpoint_state(logs_train_dir) 65 | if ckpt and ckpt.model_checkpoint_path: 66 | global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] 67 | saver.restore(sess, ckpt.model_checkpoint_path) 68 | print("successful loading,global step is %s" % global_step) 69 | else: 70 | print("no checkpoint file founded") 71 | 72 | maps = sess.run(map, feed_dict={x:concat_images}) 73 | print(maps.shape) 74 | # plt.show() 75 | #---------------------------------------------------------------------- 76 | # Scale and visualize the embedding vectors 77 | def plot_embedding(X, title=None): 78 | x_min, x_max = np.min(X, 0), np.max(X, 0) 79 | X = (X - x_min) / (x_max - x_min) 80 | 81 | plt.figure() 82 | # ax = plt.subplot(111) 83 | for i in range(X.shape[0]): 84 | plt.text(X[i, 0], X[i, 1], str(y[i]), 85 | color=plt.cm.Set1(y[i]), 86 | fontdict={'weight': 'bold', 'size': 9}) 87 | 88 | # if hasattr(offsetbox, 'AnnotationBbox'): 89 | # # only print thumbnails with matplotlib > 1.0 90 | # shown_images = np.array([[1., 1.]]) # just something big 91 | # for i in range(X.shape[0]): 92 | # dist = np.sum((X[i] - shown_images) ** 2, 1) 93 | # if np.min(dist) < 4e-3: 94 | # # don't show points that are too close 95 | # continue 96 | # shown_images = np.r_[shown_images, [X[i]]] 97 | # imagebox = offsetbox.AnnotationBbox( 98 | # offsetbox.OffsetImage(images[i], cmap=plt.cm.gray_r), 99 | # X[i]) 100 | # ax.add_artist(imagebox) 101 | plt.xticks([]), plt.yticks([]) 102 | if title is not None: 103 | plt.title(title) 104 | 105 | 106 | #---------------------------------------------------------------------- 107 | # Plot images of the digits 108 | # n_img_per_row = 20 109 | # img = np.zeros((100 * n_img_per_row, 100 * n_img_per_row)) 110 | # for i in range(n_img_per_row): 111 | # ix = 10 * i + 1 112 | # for j in range(n_img_per_row): 113 | # iy = 10 * j + 1 114 | # # print(iy) 115 | # # print(img[ix:ix + 48, iy:iy + 48].shape) 116 | # img[ix:ix + 48, iy:iy + 48] = X[i * n_img_per_row + j].reshape((48, 48)) 117 | # print('flag') 118 | # plt.imshow(img, cmap=plt.cm.binary) 119 | # plt.xticks([]) 120 | # plt.yticks([]) 121 | # plt.title('A selection from the 64-dimensional digits dataset') 122 | 123 | 124 | #---------------------------------------------------------------------- 125 | # Random 2D projection using a random unitary matrix 126 | print("Computing random projection") 127 | rp = random_projection.SparseRandomProjection(n_components=2, random_state=42) 128 | X_projected = rp.fit_transform(maps) 129 | plot_embedding(X_projected, "Random Projection of the digits") 130 | 131 | 132 | #---------------------------------------------------------------------- 133 | # Projection on to the first 2 principal components 134 | 135 | print("Computing PCA projection") 136 | t0 = time() 137 | X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(maps) 138 | plot_embedding(X_pca, 139 | "Principal Components projection of the digits (time %.2fs)" % 140 | (time() - t0)) 141 | 142 | 143 | #---------------------------------------------------------------------- 144 | # t-SNE embedding of the digits dataset 145 | 146 | print("Computing t-SNE embedding") 147 | tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) 148 | t0 = time() 149 | X_tsne = tsne.fit_transform(maps) 150 | 151 | plot_embedding(X_tsne, 152 | "t-SNE embedding of the digits (time %.2fs)" % 153 | (time() - t0)) 154 | 155 | plt.show() -------------------------------------------------------------------------------- /evaluate_one_image.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/9/19 上午9:01 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : evaluate_one_image.py 6 | # @Software : PyCharm 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | import sys 13 | 14 | import tensorflow as tf 15 | import numpy as np 16 | from PIL import Image 17 | import matplotlib.pyplot as plt 18 | from scipy import misc 19 | 20 | import data 21 | import model 22 | 23 | data_path = '/Users/zhuxinyue/ML/airplanes_faces/' 24 | logs_train_dir = 'logs/train' 25 | logs_val_dir = 'logs/val' 26 | 27 | BATCH_SIZE = 1 28 | N_CLASSES = 2 29 | image_size = 64 30 | # classes = data.classes_extraction(data_path) 31 | classes = ['airplanes', 'Faces'] 32 | 33 | def get_one_image(data_path, classes, image_size): 34 | test_images, test_clss = data.read_test_set(data_path, classes, image_size) 35 | num = test_images.shape[0] 36 | test_image_num = np.random.randint(0, num) 37 | test_image = test_images[test_image_num] 38 | test_cls = test_clss[test_image_num] 39 | print('Get Image! The Class is: ', test_cls) 40 | 41 | return test_image, test_cls 42 | 43 | def evaluate_one_image(test_image, test_cls, logs_train_dir): 44 | # display image 45 | # data.display_image(test_image) 46 | 47 | with tf.Graph().as_default(): 48 | x = tf.placeholder(tf.float32, shape=[BATCH_SIZE, image_size, image_size, 3]) 49 | 50 | test_image = np.reshape(test_image, [BATCH_SIZE, image_size, image_size, 3]) 51 | 52 | logit = model.inference(test_image, BATCH_SIZE, N_CLASSES) 53 | logit = tf.nn.softmax(logit) 54 | 55 | saver = tf.train.Saver() 56 | 57 | with tf.Session() as sess: 58 | print("Reading checkpoints...") 59 | ckpt = tf.train.get_checkpoint_state(logs_train_dir) 60 | if ckpt and ckpt.model_checkpoint_path: 61 | global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] 62 | saver.restore(sess, ckpt.model_checkpoint_path) 63 | print("successful loading,global step is %s" % global_step) 64 | else: 65 | print("no checkpoint file founded") 66 | return 67 | prediction = sess.run(logit, feed_dict={x: test_image}) 68 | # print(prediction) 69 | max_index = np.argmax(prediction) 70 | pred_index = classes[max_index] 71 | 72 | print('The true label is:', test_cls) 73 | print('The predict label is:', pred_index) 74 | # plt.show() 75 | return pred_index, max_index 76 | 77 | def evaluate_one_image_with_dir(): 78 | img_dir = input("Enter the dir of your image: ") 79 | image = Image.open(img_dir) 80 | image.show(image) 81 | image = image.resize([68, 68]) 82 | image_array= np.array(image) 83 | 84 | with tf.Graph().as_default(): 85 | BATCH_SIZE = 1 86 | N_CLASSES = 1000 87 | 88 | image = tf.cast(image_array, tf.float32) 89 | image = tf.reshape(image,[68,68,1]) 90 | image = tf.image.per_image_standardization(image) 91 | image = tf.reshape(image, [1, 68, 68, 1]) 92 | 93 | logit = model.inference(image,BATCH_SIZE,N_CLASSES) 94 | logit = tf.nn.softmax(logit) 95 | 96 | x = tf.placeholder(tf.float32,shape=[68,68]) 97 | 98 | logs_train_dir = "D:/APP/PyCharm 2017.1/task/logs_Corel50K/" 99 | saver = tf.train.Saver() 100 | 101 | with tf.Session() as sess: 102 | print("Reading checkpoints...") 103 | ckpt = tf.train.get_checkpoint_state(logs_train_dir) 104 | if ckpt and ckpt.model_checkpoint_path: 105 | global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] 106 | saver.restore(sess,ckpt.model_checkpoint_path) 107 | print("successful loading,global step is %s"%global_step) 108 | else: 109 | print("no checkpoint file founded") 110 | return 111 | prediction = sess.run(logit,feed_dict={x:image_array}) 112 | #print(prediction) 113 | max_index = np.argmax(prediction) 114 | pred_index = classes[max_index] 115 | print(max_index) 116 | 117 | def main(argv=None): # pylint: disable=unused-argument 118 | test_image, test_cls = get_one_image(data_path, classes, image_size) 119 | evaluate_one_image(test_image, test_cls, logs_train_dir) 120 | 121 | 122 | if __name__ == '__main__': 123 | tf.app.run() -------------------------------------------------------------------------------- /face_detect/FaceDetection.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Version 2.6 3 | 4 | Created by: 5 | -Grady Duncan, @aDroidman 6 | 7 | Sources: 8 | http://stackoverflow.com/questions/13211745/detect-face-then-autocrop-pictures 9 | https://gist.github.com/astanin/3097851 10 | ''' 11 | import cv2 12 | import numpy as np 13 | from PIL import Image 14 | import glob 15 | import os 16 | import sys 17 | import time 18 | 19 | # Static 20 | faceCascade = cv2.CascadeClassifier('haarcascade_frontalface_alt.xml') 21 | padding = -1 22 | boxScale = 1 23 | 24 | def DetectFace(image, faceCascade, returnImage=False): 25 | 26 | # variables 27 | min_size = (50, 50) 28 | haar_scale = 1.1 29 | min_neighbors = 3 30 | haar_flags = 0 31 | DOWNSCALE = 4 32 | 33 | # Equalize the histogram 34 | cv2.equalizeHist(image) 35 | 36 | # Detect the faces 37 | faces = faceCascade.detectMultiScale(image) 38 | print(faces) 39 | # faces = cv2.HaarDetectObjects(image, faceCascade, cv.CreateMemStorage(0), haar_scale, min_neighbors, haar_flags, min_size) 40 | 41 | # If faces are found 42 | # if faces!=None and returnImage: 43 | # for ((x, y, w, h), n) in faces: 44 | # 45 | # # Convert bounding box to two CvPoints 46 | # pt1 = (int(x), int(y)) 47 | # pt2 = (int(x + w), int(y + h)) 48 | # cv2.rectangle(image, pt1, pt2, (255, 0, 0), 5, 8, 0) 49 | 50 | if returnImage: 51 | return image 52 | else: 53 | return faces 54 | 55 | def pil2cvGrey(pil_im): 56 | pil_im = pil_im.convert('L') 57 | cv_im = np.array(pil_im) 58 | print(cv_im.shape) 59 | return cv_im 60 | 61 | def imgCrop(image, cropBox, padding): 62 | 63 | # Crop a PIL image with the provided box [x(left), y(upper), w(width), h(height)] 64 | # Calculate scale factors 65 | xPadding = max(cropBox[2] * (boxScale - 1), int(padding)) 66 | yPadding = max(cropBox[3] * (boxScale - 1), int(padding)) 67 | 68 | # Convert cv box to PIL box [left, upper, right, lower] 69 | PIL_box = [cropBox[0] - xPadding, cropBox[1] - yPadding, cropBox[0] + cropBox[2] + xPadding, cropBox[1] + cropBox[3] + yPadding] 70 | 71 | return image.crop(PIL_box) 72 | 73 | def Crop(imagePattern, outputimg, padding, webCheck): 74 | paddingCheck = True 75 | imgList = glob.glob(imagePattern) 76 | img_num = 0 77 | print(len(imgList)) 78 | while paddingCheck: 79 | if len(imgList) <= 0: 80 | return 81 | else: 82 | # Crop images 83 | for img in imgList: 84 | img_num += 1 85 | print(img_num) 86 | if img_num == len(imgList): 87 | sys.exit() 88 | pil_im = Image.open(img) 89 | cv_im = pil2cvGrey(pil_im) 90 | faces = DetectFace(cv_im, faceCascade) 91 | print('face number:', len(faces)) 92 | if len(faces)!=0: 93 | n = 1 94 | for face in faces: 95 | print(face) 96 | croppedImage = imgCrop(pil_im, face, padding) 97 | (fname, ext) = os.path.splitext(img) 98 | fname = os.path.basename(fname) 99 | croppedImage.save(outputimg + '\\' + fname + ' -c' + ext) 100 | n += 1 101 | print('Cropping:', fname) 102 | else: 103 | print('No faces found:', img) 104 | print('Closing application') 105 | time.sleep(.4) 106 | continue 107 | # Send only if capturing from webcam 108 | if webCheck: 109 | return 110 | 111 | def CropSetup(padding, webCheck): 112 | inputimg = '/Users/zhuxinyue/ML/SFEW/val/neutral/' 113 | 114 | # Input folder check 115 | if not os.path.exists(inputimg): 116 | print('Input Folder not found') 117 | outputimg = '/Users/zhuxinyue/ML/SFEW/val2/neutral/' 118 | 119 | # Create output folder if missing 120 | if not os.path.exists(outputimg): 121 | os.makedirs(outputimg) 122 | 123 | # Get padding for crop 124 | while padding < 0: 125 | padding = -1 126 | break 127 | 128 | # Sent to Crop function 129 | Crop(inputimg + '*.png', outputimg, padding, webCheck) 130 | Crop(inputimg + '*.jpg', outputimg, padding, webCheck) 131 | 132 | def main(): 133 | webCheck = False 134 | CropSetup(padding, webCheck) 135 | main() -------------------------------------------------------------------------------- /ops.py: -------------------------------------------------------------------------------- 1 | """ 2 | Most codes from https://github.com/carpedm20/DCGAN-tensorflow 3 | """ 4 | import math 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | from tensorflow.python.framework import ops 9 | from tensorflow.contrib import layers 10 | ds = tf.contrib.distributions 11 | st = tf.contrib.bayesflow.stochastic_tensor 12 | 13 | from utils import * 14 | 15 | if "concat_v2" in dir(tf): 16 | def concat(tensors, axis, *args, **kwargs): 17 | return tf.concat_v2(tensors, axis, *args, **kwargs) 18 | else: 19 | def concat(tensors, axis, *args, **kwargs): 20 | return tf.concat(tensors, axis, *args, **kwargs) 21 | 22 | def bn(x, is_training, scope): 23 | return slim.batch_norm( x, 24 | decay=0.9, 25 | updates_collections=None, 26 | epsilon=1e-5, 27 | scale=True, 28 | is_training=is_training, 29 | scope=scope) 30 | 31 | 32 | 33 | def conv_out_size_same(size, stride): 34 | return int(math.ceil(float(size) / float(stride))) 35 | 36 | def conv_cond_concat(x, y): 37 | """Concatenate conditioning vector on feature map axis.""" 38 | x_shapes = x.get_shape() 39 | y_shapes = y.get_shape() 40 | return concat([x, y*tf.ones([x_shapes[0], x_shapes[1], x_shapes[2], y_shapes[3]])], 3) 41 | 42 | def conv2d(input_, output_dim, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, name="conv2d"): 43 | with tf.variable_scope(name): 44 | w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], 45 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 46 | conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') 47 | 48 | biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) 49 | conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) 50 | 51 | return conv 52 | 53 | def resize_conv(input, out_h, out_w, out_dim, method=1, name='rc'): 54 | with tf.variable_scope(name): 55 | resize = tf.image.resize_images(input, [out_h, out_w], method=method) # Resize NearestNeighbor Method 56 | padding = np.array([0, 0, 1, 1, 1, 1, 0, 0], dtype=np.int32) 57 | padding = np.reshape(padding, [4, 2]) 58 | pad = tf.pad(resize, paddings=padding) 59 | print('padding', pad.shape) 60 | conv = conv2d(resize, out_dim, 3, 3, 1, 1) 61 | print('resize_conv', conv) 62 | return conv 63 | 64 | def deconv2d(input_, output_shape, k_h=5, k_w=5, d_h=2, d_w=2, name="deconv2d", stddev=0.02, with_w=False): 65 | with tf.variable_scope(name): 66 | # filter : [height, width, output_channels, in_channels] 67 | w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]], 68 | initializer=tf.random_normal_initializer(stddev=stddev)) 69 | 70 | try: 71 | deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, strides=[1, d_h, d_w, 1]) 72 | 73 | # Support for verisons of TensorFlow before 0.7.0 74 | except AttributeError: 75 | deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape, strides=[1, d_h, d_w, 1]) 76 | 77 | biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) 78 | deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape()) 79 | 80 | if with_w: 81 | return deconv, w, biases 82 | else: 83 | return deconv 84 | 85 | def lrelu(x, leak=0.2, scope=None): 86 | with tf.name_scope(scope, 'leak_relu', [x, leak]): 87 | if leak < 1: 88 | y = tf.maximum(x, leak * x) 89 | else: 90 | y = tf.minimum(x, leak * x) 91 | return y 92 | 93 | def linear(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False): 94 | shape = input_.get_shape().as_list() 95 | 96 | with tf.variable_scope(scope or "Linear"): 97 | matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32, 98 | tf.random_normal_initializer(stddev=stddev)) 99 | bias = tf.get_variable("bias", [output_size], 100 | initializer=tf.constant_initializer(bias_start)) 101 | if with_w: 102 | return tf.matmul(input_, matrix) + bias, matrix, bias 103 | else: 104 | return tf.matmul(input_, matrix) + bias 105 | 106 | def fc( inputs, 107 | num_outputs, 108 | activation_fn=tf.nn.relu, 109 | normalizer_fn=None, 110 | normalizer_params=None, 111 | weights_initializer=tf.random_normal_initializer(stddev=0.02), 112 | weights_regularizer=None, 113 | biases_initializer=tf.zeros_initializer(), 114 | biases_regularizer=None, 115 | reuse=None, 116 | variables_collections=None, 117 | outputs_collections=None, 118 | trainable=True, 119 | scope=None): 120 | with tf.variable_scope(scope, 'flatten_fully_connected', [inputs]): 121 | if inputs.shape.ndims > 2: 122 | inputs = slim.flatten(inputs) 123 | return slim.fully_connected(inputs, 124 | num_outputs, 125 | activation_fn, 126 | normalizer_fn, 127 | normalizer_params, 128 | weights_initializer, 129 | weights_regularizer, 130 | biases_initializer, 131 | biases_regularizer, 132 | reuse, 133 | variables_collections, 134 | outputs_collections, 135 | trainable, 136 | scope) 137 | 138 | def standard_normal(shape, **kwargs): 139 | """Create a standard Normal StochasticTensor.""" 140 | return tf.cast(st.StochasticTensor( 141 | ds.MultivariateNormalDiag(mu=tf.zeros(shape), diag_stdev=tf.ones(shape), **kwargs)), tf.float32) 142 | 143 | 144 | def encoder(input_tensor, output_size): 145 | '''Create encoder network. 146 | Args: 147 | input_tensor: a batch of flattened images [batch_size, 28*28] 148 | Returns: 149 | A tensor that expresses the encoder network 150 | ''' 151 | net = tf.reshape(input_tensor, [-1, 64, 64, 1]) 152 | net = layers.conv2d(net, 32, 5, stride=2, activation_fn=tf.nn.relu, weights_initializer=tf.contrib.layers.xavier_initializer()) 153 | net = layers.conv2d(net, 64, 5, stride=2, activation_fn=tf.nn.relu, weights_initializer=tf.contrib.layers.xavier_initializer()) 154 | net = layers.conv2d(net, 128, 5, stride=2, padding='VALID', activation_fn=tf.nn.relu, weights_initializer=tf.contrib.layers.xavier_initializer()) 155 | net = layers.dropout(net, keep_prob=0.9) 156 | net = layers.flatten(net) 157 | print(net) 158 | return layers.fully_connected(net, output_size, activation_fn=tf.nn.sigmoid, weights_initializer=tf.contrib.layers.xavier_initializer()) 159 | 160 | def last_conv(input_, stddev=0.02, reuse=False, name=None): 161 | """ Last convolutional layer of discriminator network 162 | (1 filter with size 4x4, stride 1) 163 | Args: 164 | input: 4D tensor 165 | reuse: boolean 166 | use_sigmoid: boolean (False if use lsgan) 167 | name: string, e.g. 'C64' 168 | """ 169 | with tf.variable_scope(name, reuse=reuse): 170 | weights = tf.get_variable('weights', [4, 4, input_.get_shape()[3], 1], 171 | initializer=tf.truncated_normal_initializer(stddev=stddev)) 172 | biases = tf.get_variable('biases', [1], initializer=tf.constant_initializer(0.0)) 173 | conv = tf.nn.conv2d(input_, weights, 174 | strides=[1, 1, 1, 1], padding='SAME') 175 | output = conv + biases 176 | return output 177 | 178 | def Rk(input, k, reuse=False, norm='instance', is_training=True, name=None): 179 | """ A residual block that contains two 3x3 convolutional layers 180 | with the same number of filters on both layer 181 | Args: 182 | input: 4D Tensor 183 | k: integer, number of filters (output depth) 184 | reuse: boolean 185 | name: string 186 | Returns: 187 | 4D tensor (same shape as input) 188 | """ 189 | with tf.variable_scope(name, reuse=reuse): 190 | with tf.variable_scope('layer1', reuse=reuse): 191 | weights1 = tf.get_variable('weights1', [3, 3, input.get_shape()[3], k], 192 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 193 | 194 | padded1 = tf.pad(input, [[0,0],[1,1],[1,1],[0,0]], 'REFLECT') 195 | conv1 = tf.nn.conv2d(padded1, weights1, 196 | strides=[1, 1, 1, 1], padding='VALID') 197 | normalized1 = _instance_norm(conv1) 198 | relu1 = tf.nn.relu(normalized1) 199 | 200 | with tf.variable_scope('layer2', reuse=reuse): 201 | weights2 = _weights("weights2", 202 | shape=[3, 3, relu1.get_shape()[3], k]) 203 | 204 | padded2 = tf.pad(relu1, [[0,0],[1,1],[1,1],[0,0]], 'REFLECT') 205 | conv2 = tf.nn.conv2d(padded2, weights2, 206 | strides=[1, 1, 1, 1], padding='VALID') 207 | normalized2 = _instance_norm(conv2) 208 | output = input+normalized2 209 | return output 210 | 211 | def n_res_blocks(input, reuse, norm='instance', is_training=True, n=6): 212 | depth = input.get_shape()[3] 213 | for i in range(1,n+1): 214 | output = Rk(input, depth, reuse, norm, is_training, 'R{}_{}'.format(depth, i)) 215 | input = output 216 | return output 217 | 218 | def _instance_norm(input): 219 | """ Instance Normalization 220 | """ 221 | with tf.variable_scope("instance_norm"): 222 | depth = input.get_shape()[3] 223 | scale = _weights("scale", [depth], mean=1.0) 224 | offset = _biases("offset", [depth]) 225 | mean, variance = tf.nn.moments(input, axes=[1,2], keep_dims=True) 226 | epsilon = 1e-5 227 | inv = tf.rsqrt(variance + epsilon) 228 | normalized = (input-mean)*inv 229 | return scale*normalized + offset 230 | 231 | 232 | def uk(input, k, reuse=False, name=None, output_size=None): 233 | """ A 3x3 fractional-strided-Convolution-BatchNorm-ReLU layer 234 | with k filters, stride 1/2 235 | Args: 236 | input: 4D tensor 237 | k: integer, number of filters (output depth) 238 | norm: 'instance' or 'batch' or None 239 | is_training: boolean or BoolTensor 240 | reuse: boolean 241 | name: string, e.g. 'c7sk-32' 242 | output_size: integer, desired output size of layer 243 | Returns: 244 | 4D tensor 245 | """ 246 | with tf.variable_scope(name, reuse=reuse): 247 | input_shape = input.get_shape().as_list() 248 | 249 | weights = _weights("weights", 250 | shape=[3, 3, k, input_shape[3]]) 251 | 252 | if not output_size: 253 | output_size = input_shape[1]*2 254 | output_shape = [input_shape[0], output_size, output_size, k] 255 | fsconv = tf.nn.conv2d_transpose(input, weights, 256 | output_shape=output_shape, 257 | strides=[1, 2, 2, 1], padding='SAME') 258 | normalized = _instance_norm(fsconv) 259 | output = tf.nn.relu(normalized) 260 | return output 261 | 262 | def _weights(name, shape, mean=0.0, stddev=0.02): 263 | """ Helper to create an initialized Variable 264 | Args: 265 | name: name of the variable 266 | shape: list of ints 267 | mean: mean of a Gaussian 268 | stddev: standard deviation of a Gaussian 269 | Returns: 270 | A trainable variable 271 | """ 272 | var = tf.get_variable( 273 | name, shape, 274 | initializer=tf.random_normal_initializer( 275 | mean=mean, stddev=stddev, dtype=tf.float32)) 276 | return var 277 | 278 | def _biases(name, shape, constant=0.0): 279 | """ Helper to create an initialized Bias with constant 280 | """ 281 | return tf.get_variable(name, shape, 282 | initializer=tf.constant_initializer(constant)) -------------------------------------------------------------------------------- /train_wgan.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 2017/9/21 下午10:31 3 | # @Author : Zhuxinyue_Sharon 4 | # @Email : zxybuptsee@163.com 5 | # @File : train_wgan.py 6 | # @Software : PyCharm 7 | 8 | from __future__ import absolute_import 9 | from __future__ import division 10 | from __future__ import print_function 11 | 12 | from WGAN import WGAN 13 | import sys 14 | import os 15 | os.environ['TF_CPP_MIN_LOG_LEVEL']='2' 16 | 17 | import tensorflow as tf 18 | 19 | # EPOCH = 2400 # 2399 20 | EPOCH = 1500 21 | BATCH_SIZE = 64 22 | Z_DIM = 100 23 | DATASET = 'face_emotion' 24 | CHECKPOINT_DIR = 'wgan_emotion/checkpoint/' 25 | RESULT_DIR = '/Users/zhuxinyue/ML/Gen_Caltech101/face_emotion/' 26 | LOG_DIR = 'wgan_emotion/log/' 27 | 28 | """main""" 29 | def main(): 30 | 31 | # open session 32 | with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: 33 | # declare instance for GAN 34 | classes = ['angry'] 35 | gan = WGAN(classes=classes, sess=sess, epoch=EPOCH, batch_size=BATCH_SIZE, 36 | z_dim=Z_DIM, dataset_name=DATASET, augmentation=False, aug_ratio=12, 37 | checkpoint_dir=CHECKPOINT_DIR, result_dir=RESULT_DIR, log_dir=LOG_DIR) 38 | 39 | 40 | # build graph 41 | gan.build_model() 42 | 43 | # show network architecture 44 | # show_all_variables() 45 | 46 | # launch the graph in a session 47 | gan.train() 48 | print(" [*] Training finished!") 49 | 50 | # visualize learned generator 51 | # gan.visualize_results(EPOCH - 1) 52 | print(" [*] Testing finished!") 53 | 54 | if __name__ == '__main__': 55 | main() -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Most codes from https://github.com/carpedm20/DCGAN-tensorflow 3 | """ 4 | from __future__ import division 5 | import math 6 | import random 7 | import pprint 8 | import scipy.misc 9 | import numpy as np 10 | from time import gmtime, strftime 11 | import matplotlib.pyplot as plt 12 | from scipy import misc 13 | import imgaug as ia 14 | from imgaug import augmenters as iaa 15 | 16 | import os 17 | import sys 18 | 19 | import tensorflow as tf 20 | import tensorflow.contrib.slim as slim 21 | import cv2 22 | import data 23 | 24 | def display_image(image, gray=False): 25 | dis_image = image.astype(np.uint8) 26 | plt.figure() 27 | 28 | if gray: 29 | plt.imshow(dis_image, cmap='gray') 30 | else: 31 | plt.imshow(dis_image) 32 | 33 | def load_caltech101(data_path, image_size): 34 | images = [] 35 | labels = [] 36 | files = os.listdir(data_path) 37 | 38 | for fl in files: 39 | file = data_path + fl 40 | image = misc.imread(file, mode='L') 41 | image = misc.imresize(image, [image_size, image_size]) 42 | 43 | '''test image''' 44 | # print(image.shape) 45 | # display_image(image, True) 46 | # plt.show() 47 | # sys.exit() 48 | 49 | image = np.reshape(image, [image_size, image_size, -1]) 50 | # print(image.shape) 51 | images.append(image) 52 | 53 | # label = 1.0 54 | # labels.append(label) 55 | 56 | images_uint = np.array(images, dtype=np.uint8) 57 | 58 | images = np.array(images, dtype=np.float32) 59 | # labels = np.array(labels, dtype=np.int64) 60 | 61 | print('Loding training data completed') 62 | print('images_shape:', images.shape) 63 | # print('labels_shape:', labels.shape) 64 | 65 | return images_uint, images/255 66 | 67 | def data_augmentation(ratio, images): 68 | ''' 69 | Augmentation of images [batch_size, h, w, channel] 70 | :param ratio: the number of augmentation of each image 71 | :param images: 72 | :return: [batch_size * ratio, h, w, channel], normed 73 | ''' 74 | images_aug = None 75 | seq = [] 76 | st = lambda aug: iaa.Sometimes(0.3, aug) 77 | for i in range(ratio): 78 | seq_one = iaa.Sequential([ 79 | # iaa.Fliplr(0.5), # horizontally flip 50% of all images 80 | # iaa.Flipud(0.5), # vertically flip 50% of all images 81 | # st(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), 82 | # convert images into their superpixel representation 83 | st(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width 84 | # st(iaa.GaussianBlur((0, 3.0))), # blur images with a sigma between 0 and 3.0 85 | st(iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5))), # sharpen images 86 | st(iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0))), # emboss images 87 | # search either for all edges or for directed edges 88 | st(iaa.Sometimes(0.5, 89 | iaa.EdgeDetect(alpha=(0, 0.7)), 90 | iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)), 91 | )), 92 | # st(iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.2), per_channel=0.5)), # add gaussian noise to images 93 | # st(iaa.Dropout((0.0, 0.1), per_channel=0.5)), # randomly remove up to 10% of the pixels 94 | # st(iaa.Invert(0.25, per_channel=True)), # invert color channels 95 | st(iaa.Add((-10, 10), per_channel=0.5)), # change brightness of images (by -10 to 10 of original value) 96 | st(iaa.Multiply((0.5, 1.5), per_channel=0.5)), # change brightness of images (50-150% of original value) 97 | st(iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5)), # improve or worsen the contrast 98 | st(iaa.Affine( 99 | scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, 100 | # scale images to 80-120% of their size, individually per axis 101 | translate_px={"x": (-16, 16), "y": (-16, 16)}, # translate by -16 to +16 pixels (per axis) 102 | rotate=(-45, 45), # rotate by -45 to +45 degrees 103 | shear=(-16, 16), # shear by -16 to +16 degrees 104 | order=ia.ALL, # use any of scikit-image's interpolation methods 105 | cval=(0, 255), # if mode is constant, use a cval between 0 and 255 106 | mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples) 107 | )), 108 | # st(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)) 109 | # apply elastic transformations with random strengths 110 | ], 111 | random_order=True # do all of the above in random order 112 | ) 113 | seq.append(seq_one) 114 | if i == 0: 115 | images_aug = seq[0].augment_images(images) 116 | else: 117 | aug_temp = seq[i].augment_images(images) 118 | images_aug = np.concatenate((images_aug, aug_temp), axis=0) 119 | 120 | print('Augmentation shape', images_aug.shape) 121 | images_aug = np.array(images_aug, dtype=np.float32) 122 | return images_aug/255 123 | 124 | 125 | def check_folder(log_dir): 126 | if not os.path.exists(log_dir): 127 | os.makedirs(log_dir) 128 | return log_dir 129 | 130 | def show_all_variables(): 131 | model_vars = tf.trainable_variables() 132 | slim.model_analyzer.analyze_vars(model_vars, print_info=True) 133 | 134 | def get_image(image_path, input_height, input_width, resize_height=64, resize_width=64, crop=True, grayscale=False): 135 | image = imread(image_path, grayscale) 136 | return transform(image, input_height, input_width, resize_height, resize_width, crop) 137 | 138 | def save_images(images, size, image_path): 139 | return imsave(inverse_transform(images), size, image_path) 140 | 141 | def imread(path, grayscale = False): 142 | if (grayscale): 143 | return scipy.misc.imread(path, flatten = True).astype(np.float) 144 | else: 145 | return scipy.misc.imread(path).astype(np.float) 146 | 147 | def merge_images(images, size): 148 | return inverse_transform(images) 149 | 150 | def merge(images, size): 151 | h, w = images.shape[1], images.shape[2] 152 | if (images.shape[3] in (3,4)): 153 | c = images.shape[3] 154 | img = np.zeros((h * size[0], w * size[1], c)) 155 | for idx, image in enumerate(images): 156 | i = idx % size[1] 157 | j = idx // size[1] 158 | img[j * h:j * h + h, i * w:i * w + w, :] = image 159 | return img 160 | elif images.shape[3]==1: 161 | img = np.zeros((h * size[0], w * size[1])) 162 | for idx, image in enumerate(images): 163 | i = idx % size[1] 164 | j = idx // size[1] 165 | img[j * h:j * h + h, i * w:i * w + w] = image[:,:,0] 166 | return img 167 | else: 168 | raise ValueError('in merge(images,size) images parameter ''must have dimensions: HxW or HxWx3 or HxWx4') 169 | 170 | def imsave(images, size, path): 171 | image = np.squeeze(merge(images, size)) 172 | return scipy.misc.imsave(path, image) 173 | 174 | def center_crop(x, crop_h, crop_w, resize_h=64, resize_w=64): 175 | if crop_w is None: 176 | crop_w = crop_h 177 | h, w = x.shape[:2] 178 | j = int(round((h - crop_h)/2.)) 179 | i = int(round((w - crop_w)/2.)) 180 | return scipy.misc.imresize(x[j:j+crop_h, i:i+crop_w], [resize_h, resize_w]) 181 | 182 | def transform(image, input_height, input_width, resize_height=64, resize_width=64, crop=True): 183 | if crop: 184 | cropped_image = center_crop(image, input_height, input_width, resize_height, resize_width) 185 | else: 186 | cropped_image = scipy.misc.imresize(image, [resize_height, resize_width]) 187 | return np.array(cropped_image)/127.5 - 1. 188 | 189 | def inverse_transform(images): 190 | return (images+1.)/2. 191 | 192 | """ Drawing Tools """ 193 | # borrowed from https://github.com/ykwon0407/variational_autoencoder/blob/master/variational_bayes.ipynb 194 | def save_scattered_image(z, id, z_range_x, z_range_y, name='scattered_image.jpg'): 195 | N = 10 196 | plt.figure(figsize=(8, 6)) 197 | plt.scatter(z[:, 0], z[:, 1], c=np.argmax(id, 1), marker='o', edgecolor='none', cmap=discrete_cmap(N, 'jet')) 198 | plt.colorbar(ticks=range(N)) 199 | axes = plt.gca() 200 | axes.set_xlim([-z_range_x, z_range_x]) 201 | axes.set_ylim([-z_range_y, z_range_y]) 202 | plt.grid(True) 203 | plt.savefig(name) 204 | 205 | # borrowed from https://gist.github.com/jakevdp/91077b0cae40f8f8244a 206 | def discrete_cmap(N, base_cmap=None): 207 | """Create an N-bin discrete colormap from the specified input map""" 208 | 209 | # Note that if base_cmap is a string or None, you can simply do 210 | # return plt.cm.get_cmap(base_cmap, N) 211 | # The following works for string, None, or a colormap instance: 212 | 213 | base = plt.cm.get_cmap(base_cmap) 214 | color_list = base(np.linspace(0, 1, N)) 215 | cmap_name = base.name + str(N) 216 | return base.from_list(cmap_name, color_list, N) --------------------------------------------------------------------------------