├── MNISTfashion.py ├── README.md ├── back_image.py ├── back_rand.py ├── basic.py ├── code ├── evaluate.py ├── evolve.py ├── get_data.py ├── individual.py ├── layers.py ├── main.py ├── population.py └── utils.py ├── convex.py ├── rectangle.py ├── rectangle_image.py ├── rot.py └── rot_back_image.py /MNISTfashion.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.slim as slim 3 | from tensorflow.python.ops import control_flow_ops 4 | from datetime import datetime 5 | import numpy as np 6 | import os 7 | import mnist_fashion_input as data 8 | 9 | batch_size = 128 10 | total_epochs = 100 11 | 12 | def model(): 13 | is_training = tf.placeholder(tf.bool, []) 14 | train_images, train_label = data.get_train_data(batch_size) 15 | test_images, test_label = data.get_test_data(batch_size) 16 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 17 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 18 | y_ = tf.cast(y_, tf.int64) 19 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 20 | activation_fn=tf.nn.crelu, 21 | normalizer_fn=slim.batch_norm, 22 | weights_regularizer=slim.l2_regularizer(0.005), 23 | normalizer_params={'is_training': is_training, 'decay': 0.95} 24 | ): 25 | conv1 =slim.conv2d(x, 88, [12,12], weights_initializer=tf.truncated_normal_initializer(mean=-0.09, stddev=0.01)) 26 | pool1 = slim.max_pool2d(conv1, [4,4], stride=4, padding='SAME') 27 | flatten = slim.flatten(pool1) 28 | full1 = slim.fully_connected(flatten, 1157, weights_initializer=tf.truncated_normal_initializer(mean=-0.2327, stddev=0.64162), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 29 | logits = slim.fully_connected(full1, 10, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=-0.55677732, stddev=0.061221), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 30 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 31 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 32 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 33 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 34 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 35 | 36 | lr = tf.train.exponential_decay(0.1, 37 | step, 38 | 550*30, 39 | 0.9, 40 | staircase=True) 41 | 42 | 43 | optimizer = tf.train.GradientDescentOptimizer(lr) 44 | # optimizer = tf.train.AdamOptimizer(0.001) 45 | lr_summary = tf.summary.scalar('lr', lr) 46 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 47 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 48 | if update_ops: 49 | updates = tf.group(*update_ops) 50 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 51 | 52 | loss_summary = tf.summary.scalar('loss', cross_entropy) 53 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 54 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary, lr_summary]) 55 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 56 | 57 | def train(): 58 | gpu_options = tf.GPUOptions(allow_growth=True) 59 | is_training, train_step, _, accuracy, loss, merge_summary = model() 60 | 61 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 62 | sess.run(tf.global_variables_initializer()) 63 | train_data_length = 60000 64 | test_data_length = 10000 65 | steps_in_each_epoch = (train_data_length//batch_size) 66 | total_steps = int(total_epochs*steps_in_each_epoch) 67 | coord = tf.train.Coordinator() 68 | threads = tf.train.start_queue_runners(sess, coord) 69 | try: 70 | for i in range(total_steps): 71 | if coord.should_stop(): 72 | break 73 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 74 | if i % steps_in_each_epoch == 0: 75 | test_total_step = test_data_length//batch_size 76 | test_accuracy_list = [] 77 | test_loss_list = [] 78 | for _ in range(test_total_step): 79 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 80 | test_accuracy_list.append(test_accuracy_str) 81 | test_loss_list.append(test_loss_str) 82 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 83 | 84 | except tf.errors.OutOfRangeError: 85 | print('done') 86 | finally: 87 | coord.request_stop() 88 | coord.join(threads) 89 | if __name__ =='__main__': 90 | #CUDA1 91 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 92 | tf.reset_default_graph() 93 | train() 94 | # 93.45 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --update... 2 | Now, I have release the algorithm into the dictionary "code". 3 | 4 | 5 | All codes are implemented from the architecture and weight initialization optimized by my own algorithm that will be released soon. 6 | 7 | These codes have been tested in the GPU with NVIDIA 1080, and written by Tensorflow1.3. 8 | 9 | Feel free to contact me when encouter the propoblems in reproducing the results. 10 | -------------------------------------------------------------------------------- /back_image.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Length:5, Num:644503, Mean:0.91, Std:0.03, 3 | conv[2,2,39,0.54,0.24], 4 | conv[6,6,8,0.07,0.57], 5 | pool[4,4,0.61], 6 | full[1598,-0.10436102541965764,0.4068350966991021], 7 | full[10,-0.24498672149431777,0.477889097823633] 8 | 9 | ''' 10 | 11 | import tensorflow as tf 12 | import tensorflow.contrib.slim as slim 13 | from tensorflow.python.ops import control_flow_ops 14 | from datetime import datetime 15 | import numpy as np 16 | import os 17 | import get_data as data 18 | import tensorflow as tf 19 | 20 | batch_size = 100 21 | total_epochs = 100 22 | 23 | def model(): 24 | is_training = tf.placeholder(tf.bool, []) 25 | train_images, train_label = data.get_train_data(batch_size) 26 | test_images, test_label = data.get_test_data(batch_size) 27 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 28 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 29 | y_ = tf.cast(y_, tf.int64) 30 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 31 | activation_fn=tf.nn.crelu, 32 | normalizer_fn=slim.batch_norm, 33 | weights_regularizer=slim.l2_regularizer(0.005), 34 | normalizer_params={'is_training': is_training, 'decay': 0.95} 35 | ): 36 | conv1 =slim.conv2d(x, 39, [1,1], weights_initializer=tf.truncated_normal_initializer(mean=0.54, stddev=0.24)) 37 | conv2 =slim.conv2d(conv1, 8, [6,6], weights_initializer=tf.truncated_normal_initializer(mean=0.07, stddev=0.57)) 38 | pool1 = slim.avg_pool2d(conv2, [4,4], stride=4, padding='SAME') 39 | flatten = slim.flatten(pool1) 40 | full1 = slim.fully_connected(flatten, 1598, weights_initializer=tf.truncated_normal_initializer(mean=-0.1043610, stddev=0.40), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 41 | logits = slim.fully_connected(full1, 10, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=-0.244986, stddev=0.477889), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 42 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 43 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 44 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 45 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 46 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 47 | 48 | # lr = tf.train.exponential_decay(0.1, 49 | # step, 50 | # 550*30, 51 | # 0.9, 52 | # staircase=True) 53 | # 54 | # 55 | # optimizer = tf.train.GradientDescentOptimizer(lr) 56 | optimizer = tf.train.AdamOptimizer(0.001) 57 | # lr_summary = tf.summary.scalar('lr', lr) 58 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 59 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 60 | if update_ops: 61 | updates = tf.group(*update_ops) 62 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 63 | 64 | loss_summary = tf.summary.scalar('loss', cross_entropy) 65 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 66 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 67 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 68 | 69 | def train(): 70 | gpu_options = tf.GPUOptions(allow_growth=True) 71 | is_training, train_step, _, accuracy, loss, merge_summary = model() 72 | 73 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 74 | sess.run(tf.global_variables_initializer()) 75 | train_data_length = 10000 76 | test_data_length = 50000 77 | steps_in_each_epoch = (train_data_length//batch_size) 78 | total_steps = int(total_epochs*steps_in_each_epoch) 79 | coord = tf.train.Coordinator() 80 | threads = tf.train.start_queue_runners(sess, coord) 81 | try: 82 | for i in range(total_steps): 83 | if coord.should_stop(): 84 | break 85 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 86 | if i % steps_in_each_epoch == 0: 87 | test_total_step = test_data_length//batch_size 88 | test_accuracy_list = [] 89 | test_loss_list = [] 90 | for _ in range(test_total_step): 91 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 92 | test_accuracy_list.append(test_accuracy_str) 93 | test_loss_list.append(test_loss_str) 94 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 95 | 96 | except tf.errors.OutOfRangeError: 97 | print('done') 98 | finally: 99 | coord.request_stop() 100 | coord.join(threads) 101 | if __name__ =='__main__': 102 | #CUDA2 103 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 104 | tf.reset_default_graph() 105 | train() 106 | -------------------------------------------------------------------------------- /back_rand.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.slim as slim 3 | from tensorflow.python.ops import control_flow_ops 4 | from datetime import datetime 5 | import numpy as np 6 | import os 7 | import get_data as data 8 | ''' 9 | Length:5, Num:795078, Mean:0.91, Std:0.03, 10 | conv[2,2,24,0.72,0.38], 11 | conv[4,4,11,0.26,0.13], 12 | pool[4,4,0.70], 13 | full[1445,-0.9305572400603059,0.7123493284279396], 14 | full[10,-0.05645566288281878,0.3516372640499208] 15 | 16 | 17 | ''' 18 | 19 | 20 | batch_size = 100 21 | total_epochs = 100 22 | 23 | def model(): 24 | is_training = tf.placeholder(tf.bool, []) 25 | train_images, train_label = data.get_train_data(batch_size) 26 | test_images, test_label = data.get_test_data(batch_size) 27 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 28 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 29 | y_ = tf.cast(y_, tf.int64) 30 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 31 | activation_fn=tf.nn.crelu, 32 | normalizer_fn=slim.batch_norm, 33 | weights_regularizer=slim.l2_regularizer(0.005), 34 | normalizer_params={'is_training': is_training, 'decay': 0.95} 35 | ): 36 | conv1 =slim.conv2d(x, 24, [2,2], weights_initializer=tf.truncated_normal_initializer(mean=0.72, stddev=0.38)) 37 | conv2 =slim.conv2d(conv1, 11, [4,4], weights_initializer=tf.truncated_normal_initializer(mean=0.26, stddev=0.13)) 38 | pool1 = slim.avg_pool2d(conv2, [4,4], stride=4, padding='SAME') 39 | flatten = slim.flatten(pool1) 40 | full1 = slim.fully_connected(flatten, 1445, weights_initializer=tf.truncated_normal_initializer(mean=-0.93055, stddev=0.71234), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 41 | logits = slim.fully_connected(full1, 10, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=-0.05645566, stddev=0.3516), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 42 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 43 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 44 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 45 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 46 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 47 | 48 | # lr = tf.train.exponential_decay(0.1, 49 | # step, 50 | # 550*30, 51 | # 0.9, 52 | # staircase=True) 53 | # 54 | # 55 | # optimizer = tf.train.GradientDescentOptimizer(lr) 56 | optimizer = tf.train.AdamOptimizer(0.001) 57 | # lr_summary = tf.summary.scalar('lr', lr) 58 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 59 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 60 | if update_ops: 61 | updates = tf.group(*update_ops) 62 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 63 | 64 | loss_summary = tf.summary.scalar('loss', cross_entropy) 65 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 66 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 67 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 68 | 69 | def train(): 70 | gpu_options = tf.GPUOptions(allow_growth=True) 71 | is_training, train_step, _, accuracy, loss, merge_summary = model() 72 | 73 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 74 | sess.run(tf.global_variables_initializer()) 75 | train_data_length = 10000 76 | test_data_length = 50000 77 | steps_in_each_epoch = (train_data_length//batch_size) 78 | total_steps = int(total_epochs*steps_in_each_epoch) 79 | coord = tf.train.Coordinator() 80 | threads = tf.train.start_queue_runners(sess, coord) 81 | try: 82 | for i in range(total_steps): 83 | if coord.should_stop(): 84 | break 85 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 86 | if i % steps_in_each_epoch == 0: 87 | test_total_step = test_data_length//batch_size 88 | test_accuracy_list = [] 89 | test_loss_list = [] 90 | for _ in range(test_total_step): 91 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 92 | test_accuracy_list.append(test_accuracy_str) 93 | test_loss_list.append(test_loss_str) 94 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 95 | 96 | except tf.errors.OutOfRangeError: 97 | print('done') 98 | finally: 99 | coord.request_stop() 100 | coord.join(threads) 101 | if __name__ =='__main__': 102 | #CUDA2 103 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 104 | tf.reset_default_graph() 105 | train() #96.41 106 | -------------------------------------------------------------------------------- /basic.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.slim as slim 3 | from tensorflow.python.ops import control_flow_ops 4 | from datetime import datetime 5 | import numpy as np 6 | import os 7 | import get_data as data 8 | ''' 9 | Length:5, Num:1225694, Mean:0.98, Std:0.01, 10 | conv[7,7,23,0.18,0.87], 11 | pool[2,2,0.60], 12 | pool[2,2,0.18], 13 | full[1076,0.25822218285383824,0.6626911711480307], 14 | full[10,0.0670992643760644,0.6910514760392503] 15 | ''' 16 | 17 | batch_size = 100 18 | total_epochs = 100 19 | 20 | def model(): 21 | is_training = tf.placeholder(tf.bool, []) 22 | train_images, train_label = data.get_train_data(batch_size) 23 | test_images, test_label = data.get_test_data(batch_size) 24 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 25 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 26 | y_ = tf.cast(y_, tf.int64) 27 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 28 | activation_fn=tf.nn.crelu, 29 | normalizer_fn=slim.batch_norm, 30 | weights_regularizer=slim.l2_regularizer(0.005), 31 | normalizer_params={'is_training': is_training, 'decay': 0.95} 32 | ): 33 | conv1 =slim.conv2d(x, 23, [7,7], weights_initializer=tf.truncated_normal_initializer(mean=0.18, stddev=0.87)) 34 | pool1 = slim.avg_pool2d(conv1, [2,2], stride=2, padding='SAME') 35 | pool2 = slim.max_pool2d(pool1, [2,2], stride=2, padding='SAME') 36 | flatten = slim.flatten(pool2) 37 | full1 = slim.fully_connected(flatten, 1076, weights_initializer=tf.truncated_normal_initializer(mean=0.258222, stddev=0.662691), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 38 | logits = slim.fully_connected(full1, 10, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=0.0670, stddev=0.691), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 39 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 40 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 41 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 42 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 43 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 44 | 45 | # lr = tf.train.exponential_decay(0.1, 46 | # step, 47 | # 550*30, 48 | # 0.9, 49 | # staircase=True) 50 | # 51 | # 52 | # optimizer = tf.train.GradientDescentOptimizer(lr) 53 | optimizer = tf.train.AdamOptimizer(0.001) 54 | # lr_summary = tf.summary.scalar('lr', lr) 55 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 56 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 57 | if update_ops: 58 | updates = tf.group(*update_ops) 59 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 60 | 61 | loss_summary = tf.summary.scalar('loss', cross_entropy) 62 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 63 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 64 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 65 | 66 | def train(): 67 | gpu_options = tf.GPUOptions(allow_growth=True) 68 | is_training, train_step, _, accuracy, loss, merge_summary = model() 69 | 70 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 71 | sess.run(tf.global_variables_initializer()) 72 | train_data_length = 10000 73 | test_data_length = 50000 74 | steps_in_each_epoch = (train_data_length//batch_size) 75 | total_steps = int(total_epochs*steps_in_each_epoch) 76 | coord = tf.train.Coordinator() 77 | threads = tf.train.start_queue_runners(sess, coord) 78 | try: 79 | for i in range(total_steps): 80 | if coord.should_stop(): 81 | break 82 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 83 | if i % steps_in_each_epoch == 0: 84 | test_total_step = test_data_length//batch_size 85 | test_accuracy_list = [] 86 | test_loss_list = [] 87 | for _ in range(test_total_step): 88 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 89 | test_accuracy_list.append(test_accuracy_str) 90 | test_loss_list.append(test_loss_str) 91 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 92 | 93 | except tf.errors.OutOfRangeError: 94 | print('done') 95 | finally: 96 | coord.request_stop() 97 | coord.join(threads) 98 | if __name__ =='__main__': 99 | #CUDA2 100 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 101 | tf.reset_default_graph() 102 | train() #98.72 103 | -------------------------------------------------------------------------------- /code/evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.slim as slim 3 | from tensorflow.python.ops import control_flow_ops 4 | from tensorflow.python.ops import init_ops 5 | from population import Population 6 | from individual import Individual 7 | import numpy as np 8 | import collections 9 | import timeit 10 | import os 11 | import pickle 12 | import utils 13 | from datetime import datetime 14 | import get_data 15 | 16 | 17 | 18 | 19 | class Evaluate: 20 | 21 | def __init__(self, pops, train_data, train_label, validate_data, validate_label, number_of_channel, epochs, batch_size, train_data_length, validate_data_length): 22 | self.pops = pops 23 | self.train_data = train_data # train or test data. data[0] is images and data[1] are label 24 | self.train_label = train_label 25 | self.validate_data = validate_data 26 | self.validate_label = validate_label 27 | self.number_of_channel = number_of_channel 28 | self.epochs = epochs 29 | self.batch_size = batch_size 30 | self.train_data_length = train_data_length 31 | self.validate_data_length = validate_data_length 32 | ''' 33 | Parse the chromosome in the population to the information which can be directly employed by TensorFLow 34 | ''' 35 | def parse_population(self, gen_no): 36 | save_dir = os.getcwd() + '/save_data/gen_{:03d}'.format(gen_no) 37 | tf.gfile.MakeDirs(save_dir) 38 | history_best_score = 0 39 | for i in range(self.pops.get_pop_size()): 40 | indi = self.pops.get_individual_at(i) 41 | rs_mean, rs_std, num_connections, new_best = self.parse_individual(indi, self.number_of_channel, i, save_dir, history_best_score) 42 | #rs_mean, rs_std, num_connections, new_best = np.random.random(), np.random.random(), np.random.random_integers(1000, 100000), -1 43 | indi.mean = rs_mean 44 | indi.std = rs_std 45 | indi.complxity = num_connections 46 | history_best_score = new_best 47 | list_save_path = os.getcwd() + '/save_data/gen_{:03d}/pop.txt'.format(gen_no) 48 | utils.save_append_individual(str(indi), list_save_path) 49 | 50 | pop_list = self.pops 51 | list_save_path = os.getcwd() + '/save_data/gen_{:03d}/pop.dat'.format(gen_no) 52 | with open(list_save_path, 'wb') as file_handler: 53 | pickle.dump(pop_list, file_handler) 54 | 55 | 56 | def build_graph(self, indi_index, num_of_input_channel, indi, train_data, train_label, validate_data, validate_label): 57 | is_training = tf.placeholder(tf.bool, []) 58 | X = tf.cond(is_training, lambda:train_data, lambda:validate_data) 59 | y_ = tf.cond(is_training, lambda:train_label, lambda:validate_label) 60 | true_Y = tf.cast(y_, tf.int64) 61 | 62 | name_preffix = 'I_{}'.format(indi_index) 63 | num_of_units = indi.get_layer_size() 64 | 65 | ################# variable for convolution operation####################################### 66 | last_output_feature_map_size = num_of_input_channel 67 | ############### state the connection numbers################################################ 68 | num_connections = 0 69 | ############################################################################################ 70 | output_list = [] 71 | output_list.append(X) 72 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 73 | activation_fn=tf.nn.crelu, 74 | normalizer_fn=slim.batch_norm, 75 | #weights_regularizer=slim.l2_regularizer(0.005), 76 | normalizer_params={'is_training': is_training, 'decay': 0.99}): 77 | 78 | for i in range(num_of_units): 79 | current_unit = indi.get_layer_at(i) 80 | if current_unit.type == 1: 81 | name_scope = '{}_conv_{}'.format(name_preffix, i) 82 | with tf.variable_scope(name_scope): 83 | filter_size = [current_unit.filter_width, current_unit.filter_height] 84 | mean=current_unit.weight_matrix_mean 85 | stddev=current_unit.weight_matrix_std 86 | conv_H = slim.conv2d(output_list[-1], current_unit.feature_map_size, filter_size, weights_initializer=tf.truncated_normal_initializer(mean=mean, stddev=stddev), biases_initializer=init_ops.constant_initializer(0.1, dtype=tf.float32)) 87 | output_list.append(conv_H) 88 | # update for next usage 89 | last_output_feature_map_size = current_unit.feature_map_size 90 | num_connections += current_unit.feature_map_size*current_unit.filter_width*current_unit.filter_height+current_unit.feature_map_size 91 | elif current_unit.type == 2: 92 | with tf.variable_scope('{}_pool_{}'.format(name_preffix, i)): 93 | kernel_size = [current_unit.kernel_width, current_unit.kernel_height] 94 | if current_unit.kernel_type < 0.5: 95 | pool_H = slim.max_pool2d(output_list[-1], kernel_size=kernel_size, stride=kernel_size, padding='SAME') 96 | else: 97 | pool_H = slim.avg_pool2d(output_list[-1], kernel_size=kernel_size, stride=kernel_size, padding='SAME') 98 | output_list.append(pool_H) 99 | # pooling operation does not change the number of channel size, but channge the output size 100 | last_output_feature_map_size = last_output_feature_map_size 101 | num_connections += last_output_feature_map_size 102 | elif current_unit.type == 3: 103 | with tf.variable_scope('{}_full_{}'.format(name_preffix, i)): 104 | last_unit = indi.get_layer_at(i-1) 105 | if last_unit.type != 3: # use the previous setting to calculate this input dimension 106 | input_data = slim.flatten(output_list[-1]) 107 | input_dim = input_data.get_shape()[1].value 108 | else: # current input dim should be the number of neurons in the previous hidden layer 109 | input_data = output_list[-1] 110 | input_dim = last_unit.hidden_neuron_num 111 | mean=current_unit.weight_matrix_mean 112 | stddev=current_unit.weight_matrix_std 113 | if i < num_of_units - 1: 114 | full_H = slim.fully_connected(input_data, num_outputs=current_unit.hidden_neuron_num, weights_initializer=tf.truncated_normal_initializer(mean=mean, stddev=stddev), biases_initializer=init_ops.constant_initializer(0.1, dtype=tf.float32)) 115 | else: 116 | full_H = slim.fully_connected(input_data, num_outputs=current_unit.hidden_neuron_num, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=mean, stddev=stddev), biases_initializer=init_ops.constant_initializer(0.1, dtype=tf.float32)) 117 | output_list.append(full_H) 118 | num_connections += input_dim*current_unit.hidden_neuron_num + current_unit.hidden_neuron_num 119 | else: 120 | raise NameError('No unit with type value {}'.format(current_unit.type)) 121 | 122 | 123 | with tf.name_scope('{}_loss'.format(name_preffix)): 124 | logits = output_list[-1] 125 | #regularization_loss = tf.add_n(tf.losses.get_regularization_losses()) 126 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_Y, logits=logits)) 127 | with tf.name_scope('{}_train'.format(name_preffix)): 128 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 129 | if update_ops: 130 | updates = tf.group(*update_ops) 131 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 132 | #global_step = tf.get_variable("global_step", [], initializer=tf.constant_initializer(0.0), trainable=False) 133 | #self.train_data_length//self.batch_size 134 | # lr = tf.train.exponential_decay(0.1, step, 550*30, 0.9, staircase=True) 135 | # optimizer = tf.train.GradientDescentOptimizer(lr) 136 | optimizer = tf.train.AdamOptimizer() 137 | train_op = slim.learning.create_train_op(cross_entropy, optimizer) 138 | with tf.name_scope('{}_test'.format(name_preffix)): 139 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), true_Y), tf.float32)) 140 | 141 | tf.summary.scalar('loss', cross_entropy) 142 | tf.summary.scalar('accuracy', accuracy) 143 | merge_summary = tf.summary.merge_all() 144 | 145 | return is_training, train_op, accuracy, cross_entropy, num_connections, merge_summary 146 | 147 | 148 | 149 | def parse_individual(self, indi, num_of_input_channel, indi_index, save_path, history_best_score): 150 | tf.reset_default_graph() 151 | train_data, train_label = get_data.get_train_data(self.batch_size) 152 | validate_data, validate_label = get_data.get_validate_data(self.batch_size) 153 | is_training, train_op, accuracy, cross_entropy, num_connections, merge_summary = self.build_graph(indi_index, num_of_input_channel, indi, train_data, train_label, validate_data, validate_label) 154 | with tf.Session() as sess: 155 | sess.run(tf.global_variables_initializer()) 156 | steps_in_each_epoch = (self.train_data_length//self.batch_size) 157 | total_steps = int(self.epochs*steps_in_each_epoch) 158 | coord = tf.train.Coordinator() 159 | #threads = tf.train.start_queue_runners(sess, coord) 160 | try: 161 | threads = [] 162 | for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS): 163 | threads.extend(qr.create_threads(sess, coord=coord, daemon=True, start=True)) 164 | for i in range(total_steps): 165 | if coord.should_stop(): 166 | break 167 | _, accuracy_str, loss_str, _ = sess.run([train_op, accuracy,cross_entropy, merge_summary], {is_training:True}) 168 | if i % (2*steps_in_each_epoch) == 0: 169 | test_total_step = self.validate_data_length//self.batch_size 170 | test_accuracy_list = [] 171 | test_loss_list = [] 172 | for _ in range(test_total_step): 173 | test_accuracy_str, test_loss_str = sess.run([accuracy, cross_entropy], {is_training:False}) 174 | test_accuracy_list.append(test_accuracy_str) 175 | test_loss_list.append(test_loss_str) 176 | mean_test_accu = np.mean(test_accuracy_list) 177 | mean_test_loss = np.mean(test_loss_list) 178 | print('{}, {}, indi:{}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, acc:{}'.format(datetime.now(), i // steps_in_each_epoch, indi_index, i, total_steps, loss_str, accuracy_str, mean_test_loss, mean_test_accu)) 179 | #print('{}, test_loss:{}, acc:{}'.format(datetime.now(), loss_str, accuracy_str)) 180 | #validate the last epoch 181 | test_total_step = self.validate_data_length//self.batch_size 182 | test_accuracy_list = [] 183 | test_loss_list = [] 184 | for _ in range(test_total_step): 185 | test_accuracy_str, test_loss_str = sess.run([accuracy, cross_entropy], {is_training:False}) 186 | test_accuracy_list.append(test_accuracy_str) 187 | test_loss_list.append(test_loss_str) 188 | mean_test_accu = np.mean(test_accuracy_list) 189 | mean_test_loss = np.mean(test_loss_list) 190 | print('{}, test_loss:{}, acc:{}'.format(datetime.now(), mean_test_loss, mean_test_accu)) 191 | mean_acc = mean_test_accu 192 | if mean_acc > history_best_score: 193 | save_mean_acc = tf.Variable(-1, dtype=tf.float32, name='save_mean') 194 | save_mean_acc_op = save_mean_acc.assign(mean_acc) 195 | sess.run(save_mean_acc_op) 196 | saver0 = tf.train.Saver() 197 | saver0.save(sess, save_path +'/model') 198 | saver0.export_meta_graph(save_path +'/model.meta') 199 | history_best_score = mean_acc 200 | 201 | except Exception as e: 202 | print(e) 203 | coord.request_stop(e) 204 | finally: 205 | print('finally...') 206 | coord.request_stop() 207 | coord.join(threads) 208 | 209 | return mean_test_accu, np.std(test_accuracy_list), num_connections, history_best_score 210 | 211 | 212 | -------------------------------------------------------------------------------- /code/evolve.py: -------------------------------------------------------------------------------- 1 | from population import Population 2 | from evaluate import Evaluate 3 | import numpy 4 | import tensorflow.examples.tutorials.mnist as input_data 5 | import tensorflow as tf 6 | import collections 7 | from utils import * 8 | import copy 9 | 10 | 11 | class Evolve_CNN: 12 | def __init__(self, m_prob, m_eta, x_prob, x_eta, population_size, train_data, train_label, validate_data, validate_label, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, eta): 13 | self.m_prob = m_prob 14 | self.m_eta = m_eta 15 | self.x_prob = x_prob 16 | self.x_eta = x_eta 17 | self.population_size = population_size 18 | self.train_data = train_data 19 | self.train_label = train_label 20 | self.validate_data = validate_data 21 | self.validate_label = validate_label 22 | self.epochs = epochs 23 | self.eta = eta 24 | self.number_of_channel = number_of_channel 25 | self.batch_size = batch_size 26 | self.train_data_length = train_data_length 27 | self.validate_data_length = validate_data_length 28 | 29 | def initialize_popualtion(self): 30 | print("initializing population with number {}...".format(self.population_size)) 31 | self.pops = Population(self.population_size) 32 | # all the initialized population should be saved 33 | save_populations(gen_no=-1, pops=self.pops) 34 | def evaluate_fitness(self, gen_no): 35 | print("evaluate fintesss") 36 | evaluate = Evaluate(self.pops, self.train_data, self.train_label, self.validate_data, self.validate_label, self.number_of_channel, self.epochs, self.batch_size, self.train_data_length, self.validate_data_length) 37 | evaluate.parse_population(gen_no) 38 | # # all theinitialized population should be saved 39 | save_populations(gen_no=gen_no, pops=self.pops) 40 | print(self.pops) 41 | 42 | 43 | def recombinate(self, gen_no): 44 | print("mutation and crossover...") 45 | offspring_list = [] 46 | for _ in range(int(self.pops.get_pop_size()/2)): 47 | p1 = self.tournament_selection() 48 | p2 = self.tournament_selection() 49 | # crossover 50 | offset1, offset2 = self.crossover(p1, p2) 51 | # mutation 52 | offset1.mutation() 53 | offset2.mutation() 54 | offspring_list.append(offset1) 55 | offspring_list.append(offset2) 56 | offspring_pops = Population(0) 57 | offspring_pops.set_populations(offspring_list) 58 | save_offspring(gen_no, offspring_pops) 59 | #evaluate these individuals 60 | evaluate = Evaluate(self.pops, self.train_data, self.train_label, self.validate_data, self.validate_label, self.number_of_channel, self.epochs, self.batch_size, self.train_data_length, self.validate_data_length) 61 | evaluate.parse_population(gen_no) 62 | # #save 63 | self.pops.pops.extend(offspring_pops.pops) 64 | save_populations(gen_no=gen_no, pops=self.pops) 65 | 66 | def environmental_selection(self, gen_no): 67 | assert(self.pops.get_pop_size() == 2*self.population_size) 68 | elitsam = 0.2 69 | e_count = int(np.floor(self.population_size*elitsam/2)*2) 70 | indi_list = self.pops.pops 71 | indi_list.sort(key=lambda x:x.mean, reverse=True) 72 | elistm_list = indi_list[0:e_count] 73 | 74 | left_list = indi_list[e_count:] 75 | np.random.shuffle(left_list) 76 | np.random.shuffle(left_list) 77 | 78 | for _ in range(self.population_size-e_count): 79 | i1 = randint(0, len(left_list)) 80 | i2 = randint(0, len(left_list)) 81 | winner = self.selection(left_list[i1], left_list[i2]) 82 | elistm_list.append(winner) 83 | 84 | self.pops.set_populations(elistm_list) 85 | save_populations(gen_no=gen_no, pops=self.pops) 86 | np.random.shuffle(self.pops.pops) 87 | 88 | 89 | def crossover(self, p1, p2): 90 | p1 = copy.deepcopy(p1) 91 | p2 = copy.deepcopy(p2) 92 | p1.clear_state_info() 93 | p2.clear_state_info() 94 | #for different unit, we define two list, one to save their index and the other one save unit 95 | p1_conv_index_list = [] 96 | p1_conv_layer_list = [] 97 | p1_pool_index_list = [] 98 | p1_pool_layer_list = [] 99 | p1_full_index_list = [] 100 | p1_full_layer_list = [] 101 | 102 | p2_conv_index_list = [] 103 | p2_conv_layer_list = [] 104 | p2_pool_index_list = [] 105 | p2_pool_layer_list = [] 106 | p2_full_index_list = [] 107 | p2_full_layer_list = [] 108 | 109 | for i in range(p1.get_layer_size()): 110 | unit = p1.get_layer_at(i) 111 | if unit.type == 1: 112 | p1_conv_index_list.append(i) 113 | p1_conv_layer_list.append(unit) 114 | elif unit.type == 2: 115 | p1_pool_index_list.append(i) 116 | p1_pool_layer_list.append(unit) 117 | else: 118 | p1_full_index_list.append(i) 119 | p1_full_layer_list.append(unit) 120 | 121 | 122 | for i in range(p2.get_layer_size()): 123 | unit = p2.get_layer_at(i) 124 | if unit.type == 1: 125 | p2_conv_index_list.append(i) 126 | p2_conv_layer_list.append(unit) 127 | elif unit.type == 2: 128 | p2_pool_index_list.append(i) 129 | p2_pool_layer_list.append(unit) 130 | else: 131 | p2_full_index_list.append(i) 132 | p2_full_layer_list.append(unit) 133 | 134 | #begin crossover on conn layer 135 | l = min(len(p1_conv_layer_list), len(p2_conv_layer_list)) 136 | for i in range(l): 137 | unit_p1 = p1_conv_layer_list[i] 138 | unit_p2 = p2_conv_layer_list[i] 139 | if flip(self.x_prob): 140 | #filter size 141 | this_range = p1.filter_size_range 142 | w1 = unit_p1.filter_width 143 | w2 = unit_p2.filter_width 144 | n_w1, n_w2 = self.sbx(w1, w2, this_range[0], this_range[-1], self.x_eta) 145 | unit_p1.filter_width = int(n_w1) 146 | unit_p1.filter_height = int(n_w1) 147 | unit_p2.filter_width = int(n_w2) 148 | unit_p2.filter_height = int(n_w2) 149 | #feature map size 150 | this_range = p1.featur_map_size_range 151 | s1 = unit_p1.feature_map_size 152 | s2 = unit_p2.feature_map_size 153 | n_s1, n_s2 = self.sbx(s1, s2, this_range[0], this_range[-1], self.x_eta) 154 | unit_p1.feature_map_size = int(n_s1) 155 | unit_p2.feature_map_size = int(n_s2) 156 | #mean 157 | this_range = p1.mean_range 158 | m1 = unit_p1.weight_matrix_mean 159 | m2 = unit_p2.weight_matrix_mean 160 | n_m1, n_m2 = self.sbx(m1, m2, this_range[0], this_range[-1], self.x_eta) 161 | unit_p1.weight_matrix_mean = n_m1 162 | unit_p2.weight_matrix_mean = n_m2 163 | #std 164 | this_range = p1.std_range 165 | std1 = unit_p1.weight_matrix_std 166 | std2 = unit_p2.weight_matrix_std 167 | n_std1, n_std2 = self.sbx(std1, std2, this_range[0], this_range[-1], self.x_eta) 168 | unit_p1.weight_matrix_std = n_std1 169 | unit_p2.weight_matrix_std = n_std2 170 | 171 | p1_conv_layer_list[i] = unit_p1 172 | p2_conv_layer_list[i] = unit_p2 173 | 174 | l = min(len(p1_pool_layer_list), len(p2_pool_layer_list)) 175 | for i in range(l): 176 | unit_p1 = p1_pool_layer_list[i] 177 | unit_p2 = p2_pool_layer_list[i] 178 | if flip(self.x_prob): 179 | # kernel size 180 | this_range = p1.pool_kernel_size_range 181 | k1 = np.log2(unit_p1.kernel_width) 182 | k2 = np.log2(unit_p2.kernel_width) 183 | n_k1, n_k2 = self.sbx(k1, k2, this_range[0], this_range[-1], self.x_eta) 184 | n_k1 = int(np.power(2, n_k1)) 185 | n_k2 = int(np.power(2, n_k2)) 186 | unit_p1.kernel_width = n_k1 187 | unit_p1.kernel_height = n_k1 188 | unit_p2.kernel_width = n_k2 189 | unit_p2.kernel_height = n_k2 190 | #pool type 191 | t1 = unit_p1.kernel_type 192 | t2 = unit_p2.kernel_type 193 | n_t1, n_t2 = self.sbx(t1, t2, 0, 1, self.x_eta) 194 | unit_p1.kernel_type = n_t1 195 | unit_p2.kernel_type = n_t2 196 | 197 | p1_pool_layer_list[i] = unit_p1 198 | p2_pool_layer_list[i] = unit_p2 199 | 200 | l = min(len(p1_full_layer_list), len(p2_full_layer_list)) 201 | for i in range(l-1): 202 | unit_p1 = p1_full_layer_list[i] 203 | unit_p2 = p2_full_layer_list[i] 204 | if flip(self.x_prob): 205 | this_range = p1.hidden_neurons_range 206 | n1 = unit_p1.hidden_neuron_num 207 | n2 = unit_p2.hidden_neuron_num 208 | n_n1, n_n2 = self.sbx(n1, n2, this_range[0], this_range[1], self.x_eta) 209 | unit_p1.hidden_neuron_num = int(n_n1) 210 | unit_p2.hidden_neuron_num = int(n_n2) 211 | # std amnd mean 212 | this_range = p1.mean_range 213 | m1 = unit_p1.weight_matrix_mean 214 | m2 = unit_p2.weight_matrix_mean 215 | n_m1, n_m2 = self.sbx(m1, m2, this_range[0], this_range[-1], self.x_eta) 216 | unit_p1.weight_matrix_mean = n_m1 217 | unit_p2.weight_matrix_mean = n_m2 218 | 219 | this_range = p1.std_range 220 | std1 = unit_p1.weight_matrix_std 221 | std2 = unit_p2.weight_matrix_std 222 | n_std1, n_std2 = self.sbx(std1, std2, this_range[0], this_range[-1], self.x_eta) 223 | unit_p1.weight_matrix_std = n_std1 224 | unit_p2.weight_matrix_std = n_std2 225 | 226 | p1_full_layer_list[i] = unit_p1 227 | p2_full_layer_list[i] = unit_p2 228 | 229 | # for the last full layer, only mean and std 230 | unit_p1 = p1_full_layer_list[-1] 231 | unit_p2 = p2_full_layer_list[-1] 232 | if flip(self.x_prob): 233 | # std amnd mean 234 | this_range = p1.mean_range 235 | m1 = unit_p1.weight_matrix_mean 236 | m2 = unit_p2.weight_matrix_mean 237 | n_m1, n_m2 = self.sbx(m1, m2, this_range[0], this_range[-1], self.x_eta) 238 | unit_p1.weight_matrix_mean = n_m1 239 | unit_p2.weight_matrix_mean = n_m2 240 | 241 | this_range = p1.std_range 242 | std1 = unit_p1.weight_matrix_std 243 | std2 = unit_p2.weight_matrix_std 244 | n_std1, n_std2 = self.sbx(std1, std2, this_range[0], this_range[-1], self.x_eta) 245 | unit_p1.weight_matrix_std = n_std1 246 | unit_p2.weight_matrix_std = n_std2 247 | p1_full_layer_list[-1] = unit_p1 248 | p2_full_layer_list[-1] = unit_p2 249 | 250 | p1_units = p1.indi 251 | # assign these crossovered values to the p1 and p2 252 | for i in range(len(p1_conv_index_list)): 253 | p1_units[p1_conv_index_list[i]] = p1_conv_layer_list[i] 254 | for i in range(len(p1_pool_index_list)): 255 | p1_units[p1_pool_index_list[i]] = p1_pool_layer_list[i] 256 | for i in range(len(p1_full_index_list)): 257 | p1_units[p1_full_index_list[i]] = p1_full_layer_list[i] 258 | p1.indi = p1_units 259 | 260 | p2_units = p2.indi 261 | for i in range(len(p2_conv_index_list)): 262 | p2_units[p2_conv_index_list[i]] = p2_conv_layer_list[i] 263 | for i in range(len(p2_pool_index_list)): 264 | p2_units[p2_pool_index_list[i]] = p2_pool_layer_list[i] 265 | for i in range(len(p2_full_index_list)): 266 | p2_units[p2_full_index_list[i]] = p2_full_layer_list[i] 267 | p2.indi = p2_units 268 | 269 | return p1, p2 270 | 271 | 272 | 273 | def sbx_test(self, v1, v2, xl, xu, eta): 274 | return 0.1, 0.5 275 | 276 | def sbx(self, v1, v2, xl, xu, eta): 277 | if flip(0.5): 278 | if abs(v1-v2)>1e-14: 279 | x1 = min(v1, v2) 280 | x2 = max(v1, v2) 281 | r = rand() 282 | beta = 1.0 + (2.0 * (x1 - xl) / (x2 - x1)) 283 | alpha = 2.0 - beta**-(eta + 1) 284 | if r <= 1.0 / alpha: 285 | beta_q = (r * alpha)**(1.0 / (eta + 1)) 286 | else: 287 | beta_q = (1.0 / (2.0 - r * alpha))**(1.0 / (eta + 1)) 288 | c1 = 0.5 * (x1 + x2 - beta_q * (x2 - x1)) 289 | beta = 1.0 + (2.0 * (xu - x2) / (x2 - x1)) 290 | alpha = 2.0 - beta**-(eta + 1) 291 | if r <= 1.0 / alpha: 292 | beta_q = (r * alpha)**(1.0 / (eta + 1)) 293 | else: 294 | beta_q = (1.0 / (2.0 - r * alpha))**(1.0 / (eta + 1)) 295 | c2 = 0.5 * (x1 + x2 + beta_q * (x2 - x1)) 296 | c1 = min(max(c1, xl), xu) 297 | c2 = min(max(c2, xl), xu) 298 | if flip(0.5): 299 | return c2, c1 300 | else: 301 | return c1, c2 302 | else: 303 | return v1, v2 304 | else: 305 | return v1, v2 306 | 307 | 308 | def tournament_selection(self): 309 | ind1_id = randint(0, self.pops.get_pop_size()) 310 | ind2_id = randint(0, self.pops.get_pop_size()) 311 | ind1 = self.pops.get_individual_at(ind1_id) 312 | ind2 = self.pops.get_individual_at(ind2_id) 313 | winner = self.selection(ind1, ind2) 314 | return winner 315 | 316 | def selection(self, ind1, ind2): 317 | mean_threshold = 0.05 318 | complexity_threhold = 100 319 | if ind1.mean > ind2.mean: 320 | if ind1.mean - ind2.mean > mean_threshold: 321 | return ind1 322 | else: 323 | if ind2.complxity < (ind1.complxity-complexity_threhold): 324 | return ind2 325 | else: 326 | return ind1 327 | else: 328 | if ind2.mean - ind1.mean > mean_threshold: 329 | return ind2 330 | else: 331 | if ind1.complxity < (ind2.complxity-complexity_threhold): 332 | return ind1 333 | else: 334 | return ind2 335 | 336 | 337 | 338 | if __name__ == '__main__': 339 | ev = Evolve_CNN(m_prob=0.2, m_eta=0.05, x_prob=0.9, x_eta=0.05, population_size=10, train_data=None, validate_data=None, epochs=1, eta=1) 340 | ev.initialize_popualtion() 341 | print(ev.pops) 342 | print('='*100) 343 | ind1 = ev.tournament_selection() 344 | ind2 = ev.tournament_selection() 345 | print('p1->', ind1) 346 | print('p2->', ind2) 347 | print('='*100) 348 | new_p1, new_p2 = ev.crossover(ind1, ind2) 349 | print('np1->', new_p1) 350 | print('np2->', new_p2) 351 | print('='*100) 352 | new_p1.mutation() 353 | new_p2.mutation() 354 | print('nnp1->', new_p1) 355 | print('nnp2->', new_p2) 356 | 357 | 358 | 359 | -------------------------------------------------------------------------------- /code/get_data.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import scipy.io as io 3 | import numpy as np 4 | import sklearn.preprocessing as pre 5 | import os 6 | 7 | def get_general_image(path, name, num): 8 | data = io.loadmat(path) 9 | data = data[name].astype(np.float32) 10 | data = np.reshape(data, [num, 28, 28, 1], order='F') 11 | return data 12 | def get_general_label(path, name): 13 | label = io.loadmat(path) 14 | label = label[name] 15 | label = np.squeeze(label.astype(np.int32)) 16 | return label 17 | 18 | def get_mnist_train_data(): 19 | train_images_path = '/am/lido/home/yanan/training_data/rectangles_images/train_images.mat' 20 | train_label_path = '/am/lido/home/yanan/training_data/rectangles_images/train_label.mat' 21 | 22 | train_data = get_general_image(train_images_path, 'train_images', 10000) 23 | train_label = get_general_label(train_label_path, 'train_label') 24 | 25 | 26 | return train_data, train_label 27 | 28 | 29 | def get_mnist_test_data(): 30 | test_images_path = '/am/lido/home/yanan/training_data/rectangles_images/test_images.mat' 31 | test_label_path = '/am/lido/home/yanan/training_data/rectangles_images/test_label.mat' 32 | 33 | 34 | test_data = get_general_image(test_images_path, 'test_images', 50000) 35 | test_label = get_general_label(test_label_path, 'test_label') 36 | 37 | return test_data, test_label 38 | 39 | 40 | def get_mnist_validate_data(): 41 | validate_images_path = '/am/lido/home/yanan/training_data/rectangles_images/validate_images.mat' 42 | validate_label_path = '/am/lido/home/yanan/training_data/rectangles_images/validate_label.mat' 43 | 44 | validate_data = get_general_image(validate_images_path, 'validate_images', 2000) 45 | validate_label = get_general_label(validate_label_path, 'validate_label') 46 | 47 | return validate_data, validate_label 48 | 49 | 50 | def get_standard_train_data(name): 51 | data_path = '/am/lido/home/yanan/training_data/back-{}/train_images.npy'.format(name) 52 | label_path = '/am/lido/home/yanan/training_data/back-{}/train_label.npy'.format(name) 53 | data = np.load(data_path) 54 | label = np.load(label_path) 55 | return data, label 56 | 57 | def get_standard_validate_data(name): 58 | data_path = '/am/lido/home/yanan/training_data/back-{}/validate_images.npy'.format(name) 59 | label_path = '/am/lido/home/yanan/training_data/back-{}/validate_label.npy'.format(name) 60 | data = np.load(data_path) 61 | label = np.load(label_path) 62 | return data, label 63 | 64 | def get_standard_test_data(name): 65 | data_path = '/am/lido/home/yanan/training_data/back-{}/test_images.npy'.format(name) 66 | label_path = '/am/lido/home/yanan/training_data/back-{}/test_label.npy'.format(name) 67 | data = np.load(data_path) 68 | label = np.load(label_path) 69 | return data, label 70 | 71 | 72 | 73 | def get_train_data(batch_size): 74 | t_image, t_label = get_mnist_train_data() 75 | train_image = tf.cast(t_image, tf.float32) 76 | train_label = tf.cast(t_label, tf.int32) 77 | single_image, single_label = tf.train.slice_input_producer([train_image, train_label], shuffle=True) 78 | single_image = tf.image.per_image_standardization(single_image) 79 | image_batch, label_batch = tf.train.batch([single_image, single_label], batch_size=batch_size, num_threads=2, capacity=batch_size*3) 80 | return image_batch, label_batch 81 | 82 | def get_validate_data(batch_size): 83 | t_image, t_label = get_mnist_validate_data() 84 | validate_image = tf.cast(t_image, tf.float32) 85 | validate_label = tf.cast(t_label, tf.int32) 86 | single_image, single_label = tf.train.slice_input_producer([validate_image, validate_label], shuffle=False) 87 | single_image = tf.image.per_image_standardization(single_image) 88 | image_batch, label_batch = tf.train.batch([single_image, single_label], batch_size=batch_size, num_threads=2, capacity=batch_size*3) 89 | return image_batch, label_batch 90 | 91 | 92 | def get_test_data(batch_size): 93 | t_image, t_label = get_mnist_test_data() 94 | test_image = tf.cast(t_image, tf.float32) 95 | test_label = tf.cast(t_label, tf.int32) 96 | single_image, single_label = tf.train.slice_input_producer([test_image, test_label], shuffle=False) 97 | single_image = tf.image.per_image_standardization(single_image) 98 | image_batch, label_batch = tf.train.batch([single_image, single_label], batch_size=batch_size, num_threads=2, capacity=batch_size*3) 99 | return image_batch, label_batch 100 | 101 | def tf_standalized(data): 102 | image = tf.placeholder(tf.float32, shape=[28,28,1]) 103 | scale_data = tf.image.per_image_standardization(image) 104 | data_list = [] 105 | with tf.Session() as sess: 106 | sess.run(tf.global_variables_initializer()) 107 | data_length = data.shape[0] 108 | for i in range(data_length): 109 | standard_data = sess.run(scale_data, {image:data[i]}) 110 | print(i, data_length) 111 | data_list.append(standard_data) 112 | return np.array(data_list) 113 | 114 | 115 | if __name__ =='__main__': 116 | name = 'random' 117 | data, label = get_standard_test_data(name) 118 | print(data.shape, label.shape, data.dtype, label.dtype) 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | # def get_mnist_train_batch(batch_size, capacity=1000): 128 | # mnist = input_data.read_data_sets('MNIST_data', reshape=False, one_hot=True) 129 | # train_images = tf.cast(mnist.train.images, tf.float32) 130 | # train_labels = tf.cast(mnist.train.labels, tf.int32) 131 | # input_queue = tf.train.slice_input_producer([train_images, train_labels],shuffle=True, capacity=capacity, name='input_queue') 132 | # images_batch, labels_batch = tf.train.batch(input_queue, batch_size=batch_size, num_threads=3, capacity=capacity) 133 | # return images_batch, labels_batch 134 | # batch_images, batch_labels = get_mnist_train_batch(100) 135 | # with tf.Session() as sess: 136 | # sess.run(tf.global_variables_initializer()) 137 | # coord = tf.train.Coordinator() 138 | # threads = tf.train.start_queue_runners(sess, coord) 139 | # i = 0 140 | # try: 141 | # while not coord.should_stop(): 142 | # 143 | # batch_images_v, batch_labels_v = sess.run([batch_images, batch_labels]) 144 | # print(i) 145 | # i += 1 146 | # 147 | # except tf.errors.OutOfRangeError: 148 | # print('done') 149 | # finally: 150 | # coord.request_stop() 151 | # coord.join(threads) -------------------------------------------------------------------------------- /code/individual.py: -------------------------------------------------------------------------------- 1 | from layers import ConvLayer, PoolLayer, FullLayer 2 | import numpy as np 3 | import random 4 | from utils import * 5 | 6 | class Individual: 7 | 8 | def __init__(self, x_prob=0.9, x_eta=0.05, m_prob=0.2, m_eta=0.05): 9 | self.indi = [] 10 | self.x_prob = x_prob 11 | self.x_eta = x_eta 12 | self.m_prob = m_prob 13 | self.m_eta = m_eta 14 | self.mean = 0 15 | self.std = 0 16 | self.complxity = 0 17 | 18 | ##################### 19 | self.featur_map_size_range = [3, 50] 20 | self.filter_size_range = [2, 20] 21 | self.pool_kernel_size_range = [1, 2] 22 | self.hidden_neurons_range = [1000, 2000] 23 | self.mean_range = [-1,1] 24 | self.std_range = [0,1] 25 | 26 | def clear_state_info(self): 27 | self.complxity = 0 28 | self.mean = 0 29 | self.std = 0 30 | 31 | ''' 32 | initialize a simle CNN network including one convolutional layer, one pooling layer, and one full connection layer 33 | ''' 34 | def initialize(self): 35 | self.indi = self.init_one_individual() 36 | 37 | def init_one_individual(self): 38 | init_num_conv = np.random.randint(1, 3) 39 | init_num_pool = np.random.randint(1, 3) 40 | init_num_full = np.random.randint(1, 3) 41 | _list = [] 42 | for _ in range(init_num_conv): 43 | _list.append(self.add_a_random_conv_layer()) 44 | for _ in range(init_num_pool): 45 | _list.append(self.add_a_random_pool_layer()) 46 | for _ in range(init_num_full-1): 47 | _list.append(self.add_a_random_full_layer()) 48 | _list.append(self.add_a_common_full_layer()) 49 | return _list 50 | 51 | def get_layer_at(self,i): 52 | return self.indi[i] 53 | 54 | def get_layer_size(self): 55 | return len(self.indi) 56 | def init_mean(self): 57 | return np.random.random()*(self.mean_range[1] - self.mean_range[0]) + self.mean_range[0] 58 | def init_std(self): 59 | return np.random.random()*(self.std_range[1] - self.std_range[0]) + self.std_range[0] 60 | 61 | def init_feature_size(self): 62 | return np.random.randint(self.filter_size_range[0], self.filter_size_range[1]) 63 | def init_feature_map_size(self): 64 | return np.random.randint(self.featur_map_size_range[0], self.featur_map_size_range[1]) 65 | def init_kernel_size(self): 66 | kernel_size_num = len(self.pool_kernel_size_range) 67 | n = np.random.randint(kernel_size_num) 68 | return np.power(2, self.pool_kernel_size_range[n]) 69 | def init_hidden_neuron_size(self): 70 | return np.random.randint(self.hidden_neurons_range[0], self.hidden_neurons_range[1]) 71 | 72 | def mutation(self): 73 | if flip(self.m_prob): 74 | #for the units 75 | unit_list = [] 76 | for i in range(self.get_layer_size()-1): 77 | cur_unit = self.get_layer_at(i) 78 | if flip(0.5): 79 | #mutation 80 | p_op = self.mutation_ope(rand()) 81 | max_length = 6 82 | current_length = (len(unit_list) + self.get_layer_size() - i-1) 83 | if p_op == 0: #add a new 84 | 85 | if current_length < max_length: # when length exceeds this length, only mutation no add new unit 86 | unit_list.append(self.generate_a_new_layer(cur_unit.type, self.get_layer_size())) 87 | unit_list.append(cur_unit) 88 | else: 89 | updated_unit = self.mutation_a_unit(cur_unit, self.m_eta) 90 | unit_list.append(updated_unit) 91 | if p_op == 1: #modify the lement 92 | updated_unit = self.mutation_a_unit(cur_unit, self.m_eta) 93 | unit_list.append(updated_unit) 94 | 95 | else: 96 | unit_list.append(cur_unit) 97 | #avoid all units have been removed, add a full layer 98 | if len(unit_list) == 0: 99 | unit_list.append(self.add_a_random_conv_layer()) 100 | unit_list.append(self.add_a_random_pool_layer()) 101 | unit_list.append(self.get_layer_at(-1)) 102 | # judge the first unit and the second unit 103 | if unit_list[0].type != 1: 104 | unit_list.insert(0, self.add_a_random_conv_layer()) 105 | self.indi = unit_list 106 | 107 | 108 | def mutation_a_unit(self, unit, eta): 109 | if unit.type ==1: 110 | #mutate a conv layer 111 | return self.mutate_conv_unit(unit, eta) 112 | elif unit.type ==2: 113 | #mutate a pool layer 114 | return self.mutate_pool_unit(unit, eta) 115 | else: 116 | #mutate a full layer 117 | return self.mutate_full_layer(unit, eta) 118 | 119 | def mutate_conv_unit(self, unit, eta): 120 | # feature map size, feature map number, mean std 121 | fms = unit.filter_width 122 | fmn = unit.feature_map_size 123 | mean = unit.weight_matrix_mean 124 | std = unit.weight_matrix_std 125 | 126 | new_fms = int(self.pm(self.filter_size_range[0], self.filter_size_range[-1], fms, eta)) 127 | new_fmn = int(self.pm(self.featur_map_size_range[0], self.featur_map_size_range[1], fmn, eta)) 128 | new_mean = self.pm(self.mean_range[0], self.mean_range[1], mean, eta) 129 | new_std = self.pm(self.std_range[0], self.std_range[1], std, eta) 130 | conv_layer = ConvLayer(filter_size=[new_fms, new_fms], feature_map_size=new_fmn, weight_matrix=[new_mean, new_std]) 131 | return conv_layer 132 | 133 | def mutate_pool_unit(self, unit, eta): 134 | #kernel size, pool_type 135 | ksize = np.log2(unit.kernel_width) 136 | pool_type = unit.kernel_type 137 | 138 | new_ksize = self.pm(self.pool_kernel_size_range[0], self.pool_kernel_size_range[-1], ksize, eta) 139 | new_ksize = int(np.power(2, new_ksize)) 140 | new_pool_type = self.pm(0, 1, pool_type, eta) 141 | pool_layer = PoolLayer(kernel_size=[new_ksize,new_ksize], pool_type=new_pool_type) 142 | return pool_layer 143 | 144 | def mutate_full_layer(self, unit, eta): 145 | #num of hidden neurons, mean ,std 146 | n_hidden = unit.hidden_neuron_num 147 | mean = unit.weight_matrix_mean 148 | std = unit.weight_matrix_std 149 | 150 | new_n_hidden = int(self.pm(self.hidden_neurons_range [0], self.hidden_neurons_range[-1], n_hidden, eta)) 151 | new_mean = self.pm(self.mean_range[0], self.mean_range[1], mean, eta) 152 | new_std = self.pm(self.std_range[0], self.std_range[1], std, eta) 153 | full_layer = FullLayer(hidden_neuron_num=new_n_hidden, weight_matrix=[new_mean, new_std]) 154 | return full_layer 155 | 156 | #0 add, 1 modify 2delete 157 | 158 | def mutation_ope(self, r): 159 | if r < 0.33: 160 | return 1 161 | elif r >0.66: 162 | return 2 163 | else: 164 | return 0 165 | 166 | def add_a_common_full_layer(self): 167 | mean = self.init_mean() 168 | std = self.init_std() 169 | full_layer = FullLayer(hidden_neuron_num=2, weight_matrix=[mean, std]) 170 | return full_layer 171 | def add_a_random_full_layer(self): 172 | mean = self.init_mean() 173 | std = self.init_std() 174 | hidden_neuron_num = self.init_hidden_neuron_size() 175 | full_layer = FullLayer(hidden_neuron_num=hidden_neuron_num, weight_matrix=[mean, std]) 176 | return full_layer 177 | def add_a_random_conv_layer(self): 178 | s1 = self.init_feature_size() 179 | filter_size=s1,s1 180 | feature_map_size = self.init_feature_map_size() 181 | mean = self.init_mean() 182 | std = self.init_std() 183 | conv_layer = ConvLayer(filter_size=filter_size, feature_map_size=feature_map_size, weight_matrix=[mean, std]) 184 | return conv_layer 185 | def add_a_random_pool_layer(self): 186 | s1 = self.init_kernel_size() 187 | kernel_size=s1, s1 188 | pool_type=np.random.random(size=1) 189 | pool_layer = PoolLayer(kernel_size=kernel_size, pool_type=pool_type[0]) 190 | return pool_layer 191 | 192 | 193 | def generate_a_new_layer(self, current_unit_type, unit_length): 194 | if current_unit_type == 3: 195 | #judge if current length = 1, add conv or pool 196 | if unit_length == 1: 197 | if random.random() < 0.5: 198 | return self.add_a_random_conv_layer() 199 | else: 200 | return self.add_a_random_pool_layer() 201 | else: 202 | return self.add_a_random_full_layer() 203 | else: 204 | r = random.random() 205 | if r <0.5: 206 | return self.add_a_random_conv_layer() 207 | else: 208 | return self.add_a_random_pool_layer() 209 | 210 | 211 | 212 | def pm(self, xl, xu, x, eta): 213 | delta_1 = (x - xl) / (xu - xl) 214 | delta_2 = (xu - x) / (xu - xl) 215 | rand = np.random.random() 216 | mut_pow = 1.0 / (eta + 1.) 217 | if rand < 0.5: 218 | xy = 1.0 - delta_1 219 | val = 2.0 * rand + (1.0 - 2.0 * rand) * xy**(eta + 1) 220 | delta_q = val**mut_pow - 1.0 221 | else: 222 | xy = 1.0 - delta_2 223 | val = 2.0 * (1.0 - rand) + 2.0 * (rand - 0.5) * xy**(eta + 1) 224 | delta_q = 1.0 - val**mut_pow 225 | x = x + delta_q * (xu - xl) 226 | x = min(max(x, xl), xu) 227 | return x 228 | 229 | 230 | 231 | 232 | 233 | 234 | def __str__(self): 235 | 236 | 237 | str_ = [] 238 | str_.append('Length:{}, Num:{}'.format(self.get_layer_size(), self.complxity)) 239 | str_.append('Mean:{:.2f}'.format(self.mean)) 240 | str_.append('Std:{:.2f}'.format(self.std)) 241 | 242 | for i in range(self.get_layer_size()): 243 | unit = self.get_layer_at(i) 244 | if unit.type == 1: 245 | str_.append("conv[{},{},{},{:.2f},{:.2f}]".format(unit.filter_width, unit.filter_height, unit.feature_map_size, unit.weight_matrix_mean, unit.weight_matrix_std)) 246 | elif unit.type ==2: 247 | str_.append("pool[{},{},{:.2f}]".format(unit.kernel_width, unit.kernel_height, unit.kernel_type)) 248 | elif unit.type ==3: 249 | str_.append("full[{},{},{}]".format(unit.hidden_neuron_num, unit.weight_matrix_mean, unit.weight_matrix_std)) 250 | else: 251 | raise Exception("Incorrect unit flag") 252 | return ', '.join(str_) 253 | 254 | 255 | if __name__ =='__main__': 256 | ind = Individual() 257 | print(ind.randint(1,10)) 258 | 259 | 260 | -------------------------------------------------------------------------------- /code/layers.py: -------------------------------------------------------------------------------- 1 | ''' 2 | All three kinds of layers can be initialized with their default parameters 3 | ''' 4 | 5 | class ConvLayer: 6 | def __init__(self, filter_size=[2,2], feature_map_size=8, weight_matrix=[0.0,1.0]): 7 | self.filter_width = filter_size[0] 8 | self.filter_height = filter_size[1] 9 | self.feature_map_size = feature_map_size 10 | self.weight_matrix_mean = weight_matrix[0] 11 | self.weight_matrix_std = weight_matrix[1] 12 | self.type = 1 13 | def __str__(self): 14 | return "Conv Layer: filter:[{0},{1}], feature map number:{2}, weight:[{3},{4}]".format(self.filter_width, self.filter_height, self.feature_map_size, self.weight_matrix_mean, self.weight_matrix_std) 15 | 16 | class PoolLayer: 17 | def __init__(self, kernel_size=[2,2], pool_type=0.1): 18 | self.kernel_width = kernel_size[0] 19 | self.kernel_height = kernel_size[1] 20 | self.kernel_type = pool_type # values below 0.5 means max other wise mean 21 | self.type = 2 22 | 23 | def __str__(self): 24 | return "Pool Layer: kernel:[{0},{1}], type:{2}".format(self.kernel_width, self.kernel_height, "max" if self.kernel_type<0.5 else "mean") 25 | 26 | class FullLayer: 27 | def __init__(self, hidden_neuron_num=10, weight_matrix=[0.0,1.0]): 28 | self.hidden_neuron_num = hidden_neuron_num 29 | self.weight_matrix_mean = weight_matrix[0] 30 | self.weight_matrix_std = weight_matrix[1] 31 | self.type = 3 32 | 33 | def __str__(self): 34 | return "Full Layer: hidden neurons:{}, weight:[{},{}]".format(self.hidden_neuron_num, self.weight_matrix_mean, self.weight_matrix_std) -------------------------------------------------------------------------------- /code/main.py: -------------------------------------------------------------------------------- 1 | from evolve import Evolve_CNN 2 | from utils import * 3 | import tensorflow as tf 4 | 5 | def begin_evolve(m_prob, m_eta, x_prob, x_eta, pop_size, train_data, train_label, validate_data, validation_label, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, total_generation_number, eta): 6 | cnn = Evolve_CNN(m_prob, m_eta, x_prob, x_eta, pop_size, train_data, train_label, validate_data, validation_label, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, eta) 7 | cnn.initialize_popualtion() 8 | cnn.evaluate_fitness(0) 9 | for cur_gen_no in range(total_generation_number): 10 | print('The {}/{} generation'.format(cur_gen_no+1, total_generation_number)) 11 | cnn.recombinate(cur_gen_no+1) 12 | cnn.environmental_selection(cur_gen_no+1) 13 | 14 | def restart_evolve(m_prob, m_eta, x_prob, x_eta, pop_size, train_data, train_label, validate_data, validation_label, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, total_gene_number, eta): 15 | gen_no, pops, _= load_population() 16 | cnn = Evolve_CNN(m_prob, m_eta, x_prob, x_eta, pop_size, train_data, train_label, validate_data, validation_label, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, eta) 17 | cnn.pops = pops 18 | if gen_no < 0: # go to evaluate 19 | print('first to evaluate...') 20 | cnn.evaluate_fitness(1) 21 | else: 22 | for cure_gen_no in range(gen_no+1, total_gene_number+1): 23 | print('Continue to evolve from the {}/{} generation...'.format(cure_gen_no, total_gene_number)) 24 | cnn.recombinate(cure_gen_no) 25 | cnn.environmental_selection(cure_gen_no) 26 | 27 | 28 | 29 | if __name__ == '__main__': 30 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 31 | tf.logging.set_verbosity(tf.logging.ERROR) 32 | if not tf.gfile.Exists('./save_data'): 33 | tf.gfile.MkDir('./save_data') 34 | 35 | #train_data, validation_data, test_data = get_mnist_data() 36 | batch_size = 100 37 | tf.reset_default_graph() 38 | number_of_channel = 1 39 | train_data_length = 10000 40 | validate_data_length = 2000 41 | total_generation_number = 50# total generation number 42 | pop_size = 50 43 | epochs = 10 44 | eta = 1/20 45 | #CUDA1 46 | #begin_evolve(0.9, 0.05, 0.2, 0.05, pop_size, None, None, None, None, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, total_generation_number, eta) 47 | restart_evolve(0.9, 0.05, 0.2, 0.05, pop_size, None, None, None, None, number_of_channel, epochs, batch_size, train_data_length, validate_data_length, total_generation_number, eta) 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /code/population.py: -------------------------------------------------------------------------------- 1 | from individual import Individual 2 | from utils import * 3 | 4 | class Population: 5 | 6 | def __init__(self, num_pops): 7 | self.num_pops = num_pops 8 | self.pops = [] 9 | for i in range(num_pops): 10 | indi = Individual() 11 | indi.initialize() 12 | self.pops.append(indi) 13 | 14 | def get_individual_at(self, i): 15 | return self.pops[i] 16 | 17 | def get_pop_size(self): 18 | return len(self.pops) 19 | 20 | def set_populations(self, new_pops): 21 | self.pops = new_pops 22 | 23 | 24 | 25 | 26 | 27 | def __str__(self): 28 | _str = [] 29 | for i in range(self.get_pop_size()): 30 | _str.append(str(self.get_individual_at(i))) 31 | return '\n'.join(_str) -------------------------------------------------------------------------------- /code/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import pickle 4 | from time import gmtime, strftime 5 | from population import * 6 | from individual import * 7 | 8 | def get_data_path(): 9 | return os.getcwd() + '/pops.dat' 10 | 11 | def save_populations(gen_no, pops): 12 | data = {'gen_no':gen_no, 'pops':pops, 'create_time':strftime("%Y-%m-%d %H:%M:%S", gmtime())} 13 | path = get_data_path() 14 | with open(path, 'wb') as file_handler: 15 | pickle.dump(data, file_handler) 16 | 17 | def load_population(): 18 | path = get_data_path() 19 | with open(path, 'rb') as file_handler: 20 | data = pickle.load(file_handler) 21 | return data['gen_no'], data['pops'],data['create_time'] 22 | 23 | def save_offspring(gen_no, pops): 24 | data = {'gen_no':gen_no, 'pops':pops, 'create_time':strftime("%Y-%m-%d %H:%M:%S", gmtime())} 25 | path = os.getcwd() + '/offsprings_data/gen_{}.dat'.format(gen_no) 26 | with open(path, 'wb') as file_handler: 27 | pickle.dump(data, file_handler) 28 | 29 | def load_save_log_data(): 30 | file_name = '/am/lido/home/yanan/eclipse-workspace/Ver3/pops.dat' 31 | with open(file_name, 'br') as file_h: 32 | data = pickle.load(file_h) 33 | print(data) 34 | pops = data['pops'].pops 35 | for i in range(len(pops)): 36 | print(pops[i]) 37 | 38 | def save_append_individual(indi, file_path): 39 | with open(file_path, 'a') as myfile: 40 | myfile.write(indi) 41 | myfile.write("\n") 42 | def randint(low, high): 43 | return np.random.random_integers(low, high-1) 44 | 45 | def rand(): 46 | return np.random.random() 47 | 48 | def flip(f): 49 | if rand() <= f: 50 | return True 51 | else: 52 | return False 53 | 54 | if __name__ =='__main__': 55 | load_save_log_data() 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /convex.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Length:5, Num:4359906, Mean:0.83, Std:0.04, 3 | conv[2,2,8,-0.32,0.51], 4 | conv[18,18,24,-0.13,0.17], 5 | pool[4,4,0.16], 6 | full[1848,-0.1959930992150578,0.8667409022008107], 7 | full[2,0.024912417329488123,0.9067530511189] 8 | 9 | 10 | ''' 11 | 12 | import tensorflow as tf 13 | import tensorflow.contrib.slim as slim 14 | from tensorflow.python.ops import control_flow_ops 15 | from datetime import datetime 16 | import numpy as np 17 | import os 18 | import get_data as data 19 | import tensorflow as tf 20 | 21 | batch_size = 100 22 | total_epochs = 100 23 | 24 | def model(): 25 | is_training = tf.placeholder(tf.bool, []) 26 | train_images, train_label = data.get_train_data(batch_size) 27 | test_images, test_label = data.get_test_data(batch_size) 28 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 29 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 30 | y_ = tf.cast(y_, tf.int64) 31 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 32 | activation_fn=tf.nn.crelu, 33 | normalizer_fn=slim.batch_norm, 34 | weights_regularizer=slim.l2_regularizer(0.005), 35 | normalizer_params={'is_training': is_training, 'decay': 0.95} 36 | ): 37 | conv1 =slim.conv2d(x, 8, [2,2], weights_initializer=tf.truncated_normal_initializer(mean=-0.32, stddev=0.51)) 38 | conv2 =slim.conv2d(conv1, 24, [18,18], weights_initializer=tf.truncated_normal_initializer(mean=-0.13, stddev=0.17)) 39 | pool1 = slim.max_pool2d(conv2, [4,4], stride=4, padding='SAME') 40 | flatten = slim.flatten(pool1) 41 | full1 = slim.fully_connected(flatten, 1848, weights_initializer=tf.truncated_normal_initializer(mean=-0.195993, stddev=0.866740), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 42 | logits = slim.fully_connected(full1, 2, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=0.02491, stddev=0.90), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 43 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 44 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 45 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 46 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 47 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 48 | 49 | # lr = tf.train.exponential_decay(0.1, 50 | # step, 51 | # 550*30, 52 | # 0.9, 53 | # staircase=True) 54 | # 55 | # 56 | # optimizer = tf.train.GradientDescentOptimizer(lr) 57 | optimizer = tf.train.AdamOptimizer(0.001) 58 | # lr_summary = tf.summary.scalar('lr', lr) 59 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 60 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 61 | if update_ops: 62 | updates = tf.group(*update_ops) 63 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 64 | 65 | loss_summary = tf.summary.scalar('loss', cross_entropy) 66 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 67 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 68 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 69 | 70 | def train(): 71 | gpu_options = tf.GPUOptions(allow_growth=True) 72 | is_training, train_step, _, accuracy, loss, merge_summary = model() 73 | 74 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 75 | sess.run(tf.global_variables_initializer()) 76 | train_data_length = 10000 77 | test_data_length = 50000 78 | steps_in_each_epoch = (train_data_length//batch_size) 79 | total_steps = int(total_epochs*steps_in_each_epoch) 80 | coord = tf.train.Coordinator() 81 | threads = tf.train.start_queue_runners(sess, coord) 82 | try: 83 | for i in range(total_steps): 84 | if coord.should_stop(): 85 | break 86 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 87 | if i % steps_in_each_epoch == 0: 88 | test_total_step = test_data_length//batch_size 89 | test_accuracy_list = [] 90 | test_loss_list = [] 91 | for _ in range(test_total_step): 92 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 93 | test_accuracy_list.append(test_accuracy_str) 94 | test_loss_list.append(test_loss_str) 95 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 96 | 97 | except tf.errors.OutOfRangeError: 98 | print('done') 99 | finally: 100 | coord.request_stop() 101 | coord.join(threads) 102 | if __name__ =='__main__': 103 | #CUDA2 104 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 105 | tf.reset_default_graph() 106 | train()#94.61 107 | -------------------------------------------------------------------------------- /rectangle.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Length:6, Num:386037, Mean:1.00, Std:0.01, 3 | conv[14,14,9,-0.18,0.68], 4 | conv[18,18,13,-0.26,0.93], 5 | pool[2,2,0.94], 6 | pool[4,4,0.57], 7 | full[1801,0.6176560743803923,0.9038750398909305], 8 | full[2,0.19075779624924527,0.47152749823228046] 9 | 10 | 11 | ''' 12 | 13 | import tensorflow as tf 14 | import tensorflow.contrib.slim as slim 15 | from tensorflow.python.ops import control_flow_ops 16 | from datetime import datetime 17 | import numpy as np 18 | import os 19 | import get_data as data 20 | import tensorflow as tf 21 | 22 | batch_size = 100 23 | total_epochs = 100 24 | 25 | def model(): 26 | is_training = tf.placeholder(tf.bool, []) 27 | train_images, train_label = data.get_train_data(batch_size) 28 | test_images, test_label = data.get_test_data(batch_size) 29 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 30 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 31 | y_ = tf.cast(y_, tf.int64) 32 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 33 | activation_fn=tf.nn.crelu, 34 | normalizer_fn=slim.batch_norm, 35 | weights_regularizer=slim.l2_regularizer(0.005), 36 | normalizer_params={'is_training': is_training, 'decay': 0.95} 37 | ): 38 | conv1 =slim.conv2d(x, 9, [14,14], weights_initializer=tf.truncated_normal_initializer(mean=-0.18, stddev=0.68)) 39 | conv2 =slim.conv2d(conv1, 13, [18,18], weights_initializer=tf.truncated_normal_initializer(mean=-0.26, stddev=0.93)) 40 | pool1 = slim.avg_pool2d(conv2, [2,2], stride=2, padding='SAME') 41 | pool2 = slim.avg_pool2d(pool1, [4,4], stride=4, padding='SAME') 42 | flatten = slim.flatten(pool2) 43 | full1 = slim.fully_connected(flatten, 1801, weights_initializer=tf.truncated_normal_initializer(mean=0.6176560, stddev=0.90), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 44 | logits = slim.fully_connected(full1, 2, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=0.190, stddev=0.4715), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 45 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 46 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 47 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 48 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 49 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 50 | 51 | # lr = tf.train.exponential_decay(0.1, 52 | # step, 53 | # 550*30, 54 | # 0.9, 55 | # staircase=True) 56 | # 57 | # 58 | # optimizer = tf.train.GradientDescentOptimizer(lr) 59 | optimizer = tf.train.AdamOptimizer(0.001) 60 | # lr_summary = tf.summary.scalar('lr', lr) 61 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 62 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 63 | if update_ops: 64 | updates = tf.group(*update_ops) 65 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 66 | 67 | loss_summary = tf.summary.scalar('loss', cross_entropy) 68 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 69 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 70 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 71 | 72 | def train(): 73 | gpu_options = tf.GPUOptions(allow_growth=True) 74 | is_training, train_step, _, accuracy, loss, merge_summary = model() 75 | 76 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 77 | sess.run(tf.global_variables_initializer()) 78 | train_data_length = 1000 79 | test_data_length = 50000 80 | steps_in_each_epoch = (train_data_length//batch_size) 81 | total_steps = int(total_epochs*steps_in_each_epoch) 82 | coord = tf.train.Coordinator() 83 | threads = tf.train.start_queue_runners(sess, coord) 84 | try: 85 | for i in range(total_steps): 86 | if coord.should_stop(): 87 | break 88 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 89 | if i % steps_in_each_epoch == 0: 90 | test_total_step = test_data_length//batch_size 91 | test_accuracy_list = [] 92 | test_loss_list = [] 93 | for _ in range(test_total_step): 94 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 95 | test_accuracy_list.append(test_accuracy_str) 96 | test_loss_list.append(test_loss_str) 97 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 98 | 99 | except tf.errors.OutOfRangeError: 100 | print('done') 101 | finally: 102 | coord.request_stop() 103 | coord.join(threads) 104 | if __name__ =='__main__': 105 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 106 | tf.reset_default_graph() 107 | train()#99.993 108 | -------------------------------------------------------------------------------- /rectangle_image.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Length:6, Num:6308, Mean:0.88, Std:0.04, 3 | conv[9,9,48,-0.08,0.63], 4 | pool[4,4,0.03], 5 | conv[7,7,43,-0.23,0.22], 6 | pool[4,4,0.01], 7 | pool[3,3,0.83], 8 | full[2,0.7260451837268884,0.3979923000550264] 9 | 10 | ''' 11 | 12 | import tensorflow as tf 13 | import tensorflow.contrib.slim as slim 14 | from tensorflow.python.ops import control_flow_ops 15 | from datetime import datetime 16 | import numpy as np 17 | import os 18 | import get_data as data 19 | import tensorflow as tf 20 | 21 | batch_size = 100 22 | total_epochs = 100 23 | 24 | def model(): 25 | is_training = tf.placeholder(tf.bool, []) 26 | train_images, train_label = data.get_train_data(batch_size) 27 | test_images, test_label = data.get_test_data(batch_size) 28 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 29 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 30 | y_ = tf.cast(y_, tf.int64) 31 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 32 | activation_fn=tf.nn.crelu, 33 | normalizer_fn=slim.batch_norm, 34 | weights_regularizer=slim.l2_regularizer(0.005), 35 | normalizer_params={'is_training': is_training, 'decay': 0.95} 36 | ): 37 | conv1 =slim.conv2d(x, 48, [9,9], weights_initializer=tf.truncated_normal_initializer(mean=-0.08, stddev=0.63)) 38 | pool1 = slim.max_pool2d(conv1, [4,4], stride=4, padding='SAME') 39 | conv2 =slim.conv2d(pool1, 43, [7,7], weights_initializer=tf.truncated_normal_initializer(mean=-0.23, stddev=0.22)) 40 | pool2 = slim.max_pool2d(conv2, [4,4], stride=4, padding='SAME') 41 | pool3 = slim.avg_pool2d(pool2, [3,3], stride=3, padding='SAME') 42 | flatten = slim.flatten(pool3) 43 | logits = slim.fully_connected(flatten, 2, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=0.726, stddev=0.397992), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 44 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 45 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 46 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 47 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 48 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 49 | 50 | # lr = tf.train.exponential_decay(0.1, 51 | # step, 52 | # 550*30, 53 | # 0.9, 54 | # staircase=True) 55 | # 56 | # 57 | # optimizer = tf.train.GradientDescentOptimizer(lr) 58 | optimizer = tf.train.AdamOptimizer(0.001) 59 | # lr_summary = tf.summary.scalar('lr', lr) 60 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 61 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 62 | if update_ops: 63 | updates = tf.group(*update_ops) 64 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 65 | 66 | loss_summary = tf.summary.scalar('loss', cross_entropy) 67 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 68 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 69 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 70 | 71 | def train(): 72 | gpu_options = tf.GPUOptions(allow_growth=True) 73 | is_training, train_step, _, accuracy, loss, merge_summary = model() 74 | 75 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 76 | sess.run(tf.global_variables_initializer()) 77 | train_data_length = 6000 78 | test_data_length = 50000 79 | steps_in_each_epoch = (train_data_length//batch_size) 80 | total_steps = int(total_epochs*steps_in_each_epoch) 81 | coord = tf.train.Coordinator() 82 | threads = tf.train.start_queue_runners(sess, coord) 83 | try: 84 | for i in range(total_steps): 85 | if coord.should_stop(): 86 | break 87 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 88 | if i % steps_in_each_epoch == 0: 89 | test_total_step = test_data_length//batch_size 90 | test_accuracy_list = [] 91 | test_loss_list = [] 92 | for _ in range(test_total_step): 93 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 94 | test_accuracy_list.append(test_accuracy_str) 95 | test_loss_list.append(test_loss_str) 96 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 97 | 98 | except tf.errors.OutOfRangeError: 99 | print('done') 100 | finally: 101 | coord.request_stop() 102 | coord.join(threads) 103 | if __name__ =='__main__': 104 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 105 | tf.reset_default_graph() 106 | train()#94.03 107 | -------------------------------------------------------------------------------- /rot.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.slim as slim 3 | from tensorflow.python.ops import control_flow_ops 4 | from datetime import datetime 5 | import numpy as np 6 | import os 7 | import get_data as data 8 | ''' 9 | Length:5, Num:8534644, Mean:0.91, Std:0.04, 10 | conv[8,8,5,-0.10,0.65], 11 | conv[8,8,27,0.17,0.46], 12 | pool[2,2,0.79], 13 | full[1609,-0.0023846697741753964,0.08991355789211397], 14 | full[10,0.425395045663272,0.5545578772968163] 15 | ''' 16 | 17 | batch_size = 100 18 | total_epochs = 100 19 | 20 | def model(): 21 | is_training = tf.placeholder(tf.bool, []) 22 | train_images, train_label = data.get_train_data(batch_size) 23 | test_images, test_label = data.get_test_data(batch_size) 24 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 25 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 26 | y_ = tf.cast(y_, tf.int64) 27 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 28 | activation_fn=tf.nn.crelu, 29 | normalizer_fn=slim.batch_norm, 30 | weights_regularizer=slim.l2_regularizer(0.005), 31 | normalizer_params={'is_training': is_training, 'decay': 0.95} 32 | ): 33 | conv1 =slim.conv2d(x, 5, [8,8], weights_initializer=tf.truncated_normal_initializer(mean=-0.10, stddev=0.65)) 34 | conv2 =slim.conv2d(conv1, 27, [8,8], weights_initializer=tf.truncated_normal_initializer(mean=0.17, stddev=0.46)) 35 | pool1 = slim.avg_pool2d(conv2, [2,2], stride=2, padding='SAME') 36 | flatten = slim.flatten(pool1) 37 | full1 = slim.fully_connected(flatten, 1609, weights_initializer=tf.truncated_normal_initializer(mean=-0.002384, stddev=0.08991), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 38 | logits = slim.fully_connected(full1, 10, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=0.425395, stddev=0.5545578), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 39 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 40 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 41 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 42 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 43 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 44 | 45 | # lr = tf.train.exponential_decay(0.1, 46 | # step, 47 | # 550*30, 48 | # 0.9, 49 | # staircase=True) 50 | # 51 | # 52 | # optimizer = tf.train.GradientDescentOptimizer(lr) 53 | optimizer = tf.train.AdamOptimizer(0.001) 54 | # lr_summary = tf.summary.scalar('lr', lr) 55 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 56 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 57 | if update_ops: 58 | updates = tf.group(*update_ops) 59 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 60 | 61 | loss_summary = tf.summary.scalar('loss', cross_entropy) 62 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 63 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 64 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 65 | 66 | def train(): 67 | gpu_options = tf.GPUOptions(allow_growth=True) 68 | is_training, train_step, _, accuracy, loss, merge_summary = model() 69 | 70 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 71 | sess.run(tf.global_variables_initializer()) 72 | train_data_length = 10000 73 | test_data_length = 50000 74 | steps_in_each_epoch = (train_data_length//batch_size) 75 | total_steps = int(total_epochs*steps_in_each_epoch) 76 | coord = tf.train.Coordinator() 77 | threads = tf.train.start_queue_runners(sess, coord) 78 | try: 79 | for i in range(total_steps): 80 | if coord.should_stop(): 81 | break 82 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 83 | if i % steps_in_each_epoch == 0: 84 | test_total_step = test_data_length//batch_size 85 | test_accuracy_list = [] 86 | test_loss_list = [] 87 | for _ in range(test_total_step): 88 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 89 | test_accuracy_list.append(test_accuracy_str) 90 | test_loss_list.append(test_loss_str) 91 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 92 | 93 | except tf.errors.OutOfRangeError: 94 | print('done') 95 | finally: 96 | coord.request_stop() 97 | coord.join(threads) 98 | if __name__ =='__main__': 99 | #CUDA2 100 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 101 | tf.reset_default_graph() 102 | train() #0.9455600380 103 | -------------------------------------------------------------------------------- /rot_back_image.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Length:4, Num:11512471, Mean:0.52, Std:0.04, 3 | conv[8,8,46,0.58,0.80], 4 | pool[2,2,0.93], 5 | full[1275,-0.027271153285668204,0.5305630184746687], 6 | full[10,0.15391551246327828,0.07176395426939763] 7 | 8 | ''' 9 | 10 | import tensorflow as tf 11 | import tensorflow.contrib.slim as slim 12 | from tensorflow.python.ops import control_flow_ops 13 | from datetime import datetime 14 | import numpy as np 15 | import os 16 | import get_data as data 17 | import tensorflow as tf 18 | 19 | batch_size = 100 20 | total_epochs = 100 21 | 22 | def model(): 23 | is_training = tf.placeholder(tf.bool, []) 24 | train_images, train_label = data.get_train_data(batch_size) 25 | test_images, test_label = data.get_test_data(batch_size) 26 | x = tf.cond(is_training, lambda:train_images, lambda:test_images) 27 | y_ = tf.cond(is_training, lambda:train_label, lambda:test_label) 28 | y_ = tf.cast(y_, tf.int64) 29 | with slim.arg_scope([slim.conv2d, slim.fully_connected], 30 | activation_fn=tf.nn.crelu, 31 | normalizer_fn=slim.batch_norm, 32 | weights_regularizer=slim.l2_regularizer(0.005), 33 | normalizer_params={'is_training': is_training, 'decay': 0.95} 34 | ): 35 | conv1 =slim.conv2d(x, 46, [8,8], weights_initializer=tf.truncated_normal_initializer(mean=0.58, stddev=0.80)) 36 | pool1 = slim.avg_pool2d(conv1, [2,2], stride=2, padding='SAME') 37 | flatten = slim.flatten(pool1) 38 | full1 = slim.fully_connected(flatten, 1275, weights_initializer=tf.truncated_normal_initializer(mean=-0.0272, stddev=0.530), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 39 | logits = slim.fully_connected(full1, 10, activation_fn=None, weights_initializer=tf.truncated_normal_initializer(mean=0.15391, stddev=0.71763), biases_initializer=tf.constant_initializer(0.1, dtype=tf.float32)) 40 | correct_prediction = tf.equal(tf.argmax(logits, 1), y_) 41 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 42 | regularization_loss = tf.add_n(slim.losses.get_regularization_losses()) 43 | cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_, logits=logits))+ regularization_loss 44 | step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) 45 | 46 | # lr = tf.train.exponential_decay(0.1, 47 | # step, 48 | # 550*30, 49 | # 0.9, 50 | # staircase=True) 51 | # 52 | # 53 | # optimizer = tf.train.GradientDescentOptimizer(lr) 54 | optimizer = tf.train.AdamOptimizer(0.001) 55 | # lr_summary = tf.summary.scalar('lr', lr) 56 | train_step = slim.learning.create_train_op(cross_entropy, optimizer, global_step=step) 57 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 58 | if update_ops: 59 | updates = tf.group(*update_ops) 60 | cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy) 61 | 62 | loss_summary = tf.summary.scalar('loss', cross_entropy) 63 | accuracy_summary = tf.summary.scalar('accuracy', accuracy) 64 | merge_summary = tf.summary.merge([loss_summary, accuracy_summary]) 65 | return is_training, train_step, step, accuracy, cross_entropy, merge_summary 66 | 67 | def train(): 68 | gpu_options = tf.GPUOptions(allow_growth=True) 69 | is_training, train_step, _, accuracy, loss, merge_summary = model() 70 | 71 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 72 | sess.run(tf.global_variables_initializer()) 73 | train_data_length = 10000 74 | test_data_length = 50000 75 | steps_in_each_epoch = (train_data_length//batch_size) 76 | total_steps = int(total_epochs*steps_in_each_epoch) 77 | coord = tf.train.Coordinator() 78 | threads = tf.train.start_queue_runners(sess, coord) 79 | try: 80 | for i in range(total_steps): 81 | if coord.should_stop(): 82 | break 83 | _, accuracy_str, loss_str, _ = sess.run([train_step, accuracy,loss, merge_summary], {is_training:True}) 84 | if i % steps_in_each_epoch == 0: 85 | test_total_step = test_data_length//batch_size 86 | test_accuracy_list = [] 87 | test_loss_list = [] 88 | for _ in range(test_total_step): 89 | test_accuracy_str, test_loss_str = sess.run([accuracy, loss], {is_training:False}) 90 | test_accuracy_list.append(test_accuracy_str) 91 | test_loss_list.append(test_loss_str) 92 | print('{}, {}, Step:{}/{}, train_loss:{}, acc:{}, test_loss:{}, accu:{}'.format(datetime.now(), i // steps_in_each_epoch, i, total_steps, loss_str, accuracy_str, np.mean(test_loss_list), np.mean(test_accuracy_list))) 93 | 94 | except tf.errors.OutOfRangeError: 95 | print('done') 96 | finally: 97 | coord.request_stop() 98 | coord.join(threads) 99 | if __name__ =='__main__': 100 | #CUDA2 101 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 102 | tf.reset_default_graph() 103 | train()#62.62 104 | --------------------------------------------------------------------------------