├── Denoising ├── cgp.py ├── cgp_config.py ├── cnn_model.py ├── cnn_train.py ├── example │ ├── example.pdf │ └── example.png ├── exp_main.py └── models │ ├── model.pth │ └── model_arch.txt ├── Inpainting ├── cgp.py ├── cgp_config.py ├── cnn_model.py ├── cnn_train.py ├── create_data.py ├── example │ ├── example.pdf │ └── example.png ├── exp_main.py └── models │ ├── model_SVHN_center.pth │ ├── model_arch_SVHN_center.txt │ ├── model_arch_celebA_center.txt │ └── model_celebA_center.pth ├── LICENSE └── README.md /Denoising/cgp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import csv 5 | import time 6 | import numpy as np 7 | import math 8 | 9 | # gene[f][c] f:function type, c:connection (nodeID) 10 | class Individual(object): 11 | 12 | def __init__(self, net_info, init): 13 | self.net_info = net_info 14 | self.gene = np.zeros((self.net_info.node_num + self.net_info.out_num, self.net_info.max_in_num + 1)).astype(int) 15 | self.is_active = np.empty(self.net_info.node_num + self.net_info.out_num).astype(bool) 16 | self.is_pool = np.empty(self.net_info.node_num + self.net_info.out_num).astype(bool) 17 | self.eval = None 18 | if init: 19 | print('init with specific architectures') 20 | self.init_gene_with_conv() # In the case of starting only convolution 21 | else: 22 | self.init_gene() # generate initial individual randomly 23 | 24 | def init_gene_with_conv(self): 25 | # initial architecture 26 | arch = ['S_ConvBlock_64_3'] 27 | 28 | input_layer_num = int(self.net_info.input_num / self.net_info.rows) + 1 29 | output_layer_num = int(self.net_info.out_num / self.net_info.rows) + 1 30 | layer_ids = [((self.net_info.cols - 1 - input_layer_num - output_layer_num) + i) // (len(arch)) for i in range(len(arch))] 31 | prev_id = 0 # i.e. input layer 32 | current_layer = input_layer_num 33 | block_ids = [] # *do not connect with these ids 34 | 35 | # building convolution net 36 | for i, idx in enumerate(layer_ids): 37 | 38 | current_layer += idx 39 | n = current_layer * self.net_info.rows + np.random.randint(self.net_info.rows) 40 | block_ids.append(n) 41 | self.gene[n][0] = self.net_info.func_type.index(arch[i]) 42 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 43 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 44 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 45 | if col - self.net_info.level_back >= 0 else 0 46 | 47 | self.gene[n][1] = prev_id 48 | for j in range(1, self.net_info.max_in_num): 49 | self.gene[n][j + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 50 | 51 | prev_id = n + self.net_info.input_num 52 | 53 | # output layer 54 | n = self.net_info.node_num 55 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 56 | self.gene[n][0] = np.random.randint(type_num) 57 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 58 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 59 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 60 | if col - self.net_info.level_back >= 0 else 0 61 | 62 | self.gene[n][1] = prev_id 63 | for i in range(1, self.net_info.max_in_num): 64 | self.gene[n][i + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 65 | block_ids.append(n) 66 | 67 | # intermediate node 68 | for n in range(self.net_info.node_num + self.net_info.out_num): 69 | 70 | if n in block_ids: 71 | continue 72 | 73 | # type gene 74 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 75 | self.gene[n][0] = np.random.randint(type_num) 76 | # connection gene 77 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 78 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 79 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 80 | if col - self.net_info.level_back >= 0 else 0 81 | for i in range(self.net_info.max_in_num): 82 | self.gene[n][i + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 83 | 84 | self.check_active() 85 | 86 | def init_gene(self): 87 | # intermediate node 88 | for n in range(self.net_info.node_num + self.net_info.out_num): 89 | # type gene 90 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 91 | self.gene[n][0] = np.random.randint(type_num) 92 | # connection gene 93 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 94 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 95 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 96 | if col - self.net_info.level_back >= 0 else 0 97 | for i in range(self.net_info.max_in_num): 98 | self.gene[n][i + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 99 | 100 | self.check_active() 101 | 102 | def __check_course_to_out(self, n): 103 | if not self.is_active[n]: 104 | self.is_active[n] = True 105 | t = self.gene[n][0] 106 | if n >= self.net_info.node_num: # output node 107 | in_num = self.net_info.out_in_num[t] 108 | else: # intermediate node 109 | in_num = self.net_info.func_in_num[t] 110 | 111 | for i in range(in_num): 112 | if self.gene[n][i+1] >= self.net_info.input_num: 113 | self.__check_course_to_out(self.gene[n][i+1] - self.net_info.input_num) 114 | 115 | def check_active(self): 116 | # clear 117 | self.is_active[:] = False 118 | # start from output nodes 119 | for n in range(self.net_info.out_num): 120 | self.__check_course_to_out(self.net_info.node_num + n) 121 | 122 | def __mutate(self, current, min_int, max_int): 123 | mutated_gene = current 124 | while current == mutated_gene: 125 | mutated_gene = min_int + np.random.randint(max_int - min_int) 126 | return mutated_gene 127 | 128 | def mutation(self, mutation_rate=0.01): 129 | active_check = False 130 | 131 | for n in range(self.net_info.node_num + self.net_info.out_num): 132 | t = self.gene[n][0] 133 | # mutation for type gene 134 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 135 | if np.random.rand() < mutation_rate and type_num > 1: 136 | self.gene[n][0] = self.__mutate(self.gene[n][0], 0, type_num) 137 | if self.is_active[n]: 138 | active_check = True 139 | # mutation for connection gene 140 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 141 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 142 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 143 | if col - self.net_info.level_back >= 0 else 0 144 | in_num = self.net_info.func_in_num[t] if n < self.net_info.node_num else self.net_info.out_in_num[t] 145 | for i in range(self.net_info.max_in_num): 146 | if np.random.rand() < mutation_rate and max_connect_id - min_connect_id > 1: 147 | self.gene[n][i+1] = self.__mutate(self.gene[n][i+1], min_connect_id, max_connect_id) 148 | if self.is_active[n] and i < in_num: 149 | active_check = True 150 | 151 | self.check_active() 152 | return active_check 153 | 154 | def neutral_mutation(self, mutation_rate=0.01): 155 | for n in range(self.net_info.node_num + self.net_info.out_num): 156 | t = self.gene[n][0] 157 | # mutation for type gene 158 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 159 | if not self.is_active[n] and np.random.rand() < mutation_rate and type_num > 1: 160 | self.gene[n][0] = self.__mutate(self.gene[n][0], 0, type_num) 161 | # mutation for connection gene 162 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 163 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 164 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 165 | if col - self.net_info.level_back >= 0 else 0 166 | in_num = self.net_info.func_in_num[t] if n < self.net_info.node_num else self.net_info.out_in_num[t] 167 | for i in range(self.net_info.max_in_num): 168 | if (not self.is_active[n] or i >= in_num) and np.random.rand() < mutation_rate \ 169 | and max_connect_id - min_connect_id > 1: 170 | self.gene[n][i+1] = self.__mutate(self.gene[n][i+1], min_connect_id, max_connect_id) 171 | 172 | self.check_active() 173 | return False 174 | 175 | def count_active_node(self): 176 | return self.is_active.sum() 177 | 178 | def copy(self, source): 179 | self.net_info = source.net_info 180 | self.gene = source.gene.copy() 181 | self.is_active = source.is_active.copy() 182 | self.eval = source.eval 183 | 184 | def active_net_list(self): 185 | net_list = [["input", 0]] 186 | active_cnt = np.arange(self.net_info.input_num + self.net_info.node_num + self.net_info.out_num) 187 | active_cnt[self.net_info.input_num:] = np.cumsum(self.is_active) 188 | 189 | for n, is_a in enumerate(self.is_active): 190 | if is_a: 191 | t = self.gene[n][0] 192 | if n < self.net_info.node_num: # intermediate node 193 | type_str = self.net_info.func_type[t] 194 | else: # output node 195 | type_str = self.net_info.out_type[t] 196 | 197 | connections = [active_cnt[self.gene[n][i+1]] for i in range(self.net_info.max_in_num)] 198 | net_list.append([type_str] + connections) 199 | return net_list 200 | 201 | 202 | # CGP with (1 + \lambda)-ES 203 | class CGP(object): 204 | def __init__(self, net_info, eval_func, lam=4, imgSize=32, init=False): 205 | self.lam = lam 206 | self.pop = [Individual(net_info, init) for _ in range(1 + self.lam)] 207 | self.eval_func = eval_func 208 | self.num_gen = 0 209 | self.num_eval = 0 210 | self.max_pool_num = int(math.log2(imgSize) - 2) 211 | self.init = init 212 | 213 | def _evaluation(self, pop, eval_flag): 214 | # create network list 215 | net_lists = [] 216 | active_index = np.where(eval_flag)[0] 217 | for i in active_index: 218 | net_lists.append(pop[i].active_net_list()) 219 | 220 | # evaluation 221 | fp = self.eval_func(net_lists) 222 | for i, j in enumerate(active_index): 223 | pop[j].eval = fp[i] 224 | evaluations = np.zeros(len(pop)) 225 | for i in range(len(pop)): 226 | evaluations[i] = pop[i].eval 227 | 228 | self.num_eval += len(net_lists) 229 | return evaluations 230 | 231 | def _log_data(self, net_info_type='active_only', start_time=0): 232 | log_list = [self.num_gen, self.num_eval, time.time()-start_time, self.pop[0].eval, self.pop[0].count_active_node()] 233 | if net_info_type == 'active_only': 234 | log_list.append(self.pop[0].active_net_list()) 235 | elif net_info_type == 'full': 236 | log_list += self.pop[0].gene.flatten().tolist() 237 | else: 238 | pass 239 | return log_list 240 | 241 | def _log_data_children(self, net_info_type='active_only', start_time=0, pop=None): 242 | log_list = [self.num_gen, self.num_eval, time.time()-start_time, pop.eval, pop.count_active_node()] 243 | if net_info_type == 'active_only': 244 | log_list.append(pop.active_net_list()) 245 | elif net_info_type == 'full': 246 | log_list += pop.gene.flatten().tolist() 247 | else: 248 | pass 249 | return log_list 250 | 251 | def load_log(self, log_data): 252 | self.num_gen = log_data[0] 253 | self.num_eval = log_data[1] 254 | net_info = self.pop[0].net_info 255 | self.pop[0].eval = log_data[3] 256 | self.pop[0].gene = np.array(log_data[5:]).reshape((net_info.node_num + net_info.out_num, net_info.max_in_num + 1)) 257 | self.pop[0].check_active() 258 | 259 | # Evolution CGP: 260 | # At each iteration: 261 | # - Generate lambda individuals in which at least one active node changes (i.e., forced mutation) 262 | # - Mutate the best individual with neutral mutation (unchanging the active nodes) 263 | # if the best individual is not updated. 264 | def modified_evolution(self, max_eval=100, mutation_rate=0.01, log_file='./log.txt', arch_file='./arch.txt'): 265 | with open(log_file,'w') as fw, open(arch_file, 'w') as fw_a, open('child.txt', 'w') as fw_c, open('arch_child.txt', 'w') as fw_ac: 266 | writer = csv.writer(fw, lineterminator='\n') 267 | writer_a = csv.writer(fw_a, lineterminator='\n') 268 | writer_c = csv.writer(fw_c, lineterminator='\n') 269 | writer_ac = csv.writer(fw_ac, lineterminator='\n') 270 | start_time = time.time() 271 | 272 | eval_flag = np.empty(self.lam) 273 | 274 | active_num = self.pop[0].count_active_node() 275 | if self.init: 276 | pass 277 | else: # in the case of not using an init indiviudal 278 | while active_num < self.pop[0].net_info.min_active_num or active_num > self.pop[0].net_info.max_active_num: 279 | self.pop[0].mutation(1.0) 280 | active_num = self.pop[0].count_active_node() 281 | self._evaluation([self.pop[0]], np.array([True])) 282 | print(self._log_data(net_info_type='active_only', start_time=start_time)) 283 | 284 | 285 | while self.num_gen < max_eval: 286 | self.num_gen += 1 287 | # reproduction 288 | for i in range(self.lam): 289 | eval_flag[i] = False 290 | self.pop[i + 1].copy(self.pop[0]) # copy a parent 291 | active_num = self.pop[i + 1].count_active_node() 292 | # mutation 293 | while not eval_flag[i] or active_num < self.pop[i + 1].net_info.min_active_num \ 294 | or active_num > self.pop[i + 1].net_info.max_active_num: 295 | self.pop[i + 1].copy(self.pop[0]) # copy a parent 296 | eval_flag[i] = self.pop[i + 1].mutation(mutation_rate) # mutation 297 | active_num = self.pop[i + 1].count_active_node() 298 | 299 | # evaluation and selection 300 | evaluations = self._evaluation(self.pop[1:], eval_flag=eval_flag) 301 | best_arg = evaluations.argmax() 302 | # save log 303 | for c in range(1 + self.lam): 304 | writer_c.writerow(self._log_data_children(net_info_type='full', start_time=start_time, pop=self.pop[c])) 305 | writer_ac.writerow(self._log_data_children(net_info_type='active_only', start_time=start_time, pop=self.pop[c])) 306 | 307 | if evaluations[best_arg] > self.pop[0].eval: 308 | self.pop[0].copy(self.pop[best_arg + 1]) 309 | else: 310 | self.pop[0].neutral_mutation(mutation_rate) # modify the parent 311 | 312 | # display and save log 313 | print(self._log_data(net_info_type='active_only', start_time=start_time)) 314 | writer.writerow(self._log_data(net_info_type='full', start_time=start_time)) 315 | writer_a.writerow(self._log_data(net_info_type='active_only', start_time=start_time)) 316 | -------------------------------------------------------------------------------- /Denoising/cgp_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import multiprocessing as mp 5 | import multiprocessing.pool 6 | import numpy as np 7 | import cnn_train as cnn 8 | 9 | 10 | # wrapper function for multiprocessing 11 | def arg_wrapper_mp(args): 12 | return args[0](*args[1:]) 13 | 14 | class NoDaemonProcess(mp.Process): 15 | # make 'daemon' attribute always return False 16 | def _get_daemon(self): 17 | return False 18 | def _set_daemon(self, value): 19 | pass 20 | daemon = property(_get_daemon, _set_daemon) 21 | 22 | # We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool 23 | # because the latter is only a wrapper function, not a proper class. 24 | class NoDaemonProcessPool(multiprocessing.pool.Pool): 25 | Process = NoDaemonProcess 26 | 27 | 28 | # Evaluation of CNNs 29 | def cnn_eval(net, gpu_id, epoch_num, batchsize, dataset, verbose, imgSize): 30 | 31 | print('\tgpu_id:', gpu_id, ',', net) 32 | train = cnn.CNN_train(dataset, validation=True, verbose=verbose, imgSize=imgSize, batchsize=batchsize) 33 | evaluation = train(net, gpu_id, epoch_num=epoch_num, out_model=None) 34 | print('\tgpu_id:', gpu_id, ', eval:', evaluation) 35 | return evaluation 36 | 37 | 38 | class CNNEvaluation(object): 39 | def __init__(self, gpu_num, dataset='cifar10', verbose=True, epoch_num=50, batchsize=16, imgSize=32): 40 | self.gpu_num = gpu_num 41 | self.epoch_num = epoch_num 42 | self.batchsize = batchsize 43 | self.dataset = dataset 44 | self.verbose = verbose 45 | self.imgSize = imgSize 46 | 47 | def __call__(self, net_lists): 48 | evaluations = np.zeros(len(net_lists)) 49 | for i in np.arange(0, len(net_lists), self.gpu_num): 50 | process_num = np.min((i + self.gpu_num, len(net_lists))) - i 51 | pool = NoDaemonProcessPool(process_num) 52 | arg_data = [(cnn_eval, net_lists[i+j], j, self.epoch_num, self.batchsize, self.dataset, self.verbose, self.imgSize) for j in range(process_num)] 53 | evaluations[i:i+process_num] = pool.map(arg_wrapper_mp, arg_data) 54 | pool.terminate() 55 | 56 | return evaluations 57 | 58 | 59 | # network configurations 60 | class CgpInfoConvSet(object): 61 | def __init__(self, rows=30, cols=40, level_back=40, min_active_num=8, max_active_num=50): 62 | self.input_num = 1 63 | # "S_" means that the layer has a convolution layer without downsampling. 64 | # "D_" means that the layer has a convolution layer with downsampling. 65 | # "Sum" means that the layer has a skip connection. 66 | self.func_type = ['S_ConvBlock_256_1', 'S_ConvBlock_256_3', 'S_ConvBlock_256_5', 67 | 'S_ConvBlock_128_1', 'S_ConvBlock_128_3', 'S_ConvBlock_128_5', 68 | 'S_ConvBlock_64_1', 'S_ConvBlock_64_3', 'S_ConvBlock_64_5', 69 | 'S_SumConvBlock_256_1', 'S_SumConvBlock_256_3','S_SumConvBlock_256_5', 70 | 'S_SumConvBlock_128_1', 'S_SumConvBlock_128_3','S_SumConvBlock_128_5', 71 | 'S_SumConvBlock_64_1', 'S_SumConvBlock_64_3', 'S_SumConvBlock_64_5'] 72 | 73 | self.func_in_num = [1, 1, 1, 74 | 1, 1, 1, 75 | 1, 1, 1, 76 | 1, 1, 1, 77 | 1, 1, 1, 78 | 1, 1, 1] 79 | 80 | self.out_num = 1 81 | self.out_type = ['S_DeConvBlock_1_3'] 82 | self.out_in_num = [1] 83 | 84 | # CGP network configuration 85 | self.rows = rows 86 | self.cols = cols 87 | self.node_num = rows * cols 88 | self.level_back = level_back 89 | self.min_active_num = min_active_num 90 | self.max_active_num = max_active_num 91 | 92 | self.func_type_num = len(self.func_type) 93 | self.out_type_num = len(self.out_type) 94 | self.max_in_num = np.max([np.max(self.func_in_num), np.max(self.out_in_num)]) 95 | -------------------------------------------------------------------------------- /Denoising/cnn_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | from torch.nn import init 7 | from torch.autograd import Variable 8 | from collections import OrderedDict 9 | import math 10 | import copy 11 | 12 | class ConvBlock(nn.Module): 13 | def __init__(self, in_size, out_size, kernel, stride): 14 | super(ConvBlock, self).__init__() 15 | pad_size = kernel // 2 16 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, stride=stride, padding=pad_size, bias=False), 17 | # nn.BatchNorm2d(out_size), 18 | nn.ReLU(inplace=True),) 19 | 20 | def forward(self, inputs): 21 | outputs = self.conv1(inputs) 22 | return outputs 23 | 24 | class DeConvBlock(nn.Module): 25 | def __init__(self, in_size, out_size, kernel): 26 | super(DeConvBlock, self).__init__() 27 | pad_size = kernel // 2 28 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, stride=2, padding=pad_size, output_padding=1, bias=False), 29 | # nn.BatchNorm2d(out_size), 30 | nn.ReLU(inplace=True),) 31 | 32 | def forward(self, inputs): 33 | outputs = self.conv1(inputs) 34 | return outputs 35 | 36 | class ConvBlock_last(nn.Module): 37 | def __init__(self, in_size, out_size, kernel): 38 | super(ConvBlock_last, self).__init__() 39 | pad_size = kernel // 2 40 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, padding=pad_size, bias=False)) 41 | # nn.BatchNorm2d(out_size), 42 | # nn.Tanh()) 43 | 44 | def forward(self, inputs): 45 | outputs = self.conv1(inputs) 46 | return outputs 47 | 48 | class DeConvBlock_last(nn.Module): 49 | def __init__(self, in_size, out_size, kernel): 50 | super(DeConvBlock_last, self).__init__() 51 | pad_size = kernel // 2 52 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, padding=pad_size, bias=False)) 53 | # nn.BatchNorm2d(out_size), 54 | # nn.Tanh()) 55 | 56 | def forward(self, inputs): 57 | outputs = self.conv1(inputs) 58 | return outputs 59 | 60 | class ConvBlock_s(nn.Module): 61 | def __init__(self, in_size, out_size, kernel, stride): 62 | super(ConvBlock_s, self).__init__() 63 | pad_size = kernel // 2 64 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, stride=stride, padding=pad_size, bias=False), 65 | # nn.BatchNorm2d(out_size), 66 | nn.ReLU(inplace=True),) 67 | 68 | def forward(self, inputs): 69 | outputs = self.conv1(inputs) 70 | return outputs 71 | 72 | class ConvBlock_sum(nn.Module): 73 | def __init__(self, in_size, out_size, kernel): 74 | super(ConvBlock_sum, self).__init__() 75 | pad_size = kernel // 2 76 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, padding=pad_size, bias=False), 77 | # nn.BatchNorm2d(out_size), 78 | nn.ReLU(inplace=True),) 79 | self.relu = nn.ReLU(inplace=True) 80 | 81 | def forward(self, inputs1, inputs2): 82 | outputs = self.conv1(inputs1) 83 | in_data = [outputs, inputs2] 84 | # check of the channel size 85 | small_ch_id, large_ch_id = (0, 1) if in_data[0].size(1) < in_data[1].size(1) else (1, 0) 86 | offset = int(in_data[large_ch_id].size()[1] - in_data[small_ch_id].size()[1]) 87 | if offset != 0: 88 | tmp = in_data[large_ch_id].data[:, :offset, :, :] 89 | tmp = Variable(tmp).clone() 90 | in_data[small_ch_id] = torch.cat([in_data[small_ch_id], tmp * 0], 1) 91 | out = torch.add(in_data[0], in_data[1]) 92 | return self.relu(out) 93 | 94 | class DeConvBlock_sum(nn.Module): 95 | def __init__(self, in_size, out_size, kernel): 96 | super(DeConvBlock_sum, self).__init__() 97 | pad_size = kernel // 2 98 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, stride=2, padding=pad_size, output_padding=1, bias=False), 99 | # nn.BatchNorm2d(out_size), 100 | nn.ReLU(inplace=True),) 101 | self.relu = nn.ReLU(inplace=True) 102 | 103 | def forward(self, inputs1, inputs2): 104 | outputs1 = self.conv1(inputs1) 105 | offset = outputs1.size()[2] - inputs2.size()[2] 106 | padding = 2 * [offset // 2, offset // 2] 107 | outputs2 = F.pad(inputs2, padding) 108 | out = torch.add(outputs1, outputs2) 109 | return self.relu(out) 110 | 111 | 112 | 113 | class CGP2CNN_autoencoder(nn.Module): 114 | def __init__(self, cgp, in_channel, n_class, imgSize): 115 | super(CGP2CNN_autoencoder, self).__init__() 116 | self.cgp = cgp 117 | self.pool_size = 2 118 | self.arch = OrderedDict() 119 | self.encode = [] 120 | self.decode = [] 121 | self.channel_num = [None for _ in range(len(self.cgp))] 122 | self.size = [None for _ in range(len(self.cgp))] 123 | self.channel_num[0] = in_channel 124 | self.size[0] = imgSize 125 | # encoder 126 | i = 0 127 | for name, in1 in self.cgp: 128 | if name == 'input' or 'DeConv' in name: 129 | i += 1 130 | continue 131 | key = name.split('_') 132 | down = key[0] 133 | func = key[1] 134 | out_size = int(key[2]) 135 | kernel = int(key[3]) 136 | if down == 'S': 137 | self.channel_num[i] = out_size 138 | self.size[i] = self.size[in1] 139 | if func == 'ConvBlock': 140 | self.encode.append(ConvBlock(self.channel_num[in1], out_size, kernel, stride=1)) 141 | else: 142 | self.encode.append(ConvBlock_s(self.channel_num[in1], out_size, kernel, stride=1)) 143 | else: 144 | self.channel_num[i] = out_size 145 | self.size[i] = self.size[in1] 146 | if func == 'ConvBlock': 147 | self.encode.append(ConvBlock(self.channel_num[in1], out_size, kernel, stride=2)) 148 | else: 149 | self.encode.append(ConvBlock_s(self.channel_num[in1], out_size, kernel, stride=2)) 150 | i += 1 151 | 152 | # decoder 153 | self.channel_num_d = [None for _ in range(len(self.cgp))] 154 | i -= 2 # skip the last layer 155 | self.channel_num_d[0] = self.channel_num[i] 156 | self.channel_num_d[1] = self.channel_num[i] 157 | i = 0 158 | self.cgp_inverse = copy.deepcopy(self.cgp) 159 | self.cgp_inverse.reverse() 160 | for j in range(len(self.cgp_inverse)): 161 | self.cgp_inverse[j][1] = int(math.fabs(self.cgp_inverse[j][1]-(len(self.cgp_inverse)-3))) 162 | for j in range(len(self.cgp_inverse)-1): 163 | if j == 0: 164 | i += 1 165 | continue 166 | name = self.cgp_inverse[j][0] 167 | key = name.split('_') 168 | down = key[0] 169 | func = key[1] 170 | out_size = int(key[2]) 171 | kernel = int(key[3]) 172 | self.channel_num_d[i] = out_size 173 | if down == 'S': 174 | if func == 'ConvBlock': 175 | self.decode.append(ConvBlock(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel, stride=1)) 176 | else: 177 | self.decode.append(ConvBlock_sum(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel)) 178 | else: 179 | if func == 'ConvBlock': 180 | self.decode.append(DeConvBlock(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel)) 181 | else: 182 | self.decode.append(DeConvBlock_sum(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel)) 183 | i += 1 184 | # the last layer 185 | for j in range(1): 186 | name = self.cgp_inverse[j][0] 187 | key = name.split('_') 188 | down = key[0] 189 | func = key[1] 190 | out_size = int(key[2]) 191 | kernel = int(key[3]) 192 | self.channel_num_d[-1] = out_size 193 | if down == 'S': 194 | if func == 'DeConvBlock': 195 | self.decode.append(DeConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 196 | else: 197 | self.decode.append(ConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 198 | else: 199 | if func == 'DeConvBlock': 200 | self.decode.append(DeConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 201 | else: 202 | self.decode.append(ConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 203 | 204 | self.network = self.encode + self.decode 205 | self.layer_module = nn.ModuleList(self.network) 206 | self.outputs = [None for _ in range(len(self.cgp)-1)] 207 | self.outputs_d = [None for _ in range(len(self.cgp_inverse))] 208 | self.outputs_sum = [None for _ in range(len(self.cgp_inverse))] 209 | 210 | def main(self,x): 211 | out = x 212 | outputs = self.outputs 213 | outputs[0] = x # input image 214 | outputs_d = self.outputs_d 215 | outputs_sum = self.outputs_sum 216 | nodeID = 1 217 | sumID = 0 218 | decodeID = 1 219 | flag = True 220 | for layer in self.layer_module: 221 | # encoder 222 | if nodeID <= len(self.encode): 223 | if isinstance(layer, ConvBlock_s): 224 | outputs[nodeID] = layer(outputs[self.cgp[nodeID][1]]) 225 | outputs_sum[sumID] = outputs[nodeID] 226 | sumID += 1 227 | else: 228 | outputs[nodeID] = layer(outputs[self.cgp[nodeID][1]]) 229 | # decoder 230 | elif nodeID < (len(self.decode)+len(self.encode)): 231 | if flag: 232 | outputs_d[0] = outputs[nodeID-1] 233 | outputs_d[1] = outputs[nodeID-1] 234 | del outputs 235 | flag = False 236 | if isinstance(layer, ConvBlock_sum): 237 | sumID -= 1 238 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 239 | elif isinstance(layer, DeConvBlock_sum): 240 | sumID -= 1 241 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 242 | else: 243 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]]) 244 | else: 245 | if isinstance(layer, ConvBlock_sum): 246 | sumID -= 1 247 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 248 | elif isinstance(layer, DeConvBlock_sum): 249 | sumID -= 1 250 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 251 | else: 252 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]]) 253 | decodeID += 1 254 | nodeID += 1 255 | # the last layer 256 | layer = self.layer_module[-1] 257 | out = layer(outputs_d[decodeID-1]) 258 | return out 259 | 260 | def forward(self, x, t): 261 | return self.main(x) 262 | -------------------------------------------------------------------------------- /Denoising/cnn_train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import time 5 | import math 6 | import numpy as np 7 | import torch 8 | import torch.nn as nn 9 | from torch.nn import init 10 | import torch.nn.parallel 11 | import torch.backends.cudnn as cudnn 12 | import torch.optim as optim 13 | import torch.utils.data 14 | import torchvision.datasets as dset 15 | import torchvision.transforms as transforms 16 | import torchvision.utils as vutils 17 | from torch.autograd import Variable 18 | import random 19 | from skimage.measure import compare_psnr 20 | import os 21 | 22 | from cnn_model import CGP2CNN_autoencoder 23 | 24 | 25 | def weights_init(m): 26 | classname = m.__class__.__name__ 27 | if classname.find('Conv') != -1: 28 | m.weight.data.normal_(0.0, 0.02) 29 | elif classname.find('BatchNorm') != -1: 30 | m.weight.data.normal_(1.0, 0.02) 31 | m.bias.data.fill_(0) 32 | 33 | def weights_init_normal(m): 34 | classname = m.__class__.__name__ 35 | if classname.find('Conv2d') != -1: 36 | m.apply(weights_init_normal_) 37 | elif classname.find('Linear') != -1: 38 | init.uniform(m.weight.data, 0.0, 0.02) 39 | elif classname.find('BatchNorm2d') != -1: 40 | init.uniform(m.weight.data, 1.0, 0.02) 41 | init.constant(m.bias.data, 0.0) 42 | 43 | def weights_init_normal_(m): 44 | classname = m.__class__.__name__ 45 | if classname.find('Conv') != -1: 46 | init.uniform(m.weight.data, 0.0, 0.02) 47 | elif classname.find('Linear') != -1: 48 | init.uniform(m.weight.data, 0.0, 0.02) 49 | elif classname.find('BatchNorm2d') != -1: 50 | init.uniform(m.weight.data, 1.0, 0.02) 51 | init.constant(m.bias.data, 0.0) 52 | 53 | def weights_init_xavier(m): 54 | classname = m.__class__.__name__ 55 | if classname.find('Conv') != -1: 56 | init.xavier_normal(m.weight.data, gain=1) 57 | elif classname.find('Linear') != -1: 58 | init.xavier_normal(m.weight.data, gain=1) 59 | elif classname.find('BatchNorm2d') != -1: 60 | init.uniform(m.weight.data, 1.0, 0.02) 61 | init.constant(m.bias.data, 0.0) 62 | 63 | def weights_init_kaiming(m): 64 | classname = m.__class__.__name__ 65 | if classname.find('Conv') != -1: 66 | init.kaiming_normal(m.weight.data, a=0, mode='fan_in') 67 | elif classname.find('Linear') != -1: 68 | init.kaiming_normal(m.weight.data, a=0, mode='fan_in') 69 | elif classname.find('BatchNorm2d') != -1: 70 | init.uniform(m.weight.data, 1.0, 0.02) 71 | init.constant(m.bias.data, 0.0) 72 | 73 | def weights_init_orthogonal(m): 74 | classname = m.__class__.__name__ 75 | print(classname) 76 | if classname.find('Conv') != -1: 77 | init.orthogonal(m.weight.data, gain=1) 78 | elif classname.find('Linear') != -1: 79 | init.orthogonal(m.weight.data, gain=1) 80 | elif classname.find('BatchNorm2d') != -1: 81 | init.uniform(m.weight.data, 1.0, 0.02) 82 | init.constant(m.bias.data, 0.0) 83 | 84 | def init_weights(net, init_type='normal'): 85 | print('initialization method [%s]' % init_type) 86 | if init_type == 'normal': 87 | net.apply(weights_init_normal) 88 | elif init_type == 'xavier': 89 | net.apply(weights_init_xavier) 90 | elif init_type == 'kaiming': 91 | net.apply(weights_init_kaiming) 92 | elif init_type == 'orthogonal': 93 | net.apply(weights_init_orthogonal) 94 | else: 95 | raise NotImplementedError('initialization method [%s] is not implemented' % init_type) 96 | 97 | 98 | # __init__: load dataset 99 | # __call__: training the CNN defined by CGP list 100 | class CNN_train(): 101 | def __init__(self, dataset_name, validation=True, verbose=True, imgSize=32, batchsize=16): 102 | # dataset_name: name of data set ('bsds'(color) or 'bsds_gray') 103 | # validation: [True] model train/validation mode 104 | # [False] model test mode for final evaluation of the evolved model 105 | # (raining data : all training data, test data : all test data) 106 | # verbose: flag of display 107 | self.verbose = verbose 108 | self.imgSize = imgSize 109 | self.validation = validation 110 | self.batchsize = batchsize 111 | self.dataset_name = dataset_name 112 | 113 | # load dataset 114 | if dataset_name == 'bsds' or dataset_name == 'bsds_gray': 115 | if dataset_name == 'bsds': 116 | self.n_class = 10 117 | self.channel = 3 118 | self.num_work = 2 119 | data_transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.RandomCrop(64, 0), transforms.ToTensor()]) 120 | test_data_transform = transforms.Compose([transforms.ToTensor()]) 121 | if self.validation: 122 | dataset = dset.ImageFolder(root='/dataset/BSDS500/color/train', transform=data_transform) 123 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(self.num_work)) 124 | test_dataset = dset.ImageFolder(root='/dataset/BSDS500/color/val', transform=test_data_transform) 125 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=True, num_workers=int(self.num_work)) 126 | else: 127 | dataset = dset.ImageFolder(root='/dataset/BSDS500/color/retrain', transform=data_transform) 128 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(self.num_work)) 129 | test_dataset = dset.ImageFolder(root='/dataset/BSDS500/color/test', transform=test_data_transform) 130 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=int(self.num_work)) 131 | elif dataset_name == 'bsds_gray': 132 | self.n_class = 10 133 | self.channel = 1 134 | self.num_work = 2 135 | data_transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.RandomCrop(64, 0), transforms.ToTensor()]) 136 | test_data_transform = transforms.Compose([transforms.ToTensor()]) 137 | if self.validation: 138 | dataset = dset.ImageFolder(root='/dataset/BSDS500/gray/train', transform=data_transform) 139 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(self.num_work)) 140 | test_dataset = dset.ImageFolder(root='/dataset/BSDS500/gray/val', transform=test_data_transform) 141 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=True, num_workers=int(self.num_work)) 142 | else: 143 | dataset = dset.ImageFolder(root='/dataset/BSDS500/gray/retrain', transform=data_transform) 144 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(self.num_work)) 145 | test_dataset = dset.ImageFolder(root='/dataset/BSDS500/gray/test', transform=test_data_transform) 146 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=int(self.num_work)) 147 | print('train num ', len(self.dataloader.dataset)) 148 | print('val/test num ', len(self.test_dataloader.dataset)) 149 | else: 150 | print('\tInvalid input dataset name at CNN_train()') 151 | exit(1) 152 | 153 | def __call__(self, cgp, gpuID, epoch_num=200, out_model='mymodel.model'): 154 | if self.verbose: 155 | print('GPUID :', gpuID) 156 | print('epoch_num:', epoch_num) 157 | 158 | # model 159 | torch.backends.cudnn.benchmark = True 160 | model = CGP2CNN_autoencoder(cgp, self.channel, self.n_class, self.imgSize) 161 | model.cuda(gpuID) 162 | # Loss and Optimizer 163 | criterion = nn.MSELoss() 164 | criterion.cuda(gpuID) 165 | optimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.5, 0.999)) 166 | input = torch.FloatTensor(self.batchsize, self.channel, self.imgSize, self.imgSize) 167 | input = input.cuda(gpuID) 168 | # Noise level 169 | std_list = [30,50,70] 170 | n = 50 171 | # for outputs 172 | if not os.path.exists('./outputs'): 173 | os.mkdir('./outputs') 174 | 175 | # Train loop 176 | for epoch in range(1, epoch_num+1): 177 | start_time = time.time() 178 | if self.verbose: 179 | print('epoch', epoch) 180 | train_loss = 0 181 | ite = 0 182 | for module in model.children(): 183 | module.train(True) 184 | for _, (data, _) in enumerate(self.dataloader): 185 | if self.dataset_name == 'bsds_gray': 186 | data = data[:,0:1,:,:] # for gray scale images 187 | data = data.cuda(gpuID) 188 | std = std_list[random.randint(0,len(std_list)-1)] 189 | for _ in range(1,n,1): 190 | input.resize_as_(data).copy_(data) 191 | input_ = Variable(input) 192 | data_noise = self.gaussian_noise(input_, 0.0, std) 193 | optimizer.zero_grad() 194 | try: 195 | output = model(data_noise, None) 196 | except: 197 | import traceback 198 | traceback.print_exc() 199 | return 0. 200 | loss = criterion(output, input_) 201 | train_loss += loss.data[0] 202 | loss.backward() 203 | optimizer.step() 204 | if ite == 0: 205 | vutils.save_image(data_noise.data, './noise_samples%d.png' % gpuID, normalize=False) 206 | vutils.save_image(input_.data, './org_samples%d.png' % gpuID, normalize=False) 207 | vutils.save_image(output.data, './output%d.png' % gpuID, normalize=False) 208 | ite += 1 209 | print('Train set : Average loss: {:.4f}'.format(train_loss)) 210 | print('time ', time.time()-start_time) 211 | if self.validation: 212 | if epoch == epoch_num: 213 | for module in model.children(): 214 | module.train(False) 215 | t_loss = self.__test_per_std(model, criterion, gpuID, input, std_list) 216 | else: 217 | if epoch % 10 == 0: 218 | for module in model.children(): 219 | module.train(False) 220 | t_loss = self.__test_per_std(model, criterion, gpuID, input, std_list) 221 | if epoch == 200: 222 | for param_group in optimizer.param_groups: 223 | tmp = param_group['lr'] 224 | tmp *= 0.1 225 | for param_group in optimizer.param_groups: 226 | param_group['lr'] = tmp 227 | if epoch == 400: 228 | for param_group in optimizer.param_groups: 229 | tmp = param_group['lr'] 230 | tmp *= 0.1 231 | for param_group in optimizer.param_groups: 232 | param_group['lr'] = tmp 233 | # save the model 234 | torch.save(model.state_dict(), './model_%d.pth' % int(gpuID)) 235 | return t_loss 236 | 237 | 238 | # generate gaussian noise 239 | def gaussian_noise(self, inp, mean, std): 240 | noise = Variable(inp.data.new(inp.size()).normal_(mean, std)) 241 | noise = torch.div(noise, 255.0) 242 | return inp + noise 243 | 244 | # calc PSNR by using "compare_psnr of skimage.measure" 245 | def calcPSNR(self, image1, image2): 246 | image1 *= 255 247 | image2 *= 255 248 | image1[image1>255] = 255 249 | image1[image1<0] = 0 250 | image2[image2>255] = 255 251 | image2[image2<0] = 0 252 | return compare_psnr(image1, image2, data_range=255) 253 | 254 | # For validation/test 255 | def __test_per_std(self, model, criterion, gpuID, input, std_list): 256 | test_loss = 0 257 | total_psnr = 0 258 | for std in std_list: 259 | print('std', std) 260 | ite = 0 261 | psnr = 0 262 | psnr2 = 0 263 | psnr3 = 0 264 | for _, (data, _) in enumerate(self.test_dataloader): 265 | if self.dataset_name == 'bsds_gray': 266 | data = data[:,0:1,:,:] 267 | data = data.cuda(gpuID) 268 | input.resize_as_(data).copy_(data) 269 | input_ = Variable(input, volatile=True) 270 | data_noise = self.gaussian_noise(input_, 0.0, std) 271 | try: 272 | output = model(data_noise, None) 273 | except: 274 | import traceback 275 | traceback.print_exc() 276 | return 0. 277 | loss = criterion(output, input_) 278 | psnr += -10 * math.log10(loss.data[0]) 279 | test_loss += loss.data[0] 280 | 281 | # # PSNR 282 | # img1 = (output.data).cpu().numpy() 283 | # img2 = (input_.data).cpu().numpy() 284 | # imdf = img2*255.0 - img1*255.0 285 | # imdf = imdf ** 2 286 | # rmse = np.sqrt(np.mean(imdf)) 287 | # psnr2 += 20 * math.log10(255.0/rmse) 288 | # psnr3 += self.calcPSNR(img2, img1) 289 | 290 | # save images 291 | vutils.save_image(output.data, './outputs/test_output_std%02d_%03d.png' % (int(std), int(ite)), normalize=False) 292 | vutils.save_image(data_noise.data, './outputs/test_output_std%02d_%03d_.png' % (int(std), int(ite)), normalize=False) 293 | vutils.save_image(input_.data, './outputs/test_output_std%02d_%03d__.png' % (int(std), int(ite)), normalize=False) 294 | ite += 1 295 | psnr /= (ite) 296 | # psnr2 /= (ite) 297 | # psnr3 /= (ite) 298 | test_loss /= (ite) 299 | total_psnr += psnr 300 | print('Test PSNR: {:.4f}'.format(psnr)) 301 | # print('Test PSNR2: {:.4f}'.format(psnr2)) 302 | # print('Test PSNR3: {:.4f}'.format(psnr3)) 303 | print('Test loss : {:.4f}'.format(test_loss)) 304 | 305 | total_psnr /= len(std_list) 306 | return total_psnr 307 | -------------------------------------------------------------------------------- /Denoising/example/example.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Denoising/example/example.pdf -------------------------------------------------------------------------------- /Denoising/example/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Denoising/example/example.png -------------------------------------------------------------------------------- /Denoising/exp_main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import pickle 6 | import pandas as pd 7 | 8 | from cgp import * 9 | from cgp_config import * 10 | from cnn_train import CNN_train 11 | 12 | 13 | if __name__ == '__main__': 14 | 15 | parser = argparse.ArgumentParser(description='Evolving CAE structures') 16 | parser.add_argument('--gpu_num', '-g', type=int, default=1, help='Num. of GPUs') 17 | parser.add_argument('--lam', '-l', type=int, default=2, help='Num. of offsprings') 18 | parser.add_argument('--net_info_file', default='network_info.pickle', help='Network information file name') 19 | parser.add_argument('--log_file', default='./log_cgp.txt', help='Log file name') 20 | parser.add_argument('--mode', '-m', default='evolution', help='Mode (evolution / retrain / reevolution)') 21 | parser.add_argument('--init', '-i', action='store_true') 22 | args = parser.parse_args() 23 | 24 | # --- Optimization of the CNN architecture --- 25 | if args.mode == 'evolution': 26 | # Create CGP configuration and save network information 27 | network_info = CgpInfoConvSet(rows=3, cols=20, level_back=5, min_active_num=1, max_active_num=30) 28 | with open(args.net_info_file, mode='wb') as f: 29 | pickle.dump(network_info, f) 30 | # Evaluation function for CGP (training CNN and return validation accuracy) 31 | imgSize = 64 32 | eval_f = CNNEvaluation(gpu_num=args.gpu_num, dataset='bsds_gray', verbose=True, epoch_num=20, batchsize=16, imgSize=imgSize) 33 | 34 | # Execute evolution 35 | cgp = CGP(network_info, eval_f, lam=args.lam, imgSize=imgSize, init=args.init) 36 | cgp.modified_evolution(max_eval=250, mutation_rate=0.1, log_file=args.log_file) 37 | 38 | # --- Retraining evolved architecture --- 39 | elif args.mode == 'retrain': 40 | print('Retrain') 41 | # In the case of existing log_cgp.txt 42 | # Load CGP configuration 43 | with open(args.net_info_file, mode='rb') as f: 44 | network_info = pickle.load(f) 45 | # Load network architecture 46 | cgp = CGP(network_info, None) 47 | data = pd.read_csv(args.log_file, header=None) # Load log file 48 | cgp.load_log(list(data.tail(1).values.flatten().astype(int))) # Read the log at final generation 49 | print(cgp._log_data(net_info_type='active_only', start_time=0)) 50 | # Retraining the network 51 | temp = CNN_train('bsds_gray', validation=False, verbose=True) 52 | acc = temp(cgp.pop[0].active_net_list(), 0, epoch_num=500, out_model='retrained_net.model') 53 | print(acc) 54 | 55 | # # otherwise (in the case where we do not have a log file.) 56 | # temp = CNN_train('haze1', validation=False, verbose=True, imgSize=128, batchsize=16) 57 | # cgp = [['input', 0], ['S_SumConvBlock_64_3', 0], ['S_ConvBlock_64_5', 1], ['S_SumConvBlock_128_1', 2], ['S_SumConvBlock_64_1', 3], ['S_SumConvBlock_64_5', 4], ['S_DeConvBlock_3_3', 5]] 58 | # acc = temp(cgp, 0, epoch_num=500, out_model='retrained_net.model') 59 | 60 | elif args.mode == 'reevolution': 61 | # restart evolution 62 | print('Restart Evolution') 63 | imgSize = 64 64 | with open('network_info.pickle', mode='rb') as f: 65 | network_info = pickle.load(f) 66 | eval_f = CNNEvaluation(gpu_num=args.gpu_num, dataset='bsds_gray', verbose=True, epoch_num=20, batchsize=16, imgSize=imgSize) 67 | cgp = CGP(network_info, eval_f, lam=args.lam, imgSize=imgSize) 68 | 69 | data = pd.read_csv('./log_cgp.txt', header=None) 70 | cgp.load_log(list(data.tail(1).values.flatten().astype(int))) 71 | cgp.modified_evolution(max_eval=250, mutation_rate=0.1, log_file='./log_restat.txt') 72 | 73 | else: 74 | print('Undefined mode. Please check the "-m evolution or retrain or reevolution" ') 75 | -------------------------------------------------------------------------------- /Denoising/models/model.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Denoising/models/model.pth -------------------------------------------------------------------------------- /Denoising/models/model_arch.txt: -------------------------------------------------------------------------------- 1 | cgp = [['input', 0], ['S_SumConvBlock_64_3', 0], ['S_ConvBlock_64_1', 1], ['S_ConvBlock_128_3', 2], ['S_SumConvBlock_64_1', 3], ['S_SumConvBlock_128_5', 4], ['S_ConvBlock_128_3', 5], ['S_ConvBlock_64_1', 6], ['S_DeConvBlock_1_3', 7]] 2 | 3 | # how to load the pre-trained model 4 | model = CGP2CNN_autoencoder(cgp, self.channel, self.n_class, self.imgSize) 5 | model.cuda(gpuID) 6 | param = torch.load('./models/model.pth') 7 | model.load_state_dict(param) 8 | -------------------------------------------------------------------------------- /Inpainting/cgp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Original code from https://github.com/sg-nm/cgp-cnn 5 | 6 | import csv 7 | import time 8 | import numpy as np 9 | import math 10 | 11 | # gene[f][c] f:function type, c:connection (nodeID) 12 | class Individual(object): 13 | 14 | def __init__(self, net_info, init): 15 | self.net_info = net_info 16 | self.gene = np.zeros((self.net_info.node_num + self.net_info.out_num, self.net_info.max_in_num + 1)).astype(int) 17 | self.is_active = np.empty(self.net_info.node_num + self.net_info.out_num).astype(bool) 18 | self.is_pool = np.empty(self.net_info.node_num + self.net_info.out_num).astype(bool) 19 | self.eval = None 20 | if init: 21 | print('init with specific architectures') 22 | self.init_gene_with_conv() # In the case of starting only convolution 23 | else: 24 | self.init_gene() # generate initial individual randomly 25 | 26 | def init_gene_with_conv(self): 27 | arch = ['D_ConvBlock_64_3'] 28 | 29 | input_layer_num = int(self.net_info.input_num / self.net_info.rows) + 1 30 | output_layer_num = int(self.net_info.out_num / self.net_info.rows) + 1 31 | layer_ids = [((self.net_info.cols - 1 - input_layer_num - output_layer_num) + i) // (len(arch)) for i in range(len(arch))] 32 | prev_id = 0 # i.e. input layer 33 | current_layer = input_layer_num 34 | block_ids = [] # *do not connect with these ids 35 | 36 | # building convolution net 37 | for i, idx in enumerate(layer_ids): 38 | current_layer += idx 39 | n = current_layer * self.net_info.rows + np.random.randint(self.net_info.rows) 40 | block_ids.append(n) 41 | self.gene[n][0] = self.net_info.func_type.index(arch[i]) 42 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 43 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 44 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 45 | if col - self.net_info.level_back >= 0 else 0 46 | 47 | self.gene[n][1] = prev_id 48 | for j in range(1, self.net_info.max_in_num): 49 | self.gene[n][j + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 50 | 51 | prev_id = n + self.net_info.input_num 52 | 53 | # output layer 54 | n = self.net_info.node_num 55 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 56 | self.gene[n][0] = np.random.randint(type_num) 57 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 58 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 59 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 60 | if col - self.net_info.level_back >= 0 else 0 61 | 62 | self.gene[n][1] = prev_id 63 | for i in range(1, self.net_info.max_in_num): 64 | self.gene[n][i + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 65 | block_ids.append(n) 66 | 67 | # intermediate node 68 | for n in range(self.net_info.node_num + self.net_info.out_num): 69 | 70 | if n in block_ids: 71 | continue 72 | # type gene 73 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 74 | self.gene[n][0] = np.random.randint(type_num) 75 | # connection gene 76 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 77 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 78 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 79 | if col - self.net_info.level_back >= 0 else 0 80 | for i in range(self.net_info.max_in_num): 81 | self.gene[n][i + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 82 | 83 | self.check_active() 84 | 85 | def init_gene(self): 86 | # intermediate node 87 | for n in range(self.net_info.node_num + self.net_info.out_num): 88 | # type gene 89 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 90 | self.gene[n][0] = np.random.randint(type_num) 91 | # connection gene 92 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 93 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 94 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 95 | if col - self.net_info.level_back >= 0 else 0 96 | for i in range(self.net_info.max_in_num): 97 | self.gene[n][i + 1] = min_connect_id + np.random.randint(max_connect_id - min_connect_id) 98 | 99 | self.check_active() 100 | 101 | def __check_course_to_out(self, n): 102 | if not self.is_active[n]: 103 | self.is_active[n] = True 104 | t = self.gene[n][0] 105 | if n >= self.net_info.node_num: # output node 106 | in_num = self.net_info.out_in_num[t] 107 | else: # intermediate node 108 | in_num = self.net_info.func_in_num[t] 109 | 110 | for i in range(in_num): 111 | if self.gene[n][i+1] >= self.net_info.input_num: 112 | self.__check_course_to_out(self.gene[n][i+1] - self.net_info.input_num) 113 | 114 | def check_active(self): 115 | self.is_active[:] = False 116 | # start from output nodes 117 | for n in range(self.net_info.out_num): 118 | self.__check_course_to_out(self.net_info.node_num + n) 119 | 120 | def check_pool(self): 121 | is_pool = True 122 | pool_num = 0 123 | for n in range(self.net_info.node_num + self.net_info.out_num): 124 | if self.is_active[n]: 125 | if self.gene[n][0] > 9: 126 | is_pool = False 127 | pool_num += 1 128 | return is_pool, pool_num 129 | 130 | 131 | def __mutate(self, current, min_int, max_int): 132 | mutated_gene = current 133 | while current == mutated_gene: 134 | mutated_gene = min_int + np.random.randint(max_int - min_int) 135 | return mutated_gene 136 | 137 | def mutation(self, mutation_rate=0.01): 138 | active_check = False 139 | 140 | for n in range(self.net_info.node_num + self.net_info.out_num): 141 | t = self.gene[n][0] 142 | # mutation for type gene 143 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 144 | if np.random.rand() < mutation_rate and type_num > 1: 145 | self.gene[n][0] = self.__mutate(self.gene[n][0], 0, type_num) 146 | if self.is_active[n]: 147 | active_check = True 148 | # mutation for connection gene 149 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 150 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 151 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 152 | if col - self.net_info.level_back >= 0 else 0 153 | in_num = self.net_info.func_in_num[t] if n < self.net_info.node_num else self.net_info.out_in_num[t] 154 | for i in range(self.net_info.max_in_num): 155 | if np.random.rand() < mutation_rate and max_connect_id - min_connect_id > 1: 156 | self.gene[n][i+1] = self.__mutate(self.gene[n][i+1], min_connect_id, max_connect_id) 157 | if self.is_active[n] and i < in_num: 158 | active_check = True 159 | 160 | self.check_active() 161 | return active_check 162 | 163 | def neutral_mutation(self, mutation_rate=0.01): 164 | for n in range(self.net_info.node_num + self.net_info.out_num): 165 | t = self.gene[n][0] 166 | # mutation for type gene 167 | type_num = self.net_info.func_type_num if n < self.net_info.node_num else self.net_info.out_type_num 168 | if not self.is_active[n] and np.random.rand() < mutation_rate and type_num > 1: 169 | self.gene[n][0] = self.__mutate(self.gene[n][0], 0, type_num) 170 | # mutation for connection gene 171 | col = np.min((int(n / self.net_info.rows), self.net_info.cols)) 172 | max_connect_id = col * self.net_info.rows + self.net_info.input_num 173 | min_connect_id = (col - self.net_info.level_back) * self.net_info.rows + self.net_info.input_num \ 174 | if col - self.net_info.level_back >= 0 else 0 175 | in_num = self.net_info.func_in_num[t] if n < self.net_info.node_num else self.net_info.out_in_num[t] 176 | for i in range(self.net_info.max_in_num): 177 | if (not self.is_active[n] or i >= in_num) and np.random.rand() < mutation_rate \ 178 | and max_connect_id - min_connect_id > 1: 179 | self.gene[n][i+1] = self.__mutate(self.gene[n][i+1], min_connect_id, max_connect_id) 180 | 181 | self.check_active() 182 | return False 183 | 184 | def count_active_node(self): 185 | return self.is_active.sum() 186 | 187 | def copy(self, source): 188 | self.net_info = source.net_info 189 | self.gene = source.gene.copy() 190 | self.is_active = source.is_active.copy() 191 | self.eval = source.eval 192 | 193 | def active_net_list(self): 194 | net_list = [["input", 0]] 195 | active_cnt = np.arange(self.net_info.input_num + self.net_info.node_num + self.net_info.out_num) 196 | active_cnt[self.net_info.input_num:] = np.cumsum(self.is_active) 197 | 198 | for n, is_a in enumerate(self.is_active): 199 | if is_a: 200 | t = self.gene[n][0] 201 | if n < self.net_info.node_num: # intermediate node 202 | type_str = self.net_info.func_type[t] 203 | else: # output node 204 | type_str = self.net_info.out_type[t] 205 | 206 | connections = [active_cnt[self.gene[n][i+1]] for i in range(self.net_info.max_in_num)] 207 | net_list.append([type_str] + connections) 208 | return net_list 209 | 210 | 211 | # CGP with (1 + \lambda)-ES 212 | class CGP(object): 213 | 214 | def __init__(self, net_info, eval_func, lam=4, imgSize=64, init=False): 215 | self.lam = lam 216 | self.pop = [Individual(net_info, init) for _ in range(1 + self.lam)] 217 | self.eval_func = eval_func 218 | 219 | self.num_gen = 0 220 | self.num_eval = 0 221 | self.max_pool_num = int(math.log2(imgSize) - 1) 222 | print('max down-sampling num : ', self.max_pool_num) 223 | self.init = init 224 | 225 | def _evaluation(self, pop, eval_flag): 226 | # create network list 227 | net_lists = [] 228 | active_index = np.where(eval_flag)[0] 229 | for i in active_index: 230 | net_lists.append(pop[i].active_net_list()) 231 | 232 | # evaluation 233 | fp = self.eval_func(net_lists) 234 | for i, j in enumerate(active_index): 235 | pop[j].eval = fp[i] 236 | evaluations = np.zeros(len(pop)) 237 | for i in range(len(pop)): 238 | evaluations[i] = pop[i].eval 239 | 240 | self.num_eval += len(net_lists) 241 | return evaluations 242 | 243 | def _log_data(self, net_info_type='active_only', start_time=0): 244 | log_list = [self.num_gen, self.num_eval, time.time()-start_time, self.pop[0].eval, self.pop[0].count_active_node()] 245 | if net_info_type == 'active_only': 246 | log_list.append(self.pop[0].active_net_list()) 247 | elif net_info_type == 'full': 248 | log_list += self.pop[0].gene.flatten().tolist() 249 | else: 250 | pass 251 | return log_list 252 | 253 | def _log_data_children(self, net_info_type='active_only', start_time=0, pop=None): 254 | log_list = [self.num_gen, self.num_eval, time.time()-start_time, pop.eval, pop.count_active_node()] 255 | if net_info_type == 'active_only': 256 | log_list.append(pop.active_net_list()) 257 | elif net_info_type == 'full': 258 | log_list += pop.gene.flatten().tolist() 259 | else: 260 | pass 261 | return log_list 262 | 263 | def load_log(self, log_data): 264 | self.num_gen = log_data[0] 265 | self.num_eval = log_data[1] 266 | net_info = self.pop[0].net_info 267 | self.pop[0].eval = log_data[3] 268 | self.pop[0].gene = np.array(log_data[5:]).reshape((net_info.node_num + net_info.out_num, net_info.max_in_num + 1)) 269 | self.pop[0].check_active() 270 | 271 | # Modified CGP: 272 | # At each iteration: 273 | # - Generate lambda individuals in which at least one active node changes 274 | # - Mutate the best individual with neutral mutation (unchanging the active nodes) if the best individual is not updated. 275 | def modified_evolution(self, max_eval=100, mutation_rate=0.01, log_file='./log.txt'): 276 | with open('child.txt', 'w') as fw_c : 277 | writer_c = csv.writer(fw_c, lineterminator='\n') 278 | start_time = time.time() 279 | eval_flag = np.empty(self.lam) 280 | active_num = self.pop[0].count_active_node() 281 | _, pool_num= self.pop[0].check_pool() 282 | if self.init: 283 | pass 284 | else: # in the case of not using an init indiviudal 285 | while active_num < self.pop[0].net_info.min_active_num or pool_num > self.max_pool_num: 286 | self.pop[0].mutation(1.0) 287 | active_num = self.pop[0].count_active_node() 288 | _, pool_num= self.pop[0].check_pool() 289 | self._evaluation([self.pop[0]], np.array([True])) 290 | print(self._log_data(net_info_type='active_only', start_time=start_time)) 291 | 292 | while self.num_gen < max_eval: 293 | self.num_gen += 1 294 | # reproduction 295 | for i in range(self.lam): 296 | eval_flag[i] = False 297 | self.pop[i + 1].copy(self.pop[0]) # copy a parent 298 | active_num = self.pop[i + 1].count_active_node() 299 | _, pool_num= self.pop[i + 1].check_pool() 300 | # mutation (forced mutation) 301 | while not eval_flag[i] or active_num < self.pop[i + 1].net_info.min_active_num or pool_num > self.max_pool_num: 302 | self.pop[i + 1].copy(self.pop[0]) # copy a parent 303 | eval_flag[i] = self.pop[i + 1].mutation(mutation_rate) # mutation 304 | active_num = self.pop[i + 1].count_active_node() 305 | _, pool_num= self.pop[i + 1].check_pool() 306 | 307 | # evaluation and selection 308 | evaluations = self._evaluation(self.pop[1:], eval_flag=eval_flag) 309 | best_arg = evaluations.argmax() 310 | # save 311 | f = open('arch_child.txt', 'a') 312 | writer_f = csv.writer(f, lineterminator='\n') 313 | for c in range(1 + self.lam): 314 | writer_c.writerow(self._log_data_children(net_info_type='full', start_time=start_time, pop=self.pop[c])) 315 | writer_f.writerow(self._log_data_children(net_info_type='active_only', start_time=start_time, pop=self.pop[c])) 316 | f.close() 317 | # replace the parent by the best individual 318 | if evaluations[best_arg] > self.pop[0].eval: 319 | self.pop[0].copy(self.pop[best_arg + 1]) 320 | else: 321 | self.pop[0].neutral_mutation(mutation_rate) # modify the parent (neutral mutation) 322 | 323 | # display and save log 324 | print(self._log_data(net_info_type='active_only', start_time=start_time)) 325 | fw = open(log_file, 'a') 326 | writer = csv.writer(fw, lineterminator='\n') 327 | writer.writerow(self._log_data(net_info_type='full', start_time=start_time)) 328 | fa = open('arch.txt', 'a') 329 | writer_a = csv.writer(fa, lineterminator='\n') 330 | writer_a.writerow(self._log_data(net_info_type='active_only', start_time=start_time)) 331 | fw.close() 332 | fa.close() 333 | 334 | -------------------------------------------------------------------------------- /Inpainting/cgp_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import multiprocessing as mp 5 | import multiprocessing.pool 6 | import numpy as np 7 | import cnn_train as cnn 8 | 9 | 10 | # wrapper function for multiprocessing 11 | def arg_wrapper_mp(args): 12 | return args[0](*args[1:]) 13 | 14 | class NoDaemonProcess(mp.Process): 15 | # make 'daemon' attribute always return False 16 | def _get_daemon(self): 17 | return False 18 | def _set_daemon(self, value): 19 | pass 20 | daemon = property(_get_daemon, _set_daemon) 21 | 22 | # We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool 23 | # because the latter is only a wrapper function, not a proper class. 24 | class NoDaemonProcessPool(multiprocessing.pool.Pool): 25 | Process = NoDaemonProcess 26 | 27 | 28 | # Evaluation of CNNs 29 | def cnn_eval(net, gpu_id, epoch_num, dataset, verbose, imgSize, batchsize, mask): 30 | 31 | print('\tgpu_id:', gpu_id, ',', net) 32 | train = cnn.CNN_train(dataset, validation=True, verbose=verbose, imgSize=imgSize, batchsize=batchsize) 33 | evaluation = train(net, gpu_id, epoch_num=epoch_num, out_model=None, mask_type=mask) 34 | print('\tgpu_id:', gpu_id, ', eval:', evaluation) 35 | return evaluation 36 | 37 | 38 | class CNNEvaluation(object): 39 | def __init__(self, gpu_num, epoch_num=50, dataset='cifar10', verbose=True, imgSize=64, batchsize=16, mask='center'): 40 | self.gpu_num = gpu_num 41 | self.epoch_num = epoch_num 42 | self.dataset = dataset 43 | self.verbose = verbose 44 | self.imgSize = imgSize 45 | self.batchsize = batchsize 46 | self.mask = mask 47 | 48 | def __call__(self, net_lists): 49 | evaluations = np.zeros(len(net_lists)) 50 | for i in np.arange(0, len(net_lists), self.gpu_num): 51 | process_num = np.min((i + self.gpu_num, len(net_lists))) - i 52 | pool = NoDaemonProcessPool(process_num) 53 | arg_data = [(cnn_eval, net_lists[i+j], j, self.epoch_num, self.dataset, 54 | self.verbose, self.imgSize, self.batchsize, self.mask) for j in range(process_num)] 55 | evaluations[i:i+process_num] = pool.map(arg_wrapper_mp, arg_data) 56 | pool.terminate() 57 | 58 | return evaluations 59 | 60 | 61 | # network configurations 62 | class CgpInfoConvSet(object): 63 | def __init__(self, rows=30, cols=40, level_back=40, min_active_num=8, max_active_num=50): 64 | self.input_num = 1 65 | 66 | self.func_type = ['S_SumConvBlock_256_1', 'S_SumConvBlock_256_3','S_SumConvBlock_256_5', 67 | 'S_SumConvBlock_128_1', 'S_SumConvBlock_128_3','S_SumConvBlock_128_5', 68 | 'S_SumConvBlock_64_1', 'S_SumConvBlock_64_3', 'S_SumConvBlock_64_5', 69 | 'D_ConvBlock_256_1', 'D_ConvBlock_256_3', 'D_ConvBlock_256_5', 70 | 'D_ConvBlock_128_1', 'D_ConvBlock_128_3', 'D_ConvBlock_128_5', 71 | 'D_ConvBlock_64_1', 'D_ConvBlock_64_3', 'D_ConvBlock_64_5'] 72 | 73 | self.func_in_num = [1, 1, 1, 74 | 1, 1, 1, 75 | 1, 1, 1, 76 | 1, 1, 1, 77 | 1, 1, 1, 78 | 1, 1, 1] 79 | 80 | self.out_num = 1 81 | self.out_type = ['S_DeConvBlock_3_3'] 82 | self.out_in_num = [1] 83 | 84 | # CGP network configuration 85 | self.rows = rows 86 | self.cols = cols 87 | self.node_num = rows * cols 88 | self.level_back = level_back 89 | self.min_active_num = min_active_num 90 | self.max_active_num = max_active_num 91 | 92 | self.func_type_num = len(self.func_type) 93 | self.out_type_num = len(self.out_type) 94 | self.max_in_num = np.max([np.max(self.func_in_num), np.max(self.out_in_num)]) -------------------------------------------------------------------------------- /Inpainting/cnn_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | from torch.autograd import Variable 7 | from collections import OrderedDict 8 | import math 9 | import copy 10 | import numpy as np 11 | import torch.nn.functional as F 12 | 13 | 14 | class ConvBlock(nn.Module): 15 | def __init__(self, in_size, out_size, kernel, stride): 16 | super(ConvBlock, self).__init__() 17 | pad_size = kernel // 2 18 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, stride=stride, padding=pad_size, bias=False), 19 | # nn.BatchNorm2d(out_size), 20 | nn.ReLU(inplace=True),) 21 | 22 | def forward(self, inputs): 23 | outputs = self.conv1(inputs) 24 | return outputs 25 | 26 | class DeConvBlock(nn.Module): 27 | def __init__(self, in_size, out_size, kernel): 28 | super(DeConvBlock, self).__init__() 29 | pad_size = kernel // 2 30 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, stride=2, padding=pad_size, output_padding=1, bias=False), 31 | # nn.BatchNorm2d(out_size), 32 | nn.ReLU(inplace=True),) 33 | 34 | def forward(self, inputs): 35 | outputs = self.conv1(inputs) 36 | return outputs 37 | 38 | class ConvBlock_last(nn.Module): 39 | def __init__(self, in_size, out_size, kernel): 40 | super(ConvBlock_last, self).__init__() 41 | pad_size = kernel // 2 42 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, padding=pad_size, bias=False)) 43 | # nn.BatchNorm2d(out_size), 44 | # nn.Tanh()) 45 | 46 | def forward(self, inputs): 47 | outputs = self.conv1(inputs) 48 | return outputs 49 | 50 | class DeConvBlock_last(nn.Module): 51 | def __init__(self, in_size, out_size, kernel): 52 | super(DeConvBlock_last, self).__init__() 53 | pad_size = kernel // 2 54 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, padding=pad_size, bias=False)) 55 | # nn.BatchNorm2d(out_size), 56 | # nn.Tanh()) 57 | 58 | def forward(self, inputs): 59 | outputs = self.conv1(inputs) 60 | return outputs 61 | 62 | class ConvBlockTranspose(nn.Module): 63 | def __init__(self, in_size, out_size, kernel): 64 | super(ConvBlockTranspose, self).__init__() 65 | pad_size = kernel // 2 66 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, padding=pad_size, bias=False), 67 | # nn.BatchNorm2d(out_size), 68 | nn.ReLU(inplace=True),) 69 | 70 | def forward(self, inputs): 71 | outputs = self.conv1(inputs) 72 | return outputs 73 | 74 | class ConvBlock_cat(nn.Module): 75 | def __init__(self, in_size, out_size, kernel): 76 | super(ConvBlock_cat, self).__init__() 77 | pad_size = kernel // 2 78 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, padding=pad_size, bias=False), 79 | # nn.BatchNorm2d(out_size), 80 | nn.ReLU(inplace=True),) 81 | 82 | def forward(self, inputs): 83 | outputs = self.conv1(inputs) 84 | return outputs 85 | 86 | class ConvBlock_s(nn.Module): 87 | def __init__(self, in_size, out_size, kernel, stride): 88 | super(ConvBlock_s, self).__init__() 89 | pad_size = kernel // 2 90 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, stride=stride, padding=pad_size, bias=False), 91 | # nn.BatchNorm2d(out_size), 92 | nn.ReLU(inplace=True),) 93 | 94 | def forward(self, inputs): 95 | outputs = self.conv1(inputs) 96 | return outputs 97 | 98 | class ConvBlock_sum(nn.Module): 99 | def __init__(self, in_size, out_size, kernel): 100 | super(ConvBlock_sum, self).__init__() 101 | pad_size = kernel // 2 102 | self.conv1 = nn.Sequential(nn.Conv2d(in_size, out_size, kernel, padding=pad_size, bias=False), 103 | # nn.BatchNorm2d(out_size), 104 | nn.ReLU(inplace=True),) 105 | self.relu = nn.ReLU(inplace=True) 106 | 107 | def forward(self, inputs1, inputs2): 108 | outputs = self.conv1(inputs1) 109 | in_data = [outputs, inputs2] 110 | # check of the channel size 111 | small_ch_id, large_ch_id = (0, 1) if in_data[0].size(1) < in_data[1].size(1) else (1, 0) 112 | offset = int(in_data[large_ch_id].size()[1] - in_data[small_ch_id].size()[1]) 113 | if offset != 0: 114 | tmp = in_data[large_ch_id].data[:, :offset, :, :] 115 | tmp = Variable(tmp).clone() 116 | in_data[small_ch_id] = torch.cat([in_data[small_ch_id], tmp * 0], 1) 117 | out = torch.add(in_data[0], in_data[1]) 118 | return self.relu(out) 119 | 120 | class DeConvBlock_sum(nn.Module): 121 | def __init__(self, in_size, out_size, kernel): 122 | super(DeConvBlock_sum, self).__init__() 123 | pad_size = kernel // 2 124 | self.conv1 = nn.Sequential(nn.ConvTranspose2d(in_size, out_size, kernel, stride=2, padding=pad_size, output_padding=1, bias=False), 125 | # nn.BatchNorm2d(out_size), 126 | nn.ReLU(inplace=True),) 127 | self.relu = nn.ReLU(inplace=True) 128 | 129 | def forward(self, inputs1, inputs2): 130 | outputs1 = self.conv1(inputs1) 131 | offset = outputs1.size()[2] - inputs2.size()[2] 132 | padding = 2 * [offset // 2, offset // 2] 133 | outputs2 = F.pad(inputs2, padding) 134 | out = torch.add(outputs1, outputs2) 135 | return self.relu(out) 136 | 137 | 138 | 139 | 140 | class CGP2CNN_autoencoder(nn.Module): 141 | def __init__(self, cgp, in_channel, imgSize): 142 | super(CGP2CNN_autoencoder, self).__init__() 143 | self.cgp = cgp 144 | self.pool_size = 2 145 | self.arch = OrderedDict() 146 | self.encode = [] 147 | self.decode = [] 148 | self.channel_num = [None for _ in range(len(self.cgp))] 149 | self.size = [None for _ in range(len(self.cgp))] 150 | self.channel_num[0] = in_channel 151 | self.size[0] = imgSize 152 | # encoder 153 | i = 0 154 | for name, in1 in self.cgp: 155 | if name == 'input' or 'DeConv' in name: 156 | i += 1 157 | continue 158 | key = name.split('_') 159 | down = key[0] 160 | func = key[1] 161 | out_size = int(key[2]) 162 | kernel = int(key[3]) 163 | if down == 'S': 164 | self.channel_num[i] = out_size 165 | self.size[i] = self.size[in1] 166 | if func == 'ConvBlock': 167 | self.encode.append(ConvBlock(self.channel_num[in1], out_size, kernel, stride=1)) 168 | else: 169 | self.encode.append(ConvBlock_s(self.channel_num[in1], out_size, kernel, stride=1)) 170 | else: 171 | self.channel_num[i] = out_size 172 | self.size[i] = self.size[in1] 173 | if func == 'ConvBlock': 174 | self.encode.append(ConvBlock(self.channel_num[in1], out_size, kernel, stride=2)) 175 | else: 176 | self.encode.append(ConvBlock_s(self.channel_num[in1], out_size, kernel, stride=2)) 177 | i += 1 178 | 179 | # decoder 180 | self.channel_num_d = [None for _ in range(len(self.cgp))] 181 | i -= 2 # skip the last layer 182 | self.channel_num_d[0] = self.channel_num[i] 183 | self.channel_num_d[1] = self.channel_num[i] 184 | i = 0 185 | self.cgp_inverse = copy.deepcopy(self.cgp) 186 | self.cgp_inverse.reverse() 187 | for j in range(len(self.cgp_inverse)): 188 | self.cgp_inverse[j][1] = int(math.fabs(self.cgp_inverse[j][1]-(len(self.cgp_inverse)-3))) 189 | for j in range(len(self.cgp_inverse)-1): 190 | if j == 0: 191 | i += 1 192 | continue 193 | name = self.cgp_inverse[j][0] 194 | key = name.split('_') 195 | down = key[0] 196 | func = key[1] 197 | out_size = int(key[2]) 198 | kernel = int(key[3]) 199 | self.channel_num_d[i] = out_size 200 | if down == 'S': 201 | if func == 'ConvBlock': 202 | self.decode.append(ConvBlock(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel, stride=1)) 203 | else: 204 | self.decode.append(ConvBlock_sum(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel)) 205 | else: 206 | if func == 'ConvBlock': 207 | self.decode.append(DeConvBlock(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel)) 208 | else: 209 | self.decode.append(DeConvBlock_sum(self.channel_num_d[self.cgp_inverse[j][1]], out_size, kernel)) 210 | i += 1 211 | # the last layer 212 | for j in range(1): 213 | name = self.cgp_inverse[j][0] 214 | key = name.split('_') 215 | down = key[0] 216 | func = key[1] 217 | out_size = int(key[2]) 218 | kernel = int(key[3]) 219 | self.channel_num_d[-1] = out_size 220 | if down == 'S': 221 | if func == 'DeConvBlock': 222 | self.decode.append(DeConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 223 | else: 224 | self.decode.append(ConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 225 | else: 226 | if func == 'DeConvBlock': 227 | self.decode.append(DeConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 228 | else: 229 | self.decode.append(ConvBlock_last(self.channel_num_d[i-1], out_size, kernel)) 230 | 231 | self.network = self.encode + self.decode 232 | self.layer_module = nn.ModuleList(self.network) 233 | self.train = True 234 | self.loss = None 235 | self.accuracy = None 236 | self.outputs = [None for _ in range(len(self.cgp)-1)] 237 | self.outputs_d = [None for _ in range(len(self.cgp_inverse))] 238 | self.outputs_sum = [None for _ in range(len(self.cgp_inverse))] 239 | self.param_num = 0 240 | 241 | def main(self,x): 242 | out = x 243 | outputs = self.outputs 244 | outputs[0] = x # input image 245 | outputs_d = self.outputs_d 246 | outputs_sum = self.outputs_sum 247 | nodeID = 1 248 | sumID = 0 249 | decodeID = 1 250 | flag = True 251 | for layer in self.layer_module: 252 | # encoder 253 | if nodeID <= len(self.encode): 254 | if isinstance(layer, ConvBlock_s): 255 | outputs[nodeID] = layer(outputs[self.cgp[nodeID][1]]) 256 | outputs_sum[sumID] = outputs[nodeID] 257 | sumID += 1 258 | else: 259 | outputs[nodeID] = layer(outputs[self.cgp[nodeID][1]]) 260 | # decoder 261 | elif nodeID < (len(self.decode)+len(self.encode)): 262 | if flag: 263 | outputs_d[0] = outputs[nodeID-1] 264 | outputs_d[1] = outputs[nodeID-1] 265 | del outputs 266 | flag = False 267 | if isinstance(layer, ConvBlock_sum): 268 | sumID -= 1 269 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 270 | elif isinstance(layer, DeConvBlock_sum): 271 | sumID -= 1 272 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 273 | else: 274 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]]) 275 | else: 276 | if isinstance(layer, ConvBlock_sum): 277 | sumID -= 1 278 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 279 | elif isinstance(layer, DeConvBlock_sum): 280 | sumID -= 1 281 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]], outputs_sum[sumID]) 282 | else: 283 | outputs_d[decodeID] = layer(outputs_d[self.cgp_inverse[decodeID][1]]) 284 | decodeID += 1 285 | nodeID += 1 286 | # the last layer 287 | layer = self.layer_module[-1] 288 | out = layer(outputs_d[decodeID-1]) 289 | del outputs_d 290 | return out 291 | 292 | def forward(self, x): 293 | return self.main(x) 294 | -------------------------------------------------------------------------------- /Inpainting/cnn_train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import time 5 | import os 6 | import math 7 | import numpy as np 8 | import torch 9 | import torch.nn as nn 10 | from torch.nn import init 11 | import torch.nn.parallel 12 | import torch.backends.cudnn as cudnn 13 | import torch.optim as optim 14 | import torch.utils.data 15 | import torchvision.datasets as dset 16 | import torchvision.transforms as transforms 17 | import torchvision.utils as vutils 18 | from torch.autograd import Variable 19 | import random 20 | from skimage.measure import compare_psnr 21 | 22 | from cnn_model import CGP2CNN_autoencoder 23 | 24 | 25 | def weights_init(m): 26 | classname = m.__class__.__name__ 27 | if classname.find('Conv') != -1: 28 | m.weight.data.normal_(0.0, 0.02) 29 | elif classname.find('BatchNorm') != -1: 30 | m.weight.data.normal_(1.0, 0.02) 31 | m.bias.data.fill_(0) 32 | 33 | def weights_init_normal(m): 34 | classname = m.__class__.__name__ 35 | if classname.find('Conv2d') != -1: 36 | m.apply(weights_init_normal_) 37 | elif classname.find('Linear') != -1: 38 | init.uniform(m.weight.data, 0.0, 0.02) 39 | elif classname.find('BatchNorm2d') != -1: 40 | init.uniform(m.weight.data, 1.0, 0.02) 41 | init.constant(m.bias.data, 0.0) 42 | 43 | def weights_init_normal_(m): 44 | classname = m.__class__.__name__ 45 | if classname.find('Conv') != -1: 46 | init.uniform(m.weight.data, 0.0, 0.02) 47 | elif classname.find('Linear') != -1: 48 | init.uniform(m.weight.data, 0.0, 0.02) 49 | elif classname.find('BatchNorm2d') != -1: 50 | init.uniform(m.weight.data, 1.0, 0.02) 51 | init.constant(m.bias.data, 0.0) 52 | 53 | def weights_init_xavier(m): 54 | classname = m.__class__.__name__ 55 | if classname.find('Conv') != -1: 56 | init.xavier_normal(m.weight.data, gain=1) 57 | elif classname.find('Linear') != -1: 58 | init.xavier_normal(m.weight.data, gain=1) 59 | elif classname.find('BatchNorm2d') != -1: 60 | init.uniform(m.weight.data, 1.0, 0.02) 61 | init.constant(m.bias.data, 0.0) 62 | 63 | def weights_init_kaiming(m): 64 | classname = m.__class__.__name__ 65 | if classname.find('Conv') != -1: 66 | init.kaiming_normal(m.weight.data, a=0, mode='fan_in') 67 | elif classname.find('Linear') != -1: 68 | init.kaiming_normal(m.weight.data, a=0, mode='fan_in') 69 | elif classname.find('BatchNorm2d') != -1: 70 | init.uniform(m.weight.data, 1.0, 0.02) 71 | init.constant(m.bias.data, 0.0) 72 | 73 | def weights_init_orthogonal(m): 74 | classname = m.__class__.__name__ 75 | print(classname) 76 | if classname.find('Conv') != -1: 77 | init.orthogonal(m.weight.data, gain=1) 78 | elif classname.find('Linear') != -1: 79 | init.orthogonal(m.weight.data, gain=1) 80 | elif classname.find('BatchNorm2d') != -1: 81 | init.uniform(m.weight.data, 1.0, 0.02) 82 | init.constant(m.bias.data, 0.0) 83 | 84 | def init_weights(net, init_type='normal'): 85 | print('initialization method [%s]' % init_type) 86 | if init_type == 'normal': 87 | net.apply(weights_init_normal) 88 | elif init_type == 'xavier': 89 | net.apply(weights_init_xavier) 90 | elif init_type == 'kaiming': 91 | net.apply(weights_init_kaiming) 92 | elif init_type == 'orthogonal': 93 | net.apply(weights_init_orthogonal) 94 | else: 95 | raise NotImplementedError('initialization method [%s] is not implemented' % init_type) 96 | 97 | 98 | # __init__: load dataset 99 | # __call__: training the CNN defined by CGP list 100 | class CNN_train(): 101 | def __init__(self, dataset_name, validation=True, verbose=True, imgSize=64, batchsize=16): 102 | # dataset_name: name of data set ('celebA' or 'cars' or 'svhn') 103 | # validation : [True] model train/validation mode 104 | # [False] model test mode for final evaluation of the evolved model 105 | # verbose : flag of display 106 | self.verbose = verbose 107 | self.imgSize = imgSize 108 | self.validation = validation 109 | self.batchsize = batchsize 110 | self.channel = 3 111 | num_work = 2 112 | 113 | # load dataset 114 | if dataset_name == 'svhn' or dataset_name == 'celebA' or dataset_name == 'cars': 115 | if dataset_name == 'svhn': 116 | if self.validation: 117 | dataset = dset.SVHN(root='./svhn', split='train', download=True, 118 | transform=transforms.Compose([transforms.RandomHorizontalFlip(),transforms.Scale(self.imgSize),transforms.ToTensor(),])) 119 | test_dataset = dset.SVHN(root='./svhn', split='extra', download=True, 120 | transform=transforms.Compose([transforms.Scale(self.imgSize), transforms.ToTensor(),])) 121 | else: 122 | dataset = dset.SVHN(root='./svhn', split='train', download=True, 123 | transform=transforms.Compose([transforms.Scale(self.imgSize),transforms.ToTensor(),])) 124 | test_dataset = dset.SVHN(root='./svhn', split='test', download=True, 125 | transform=transforms.Compose([transforms.Scale(self.imgSize),transforms.ToTensor(),])) 126 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(num_work), drop_last=True) 127 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=int(num_work)) 128 | elif dataset_name == 'celebA': 129 | if self.validation: 130 | data_transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.ToTensor()]) 131 | test_data_transform = transforms.Compose([transforms.ToTensor()]) 132 | dataset = dset.ImageFolder(root='/dataset/celebA/train', transform=data_transform) 133 | test_dataset = dset.ImageFolder(root='/dataset/celebA/val', transform=test_data_transform) 134 | else: 135 | data_transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.ToTensor()]) 136 | test_data_transform = transforms.Compose([transforms.ToTensor()]) 137 | dataset = dset.ImageFolder(root='/dataset/celebA/train', transform=data_transform) 138 | test_dataset = dset.ImageFolder(root='/dataset/celebA/test', transform=test_data_transform) 139 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(num_work), drop_last=True) 140 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=int(num_work)) 141 | elif dataset_name == 'cars': 142 | if self.validation: 143 | data_transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.ToTensor()]) 144 | test_data_transform = transforms.Compose([transforms.ToTensor()]) 145 | dataset = dset.ImageFolder(root='/dataset/cars/train', transform=data_transform) 146 | test_dataset = dset.ImageFolder(root='/dataset/cars/val', transform=test_data_transform) 147 | else: 148 | data_transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.ToTensor()]) 149 | test_data_transform = transforms.Compose([transforms.ToTensor()]) 150 | dataset = dset.ImageFolder(root='/dataset/cars/retrain', transform=data_transform) 151 | test_dataset = dset.ImageFolder(root='/dataset/cars/test', transform=test_data_transform) 152 | self.dataloader = torch.utils.data.DataLoader(dataset, batch_size=self.batchsize, shuffle=True, num_workers=int(num_work), drop_last=True) 153 | self.test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=int(num_work)) 154 | print('train num', len(self.dataloader.dataset)) 155 | print('test num ', len(self.test_dataloader.dataset)) 156 | else: 157 | print('\tInvalid input dataset name at CNN_train()') 158 | exit(1) 159 | 160 | def __call__(self, cgp, gpuID, epoch_num=200, out_model='mymodel.model', mask_type='center'): 161 | if self.verbose: 162 | print('GPUID :', gpuID) 163 | print('epoch_num:', epoch_num) 164 | print('mast type:', mask_type) 165 | 166 | 167 | torch.backends.cudnn.benchmark = True 168 | model = CGP2CNN_autoencoder(cgp, self.channel, self.imgSize) 169 | model.cuda(gpuID) 170 | criterion = nn.MSELoss() 171 | criterion.cuda(gpuID) 172 | optimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.5, 0.999)) 173 | 174 | input = torch.FloatTensor(self.batchsize, self.channel, self.imgSize, self.imgSize) 175 | input = input.cuda(gpuID) 176 | # center mask 177 | mask = torch.FloatTensor(1, self.channel, self.imgSize, self.imgSize).fill_(1.0) 178 | scale = 0.25 179 | l = int(self.imgSize*scale) 180 | u = int(self.imgSize*(1.0-scale)) 181 | mask[:,0, l:u, l:u] = 0.0 182 | mask[:,1, l:u, l:u] = 0.0 183 | mask[:,2, l:u, l:u] = 0.0 184 | mask = mask.cuda(gpuID) 185 | mask = Variable(mask) 186 | # for outputs 187 | if not os.path.exists('./outputs'): 188 | os.mkdir('./outputs') 189 | 190 | for epoch in range(1, epoch_num+1): 191 | start_time = time.time() 192 | if self.verbose: 193 | print('epoch', epoch) 194 | train_loss = 0 195 | ite = 0 196 | for module in model.children(): 197 | module.train(True) 198 | for _, (data, target) in enumerate(self.dataloader): 199 | # data = data[:,0:1,:,:] # in the case of using gray-scale images 200 | data, target = data.cuda(gpuID), target.cuda(gpuID) 201 | input.resize_as_(data).copy_(data) 202 | input_ = Variable(input) 203 | if mask_type == 'center': 204 | data_noise = torch.mul(input_, mask) 205 | elif mask_type == 'pixel': 206 | data_noise = self.random_pixel_mask(input_, [self.imgSize, self.imgSize], gpuID) 207 | else: 208 | data_noise = self.half_mask(input_, gpuID) 209 | optimizer.zero_grad() 210 | try: 211 | output = model(data_noise) 212 | except: 213 | import traceback 214 | traceback.print_exc() 215 | return 0. 216 | loss = criterion(output, input_) 217 | train_loss += loss.data[0] 218 | loss.backward() 219 | optimizer.step() 220 | if ite == 100: 221 | vutils.save_image(data_noise.data, './noise_samples%d.png' % gpuID, normalize=False) 222 | vutils.save_image(input_.data, './org_samples%d.png' % gpuID, normalize=False) 223 | vutils.save_image(output.data, './output%d.png' % gpuID, normalize=False) 224 | ite += 1 225 | if ite == 1000: 226 | break 227 | print('Train set : Average loss: {:.4f}'.format(train_loss)) 228 | print('time ', time.time()-start_time) 229 | if self.validation: # for evolution 230 | if epoch == epoch_num: 231 | for module in model.children(): 232 | module.train(False) 233 | t_loss = self.__test_per_std(model, criterion, gpuID, input, mask_type, mask) 234 | else: # for retrain the best model 235 | if epoch % 10 == 0: 236 | for module in model.children(): 237 | module.train(False) 238 | t_loss = self.__test_per_std(model, criterion, gpuID, input, mask_type, mask) 239 | if epoch == 200: 240 | for param_group in optimizer.param_groups: 241 | tmp = param_group['lr'] 242 | tmp *= 0.1 243 | for param_group in optimizer.param_groups: 244 | param_group['lr'] = tmp 245 | if epoch == 400: 246 | for param_group in optimizer.param_groups: 247 | tmp = param_group['lr'] 248 | tmp *= 0.1 249 | for param_group in optimizer.param_groups: 250 | param_group['lr'] = tmp 251 | 252 | model = model.cpu() 253 | torch.save(model.state_dict(), './model_%d.pth' % int(gpuID)) 254 | return t_loss 255 | 256 | 257 | def random_pixel_mask(self, inp, image_shape, gpuID, fraction_masked=0.8): 258 | mask = torch.rand(image_shape) 259 | mask[mask255] = 255 287 | image1[image1<0] = 0 288 | image2[image2>255] = 255 289 | image2[image2<0] = 0 290 | return compare_psnr(image1, image2, data_range=255) 291 | 292 | 293 | def __test_per_std(self, model, criterion, gpuID, input, mask_type, mask): 294 | test_loss = 0 295 | psnr = 0 296 | psnr2 = 0 297 | ite = 0 298 | for _, (data, target) in enumerate(self.test_dataloader): 299 | # data = data[:,0:1,:,:] # in the case of gray-scale 300 | data, target = data.cuda(gpuID), target.cuda(gpuID) 301 | input.resize_as_(data).copy_(data) 302 | input_ = Variable(input, volatile=True) 303 | if mask_type == 'center': 304 | data_noise = torch.mul(input_, mask) 305 | elif mask_type == 'pixel': 306 | data_noise = self.random_pixel_mask(input_, [self.imgSize, self.imgSize], gpuID) 307 | else: 308 | data_noise = self.half_mask(input_, gpuID) 309 | try: 310 | output = model(data_noise) 311 | except: 312 | import traceback 313 | traceback.print_exc() 314 | return 0. 315 | loss = criterion(output, input_) 316 | psnr += -10 * math.log10(loss.data[0]) 317 | test_loss += loss.data[0] 318 | 319 | if ite < 2000: 320 | vutils.save_image(output.data, './outputs/test_output_%02d.png' % int(ite), normalize=False) 321 | vutils.save_image(data_noise.data, './outputs/test_output_%02d_.png' % int(ite), normalize=False) 322 | vutils.save_image(input_.data, './outputs/test_output_%02d__.png' % int(ite), normalize=False) 323 | ite += 1 324 | 325 | # # PSNR 326 | # img1 = (output.data).cpu().numpy() 327 | # img2 = (input_.data).cpu().numpy() 328 | # imdf = img2*255.0 - img1*255.0 329 | # imdf = imdf ** 2 330 | # rmse = np.sqrt(np.mean(imdf)) 331 | # psnr2 += 20 * math.log10(255.0/rmse) 332 | 333 | psnr /= (ite) 334 | psnr2 /= (ite) 335 | test_loss /= (ite) 336 | print('Test PSNR: {:.4f}'.format(psnr)) 337 | # print('Test PSNR2: {:.4f}'.format(psnr2)) 338 | print('Test loss: {:.4f}'.format(test_loss)) 339 | 340 | return psnr 341 | -------------------------------------------------------------------------------- /Inpainting/create_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | #!/usr/bin/env python 3 | 4 | import cv2 5 | import os 6 | import random 7 | import numpy as np 8 | 9 | 10 | p_size = 48 11 | data_dir = "./celebA_org/" 12 | out_dir1 = "./dataset/celebA/train/train/" 13 | out_dir2 = "./dataset/celebA/val/val/" 14 | out_dir3 = "./dataset/celebA/test/test/" 15 | if not os.path.exists(out_dir1): 16 | os.makedirs(out_dir1) 17 | if not os.path.exists(out_dir2): 18 | os.makedirs(out_dir2) 19 | if not os.path.exists(out_dir3): 20 | os.makedirs(out_dir3) 21 | 22 | fp = open('list_landmarks_align_celeba.txt', 'r') 23 | line = fp.readline().strip() 24 | line = fp.readline().strip() 25 | count = 0 26 | sumR = 0.0 27 | sumG = 0.0 28 | sumB = 0.0 29 | patchSize = 64 30 | while line != '': 31 | line = fp.readline().strip() 32 | key = line.split() 33 | x = int(key[5]) # nose_x 34 | y = int(key[6]) # nose_y 35 | x1 = int(key[1]) # Leye_x 36 | y1 = int(key[2]) # Leye_y 37 | x2 = int(key[3]) # Reye_x 38 | y2 = int(key[4]) # Reye_y 39 | center_eye_X = int((x2+x1)/2.0) 40 | center_eye_Y = int((y2+y1)/2.0) 41 | img = cv2.imread(data_dir+key[0],1) 42 | patch = img[center_eye_Y-p_size:center_eye_Y+p_size, center_eye_X-p_size:center_eye_X+p_size, :] 43 | out = cv2.resize(patch, (patchSize,patchSize), interpolation=cv2.INTER_CUBIC) 44 | if count < 100000: 45 | cv2.imwrite(out_dir1+str("{0:05d}".format(count))+'.png', out) 46 | elif count < 101000: 47 | cv2.imwrite(out_dir2+str("{0:05d}".format(count))+'.png', out) 48 | elif count < 103000: 49 | cv2.imwrite(out_dir3+str("{0:05d}".format(count))+'.png', out) 50 | count += 1 51 | 52 | -------------------------------------------------------------------------------- /Inpainting/example/example.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Inpainting/example/example.pdf -------------------------------------------------------------------------------- /Inpainting/example/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Inpainting/example/example.png -------------------------------------------------------------------------------- /Inpainting/exp_main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import pickle 6 | import pandas as pd 7 | 8 | from cgp import * 9 | from cgp_config import * 10 | from cnn_train import CNN_train 11 | 12 | 13 | if __name__ == '__main__': 14 | 15 | parser = argparse.ArgumentParser(description='Evolving CAE structures') 16 | parser.add_argument('--gpu_num', '-g', type=int, default=1, help='Num. of GPUs') 17 | parser.add_argument('--lam', '-l', type=int, default=2, help='Num. of offsprings') 18 | parser.add_argument('--net_info_file', default='network_info.pickle', help='Network information file name') 19 | parser.add_argument('--log_file', default='./log_cgp.txt', help='Log file name') 20 | parser.add_argument('--mode', '-m', default='evolution', help='Mode (evolution / retrain / reevolution)') 21 | parser.add_argument('--init', '-i', action='store_true') 22 | parser.add_argument('--mask', '-mask', default='center', help='Mode (center / pixel / half)') 23 | args = parser.parse_args() 24 | 25 | # --- Optimization of the CNN architecture --- 26 | if args.mode == 'evolution': 27 | # Create CGP configuration and save network information 28 | network_info = CgpInfoConvSet(rows=3, cols=20, level_back=5, min_active_num=1, max_active_num=30) 29 | with open(args.net_info_file, mode='wb') as f: 30 | pickle.dump(network_info, f) 31 | # Evaluation function for CGP (training CNN and return validation accuracy) 32 | imgSize = 64 33 | eval_f = CNNEvaluation(gpu_num=args.gpu_num, epoch_num=20, dataset='celebA', verbose=True, imgSize=imgSize, batchsize=16, mask=args.mask) 34 | 35 | # evolution 36 | cgp = CGP(network_info, eval_f, lam=args.lam, imgSize=imgSize, init=args.init) 37 | cgp.modified_evolution(max_eval=250, mutation_rate=0.1, log_file=args.log_file) 38 | 39 | # --- Retraining evolved architecture --- 40 | elif args.mode == 'retrain': 41 | print('Retrain the model') 42 | # In the case of existing log_cgp.txt 43 | # Load CGP configuration 44 | with open(args.net_info_file, mode='rb') as f: 45 | network_info = pickle.load(f) 46 | 47 | # Load network architecture 48 | cgp = CGP(network_info, None) 49 | data = pd.read_csv(args.log_file, header=None) # Load log file 50 | cgp.load_log(list(data.tail(1).values.flatten().astype(int))) # Read the log at final generation 51 | print(cgp._log_data(net_info_type='active_only', start_time=0)) 52 | 53 | # Retraining the network 54 | temp = CNN_train('celebA', validation=False, verbose=True, imgSize=64, batchsize=16) 55 | acc = temp(cgp.pop[0].active_net_list(), 0, epoch_num=500, out_model='retrained_net.model', mask='center') 56 | print(acc) 57 | 58 | # # otherwise (in the case where we do not have a log file.) 59 | # temp = CNN_train('haze1', validation=False, verbose=True, imgSize=128, batchsize=16) 60 | # cgp = [['input', 0], ['S_SumConvBlock_64_3', 0], ['S_ConvBlock_64_5', 1], ['S_SumConvBlock_128_1', 2], ['S_SumConvBlock_64_1', 3], ['S_SumConvBlock_64_5', 4], ['S_DeConvBlock_3_3', 5]] 61 | # acc = temp(cgp, 0, epoch_num=500, out_model='retrained_net.model', mask_type='center') 62 | 63 | elif args.mode == 'reevolution': 64 | # restart 65 | print('Restart evolution') 66 | with open('network_info.pickle', mode='rb') as f: 67 | network_info = pickle.load(f) 68 | imgSize = 64 69 | eval_f = CNNEvaluation(gpu_num=args.gpu_num, dataset='celebA', verbose=True, epoch_num=20, imgSize=imgSize, batchsize=16, mask=args.mask) 70 | cgp = CGP(network_info, eval_f, lam=args.lam, imgSize=imgSize, init=args.init) 71 | 72 | data = pd.read_csv('./log_cgp.txt', header=None) 73 | cgp.load_log(list(data.tail(1).values.flatten().astype(int))) 74 | cgp.modified_evolution(max_eval=250, mutation_rate=0.1, log_file='./log_restat.txt') 75 | 76 | else: 77 | print('Undefined mode.') 78 | -------------------------------------------------------------------------------- /Inpainting/models/model_SVHN_center.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Inpainting/models/model_SVHN_center.pth -------------------------------------------------------------------------------- /Inpainting/models/model_arch_SVHN_center.txt: -------------------------------------------------------------------------------- 1 | cgp = [['input', 0], ['S_Sum_ConvBlock_128_5', 0], ['D_ConvBlock_128_5', 1], ['S_Sum_ConvBlock_64_5', 2], ['D_ConvBlock_64_5', 3], ['S_Sum_ConvBlock_128_3', 4], ['S_Sum_ConvBlock_128_3', 5], ['D_ConvBlock_64_3', 6], ['S_DeConvBlock_3_1', 7]] 2 | 3 | # how to load the pre-trained model 4 | model = CGP2CNN_autoencoder(cgp, self.channel, self.n_class, self.imgSize) 5 | model.cuda(gpuID) 6 | param = torch.load('./models/model_SVHN_center.pth') 7 | model.load_state_dict(param) 8 | -------------------------------------------------------------------------------- /Inpainting/models/model_arch_celebA_center.txt: -------------------------------------------------------------------------------- 1 | cgp = [['input', 0], ['S_ConvBlock_128_3', 0], ['D_ConvBlock_64_3', 1], ['S_ConvBlock_128_5', 2], ['D_ConvBlock_128_1', 3], ['S_ConvBlock_256_5', 4], ['D_ConvBlock_256_1', 5], ['S_ConvBlock_64_5', 6], ['S_DeConvBlock_3_1', 7]] 2 | 3 | # how to load the pre-trained model 4 | model = CGP2CNN_autoencoder(cgp, self.channel, self.n_class, self.imgSize) 5 | model.cuda(gpuID) 6 | param = torch.load('./models/model_celebA_center.pth') 7 | model.load_state_dict(param) 8 | -------------------------------------------------------------------------------- /Inpainting/models/model_celebA_center.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sg-nm/Evolutionary-Autoencoders/9d8cbcc5b741c1f0df18220cbffbe9682948ebeb/Inpainting/models/model_celebA_center.pth -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Masanori Suganuma 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Exploiting the Potential of Standard Convolutional Autoencoders for Image Restoration by Evolutionary Search 2 | 3 | This repository contains the code for the following paper: 4 | 5 | Masanori Suganuma, Mete Ozay, and Takayuki Okatani, "Exploiting the Potential of Standard Convolutional Autoencoders for Image Restoration by Evolutionary Search," International Conference on Machine Learning (ICML), 2018. [[PDF](http://proceedings.mlr.press/v80/suganuma18a.html)] [[arXiv](https://arxiv.org/abs/1803.00370)] 6 | 7 | If you find this work useful in your research, please cite: 8 | 9 | @inproceedings{suganumaICML2018, 10 | Author = {M. Suganuma and M. Ozay and T. Okatani}, 11 | Title = {Exploiting the Potential of Standard Convolutional Autoencoders for Image Restoration by Evolutionary Search}, 12 | Booktitle = {ICML}, 13 | Year = {2018} 14 | } 15 | 16 | 17 | Sample results on inpainting tasks: 18 | 19 | ![example](Inpainting/example/example.png "Sample inpainting results") 20 | 21 | Sample results on denoising tasks: 22 | 23 | ![example](Denoising/example/example.png "Sample denoising results") 24 | 25 | 26 | ## Requirement 27 | 28 | * Ubuntu 14.04 LTS 29 | * CUDA version 8.0 30 | * Python version 3.6.2 31 | * PyTorch version 0.2.0_4 32 | 33 | 34 | ## Usage 35 | 36 | ### Run the architecture search (denoising) 37 | 38 | ```shell 39 | python exp_main.py -i 40 | ``` 41 | 42 | ### Run the architecture search (inpainting) 43 | 44 | ```shell 45 | python exp_main.py -i -mask center 46 | ``` 47 | 48 | When you specify the `-i` option, an initial individual consists of a single convolution layer and a single deconvolution layer. 49 | To choose inpainting tasks, please specify the `-mask` option (center, pixel, half). 50 | 51 | 52 | When you use the multiple GPUs, please specify the `-g` option (default:1): 53 | 54 | ```shell 55 | python exp_main.py -g 2 56 | ``` 57 | 58 | You can set the number of offsprings with the `-l` option (default:2): 59 | 60 | ```shell 61 | python exp_main.py -l 4 62 | ``` 63 | 64 | 65 | After the execution, the files, `network_info.pickle` and `log_cgp.txt` will be generated. The file `network_info.pickle` contains the information for Cartegian genetic programming (CGP) and `log_cgp.txt` contains the log of the optimization and discovered CAE architecture's genotype lists. 66 | 67 | Some parameters (e.g., # rows and columns of CGP, and # epochs) can easily change by modifying the arguments in the script `exp_main.py`. 68 | 69 | 70 | ### Re-training (fine-tuning) 71 | 72 | To re-train the discovered architecture: 73 | 74 | ```shell 75 | python exp_main.py -m retrain 76 | ``` 77 | 78 | ### Re-evolution 79 | 80 | To re-start the evolution: 81 | 82 | ```shell 83 | python exp_main.py -m reevolution 84 | ``` 85 | 86 | 87 | ### Dataset 88 | 89 | The CelebA dataset is available [here](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html). 90 | 91 | The cars dataset is available [here](http://ai.stanford.edu/~jkrause/cars/car_dataset.html). 92 | 93 | The SVHN dataset is available [here](http://ufldl.stanford.edu/housenumbers/). 94 | 95 | The BSDS500 dataset is available [here](https://www2.eecs.berkeley.edu/Research/Projects/CS/vision/bsds/). 96 | 97 | To create the training, validation, and test sets of the CelebA dataset, please download "Align&Cropped Images" and save images to `./celebA_org`, and download "list_landmarks_landmarks_align_celeba.txt" from [here.](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) 98 | Then please run a following code: 99 | 100 | ```shell 101 | python create_data.py 102 | ``` 103 | 104 | 105 | 106 | --------------------------------------------------------------------------------