├── .gitattributes ├── Images ├── content_coast.jpg ├── content_evening_city.jpg ├── content_machu_picchu.jpg ├── content_peru_rocks.jpg ├── content_rocky_lake.jpg ├── content_tree.jpg ├── style_group7_moutains.jpg ├── style_line_architecture.jpg ├── style_monet_sunset.jpg ├── style_starry_night.jpg ├── style_vandrie_yellow_forest.jpg └── style_vangogh_tree.jpg ├── N_A_A_S.py └── vgg_conv_weights.pth /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /Images/content_coast.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/content_coast.jpg -------------------------------------------------------------------------------- /Images/content_evening_city.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/content_evening_city.jpg -------------------------------------------------------------------------------- /Images/content_machu_picchu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/content_machu_picchu.jpg -------------------------------------------------------------------------------- /Images/content_peru_rocks.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/content_peru_rocks.jpg -------------------------------------------------------------------------------- /Images/content_rocky_lake.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/content_rocky_lake.jpg -------------------------------------------------------------------------------- /Images/content_tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/content_tree.jpg -------------------------------------------------------------------------------- /Images/style_group7_moutains.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/style_group7_moutains.jpg -------------------------------------------------------------------------------- /Images/style_line_architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/style_line_architecture.jpg -------------------------------------------------------------------------------- /Images/style_monet_sunset.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/style_monet_sunset.jpg -------------------------------------------------------------------------------- /Images/style_starry_night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/style_starry_night.jpg -------------------------------------------------------------------------------- /Images/style_vandrie_yellow_forest.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/style_vandrie_yellow_forest.jpg -------------------------------------------------------------------------------- /Images/style_vangogh_tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/Images/style_vangogh_tree.jpg -------------------------------------------------------------------------------- /N_A_A_S.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Dec 27 08:33:31 2018 4 | 5 | @author: Faris 6 | """ 7 | 8 | #-----IMPORTS AND DIRECTORIES----- 9 | import time 10 | import os 11 | 12 | image_dir = 'PATH TO IMAGES' 13 | model_dir = 'PATH TO MODEL' 14 | 15 | import torch 16 | from torch.autograd import Variable 17 | import torch.nn as nn 18 | import torch.nn.functional as F 19 | from torch import optim 20 | 21 | import torchvision 22 | from torchvision import transforms 23 | 24 | from PIL import Image 25 | from collections import OrderedDict 26 | 27 | import matplotlib.pyplot as plt 28 | 29 | #------------------------------ 30 | #-----VGG MODEL DEFINITION----- 31 | #------------------------------ 32 | 33 | #CAN RETURN OUTPUT FROM ANY LAYER 34 | class VGG(nn.Module): 35 | def __init__(self, pool='max'): 36 | super(VGG, self).__init__() 37 | #CONV LAYERS 38 | self.conv1_1 = nn.Conv2d(3, 64, kernel_size = 3, padding = 1) 39 | self.conv1_2 = nn.Conv2d(64, 64, kernel_size = 3, padding = 1) 40 | 41 | self.conv2_1 = nn.Conv2d(64, 128, kernel_size = 3, padding = 1) 42 | self.conv2_2 = nn.Conv2d(128, 128, kernel_size = 3, padding = 1) 43 | 44 | self.conv3_1 = nn.Conv2d(128, 256, kernel_size = 3, padding = 1) 45 | self.conv3_2 = nn.Conv2d(256, 256, kernel_size = 3, padding = 1) 46 | self.conv3_3 = nn.Conv2d(256, 256, kernel_size = 3, padding = 1) 47 | self.conv3_4 = nn.Conv2d(256, 256, kernel_size = 3, padding = 1) 48 | 49 | self.conv4_1 = nn.Conv2d(256, 512, kernel_size = 3, padding = 1) 50 | self.conv4_2 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 51 | self.conv4_3 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 52 | self.conv4_4 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 53 | 54 | self.conv5_1 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 55 | self.conv5_2 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 56 | self.conv5_3 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 57 | self.conv5_4 = nn.Conv2d(512, 512, kernel_size = 3, padding = 1) 58 | 59 | #HANDLE POOLING OPTIONS 60 | #MAX POOLING 61 | if pool == 'max': 62 | self.pool1 = nn.MaxPool2d(kernel_size = 2, stride = 2) 63 | self.pool2 = nn.MaxPool2d(kernel_size = 2, stride = 2) 64 | self.pool3 = nn.MaxPool2d(kernel_size = 2, stride = 2) 65 | self.pool4 = nn.MaxPool2d(kernel_size = 2, stride = 2) 66 | self.pool5 = nn.MaxPool2d(kernel_size = 2, stride = 2) 67 | #AVERAGE POOLING 68 | elif pool == 'avg': 69 | self.pool1 = nn.AvgPool2d(kernel_size = 2, stride = 2) 70 | self.pool2 = nn.AvgPool2d(kernel_size = 2, stride = 2) 71 | self.pool3 = nn.AvgPool2d(kernel_size = 2, stride = 2) 72 | self.pool4 = nn.AvgPool2d(kernel_size = 2, stride = 2) 73 | self.pool5 = nn.AvgPool2d(kernel_size = 2, stride = 2) 74 | 75 | #FORWARD PROP 76 | def forward(self, x, out_keys): 77 | out = {} 78 | 79 | out['r11'] = F.relu(self.conv1_1(x)) 80 | out['r12'] = F.relu(self.conv1_2(out['r11'])) 81 | out['p1'] = self.pool1(out['r12']) 82 | 83 | out['r21'] = F.relu(self.conv2_1(out['p1'])) 84 | out['r22'] = F.relu(self.conv2_2(out['r21'])) 85 | out['p2'] = self.pool2(out['r22']) 86 | 87 | out['r31'] = F.relu(self.conv3_1(out['p2'])) 88 | out['r32'] = F.relu(self.conv3_2(out['r31'])) 89 | out['r33'] = F.relu(self.conv3_3(out['r32'])) 90 | out['r34'] = F.relu(self.conv3_4(out['r33'])) 91 | out['p3'] = self.pool3(out['r34']) 92 | 93 | out['r41'] = F.relu(self.conv4_1(out['p3'])) 94 | out['r42'] = F.relu(self.conv4_2(out['r41'])) 95 | out['r43'] = F.relu(self.conv4_3(out['r42'])) 96 | out['r44'] = F.relu(self.conv4_4(out['r43'])) 97 | out['p4'] = self.pool4(out['r44']) 98 | 99 | out['r51'] = F.relu(self.conv5_1(out['p4'])) 100 | out['r52'] = F.relu(self.conv5_2(out['r51'])) 101 | out['r53'] = F.relu(self.conv5_3(out['r52'])) 102 | out['r54'] = F.relu(self.conv5_4(out['r53'])) 103 | out['p5'] = self.pool5(out['r54']) 104 | 105 | 106 | #RETURN DESIRED ACTIVATIONs 107 | return [out[key] for key in out_keys] 108 | 109 | #---------------------------------------------------- 110 | #-----COMPUTING GRAM MATRIX AND GRAM MATRIX LOSS-----.0 111 | #---------------------------------------------------- 112 | 113 | #GRAM MATRICES ARE USED TO MEASURE STYLE LOSS 114 | #MATRIX 115 | class GramMatrix(nn.Module): 116 | def forward(self, input): 117 | b, c, w, h = input.size() 118 | F = input.view(b, c, h * w) 119 | #COMPUTES GRAM MATRIX BY MULTIPLYING INPUT BY TRANPOSE OF ITSELF 120 | G = torch.bmm(F, F.transpose(1, 2)) 121 | G.div_(h*w) 122 | return G 123 | 124 | #LOSS 125 | class GramMSELoss(nn.Module): 126 | def forward(self, input, target): 127 | out = nn.MSELoss()(GramMatrix()(input), target) 128 | return out 129 | 130 | #-------------------------- 131 | #-----IMAGE PROCESSING----- 132 | #-------------------------- 133 | 134 | img_size = 512 135 | 136 | #PRE-PROCESSING 137 | prep = transforms.Compose([transforms.Scale(img_size), 138 | transforms.ToTensor(), 139 | transforms.Lambda(lambda x: x[torch.LongTensor([2, 1, 0])]), #CONVERT TO BGR FOR VGG NET 140 | transforms.Normalize(mean = [0.40760392, 0.45795686, 0.48501961], std = [1, 1, 1]), #SUBTRACT IMAGENET MEAN 141 | transforms.Lambda(lambda x: x.mul_(255)), #VGG WAS TRAINED WITH PIXEL VALUES 0-255 142 | ]) 143 | 144 | #POST PROCESSING A 145 | postpa = transforms.Compose([transforms.Lambda(lambda x: x.mul_(1./255)), 146 | transforms.Normalize(mean = [-0.40760392, -0.45795686, -0.48501961], std = [1, 1, 1]), 147 | transforms.Lambda(lambda x: x[torch.LongTensor([2,1,0])]), 148 | ]) 149 | 150 | #POST PROCESSING B 151 | postpb = transforms.Compose([transforms.ToPILImage()]) 152 | 153 | #POST PROCESSING FUNCTION INCORPORATES A AND B, AND CLIPS PIXEL VALUES WHICH ARE OUT OF RANGE 154 | def postp(tensor): 155 | t = postpa(tensor) 156 | t[t>1] = 1 157 | t[t<0] = 0 158 | img = postpb(t) 159 | return img 160 | 161 | #--------------------------- 162 | #-----PREPARING NETWORK----- 163 | #--------------------------- 164 | 165 | vgg = VGG() 166 | 167 | vgg.load_state_dict(torch.load(model_dir + 'vgg_conv_weights.pth')) 168 | for param in vgg.parameters(): 169 | param.requires_grad = False 170 | if torch.cuda.is_available(): 171 | vgg.cuda() 172 | 173 | #-----LOADING AND PREPARING IMAGES----- 174 | img_dirs = [image_dir, image_dir] 175 | 176 | #IMAGE LOADING ORDER: STYLE, CONTENT 177 | img_names = ['style_pointillism_kingdom.jpg', 'content_hongkong.jpg'] 178 | imgs = [Image.open(img_dirs[i] + name) for i, name in enumerate(img_names)] 179 | imgs_torch = [prep(img) for img in imgs] 180 | 181 | #HANDLE CUDA 182 | if torch.cuda.is_available(): 183 | imgs_torch = [Variable(img.unsqueeze(0)).cuda() for img in imgs_torch] 184 | else: 185 | imgs_torch = [Variable(img.unsqueeze(0)) for img in imgs_torch] 186 | style_img, content_img = imgs_torch 187 | for img in imgs_torch: 188 | print("Image size: ", img.size()) 189 | 190 | #SET UP IMAGE TO BE OPTIMIZED 191 | #CAN BE INITIALIZED RANDOMLY OR AS A CLONE OF CONTENT IMAGE, AS DONE BELOW 192 | opt_img = Variable(content_img.clone(), requires_grad = True) 193 | print(content_img.size()) 194 | print(opt_img.size()) 195 | 196 | #DISPLAY IMAGES 197 | for img in imgs: 198 | plt.grid(None) 199 | plt.imshow(img) 200 | plt.show() 201 | 202 | #---------------------------- 203 | #-----SETUP FOR TRAINING----- 204 | #---------------------------- 205 | #LAYERS FOR STYLE AND CONTENT LOSS 206 | style_layers = ['r11', 'r12', 'r31', 'r41', 'r51'] 207 | content_layers = ['r42'] 208 | loss_layers = style_layers + content_layers 209 | 210 | #CREATING LOSS FUNCTION 211 | loss_fns = [GramMSELoss()] * len(style_layers) + [nn.MSELoss()] * len(content_layers) 212 | if torch.cuda.is_available(): 213 | loss_fns = [loss_fn.cuda() for loss_fn in loss_fns] 214 | 215 | #SETUP WEIGHTS FOR LOSS LAYERS 216 | style_weights = [1e3/n**2 for n in [64, 128, 256, 512, 512]] 217 | content_weights = [1e0] 218 | weights = style_weights + content_weights 219 | 220 | #CREATE OPTIMIZATION TARGETS 221 | style_targets = [GramMatrix()(A).detach() for A in vgg(style_img, style_layers)] 222 | content_targets = [A.detach() for A in vgg(content_img, content_layers)] 223 | targets = style_targets + content_targets 224 | 225 | #----------------------- 226 | #-----TRAINING LOOP----- 227 | #----------------------- 228 | max_iter = 500 229 | show_iter = 50 230 | optimizer = optim.LBFGS([opt_img]) 231 | print(opt_img.size()) 232 | print(content_img.size()) 233 | n_iter = [0] 234 | 235 | #ENTER LOOP 236 | while n_iter[0] <= max_iter: 237 | 238 | def closure(): 239 | optimizer.zero_grad() 240 | 241 | #FORWARD 242 | out = vgg(opt_img, loss_layers) 243 | 244 | #LOSS 245 | layer_losses = [weights[a] * loss_fns[a](A, targets[a]) for a,A in enumerate(out)] 246 | loss = sum(layer_losses) 247 | 248 | #BACKWARDS 249 | loss.backward() 250 | 251 | #TRACK PROGRESS 252 | n_iter[0] += 1 253 | if n_iter[0] % show_iter == (show_iter - 1): 254 | print('Iteration: %d,\tLoss: %f' % (n_iter[0] + 1, loss.data[0])) 255 | 256 | return loss 257 | 258 | optimizer.step(closure) 259 | 260 | #----------------- 261 | #-----RESULTS----- 262 | #----------------- 263 | print(float(opt_img.size(3))) 264 | print(float(content_img.size(3))) 265 | out_img = postp(opt_img.data[0].cpu().squeeze()) 266 | print(float(prep(out_img).size(2))) 267 | plt.grid(None) 268 | plt.imshow(out_img) 269 | plt.gcf().set_size_inches(10, 10) 270 | 271 | 272 | #<--------------------------------------------------------------------------------> 273 | #<--------------------------------------------------------------------------------> 274 | #<-------------------SECTION 2: PRODUCING HIGH RESOLUTION OUTPUT-------------------> 275 | #<--------------------------------------------------------------------------------> 276 | #<--------------------------------------------------------------------------------> 277 | 278 | #--------------------------- 279 | #-----HR PRE-PROCESSING----- 280 | #--------------------------- 281 | img_size_hr = 800 #FOR 8GB GPU, CAN MAKE LARGER IF YOU HAVE MORE 282 | 283 | prep_hr = transforms.Compose([ 284 | transforms.Scale(img_size_hr), 285 | transforms.ToTensor(), 286 | transforms.Lambda(lambda x: x[torch.LongTensor([2, 1, 0])]), 287 | transforms.Normalize(mean=[0.40760392, 0.45795686, 0.48501961], std = [1, 1, 1]), 288 | transforms.Lambda(lambda x: x.mul_(255)) 289 | ]) 290 | 291 | #PREPARE CONTENT IMAGE 292 | content_img = postp(content_img.data[0].cpu().squeeze()) 293 | content_img = prep_hr(content_img) 294 | 295 | 296 | #IMAGES TORCH 297 | imgs_torch = [prep_hr(imgs[0]), content_img] 298 | 299 | if torch.cuda.is_available(): 300 | imgs_torch = [Variable(img.unsqueeze(0).cuda()) for img in imgs_torch] 301 | else: 302 | imgs_torch = [Variable(img.unsqueeze(0)) for img in imgs_torch] 303 | 304 | style_img, content_img = imgs_torch 305 | 306 | #CHANGE OPTIMIZATION IMAGE TO UPSIZED LOW RES VERSION 307 | opt_img = prep_hr(out_img).unsqueeze(0) 308 | opt_img = Variable(opt_img.type_as(content_img.data), requires_grad = True) 309 | 310 | print(float(content_img.size(3)), float(opt_img.size(3))) 311 | #----------------------------------------- 312 | #-----PREPARE HR OPTIMIZATION TARGETS----- 313 | #----------------------------------------- 314 | style_targets = [GramMatrix()(A).detach() for A in vgg(style_img, style_layers)] 315 | content_targets = [A.detach() for A in vgg(content_img, content_layers)] 316 | targets = style_targets + content_targets 317 | 318 | #------------------------------------------------------------ 319 | #-----RUNNING STYLE TRANSFER FOR HIGH RESOLUTION OUTPUT------ 320 | #------------------------------------------------------------ 321 | max_iter_hr = 200 322 | optimizer = optim.LBFGS([opt_img]) 323 | n_iter = [0] 324 | 325 | while n_iter[0] <= max_iter_hr: 326 | 327 | def closure(): 328 | optimizer.zero_grad() 329 | out = vgg(opt_img, loss_layers) 330 | layer_losses = [weights[a] * loss_fns[a](A, targets[a]) for a,A in enumerate(out)] 331 | loss = sum(layer_losses) 332 | loss.backward() 333 | n_iter[0] += 1 334 | 335 | #DISPLAY LOSS 336 | if n_iter[0] % show_iter == (show_iter - 1): 337 | print('Iteration: %d,\tLoss: %f' % (n_iter[0] + 1, loss.data[0])) 338 | 339 | return loss 340 | 341 | optimizer.step(closure) 342 | 343 | #--------------------- 344 | #-----HR RESULTS------ 345 | #--------------------- 346 | out_img_hr = postp(opt_img.data[0].cpu().squeeze()) 347 | plt.grid(None) 348 | plt.imshow(out_img_hr) 349 | plt.gcf().set_size_inches(10, 10) 350 | 351 | -------------------------------------------------------------------------------- /vgg_conv_weights.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FarisNolan/Neural_Algorithm_Artistic_Style/e4871f5487c7094e113fe805e21fc0b48a621ec4/vgg_conv_weights.pth --------------------------------------------------------------------------------