├── 0_Dataset ├── Gaussian_Noise │ ├── Clean │ │ └── Peppers_clean.npz │ ├── Gaussian_Noise_2 │ │ ├── Peppers.png │ │ └── Peppers_Gaussian_Noise_2.npz │ └── Gaussian_Noise_4 │ │ ├── Peppers.png │ │ └── Peppers_Gaussian_Noise_4.npz ├── Inpainting │ └── 0_IP11 │ │ ├── .idea │ │ ├── .gitignore │ │ ├── 0_IP11.iml │ │ ├── inspectionProfiles │ │ │ ├── Project_Default.xml │ │ │ └── profiles_settings.xml │ │ ├── misc.xml │ │ └── modules.xml │ │ ├── IP_Dataset_0.1 │ │ ├── barbara_img_0.1.png │ │ ├── barbara_mask_0.1.png │ │ └── barbara_vis_0.1.png │ │ ├── Original_Images │ │ └── barbara.png │ │ └── generate_mask.py └── Super_Resolution │ └── Set5 │ └── baby.png ├── 1_Denoising ├── BN_Net.py ├── Main_Start.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── common.cpython-36.pyc │ │ ├── common.cpython-37.pyc │ │ ├── downsampler.cpython-36.pyc │ │ ├── downsampler.cpython-37.pyc │ │ ├── resnet.cpython-36.pyc │ │ ├── resnet.cpython-37.pyc │ │ ├── skip.cpython-36.pyc │ │ ├── skip.cpython-37.pyc │ │ ├── texture_nets.cpython-36.pyc │ │ ├── texture_nets.cpython-37.pyc │ │ ├── unet.cpython-36.pyc │ │ └── unet.cpython-37.pyc │ ├── common.py │ ├── dcgan.py │ ├── downsampler.py │ ├── resnet.py │ ├── skip.py │ ├── texture_nets.py │ └── unet.py ├── psnr_ssim.py ├── train_denoising.py ├── util.py └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── common_utils.cpython-36.pyc │ ├── common_utils.cpython-37.pyc │ ├── denoising_utils.cpython-36.pyc │ └── denoising_utils.cpython-37.pyc │ ├── common_utils.py │ ├── denoising_utils.py │ ├── feature_inversion_utils.py │ ├── inpainting_utils.py │ ├── matcher.py │ ├── perceptual_loss │ ├── __init__.py │ ├── matcher.py │ ├── perceptual_loss.py │ └── vgg_modified.py │ └── sr_utils.py ├── 2_Super_Resolution ├── BN_Net.py ├── Main_Start.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── common.cpython-36.pyc │ │ ├── common.cpython-37.pyc │ │ ├── downsampler.cpython-36.pyc │ │ ├── downsampler.cpython-37.pyc │ │ ├── resnet.cpython-36.pyc │ │ ├── resnet.cpython-37.pyc │ │ ├── skip.cpython-36.pyc │ │ ├── skip.cpython-37.pyc │ │ ├── texture_nets.cpython-36.pyc │ │ ├── texture_nets.cpython-37.pyc │ │ ├── unet.cpython-36.pyc │ │ └── unet.cpython-37.pyc │ ├── common.py │ ├── dcgan.py │ ├── downsampler.py │ ├── resnet.py │ ├── skip.py │ ├── texture_nets.py │ └── unet.py ├── psnr_ssim.py ├── train_sr.py ├── util.py └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── common_utils.cpython-36.pyc │ ├── common_utils.cpython-37.pyc │ ├── denoising_utils.cpython-36.pyc │ ├── denoising_utils.cpython-37.pyc │ └── sr_utils.cpython-36.pyc │ ├── common_utils.py │ ├── denoising_utils.py │ ├── feature_inversion_utils.py │ ├── inpainting_utils.py │ ├── matcher.py │ ├── perceptual_loss │ ├── __init__.py │ ├── matcher.py │ ├── perceptual_loss.py │ └── vgg_modified.py │ └── sr_utils.py ├── 3_Inpainting ├── BN_Net.py ├── Main_Start.py ├── include │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── compression.cpython-36.pyc │ │ ├── decoder.cpython-36.pyc │ │ ├── fit.cpython-36.pyc │ │ ├── helpers.cpython-36.pyc │ │ ├── visualize.cpython-36.pyc │ │ └── wavelet.cpython-36.pyc │ ├── compression.py │ ├── decoder.py │ ├── fit.py │ ├── helpers.py │ ├── visualize.py │ └── wavelet.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── common.cpython-36.pyc │ │ ├── common.cpython-37.pyc │ │ ├── downsampler.cpython-36.pyc │ │ ├── downsampler.cpython-37.pyc │ │ ├── resnet.cpython-36.pyc │ │ ├── resnet.cpython-37.pyc │ │ ├── skip.cpython-36.pyc │ │ ├── skip.cpython-37.pyc │ │ ├── texture_nets.cpython-36.pyc │ │ ├── texture_nets.cpython-37.pyc │ │ ├── unet.cpython-36.pyc │ │ └── unet.cpython-37.pyc │ ├── common.py │ ├── dcgan.py │ ├── downsampler.py │ ├── resnet.py │ ├── skip.py │ ├── texture_nets.py │ └── unet.py ├── psnr_ssim.py ├── train_ip.py ├── util.py └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── common_utils.cpython-36.pyc │ ├── common_utils.cpython-37.pyc │ ├── denoising_utils.cpython-36.pyc │ └── denoising_utils.cpython-37.pyc │ ├── common_utils.py │ ├── denoising_utils.py │ ├── feature_inversion_utils.py │ ├── inpainting_utils.py │ ├── matcher.py │ ├── perceptual_loss │ ├── __init__.py │ ├── matcher.py │ ├── perceptual_loss.py │ └── vgg_modified.py │ └── sr_utils.py ├── LICENSE ├── README.md └── environment.yml /0_Dataset/Gaussian_Noise/Clean/Peppers_clean.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Clean/Peppers_clean.npz -------------------------------------------------------------------------------- /0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers.png -------------------------------------------------------------------------------- /0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers_Gaussian_Noise_2.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers_Gaussian_Noise_2.npz -------------------------------------------------------------------------------- /0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers.png -------------------------------------------------------------------------------- /0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers_Gaussian_Noise_4.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers_Gaussian_Noise_4.npz -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/.idea/0_IP11.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_img_0.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_img_0.1.png -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_mask_0.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_mask_0.1.png -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_vis_0.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_vis_0.1.png -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/Original_Images/barbara.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/Original_Images/barbara.png -------------------------------------------------------------------------------- /0_Dataset/Inpainting/0_IP11/generate_mask.py: -------------------------------------------------------------------------------- 1 | ### Generate inpainting mask 2 | ### The mask is a binary matrix that randomly drops 50% pixels 3 | 4 | import numpy as np 5 | import skimage.io as ski 6 | from skimage.transform import resize 7 | import matplotlib.pyplot as plt 8 | 9 | 10 | 11 | 12 | def get_bernoulli_mask(image_shape, zero_fraction=0.5): 13 | img_mask_np = (np.random.random_sample(size=image_shape) > zero_fraction).astype(int) 14 | return img_mask_np 15 | 16 | 17 | 18 | if __name__ == '__main__': 19 | # a = ski.imread('IP_Dataset/barbara_mask.png') 20 | # b = ski.imread('IP_Dataset/barbara_img.png') 21 | # plt.imshow(a * b, cmap='gray') 22 | # plt.show() 23 | 24 | zero_fraction = 0.5 ### how many % of pixels we want to drop/mask 25 | image_list = ['barbara', 'boat', 'Cameraman256', 'couple', 26 | 'fingerprint', 'hill', 'house', 'Lena512', 27 | 'man', 'montage', 'peppers256'] 28 | 29 | for cur_img in image_list: 30 | cur_img_path = 'Original_Images/{}.png'.format(cur_img) 31 | cur_img_data = ski.imread(cur_img_path) 32 | h, w = cur_img_data.shape 33 | 34 | if h!=512: 35 | cur_img_data = resize(cur_img_data,(512,512)) 36 | 37 | # now, let's get is mask 38 | cur_mask_data = get_bernoulli_mask(cur_img_data.shape, zero_fraction=zero_fraction) 39 | 40 | # plt.figure() 41 | # plt.hist(cur_mask_data.flatten()) 42 | # plt.show() 43 | 44 | # now, let's save the clean image and its mask 45 | cur_IP_img_file = 'IP_Dataset_{}/{}_img_{}.png'.format(zero_fraction,cur_img,zero_fraction) 46 | ski.imsave(cur_IP_img_file, cur_img_data) 47 | 48 | cur_IP_mask_file = 'IP_Dataset_{}/{}_mask_{}.png'.format(zero_fraction,cur_img,zero_fraction) 49 | ski.imsave(cur_IP_mask_file,cur_mask_data) 50 | 51 | # visualize them 52 | plt.figure() 53 | plt.subplot(1,3,1) 54 | plt.imshow(cur_img_data, cmap='gray') 55 | plt.subplot(1,3,2) 56 | plt.imshow(cur_mask_data, cmap='gray') 57 | plt.subplot(1, 3, 3) 58 | plt.imshow(cur_img_data*cur_mask_data, cmap='gray') 59 | plt.savefig('IP_Dataset_{}/{}_vis_{}.png'.format(zero_fraction, cur_img, zero_fraction)) 60 | plt.close() 61 | 62 | 63 | -------------------------------------------------------------------------------- /0_Dataset/Super_Resolution/Set5/baby.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Super_Resolution/Set5/baby.png -------------------------------------------------------------------------------- /1_Denoising/BN_Net.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | #----------------------------------------------------------- 4 | #----------------------------------------------------------- 5 | # BN for the input seed 6 | #----------------------------------------------------------- 7 | #----------------------------------------------------------- 8 | class BNNet(nn.Module): 9 | def __init__(self,num_channel): 10 | super(BNNet, self).__init__() 11 | self.bn = nn.BatchNorm2d(num_channel) 12 | 13 | def forward(self, input_data): 14 | output_data = self.bn(input_data) 15 | return output_data 16 | -------------------------------------------------------------------------------- /1_Denoising/Main_Start.py: -------------------------------------------------------------------------------- 1 | from train_denoising import * 2 | import time 3 | import datetime 4 | 5 | #----------------------------------------------------------- 6 | #----------------------------------------------------------- 7 | # This is the Main of the code 8 | #----------------------------------------------------------- 9 | #----------------------------------------------------------- 10 | 11 | if __name__ == '__main__': 12 | # Set random seed for reproducibility 13 | manualSeed = 100 14 | manualSeed = random.randint(1, 10000) # use if you want new results 15 | print("Random Seed: ", manualSeed) 16 | random.seed(manualSeed) 17 | torch.manual_seed(manualSeed) 18 | 19 | # Set up hyper-parameters 20 | #----------------------------------------------------------- 21 | gpu = 0 22 | corr_level = 2 23 | 24 | corr_type = 'Gaussian_Noise' 25 | 26 | corr_type_level = corr_type + '_' + str(corr_level) 27 | print_step = 1 28 | max_epoch = 5000 29 | max_epoch = int(max_epoch / print_step) * print_step + 1 30 | 31 | tv_weight = 0.45 32 | learning_rate_model = 1e-1 33 | OPTIMIZER = 'Adam' 34 | LOSS = "MSE"# or 'Huber' or "L1" 35 | 36 | input_w = 512 37 | input_h = 512 38 | input_c = 3 39 | width = 64 40 | #----------------------------------------------------------- 41 | 42 | image_list = ['Peppers'] 43 | 44 | #image_list = ['Baboon', 'F16', 'House', 45 | #'kodim01', 'kodim02', 'kodim03', 46 | #'kodim12', 'Lena', 'Peppers'] 47 | ###################################### Processing images one by one ################################# 48 | for cur_image in image_list: 49 | 50 | clean_image_path = '../0_Dataset/{}/Clean/{}_clean.npz'.format(corr_type, cur_image) 51 | corrupted_image_path = '../0_Dataset/{}/{}/{}_{}.npz'.format(corr_type,corr_type_level,cur_image,corr_type_level) 52 | 53 | DIP_train(learning_rate_model, 54 | OPTIMIZER, 55 | LOSS, 56 | width, 57 | input_w, 58 | input_h, 59 | input_c, 60 | clean_image_path, 61 | corrupted_image_path, 62 | max_epoch, 63 | print_step, 64 | gpu, 65 | cur_image, 66 | tv_weight 67 | ) -------------------------------------------------------------------------------- /1_Denoising/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .skip import skip 2 | from .texture_nets import get_texture_nets 3 | from .resnet import ResNet 4 | from .unet import UNet 5 | 6 | import torch.nn as nn 7 | 8 | def get_net(input_depth, NET_TYPE, pad, upsample_mode, n_channels=3, act_fun='LeakyReLU', skip_n33d=128, skip_n33u=128, skip_n11=4, num_scales=5, downsample_mode='stride'): 9 | if NET_TYPE == 'ResNet': 10 | # TODO 11 | net = ResNet(input_depth, 3, 10, 16, 1, nn.BatchNorm2d, False) 12 | elif NET_TYPE == 'skip': 13 | net = skip(input_depth, n_channels, num_channels_down = [skip_n33d]*num_scales if isinstance(skip_n33d, int) else skip_n33d, 14 | num_channels_up = [skip_n33u]*num_scales if isinstance(skip_n33u, int) else skip_n33u, 15 | num_channels_skip = [skip_n11]*num_scales if isinstance(skip_n11, int) else skip_n11, 16 | upsample_mode=upsample_mode, downsample_mode=downsample_mode, 17 | need_sigmoid=True, need_bias=True, pad=pad, act_fun=act_fun) 18 | 19 | elif NET_TYPE == 'texture_nets': 20 | net = get_texture_nets(inp=input_depth, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False,pad=pad) 21 | 22 | elif NET_TYPE =='UNet': 23 | net = UNet(num_input_channels=input_depth, num_output_channels=3, 24 | feature_scale=4, more_layers=0, concat_x=False, 25 | upsample_mode=upsample_mode, pad=pad, norm_layer=nn.BatchNorm2d, need_sigmoid=True, need_bias=True) 26 | elif NET_TYPE == 'identity': 27 | assert input_depth == 3 28 | net = nn.Sequential() 29 | else: 30 | assert False 31 | 32 | return net -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/common.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/common.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/common.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/common.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/downsampler.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/downsampler.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/downsampler.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/downsampler.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/resnet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/resnet.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/skip.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/skip.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/skip.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/skip.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/texture_nets.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/texture_nets.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/texture_nets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/texture_nets.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/unet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/unet.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/models/__pycache__/unet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/unet.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/models/common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from .downsampler import Downsampler 5 | 6 | def add_module(self, module): 7 | self.add_module(str(len(self) + 1), module) 8 | 9 | torch.nn.Module.add = add_module 10 | 11 | class Concat(nn.Module): 12 | def __init__(self, dim, *args): 13 | super(Concat, self).__init__() 14 | self.dim = dim 15 | 16 | for idx, module in enumerate(args): 17 | self.add_module(str(idx), module) 18 | 19 | def forward(self, input): 20 | inputs = [] 21 | for module in self._modules.values(): 22 | inputs.append(module(input)) 23 | 24 | inputs_shapes2 = [x.shape[2] for x in inputs] 25 | inputs_shapes3 = [x.shape[3] for x in inputs] 26 | 27 | if np.all(np.array(inputs_shapes2) == min(inputs_shapes2)) and np.all(np.array(inputs_shapes3) == min(inputs_shapes3)): 28 | inputs_ = inputs 29 | else: 30 | target_shape2 = min(inputs_shapes2) 31 | target_shape3 = min(inputs_shapes3) 32 | 33 | inputs_ = [] 34 | for inp in inputs: 35 | diff2 = (inp.size(2) - target_shape2) // 2 36 | diff3 = (inp.size(3) - target_shape3) // 2 37 | inputs_.append(inp[:, :, diff2: diff2 + target_shape2, diff3:diff3 + target_shape3]) 38 | 39 | return torch.cat(inputs_, dim=self.dim) 40 | 41 | def __len__(self): 42 | return len(self._modules) 43 | 44 | 45 | class GenNoise(nn.Module): 46 | def __init__(self, dim2): 47 | super(GenNoise, self).__init__() 48 | self.dim2 = dim2 49 | 50 | def forward(self, input): 51 | a = list(input.size()) 52 | a[1] = self.dim2 53 | # print (input.data.type()) 54 | 55 | b = torch.zeros(a).type_as(input.data) 56 | b.normal_() 57 | 58 | x = torch.autograd.Variable(b) 59 | 60 | return x 61 | 62 | 63 | class Swish(nn.Module): 64 | """ 65 | https://arxiv.org/abs/1710.05941 66 | The hype was so huge that I could not help but try it 67 | """ 68 | def __init__(self): 69 | super(Swish, self).__init__() 70 | self.s = nn.Sigmoid() 71 | 72 | def forward(self, x): 73 | return x * self.s(x) 74 | 75 | 76 | def act(act_fun = 'LeakyReLU'): 77 | ''' 78 | Either string defining an activation function or module (e.g. nn.ReLU) 79 | ''' 80 | if isinstance(act_fun, str): 81 | if act_fun == 'LeakyReLU': 82 | return nn.LeakyReLU(0.2, inplace=True) 83 | elif act_fun == 'Swish': 84 | return Swish() 85 | elif act_fun == 'ELU': 86 | return nn.ELU() 87 | elif act_fun == 'none': 88 | return nn.Sequential() 89 | else: 90 | assert False 91 | else: 92 | return act_fun() 93 | 94 | 95 | def bn(num_features): 96 | return nn.BatchNorm2d(num_features) 97 | 98 | 99 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero', downsample_mode='stride'): 100 | downsampler = None 101 | if stride != 1 and downsample_mode != 'stride': 102 | 103 | if downsample_mode == 'avg': 104 | downsampler = nn.AvgPool2d(stride, stride) 105 | elif downsample_mode == 'max': 106 | downsampler = nn.MaxPool2d(stride, stride) 107 | elif downsample_mode in ['lanczos2', 'lanczos3']: 108 | downsampler = Downsampler(n_planes=out_f, factor=stride, kernel_type=downsample_mode, phase=0.5, preserve_size=True) 109 | else: 110 | assert False 111 | 112 | stride = 1 113 | 114 | padder = None 115 | to_pad = int((kernel_size - 1) / 2) 116 | if pad == 'reflection': 117 | padder = nn.ReflectionPad2d(to_pad) 118 | to_pad = 0 119 | 120 | convolver = nn.Conv2d(in_f, out_f, kernel_size, stride, padding=to_pad, bias=bias) 121 | 122 | 123 | layers = filter(lambda x: x is not None, [padder, convolver, downsampler]) 124 | return nn.Sequential(*layers) -------------------------------------------------------------------------------- /1_Denoising/models/dcgan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | def dcgan(inp=2, 5 | ndf=32, 6 | num_ups=4, need_sigmoid=True, need_bias=True, pad='zero', upsample_mode='nearest', need_convT = True): 7 | 8 | layers= [nn.ConvTranspose2d(inp, ndf, kernel_size=3, stride=1, padding=0, bias=False), 9 | nn.BatchNorm2d(ndf), 10 | nn.LeakyReLU(True)] 11 | 12 | for i in range(num_ups-3): 13 | if need_convT: 14 | layers += [ nn.ConvTranspose2d(ndf, ndf, kernel_size=4, stride=2, padding=1, bias=False), 15 | nn.BatchNorm2d(ndf), 16 | nn.LeakyReLU(True)] 17 | else: 18 | layers += [ nn.Upsample(scale_factor=2, mode=upsample_mode), 19 | nn.Conv2d(ndf, ndf, kernel_size=3, stride=1, padding=1, bias=False), 20 | nn.BatchNorm2d(ndf), 21 | nn.LeakyReLU(True)] 22 | 23 | if need_convT: 24 | layers += [nn.ConvTranspose2d(ndf, 3, 4, 2, 1, bias=False),] 25 | else: 26 | layers += [nn.Upsample(scale_factor=2, mode='bilinear'), 27 | nn.Conv2d(ndf, 3, kernel_size=3, stride=1, padding=1, bias=False)] 28 | 29 | 30 | if need_sigmoid: 31 | layers += [nn.Sigmoid()] 32 | 33 | model =nn.Sequential(*layers) 34 | return model -------------------------------------------------------------------------------- /1_Denoising/models/downsampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | class Downsampler(nn.Module): 6 | ''' 7 | http://www.realitypixels.com/turk/computergraphics/ResamplingFilters.pdf 8 | ''' 9 | def __init__(self, n_planes, factor, kernel_type, phase=0, kernel_width=None, support=None, sigma=None, preserve_size=False): 10 | super(Downsampler, self).__init__() 11 | 12 | assert phase in [0, 0.5], 'phase should be 0 or 0.5' 13 | 14 | if kernel_type == 'lanczos2': 15 | support = 2 16 | kernel_width = 4 * factor + 1 17 | kernel_type_ = 'lanczos' 18 | 19 | elif kernel_type == 'lanczos3': 20 | support = 3 21 | kernel_width = 6 * factor + 1 22 | kernel_type_ = 'lanczos' 23 | 24 | elif kernel_type == 'gauss12': 25 | kernel_width = 7 26 | sigma = 1/2 27 | kernel_type_ = 'gauss' 28 | 29 | elif kernel_type == 'gauss1sq2': 30 | kernel_width = 9 31 | sigma = 1./np.sqrt(2) 32 | kernel_type_ = 'gauss' 33 | 34 | elif kernel_type in ['lanczos', 'gauss', 'box']: 35 | kernel_type_ = kernel_type 36 | 37 | else: 38 | assert False, 'wrong name kernel' 39 | 40 | 41 | # note that `kernel width` will be different to actual size for phase = 1/2 42 | self.kernel = get_kernel(factor, kernel_type_, phase, kernel_width, support=support, sigma=sigma) 43 | 44 | downsampler = nn.Conv2d(n_planes, n_planes, kernel_size=self.kernel.shape, stride=factor, padding=0) 45 | downsampler.weight.data[:] = 0 46 | downsampler.bias.data[:] = 0 47 | 48 | kernel_torch = torch.from_numpy(self.kernel) 49 | for i in range(n_planes): 50 | downsampler.weight.data[i, i] = kernel_torch 51 | 52 | self.downsampler_ = downsampler 53 | 54 | if preserve_size: 55 | 56 | if self.kernel.shape[0] % 2 == 1: 57 | pad = int((self.kernel.shape[0] - 1) / 2.) 58 | else: 59 | pad = int((self.kernel.shape[0] - factor) / 2.) 60 | 61 | self.padding = nn.ReplicationPad2d(pad) 62 | 63 | self.preserve_size = preserve_size 64 | 65 | def forward(self, input): 66 | if self.preserve_size: 67 | x = self.padding(input) 68 | else: 69 | x= input 70 | self.x = x 71 | return self.downsampler_(x) 72 | 73 | def get_kernel(factor, kernel_type, phase, kernel_width, support=None, sigma=None): 74 | assert kernel_type in ['lanczos', 'gauss', 'box'] 75 | 76 | # factor = float(factor) 77 | if phase == 0.5 and kernel_type != 'box': 78 | kernel = np.zeros([kernel_width - 1, kernel_width - 1]) 79 | else: 80 | kernel = np.zeros([kernel_width, kernel_width]) 81 | 82 | 83 | if kernel_type == 'box': 84 | assert phase == 0.5, 'Box filter is always half-phased' 85 | kernel[:] = 1./(kernel_width * kernel_width) 86 | 87 | elif kernel_type == 'gauss': 88 | assert sigma, 'sigma is not specified' 89 | assert phase != 0.5, 'phase 1/2 for gauss not implemented' 90 | 91 | center = (kernel_width + 1.)/2. 92 | print(center, kernel_width) 93 | sigma_sq = sigma * sigma 94 | 95 | for i in range(1, kernel.shape[0] + 1): 96 | for j in range(1, kernel.shape[1] + 1): 97 | di = (i - center)/2. 98 | dj = (j - center)/2. 99 | kernel[i - 1][j - 1] = np.exp(-(di * di + dj * dj)/(2 * sigma_sq)) 100 | kernel[i - 1][j - 1] = kernel[i - 1][j - 1]/(2. * np.pi * sigma_sq) 101 | elif kernel_type == 'lanczos': 102 | assert support, 'support is not specified' 103 | center = (kernel_width + 1) / 2. 104 | 105 | for i in range(1, kernel.shape[0] + 1): 106 | for j in range(1, kernel.shape[1] + 1): 107 | 108 | if phase == 0.5: 109 | di = abs(i + 0.5 - center) / factor 110 | dj = abs(j + 0.5 - center) / factor 111 | else: 112 | di = abs(i - center) / factor 113 | dj = abs(j - center) / factor 114 | 115 | 116 | pi_sq = np.pi * np.pi 117 | 118 | val = 1 119 | if di != 0: 120 | val = val * support * np.sin(np.pi * di) * np.sin(np.pi * di / support) 121 | val = val / (np.pi * np.pi * di * di) 122 | 123 | if dj != 0: 124 | val = val * support * np.sin(np.pi * dj) * np.sin(np.pi * dj / support) 125 | val = val / (np.pi * np.pi * dj * dj) 126 | 127 | kernel[i - 1][j - 1] = val 128 | 129 | 130 | else: 131 | assert False, 'wrong method name' 132 | 133 | kernel /= kernel.sum() 134 | 135 | return kernel 136 | 137 | #a = Downsampler(n_planes=3, factor=2, kernel_type='lanczos2', phase='1', preserve_size=True) 138 | 139 | 140 | 141 | 142 | 143 | 144 | ################# 145 | # Learnable downsampler 146 | 147 | # KS = 32 148 | # dow = nn.Sequential(nn.ReplicationPad2d(int((KS - factor) / 2.)), nn.Conv2d(1,1,KS,factor)) 149 | 150 | # class Apply(nn.Module): 151 | # def __init__(self, what, dim, *args): 152 | # super(Apply, self).__init__() 153 | # self.dim = dim 154 | 155 | # self.what = what 156 | 157 | # def forward(self, input): 158 | # inputs = [] 159 | # for i in range(input.size(self.dim)): 160 | # inputs.append(self.what(input.narrow(self.dim, i, 1))) 161 | 162 | # return torch.cat(inputs, dim=self.dim) 163 | 164 | # def __len__(self): 165 | # return len(self._modules) 166 | 167 | # downs = Apply(dow, 1) 168 | # downs.type(dtype)(net_input.type(dtype)).size() 169 | -------------------------------------------------------------------------------- /1_Denoising/models/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from numpy.random import normal 4 | from numpy.linalg import svd 5 | from math import sqrt 6 | import torch.nn.init 7 | from .common import * 8 | 9 | class ResidualSequential(nn.Sequential): 10 | def __init__(self, *args): 11 | super(ResidualSequential, self).__init__(*args) 12 | 13 | def forward(self, x): 14 | out = super(ResidualSequential, self).forward(x) 15 | # print(x.size(), out.size()) 16 | x_ = None 17 | if out.size(2) != x.size(2) or out.size(3) != x.size(3): 18 | diff2 = x.size(2) - out.size(2) 19 | diff3 = x.size(3) - out.size(3) 20 | # print(1) 21 | x_ = x[:, :, diff2 /2:out.size(2) + diff2 / 2, diff3 / 2:out.size(3) + diff3 / 2] 22 | else: 23 | x_ = x 24 | return out + x_ 25 | 26 | def eval(self): 27 | print(2) 28 | for m in self.modules(): 29 | m.eval() 30 | exit() 31 | 32 | 33 | def get_block(num_channels, norm_layer, act_fun): 34 | layers = [ 35 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False), 36 | norm_layer(num_channels, affine=True), 37 | act(act_fun), 38 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False), 39 | norm_layer(num_channels, affine=True), 40 | ] 41 | return layers 42 | 43 | 44 | class ResNet(nn.Module): 45 | def __init__(self, num_input_channels, num_output_channels, num_blocks, num_channels, need_residual=True, act_fun='LeakyReLU', need_sigmoid=True, norm_layer=nn.BatchNorm2d, pad='reflection'): 46 | ''' 47 | pad = 'start|zero|replication' 48 | ''' 49 | super(ResNet, self).__init__() 50 | 51 | if need_residual: 52 | s = ResidualSequential 53 | else: 54 | s = nn.Sequential 55 | 56 | stride = 1 57 | # First layers 58 | layers = [ 59 | # nn.ReplicationPad2d(num_blocks * 2 * stride + 3), 60 | conv(num_input_channels, num_channels, 3, stride=1, bias=True, pad=pad), 61 | act(act_fun) 62 | ] 63 | # Residual blocks 64 | # layers_residual = [] 65 | for i in range(num_blocks): 66 | layers += [s(*get_block(num_channels, norm_layer, act_fun))] 67 | 68 | layers += [ 69 | nn.Conv2d(num_channels, num_channels, 3, 1, 1), 70 | norm_layer(num_channels, affine=True) 71 | ] 72 | 73 | # if need_residual: 74 | # layers += [ResidualSequential(*layers_residual)] 75 | # else: 76 | # layers += [Sequential(*layers_residual)] 77 | 78 | # if factor >= 2: 79 | # # Do upsampling if needed 80 | # layers += [ 81 | # nn.Conv2d(num_channels, num_channels * 82 | # factor ** 2, 3, 1), 83 | # nn.PixelShuffle(factor), 84 | # act(act_fun) 85 | # ] 86 | layers += [ 87 | conv(num_channels, num_output_channels, 3, 1, bias=True, pad=pad), 88 | nn.Sigmoid() 89 | ] 90 | self.model = nn.Sequential(*layers) 91 | 92 | def forward(self, input): 93 | return self.model(input) 94 | 95 | def eval(self): 96 | self.model.eval() 97 | -------------------------------------------------------------------------------- /1_Denoising/models/skip.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .common import * 4 | 5 | def skip( 6 | num_input_channels=2, num_output_channels=3, 7 | num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4], 8 | filter_size_down=3, filter_size_up=3, filter_skip_size=1, 9 | need_sigmoid=True, need_bias=True, 10 | pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU', 11 | need1x1_up=True): 12 | """Assembles encoder-decoder with skip connections. 13 | 14 | Arguments: 15 | act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU) 16 | pad (string): zero|reflection (default: 'zero') 17 | upsample_mode (string): 'nearest|bilinear' (default: 'nearest') 18 | downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride') 19 | 20 | """ 21 | assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip) 22 | 23 | n_scales = len(num_channels_down) 24 | 25 | if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) : 26 | upsample_mode = [upsample_mode]*n_scales 27 | 28 | if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)): 29 | downsample_mode = [downsample_mode]*n_scales 30 | 31 | if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) : 32 | filter_size_down = [filter_size_down]*n_scales 33 | 34 | if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) : 35 | filter_size_up = [filter_size_up]*n_scales 36 | 37 | last_scale = n_scales - 1 38 | 39 | cur_depth = None 40 | 41 | model = nn.Sequential() 42 | model_tmp = model 43 | 44 | input_depth = num_input_channels 45 | for i in range(len(num_channels_down)): 46 | 47 | deeper = nn.Sequential() 48 | skip = nn.Sequential() 49 | 50 | if num_channels_skip[i] != 0: 51 | model_tmp.add(Concat(1, skip, deeper)) 52 | else: 53 | model_tmp.add(deeper) 54 | 55 | model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i]))) 56 | 57 | if num_channels_skip[i] != 0: 58 | skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad)) 59 | skip.add(bn(num_channels_skip[i])) 60 | skip.add(act(act_fun)) 61 | 62 | # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part)) 63 | 64 | deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i])) 65 | deeper.add(bn(num_channels_down[i])) 66 | deeper.add(act(act_fun)) 67 | 68 | deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad)) 69 | deeper.add(bn(num_channels_down[i])) 70 | deeper.add(act(act_fun)) 71 | 72 | deeper_main = nn.Sequential() 73 | 74 | if i == len(num_channels_down) - 1: 75 | # The deepest 76 | k = num_channels_down[i] 77 | else: 78 | deeper.add(deeper_main) 79 | k = num_channels_up[i + 1] 80 | 81 | deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i])) 82 | 83 | model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad)) 84 | model_tmp.add(bn(num_channels_up[i])) 85 | model_tmp.add(act(act_fun)) 86 | 87 | 88 | if need1x1_up: 89 | model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad)) 90 | model_tmp.add(bn(num_channels_up[i])) 91 | model_tmp.add(act(act_fun)) 92 | 93 | input_depth = num_channels_down[i] 94 | model_tmp = deeper_main 95 | 96 | model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad)) 97 | if need_sigmoid: 98 | model.add(nn.Sigmoid()) 99 | 100 | return model 101 | -------------------------------------------------------------------------------- /1_Denoising/models/texture_nets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .common import * 4 | 5 | 6 | normalization = nn.BatchNorm2d 7 | 8 | 9 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero'): 10 | if pad == 'zero': 11 | return nn.Conv2d(in_f, out_f, kernel_size, stride, padding=(kernel_size - 1) / 2, bias=bias) 12 | elif pad == 'reflection': 13 | layers = [nn.ReflectionPad2d((kernel_size - 1) / 2), 14 | nn.Conv2d(in_f, out_f, kernel_size, stride, padding=0, bias=bias)] 15 | return nn.Sequential(*layers) 16 | 17 | def get_texture_nets(inp=3, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False, pad='zero', need_sigmoid=False, conv_num=8, upsample_mode='nearest'): 18 | 19 | 20 | for i in range(len(ratios)): 21 | j = i + 1 22 | 23 | seq = nn.Sequential() 24 | 25 | tmp = nn.AvgPool2d(ratios[i], ratios[i]) 26 | 27 | seq.add(tmp) 28 | if fill_noise: 29 | seq.add(GenNoise(inp)) 30 | 31 | seq.add(conv(inp, conv_num, 3, pad=pad)) 32 | seq.add(normalization(conv_num)) 33 | seq.add(act()) 34 | 35 | seq.add(conv(conv_num, conv_num, 3, pad=pad)) 36 | seq.add(normalization(conv_num)) 37 | seq.add(act()) 38 | 39 | seq.add(conv(conv_num, conv_num, 1, pad=pad)) 40 | seq.add(normalization(conv_num)) 41 | seq.add(act()) 42 | 43 | if i == 0: 44 | seq.add(nn.Upsample(scale_factor=2, mode=upsample_mode)) 45 | cur = seq 46 | else: 47 | 48 | cur_temp = cur 49 | 50 | cur = nn.Sequential() 51 | 52 | # Batch norm before merging 53 | seq.add(normalization(conv_num)) 54 | cur_temp.add(normalization(conv_num * (j - 1))) 55 | 56 | cur.add(Concat(1, cur_temp, seq)) 57 | 58 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad)) 59 | cur.add(normalization(conv_num * j)) 60 | cur.add(act()) 61 | 62 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad)) 63 | cur.add(normalization(conv_num * j)) 64 | cur.add(act()) 65 | 66 | cur.add(conv(conv_num * j, conv_num * j, 1, pad=pad)) 67 | cur.add(normalization(conv_num * j)) 68 | cur.add(act()) 69 | 70 | if i == len(ratios) - 1: 71 | cur.add(conv(conv_num * j, 3, 1, pad=pad)) 72 | else: 73 | cur.add(nn.Upsample(scale_factor=2, mode=upsample_mode)) 74 | 75 | model = cur 76 | if need_sigmoid: 77 | model.add(nn.Sigmoid()) 78 | 79 | return model 80 | -------------------------------------------------------------------------------- /1_Denoising/psnr_ssim.py: -------------------------------------------------------------------------------- 1 | ### This script will calcualte psnr and ssim 2 | import numpy as np 3 | from skimage.metrics import peak_signal_noise_ratio 4 | from skimage.measure import compare_ssim 5 | 6 | 7 | def calcualte_PSNR(im_true, im_test): 8 | im_true = np.transpose(im_true,(1,2,0)) 9 | im_test = np.transpose(im_test, (1, 2, 0)) 10 | psnr_value = peak_signal_noise_ratio(im_true, im_test) 11 | return psnr_value 12 | 13 | def calcualte_SSIM(im_true, im_test, multichannel): # multichannel=True for RGB images 14 | im_true = np.transpose(im_true, (1, 2, 0)) 15 | im_test = np.transpose(im_test, (1, 2, 0)) 16 | ssim_value = compare_ssim(im_true, im_test, multichannel=multichannel, data_range=im_test.max() - im_test.min()) 17 | return ssim_value 18 | 19 | if __name__ == '__main__': 20 | pass -------------------------------------------------------------------------------- /1_Denoising/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__init__.py -------------------------------------------------------------------------------- /1_Denoising/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/utils/__pycache__/common_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/common_utils.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/utils/__pycache__/common_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/common_utils.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/utils/__pycache__/denoising_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/denoising_utils.cpython-36.pyc -------------------------------------------------------------------------------- /1_Denoising/utils/__pycache__/denoising_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/denoising_utils.cpython-37.pyc -------------------------------------------------------------------------------- /1_Denoising/utils/common_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision 4 | import sys 5 | 6 | import numpy as np 7 | from PIL import Image 8 | import PIL 9 | import numpy as np 10 | 11 | import matplotlib.pyplot as plt 12 | 13 | def crop_image(img, d=32): 14 | '''Make dimensions divisible by `d`''' 15 | 16 | new_size = (img.size[0] - img.size[0] % d, 17 | img.size[1] - img.size[1] % d) 18 | 19 | bbox = [ 20 | int((img.size[0] - new_size[0])/2), 21 | int((img.size[1] - new_size[1])/2), 22 | int((img.size[0] + new_size[0])/2), 23 | int((img.size[1] + new_size[1])/2), 24 | ] 25 | 26 | img_cropped = img.crop(bbox) 27 | return img_cropped 28 | 29 | def get_params(opt_over, net, net_input, downsampler=None): 30 | '''Returns parameters that we want to optimize over. 31 | 32 | Args: 33 | opt_over: comma separated list, e.g. "net,input" or "net" 34 | net: network 35 | net_input: torch.Tensor that stores input `z` 36 | ''' 37 | opt_over_list = opt_over.split(',') 38 | params = [] 39 | 40 | for opt in opt_over_list: 41 | 42 | if opt == 'net': 43 | params += [x for x in net.parameters() ] 44 | elif opt=='down': 45 | assert downsampler is not None 46 | params = [x for x in downsampler.parameters()] 47 | elif opt == 'input': 48 | net_input.requires_grad = True 49 | params += [net_input] 50 | else: 51 | assert False, 'what is it?' 52 | 53 | return params 54 | 55 | def get_image_grid(images_np, nrow=8): 56 | '''Creates a grid from a list of images by concatenating them.''' 57 | images_torch = [torch.from_numpy(x) for x in images_np] 58 | torch_grid = torchvision.utils.make_grid(images_torch, nrow) 59 | 60 | return torch_grid.numpy() 61 | 62 | def plot_image_grid(images_np, nrow =8, factor=1, interpolation='lanczos'): 63 | """Draws images in a grid 64 | 65 | Args: 66 | images_np: list of images, each image is np.array of size 3xHxW of 1xHxW 67 | nrow: how many images will be in one row 68 | factor: size if the plt.figure 69 | interpolation: interpolation used in plt.imshow 70 | """ 71 | n_channels = max(x.shape[0] for x in images_np) 72 | assert (n_channels == 3) or (n_channels == 1), "images should have 1 or 3 channels" 73 | 74 | images_np = [x if (x.shape[0] == n_channels) else np.concatenate([x, x, x], axis=0) for x in images_np] 75 | 76 | grid = get_image_grid(images_np, nrow) 77 | 78 | plt.figure(figsize=(len(images_np) + factor, 12 + factor)) 79 | 80 | if images_np[0].shape[0] == 1: 81 | plt.imshow(grid[0], cmap='gray', interpolation=interpolation) 82 | else: 83 | plt.imshow(grid.transpose(1, 2, 0), interpolation=interpolation) 84 | 85 | plt.show() 86 | 87 | return grid 88 | 89 | def load(path): 90 | """Load PIL image.""" 91 | img = Image.open(path) 92 | return img 93 | 94 | def get_image(path, imsize=-1): 95 | """Load an image and resize to a cpecific size. 96 | 97 | Args: 98 | path: path to image 99 | imsize: tuple or scalar with dimensions; -1 for `no resize` 100 | """ 101 | img = load(path) 102 | 103 | if isinstance(imsize, int): 104 | imsize = (imsize, imsize) 105 | 106 | if imsize[0]!= -1 and img.size != imsize: 107 | if imsize[0] > img.size[0]: 108 | img = img.resize(imsize, Image.BICUBIC) 109 | else: 110 | img = img.resize(imsize, Image.ANTIALIAS) 111 | 112 | img_np = pil_to_np(img) 113 | 114 | return img, img_np 115 | 116 | 117 | 118 | def fill_noise(x, noise_type): 119 | """Fills tensor `x` with noise of type `noise_type`.""" 120 | if noise_type == 'u': 121 | x.uniform_() 122 | elif noise_type == 'n': 123 | x.normal_() 124 | else: 125 | assert False 126 | 127 | def get_noise(input_depth, method, spatial_size, noise_type='u', var=1./10): 128 | """Returns a pytorch.Tensor of size (1 x `input_depth` x `spatial_size[0]` x `spatial_size[1]`) 129 | initialized in a specific way. 130 | Args: 131 | input_depth: number of channels in the tensor 132 | method: `noise` for fillting tensor with noise; `meshgrid` for np.meshgrid 133 | spatial_size: spatial size of the tensor to initialize 134 | noise_type: 'u' for uniform; 'n' for normal 135 | var: a factor, a noise will be multiplicated by. Basically it is standard deviation scaler. 136 | """ 137 | if isinstance(spatial_size, int): 138 | spatial_size = (spatial_size, spatial_size) 139 | if method == 'noise': 140 | shape = [1, input_depth, spatial_size[0], spatial_size[1]] 141 | net_input = torch.zeros(shape) 142 | 143 | fill_noise(net_input, noise_type) 144 | net_input *= var 145 | elif method == 'meshgrid': 146 | assert input_depth == 2 147 | X, Y = np.meshgrid(np.arange(0, spatial_size[1])/float(spatial_size[1]-1), np.arange(0, spatial_size[0])/float(spatial_size[0]-1)) 148 | meshgrid = np.concatenate([X[None,:], Y[None,:]]) 149 | net_input= np_to_torch(meshgrid) 150 | else: 151 | assert False 152 | 153 | return net_input 154 | 155 | def pil_to_np(img_PIL): 156 | '''Converts image in PIL format to np.array. 157 | 158 | From W x H x C [0...255] to C x W x H [0..1] 159 | ''' 160 | ar = np.array(img_PIL) 161 | 162 | if len(ar.shape) == 3: 163 | ar = ar.transpose(2,0,1) 164 | else: 165 | ar = ar[None, ...] 166 | 167 | return ar.astype(np.float32) / 255. 168 | 169 | def np_to_pil(img_np): 170 | '''Converts image in np.array format to PIL image. 171 | 172 | From C x W x H [0..1] to W x H x C [0...255] 173 | ''' 174 | ar = np.clip(img_np*255,0,255).astype(np.uint8) 175 | 176 | if img_np.shape[0] == 1: 177 | ar = ar[0] 178 | else: 179 | ar = ar.transpose(1, 2, 0) 180 | 181 | return Image.fromarray(ar) 182 | 183 | def np_to_torch(img_np): 184 | '''Converts image in numpy.array to torch.Tensor. 185 | 186 | From C x W x H [0..1] to C x W x H [0..1] 187 | ''' 188 | return torch.from_numpy(img_np)[None, :] 189 | 190 | def torch_to_np(img_var): 191 | '''Converts an image in torch.Tensor format to np.array. 192 | 193 | From 1 x C x W x H [0..1] to C x W x H [0..1] 194 | ''' 195 | return img_var.detach().cpu().numpy()[0] 196 | 197 | 198 | def optimize(optimizer_type, parameters, closure, LR, num_iter): 199 | """Runs optimization loop. 200 | 201 | Args: 202 | optimizer_type: 'LBFGS' of 'adam' 203 | parameters: list of Tensors to optimize over 204 | closure: function, that returns loss variable 205 | LR: learning rate 206 | num_iter: number of iterations 207 | """ 208 | if optimizer_type == 'LBFGS': 209 | # Do several steps with adam first 210 | optimizer = torch.optim.Adam(parameters, lr=0.001) 211 | for j in range(100): 212 | optimizer.zero_grad() 213 | closure() 214 | optimizer.step() 215 | 216 | print('Starting optimization with LBFGS') 217 | def closure2(): 218 | optimizer.zero_grad() 219 | return closure() 220 | optimizer = torch.optim.LBFGS(parameters, max_iter=num_iter, lr=LR, tolerance_grad=-1, tolerance_change=-1) 221 | optimizer.step(closure2) 222 | 223 | elif optimizer_type == 'adam': 224 | print('Starting optimization with ADAM') 225 | optimizer = torch.optim.Adam(parameters, lr=LR) 226 | 227 | for j in range(num_iter): 228 | optimizer.zero_grad() 229 | closure() 230 | optimizer.step() 231 | else: 232 | assert False -------------------------------------------------------------------------------- /1_Denoising/utils/denoising_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from .common_utils import * 3 | 4 | 5 | 6 | def get_noisy_image(img_np, sigma): 7 | """Adds Gaussian noise to an image. 8 | 9 | Args: 10 | img_np: image, np.array with values from 0 to 1 11 | sigma: std of the noise 12 | """ 13 | img_noisy_np = np.clip(img_np + np.random.normal(scale=sigma, size=img_np.shape), 0, 1).astype(np.float32) 14 | img_noisy_pil = np_to_pil(img_noisy_np) 15 | 16 | return img_noisy_pil, img_noisy_np -------------------------------------------------------------------------------- /1_Denoising/utils/feature_inversion_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.transforms as transforms 4 | import torchvision.models as models 5 | from .matcher import Matcher 6 | import os 7 | from collections import OrderedDict 8 | 9 | class View(nn.Module): 10 | def __init__(self): 11 | super(View, self).__init__() 12 | 13 | def forward(self, x): 14 | return x.view(-1) 15 | 16 | def get_vanilla_vgg_features(cut_idx=-1): 17 | if not os.path.exists('vgg_features.pth'): 18 | os.system( 19 | 'wget --no-check-certificate -N https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth') 20 | vgg_weights = torch.load('vgg19-d01eb7cb.pth') 21 | # fix compatibility issues 22 | map = {'classifier.6.weight':u'classifier.7.weight', 'classifier.6.bias':u'classifier.7.bias'} 23 | vgg_weights = OrderedDict([(map[k] if k in map else k,v) for k,v in vgg_weights.iteritems()]) 24 | 25 | 26 | 27 | model = models.vgg19() 28 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values()) 29 | 30 | 31 | model.load_state_dict(vgg_weights) 32 | 33 | torch.save(model.features, 'vgg_features.pth') 34 | torch.save(model.classifier, 'vgg_classifier.pth') 35 | 36 | vgg = torch.load('vgg_features.pth') 37 | if cut_idx > 36: 38 | vgg_classifier = torch.load('vgg_classifier.pth') 39 | vgg = nn.Sequential(*(vgg._modules.values() + vgg_classifier._modules.values())) 40 | 41 | vgg.eval() 42 | 43 | return vgg 44 | 45 | 46 | def get_matcher(net, opt): 47 | idxs = [x for x in opt['layers'].split(',')] 48 | matcher = Matcher(opt['what']) 49 | 50 | def hook(module, input, output): 51 | matcher(module, output) 52 | 53 | for i in idxs: 54 | net._modules[i].register_forward_hook(hook) 55 | 56 | return matcher 57 | 58 | 59 | 60 | def get_vgg(cut_idx=-1): 61 | f = get_vanilla_vgg_features(cut_idx) 62 | 63 | if cut_idx > 0: 64 | num_modules = len(f._modules) 65 | keys_to_delete = [f._modules.keys()[x] for x in range(cut_idx, num_modules)] 66 | for k in keys_to_delete: 67 | del f._modules[k] 68 | 69 | return f 70 | 71 | def vgg_preprocess_var(var): 72 | (r, g, b) = torch.chunk(var, 3, dim=1) 73 | bgr = torch.cat((b, g, r), 1) 74 | out = bgr * 255 - torch.autograd.Variable(vgg_mean[None, ...]).type(var.type()).expand_as(bgr) 75 | return out 76 | 77 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1) 78 | 79 | 80 | 81 | def get_preprocessor(imsize): 82 | def vgg_preprocess(tensor): 83 | (r, g, b) = torch.chunk(tensor, 3, dim=0) 84 | bgr = torch.cat((b, g, r), 0) 85 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr) 86 | return out 87 | preprocess = transforms.Compose([ 88 | transforms.Resize(imsize), 89 | transforms.ToTensor(), 90 | transforms.Lambda(vgg_preprocess) 91 | ]) 92 | 93 | return preprocess 94 | 95 | 96 | def get_deprocessor(): 97 | def vgg_deprocess(tensor): 98 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0 99 | (b, g, r) = torch.chunk(bgr, 3, dim=0) 100 | rgb = torch.cat((r, g, b), 0) 101 | return rgb 102 | deprocess = transforms.Compose([ 103 | transforms.Lambda(vgg_deprocess), 104 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)), 105 | transforms.ToPILImage() 106 | ]) 107 | return deprocess 108 | -------------------------------------------------------------------------------- /1_Denoising/utils/inpainting_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | import PIL.ImageDraw as ImageDraw 4 | import PIL.ImageFont as ImageFont 5 | from .common_utils import * 6 | 7 | def get_text_mask(for_image, sz=20): 8 | font_fname = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf' 9 | font_size = sz 10 | font = ImageFont.truetype(font_fname, font_size) 11 | 12 | img_mask = Image.fromarray(np.array(for_image)*0+255) 13 | draw = ImageDraw.Draw(img_mask) 14 | draw.text((128, 128), "hello world", font=font, fill='rgb(0, 0, 0)') 15 | 16 | return img_mask 17 | 18 | def get_bernoulli_mask(for_image, zero_fraction=0.95): 19 | img_mask_np=(np.random.random_sample(size=pil_to_np(for_image).shape) > zero_fraction).astype(int) 20 | img_mask = np_to_pil(img_mask_np) 21 | 22 | return img_mask 23 | -------------------------------------------------------------------------------- /1_Denoising/utils/matcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class Matcher: 5 | def __init__(self, how='gram_matrix', loss='mse'): 6 | self.mode = 'store' 7 | self.stored = {} 8 | self.losses = {} 9 | 10 | if how in all_features.keys(): 11 | self.get_statistics = all_features[how] 12 | else: 13 | assert False 14 | pass 15 | 16 | if loss in all_losses.keys(): 17 | self.loss = all_losses[loss] 18 | else: 19 | assert False 20 | 21 | def __call__(self, module, features): 22 | statistics = self.get_statistics(features) 23 | 24 | self.statistics = statistics 25 | if self.mode == 'store': 26 | self.stored[module] = statistics.detach().clone() 27 | elif self.mode == 'match': 28 | self.losses[module] = self.loss(statistics, self.stored[module]) 29 | 30 | def clean(self): 31 | self.losses = {} 32 | 33 | def gram_matrix(x): 34 | (b, ch, h, w) = x.size() 35 | features = x.view(b, ch, w * h) 36 | features_t = features.transpose(1, 2) 37 | gram = features.bmm(features_t) / (ch * h * w) 38 | return gram 39 | 40 | 41 | def features(x): 42 | return x 43 | 44 | 45 | all_features = { 46 | 'gram_matrix': gram_matrix, 47 | 'features': features, 48 | } 49 | 50 | all_losses = { 51 | 'mse': nn.MSELoss(), 52 | 'smoothL1': nn.SmoothL1Loss(), 53 | 'L1': nn.L1Loss(), 54 | } 55 | -------------------------------------------------------------------------------- /1_Denoising/utils/perceptual_loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/perceptual_loss/__init__.py -------------------------------------------------------------------------------- /1_Denoising/utils/perceptual_loss/matcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class Matcher: 6 | def __init__(self, how='gram_matrix', loss='mse', map_index=933): 7 | self.mode = 'store' 8 | self.stored = {} 9 | self.losses = {} 10 | 11 | if how in all_features.keys(): 12 | self.get_statistics = all_features[how] 13 | else: 14 | assert False 15 | pass 16 | 17 | if loss in all_losses.keys(): 18 | self.loss = all_losses[loss] 19 | else: 20 | assert False 21 | 22 | self.map_index = map_index 23 | self.method = 'match' 24 | 25 | 26 | def __call__(self, module, features): 27 | statistics = self.get_statistics(features) 28 | 29 | self.statistics = statistics 30 | if self.mode == 'store': 31 | self.stored[module] = statistics.detach() 32 | 33 | elif self.mode == 'match': 34 | 35 | if statistics.ndimension() == 2: 36 | 37 | if self.method == 'maximize': 38 | self.losses[module] = - statistics[0, self.map_index] 39 | else: 40 | self.losses[module] = torch.abs(300 - statistics[0, self.map_index]) 41 | 42 | else: 43 | ws = self.window_size 44 | 45 | t = statistics.detach() * 0 46 | 47 | s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0 48 | t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0 49 | t_cc[:, self.map_index,...] = 1 50 | 51 | if self.method == 'maximize': 52 | self.losses[module] = -(s_cc * t_cc.contiguous()).sum() 53 | else: 54 | self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum() 55 | 56 | 57 | def clean(self): 58 | self.losses = {} 59 | 60 | def gram_matrix(x): 61 | (b, ch, h, w) = x.size() 62 | features = x.view(b, ch, w * h) 63 | features_t = features.transpose(1, 2) 64 | gram = features.bmm(features_t) / (ch * h * w) 65 | return gram 66 | 67 | 68 | def features(x): 69 | return x 70 | 71 | 72 | all_features = { 73 | 'gram_matrix': gram_matrix, 74 | 'features': features, 75 | } 76 | 77 | all_losses = { 78 | 'mse': nn.MSELoss(), 79 | 'smoothL1': nn.SmoothL1Loss(), 80 | 'L1': nn.L1Loss(), 81 | } 82 | -------------------------------------------------------------------------------- /1_Denoising/utils/perceptual_loss/perceptual_loss.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn as nn 4 | import torchvision.transforms as transforms 5 | import torchvision.models as models 6 | from .matcher import Matcher 7 | from collections import OrderedDict 8 | 9 | from torchvision.models.vgg import model_urls 10 | from torchvision.models import vgg19 11 | from torch.autograd import Variable 12 | 13 | from .vgg_modified import VGGModified 14 | 15 | def get_pretrained_net(name): 16 | """Loads pretrained network""" 17 | if name == 'alexnet_caffe': 18 | if not os.path.exists('alexnet-torch_py3.pth'): 19 | print('Downloading AlexNet') 20 | os.system('wget -O alexnet-torch_py3.pth --no-check-certificate -nc https://box.skoltech.ru/index.php/s/77xSWvrDN0CiQtK/download') 21 | return torch.load('alexnet-torch_py3.pth') 22 | elif name == 'vgg19_caffe': 23 | if not os.path.exists('vgg19-caffe-py3.pth'): 24 | print('Downloading VGG-19') 25 | os.system('wget -O vgg19-caffe-py3.pth --no-check-certificate -nc https://box.skoltech.ru/index.php/s/HPcOFQTjXxbmp4X/download') 26 | 27 | vgg = get_vgg19_caffe() 28 | 29 | return vgg 30 | elif name == 'vgg16_caffe': 31 | if not os.path.exists('vgg16-caffe-py3.pth'): 32 | print('Downloading VGG-16') 33 | os.system('wget -O vgg16-caffe-py3.pth --no-check-certificate -nc https://box.skoltech.ru/index.php/s/TUZ62HnPKWdxyLr/download') 34 | 35 | vgg = get_vgg16_caffe() 36 | 37 | return vgg 38 | elif name == 'vgg19_pytorch_modified': 39 | # os.system('wget -O data/feature_inversion/vgg19-caffe.pth --no-check-certificate -nc https://www.dropbox.com/s/xlbdo688dy4keyk/vgg19-caffe.pth?dl=1') 40 | 41 | model = VGGModified(vgg19(pretrained=False), 0.2) 42 | model.load_state_dict(torch.load('vgg_pytorch_modified.pkl')['state_dict']) 43 | 44 | return model 45 | else: 46 | assert False 47 | 48 | 49 | class PerceputalLoss(nn.modules.loss._Loss): 50 | """ 51 | Assumes input image is in range [0,1] if `input_range` is 'sigmoid', [-1, 1] if 'tanh' 52 | """ 53 | def __init__(self, input_range='sigmoid', 54 | net_type = 'vgg_torch', 55 | input_preprocessing='corresponding', 56 | match=[{'layers':[11,20,29],'what':'features'}]): 57 | 58 | if input_range not in ['sigmoid', 'tanh']: 59 | assert False 60 | 61 | self.net = get_pretrained_net(net_type).cuda() 62 | 63 | self.matchers = [get_matcher(self.net, match_opts) for match_opts in match] 64 | 65 | preprocessing_correspondence = { 66 | 'vgg19_torch': vgg_preprocess_caffe, 67 | 'vgg16_torch': vgg_preprocess_caffe, 68 | 'vgg19_pytorch': vgg_preprocess_pytorch, 69 | 'vgg19_pytorch_modified': vgg_preprocess_pytorch, 70 | } 71 | 72 | if input_preprocessing == 'corresponding': 73 | self.preprocess_input = preprocessing_correspondence[net_type] 74 | else: 75 | self.preprocessing = preprocessing_correspondence[input_preprocessing] 76 | 77 | def preprocess_input(self, x): 78 | if self.input_range == 'tanh': 79 | x = (x + 1.) / 2. 80 | 81 | return self.preprocess(x) 82 | 83 | def __call__(self, x, y): 84 | 85 | # for 86 | self.matcher_content.mode = 'store' 87 | self.net(self.preprocess_input(y)); 88 | 89 | self.matcher_content.mode = 'match' 90 | self.net(self.preprocess_input(x)); 91 | 92 | return sum([sum(matcher.losses.values()) for matcher in self.matchers]) 93 | 94 | 95 | def get_vgg19_caffe(): 96 | model = vgg19() 97 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values()) 98 | vgg = model.features 99 | vgg_classifier = model.classifier 100 | 101 | names = ['conv1_1','relu1_1','conv1_2','relu1_2','pool1', 102 | 'conv2_1','relu2_1','conv2_2','relu2_2','pool2', 103 | 'conv3_1','relu3_1','conv3_2','relu3_2','conv3_3','relu3_3','conv3_4','relu3_4','pool3', 104 | 'conv4_1','relu4_1','conv4_2','relu4_2','conv4_3','relu4_3','conv4_4','relu4_4','pool4', 105 | 'conv5_1','relu5_1','conv5_2','relu5_2','conv5_3','relu5_3','conv5_4','relu5_4','pool5', 106 | 'torch_view','fc6','relu6','drop6','fc7','relu7','drop7','fc8'] 107 | 108 | model = nn.Sequential() 109 | for n, m in zip(names, list(vgg) + list(vgg_classifier)): 110 | model.add_module(n, m) 111 | 112 | model.load_state_dict(torch.load('vgg19-caffe-py3.pth')) 113 | 114 | return model 115 | 116 | def get_vgg16_caffe(): 117 | vgg = torch.load('vgg16-caffe-py3.pth') 118 | 119 | names = ['conv1_1','relu1_1','conv1_2','relu1_2','pool1', 120 | 'conv2_1','relu2_1','conv2_2','relu2_2','pool2', 121 | 'conv3_1','relu3_1','conv3_2','relu3_2','conv3_3','relu3_3','pool3', 122 | 'conv4_1','relu4_1','conv4_2','relu4_2','conv4_3','relu4_3','pool4', 123 | 'conv5_1','relu5_1','conv5_2','relu5_2','conv5_3','relu5_3','pool5', 124 | 'torch_view','fc6','relu6','drop6','fc7','relu7','fc8'] 125 | 126 | model = nn.Sequential() 127 | for n, m in zip(names, list(vgg)): 128 | model.add_module(n, m) 129 | 130 | # model.load_state_dict(torch.load('vgg19-caffe-py3.pth')) 131 | 132 | return model 133 | 134 | 135 | class View(nn.Module): 136 | def __init__(self): 137 | super(View, self).__init__() 138 | 139 | def forward(self, x): 140 | return x.view(x.size(0), -1) 141 | 142 | 143 | def get_matcher(vgg, opt): 144 | # idxs = [int(x) for x in opt['layers'].split(',')] 145 | matcher = Matcher(opt['what'], 'mse', opt['map_idx']) 146 | 147 | def hook(module, input, output): 148 | matcher(module, output) 149 | 150 | for layer_name in opt['layers']: 151 | vgg._modules[layer_name].register_forward_hook(hook) 152 | 153 | return matcher 154 | 155 | 156 | def get_vgg(cut_idx=-1, vgg_type='pytorch'): 157 | f = get_vanilla_vgg_features(cut_idx, vgg_type) 158 | 159 | keys = [x for x in cnn._modules.keys()] 160 | max_idx = max(keys.index(x) for x in opt_content['layers'].split(',')) 161 | for k in keys[max_idx+1:]: 162 | cnn._modules.pop(k) 163 | 164 | return f 165 | 166 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1) 167 | def vgg_preprocess_caffe(var): 168 | (r, g, b) = torch.chunk(var, 3, dim=1) 169 | bgr = torch.cat((b, g, r), 1) 170 | out = bgr * 255 - torch.autograd.Variable(vgg_mean).type(var.type()) 171 | return out 172 | 173 | 174 | 175 | mean_pytorch = Variable(torch.Tensor([0.485, 0.456, 0.406]).view(3, 1, 1)) 176 | std_pytorch = Variable(torch.Tensor([0.229, 0.224, 0.225]).view(3, 1, 1)) 177 | 178 | def vgg_preprocess_pytorch(var): 179 | return (var - mean_pytorch.type_as(var))/std_pytorch.type_as(var) 180 | 181 | 182 | 183 | def get_preprocessor(imsize): 184 | def vgg_preprocess(tensor): 185 | (r, g, b) = torch.chunk(tensor, 3, dim=0) 186 | bgr = torch.cat((b, g, r), 0) 187 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr) 188 | return out 189 | preprocess = transforms.Compose([ 190 | transforms.Resize(imsize), 191 | transforms.ToTensor(), 192 | transforms.Lambda(vgg_preprocess) 193 | ]) 194 | 195 | return preprocess 196 | 197 | 198 | def get_deprocessor(): 199 | def vgg_deprocess(tensor): 200 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0 201 | (b, g, r) = torch.chunk(bgr, 3, dim=0) 202 | rgb = torch.cat((r, g, b), 0) 203 | return rgb 204 | deprocess = transforms.Compose([ 205 | transforms.Lambda(vgg_deprocess), 206 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)), 207 | transforms.ToPILImage() 208 | ]) 209 | return deprocess 210 | 211 | -------------------------------------------------------------------------------- /1_Denoising/utils/perceptual_loss/vgg_modified.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | class VGGModified(nn.Module): 4 | def __init__(self, vgg19_orig, slope=0.01): 5 | super(VGGModified, self).__init__() 6 | self.features = nn.Sequential() 7 | 8 | self.features.add_module(str(0), vgg19_orig.features[0]) 9 | self.features.add_module(str(1), nn.LeakyReLU(slope, True)) 10 | self.features.add_module(str(2), vgg19_orig.features[2]) 11 | self.features.add_module(str(3), nn.LeakyReLU(slope, True)) 12 | self.features.add_module(str(4), nn.AvgPool2d((2,2), (2,2))) 13 | 14 | self.features.add_module(str(5), vgg19_orig.features[5]) 15 | self.features.add_module(str(6), nn.LeakyReLU(slope, True)) 16 | self.features.add_module(str(7), vgg19_orig.features[7]) 17 | self.features.add_module(str(8), nn.LeakyReLU(slope, True)) 18 | self.features.add_module(str(9), nn.AvgPool2d((2,2), (2,2))) 19 | 20 | self.features.add_module(str(10), vgg19_orig.features[10]) 21 | self.features.add_module(str(11), nn.LeakyReLU(slope, True)) 22 | self.features.add_module(str(12), vgg19_orig.features[12]) 23 | self.features.add_module(str(13), nn.LeakyReLU(slope, True)) 24 | self.features.add_module(str(14), vgg19_orig.features[14]) 25 | self.features.add_module(str(15), nn.LeakyReLU(slope, True)) 26 | self.features.add_module(str(16), vgg19_orig.features[16]) 27 | self.features.add_module(str(17), nn.LeakyReLU(slope, True)) 28 | self.features.add_module(str(18), nn.AvgPool2d((2,2), (2,2))) 29 | 30 | self.features.add_module(str(19), vgg19_orig.features[19]) 31 | self.features.add_module(str(20), nn.LeakyReLU(slope, True)) 32 | self.features.add_module(str(21), vgg19_orig.features[21]) 33 | self.features.add_module(str(22), nn.LeakyReLU(slope, True)) 34 | self.features.add_module(str(23), vgg19_orig.features[23]) 35 | self.features.add_module(str(24), nn.LeakyReLU(slope, True)) 36 | self.features.add_module(str(25), vgg19_orig.features[25]) 37 | self.features.add_module(str(26), nn.LeakyReLU(slope, True)) 38 | self.features.add_module(str(27), nn.AvgPool2d((2,2), (2,2))) 39 | 40 | self.features.add_module(str(28), vgg19_orig.features[28]) 41 | self.features.add_module(str(29), nn.LeakyReLU(slope, True)) 42 | self.features.add_module(str(30), vgg19_orig.features[30]) 43 | self.features.add_module(str(31), nn.LeakyReLU(slope, True)) 44 | self.features.add_module(str(32), vgg19_orig.features[32]) 45 | self.features.add_module(str(33), nn.LeakyReLU(slope, True)) 46 | self.features.add_module(str(34), vgg19_orig.features[34]) 47 | self.features.add_module(str(35), nn.LeakyReLU(slope, True)) 48 | self.features.add_module(str(36), nn.AvgPool2d((2,2), (2,2))) 49 | 50 | self.classifier = nn.Sequential() 51 | 52 | self.classifier.add_module(str(0), vgg19_orig.classifier[0]) 53 | self.classifier.add_module(str(1), nn.LeakyReLU(slope, True)) 54 | self.classifier.add_module(str(2), nn.Dropout2d(p = 0.5)) 55 | self.classifier.add_module(str(3), vgg19_orig.classifier[3]) 56 | self.classifier.add_module(str(4), nn.LeakyReLU(slope, True)) 57 | self.classifier.add_module(str(5), nn.Dropout2d(p = 0.5)) 58 | self.classifier.add_module(str(6), vgg19_orig.classifier[6]) 59 | 60 | def forward(self, x): 61 | return self.classifier(self.features.forward(x)) -------------------------------------------------------------------------------- /1_Denoising/utils/sr_utils.py: -------------------------------------------------------------------------------- 1 | from .common_utils import * 2 | 3 | def put_in_center(img_np, target_size): 4 | img_out = np.zeros([3, target_size[0], target_size[1]]) 5 | 6 | bbox = [ 7 | int((target_size[0] - img_np.shape[1]) / 2), 8 | int((target_size[1] - img_np.shape[2]) / 2), 9 | int((target_size[0] + img_np.shape[1]) / 2), 10 | int((target_size[1] + img_np.shape[2]) / 2), 11 | ] 12 | 13 | img_out[:, bbox[0]:bbox[2], bbox[1]:bbox[3]] = img_np 14 | 15 | return img_out 16 | 17 | 18 | def load_LR_HR_imgs_sr(fname, imsize, factor, enforse_div32=None): 19 | '''Loads an image, resizes it, center crops and downscales. 20 | 21 | Args: 22 | fname: path to the image 23 | imsize: new size for the image, -1 for no resizing 24 | factor: downscaling factor 25 | enforse_div32: if 'CROP' center crops an image, so that its dimensions are divisible by 32. 26 | ''' 27 | img_orig_pil, img_orig_np = get_image(fname, -1) 28 | 29 | if imsize != -1: 30 | img_orig_pil, img_orig_np = get_image(fname, imsize) 31 | 32 | # For comparison with GT 33 | if enforse_div32 == 'CROP': 34 | new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 32, 35 | img_orig_pil.size[1] - img_orig_pil.size[1] % 32) 36 | 37 | bbox = [ 38 | (img_orig_pil.size[0] - new_size[0])/2, 39 | (img_orig_pil.size[1] - new_size[1])/2, 40 | (img_orig_pil.size[0] + new_size[0])/2, 41 | (img_orig_pil.size[1] + new_size[1])/2, 42 | ] 43 | 44 | img_HR_pil = img_orig_pil.crop(bbox) 45 | img_HR_np = pil_to_np(img_HR_pil) 46 | else: 47 | img_HR_pil, img_HR_np = img_orig_pil, img_orig_np 48 | 49 | LR_size = [ 50 | img_HR_pil.size[0] // factor, 51 | img_HR_pil.size[1] // factor 52 | ] 53 | 54 | img_LR_pil = img_HR_pil.resize(LR_size, Image.ANTIALIAS) 55 | img_LR_np = pil_to_np(img_LR_pil) 56 | 57 | print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str (img_LR_pil.size))) 58 | 59 | return { 60 | 'orig_pil': img_orig_pil, 61 | 'orig_np': img_orig_np, 62 | 'LR_pil': img_LR_pil, 63 | 'LR_np': img_LR_np, 64 | 'HR_pil': img_HR_pil, 65 | 'HR_np': img_HR_np 66 | } 67 | 68 | 69 | def get_baselines(img_LR_pil, img_HR_pil): 70 | '''Gets `bicubic`, sharpened bicubic and `nearest` baselines.''' 71 | img_bicubic_pil = img_LR_pil.resize(img_HR_pil.size, Image.BICUBIC) 72 | img_bicubic_np = pil_to_np(img_bicubic_pil) 73 | 74 | img_nearest_pil = img_LR_pil.resize(img_HR_pil.size, Image.NEAREST) 75 | img_nearest_np = pil_to_np(img_nearest_pil) 76 | 77 | img_bic_sharp_pil = img_bicubic_pil.filter(PIL.ImageFilter.UnsharpMask()) 78 | img_bic_sharp_np = pil_to_np(img_bic_sharp_pil) 79 | 80 | return img_bicubic_np, img_bic_sharp_np, img_nearest_np 81 | 82 | 83 | 84 | def tv_loss(x, beta = 0.5): 85 | '''Calculates TV loss for an image `x`. 86 | 87 | Args: 88 | x: image, torch.Variable of torch.Tensor 89 | beta: See https://arxiv.org/abs/1412.0035 (fig. 2) to see effect of `beta` 90 | ''' 91 | dh = torch.pow(x[:,:,:,1:] - x[:,:,:,:-1], 2) 92 | dw = torch.pow(x[:,:,1:,:] - x[:,:,:-1,:], 2) 93 | 94 | return torch.sum(torch.pow(dh[:, :, :-1] + dw[:, :, :, :-1], beta)) 95 | -------------------------------------------------------------------------------- /2_Super_Resolution/BN_Net.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | 5 | #----------------------------------------------------------- 6 | #----------------------------------------------------------- 7 | # BN for the input seed 8 | #----------------------------------------------------------- 9 | #----------------------------------------------------------- 10 | class BNNet(nn.Module): 11 | def __init__(self,num_channel): 12 | super(BNNet, self).__init__() 13 | self.bn = nn.BatchNorm2d(num_channel) 14 | 15 | def forward(self, input_data): 16 | output_data = self.bn(input_data) 17 | return output_data 18 | -------------------------------------------------------------------------------- /2_Super_Resolution/Main_Start.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from train_sr import * 4 | import time 5 | import datetime 6 | # from include import * 7 | from utils import * 8 | from utils.sr_utils import * 9 | from utils.common_utils import * 10 | 11 | 12 | 13 | 14 | #----------------------------------------------------------- 15 | #----------------------------------------------------------- 16 | # This is the Main of the code 17 | #----------------------------------------------------------- 18 | #----------------------------------------------------------- 19 | 20 | 21 | if __name__ == '__main__': 22 | # Set random seed for reproducibility 23 | manualSeed = 100 24 | manualSeed = random.randint(1, 10000) # use if you want new results 25 | print("Random Seed: ", manualSeed) 26 | random.seed(manualSeed) 27 | torch.manual_seed(manualSeed) 28 | 29 | ##### Set up hyper-parameters 30 | #----------------------------------------------------------- 31 | gpu = 1 32 | 33 | print_step = 1 34 | max_epoch = 5000 35 | max_epoch = int(max_epoch / print_step) * print_step + 1 36 | 37 | #----------------------------------------------------------- 38 | tv_weight = 0.75 # 39 | learning_rate_model = 0.5 40 | OPTIMIZER = 'Adam' 41 | LOSS = "MSE"#'Huber' or "L1" 42 | 43 | factor = 4 44 | 45 | # image_list = ['Peppers'] 46 | 47 | # image_list = ['baby', 'bird', 'butterfly', 'head', 'woman', 48 | # 'baboon', 'barbara', 'bridge', 'coastguard', 'comic', 'face', 'flowers', 49 | # 'foreman', 'lenna', 'man', 'monarch', 'pepper', 'ppt3', 'zebra'] 50 | 51 | # Set5 = ['baby', 'bird', 'butterfly', 'head', 'woman'] 52 | 53 | # Set14 = ['baboon', 'barbara', 'bridge', 'coastguard', 'comic', 'face', 'flowers', 54 | # 'foreman', 'lenna', 'man', 'monarch', 'pepper', 'ppt3', 'zebra'] 55 | 56 | Set5 = ['baby'] 57 | 58 | Set14 = ['baboon', 'barbara', 'bridge', 'coastguard', 'comic', 'face', 'flowers', 59 | 'foreman', 'lenna', 'man', 'monarch', 'pepper', 'ppt3', 'zebra'] 60 | 61 | 62 | image_list = Set5 63 | ###################################### Processing images one by one ################################# 64 | for cur_image in image_list: 65 | # set up the path info 66 | if cur_image in Set5: 67 | path = '../0_Dataset/Super_Resolution/Set5/' 68 | elif cur_image in Set14: 69 | path = '../0_Dataset/Super_Resolution/Set14/' 70 | else: 71 | assert False, 'Error: please put your images under Set5 or Set14!' 72 | 73 | #---load image 74 | PLOT = False 75 | imsize = -1 76 | enforse_div32 = 'CROP' # set to be NONE for Set5/14 'CROP' # we usually need the dimensions to be divisible by a power of two (32 in this case) 77 | # Starts here 78 | path_to_image = os.path.join(path, cur_image+'.png') 79 | imgs = load_LR_HR_imgs_sr(path_to_image, imsize, factor, enforse_div32) 80 | 81 | imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np'] = get_baselines(imgs['LR_pil'], imgs['HR_pil']) 82 | 83 | if PLOT: 84 | plot_image_grid([imgs['HR_np'], imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np']], 4, 12) 85 | print('PSNR bicubic: %.4f PSNR nearest: %.4f' % ( 86 | compare_psnr(imgs['HR_np'], imgs['bicubic_np']), 87 | compare_psnr(imgs['HR_np'], imgs['nearest_np']))) 88 | 89 | ### set up input noise width, height, and channel 90 | HR_np_shape = imgs['HR_np'].shape 91 | input_h = HR_np_shape[1] 92 | input_w = HR_np_shape[2] 93 | input_c = 1 94 | width = 128 95 | 96 | DIP_train(imgs, 97 | learning_rate_model, 98 | OPTIMIZER, 99 | LOSS, 100 | width, 101 | input_w, 102 | input_h, 103 | input_c, 104 | max_epoch, 105 | print_step, 106 | gpu, 107 | cur_image, 108 | tv_weight, 109 | factor 110 | ) 111 | -------------------------------------------------------------------------------- /2_Super_Resolution/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .skip import skip 2 | from .texture_nets import get_texture_nets 3 | from .resnet import ResNet 4 | from .unet import UNet 5 | 6 | import torch.nn as nn 7 | 8 | def get_net(input_depth, NET_TYPE, pad, upsample_mode, n_channels=3, act_fun='LeakyReLU', skip_n33d=128, skip_n33u=128, skip_n11=4, num_scales=5, downsample_mode='stride'): 9 | if NET_TYPE == 'ResNet': 10 | # TODO 11 | net = ResNet(input_depth, 3, 10, 16, 1, nn.BatchNorm2d, False) 12 | elif NET_TYPE == 'skip': 13 | net = skip(input_depth, n_channels, num_channels_down = [skip_n33d]*num_scales if isinstance(skip_n33d, int) else skip_n33d, 14 | num_channels_up = [skip_n33u]*num_scales if isinstance(skip_n33u, int) else skip_n33u, 15 | num_channels_skip = [skip_n11]*num_scales if isinstance(skip_n11, int) else skip_n11, 16 | upsample_mode=upsample_mode, downsample_mode=downsample_mode, 17 | need_sigmoid=True, need_bias=True, pad=pad, act_fun=act_fun) 18 | 19 | elif NET_TYPE == 'texture_nets': 20 | net = get_texture_nets(inp=input_depth, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False,pad=pad) 21 | 22 | elif NET_TYPE =='UNet': 23 | net = UNet(num_input_channels=input_depth, num_output_channels=3, 24 | feature_scale=4, more_layers=0, concat_x=False, 25 | upsample_mode=upsample_mode, pad=pad, norm_layer=nn.BatchNorm2d, need_sigmoid=True, need_bias=True) 26 | elif NET_TYPE == 'identity': 27 | assert input_depth == 3 28 | net = nn.Sequential() 29 | else: 30 | assert False 31 | 32 | return net -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/common.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/common.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/common.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/common.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/downsampler.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/downsampler.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/downsampler.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/downsampler.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/resnet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/resnet.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/skip.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/skip.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/skip.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/skip.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/texture_nets.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/texture_nets.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/texture_nets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/texture_nets.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/unet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/unet.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/__pycache__/unet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/unet.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/models/common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from .downsampler import Downsampler 5 | 6 | def add_module(self, module): 7 | self.add_module(str(len(self) + 1), module) 8 | 9 | torch.nn.Module.add = add_module 10 | 11 | class Concat(nn.Module): 12 | def __init__(self, dim, *args): 13 | super(Concat, self).__init__() 14 | self.dim = dim 15 | 16 | for idx, module in enumerate(args): 17 | self.add_module(str(idx), module) 18 | 19 | def forward(self, input): 20 | inputs = [] 21 | for module in self._modules.values(): 22 | inputs.append(module(input)) 23 | 24 | inputs_shapes2 = [x.shape[2] for x in inputs] 25 | inputs_shapes3 = [x.shape[3] for x in inputs] 26 | 27 | if np.all(np.array(inputs_shapes2) == min(inputs_shapes2)) and np.all(np.array(inputs_shapes3) == min(inputs_shapes3)): 28 | inputs_ = inputs 29 | else: 30 | target_shape2 = min(inputs_shapes2) 31 | target_shape3 = min(inputs_shapes3) 32 | 33 | inputs_ = [] 34 | for inp in inputs: 35 | diff2 = (inp.size(2) - target_shape2) // 2 36 | diff3 = (inp.size(3) - target_shape3) // 2 37 | inputs_.append(inp[:, :, diff2: diff2 + target_shape2, diff3:diff3 + target_shape3]) 38 | 39 | return torch.cat(inputs_, dim=self.dim) 40 | 41 | def __len__(self): 42 | return len(self._modules) 43 | 44 | 45 | class GenNoise(nn.Module): 46 | def __init__(self, dim2): 47 | super(GenNoise, self).__init__() 48 | self.dim2 = dim2 49 | 50 | def forward(self, input): 51 | a = list(input.size()) 52 | a[1] = self.dim2 53 | # print (input.data.type()) 54 | 55 | b = torch.zeros(a).type_as(input.data) 56 | b.normal_() 57 | 58 | x = torch.autograd.Variable(b) 59 | 60 | return x 61 | 62 | 63 | class Swish(nn.Module): 64 | """ 65 | https://arxiv.org/abs/1710.05941 66 | The hype was so huge that I could not help but try it 67 | """ 68 | def __init__(self): 69 | super(Swish, self).__init__() 70 | self.s = nn.Sigmoid() 71 | 72 | def forward(self, x): 73 | return x * self.s(x) 74 | 75 | 76 | def act(act_fun = 'LeakyReLU'): 77 | ''' 78 | Either string defining an activation function or module (e.g. nn.ReLU) 79 | ''' 80 | if isinstance(act_fun, str): 81 | if act_fun == 'LeakyReLU': 82 | return nn.LeakyReLU(0.2, inplace=True) 83 | elif act_fun == 'Swish': 84 | return Swish() 85 | elif act_fun == 'ELU': 86 | return nn.ELU() 87 | elif act_fun == 'none': 88 | return nn.Sequential() 89 | else: 90 | assert False 91 | else: 92 | return act_fun() 93 | 94 | 95 | def bn(num_features): 96 | return nn.BatchNorm2d(num_features) 97 | 98 | 99 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero', downsample_mode='stride'): 100 | downsampler = None 101 | if stride != 1 and downsample_mode != 'stride': 102 | 103 | if downsample_mode == 'avg': 104 | downsampler = nn.AvgPool2d(stride, stride) 105 | elif downsample_mode == 'max': 106 | downsampler = nn.MaxPool2d(stride, stride) 107 | elif downsample_mode in ['lanczos2', 'lanczos3']: 108 | downsampler = Downsampler(n_planes=out_f, factor=stride, kernel_type=downsample_mode, phase=0.5, preserve_size=True) 109 | else: 110 | assert False 111 | 112 | stride = 1 113 | 114 | padder = None 115 | to_pad = int((kernel_size - 1) / 2) 116 | if pad == 'reflection': 117 | padder = nn.ReflectionPad2d(to_pad) 118 | to_pad = 0 119 | 120 | convolver = nn.Conv2d(in_f, out_f, kernel_size, stride, padding=to_pad, bias=bias) 121 | 122 | 123 | layers = filter(lambda x: x is not None, [padder, convolver, downsampler]) 124 | return nn.Sequential(*layers) -------------------------------------------------------------------------------- /2_Super_Resolution/models/dcgan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | def dcgan(inp=2, 5 | ndf=32, 6 | num_ups=4, need_sigmoid=True, need_bias=True, pad='zero', upsample_mode='nearest', need_convT = True): 7 | 8 | layers= [nn.ConvTranspose2d(inp, ndf, kernel_size=3, stride=1, padding=0, bias=False), 9 | nn.BatchNorm2d(ndf), 10 | nn.LeakyReLU(True)] 11 | 12 | for i in range(num_ups-3): 13 | if need_convT: 14 | layers += [ nn.ConvTranspose2d(ndf, ndf, kernel_size=4, stride=2, padding=1, bias=False), 15 | nn.BatchNorm2d(ndf), 16 | nn.LeakyReLU(True)] 17 | else: 18 | layers += [ nn.Upsample(scale_factor=2, mode=upsample_mode), 19 | nn.Conv2d(ndf, ndf, kernel_size=3, stride=1, padding=1, bias=False), 20 | nn.BatchNorm2d(ndf), 21 | nn.LeakyReLU(True)] 22 | 23 | if need_convT: 24 | layers += [nn.ConvTranspose2d(ndf, 3, 4, 2, 1, bias=False),] 25 | else: 26 | layers += [nn.Upsample(scale_factor=2, mode='bilinear'), 27 | nn.Conv2d(ndf, 3, kernel_size=3, stride=1, padding=1, bias=False)] 28 | 29 | 30 | if need_sigmoid: 31 | layers += [nn.Sigmoid()] 32 | 33 | model =nn.Sequential(*layers) 34 | return model -------------------------------------------------------------------------------- /2_Super_Resolution/models/downsampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | class Downsampler(nn.Module): 6 | ''' 7 | http://www.realitypixels.com/turk/computergraphics/ResamplingFilters.pdf 8 | ''' 9 | def __init__(self, n_planes, factor, kernel_type, phase=0, kernel_width=None, support=None, sigma=None, preserve_size=False): 10 | super(Downsampler, self).__init__() 11 | 12 | assert phase in [0, 0.5], 'phase should be 0 or 0.5' 13 | 14 | if kernel_type == 'lanczos2': 15 | support = 2 16 | kernel_width = 4 * factor + 1 17 | kernel_type_ = 'lanczos' 18 | 19 | elif kernel_type == 'lanczos3': 20 | support = 3 21 | kernel_width = 6 * factor + 1 22 | kernel_type_ = 'lanczos' 23 | 24 | elif kernel_type == 'gauss12': 25 | kernel_width = 7 26 | sigma = 1/2 27 | kernel_type_ = 'gauss' 28 | 29 | elif kernel_type == 'gauss1sq2': 30 | kernel_width = 9 31 | sigma = 1./np.sqrt(2) 32 | kernel_type_ = 'gauss' 33 | 34 | elif kernel_type in ['lanczos', 'gauss', 'box']: 35 | kernel_type_ = kernel_type 36 | 37 | else: 38 | assert False, 'wrong name kernel' 39 | 40 | 41 | # note that `kernel width` will be different to actual size for phase = 1/2 42 | self.kernel = get_kernel(factor, kernel_type_, phase, kernel_width, support=support, sigma=sigma) 43 | 44 | downsampler = nn.Conv2d(n_planes, n_planes, kernel_size=self.kernel.shape, stride=factor, padding=0) 45 | downsampler.weight.data[:] = 0 46 | downsampler.bias.data[:] = 0 47 | 48 | kernel_torch = torch.from_numpy(self.kernel) 49 | for i in range(n_planes): 50 | downsampler.weight.data[i, i] = kernel_torch 51 | 52 | self.downsampler_ = downsampler 53 | 54 | if preserve_size: 55 | 56 | if self.kernel.shape[0] % 2 == 1: 57 | pad = int((self.kernel.shape[0] - 1) / 2.) 58 | else: 59 | pad = int((self.kernel.shape[0] - factor) / 2.) 60 | 61 | self.padding = nn.ReplicationPad2d(pad) 62 | 63 | self.preserve_size = preserve_size 64 | 65 | def forward(self, input): 66 | if self.preserve_size: 67 | x = self.padding(input) 68 | else: 69 | x= input 70 | self.x = x 71 | return self.downsampler_(x) 72 | 73 | def get_kernel(factor, kernel_type, phase, kernel_width, support=None, sigma=None): 74 | assert kernel_type in ['lanczos', 'gauss', 'box'] 75 | 76 | # factor = float(factor) 77 | if phase == 0.5 and kernel_type != 'box': 78 | kernel = np.zeros([kernel_width - 1, kernel_width - 1]) 79 | else: 80 | kernel = np.zeros([kernel_width, kernel_width]) 81 | 82 | 83 | if kernel_type == 'box': 84 | assert phase == 0.5, 'Box filter is always half-phased' 85 | kernel[:] = 1./(kernel_width * kernel_width) 86 | 87 | elif kernel_type == 'gauss': 88 | assert sigma, 'sigma is not specified' 89 | assert phase != 0.5, 'phase 1/2 for gauss not implemented' 90 | 91 | center = (kernel_width + 1.)/2. 92 | print(center, kernel_width) 93 | sigma_sq = sigma * sigma 94 | 95 | for i in range(1, kernel.shape[0] + 1): 96 | for j in range(1, kernel.shape[1] + 1): 97 | di = (i - center)/2. 98 | dj = (j - center)/2. 99 | kernel[i - 1][j - 1] = np.exp(-(di * di + dj * dj)/(2 * sigma_sq)) 100 | kernel[i - 1][j - 1] = kernel[i - 1][j - 1]/(2. * np.pi * sigma_sq) 101 | elif kernel_type == 'lanczos': 102 | assert support, 'support is not specified' 103 | center = (kernel_width + 1) / 2. 104 | 105 | for i in range(1, kernel.shape[0] + 1): 106 | for j in range(1, kernel.shape[1] + 1): 107 | 108 | if phase == 0.5: 109 | di = abs(i + 0.5 - center) / factor 110 | dj = abs(j + 0.5 - center) / factor 111 | else: 112 | di = abs(i - center) / factor 113 | dj = abs(j - center) / factor 114 | 115 | 116 | pi_sq = np.pi * np.pi 117 | 118 | val = 1 119 | if di != 0: 120 | val = val * support * np.sin(np.pi * di) * np.sin(np.pi * di / support) 121 | val = val / (np.pi * np.pi * di * di) 122 | 123 | if dj != 0: 124 | val = val * support * np.sin(np.pi * dj) * np.sin(np.pi * dj / support) 125 | val = val / (np.pi * np.pi * dj * dj) 126 | 127 | kernel[i - 1][j - 1] = val 128 | 129 | 130 | else: 131 | assert False, 'wrong method name' 132 | 133 | kernel /= kernel.sum() 134 | 135 | return kernel 136 | 137 | #a = Downsampler(n_planes=3, factor=2, kernel_type='lanczos2', phase='1', preserve_size=True) 138 | 139 | 140 | 141 | 142 | 143 | 144 | ################# 145 | # Learnable downsampler 146 | 147 | # KS = 32 148 | # dow = nn.Sequential(nn.ReplicationPad2d(int((KS - factor) / 2.)), nn.Conv2d(1,1,KS,factor)) 149 | 150 | # class Apply(nn.Module): 151 | # def __init__(self, what, dim, *args): 152 | # super(Apply, self).__init__() 153 | # self.dim = dim 154 | 155 | # self.what = what 156 | 157 | # def forward(self, input): 158 | # inputs = [] 159 | # for i in range(input.size(self.dim)): 160 | # inputs.append(self.what(input.narrow(self.dim, i, 1))) 161 | 162 | # return torch.cat(inputs, dim=self.dim) 163 | 164 | # def __len__(self): 165 | # return len(self._modules) 166 | 167 | # downs = Apply(dow, 1) 168 | # downs.type(dtype)(net_input.type(dtype)).size() 169 | -------------------------------------------------------------------------------- /2_Super_Resolution/models/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from numpy.random import normal 4 | from numpy.linalg import svd 5 | from math import sqrt 6 | import torch.nn.init 7 | from .common import * 8 | 9 | class ResidualSequential(nn.Sequential): 10 | def __init__(self, *args): 11 | super(ResidualSequential, self).__init__(*args) 12 | 13 | def forward(self, x): 14 | out = super(ResidualSequential, self).forward(x) 15 | # print(x.size(), out.size()) 16 | x_ = None 17 | if out.size(2) != x.size(2) or out.size(3) != x.size(3): 18 | diff2 = x.size(2) - out.size(2) 19 | diff3 = x.size(3) - out.size(3) 20 | # print(1) 21 | x_ = x[:, :, diff2 /2:out.size(2) + diff2 / 2, diff3 / 2:out.size(3) + diff3 / 2] 22 | else: 23 | x_ = x 24 | return out + x_ 25 | 26 | def eval(self): 27 | print(2) 28 | for m in self.modules(): 29 | m.eval() 30 | exit() 31 | 32 | 33 | def get_block(num_channels, norm_layer, act_fun): 34 | layers = [ 35 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False), 36 | norm_layer(num_channels, affine=True), 37 | act(act_fun), 38 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False), 39 | norm_layer(num_channels, affine=True), 40 | ] 41 | return layers 42 | 43 | 44 | class ResNet(nn.Module): 45 | def __init__(self, num_input_channels, num_output_channels, num_blocks, num_channels, need_residual=True, act_fun='LeakyReLU', need_sigmoid=True, norm_layer=nn.BatchNorm2d, pad='reflection'): 46 | ''' 47 | pad = 'start|zero|replication' 48 | ''' 49 | super(ResNet, self).__init__() 50 | 51 | if need_residual: 52 | s = ResidualSequential 53 | else: 54 | s = nn.Sequential 55 | 56 | stride = 1 57 | # First layers 58 | layers = [ 59 | # nn.ReplicationPad2d(num_blocks * 2 * stride + 3), 60 | conv(num_input_channels, num_channels, 3, stride=1, bias=True, pad=pad), 61 | act(act_fun) 62 | ] 63 | # Residual blocks 64 | # layers_residual = [] 65 | for i in range(num_blocks): 66 | layers += [s(*get_block(num_channels, norm_layer, act_fun))] 67 | 68 | layers += [ 69 | nn.Conv2d(num_channels, num_channels, 3, 1, 1), 70 | norm_layer(num_channels, affine=True) 71 | ] 72 | 73 | # if need_residual: 74 | # layers += [ResidualSequential(*layers_residual)] 75 | # else: 76 | # layers += [Sequential(*layers_residual)] 77 | 78 | # if factor >= 2: 79 | # # Do upsampling if needed 80 | # layers += [ 81 | # nn.Conv2d(num_channels, num_channels * 82 | # factor ** 2, 3, 1), 83 | # nn.PixelShuffle(factor), 84 | # act(act_fun) 85 | # ] 86 | layers += [ 87 | conv(num_channels, num_output_channels, 3, 1, bias=True, pad=pad), 88 | nn.Sigmoid() 89 | ] 90 | self.model = nn.Sequential(*layers) 91 | 92 | def forward(self, input): 93 | return self.model(input) 94 | 95 | def eval(self): 96 | self.model.eval() 97 | -------------------------------------------------------------------------------- /2_Super_Resolution/models/skip.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .common import * 4 | 5 | def skip( 6 | num_input_channels=2, num_output_channels=3, 7 | num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4], 8 | filter_size_down=3, filter_size_up=3, filter_skip_size=1, 9 | need_sigmoid=True, need_bias=True, 10 | pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU', 11 | need1x1_up=True): 12 | """Assembles encoder-decoder with skip connections. 13 | 14 | Arguments: 15 | act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU) 16 | pad (string): zero|reflection (default: 'zero') 17 | upsample_mode (string): 'nearest|bilinear' (default: 'nearest') 18 | downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride') 19 | 20 | """ 21 | assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip) 22 | 23 | n_scales = len(num_channels_down) 24 | 25 | if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) : 26 | upsample_mode = [upsample_mode]*n_scales 27 | 28 | if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)): 29 | downsample_mode = [downsample_mode]*n_scales 30 | 31 | if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) : 32 | filter_size_down = [filter_size_down]*n_scales 33 | 34 | if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) : 35 | filter_size_up = [filter_size_up]*n_scales 36 | 37 | last_scale = n_scales - 1 38 | 39 | cur_depth = None 40 | 41 | model = nn.Sequential() 42 | model_tmp = model 43 | 44 | input_depth = num_input_channels 45 | for i in range(len(num_channels_down)): 46 | 47 | deeper = nn.Sequential() 48 | skip = nn.Sequential() 49 | 50 | if num_channels_skip[i] != 0: 51 | model_tmp.add(Concat(1, skip, deeper)) 52 | else: 53 | model_tmp.add(deeper) 54 | 55 | model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i]))) 56 | 57 | if num_channels_skip[i] != 0: 58 | skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad)) 59 | skip.add(bn(num_channels_skip[i])) 60 | skip.add(act(act_fun)) 61 | 62 | # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part)) 63 | 64 | deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i])) 65 | deeper.add(bn(num_channels_down[i])) 66 | deeper.add(act(act_fun)) 67 | 68 | deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad)) 69 | deeper.add(bn(num_channels_down[i])) 70 | deeper.add(act(act_fun)) 71 | 72 | deeper_main = nn.Sequential() 73 | 74 | if i == len(num_channels_down) - 1: 75 | # The deepest 76 | k = num_channels_down[i] 77 | else: 78 | deeper.add(deeper_main) 79 | k = num_channels_up[i + 1] 80 | 81 | deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i])) 82 | 83 | model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad)) 84 | model_tmp.add(bn(num_channels_up[i])) 85 | model_tmp.add(act(act_fun)) 86 | 87 | 88 | if need1x1_up: 89 | model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad)) 90 | model_tmp.add(bn(num_channels_up[i])) 91 | model_tmp.add(act(act_fun)) 92 | 93 | input_depth = num_channels_down[i] 94 | model_tmp = deeper_main 95 | 96 | model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad)) 97 | if need_sigmoid: 98 | model.add(nn.Sigmoid()) 99 | 100 | return model 101 | -------------------------------------------------------------------------------- /2_Super_Resolution/models/texture_nets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .common import * 4 | 5 | 6 | normalization = nn.BatchNorm2d 7 | 8 | 9 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero'): 10 | if pad == 'zero': 11 | return nn.Conv2d(in_f, out_f, kernel_size, stride, padding=(kernel_size - 1) / 2, bias=bias) 12 | elif pad == 'reflection': 13 | layers = [nn.ReflectionPad2d((kernel_size - 1) / 2), 14 | nn.Conv2d(in_f, out_f, kernel_size, stride, padding=0, bias=bias)] 15 | return nn.Sequential(*layers) 16 | 17 | def get_texture_nets(inp=3, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False, pad='zero', need_sigmoid=False, conv_num=8, upsample_mode='nearest'): 18 | 19 | 20 | for i in range(len(ratios)): 21 | j = i + 1 22 | 23 | seq = nn.Sequential() 24 | 25 | tmp = nn.AvgPool2d(ratios[i], ratios[i]) 26 | 27 | seq.add(tmp) 28 | if fill_noise: 29 | seq.add(GenNoise(inp)) 30 | 31 | seq.add(conv(inp, conv_num, 3, pad=pad)) 32 | seq.add(normalization(conv_num)) 33 | seq.add(act()) 34 | 35 | seq.add(conv(conv_num, conv_num, 3, pad=pad)) 36 | seq.add(normalization(conv_num)) 37 | seq.add(act()) 38 | 39 | seq.add(conv(conv_num, conv_num, 1, pad=pad)) 40 | seq.add(normalization(conv_num)) 41 | seq.add(act()) 42 | 43 | if i == 0: 44 | seq.add(nn.Upsample(scale_factor=2, mode=upsample_mode)) 45 | cur = seq 46 | else: 47 | 48 | cur_temp = cur 49 | 50 | cur = nn.Sequential() 51 | 52 | # Batch norm before merging 53 | seq.add(normalization(conv_num)) 54 | cur_temp.add(normalization(conv_num * (j - 1))) 55 | 56 | cur.add(Concat(1, cur_temp, seq)) 57 | 58 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad)) 59 | cur.add(normalization(conv_num * j)) 60 | cur.add(act()) 61 | 62 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad)) 63 | cur.add(normalization(conv_num * j)) 64 | cur.add(act()) 65 | 66 | cur.add(conv(conv_num * j, conv_num * j, 1, pad=pad)) 67 | cur.add(normalization(conv_num * j)) 68 | cur.add(act()) 69 | 70 | if i == len(ratios) - 1: 71 | cur.add(conv(conv_num * j, 3, 1, pad=pad)) 72 | else: 73 | cur.add(nn.Upsample(scale_factor=2, mode=upsample_mode)) 74 | 75 | model = cur 76 | if need_sigmoid: 77 | model.add(nn.Sigmoid()) 78 | 79 | return model 80 | -------------------------------------------------------------------------------- /2_Super_Resolution/psnr_ssim.py: -------------------------------------------------------------------------------- 1 | ### This script will calcualte psnr and ssim 2 | import numpy as np 3 | from skimage.metrics import peak_signal_noise_ratio 4 | from skimage.measure import compare_ssim 5 | 6 | 7 | def calcualte_PSNR(im_true, im_test): 8 | im_true = np.transpose(im_true,(1,2,0)) 9 | im_test = np.transpose(im_test, (1, 2, 0)) 10 | psnr_value = peak_signal_noise_ratio(im_true, im_test) 11 | return psnr_value 12 | 13 | 14 | def calcualte_SSIM(im_true, im_test, multichannel): # multichannel=True for RGB images 15 | im_true = np.transpose(im_true, (1, 2, 0)) 16 | im_test = np.transpose(im_test, (1, 2, 0)) 17 | ssim_value = compare_ssim(im_true, im_test, multichannel=multichannel, data_range=im_test.max() - im_test.min()) 18 | return ssim_value 19 | 20 | 21 | 22 | 23 | 24 | if __name__ == '__main__': 25 | pass -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__init__.py -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/common_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/common_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/common_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/common_utils.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-37.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/__pycache__/sr_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/sr_utils.cpython-36.pyc -------------------------------------------------------------------------------- /2_Super_Resolution/utils/common_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision 4 | import sys 5 | 6 | import numpy as np 7 | from PIL import Image 8 | import PIL 9 | import numpy as np 10 | 11 | import matplotlib.pyplot as plt 12 | 13 | def crop_image(img, d=32): 14 | '''Make dimensions divisible by `d`''' 15 | 16 | new_size = (img.size[0] - img.size[0] % d, 17 | img.size[1] - img.size[1] % d) 18 | 19 | bbox = [ 20 | int((img.size[0] - new_size[0])/2), 21 | int((img.size[1] - new_size[1])/2), 22 | int((img.size[0] + new_size[0])/2), 23 | int((img.size[1] + new_size[1])/2), 24 | ] 25 | 26 | img_cropped = img.crop(bbox) 27 | return img_cropped 28 | 29 | def get_params(opt_over, net, net_input, downsampler=None): 30 | '''Returns parameters that we want to optimize over. 31 | 32 | Args: 33 | opt_over: comma separated list, e.g. "net,input" or "net" 34 | net: network 35 | net_input: torch.Tensor that stores input `z` 36 | ''' 37 | opt_over_list = opt_over.split(',') 38 | params = [] 39 | 40 | for opt in opt_over_list: 41 | 42 | if opt == 'net': 43 | params += [x for x in net.parameters() ] 44 | elif opt=='down': 45 | assert downsampler is not None 46 | params = [x for x in downsampler.parameters()] 47 | elif opt == 'input': 48 | net_input.requires_grad = True 49 | params += [net_input] 50 | else: 51 | assert False, 'what is it?' 52 | 53 | return params 54 | 55 | def get_image_grid(images_np, nrow=8): 56 | '''Creates a grid from a list of images by concatenating them.''' 57 | images_torch = [torch.from_numpy(x) for x in images_np] 58 | torch_grid = torchvision.utils.make_grid(images_torch, nrow) 59 | 60 | return torch_grid.numpy() 61 | 62 | def plot_image_grid(images_np, nrow =8, factor=1, interpolation='lanczos'): 63 | """Draws images in a grid 64 | 65 | Args: 66 | images_np: list of images, each image is np.array of size 3xHxW of 1xHxW 67 | nrow: how many images will be in one row 68 | factor: size if the plt.figure 69 | interpolation: interpolation used in plt.imshow 70 | """ 71 | n_channels = max(x.shape[0] for x in images_np) 72 | assert (n_channels == 3) or (n_channels == 1), "images should have 1 or 3 channels" 73 | 74 | images_np = [x if (x.shape[0] == n_channels) else np.concatenate([x, x, x], axis=0) for x in images_np] 75 | 76 | grid = get_image_grid(images_np, nrow) 77 | 78 | plt.figure(figsize=(len(images_np) + factor, 12 + factor)) 79 | 80 | if images_np[0].shape[0] == 1: 81 | plt.imshow(grid[0], cmap='gray', interpolation=interpolation) 82 | else: 83 | plt.imshow(grid.transpose(1, 2, 0), interpolation=interpolation) 84 | 85 | plt.show() 86 | 87 | return grid 88 | 89 | def load(path): 90 | """Load PIL image.""" 91 | img = Image.open(path) 92 | return img 93 | 94 | def get_image(path, imsize=-1): 95 | """Load an image and resize to a cpecific size. 96 | 97 | Args: 98 | path: path to image 99 | imsize: tuple or scalar with dimensions; -1 for `no resize` 100 | """ 101 | img = load(path) 102 | 103 | if isinstance(imsize, int): 104 | imsize = (imsize, imsize) 105 | 106 | if imsize[0]!= -1 and img.size != imsize: 107 | if imsize[0] > img.size[0]: 108 | img = img.resize(imsize, Image.BICUBIC) 109 | else: 110 | img = img.resize(imsize, Image.ANTIALIAS) 111 | 112 | img_np = pil_to_np(img) 113 | 114 | return img, img_np 115 | 116 | 117 | 118 | def fill_noise(x, noise_type): 119 | """Fills tensor `x` with noise of type `noise_type`.""" 120 | if noise_type == 'u': 121 | x.uniform_() 122 | elif noise_type == 'n': 123 | x.normal_() 124 | else: 125 | assert False 126 | 127 | def get_noise(input_depth, method, spatial_size, noise_type='u', var=1./10): 128 | """Returns a pytorch.Tensor of size (1 x `input_depth` x `spatial_size[0]` x `spatial_size[1]`) 129 | initialized in a specific way. 130 | Args: 131 | input_depth: number of channels in the tensor 132 | method: `noise` for fillting tensor with noise; `meshgrid` for np.meshgrid 133 | spatial_size: spatial size of the tensor to initialize 134 | noise_type: 'u' for uniform; 'n' for normal 135 | var: a factor, a noise will be multiplicated by. Basically it is standard deviation scaler. 136 | """ 137 | if isinstance(spatial_size, int): 138 | spatial_size = (spatial_size, spatial_size) 139 | if method == 'noise': 140 | shape = [1, input_depth, spatial_size[0], spatial_size[1]] 141 | net_input = torch.zeros(shape) 142 | 143 | fill_noise(net_input, noise_type) 144 | net_input *= var 145 | elif method == 'meshgrid': 146 | assert input_depth == 2 147 | X, Y = np.meshgrid(np.arange(0, spatial_size[1])/float(spatial_size[1]-1), np.arange(0, spatial_size[0])/float(spatial_size[0]-1)) 148 | meshgrid = np.concatenate([X[None,:], Y[None,:]]) 149 | net_input= np_to_torch(meshgrid) 150 | else: 151 | assert False 152 | 153 | return net_input 154 | 155 | def pil_to_np(img_PIL): 156 | '''Converts image in PIL format to np.array. 157 | 158 | From W x H x C [0...255] to C x W x H [0..1] 159 | ''' 160 | ar = np.array(img_PIL) 161 | 162 | if len(ar.shape) == 3: 163 | ar = ar.transpose(2,0,1) 164 | else: 165 | ar = ar[None, ...] 166 | 167 | return ar.astype(np.float32) / 255. 168 | 169 | def np_to_pil(img_np): 170 | '''Converts image in np.array format to PIL image. 171 | 172 | From C x W x H [0..1] to W x H x C [0...255] 173 | ''' 174 | ar = np.clip(img_np*255,0,255).astype(np.uint8) 175 | 176 | if img_np.shape[0] == 1: 177 | ar = ar[0] 178 | else: 179 | ar = ar.transpose(1, 2, 0) 180 | 181 | return Image.fromarray(ar) 182 | 183 | def np_to_torch(img_np): 184 | '''Converts image in numpy.array to torch.Tensor. 185 | 186 | From C x W x H [0..1] to C x W x H [0..1] 187 | ''' 188 | return torch.from_numpy(img_np)[None, :] 189 | 190 | def torch_to_np(img_var): 191 | '''Converts an image in torch.Tensor format to np.array. 192 | 193 | From 1 x C x W x H [0..1] to C x W x H [0..1] 194 | ''' 195 | return img_var.detach().cpu().numpy()[0] 196 | 197 | 198 | def optimize(optimizer_type, parameters, closure, LR, num_iter): 199 | """Runs optimization loop. 200 | 201 | Args: 202 | optimizer_type: 'LBFGS' of 'adam' 203 | parameters: list of Tensors to optimize over 204 | closure: function, that returns loss variable 205 | LR: learning rate 206 | num_iter: number of iterations 207 | """ 208 | if optimizer_type == 'LBFGS': 209 | # Do several steps with adam first 210 | optimizer = torch.optim.Adam(parameters, lr=0.001) 211 | for j in range(100): 212 | optimizer.zero_grad() 213 | closure() 214 | optimizer.step() 215 | 216 | print('Starting optimization with LBFGS') 217 | def closure2(): 218 | optimizer.zero_grad() 219 | return closure() 220 | optimizer = torch.optim.LBFGS(parameters, max_iter=num_iter, lr=LR, tolerance_grad=-1, tolerance_change=-1) 221 | optimizer.step(closure2) 222 | 223 | elif optimizer_type == 'adam': 224 | print('Starting optimization with ADAM') 225 | optimizer = torch.optim.Adam(parameters, lr=LR) 226 | 227 | for j in range(num_iter): 228 | optimizer.zero_grad() 229 | closure() 230 | optimizer.step() 231 | else: 232 | assert False -------------------------------------------------------------------------------- /2_Super_Resolution/utils/denoising_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from .common_utils import * 3 | 4 | 5 | 6 | def get_noisy_image(img_np, sigma): 7 | """Adds Gaussian noise to an image. 8 | 9 | Args: 10 | img_np: image, np.array with values from 0 to 1 11 | sigma: std of the noise 12 | """ 13 | img_noisy_np = np.clip(img_np + np.random.normal(scale=sigma, size=img_np.shape), 0, 1).astype(np.float32) 14 | img_noisy_pil = np_to_pil(img_noisy_np) 15 | 16 | return img_noisy_pil, img_noisy_np -------------------------------------------------------------------------------- /2_Super_Resolution/utils/feature_inversion_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.transforms as transforms 4 | import torchvision.models as models 5 | from .matcher import Matcher 6 | import os 7 | from collections import OrderedDict 8 | 9 | class View(nn.Module): 10 | def __init__(self): 11 | super(View, self).__init__() 12 | 13 | def forward(self, x): 14 | return x.view(-1) 15 | 16 | def get_vanilla_vgg_features(cut_idx=-1): 17 | if not os.path.exists('vgg_features.pth'): 18 | os.system( 19 | 'wget --no-check-certificate -N https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth') 20 | vgg_weights = torch.load('vgg19-d01eb7cb.pth') 21 | # fix compatibility issues 22 | map = {'classifier.6.weight':u'classifier.7.weight', 'classifier.6.bias':u'classifier.7.bias'} 23 | vgg_weights = OrderedDict([(map[k] if k in map else k,v) for k,v in vgg_weights.iteritems()]) 24 | 25 | 26 | 27 | model = models.vgg19() 28 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values()) 29 | 30 | 31 | model.load_state_dict(vgg_weights) 32 | 33 | torch.save(model.features, 'vgg_features.pth') 34 | torch.save(model.classifier, 'vgg_classifier.pth') 35 | 36 | vgg = torch.load('vgg_features.pth') 37 | if cut_idx > 36: 38 | vgg_classifier = torch.load('vgg_classifier.pth') 39 | vgg = nn.Sequential(*(vgg._modules.values() + vgg_classifier._modules.values())) 40 | 41 | vgg.eval() 42 | 43 | return vgg 44 | 45 | 46 | def get_matcher(net, opt): 47 | idxs = [x for x in opt['layers'].split(',')] 48 | matcher = Matcher(opt['what']) 49 | 50 | def hook(module, input, output): 51 | matcher(module, output) 52 | 53 | for i in idxs: 54 | net._modules[i].register_forward_hook(hook) 55 | 56 | return matcher 57 | 58 | 59 | 60 | def get_vgg(cut_idx=-1): 61 | f = get_vanilla_vgg_features(cut_idx) 62 | 63 | if cut_idx > 0: 64 | num_modules = len(f._modules) 65 | keys_to_delete = [f._modules.keys()[x] for x in range(cut_idx, num_modules)] 66 | for k in keys_to_delete: 67 | del f._modules[k] 68 | 69 | return f 70 | 71 | def vgg_preprocess_var(var): 72 | (r, g, b) = torch.chunk(var, 3, dim=1) 73 | bgr = torch.cat((b, g, r), 1) 74 | out = bgr * 255 - torch.autograd.Variable(vgg_mean[None, ...]).type(var.type()).expand_as(bgr) 75 | return out 76 | 77 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1) 78 | 79 | 80 | 81 | def get_preprocessor(imsize): 82 | def vgg_preprocess(tensor): 83 | (r, g, b) = torch.chunk(tensor, 3, dim=0) 84 | bgr = torch.cat((b, g, r), 0) 85 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr) 86 | return out 87 | preprocess = transforms.Compose([ 88 | transforms.Resize(imsize), 89 | transforms.ToTensor(), 90 | transforms.Lambda(vgg_preprocess) 91 | ]) 92 | 93 | return preprocess 94 | 95 | 96 | def get_deprocessor(): 97 | def vgg_deprocess(tensor): 98 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0 99 | (b, g, r) = torch.chunk(bgr, 3, dim=0) 100 | rgb = torch.cat((r, g, b), 0) 101 | return rgb 102 | deprocess = transforms.Compose([ 103 | transforms.Lambda(vgg_deprocess), 104 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)), 105 | transforms.ToPILImage() 106 | ]) 107 | return deprocess 108 | -------------------------------------------------------------------------------- /2_Super_Resolution/utils/inpainting_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | import PIL.ImageDraw as ImageDraw 4 | import PIL.ImageFont as ImageFont 5 | from .common_utils import * 6 | 7 | def get_text_mask(for_image, sz=20): 8 | font_fname = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf' 9 | font_size = sz 10 | font = ImageFont.truetype(font_fname, font_size) 11 | 12 | img_mask = Image.fromarray(np.array(for_image)*0+255) 13 | draw = ImageDraw.Draw(img_mask) 14 | draw.text((128, 128), "hello world", font=font, fill='rgb(0, 0, 0)') 15 | 16 | return img_mask 17 | 18 | def get_bernoulli_mask(for_image, zero_fraction=0.95): 19 | img_mask_np=(np.random.random_sample(size=pil_to_np(for_image).shape) > zero_fraction).astype(int) 20 | img_mask = np_to_pil(img_mask_np) 21 | 22 | return img_mask 23 | -------------------------------------------------------------------------------- /2_Super_Resolution/utils/matcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class Matcher: 5 | def __init__(self, how='gram_matrix', loss='mse'): 6 | self.mode = 'store' 7 | self.stored = {} 8 | self.losses = {} 9 | 10 | if how in all_features.keys(): 11 | self.get_statistics = all_features[how] 12 | else: 13 | assert False 14 | pass 15 | 16 | if loss in all_losses.keys(): 17 | self.loss = all_losses[loss] 18 | else: 19 | assert False 20 | 21 | def __call__(self, module, features): 22 | statistics = self.get_statistics(features) 23 | 24 | self.statistics = statistics 25 | if self.mode == 'store': 26 | self.stored[module] = statistics.detach().clone() 27 | elif self.mode == 'match': 28 | self.losses[module] = self.loss(statistics, self.stored[module]) 29 | 30 | def clean(self): 31 | self.losses = {} 32 | 33 | def gram_matrix(x): 34 | (b, ch, h, w) = x.size() 35 | features = x.view(b, ch, w * h) 36 | features_t = features.transpose(1, 2) 37 | gram = features.bmm(features_t) / (ch * h * w) 38 | return gram 39 | 40 | 41 | def features(x): 42 | return x 43 | 44 | 45 | all_features = { 46 | 'gram_matrix': gram_matrix, 47 | 'features': features, 48 | } 49 | 50 | all_losses = { 51 | 'mse': nn.MSELoss(), 52 | 'smoothL1': nn.SmoothL1Loss(), 53 | 'L1': nn.L1Loss(), 54 | } 55 | -------------------------------------------------------------------------------- /2_Super_Resolution/utils/perceptual_loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/perceptual_loss/__init__.py -------------------------------------------------------------------------------- /2_Super_Resolution/utils/perceptual_loss/matcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class Matcher: 6 | def __init__(self, how='gram_matrix', loss='mse', map_index=933): 7 | self.mode = 'store' 8 | self.stored = {} 9 | self.losses = {} 10 | 11 | if how in all_features.keys(): 12 | self.get_statistics = all_features[how] 13 | else: 14 | assert False 15 | pass 16 | 17 | if loss in all_losses.keys(): 18 | self.loss = all_losses[loss] 19 | else: 20 | assert False 21 | 22 | self.map_index = map_index 23 | self.method = 'match' 24 | 25 | 26 | def __call__(self, module, features): 27 | statistics = self.get_statistics(features) 28 | 29 | self.statistics = statistics 30 | if self.mode == 'store': 31 | self.stored[module] = statistics.detach() 32 | 33 | elif self.mode == 'match': 34 | 35 | if statistics.ndimension() == 2: 36 | 37 | if self.method == 'maximize': 38 | self.losses[module] = - statistics[0, self.map_index] 39 | else: 40 | self.losses[module] = torch.abs(300 - statistics[0, self.map_index]) 41 | 42 | else: 43 | ws = self.window_size 44 | 45 | t = statistics.detach() * 0 46 | 47 | s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0 48 | t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0 49 | t_cc[:, self.map_index,...] = 1 50 | 51 | if self.method == 'maximize': 52 | self.losses[module] = -(s_cc * t_cc.contiguous()).sum() 53 | else: 54 | self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum() 55 | 56 | 57 | def clean(self): 58 | self.losses = {} 59 | 60 | def gram_matrix(x): 61 | (b, ch, h, w) = x.size() 62 | features = x.view(b, ch, w * h) 63 | features_t = features.transpose(1, 2) 64 | gram = features.bmm(features_t) / (ch * h * w) 65 | return gram 66 | 67 | 68 | def features(x): 69 | return x 70 | 71 | 72 | all_features = { 73 | 'gram_matrix': gram_matrix, 74 | 'features': features, 75 | } 76 | 77 | all_losses = { 78 | 'mse': nn.MSELoss(), 79 | 'smoothL1': nn.SmoothL1Loss(), 80 | 'L1': nn.L1Loss(), 81 | } 82 | -------------------------------------------------------------------------------- /2_Super_Resolution/utils/perceptual_loss/vgg_modified.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | class VGGModified(nn.Module): 4 | def __init__(self, vgg19_orig, slope=0.01): 5 | super(VGGModified, self).__init__() 6 | self.features = nn.Sequential() 7 | 8 | self.features.add_module(str(0), vgg19_orig.features[0]) 9 | self.features.add_module(str(1), nn.LeakyReLU(slope, True)) 10 | self.features.add_module(str(2), vgg19_orig.features[2]) 11 | self.features.add_module(str(3), nn.LeakyReLU(slope, True)) 12 | self.features.add_module(str(4), nn.AvgPool2d((2,2), (2,2))) 13 | 14 | self.features.add_module(str(5), vgg19_orig.features[5]) 15 | self.features.add_module(str(6), nn.LeakyReLU(slope, True)) 16 | self.features.add_module(str(7), vgg19_orig.features[7]) 17 | self.features.add_module(str(8), nn.LeakyReLU(slope, True)) 18 | self.features.add_module(str(9), nn.AvgPool2d((2,2), (2,2))) 19 | 20 | self.features.add_module(str(10), vgg19_orig.features[10]) 21 | self.features.add_module(str(11), nn.LeakyReLU(slope, True)) 22 | self.features.add_module(str(12), vgg19_orig.features[12]) 23 | self.features.add_module(str(13), nn.LeakyReLU(slope, True)) 24 | self.features.add_module(str(14), vgg19_orig.features[14]) 25 | self.features.add_module(str(15), nn.LeakyReLU(slope, True)) 26 | self.features.add_module(str(16), vgg19_orig.features[16]) 27 | self.features.add_module(str(17), nn.LeakyReLU(slope, True)) 28 | self.features.add_module(str(18), nn.AvgPool2d((2,2), (2,2))) 29 | 30 | self.features.add_module(str(19), vgg19_orig.features[19]) 31 | self.features.add_module(str(20), nn.LeakyReLU(slope, True)) 32 | self.features.add_module(str(21), vgg19_orig.features[21]) 33 | self.features.add_module(str(22), nn.LeakyReLU(slope, True)) 34 | self.features.add_module(str(23), vgg19_orig.features[23]) 35 | self.features.add_module(str(24), nn.LeakyReLU(slope, True)) 36 | self.features.add_module(str(25), vgg19_orig.features[25]) 37 | self.features.add_module(str(26), nn.LeakyReLU(slope, True)) 38 | self.features.add_module(str(27), nn.AvgPool2d((2,2), (2,2))) 39 | 40 | self.features.add_module(str(28), vgg19_orig.features[28]) 41 | self.features.add_module(str(29), nn.LeakyReLU(slope, True)) 42 | self.features.add_module(str(30), vgg19_orig.features[30]) 43 | self.features.add_module(str(31), nn.LeakyReLU(slope, True)) 44 | self.features.add_module(str(32), vgg19_orig.features[32]) 45 | self.features.add_module(str(33), nn.LeakyReLU(slope, True)) 46 | self.features.add_module(str(34), vgg19_orig.features[34]) 47 | self.features.add_module(str(35), nn.LeakyReLU(slope, True)) 48 | self.features.add_module(str(36), nn.AvgPool2d((2,2), (2,2))) 49 | 50 | self.classifier = nn.Sequential() 51 | 52 | self.classifier.add_module(str(0), vgg19_orig.classifier[0]) 53 | self.classifier.add_module(str(1), nn.LeakyReLU(slope, True)) 54 | self.classifier.add_module(str(2), nn.Dropout2d(p = 0.5)) 55 | self.classifier.add_module(str(3), vgg19_orig.classifier[3]) 56 | self.classifier.add_module(str(4), nn.LeakyReLU(slope, True)) 57 | self.classifier.add_module(str(5), nn.Dropout2d(p = 0.5)) 58 | self.classifier.add_module(str(6), vgg19_orig.classifier[6]) 59 | 60 | def forward(self, x): 61 | return self.classifier(self.features.forward(x)) -------------------------------------------------------------------------------- /2_Super_Resolution/utils/sr_utils.py: -------------------------------------------------------------------------------- 1 | from .common_utils import * 2 | 3 | def put_in_center(img_np, target_size): 4 | img_out = np.zeros([3, target_size[0], target_size[1]]) 5 | 6 | bbox = [ 7 | int((target_size[0] - img_np.shape[1]) / 2), 8 | int((target_size[1] - img_np.shape[2]) / 2), 9 | int((target_size[0] + img_np.shape[1]) / 2), 10 | int((target_size[1] + img_np.shape[2]) / 2), 11 | ] 12 | 13 | img_out[:, bbox[0]:bbox[2], bbox[1]:bbox[3]] = img_np 14 | 15 | return img_out 16 | 17 | 18 | def load_LR_HR_imgs_sr(fname, imsize, factor, enforse_div32=None): 19 | '''Loads an image, resizes it, center crops and downscales. 20 | 21 | Args: 22 | fname: path to the image 23 | imsize: new size for the image, -1 for no resizing 24 | factor: downscaling factor 25 | enforse_div32: if 'CROP' center crops an image, so that its dimensions are divisible by 32. 26 | ''' 27 | img_orig_pil, img_orig_np = get_image(fname, -1) 28 | 29 | if imsize != -1: 30 | img_orig_pil, img_orig_np = get_image(fname, imsize) 31 | 32 | # For comparison with GT 33 | if enforse_div32 == 'CROP': 34 | new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 32, 35 | img_orig_pil.size[1] - img_orig_pil.size[1] % 32) 36 | 37 | bbox = [ 38 | (img_orig_pil.size[0] - new_size[0])/2, 39 | (img_orig_pil.size[1] - new_size[1])/2, 40 | (img_orig_pil.size[0] + new_size[0])/2, 41 | (img_orig_pil.size[1] + new_size[1])/2, 42 | ] 43 | 44 | img_HR_pil = img_orig_pil.crop(bbox) 45 | img_HR_np = pil_to_np(img_HR_pil) 46 | else: 47 | img_HR_pil, img_HR_np = img_orig_pil, img_orig_np 48 | 49 | LR_size = [ 50 | img_HR_pil.size[0] // factor, 51 | img_HR_pil.size[1] // factor 52 | ] 53 | 54 | img_LR_pil = img_HR_pil.resize(LR_size, Image.ANTIALIAS) 55 | img_LR_np = pil_to_np(img_LR_pil) 56 | 57 | print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str (img_LR_pil.size))) 58 | 59 | return { 60 | 'orig_pil': img_orig_pil, 61 | 'orig_np': img_orig_np, 62 | 'LR_pil': img_LR_pil, 63 | 'LR_np': img_LR_np, 64 | 'HR_pil': img_HR_pil, 65 | 'HR_np': img_HR_np 66 | } 67 | 68 | 69 | def get_baselines(img_LR_pil, img_HR_pil): 70 | '''Gets `bicubic`, sharpened bicubic and `nearest` baselines.''' 71 | img_bicubic_pil = img_LR_pil.resize(img_HR_pil.size, Image.BICUBIC) 72 | img_bicubic_np = pil_to_np(img_bicubic_pil) 73 | 74 | img_nearest_pil = img_LR_pil.resize(img_HR_pil.size, Image.NEAREST) 75 | img_nearest_np = pil_to_np(img_nearest_pil) 76 | 77 | img_bic_sharp_pil = img_bicubic_pil.filter(PIL.ImageFilter.UnsharpMask()) 78 | img_bic_sharp_np = pil_to_np(img_bic_sharp_pil) 79 | 80 | return img_bicubic_np, img_bic_sharp_np, img_nearest_np 81 | 82 | 83 | 84 | def tv_loss(x, beta = 0.5): 85 | '''Calculates TV loss for an image `x`. 86 | 87 | Args: 88 | x: image, torch.Variable of torch.Tensor 89 | beta: See https://arxiv.org/abs/1412.0035 (fig. 2) to see effect of `beta` 90 | ''' 91 | dh = torch.pow(x[:,:,:,1:] - x[:,:,:,:-1], 2) 92 | dw = torch.pow(x[:,:,1:,:] - x[:,:,:-1,:], 2) 93 | 94 | return torch.sum(torch.pow(dh[:, :, :-1] + dw[:, :, :, :-1], beta)) 95 | -------------------------------------------------------------------------------- /3_Inpainting/BN_Net.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | 6 | #----------------------------------------------------------- 7 | #----------------------------------------------------------- 8 | # BN for the input seed 9 | #----------------------------------------------------------- 10 | #----------------------------------------------------------- 11 | class BNNet(nn.Module): 12 | def __init__(self,num_channel): 13 | super(BNNet, self).__init__() 14 | self.bn = nn.BatchNorm2d(num_channel) 15 | 16 | def forward(self, input_data): 17 | output_data = self.bn(input_data) 18 | return output_data 19 | -------------------------------------------------------------------------------- /3_Inpainting/Main_Start.py: -------------------------------------------------------------------------------- 1 | from train_ip import * 2 | import time 3 | import datetime 4 | from include import * 5 | 6 | 7 | 8 | 9 | #----------------------------------------------------------- 10 | #----------------------------------------------------------- 11 | # This is the Main of the code 12 | #----------------------------------------------------------- 13 | #----------------------------------------------------------- 14 | 15 | if __name__ == '__main__': 16 | # Set random seed for reproducibility 17 | manualSeed = 100 18 | manualSeed = random.randint(1, 10000) # use if you want new results 19 | print("Random Seed: ", manualSeed) 20 | random.seed(manualSeed) 21 | torch.manual_seed(manualSeed) 22 | 23 | ##### Set up hyper-parameters 24 | #----------------------------------------------------------- 25 | gpu = 2 26 | 27 | print_step = 1 28 | max_epoch = 5000 29 | max_epoch = int(max_epoch / print_step) * print_step + 1 30 | 31 | 32 | tv_weight = 1e-2 33 | learning_rate_model = 0.05 34 | 35 | OPTIMIZER = 'Adam' 36 | LOSS = "MSE"#'Huber' or "L1" 37 | 38 | IP_ratio = 0.1 39 | 40 | input_w = 512 41 | input_h = 512 42 | input_c = 2 43 | width = 128 44 | #----------------------------------------------------------- 45 | 46 | # image_list = ['Peppers'] 47 | 48 | #image_list = ['couple','fingerprint', 'hill', 'house', 'Lena512','man', 'montage', 'peppers256'] 49 | 50 | image_list = ['barbara'] 51 | 52 | ###################################### Processing images one by one ################################# 53 | for cur_image in image_list: 54 | path = '../0_Dataset/Inpainting/0_IP11/IP_Dataset_{}/'.format(IP_ratio) 55 | 56 | # load clean image 57 | img_path = path + cur_image + "_img_{}.png".format(IP_ratio) 58 | img_pil = Image.open(img_path) 59 | img_np = pil_to_np(img_pil) 60 | 61 | # ---------------------------- 62 | # ----- Clean Var----- 63 | # ---------------------------- 64 | img_clean_var = np_to_var(img_np).type(dtype) 65 | 66 | output_depth = img_np.shape[0] 67 | 68 | # load its mask 69 | mask_path = path + cur_image + '_mask_{}.png'.format(IP_ratio) 70 | img_mask_pil = Image.open(mask_path) 71 | mask_np = pil_to_np(img_mask_pil) 72 | mask_np = np.array([mask_np[0, :, :] / np.max(mask_np)] * output_depth) 73 | 74 | # ---------------------------- 75 | # ----- Mask Var----- 76 | # ---------------------------- 77 | mask_var = np_to_var(mask_np).type(dtype) 78 | 79 | ##################Generate inpainted image 80 | # ---------------------------- 81 | # ----- Noisy Var----- 82 | # ---------------------------- 83 | img_noisy_var = img_clean_var * mask_var 84 | img_noisy_np = var_to_np(img_noisy_var) 85 | 86 | 87 | DIP_train(img_clean_var, 88 | img_noisy_var, 89 | mask_var, 90 | learning_rate_model, 91 | OPTIMIZER, 92 | LOSS, 93 | width, 94 | input_w, 95 | input_h, 96 | input_c, 97 | max_epoch, 98 | print_step, 99 | gpu, 100 | cur_image, 101 | tv_weight 102 | ) 103 | -------------------------------------------------------------------------------- /3_Inpainting/include/__init__.py: -------------------------------------------------------------------------------- 1 | from .wavelet import * 2 | from .decoder import * 3 | from .visualize import * 4 | from .fit import * 5 | from .helpers import * 6 | from .compression import * -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/compression.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/compression.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/decoder.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/decoder.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/fit.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/fit.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/helpers.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/helpers.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/visualize.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/visualize.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/__pycache__/wavelet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/wavelet.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/include/compression.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Variable 2 | import torch 3 | import torch.optim 4 | import copy 5 | import numpy as np 6 | 7 | from .helpers import * 8 | from .decoder import * 9 | from .fit import * 10 | from .wavelet import * 11 | 12 | def rep_error_deep_decoder(img_np,k=128,convert2ycbcr=False): 13 | ''' 14 | mse obtained by representing img_np with the deep decoder 15 | ''' 16 | output_depth = img_np.shape[0] 17 | if output_depth == 3 and convert2ycbcr: 18 | img = rgb2ycbcr(img_np) 19 | else: 20 | img = img_np 21 | img_var = np_to_var(img).type(dtype) 22 | 23 | num_channels = [k]*5 24 | net = decodernwv2(output_depth,num_channels_up=num_channels,bn_before_act=True).type(dtype) 25 | rnd = 500 26 | numit = 15000 27 | rn = 0.005 28 | mse_n, mse_t, ni, net = fit( num_channels=num_channels, 29 | reg_noise_std=rn, 30 | reg_noise_decayevery = rnd, 31 | num_iter=numit, 32 | LR=0.004, 33 | img_noisy_var=img_var, 34 | net=net, 35 | img_clean_var=img_var, 36 | find_best=True, 37 | ) 38 | out_img = net(ni.type(dtype)).data.cpu().numpy()[0] 39 | if output_depth == 3 and convert2ycbcr: 40 | out_img = ycbcr2rgb(out_img) 41 | return psnr(out_img,img_np), out_img, num_param(net) 42 | 43 | def rep_error_wavelet(img_np,ncoeff=300): 44 | ''' 45 | mse obtained by representing img_np with wavelet thresholding 46 | ncoff coefficients are retained per color channel 47 | ''' 48 | if img_np.shape[0] == 1: 49 | img_np = img_np[0,:,:] 50 | out_img_np = denoise_wavelet(img_np, ncoeff=ncoeff, multichannel=False, convert2ycbcr=True, mode='hard') 51 | else: 52 | img_np = np.transpose(img_np) 53 | out_img_np = denoise_wavelet(img_np, ncoeff=ncoeff, multichannel=True, convert2ycbcr=True, mode='hard') 54 | # img_np = np.array([img_np[:,:,0],img_np[:,:,1],img_np[:,:,2]]) 55 | return psnr(out_img_np,img_np), out_img_np 56 | 57 | def myimgshow(plt,img): 58 | if(img.shape[0] == 1): 59 | plt.imshow(np.clip(img[0],0,1),cmap='Greys',interpolation='none') 60 | else: 61 | plt.imshow(np.clip(img.transpose(1, 2, 0),0,1),interpolation='none') 62 | 63 | -------------------------------------------------------------------------------- /3_Inpainting/include/fit.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Variable 2 | import torch 3 | import torch.optim 4 | import copy 5 | import numpy as np 6 | from scipy.linalg import hadamard 7 | 8 | from .helpers import * 9 | 10 | dtype = torch.cuda.FloatTensor 11 | #dtype = torch.FloatTensor 12 | 13 | 14 | def exp_lr_scheduler(optimizer, epoch, init_lr=0.001, lr_decay_epoch=500): 15 | """Decay learning rate by a factor of 0.1 every lr_decay_epoch epochs.""" 16 | lr = init_lr * (0.65**(epoch // lr_decay_epoch)) 17 | 18 | if epoch % lr_decay_epoch == 0: 19 | print('LR is set to {}'.format(lr)) 20 | 21 | for param_group in optimizer.param_groups: 22 | param_group['lr'] = lr 23 | 24 | return optimizer 25 | 26 | 27 | 28 | def fit(net, 29 | img_noisy_var, 30 | num_channels, 31 | img_clean_var, 32 | num_iter = 5000, 33 | LR = 0.01, 34 | OPTIMIZER='adam', 35 | opt_input = False, 36 | reg_noise_std = 0, 37 | reg_noise_decayevery = 100000, 38 | mask_var = None, 39 | apply_f = None, 40 | lr_decay_epoch = 0, 41 | net_input = None, 42 | net_input_gen = "random", 43 | find_best=False, 44 | weight_decay=0, 45 | ): 46 | 47 | if net_input is not None: 48 | print("input provided") 49 | else: 50 | # feed uniform noise into the network 51 | totalupsample = 2**len(num_channels) 52 | width = int(img_clean_var.data.shape[2]/totalupsample) 53 | height = int(img_clean_var.data.shape[3]/totalupsample) 54 | shape = [1,num_channels[0], width, height] 55 | print("shape: ", shape) 56 | net_input = Variable(torch.zeros(shape)) 57 | net_input.data.uniform_() 58 | net_input.data *= 1./10 59 | 60 | net_input_saved = net_input.data.clone() 61 | noise = net_input.data.clone() 62 | p = [x for x in net.parameters() ] 63 | 64 | if(opt_input == True): # optimizer over the input as well 65 | net_input.requires_grad = True 66 | p += [net_input] 67 | 68 | mse_wrt_noisy = np.zeros(num_iter) 69 | mse_wrt_truth = np.zeros(num_iter) 70 | 71 | if OPTIMIZER == 'SGD': 72 | print("optimize with SGD", LR) 73 | optimizer = torch.optim.SGD(p, lr=LR,momentum=0.9,weight_decay=weight_decay) 74 | elif OPTIMIZER == 'adam': 75 | print("optimize with adam", LR) 76 | optimizer = torch.optim.Adam(p, lr=LR,weight_decay=weight_decay) 77 | elif OPTIMIZER == 'LBFGS': 78 | print("optimize with LBFGS", LR) 79 | optimizer = torch.optim.LBFGS(p, lr=LR) 80 | 81 | mse = torch.nn.MSELoss() #.type(dtype) 82 | noise_energy = mse(img_noisy_var, img_clean_var) 83 | 84 | if find_best: 85 | best_net = copy.deepcopy(net) 86 | best_mse = 1000000.0 87 | 88 | for i in range(num_iter): 89 | 90 | if lr_decay_epoch is not 0: 91 | optimizer = exp_lr_scheduler(optimizer, i, init_lr=LR, lr_decay_epoch=lr_decay_epoch) 92 | if reg_noise_std > 0: 93 | if i % reg_noise_decayevery == 0: 94 | reg_noise_std *= 0.7 95 | net_input = Variable(net_input_saved + (noise.normal_() * reg_noise_std)) 96 | 97 | def closure(): 98 | optimizer.zero_grad() 99 | out = net(net_input.type(dtype)) 100 | 101 | # training loss 102 | if mask_var is not None: 103 | loss = mse( out * mask_var , img_noisy_var * mask_var ) 104 | elif apply_f: 105 | loss = mse( apply_f(out) , img_noisy_var ) 106 | else: 107 | loss = mse(out, img_noisy_var) 108 | 109 | loss.backward() 110 | mse_wrt_noisy[i] = loss.data.cpu().numpy() 111 | 112 | 113 | # the actual loss 114 | true_loss = mse(Variable(out.data, requires_grad=False), img_clean_var) 115 | mse_wrt_truth[i] = true_loss.data.cpu().numpy() 116 | if i % 10 == 0: 117 | out2 = net(Variable(net_input_saved).type(dtype)) 118 | loss2 = mse(out2, img_clean_var) 119 | print ('Iteration %05d Train loss %f Actual loss %f Actual loss orig %f Noise Energy %f' % (i, loss.data,true_loss.data,loss2.data,noise_energy.data), '\r', end='') 120 | return loss 121 | 122 | 123 | #if OPTIMIZER == 'LBFGS': 124 | # if i < 100: 125 | # optimizer = torch.optim.Adam(p, lr=LR) 126 | # else: 127 | # optimizer = torch.optim.LBFGS(p, lr=LR) 128 | 129 | 130 | loss = optimizer.step(closure) 131 | 132 | if find_best: 133 | # if training loss improves by at least one percent, we found a new best net 134 | if best_mse > 1.005*loss.data: 135 | best_mse = loss.data 136 | best_net = copy.deepcopy(net) 137 | 138 | 139 | if find_best: 140 | net = best_net 141 | return mse_wrt_noisy, mse_wrt_truth,net_input_saved, net 142 | 143 | 144 | 145 | 146 | 147 | ### weight regularization 148 | #if orth_reg > 0: 149 | # for name, param in net.named_parameters(): 150 | # consider all the conv weights, but the last one which only combines colors 151 | # if '.1.weight' in name and str( len(net)-1 ) not in name: 152 | # param_flat = param.view(param.shape[0], -1) 153 | # sym = torch.mm(param_flat, torch.t(param_flat)) 154 | # sym -= Variable(torch.eye(param_flat.shape[0])).type(dtype) 155 | # loss = loss + (orth_reg * sym.sum().type(dtype) ) 156 | ### -------------------------------------------------------------------------------- /3_Inpainting/include/helpers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision 4 | import sys 5 | 6 | import numpy as np 7 | from PIL import Image 8 | import PIL 9 | import numpy as np 10 | 11 | from torch.autograd import Variable 12 | 13 | 14 | 15 | 16 | 17 | import numpy as np 18 | import torch 19 | import matplotlib.pyplot as plt 20 | 21 | from PIL import Image 22 | import PIL 23 | 24 | from torch.autograd import Variable 25 | 26 | def load_and_crop(imgname,target_width=512,target_height=512): 27 | ''' 28 | imgname: string of image location 29 | load an image, and center-crop if the image is large enough, else return none 30 | ''' 31 | img = Image.open(imgname) 32 | width, height = img.size 33 | if width <= target_width or height <= target_height: 34 | return None 35 | 36 | left = (width - target_width)/2 37 | top = (height - target_height)/2 38 | right = (width + target_width)/2 39 | bottom = (height + target_height)/2 40 | 41 | return img.crop((left, top, right, bottom)) 42 | 43 | def save_np_img(img,filename): 44 | if(img.shape[0] == 1): 45 | plt.imshow(np.clip(img[0],0,1),cmap='Greys',interpolation='nearest') 46 | else: 47 | plt.imshow(np.clip(img.transpose(1, 2, 0),0,1)) 48 | plt.axis('off') 49 | plt.savefig(filename, bbox_inches='tight') 50 | plt.close() 51 | 52 | def np_to_tensor(img_np): 53 | '''Converts image in numpy.array to torch.Tensor. 54 | 55 | From C x W x H [0..1] to C x W x H [0..1] 56 | ''' 57 | return torch.from_numpy(img_np) 58 | 59 | def np_to_var(img_np, dtype = torch.cuda.FloatTensor): 60 | '''Converts image in numpy.array to torch.Variable. 61 | 62 | From C x W x H [0..1] to 1 x C x W x H [0..1] 63 | ''' 64 | return Variable(np_to_tensor(img_np)[None, :]) 65 | 66 | def var_to_np(img_var): 67 | '''Converts an image in torch.Variable format to np.array. 68 | 69 | From 1 x C x W x H [0..1] to C x W x H [0..1] 70 | ''' 71 | return img_var.data.cpu().numpy()[0] 72 | 73 | 74 | def pil_to_np(img_PIL): 75 | '''Converts image in PIL format to np.array. 76 | 77 | From W x H x C [0...255] to C x W x H [0..1] 78 | ''' 79 | ar = np.array(img_PIL) 80 | 81 | if len(ar.shape) == 3: 82 | ar = ar.transpose(2,0,1) 83 | else: 84 | ar = ar[None, ...] 85 | 86 | return ar.astype(np.float32) / 255. 87 | 88 | 89 | def rgb2ycbcr(img): 90 | #out = color.rgb2ycbcr( img.transpose(1, 2, 0) ) 91 | #return out.transpose(2,0,1)/256. 92 | r,g,b = img[0],img[1],img[2] 93 | y = 0.299*r+0.587*g+0.114*b 94 | cb = 0.5 - 0.168736*r - 0.331264*g + 0.5*b 95 | cr = 0.5 + 0.5*r - 0.418588*g - 0.081312*b 96 | return np.array([y,cb,cr]) 97 | 98 | def ycbcr2rgb(img): 99 | #out = color.ycbcr2rgb( 256.*img.transpose(1, 2, 0) ) 100 | #return (out.transpose(2,0,1) - np.min(out))/(np.max(out)-np.min(out)) 101 | y,cb,cr = img[0],img[1],img[2] 102 | r = y + 1.402*(cr-0.5) 103 | g = y - 0.344136*(cb-0.5) - 0.714136*(cr-0.5) 104 | b = y + 1.772*(cb - 0.5) 105 | return np.array([r,g,b]) 106 | 107 | 108 | 109 | def mse(x_hat,x_true,maxv=1.): 110 | x_hat = x_hat.flatten() 111 | x_true = x_true.flatten() 112 | mse = np.mean(np.square(x_hat-x_true)) 113 | energy = np.mean(np.square(x_true)) 114 | return mse/energy 115 | 116 | def psnr(x_hat,x_true,maxv=1.): 117 | x_hat = x_hat.flatten() 118 | x_true = x_true.flatten() 119 | mse=np.mean(np.square(x_hat-x_true)) 120 | psnr_ = 10.*np.log(maxv**2/mse)/np.log(10.) 121 | return psnr_ 122 | 123 | def num_param(net): 124 | s = sum([np.prod(list(p.size())) for p in net.parameters()]); 125 | return s 126 | #print('Number of params: %d' % s) 127 | 128 | def rgb2gray(rgb): 129 | r, g, b = rgb[0,:,:], rgb[1,:,:], rgb[2,:,:] 130 | gray = 0.2989 * r + 0.5870 * g + 0.1140 * b 131 | return np.array([gray]) 132 | 133 | def savemtx_for_logplot(A,filename = "exp.dat"): 134 | ind = sorted(list(set([int(i) for i in np.geomspace(1, len(A[0])-1 ,num=700)]))) 135 | A = [ [a[i] for i in ind] for a in A] 136 | X = np.array([ind] + A) 137 | np.savetxt(filename, X.T, delimiter=' ') 138 | -------------------------------------------------------------------------------- /3_Inpainting/include/visualize.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | from torch.autograd import Variable 3 | import torch 4 | import torch.optim 5 | import numpy as np 6 | from collections import Iterable 7 | 8 | 9 | dtype = torch.cuda.FloatTensor 10 | #dtype = torch.FloatTensor 11 | 12 | def save_np_img(img,filename): 13 | if(img.shape[0] == 1): 14 | plt.imshow(np.clip(img[0],0,1)) 15 | else: 16 | plt.imshow(np.clip(img.transpose(1, 2, 0),0,1)) 17 | plt.axis('off') 18 | plt.savefig(filename, bbox_inches='tight') 19 | plt.close() 20 | 21 | def apply_until(net_input,net,n = 100): 22 | # applies function by funtion of a network 23 | for i,fun in enumerate(net): 24 | if i>=n: 25 | break 26 | if i==0: 27 | out = fun(net_input.type(dtype)) 28 | else: 29 | out = fun(out) 30 | print(i, "last func. applied:", net[i-1]) 31 | if n == 0: 32 | return net_input 33 | else: 34 | return out 35 | 36 | 37 | from math import ceil 38 | 39 | 40 | # given a lists of images as np-arrays, plot them as a row# given 41 | def plot_image_grid(imgs,nrows=10): 42 | ncols = ceil( len(imgs)/nrows ) 43 | nrows = min(nrows,len(imgs)) 44 | fig, axes = plt.subplots(nrows=nrows, ncols=ncols, sharex=True, sharey=True,figsize=(ncols,nrows),squeeze=False) 45 | for i, row in enumerate(axes): 46 | for j, ax in enumerate(row): 47 | ax.imshow(imgs[j*nrows+i], cmap='Greys_r', interpolation='none') 48 | ax.get_xaxis().set_visible(False) 49 | ax.get_yaxis().set_visible(False) 50 | fig.tight_layout(pad=0.1) 51 | return fig 52 | 53 | def save_tensor(out,filename,nrows=8): 54 | imgs = [img for img in out.data.cpu().numpy()[0]] 55 | fig = plot_image_grid(imgs,nrows=nrows) 56 | plt.savefig(filename) 57 | plt.close() 58 | 59 | -------------------------------------------------------------------------------- /3_Inpainting/include/wavelet.py: -------------------------------------------------------------------------------- 1 | #import matplotlib.pyplot as plt 2 | import numpy as np 3 | import numbers 4 | import pywt 5 | import scipy 6 | import skimage.color as color 7 | from skimage.restoration import (denoise_wavelet, estimate_sigma) 8 | from skimage import data, img_as_float 9 | from skimage.util import random_noise 10 | from skimage.measure import compare_psnr 11 | from include import * 12 | 13 | def _wavelet_threshold(image, wavelet, ncoeff = None, threshold=None, mode='soft', wavelet_levels=None): 14 | 15 | wavelet = pywt.Wavelet(wavelet) 16 | 17 | # original_extent is used to workaround PyWavelets issue #80 18 | # odd-sized input results in an image with 1 extra sample after waverecn 19 | original_extent = [slice(s) for s in image.shape] 20 | 21 | # Determine the number of wavelet decomposition levels 22 | if wavelet_levels is None: 23 | # Determine the maximum number of possible levels for image 24 | dlen = wavelet.dec_len 25 | wavelet_levels = np.min( 26 | [pywt.dwt_max_level(s, dlen) for s in image.shape]) 27 | 28 | # Skip coarsest wavelet scales (see Notes in docstring). 29 | wavelet_levels = max(wavelet_levels - 3, 1) 30 | 31 | coeffs = pywt.wavedecn(image, wavelet=wavelet, level=wavelet_levels) 32 | # Detail coefficients at each decomposition level 33 | dcoeffs = coeffs[1:] 34 | 35 | a = [] 36 | for level in dcoeffs: 37 | for key in level: 38 | a += [np.ndarray.flatten(level[key])] 39 | a = np.concatenate(a) 40 | a = np.sort( np.abs(a) ) 41 | 42 | sh = coeffs[0].shape 43 | basecoeffs = sh[0]*sh[1] 44 | threshold = a[- (ncoeff - basecoeffs)] 45 | 46 | # A single threshold for all coefficient arrays 47 | denoised_detail = [{key: pywt.threshold(level[key],value=threshold, 48 | mode=mode) for key in level} for level in dcoeffs] 49 | 50 | denoised_coeffs = [coeffs[0]] + denoised_detail 51 | return pywt.waverecn(denoised_coeffs, wavelet)[original_extent] 52 | 53 | 54 | def denoise_wavelet(image, ncoeff=None, wavelet='db1', mode='hard', 55 | wavelet_levels=None, multichannel=False, 56 | convert2ycbcr=False): 57 | 58 | image = img_as_float(image) 59 | 60 | 61 | if multichannel: 62 | if convert2ycbcr: 63 | out = color.rgb2ycbcr(image) 64 | for i in range(3): 65 | # renormalizing this color channel to live in [0, 1] 66 | min, max = out[..., i].min(), out[..., i].max() 67 | channel = out[..., i] - min 68 | channel /= max - min 69 | out[..., i] = denoise_wavelet(channel, wavelet=wavelet,ncoeff=ncoeff, 70 | mode=mode, 71 | wavelet_levels=wavelet_levels) 72 | 73 | out[..., i] = out[..., i] * (max - min) 74 | out[..., i] += min 75 | out = color.ycbcr2rgb(out) 76 | else: 77 | out = np.empty_like(image) 78 | for c in range(image.shape[-1]): 79 | out[..., c] = _wavelet_threshold(image[..., c],ncoeff=ncoeff, 80 | wavelet=wavelet, mode=mode, 81 | wavelet_levels=wavelet_levels) 82 | else: 83 | out = _wavelet_threshold(image, wavelet=wavelet, mode=mode,ncoeff=ncoeff, 84 | wavelet_levels=wavelet_levels) 85 | 86 | clip_range = (-1, 1) if image.min() < 0 else (0, 1) 87 | return np.clip(out, *clip_range) 88 | 89 | 90 | -------------------------------------------------------------------------------- /3_Inpainting/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .skip import skip 2 | from .texture_nets import get_texture_nets 3 | from .resnet import ResNet 4 | from .unet import UNet 5 | 6 | import torch.nn as nn 7 | 8 | def get_net(input_depth, NET_TYPE, pad, upsample_mode, n_channels=3, act_fun='LeakyReLU', skip_n33d=128, skip_n33u=128, skip_n11=4, num_scales=5, downsample_mode='stride'): 9 | if NET_TYPE == 'ResNet': 10 | # TODO 11 | net = ResNet(input_depth, 3, 10, 16, 1, nn.BatchNorm2d, False) 12 | elif NET_TYPE == 'skip': 13 | net = skip(input_depth, n_channels, num_channels_down = [skip_n33d]*num_scales if isinstance(skip_n33d, int) else skip_n33d, 14 | num_channels_up = [skip_n33u]*num_scales if isinstance(skip_n33u, int) else skip_n33u, 15 | num_channels_skip = [skip_n11]*num_scales if isinstance(skip_n11, int) else skip_n11, 16 | upsample_mode=upsample_mode, downsample_mode=downsample_mode, 17 | need_sigmoid=True, need_bias=True, pad=pad, act_fun=act_fun) 18 | 19 | elif NET_TYPE == 'texture_nets': 20 | net = get_texture_nets(inp=input_depth, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False,pad=pad) 21 | 22 | elif NET_TYPE =='UNet': 23 | net = UNet(num_input_channels=input_depth, num_output_channels=3, 24 | feature_scale=4, more_layers=0, concat_x=False, 25 | upsample_mode=upsample_mode, pad=pad, norm_layer=nn.BatchNorm2d, need_sigmoid=True, need_bias=True) 26 | elif NET_TYPE == 'identity': 27 | assert input_depth == 3 28 | net = nn.Sequential() 29 | else: 30 | assert False 31 | 32 | return net -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/common.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/common.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/common.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/common.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/downsampler.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/downsampler.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/downsampler.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/downsampler.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/resnet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/resnet.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/skip.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/skip.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/skip.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/skip.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/texture_nets.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/texture_nets.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/texture_nets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/texture_nets.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/unet.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/unet.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/__pycache__/unet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/unet.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/models/common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from .downsampler import Downsampler 5 | 6 | def add_module(self, module): 7 | self.add_module(str(len(self) + 1), module) 8 | 9 | torch.nn.Module.add = add_module 10 | 11 | class Concat(nn.Module): 12 | def __init__(self, dim, *args): 13 | super(Concat, self).__init__() 14 | self.dim = dim 15 | 16 | for idx, module in enumerate(args): 17 | self.add_module(str(idx), module) 18 | 19 | def forward(self, input): 20 | inputs = [] 21 | for module in self._modules.values(): 22 | inputs.append(module(input)) 23 | 24 | inputs_shapes2 = [x.shape[2] for x in inputs] 25 | inputs_shapes3 = [x.shape[3] for x in inputs] 26 | 27 | if np.all(np.array(inputs_shapes2) == min(inputs_shapes2)) and np.all(np.array(inputs_shapes3) == min(inputs_shapes3)): 28 | inputs_ = inputs 29 | else: 30 | target_shape2 = min(inputs_shapes2) 31 | target_shape3 = min(inputs_shapes3) 32 | 33 | inputs_ = [] 34 | for inp in inputs: 35 | diff2 = (inp.size(2) - target_shape2) // 2 36 | diff3 = (inp.size(3) - target_shape3) // 2 37 | inputs_.append(inp[:, :, diff2: diff2 + target_shape2, diff3:diff3 + target_shape3]) 38 | 39 | return torch.cat(inputs_, dim=self.dim) 40 | 41 | def __len__(self): 42 | return len(self._modules) 43 | 44 | 45 | class GenNoise(nn.Module): 46 | def __init__(self, dim2): 47 | super(GenNoise, self).__init__() 48 | self.dim2 = dim2 49 | 50 | def forward(self, input): 51 | a = list(input.size()) 52 | a[1] = self.dim2 53 | # print (input.data.type()) 54 | 55 | b = torch.zeros(a).type_as(input.data) 56 | b.normal_() 57 | 58 | x = torch.autograd.Variable(b) 59 | 60 | return x 61 | 62 | 63 | class Swish(nn.Module): 64 | """ 65 | https://arxiv.org/abs/1710.05941 66 | The hype was so huge that I could not help but try it 67 | """ 68 | def __init__(self): 69 | super(Swish, self).__init__() 70 | self.s = nn.Sigmoid() 71 | 72 | def forward(self, x): 73 | return x * self.s(x) 74 | 75 | 76 | def act(act_fun = 'LeakyReLU'): 77 | ''' 78 | Either string defining an activation function or module (e.g. nn.ReLU) 79 | ''' 80 | if isinstance(act_fun, str): 81 | if act_fun == 'LeakyReLU': 82 | return nn.LeakyReLU(0.2, inplace=True) 83 | elif act_fun == 'Swish': 84 | return Swish() 85 | elif act_fun == 'ELU': 86 | return nn.ELU() 87 | elif act_fun == 'none': 88 | return nn.Sequential() 89 | else: 90 | assert False 91 | else: 92 | return act_fun() 93 | 94 | 95 | def bn(num_features): 96 | return nn.BatchNorm2d(num_features) 97 | 98 | 99 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero', downsample_mode='stride'): 100 | downsampler = None 101 | if stride != 1 and downsample_mode != 'stride': 102 | 103 | if downsample_mode == 'avg': 104 | downsampler = nn.AvgPool2d(stride, stride) 105 | elif downsample_mode == 'max': 106 | downsampler = nn.MaxPool2d(stride, stride) 107 | elif downsample_mode in ['lanczos2', 'lanczos3']: 108 | downsampler = Downsampler(n_planes=out_f, factor=stride, kernel_type=downsample_mode, phase=0.5, preserve_size=True) 109 | else: 110 | assert False 111 | 112 | stride = 1 113 | 114 | padder = None 115 | to_pad = int((kernel_size - 1) / 2) 116 | if pad == 'reflection': 117 | padder = nn.ReflectionPad2d(to_pad) 118 | to_pad = 0 119 | 120 | convolver = nn.Conv2d(in_f, out_f, kernel_size, stride, padding=to_pad, bias=bias) 121 | 122 | 123 | layers = filter(lambda x: x is not None, [padder, convolver, downsampler]) 124 | return nn.Sequential(*layers) -------------------------------------------------------------------------------- /3_Inpainting/models/dcgan.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | def dcgan(inp=2, 5 | ndf=32, 6 | num_ups=4, need_sigmoid=True, need_bias=True, pad='zero', upsample_mode='nearest', need_convT = True): 7 | 8 | layers= [nn.ConvTranspose2d(inp, ndf, kernel_size=3, stride=1, padding=0, bias=False), 9 | nn.BatchNorm2d(ndf), 10 | nn.LeakyReLU(True)] 11 | 12 | for i in range(num_ups-3): 13 | if need_convT: 14 | layers += [ nn.ConvTranspose2d(ndf, ndf, kernel_size=4, stride=2, padding=1, bias=False), 15 | nn.BatchNorm2d(ndf), 16 | nn.LeakyReLU(True)] 17 | else: 18 | layers += [ nn.Upsample(scale_factor=2, mode=upsample_mode), 19 | nn.Conv2d(ndf, ndf, kernel_size=3, stride=1, padding=1, bias=False), 20 | nn.BatchNorm2d(ndf), 21 | nn.LeakyReLU(True)] 22 | 23 | if need_convT: 24 | layers += [nn.ConvTranspose2d(ndf, 3, 4, 2, 1, bias=False),] 25 | else: 26 | layers += [nn.Upsample(scale_factor=2, mode='bilinear'), 27 | nn.Conv2d(ndf, 3, kernel_size=3, stride=1, padding=1, bias=False)] 28 | 29 | 30 | if need_sigmoid: 31 | layers += [nn.Sigmoid()] 32 | 33 | model =nn.Sequential(*layers) 34 | return model -------------------------------------------------------------------------------- /3_Inpainting/models/downsampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | class Downsampler(nn.Module): 6 | ''' 7 | http://www.realitypixels.com/turk/computergraphics/ResamplingFilters.pdf 8 | ''' 9 | def __init__(self, n_planes, factor, kernel_type, phase=0, kernel_width=None, support=None, sigma=None, preserve_size=False): 10 | super(Downsampler, self).__init__() 11 | 12 | assert phase in [0, 0.5], 'phase should be 0 or 0.5' 13 | 14 | if kernel_type == 'lanczos2': 15 | support = 2 16 | kernel_width = 4 * factor + 1 17 | kernel_type_ = 'lanczos' 18 | 19 | elif kernel_type == 'lanczos3': 20 | support = 3 21 | kernel_width = 6 * factor + 1 22 | kernel_type_ = 'lanczos' 23 | 24 | elif kernel_type == 'gauss12': 25 | kernel_width = 7 26 | sigma = 1/2 27 | kernel_type_ = 'gauss' 28 | 29 | elif kernel_type == 'gauss1sq2': 30 | kernel_width = 9 31 | sigma = 1./np.sqrt(2) 32 | kernel_type_ = 'gauss' 33 | 34 | elif kernel_type in ['lanczos', 'gauss', 'box']: 35 | kernel_type_ = kernel_type 36 | 37 | else: 38 | assert False, 'wrong name kernel' 39 | 40 | 41 | # note that `kernel width` will be different to actual size for phase = 1/2 42 | self.kernel = get_kernel(factor, kernel_type_, phase, kernel_width, support=support, sigma=sigma) 43 | 44 | downsampler = nn.Conv2d(n_planes, n_planes, kernel_size=self.kernel.shape, stride=factor, padding=0) 45 | downsampler.weight.data[:] = 0 46 | downsampler.bias.data[:] = 0 47 | 48 | kernel_torch = torch.from_numpy(self.kernel) 49 | for i in range(n_planes): 50 | downsampler.weight.data[i, i] = kernel_torch 51 | 52 | self.downsampler_ = downsampler 53 | 54 | if preserve_size: 55 | 56 | if self.kernel.shape[0] % 2 == 1: 57 | pad = int((self.kernel.shape[0] - 1) / 2.) 58 | else: 59 | pad = int((self.kernel.shape[0] - factor) / 2.) 60 | 61 | self.padding = nn.ReplicationPad2d(pad) 62 | 63 | self.preserve_size = preserve_size 64 | 65 | def forward(self, input): 66 | if self.preserve_size: 67 | x = self.padding(input) 68 | else: 69 | x= input 70 | self.x = x 71 | return self.downsampler_(x) 72 | 73 | def get_kernel(factor, kernel_type, phase, kernel_width, support=None, sigma=None): 74 | assert kernel_type in ['lanczos', 'gauss', 'box'] 75 | 76 | # factor = float(factor) 77 | if phase == 0.5 and kernel_type != 'box': 78 | kernel = np.zeros([kernel_width - 1, kernel_width - 1]) 79 | else: 80 | kernel = np.zeros([kernel_width, kernel_width]) 81 | 82 | 83 | if kernel_type == 'box': 84 | assert phase == 0.5, 'Box filter is always half-phased' 85 | kernel[:] = 1./(kernel_width * kernel_width) 86 | 87 | elif kernel_type == 'gauss': 88 | assert sigma, 'sigma is not specified' 89 | assert phase != 0.5, 'phase 1/2 for gauss not implemented' 90 | 91 | center = (kernel_width + 1.)/2. 92 | print(center, kernel_width) 93 | sigma_sq = sigma * sigma 94 | 95 | for i in range(1, kernel.shape[0] + 1): 96 | for j in range(1, kernel.shape[1] + 1): 97 | di = (i - center)/2. 98 | dj = (j - center)/2. 99 | kernel[i - 1][j - 1] = np.exp(-(di * di + dj * dj)/(2 * sigma_sq)) 100 | kernel[i - 1][j - 1] = kernel[i - 1][j - 1]/(2. * np.pi * sigma_sq) 101 | elif kernel_type == 'lanczos': 102 | assert support, 'support is not specified' 103 | center = (kernel_width + 1) / 2. 104 | 105 | for i in range(1, kernel.shape[0] + 1): 106 | for j in range(1, kernel.shape[1] + 1): 107 | 108 | if phase == 0.5: 109 | di = abs(i + 0.5 - center) / factor 110 | dj = abs(j + 0.5 - center) / factor 111 | else: 112 | di = abs(i - center) / factor 113 | dj = abs(j - center) / factor 114 | 115 | 116 | pi_sq = np.pi * np.pi 117 | 118 | val = 1 119 | if di != 0: 120 | val = val * support * np.sin(np.pi * di) * np.sin(np.pi * di / support) 121 | val = val / (np.pi * np.pi * di * di) 122 | 123 | if dj != 0: 124 | val = val * support * np.sin(np.pi * dj) * np.sin(np.pi * dj / support) 125 | val = val / (np.pi * np.pi * dj * dj) 126 | 127 | kernel[i - 1][j - 1] = val 128 | 129 | 130 | else: 131 | assert False, 'wrong method name' 132 | 133 | kernel /= kernel.sum() 134 | 135 | return kernel 136 | 137 | #a = Downsampler(n_planes=3, factor=2, kernel_type='lanczos2', phase='1', preserve_size=True) 138 | 139 | 140 | 141 | 142 | 143 | 144 | ################# 145 | # Learnable downsampler 146 | 147 | # KS = 32 148 | # dow = nn.Sequential(nn.ReplicationPad2d(int((KS - factor) / 2.)), nn.Conv2d(1,1,KS,factor)) 149 | 150 | # class Apply(nn.Module): 151 | # def __init__(self, what, dim, *args): 152 | # super(Apply, self).__init__() 153 | # self.dim = dim 154 | 155 | # self.what = what 156 | 157 | # def forward(self, input): 158 | # inputs = [] 159 | # for i in range(input.size(self.dim)): 160 | # inputs.append(self.what(input.narrow(self.dim, i, 1))) 161 | 162 | # return torch.cat(inputs, dim=self.dim) 163 | 164 | # def __len__(self): 165 | # return len(self._modules) 166 | 167 | # downs = Apply(dow, 1) 168 | # downs.type(dtype)(net_input.type(dtype)).size() 169 | -------------------------------------------------------------------------------- /3_Inpainting/models/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from numpy.random import normal 4 | from numpy.linalg import svd 5 | from math import sqrt 6 | import torch.nn.init 7 | from .common import * 8 | 9 | class ResidualSequential(nn.Sequential): 10 | def __init__(self, *args): 11 | super(ResidualSequential, self).__init__(*args) 12 | 13 | def forward(self, x): 14 | out = super(ResidualSequential, self).forward(x) 15 | # print(x.size(), out.size()) 16 | x_ = None 17 | if out.size(2) != x.size(2) or out.size(3) != x.size(3): 18 | diff2 = x.size(2) - out.size(2) 19 | diff3 = x.size(3) - out.size(3) 20 | # print(1) 21 | x_ = x[:, :, diff2 /2:out.size(2) + diff2 / 2, diff3 / 2:out.size(3) + diff3 / 2] 22 | else: 23 | x_ = x 24 | return out + x_ 25 | 26 | def eval(self): 27 | print(2) 28 | for m in self.modules(): 29 | m.eval() 30 | exit() 31 | 32 | 33 | def get_block(num_channels, norm_layer, act_fun): 34 | layers = [ 35 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False), 36 | norm_layer(num_channels, affine=True), 37 | act(act_fun), 38 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False), 39 | norm_layer(num_channels, affine=True), 40 | ] 41 | return layers 42 | 43 | 44 | class ResNet(nn.Module): 45 | def __init__(self, num_input_channels, num_output_channels, num_blocks, num_channels, need_residual=True, act_fun='LeakyReLU', need_sigmoid=True, norm_layer=nn.BatchNorm2d, pad='reflection'): 46 | ''' 47 | pad = 'start|zero|replication' 48 | ''' 49 | super(ResNet, self).__init__() 50 | 51 | if need_residual: 52 | s = ResidualSequential 53 | else: 54 | s = nn.Sequential 55 | 56 | stride = 1 57 | # First layers 58 | layers = [ 59 | # nn.ReplicationPad2d(num_blocks * 2 * stride + 3), 60 | conv(num_input_channels, num_channels, 3, stride=1, bias=True, pad=pad), 61 | act(act_fun) 62 | ] 63 | # Residual blocks 64 | # layers_residual = [] 65 | for i in range(num_blocks): 66 | layers += [s(*get_block(num_channels, norm_layer, act_fun))] 67 | 68 | layers += [ 69 | nn.Conv2d(num_channels, num_channels, 3, 1, 1), 70 | norm_layer(num_channels, affine=True) 71 | ] 72 | 73 | # if need_residual: 74 | # layers += [ResidualSequential(*layers_residual)] 75 | # else: 76 | # layers += [Sequential(*layers_residual)] 77 | 78 | # if factor >= 2: 79 | # # Do upsampling if needed 80 | # layers += [ 81 | # nn.Conv2d(num_channels, num_channels * 82 | # factor ** 2, 3, 1), 83 | # nn.PixelShuffle(factor), 84 | # act(act_fun) 85 | # ] 86 | layers += [ 87 | conv(num_channels, num_output_channels, 3, 1, bias=True, pad=pad), 88 | nn.Sigmoid() 89 | ] 90 | self.model = nn.Sequential(*layers) 91 | 92 | def forward(self, input): 93 | return self.model(input) 94 | 95 | def eval(self): 96 | self.model.eval() 97 | -------------------------------------------------------------------------------- /3_Inpainting/models/skip.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .common import * 4 | 5 | def skip( 6 | num_input_channels=2, num_output_channels=3, 7 | num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4], 8 | filter_size_down=3, filter_size_up=3, filter_skip_size=1, 9 | need_sigmoid=True, need_bias=True, 10 | pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU', 11 | need1x1_up=True): 12 | """Assembles encoder-decoder with skip connections. 13 | 14 | Arguments: 15 | act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU) 16 | pad (string): zero|reflection (default: 'zero') 17 | upsample_mode (string): 'nearest|bilinear' (default: 'nearest') 18 | downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride') 19 | 20 | """ 21 | assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip) 22 | 23 | n_scales = len(num_channels_down) 24 | 25 | if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) : 26 | upsample_mode = [upsample_mode]*n_scales 27 | 28 | if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)): 29 | downsample_mode = [downsample_mode]*n_scales 30 | 31 | if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) : 32 | filter_size_down = [filter_size_down]*n_scales 33 | 34 | if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) : 35 | filter_size_up = [filter_size_up]*n_scales 36 | 37 | last_scale = n_scales - 1 38 | 39 | cur_depth = None 40 | 41 | model = nn.Sequential() 42 | model_tmp = model 43 | 44 | input_depth = num_input_channels 45 | for i in range(len(num_channels_down)): 46 | 47 | deeper = nn.Sequential() 48 | skip = nn.Sequential() 49 | 50 | if num_channels_skip[i] != 0: 51 | model_tmp.add(Concat(1, skip, deeper)) 52 | else: 53 | model_tmp.add(deeper) 54 | 55 | model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i]))) 56 | 57 | if num_channels_skip[i] != 0: 58 | skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad)) 59 | skip.add(bn(num_channels_skip[i])) 60 | skip.add(act(act_fun)) 61 | 62 | # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part)) 63 | 64 | deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i])) 65 | deeper.add(bn(num_channels_down[i])) 66 | deeper.add(act(act_fun)) 67 | 68 | deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad)) 69 | deeper.add(bn(num_channels_down[i])) 70 | deeper.add(act(act_fun)) 71 | 72 | deeper_main = nn.Sequential() 73 | 74 | if i == len(num_channels_down) - 1: 75 | # The deepest 76 | k = num_channels_down[i] 77 | else: 78 | deeper.add(deeper_main) 79 | k = num_channels_up[i + 1] 80 | 81 | deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i])) 82 | 83 | model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad)) 84 | model_tmp.add(bn(num_channels_up[i])) 85 | model_tmp.add(act(act_fun)) 86 | 87 | 88 | if need1x1_up: 89 | model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad)) 90 | model_tmp.add(bn(num_channels_up[i])) 91 | model_tmp.add(act(act_fun)) 92 | 93 | input_depth = num_channels_down[i] 94 | model_tmp = deeper_main 95 | 96 | model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad)) 97 | if need_sigmoid: 98 | model.add(nn.Sigmoid()) 99 | 100 | return model 101 | -------------------------------------------------------------------------------- /3_Inpainting/models/texture_nets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .common import * 4 | 5 | 6 | normalization = nn.BatchNorm2d 7 | 8 | 9 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero'): 10 | if pad == 'zero': 11 | return nn.Conv2d(in_f, out_f, kernel_size, stride, padding=(kernel_size - 1) / 2, bias=bias) 12 | elif pad == 'reflection': 13 | layers = [nn.ReflectionPad2d((kernel_size - 1) / 2), 14 | nn.Conv2d(in_f, out_f, kernel_size, stride, padding=0, bias=bias)] 15 | return nn.Sequential(*layers) 16 | 17 | def get_texture_nets(inp=3, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False, pad='zero', need_sigmoid=False, conv_num=8, upsample_mode='nearest'): 18 | 19 | 20 | for i in range(len(ratios)): 21 | j = i + 1 22 | 23 | seq = nn.Sequential() 24 | 25 | tmp = nn.AvgPool2d(ratios[i], ratios[i]) 26 | 27 | seq.add(tmp) 28 | if fill_noise: 29 | seq.add(GenNoise(inp)) 30 | 31 | seq.add(conv(inp, conv_num, 3, pad=pad)) 32 | seq.add(normalization(conv_num)) 33 | seq.add(act()) 34 | 35 | seq.add(conv(conv_num, conv_num, 3, pad=pad)) 36 | seq.add(normalization(conv_num)) 37 | seq.add(act()) 38 | 39 | seq.add(conv(conv_num, conv_num, 1, pad=pad)) 40 | seq.add(normalization(conv_num)) 41 | seq.add(act()) 42 | 43 | if i == 0: 44 | seq.add(nn.Upsample(scale_factor=2, mode=upsample_mode)) 45 | cur = seq 46 | else: 47 | 48 | cur_temp = cur 49 | 50 | cur = nn.Sequential() 51 | 52 | # Batch norm before merging 53 | seq.add(normalization(conv_num)) 54 | cur_temp.add(normalization(conv_num * (j - 1))) 55 | 56 | cur.add(Concat(1, cur_temp, seq)) 57 | 58 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad)) 59 | cur.add(normalization(conv_num * j)) 60 | cur.add(act()) 61 | 62 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad)) 63 | cur.add(normalization(conv_num * j)) 64 | cur.add(act()) 65 | 66 | cur.add(conv(conv_num * j, conv_num * j, 1, pad=pad)) 67 | cur.add(normalization(conv_num * j)) 68 | cur.add(act()) 69 | 70 | if i == len(ratios) - 1: 71 | cur.add(conv(conv_num * j, 3, 1, pad=pad)) 72 | else: 73 | cur.add(nn.Upsample(scale_factor=2, mode=upsample_mode)) 74 | 75 | model = cur 76 | if need_sigmoid: 77 | model.add(nn.Sigmoid()) 78 | 79 | return model 80 | -------------------------------------------------------------------------------- /3_Inpainting/psnr_ssim.py: -------------------------------------------------------------------------------- 1 | ### This script will calcualte psnr and ssim 2 | import numpy as np 3 | from skimage.metrics import peak_signal_noise_ratio 4 | from skimage.measure import compare_ssim 5 | 6 | 7 | def calcualte_PSNR(im_true, im_test): 8 | im_true = np.transpose(im_true,(1,2,0)) 9 | im_test = np.transpose(im_test, (1, 2, 0)) 10 | psnr_value = peak_signal_noise_ratio(im_true, im_test) 11 | return psnr_value 12 | 13 | 14 | def calcualte_SSIM(im_true, im_test, multichannel): # multichannel=True for RGB images 15 | im_true = np.transpose(im_true, (1, 2, 0)) 16 | im_test = np.transpose(im_test, (1, 2, 0)) 17 | ssim_value = compare_ssim(im_true, im_test, multichannel=multichannel, data_range=im_test.max() - im_test.min()) 18 | return ssim_value 19 | 20 | 21 | 22 | 23 | 24 | if __name__ == '__main__': 25 | pass -------------------------------------------------------------------------------- /3_Inpainting/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__init__.py -------------------------------------------------------------------------------- /3_Inpainting/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/utils/__pycache__/common_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/common_utils.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/utils/__pycache__/common_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/common_utils.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/utils/__pycache__/denoising_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/denoising_utils.cpython-36.pyc -------------------------------------------------------------------------------- /3_Inpainting/utils/__pycache__/denoising_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/denoising_utils.cpython-37.pyc -------------------------------------------------------------------------------- /3_Inpainting/utils/common_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision 4 | import sys 5 | 6 | import numpy as np 7 | from PIL import Image 8 | import PIL 9 | import numpy as np 10 | 11 | import matplotlib.pyplot as plt 12 | 13 | def crop_image(img, d=32): 14 | '''Make dimensions divisible by `d`''' 15 | 16 | new_size = (img.size[0] - img.size[0] % d, 17 | img.size[1] - img.size[1] % d) 18 | 19 | bbox = [ 20 | int((img.size[0] - new_size[0])/2), 21 | int((img.size[1] - new_size[1])/2), 22 | int((img.size[0] + new_size[0])/2), 23 | int((img.size[1] + new_size[1])/2), 24 | ] 25 | 26 | img_cropped = img.crop(bbox) 27 | return img_cropped 28 | 29 | def get_params(opt_over, net, net_input, downsampler=None): 30 | '''Returns parameters that we want to optimize over. 31 | 32 | Args: 33 | opt_over: comma separated list, e.g. "net,input" or "net" 34 | net: network 35 | net_input: torch.Tensor that stores input `z` 36 | ''' 37 | opt_over_list = opt_over.split(',') 38 | params = [] 39 | 40 | for opt in opt_over_list: 41 | 42 | if opt == 'net': 43 | params += [x for x in net.parameters() ] 44 | elif opt=='down': 45 | assert downsampler is not None 46 | params = [x for x in downsampler.parameters()] 47 | elif opt == 'input': 48 | net_input.requires_grad = True 49 | params += [net_input] 50 | else: 51 | assert False, 'what is it?' 52 | 53 | return params 54 | 55 | def get_image_grid(images_np, nrow=8): 56 | '''Creates a grid from a list of images by concatenating them.''' 57 | images_torch = [torch.from_numpy(x) for x in images_np] 58 | torch_grid = torchvision.utils.make_grid(images_torch, nrow) 59 | 60 | return torch_grid.numpy() 61 | 62 | def plot_image_grid(images_np, nrow =8, factor=1, interpolation='lanczos'): 63 | """Draws images in a grid 64 | 65 | Args: 66 | images_np: list of images, each image is np.array of size 3xHxW of 1xHxW 67 | nrow: how many images will be in one row 68 | factor: size if the plt.figure 69 | interpolation: interpolation used in plt.imshow 70 | """ 71 | n_channels = max(x.shape[0] for x in images_np) 72 | assert (n_channels == 3) or (n_channels == 1), "images should have 1 or 3 channels" 73 | 74 | images_np = [x if (x.shape[0] == n_channels) else np.concatenate([x, x, x], axis=0) for x in images_np] 75 | 76 | grid = get_image_grid(images_np, nrow) 77 | 78 | plt.figure(figsize=(len(images_np) + factor, 12 + factor)) 79 | 80 | if images_np[0].shape[0] == 1: 81 | plt.imshow(grid[0], cmap='gray', interpolation=interpolation) 82 | else: 83 | plt.imshow(grid.transpose(1, 2, 0), interpolation=interpolation) 84 | 85 | plt.show() 86 | 87 | return grid 88 | 89 | def load(path): 90 | """Load PIL image.""" 91 | img = Image.open(path) 92 | return img 93 | 94 | def get_image(path, imsize=-1): 95 | """Load an image and resize to a cpecific size. 96 | 97 | Args: 98 | path: path to image 99 | imsize: tuple or scalar with dimensions; -1 for `no resize` 100 | """ 101 | img = load(path) 102 | 103 | if isinstance(imsize, int): 104 | imsize = (imsize, imsize) 105 | 106 | if imsize[0]!= -1 and img.size != imsize: 107 | if imsize[0] > img.size[0]: 108 | img = img.resize(imsize, Image.BICUBIC) 109 | else: 110 | img = img.resize(imsize, Image.ANTIALIAS) 111 | 112 | img_np = pil_to_np(img) 113 | 114 | return img, img_np 115 | 116 | 117 | 118 | def fill_noise(x, noise_type): 119 | """Fills tensor `x` with noise of type `noise_type`.""" 120 | if noise_type == 'u': 121 | x.uniform_() 122 | elif noise_type == 'n': 123 | x.normal_() 124 | else: 125 | assert False 126 | 127 | def get_noise(input_depth, method, spatial_size, noise_type='u', var=1./10): 128 | """Returns a pytorch.Tensor of size (1 x `input_depth` x `spatial_size[0]` x `spatial_size[1]`) 129 | initialized in a specific way. 130 | Args: 131 | input_depth: number of channels in the tensor 132 | method: `noise` for fillting tensor with noise; `meshgrid` for np.meshgrid 133 | spatial_size: spatial size of the tensor to initialize 134 | noise_type: 'u' for uniform; 'n' for normal 135 | var: a factor, a noise will be multiplicated by. Basically it is standard deviation scaler. 136 | """ 137 | if isinstance(spatial_size, int): 138 | spatial_size = (spatial_size, spatial_size) 139 | if method == 'noise': 140 | shape = [1, input_depth, spatial_size[0], spatial_size[1]] 141 | net_input = torch.zeros(shape) 142 | 143 | fill_noise(net_input, noise_type) 144 | net_input *= var 145 | elif method == 'meshgrid': 146 | assert input_depth == 2 147 | X, Y = np.meshgrid(np.arange(0, spatial_size[1])/float(spatial_size[1]-1), np.arange(0, spatial_size[0])/float(spatial_size[0]-1)) 148 | meshgrid = np.concatenate([X[None,:], Y[None,:]]) 149 | net_input= np_to_torch(meshgrid) 150 | else: 151 | assert False 152 | 153 | return net_input 154 | 155 | def pil_to_np(img_PIL): 156 | '''Converts image in PIL format to np.array. 157 | 158 | From W x H x C [0...255] to C x W x H [0..1] 159 | ''' 160 | ar = np.array(img_PIL) 161 | 162 | if len(ar.shape) == 3: 163 | ar = ar.transpose(2,0,1) 164 | else: 165 | ar = ar[None, ...] 166 | 167 | return ar.astype(np.float32) / 255. 168 | 169 | def np_to_pil(img_np): 170 | '''Converts image in np.array format to PIL image. 171 | 172 | From C x W x H [0..1] to W x H x C [0...255] 173 | ''' 174 | ar = np.clip(img_np*255,0,255).astype(np.uint8) 175 | 176 | if img_np.shape[0] == 1: 177 | ar = ar[0] 178 | else: 179 | ar = ar.transpose(1, 2, 0) 180 | 181 | return Image.fromarray(ar) 182 | 183 | def np_to_torch(img_np): 184 | '''Converts image in numpy.array to torch.Tensor. 185 | 186 | From C x W x H [0..1] to C x W x H [0..1] 187 | ''' 188 | return torch.from_numpy(img_np)[None, :] 189 | 190 | def torch_to_np(img_var): 191 | '''Converts an image in torch.Tensor format to np.array. 192 | 193 | From 1 x C x W x H [0..1] to C x W x H [0..1] 194 | ''' 195 | return img_var.detach().cpu().numpy()[0] 196 | 197 | 198 | def optimize(optimizer_type, parameters, closure, LR, num_iter): 199 | """Runs optimization loop. 200 | 201 | Args: 202 | optimizer_type: 'LBFGS' of 'adam' 203 | parameters: list of Tensors to optimize over 204 | closure: function, that returns loss variable 205 | LR: learning rate 206 | num_iter: number of iterations 207 | """ 208 | if optimizer_type == 'LBFGS': 209 | # Do several steps with adam first 210 | optimizer = torch.optim.Adam(parameters, lr=0.001) 211 | for j in range(100): 212 | optimizer.zero_grad() 213 | closure() 214 | optimizer.step() 215 | 216 | print('Starting optimization with LBFGS') 217 | def closure2(): 218 | optimizer.zero_grad() 219 | return closure() 220 | optimizer = torch.optim.LBFGS(parameters, max_iter=num_iter, lr=LR, tolerance_grad=-1, tolerance_change=-1) 221 | optimizer.step(closure2) 222 | 223 | elif optimizer_type == 'adam': 224 | print('Starting optimization with ADAM') 225 | optimizer = torch.optim.Adam(parameters, lr=LR) 226 | 227 | for j in range(num_iter): 228 | optimizer.zero_grad() 229 | closure() 230 | optimizer.step() 231 | else: 232 | assert False -------------------------------------------------------------------------------- /3_Inpainting/utils/denoising_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from .common_utils import * 3 | 4 | 5 | 6 | def get_noisy_image(img_np, sigma): 7 | """Adds Gaussian noise to an image. 8 | 9 | Args: 10 | img_np: image, np.array with values from 0 to 1 11 | sigma: std of the noise 12 | """ 13 | img_noisy_np = np.clip(img_np + np.random.normal(scale=sigma, size=img_np.shape), 0, 1).astype(np.float32) 14 | img_noisy_pil = np_to_pil(img_noisy_np) 15 | 16 | return img_noisy_pil, img_noisy_np -------------------------------------------------------------------------------- /3_Inpainting/utils/feature_inversion_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.transforms as transforms 4 | import torchvision.models as models 5 | from .matcher import Matcher 6 | import os 7 | from collections import OrderedDict 8 | 9 | class View(nn.Module): 10 | def __init__(self): 11 | super(View, self).__init__() 12 | 13 | def forward(self, x): 14 | return x.view(-1) 15 | 16 | def get_vanilla_vgg_features(cut_idx=-1): 17 | if not os.path.exists('vgg_features.pth'): 18 | os.system( 19 | 'wget --no-check-certificate -N https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth') 20 | vgg_weights = torch.load('vgg19-d01eb7cb.pth') 21 | # fix compatibility issues 22 | map = {'classifier.6.weight':u'classifier.7.weight', 'classifier.6.bias':u'classifier.7.bias'} 23 | vgg_weights = OrderedDict([(map[k] if k in map else k,v) for k,v in vgg_weights.iteritems()]) 24 | 25 | 26 | 27 | model = models.vgg19() 28 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values()) 29 | 30 | 31 | model.load_state_dict(vgg_weights) 32 | 33 | torch.save(model.features, 'vgg_features.pth') 34 | torch.save(model.classifier, 'vgg_classifier.pth') 35 | 36 | vgg = torch.load('vgg_features.pth') 37 | if cut_idx > 36: 38 | vgg_classifier = torch.load('vgg_classifier.pth') 39 | vgg = nn.Sequential(*(vgg._modules.values() + vgg_classifier._modules.values())) 40 | 41 | vgg.eval() 42 | 43 | return vgg 44 | 45 | 46 | def get_matcher(net, opt): 47 | idxs = [x for x in opt['layers'].split(',')] 48 | matcher = Matcher(opt['what']) 49 | 50 | def hook(module, input, output): 51 | matcher(module, output) 52 | 53 | for i in idxs: 54 | net._modules[i].register_forward_hook(hook) 55 | 56 | return matcher 57 | 58 | 59 | 60 | def get_vgg(cut_idx=-1): 61 | f = get_vanilla_vgg_features(cut_idx) 62 | 63 | if cut_idx > 0: 64 | num_modules = len(f._modules) 65 | keys_to_delete = [f._modules.keys()[x] for x in range(cut_idx, num_modules)] 66 | for k in keys_to_delete: 67 | del f._modules[k] 68 | 69 | return f 70 | 71 | def vgg_preprocess_var(var): 72 | (r, g, b) = torch.chunk(var, 3, dim=1) 73 | bgr = torch.cat((b, g, r), 1) 74 | out = bgr * 255 - torch.autograd.Variable(vgg_mean[None, ...]).type(var.type()).expand_as(bgr) 75 | return out 76 | 77 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1) 78 | 79 | 80 | 81 | def get_preprocessor(imsize): 82 | def vgg_preprocess(tensor): 83 | (r, g, b) = torch.chunk(tensor, 3, dim=0) 84 | bgr = torch.cat((b, g, r), 0) 85 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr) 86 | return out 87 | preprocess = transforms.Compose([ 88 | transforms.Resize(imsize), 89 | transforms.ToTensor(), 90 | transforms.Lambda(vgg_preprocess) 91 | ]) 92 | 93 | return preprocess 94 | 95 | 96 | def get_deprocessor(): 97 | def vgg_deprocess(tensor): 98 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0 99 | (b, g, r) = torch.chunk(bgr, 3, dim=0) 100 | rgb = torch.cat((r, g, b), 0) 101 | return rgb 102 | deprocess = transforms.Compose([ 103 | transforms.Lambda(vgg_deprocess), 104 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)), 105 | transforms.ToPILImage() 106 | ]) 107 | return deprocess 108 | -------------------------------------------------------------------------------- /3_Inpainting/utils/inpainting_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | import PIL.ImageDraw as ImageDraw 4 | import PIL.ImageFont as ImageFont 5 | from .common_utils import * 6 | 7 | def get_text_mask(for_image, sz=20): 8 | font_fname = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf' 9 | font_size = sz 10 | font = ImageFont.truetype(font_fname, font_size) 11 | 12 | img_mask = Image.fromarray(np.array(for_image)*0+255) 13 | draw = ImageDraw.Draw(img_mask) 14 | draw.text((128, 128), "hello world", font=font, fill='rgb(0, 0, 0)') 15 | 16 | return img_mask 17 | 18 | def get_bernoulli_mask(for_image, zero_fraction=0.95): 19 | img_mask_np=(np.random.random_sample(size=pil_to_np(for_image).shape) > zero_fraction).astype(int) 20 | img_mask = np_to_pil(img_mask_np) 21 | 22 | return img_mask 23 | -------------------------------------------------------------------------------- /3_Inpainting/utils/matcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class Matcher: 5 | def __init__(self, how='gram_matrix', loss='mse'): 6 | self.mode = 'store' 7 | self.stored = {} 8 | self.losses = {} 9 | 10 | if how in all_features.keys(): 11 | self.get_statistics = all_features[how] 12 | else: 13 | assert False 14 | pass 15 | 16 | if loss in all_losses.keys(): 17 | self.loss = all_losses[loss] 18 | else: 19 | assert False 20 | 21 | def __call__(self, module, features): 22 | statistics = self.get_statistics(features) 23 | 24 | self.statistics = statistics 25 | if self.mode == 'store': 26 | self.stored[module] = statistics.detach().clone() 27 | elif self.mode == 'match': 28 | self.losses[module] = self.loss(statistics, self.stored[module]) 29 | 30 | def clean(self): 31 | self.losses = {} 32 | 33 | def gram_matrix(x): 34 | (b, ch, h, w) = x.size() 35 | features = x.view(b, ch, w * h) 36 | features_t = features.transpose(1, 2) 37 | gram = features.bmm(features_t) / (ch * h * w) 38 | return gram 39 | 40 | 41 | def features(x): 42 | return x 43 | 44 | 45 | all_features = { 46 | 'gram_matrix': gram_matrix, 47 | 'features': features, 48 | } 49 | 50 | all_losses = { 51 | 'mse': nn.MSELoss(), 52 | 'smoothL1': nn.SmoothL1Loss(), 53 | 'L1': nn.L1Loss(), 54 | } 55 | -------------------------------------------------------------------------------- /3_Inpainting/utils/perceptual_loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/perceptual_loss/__init__.py -------------------------------------------------------------------------------- /3_Inpainting/utils/perceptual_loss/matcher.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class Matcher: 6 | def __init__(self, how='gram_matrix', loss='mse', map_index=933): 7 | self.mode = 'store' 8 | self.stored = {} 9 | self.losses = {} 10 | 11 | if how in all_features.keys(): 12 | self.get_statistics = all_features[how] 13 | else: 14 | assert False 15 | pass 16 | 17 | if loss in all_losses.keys(): 18 | self.loss = all_losses[loss] 19 | else: 20 | assert False 21 | 22 | self.map_index = map_index 23 | self.method = 'match' 24 | 25 | 26 | def __call__(self, module, features): 27 | statistics = self.get_statistics(features) 28 | 29 | self.statistics = statistics 30 | if self.mode == 'store': 31 | self.stored[module] = statistics.detach() 32 | 33 | elif self.mode == 'match': 34 | 35 | if statistics.ndimension() == 2: 36 | 37 | if self.method == 'maximize': 38 | self.losses[module] = - statistics[0, self.map_index] 39 | else: 40 | self.losses[module] = torch.abs(300 - statistics[0, self.map_index]) 41 | 42 | else: 43 | ws = self.window_size 44 | 45 | t = statistics.detach() * 0 46 | 47 | s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0 48 | t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0 49 | t_cc[:, self.map_index,...] = 1 50 | 51 | if self.method == 'maximize': 52 | self.losses[module] = -(s_cc * t_cc.contiguous()).sum() 53 | else: 54 | self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum() 55 | 56 | 57 | def clean(self): 58 | self.losses = {} 59 | 60 | def gram_matrix(x): 61 | (b, ch, h, w) = x.size() 62 | features = x.view(b, ch, w * h) 63 | features_t = features.transpose(1, 2) 64 | gram = features.bmm(features_t) / (ch * h * w) 65 | return gram 66 | 67 | 68 | def features(x): 69 | return x 70 | 71 | 72 | all_features = { 73 | 'gram_matrix': gram_matrix, 74 | 'features': features, 75 | } 76 | 77 | all_losses = { 78 | 'mse': nn.MSELoss(), 79 | 'smoothL1': nn.SmoothL1Loss(), 80 | 'L1': nn.L1Loss(), 81 | } 82 | -------------------------------------------------------------------------------- /3_Inpainting/utils/perceptual_loss/vgg_modified.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | class VGGModified(nn.Module): 4 | def __init__(self, vgg19_orig, slope=0.01): 5 | super(VGGModified, self).__init__() 6 | self.features = nn.Sequential() 7 | 8 | self.features.add_module(str(0), vgg19_orig.features[0]) 9 | self.features.add_module(str(1), nn.LeakyReLU(slope, True)) 10 | self.features.add_module(str(2), vgg19_orig.features[2]) 11 | self.features.add_module(str(3), nn.LeakyReLU(slope, True)) 12 | self.features.add_module(str(4), nn.AvgPool2d((2,2), (2,2))) 13 | 14 | self.features.add_module(str(5), vgg19_orig.features[5]) 15 | self.features.add_module(str(6), nn.LeakyReLU(slope, True)) 16 | self.features.add_module(str(7), vgg19_orig.features[7]) 17 | self.features.add_module(str(8), nn.LeakyReLU(slope, True)) 18 | self.features.add_module(str(9), nn.AvgPool2d((2,2), (2,2))) 19 | 20 | self.features.add_module(str(10), vgg19_orig.features[10]) 21 | self.features.add_module(str(11), nn.LeakyReLU(slope, True)) 22 | self.features.add_module(str(12), vgg19_orig.features[12]) 23 | self.features.add_module(str(13), nn.LeakyReLU(slope, True)) 24 | self.features.add_module(str(14), vgg19_orig.features[14]) 25 | self.features.add_module(str(15), nn.LeakyReLU(slope, True)) 26 | self.features.add_module(str(16), vgg19_orig.features[16]) 27 | self.features.add_module(str(17), nn.LeakyReLU(slope, True)) 28 | self.features.add_module(str(18), nn.AvgPool2d((2,2), (2,2))) 29 | 30 | self.features.add_module(str(19), vgg19_orig.features[19]) 31 | self.features.add_module(str(20), nn.LeakyReLU(slope, True)) 32 | self.features.add_module(str(21), vgg19_orig.features[21]) 33 | self.features.add_module(str(22), nn.LeakyReLU(slope, True)) 34 | self.features.add_module(str(23), vgg19_orig.features[23]) 35 | self.features.add_module(str(24), nn.LeakyReLU(slope, True)) 36 | self.features.add_module(str(25), vgg19_orig.features[25]) 37 | self.features.add_module(str(26), nn.LeakyReLU(slope, True)) 38 | self.features.add_module(str(27), nn.AvgPool2d((2,2), (2,2))) 39 | 40 | self.features.add_module(str(28), vgg19_orig.features[28]) 41 | self.features.add_module(str(29), nn.LeakyReLU(slope, True)) 42 | self.features.add_module(str(30), vgg19_orig.features[30]) 43 | self.features.add_module(str(31), nn.LeakyReLU(slope, True)) 44 | self.features.add_module(str(32), vgg19_orig.features[32]) 45 | self.features.add_module(str(33), nn.LeakyReLU(slope, True)) 46 | self.features.add_module(str(34), vgg19_orig.features[34]) 47 | self.features.add_module(str(35), nn.LeakyReLU(slope, True)) 48 | self.features.add_module(str(36), nn.AvgPool2d((2,2), (2,2))) 49 | 50 | self.classifier = nn.Sequential() 51 | 52 | self.classifier.add_module(str(0), vgg19_orig.classifier[0]) 53 | self.classifier.add_module(str(1), nn.LeakyReLU(slope, True)) 54 | self.classifier.add_module(str(2), nn.Dropout2d(p = 0.5)) 55 | self.classifier.add_module(str(3), vgg19_orig.classifier[3]) 56 | self.classifier.add_module(str(4), nn.LeakyReLU(slope, True)) 57 | self.classifier.add_module(str(5), nn.Dropout2d(p = 0.5)) 58 | self.classifier.add_module(str(6), vgg19_orig.classifier[6]) 59 | 60 | def forward(self, x): 61 | return self.classifier(self.features.forward(x)) -------------------------------------------------------------------------------- /3_Inpainting/utils/sr_utils.py: -------------------------------------------------------------------------------- 1 | from .common_utils import * 2 | 3 | def put_in_center(img_np, target_size): 4 | img_out = np.zeros([3, target_size[0], target_size[1]]) 5 | 6 | bbox = [ 7 | int((target_size[0] - img_np.shape[1]) / 2), 8 | int((target_size[1] - img_np.shape[2]) / 2), 9 | int((target_size[0] + img_np.shape[1]) / 2), 10 | int((target_size[1] + img_np.shape[2]) / 2), 11 | ] 12 | 13 | img_out[:, bbox[0]:bbox[2], bbox[1]:bbox[3]] = img_np 14 | 15 | return img_out 16 | 17 | 18 | def load_LR_HR_imgs_sr(fname, imsize, factor, enforse_div32=None): 19 | '''Loads an image, resizes it, center crops and downscales. 20 | 21 | Args: 22 | fname: path to the image 23 | imsize: new size for the image, -1 for no resizing 24 | factor: downscaling factor 25 | enforse_div32: if 'CROP' center crops an image, so that its dimensions are divisible by 32. 26 | ''' 27 | img_orig_pil, img_orig_np = get_image(fname, -1) 28 | 29 | if imsize != -1: 30 | img_orig_pil, img_orig_np = get_image(fname, imsize) 31 | 32 | # For comparison with GT 33 | if enforse_div32 == 'CROP': 34 | new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 32, 35 | img_orig_pil.size[1] - img_orig_pil.size[1] % 32) 36 | 37 | bbox = [ 38 | (img_orig_pil.size[0] - new_size[0])/2, 39 | (img_orig_pil.size[1] - new_size[1])/2, 40 | (img_orig_pil.size[0] + new_size[0])/2, 41 | (img_orig_pil.size[1] + new_size[1])/2, 42 | ] 43 | 44 | img_HR_pil = img_orig_pil.crop(bbox) 45 | img_HR_np = pil_to_np(img_HR_pil) 46 | else: 47 | img_HR_pil, img_HR_np = img_orig_pil, img_orig_np 48 | 49 | LR_size = [ 50 | img_HR_pil.size[0] // factor, 51 | img_HR_pil.size[1] // factor 52 | ] 53 | 54 | img_LR_pil = img_HR_pil.resize(LR_size, Image.ANTIALIAS) 55 | img_LR_np = pil_to_np(img_LR_pil) 56 | 57 | print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str (img_LR_pil.size))) 58 | 59 | return { 60 | 'orig_pil': img_orig_pil, 61 | 'orig_np': img_orig_np, 62 | 'LR_pil': img_LR_pil, 63 | 'LR_np': img_LR_np, 64 | 'HR_pil': img_HR_pil, 65 | 'HR_np': img_HR_np 66 | } 67 | 68 | 69 | def get_baselines(img_LR_pil, img_HR_pil): 70 | '''Gets `bicubic`, sharpened bicubic and `nearest` baselines.''' 71 | img_bicubic_pil = img_LR_pil.resize(img_HR_pil.size, Image.BICUBIC) 72 | img_bicubic_np = pil_to_np(img_bicubic_pil) 73 | 74 | img_nearest_pil = img_LR_pil.resize(img_HR_pil.size, Image.NEAREST) 75 | img_nearest_np = pil_to_np(img_nearest_pil) 76 | 77 | img_bic_sharp_pil = img_bicubic_pil.filter(PIL.ImageFilter.UnsharpMask()) 78 | img_bic_sharp_np = pil_to_np(img_bic_sharp_pil) 79 | 80 | return img_bicubic_np, img_bic_sharp_np, img_nearest_np 81 | 82 | 83 | 84 | def tv_loss(x, beta = 0.5): 85 | '''Calculates TV loss for an image `x`. 86 | 87 | Args: 88 | x: image, torch.Variable of torch.Tensor 89 | beta: See https://arxiv.org/abs/1412.0035 (fig. 2) to see effect of `beta` 90 | ''' 91 | dh = torch.pow(x[:,:,:,1:] - x[:,:,:,:-1], 2) 92 | dw = torch.pow(x[:,:,1:,:] - x[:,:,:-1,:], 2) 93 | 94 | return torch.sum(torch.pow(dh[:, :, :-1] + dw[:, :, :, :-1], beta)) 95 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 GLOVEX @ UMN 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep-Random-Projector 2 | ## Deep Random Projector: Accelerated Deep Image Prior 3 | 4 | This is the official implementation of our paper *Deep Random Projector: Accelerated Deep Image Prior* which has been accepted to the [CVPR2023](https://cvpr2023.thecvf.com/). You can find our paper via [this link](https://openaccess.thecvf.com/content/CVPR2023/html/Li_Deep_Random_Projector_Accelerated_Deep_Image_Prior_CVPR_2023_paper.html). 5 | 6 | 7 | ## Set Up the Environment 8 | 9 | 1. Get and clone the github repository: 10 | 11 | `git clone https://github.com/sun-umn/Deep-Random-Projector/` 12 | 13 | 2. Switch to `Deep-Random-Projector` : 14 | 15 | `cd XXX/Deep-Random-Projector` 16 | (*Note*: `XXX` here indicates the upper directory of `Deep-Random-Projector`. For example, if you clone `Deep-Random-Projector` under `/home/Download`, then you should replace `XXX` with `/home/Download`.) 17 | 18 | 3. Create a new conda environment with the YML file we provide: 19 | 20 | `conda env create -f environment.yml` 21 | 22 | 4. Activate conda environment and you are now ready to explpre the codes/models! 23 | 24 | `conda activate pytorch_py3.6` 25 | 26 | 27 | ## Explore the Codes/Models 28 | 29 | - **0_Dataset**: we provide a ready-to-use *example* dataset. 30 | 31 | - **1_Denoising**: the code for image denoising. 32 | 33 | - **2_Super_Resolution**: the code for image super-solution. 34 | 35 | - **3_Inpainting**: the code for image inpainting. 36 | 37 | We have provided the detailed inline comments in each Python file. You can modify any parameters you want to explore the models. Or, you can try our method by simply running the command below: 38 | 39 | ``` 40 | python Main_Start.py 41 | ``` 42 | 43 | 44 | ## Citation/BibTex 45 | 46 | More technical details and experimental results can be found in our paper: 47 | 48 | Taihui Li, Hengkang Wang, Zhong Zhuang, and Ju Sun. "Deep Random Projector: Accelerated Deep Image Prior." In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition, pp. 18176-18185. 2023. 49 | 50 | ``` 51 | @inproceedings{li2023deep, 52 | title={Deep Random Projector: Accelerated Deep Image Prior}, 53 | author={Li, Taihui and Wang, Hengkang and Zhuang, Zhong and Sun, Ju}, 54 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 55 | pages={18176--18185}, 56 | year={2023} 57 | } 58 | ``` 59 | 60 | 61 | ## Acknowledgements 62 | Our code is based on the [deep image prior (DIP)](https://github.com/DmitryUlyanov/deep-image-prior) and [deep decoder (DD)](https://github.com/reinhardh/supplement_deep_decoder). 63 | 64 | 65 | ## Contact 66 | - Taihui Li, lixx5027@umn.edu, [https://taihui.github.io/](https://taihui.github.io/) 67 | - Hengkang Wang, wang9881@umn.edu, [https://www.linkedin.com/in/hengkang-henry-wang-a1b293104/](https://www.linkedin.com/in/hengkang-henry-wang-a1b293104/) 68 | - Zhong Zhuang, zhuan143@umn.edu, [https://scholar.google.com/citations?user=rGGxUQEAAAAJ](https://scholar.google.com/citations?user=rGGxUQEAAAAJ) 69 | - Ju Sun, jusun@umn.edu, [https://sunju.org/](https://sunju.org/) 70 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: pytorch_py3.6 2 | channels: 3 | - pytorch 4 | - defaults 5 | dependencies: 6 | - _libgcc_mutex=0.1=main 7 | - blas=1.0=mkl 8 | - bzip2=1.0.8=h7b6447c_0 9 | - ca-certificates=2021.1.19=h06a4308_1 10 | - certifi=2020.12.5=py36h06a4308_0 11 | - cudatoolkit=10.2.89=hfd86e86_1 12 | - dataclasses=0.8=pyh4f3eec9_6 13 | - ffmpeg=4.3=hf484d3e_0 14 | - freetype=2.10.4=h5ab3b9f_0 15 | - gmp=6.2.1=h2531618_2 16 | - gnutls=3.6.5=h71b1129_1002 17 | - intel-openmp=2020.2=254 18 | - jpeg=9b=h024ee3a_2 19 | - lame=3.100=h7b6447c_0 20 | - lcms2=2.11=h396b838_0 21 | - ld_impl_linux-64=2.33.1=h53a641e_7 22 | - libffi=3.3=he6710b0_2 23 | - libgcc-ng=9.1.0=hdf63c60_0 24 | - libiconv=1.15=h63c8f33_5 25 | - libpng=1.6.37=hbc83047_0 26 | - libstdcxx-ng=9.1.0=hdf63c60_0 27 | - libtiff=4.2.0=h3942068_0 28 | - libuv=1.40.0=h7b6447c_0 29 | - libwebp-base=1.2.0=h27cfd23_0 30 | - lz4-c=1.9.3=h2531618_0 31 | - mkl=2020.2=256 32 | - mkl-service=2.3.0=py36he8ac12f_0 33 | - mkl_fft=1.3.0=py36h54f3939_0 34 | - mkl_random=1.1.1=py36h0573a6f_0 35 | - ncurses=6.2=he6710b0_1 36 | - nettle=3.4.1=hbb512f6_0 37 | - ninja=1.10.2=py36hff7bd54_0 38 | - numpy=1.19.2=py36h54aff64_0 39 | - numpy-base=1.19.2=py36hfa32c7d_0 40 | - olefile=0.46=py36_0 41 | - openh264=2.1.0=hd408876_0 42 | - openssl=1.1.1j=h27cfd23_0 43 | - pillow=8.1.2=py36he98fc37_0 44 | - pip=21.0.1=py36h06a4308_0 45 | - python=3.6.13=hdb3f193_0 46 | - pytorch=1.8.0=py3.6_cuda10.2_cudnn7.6.5_0 47 | - readline=8.1=h27cfd23_0 48 | - setuptools=52.0.0=py36h06a4308_0 49 | - six=1.15.0=py36h06a4308_0 50 | - sqlite=3.35.1=hdfb4753_0 51 | - tk=8.6.10=hbc83047_0 52 | - torchaudio=0.8.0=py36 53 | - torchvision=0.9.0=py36_cu102 54 | - typing_extensions=3.7.4.3=pyha847dfd_0 55 | - wheel=0.36.2=pyhd3eb1b0_0 56 | - xz=5.2.5=h7b6447c_0 57 | - zlib=1.2.11=h7b6447c_3 58 | - zstd=1.4.5=h9ceee32_0 59 | - pip: 60 | - backcall==0.2.0 61 | - charset-normalizer==2.0.4 62 | - cycler==0.10.0 63 | - decorator==4.4.2 64 | - idna==3.2 65 | - imageio==2.9.0 66 | - ipython==7.16.3 67 | - ipython-genutils==0.2.0 68 | - jedi==0.17.2 69 | - joblib==1.0.1 70 | - kiwisolver==1.3.1 71 | - matplotlib==3.3.4 72 | - networkx==2.5 73 | - opencv-python==4.5.1.48 74 | - pandas==1.1.5 75 | - parso==0.7.1 76 | - pexpect==4.8.0 77 | - pickleshare==0.7.5 78 | - prompt-toolkit==3.0.29 79 | - ptyprocess==0.7.0 80 | - pygments==2.12.0 81 | - pyparsing==2.4.7 82 | - python-dateutil==2.8.1 83 | - pytz==2021.1 84 | - pywavelets==1.1.1 85 | - requests==2.26.0 86 | - scikit-image==0.17.2 87 | - scikit-learn==0.24.1 88 | - scipy==1.5.4 89 | - seaborn==0.11.1 90 | - tdqm==0.0.1 91 | - threadpoolctl==2.1.0 92 | - tifffile==2020.9.3 93 | - tqdm==4.59.0 94 | - traitlets==4.3.3 95 | - urllib3==1.26.6 96 | - wand==0.6.6 97 | - wcwidth==0.2.5 98 | prefix: /home/taihui/anaconda3/envs/pytorch_py3.6 99 | --------------------------------------------------------------------------------