├── 0_Dataset
├── Gaussian_Noise
│ ├── Clean
│ │ └── Peppers_clean.npz
│ ├── Gaussian_Noise_2
│ │ ├── Peppers.png
│ │ └── Peppers_Gaussian_Noise_2.npz
│ └── Gaussian_Noise_4
│ │ ├── Peppers.png
│ │ └── Peppers_Gaussian_Noise_4.npz
├── Inpainting
│ └── 0_IP11
│ │ ├── .idea
│ │ ├── .gitignore
│ │ ├── 0_IP11.iml
│ │ ├── inspectionProfiles
│ │ │ ├── Project_Default.xml
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ └── modules.xml
│ │ ├── IP_Dataset_0.1
│ │ ├── barbara_img_0.1.png
│ │ ├── barbara_mask_0.1.png
│ │ └── barbara_vis_0.1.png
│ │ ├── Original_Images
│ │ └── barbara.png
│ │ └── generate_mask.py
└── Super_Resolution
│ └── Set5
│ └── baby.png
├── 1_Denoising
├── BN_Net.py
├── Main_Start.py
├── models
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── __init__.cpython-37.pyc
│ │ ├── common.cpython-36.pyc
│ │ ├── common.cpython-37.pyc
│ │ ├── downsampler.cpython-36.pyc
│ │ ├── downsampler.cpython-37.pyc
│ │ ├── resnet.cpython-36.pyc
│ │ ├── resnet.cpython-37.pyc
│ │ ├── skip.cpython-36.pyc
│ │ ├── skip.cpython-37.pyc
│ │ ├── texture_nets.cpython-36.pyc
│ │ ├── texture_nets.cpython-37.pyc
│ │ ├── unet.cpython-36.pyc
│ │ └── unet.cpython-37.pyc
│ ├── common.py
│ ├── dcgan.py
│ ├── downsampler.py
│ ├── resnet.py
│ ├── skip.py
│ ├── texture_nets.py
│ └── unet.py
├── psnr_ssim.py
├── train_denoising.py
├── util.py
└── utils
│ ├── __init__.py
│ ├── __pycache__
│ ├── __init__.cpython-36.pyc
│ ├── __init__.cpython-37.pyc
│ ├── common_utils.cpython-36.pyc
│ ├── common_utils.cpython-37.pyc
│ ├── denoising_utils.cpython-36.pyc
│ └── denoising_utils.cpython-37.pyc
│ ├── common_utils.py
│ ├── denoising_utils.py
│ ├── feature_inversion_utils.py
│ ├── inpainting_utils.py
│ ├── matcher.py
│ ├── perceptual_loss
│ ├── __init__.py
│ ├── matcher.py
│ ├── perceptual_loss.py
│ └── vgg_modified.py
│ └── sr_utils.py
├── 2_Super_Resolution
├── BN_Net.py
├── Main_Start.py
├── models
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── __init__.cpython-37.pyc
│ │ ├── common.cpython-36.pyc
│ │ ├── common.cpython-37.pyc
│ │ ├── downsampler.cpython-36.pyc
│ │ ├── downsampler.cpython-37.pyc
│ │ ├── resnet.cpython-36.pyc
│ │ ├── resnet.cpython-37.pyc
│ │ ├── skip.cpython-36.pyc
│ │ ├── skip.cpython-37.pyc
│ │ ├── texture_nets.cpython-36.pyc
│ │ ├── texture_nets.cpython-37.pyc
│ │ ├── unet.cpython-36.pyc
│ │ └── unet.cpython-37.pyc
│ ├── common.py
│ ├── dcgan.py
│ ├── downsampler.py
│ ├── resnet.py
│ ├── skip.py
│ ├── texture_nets.py
│ └── unet.py
├── psnr_ssim.py
├── train_sr.py
├── util.py
└── utils
│ ├── __init__.py
│ ├── __pycache__
│ ├── __init__.cpython-36.pyc
│ ├── __init__.cpython-37.pyc
│ ├── common_utils.cpython-36.pyc
│ ├── common_utils.cpython-37.pyc
│ ├── denoising_utils.cpython-36.pyc
│ ├── denoising_utils.cpython-37.pyc
│ └── sr_utils.cpython-36.pyc
│ ├── common_utils.py
│ ├── denoising_utils.py
│ ├── feature_inversion_utils.py
│ ├── inpainting_utils.py
│ ├── matcher.py
│ ├── perceptual_loss
│ ├── __init__.py
│ ├── matcher.py
│ ├── perceptual_loss.py
│ └── vgg_modified.py
│ └── sr_utils.py
├── 3_Inpainting
├── BN_Net.py
├── Main_Start.py
├── include
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── compression.cpython-36.pyc
│ │ ├── decoder.cpython-36.pyc
│ │ ├── fit.cpython-36.pyc
│ │ ├── helpers.cpython-36.pyc
│ │ ├── visualize.cpython-36.pyc
│ │ └── wavelet.cpython-36.pyc
│ ├── compression.py
│ ├── decoder.py
│ ├── fit.py
│ ├── helpers.py
│ ├── visualize.py
│ └── wavelet.py
├── models
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── __init__.cpython-37.pyc
│ │ ├── common.cpython-36.pyc
│ │ ├── common.cpython-37.pyc
│ │ ├── downsampler.cpython-36.pyc
│ │ ├── downsampler.cpython-37.pyc
│ │ ├── resnet.cpython-36.pyc
│ │ ├── resnet.cpython-37.pyc
│ │ ├── skip.cpython-36.pyc
│ │ ├── skip.cpython-37.pyc
│ │ ├── texture_nets.cpython-36.pyc
│ │ ├── texture_nets.cpython-37.pyc
│ │ ├── unet.cpython-36.pyc
│ │ └── unet.cpython-37.pyc
│ ├── common.py
│ ├── dcgan.py
│ ├── downsampler.py
│ ├── resnet.py
│ ├── skip.py
│ ├── texture_nets.py
│ └── unet.py
├── psnr_ssim.py
├── train_ip.py
├── util.py
└── utils
│ ├── __init__.py
│ ├── __pycache__
│ ├── __init__.cpython-36.pyc
│ ├── __init__.cpython-37.pyc
│ ├── common_utils.cpython-36.pyc
│ ├── common_utils.cpython-37.pyc
│ ├── denoising_utils.cpython-36.pyc
│ └── denoising_utils.cpython-37.pyc
│ ├── common_utils.py
│ ├── denoising_utils.py
│ ├── feature_inversion_utils.py
│ ├── inpainting_utils.py
│ ├── matcher.py
│ ├── perceptual_loss
│ ├── __init__.py
│ ├── matcher.py
│ ├── perceptual_loss.py
│ └── vgg_modified.py
│ └── sr_utils.py
├── LICENSE
├── README.md
└── environment.yml
/0_Dataset/Gaussian_Noise/Clean/Peppers_clean.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Clean/Peppers_clean.npz
--------------------------------------------------------------------------------
/0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers.png
--------------------------------------------------------------------------------
/0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers_Gaussian_Noise_2.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_2/Peppers_Gaussian_Noise_2.npz
--------------------------------------------------------------------------------
/0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers.png
--------------------------------------------------------------------------------
/0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers_Gaussian_Noise_4.npz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Gaussian_Noise/Gaussian_Noise_4/Peppers_Gaussian_Noise_4.npz
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /shelf/
3 | /workspace.xml
4 | # Editor-based HTTP Client requests
5 | /httpRequests/
6 | # Datasource local storage ignored files
7 | /dataSources/
8 | /dataSources.local.xml
9 |
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/.idea/0_IP11.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_img_0.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_img_0.1.png
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_mask_0.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_mask_0.1.png
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_vis_0.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/IP_Dataset_0.1/barbara_vis_0.1.png
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/Original_Images/barbara.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Inpainting/0_IP11/Original_Images/barbara.png
--------------------------------------------------------------------------------
/0_Dataset/Inpainting/0_IP11/generate_mask.py:
--------------------------------------------------------------------------------
1 | ### Generate inpainting mask
2 | ### The mask is a binary matrix that randomly drops 50% pixels
3 |
4 | import numpy as np
5 | import skimage.io as ski
6 | from skimage.transform import resize
7 | import matplotlib.pyplot as plt
8 |
9 |
10 |
11 |
12 | def get_bernoulli_mask(image_shape, zero_fraction=0.5):
13 | img_mask_np = (np.random.random_sample(size=image_shape) > zero_fraction).astype(int)
14 | return img_mask_np
15 |
16 |
17 |
18 | if __name__ == '__main__':
19 | # a = ski.imread('IP_Dataset/barbara_mask.png')
20 | # b = ski.imread('IP_Dataset/barbara_img.png')
21 | # plt.imshow(a * b, cmap='gray')
22 | # plt.show()
23 |
24 | zero_fraction = 0.5 ### how many % of pixels we want to drop/mask
25 | image_list = ['barbara', 'boat', 'Cameraman256', 'couple',
26 | 'fingerprint', 'hill', 'house', 'Lena512',
27 | 'man', 'montage', 'peppers256']
28 |
29 | for cur_img in image_list:
30 | cur_img_path = 'Original_Images/{}.png'.format(cur_img)
31 | cur_img_data = ski.imread(cur_img_path)
32 | h, w = cur_img_data.shape
33 |
34 | if h!=512:
35 | cur_img_data = resize(cur_img_data,(512,512))
36 |
37 | # now, let's get is mask
38 | cur_mask_data = get_bernoulli_mask(cur_img_data.shape, zero_fraction=zero_fraction)
39 |
40 | # plt.figure()
41 | # plt.hist(cur_mask_data.flatten())
42 | # plt.show()
43 |
44 | # now, let's save the clean image and its mask
45 | cur_IP_img_file = 'IP_Dataset_{}/{}_img_{}.png'.format(zero_fraction,cur_img,zero_fraction)
46 | ski.imsave(cur_IP_img_file, cur_img_data)
47 |
48 | cur_IP_mask_file = 'IP_Dataset_{}/{}_mask_{}.png'.format(zero_fraction,cur_img,zero_fraction)
49 | ski.imsave(cur_IP_mask_file,cur_mask_data)
50 |
51 | # visualize them
52 | plt.figure()
53 | plt.subplot(1,3,1)
54 | plt.imshow(cur_img_data, cmap='gray')
55 | plt.subplot(1,3,2)
56 | plt.imshow(cur_mask_data, cmap='gray')
57 | plt.subplot(1, 3, 3)
58 | plt.imshow(cur_img_data*cur_mask_data, cmap='gray')
59 | plt.savefig('IP_Dataset_{}/{}_vis_{}.png'.format(zero_fraction, cur_img, zero_fraction))
60 | plt.close()
61 |
62 |
63 |
--------------------------------------------------------------------------------
/0_Dataset/Super_Resolution/Set5/baby.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/0_Dataset/Super_Resolution/Set5/baby.png
--------------------------------------------------------------------------------
/1_Denoising/BN_Net.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 |
3 | #-----------------------------------------------------------
4 | #-----------------------------------------------------------
5 | # BN for the input seed
6 | #-----------------------------------------------------------
7 | #-----------------------------------------------------------
8 | class BNNet(nn.Module):
9 | def __init__(self,num_channel):
10 | super(BNNet, self).__init__()
11 | self.bn = nn.BatchNorm2d(num_channel)
12 |
13 | def forward(self, input_data):
14 | output_data = self.bn(input_data)
15 | return output_data
16 |
--------------------------------------------------------------------------------
/1_Denoising/Main_Start.py:
--------------------------------------------------------------------------------
1 | from train_denoising import *
2 | import time
3 | import datetime
4 |
5 | #-----------------------------------------------------------
6 | #-----------------------------------------------------------
7 | # This is the Main of the code
8 | #-----------------------------------------------------------
9 | #-----------------------------------------------------------
10 |
11 | if __name__ == '__main__':
12 | # Set random seed for reproducibility
13 | manualSeed = 100
14 | manualSeed = random.randint(1, 10000) # use if you want new results
15 | print("Random Seed: ", manualSeed)
16 | random.seed(manualSeed)
17 | torch.manual_seed(manualSeed)
18 |
19 | # Set up hyper-parameters
20 | #-----------------------------------------------------------
21 | gpu = 0
22 | corr_level = 2
23 |
24 | corr_type = 'Gaussian_Noise'
25 |
26 | corr_type_level = corr_type + '_' + str(corr_level)
27 | print_step = 1
28 | max_epoch = 5000
29 | max_epoch = int(max_epoch / print_step) * print_step + 1
30 |
31 | tv_weight = 0.45
32 | learning_rate_model = 1e-1
33 | OPTIMIZER = 'Adam'
34 | LOSS = "MSE"# or 'Huber' or "L1"
35 |
36 | input_w = 512
37 | input_h = 512
38 | input_c = 3
39 | width = 64
40 | #-----------------------------------------------------------
41 |
42 | image_list = ['Peppers']
43 |
44 | #image_list = ['Baboon', 'F16', 'House',
45 | #'kodim01', 'kodim02', 'kodim03',
46 | #'kodim12', 'Lena', 'Peppers']
47 | ###################################### Processing images one by one #################################
48 | for cur_image in image_list:
49 |
50 | clean_image_path = '../0_Dataset/{}/Clean/{}_clean.npz'.format(corr_type, cur_image)
51 | corrupted_image_path = '../0_Dataset/{}/{}/{}_{}.npz'.format(corr_type,corr_type_level,cur_image,corr_type_level)
52 |
53 | DIP_train(learning_rate_model,
54 | OPTIMIZER,
55 | LOSS,
56 | width,
57 | input_w,
58 | input_h,
59 | input_c,
60 | clean_image_path,
61 | corrupted_image_path,
62 | max_epoch,
63 | print_step,
64 | gpu,
65 | cur_image,
66 | tv_weight
67 | )
--------------------------------------------------------------------------------
/1_Denoising/models/__init__.py:
--------------------------------------------------------------------------------
1 | from .skip import skip
2 | from .texture_nets import get_texture_nets
3 | from .resnet import ResNet
4 | from .unet import UNet
5 |
6 | import torch.nn as nn
7 |
8 | def get_net(input_depth, NET_TYPE, pad, upsample_mode, n_channels=3, act_fun='LeakyReLU', skip_n33d=128, skip_n33u=128, skip_n11=4, num_scales=5, downsample_mode='stride'):
9 | if NET_TYPE == 'ResNet':
10 | # TODO
11 | net = ResNet(input_depth, 3, 10, 16, 1, nn.BatchNorm2d, False)
12 | elif NET_TYPE == 'skip':
13 | net = skip(input_depth, n_channels, num_channels_down = [skip_n33d]*num_scales if isinstance(skip_n33d, int) else skip_n33d,
14 | num_channels_up = [skip_n33u]*num_scales if isinstance(skip_n33u, int) else skip_n33u,
15 | num_channels_skip = [skip_n11]*num_scales if isinstance(skip_n11, int) else skip_n11,
16 | upsample_mode=upsample_mode, downsample_mode=downsample_mode,
17 | need_sigmoid=True, need_bias=True, pad=pad, act_fun=act_fun)
18 |
19 | elif NET_TYPE == 'texture_nets':
20 | net = get_texture_nets(inp=input_depth, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False,pad=pad)
21 |
22 | elif NET_TYPE =='UNet':
23 | net = UNet(num_input_channels=input_depth, num_output_channels=3,
24 | feature_scale=4, more_layers=0, concat_x=False,
25 | upsample_mode=upsample_mode, pad=pad, norm_layer=nn.BatchNorm2d, need_sigmoid=True, need_bias=True)
26 | elif NET_TYPE == 'identity':
27 | assert input_depth == 3
28 | net = nn.Sequential()
29 | else:
30 | assert False
31 |
32 | return net
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/__init__.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/common.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/common.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/common.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/common.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/downsampler.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/downsampler.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/downsampler.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/downsampler.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/resnet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/resnet.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/resnet.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/resnet.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/skip.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/skip.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/skip.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/skip.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/texture_nets.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/texture_nets.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/texture_nets.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/texture_nets.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/unet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/unet.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/__pycache__/unet.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/models/__pycache__/unet.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/models/common.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 | from .downsampler import Downsampler
5 |
6 | def add_module(self, module):
7 | self.add_module(str(len(self) + 1), module)
8 |
9 | torch.nn.Module.add = add_module
10 |
11 | class Concat(nn.Module):
12 | def __init__(self, dim, *args):
13 | super(Concat, self).__init__()
14 | self.dim = dim
15 |
16 | for idx, module in enumerate(args):
17 | self.add_module(str(idx), module)
18 |
19 | def forward(self, input):
20 | inputs = []
21 | for module in self._modules.values():
22 | inputs.append(module(input))
23 |
24 | inputs_shapes2 = [x.shape[2] for x in inputs]
25 | inputs_shapes3 = [x.shape[3] for x in inputs]
26 |
27 | if np.all(np.array(inputs_shapes2) == min(inputs_shapes2)) and np.all(np.array(inputs_shapes3) == min(inputs_shapes3)):
28 | inputs_ = inputs
29 | else:
30 | target_shape2 = min(inputs_shapes2)
31 | target_shape3 = min(inputs_shapes3)
32 |
33 | inputs_ = []
34 | for inp in inputs:
35 | diff2 = (inp.size(2) - target_shape2) // 2
36 | diff3 = (inp.size(3) - target_shape3) // 2
37 | inputs_.append(inp[:, :, diff2: diff2 + target_shape2, diff3:diff3 + target_shape3])
38 |
39 | return torch.cat(inputs_, dim=self.dim)
40 |
41 | def __len__(self):
42 | return len(self._modules)
43 |
44 |
45 | class GenNoise(nn.Module):
46 | def __init__(self, dim2):
47 | super(GenNoise, self).__init__()
48 | self.dim2 = dim2
49 |
50 | def forward(self, input):
51 | a = list(input.size())
52 | a[1] = self.dim2
53 | # print (input.data.type())
54 |
55 | b = torch.zeros(a).type_as(input.data)
56 | b.normal_()
57 |
58 | x = torch.autograd.Variable(b)
59 |
60 | return x
61 |
62 |
63 | class Swish(nn.Module):
64 | """
65 | https://arxiv.org/abs/1710.05941
66 | The hype was so huge that I could not help but try it
67 | """
68 | def __init__(self):
69 | super(Swish, self).__init__()
70 | self.s = nn.Sigmoid()
71 |
72 | def forward(self, x):
73 | return x * self.s(x)
74 |
75 |
76 | def act(act_fun = 'LeakyReLU'):
77 | '''
78 | Either string defining an activation function or module (e.g. nn.ReLU)
79 | '''
80 | if isinstance(act_fun, str):
81 | if act_fun == 'LeakyReLU':
82 | return nn.LeakyReLU(0.2, inplace=True)
83 | elif act_fun == 'Swish':
84 | return Swish()
85 | elif act_fun == 'ELU':
86 | return nn.ELU()
87 | elif act_fun == 'none':
88 | return nn.Sequential()
89 | else:
90 | assert False
91 | else:
92 | return act_fun()
93 |
94 |
95 | def bn(num_features):
96 | return nn.BatchNorm2d(num_features)
97 |
98 |
99 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero', downsample_mode='stride'):
100 | downsampler = None
101 | if stride != 1 and downsample_mode != 'stride':
102 |
103 | if downsample_mode == 'avg':
104 | downsampler = nn.AvgPool2d(stride, stride)
105 | elif downsample_mode == 'max':
106 | downsampler = nn.MaxPool2d(stride, stride)
107 | elif downsample_mode in ['lanczos2', 'lanczos3']:
108 | downsampler = Downsampler(n_planes=out_f, factor=stride, kernel_type=downsample_mode, phase=0.5, preserve_size=True)
109 | else:
110 | assert False
111 |
112 | stride = 1
113 |
114 | padder = None
115 | to_pad = int((kernel_size - 1) / 2)
116 | if pad == 'reflection':
117 | padder = nn.ReflectionPad2d(to_pad)
118 | to_pad = 0
119 |
120 | convolver = nn.Conv2d(in_f, out_f, kernel_size, stride, padding=to_pad, bias=bias)
121 |
122 |
123 | layers = filter(lambda x: x is not None, [padder, convolver, downsampler])
124 | return nn.Sequential(*layers)
--------------------------------------------------------------------------------
/1_Denoising/models/dcgan.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | def dcgan(inp=2,
5 | ndf=32,
6 | num_ups=4, need_sigmoid=True, need_bias=True, pad='zero', upsample_mode='nearest', need_convT = True):
7 |
8 | layers= [nn.ConvTranspose2d(inp, ndf, kernel_size=3, stride=1, padding=0, bias=False),
9 | nn.BatchNorm2d(ndf),
10 | nn.LeakyReLU(True)]
11 |
12 | for i in range(num_ups-3):
13 | if need_convT:
14 | layers += [ nn.ConvTranspose2d(ndf, ndf, kernel_size=4, stride=2, padding=1, bias=False),
15 | nn.BatchNorm2d(ndf),
16 | nn.LeakyReLU(True)]
17 | else:
18 | layers += [ nn.Upsample(scale_factor=2, mode=upsample_mode),
19 | nn.Conv2d(ndf, ndf, kernel_size=3, stride=1, padding=1, bias=False),
20 | nn.BatchNorm2d(ndf),
21 | nn.LeakyReLU(True)]
22 |
23 | if need_convT:
24 | layers += [nn.ConvTranspose2d(ndf, 3, 4, 2, 1, bias=False),]
25 | else:
26 | layers += [nn.Upsample(scale_factor=2, mode='bilinear'),
27 | nn.Conv2d(ndf, 3, kernel_size=3, stride=1, padding=1, bias=False)]
28 |
29 |
30 | if need_sigmoid:
31 | layers += [nn.Sigmoid()]
32 |
33 | model =nn.Sequential(*layers)
34 | return model
--------------------------------------------------------------------------------
/1_Denoising/models/downsampler.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 | import torch.nn as nn
4 |
5 | class Downsampler(nn.Module):
6 | '''
7 | http://www.realitypixels.com/turk/computergraphics/ResamplingFilters.pdf
8 | '''
9 | def __init__(self, n_planes, factor, kernel_type, phase=0, kernel_width=None, support=None, sigma=None, preserve_size=False):
10 | super(Downsampler, self).__init__()
11 |
12 | assert phase in [0, 0.5], 'phase should be 0 or 0.5'
13 |
14 | if kernel_type == 'lanczos2':
15 | support = 2
16 | kernel_width = 4 * factor + 1
17 | kernel_type_ = 'lanczos'
18 |
19 | elif kernel_type == 'lanczos3':
20 | support = 3
21 | kernel_width = 6 * factor + 1
22 | kernel_type_ = 'lanczos'
23 |
24 | elif kernel_type == 'gauss12':
25 | kernel_width = 7
26 | sigma = 1/2
27 | kernel_type_ = 'gauss'
28 |
29 | elif kernel_type == 'gauss1sq2':
30 | kernel_width = 9
31 | sigma = 1./np.sqrt(2)
32 | kernel_type_ = 'gauss'
33 |
34 | elif kernel_type in ['lanczos', 'gauss', 'box']:
35 | kernel_type_ = kernel_type
36 |
37 | else:
38 | assert False, 'wrong name kernel'
39 |
40 |
41 | # note that `kernel width` will be different to actual size for phase = 1/2
42 | self.kernel = get_kernel(factor, kernel_type_, phase, kernel_width, support=support, sigma=sigma)
43 |
44 | downsampler = nn.Conv2d(n_planes, n_planes, kernel_size=self.kernel.shape, stride=factor, padding=0)
45 | downsampler.weight.data[:] = 0
46 | downsampler.bias.data[:] = 0
47 |
48 | kernel_torch = torch.from_numpy(self.kernel)
49 | for i in range(n_planes):
50 | downsampler.weight.data[i, i] = kernel_torch
51 |
52 | self.downsampler_ = downsampler
53 |
54 | if preserve_size:
55 |
56 | if self.kernel.shape[0] % 2 == 1:
57 | pad = int((self.kernel.shape[0] - 1) / 2.)
58 | else:
59 | pad = int((self.kernel.shape[0] - factor) / 2.)
60 |
61 | self.padding = nn.ReplicationPad2d(pad)
62 |
63 | self.preserve_size = preserve_size
64 |
65 | def forward(self, input):
66 | if self.preserve_size:
67 | x = self.padding(input)
68 | else:
69 | x= input
70 | self.x = x
71 | return self.downsampler_(x)
72 |
73 | def get_kernel(factor, kernel_type, phase, kernel_width, support=None, sigma=None):
74 | assert kernel_type in ['lanczos', 'gauss', 'box']
75 |
76 | # factor = float(factor)
77 | if phase == 0.5 and kernel_type != 'box':
78 | kernel = np.zeros([kernel_width - 1, kernel_width - 1])
79 | else:
80 | kernel = np.zeros([kernel_width, kernel_width])
81 |
82 |
83 | if kernel_type == 'box':
84 | assert phase == 0.5, 'Box filter is always half-phased'
85 | kernel[:] = 1./(kernel_width * kernel_width)
86 |
87 | elif kernel_type == 'gauss':
88 | assert sigma, 'sigma is not specified'
89 | assert phase != 0.5, 'phase 1/2 for gauss not implemented'
90 |
91 | center = (kernel_width + 1.)/2.
92 | print(center, kernel_width)
93 | sigma_sq = sigma * sigma
94 |
95 | for i in range(1, kernel.shape[0] + 1):
96 | for j in range(1, kernel.shape[1] + 1):
97 | di = (i - center)/2.
98 | dj = (j - center)/2.
99 | kernel[i - 1][j - 1] = np.exp(-(di * di + dj * dj)/(2 * sigma_sq))
100 | kernel[i - 1][j - 1] = kernel[i - 1][j - 1]/(2. * np.pi * sigma_sq)
101 | elif kernel_type == 'lanczos':
102 | assert support, 'support is not specified'
103 | center = (kernel_width + 1) / 2.
104 |
105 | for i in range(1, kernel.shape[0] + 1):
106 | for j in range(1, kernel.shape[1] + 1):
107 |
108 | if phase == 0.5:
109 | di = abs(i + 0.5 - center) / factor
110 | dj = abs(j + 0.5 - center) / factor
111 | else:
112 | di = abs(i - center) / factor
113 | dj = abs(j - center) / factor
114 |
115 |
116 | pi_sq = np.pi * np.pi
117 |
118 | val = 1
119 | if di != 0:
120 | val = val * support * np.sin(np.pi * di) * np.sin(np.pi * di / support)
121 | val = val / (np.pi * np.pi * di * di)
122 |
123 | if dj != 0:
124 | val = val * support * np.sin(np.pi * dj) * np.sin(np.pi * dj / support)
125 | val = val / (np.pi * np.pi * dj * dj)
126 |
127 | kernel[i - 1][j - 1] = val
128 |
129 |
130 | else:
131 | assert False, 'wrong method name'
132 |
133 | kernel /= kernel.sum()
134 |
135 | return kernel
136 |
137 | #a = Downsampler(n_planes=3, factor=2, kernel_type='lanczos2', phase='1', preserve_size=True)
138 |
139 |
140 |
141 |
142 |
143 |
144 | #################
145 | # Learnable downsampler
146 |
147 | # KS = 32
148 | # dow = nn.Sequential(nn.ReplicationPad2d(int((KS - factor) / 2.)), nn.Conv2d(1,1,KS,factor))
149 |
150 | # class Apply(nn.Module):
151 | # def __init__(self, what, dim, *args):
152 | # super(Apply, self).__init__()
153 | # self.dim = dim
154 |
155 | # self.what = what
156 |
157 | # def forward(self, input):
158 | # inputs = []
159 | # for i in range(input.size(self.dim)):
160 | # inputs.append(self.what(input.narrow(self.dim, i, 1)))
161 |
162 | # return torch.cat(inputs, dim=self.dim)
163 |
164 | # def __len__(self):
165 | # return len(self._modules)
166 |
167 | # downs = Apply(dow, 1)
168 | # downs.type(dtype)(net_input.type(dtype)).size()
169 |
--------------------------------------------------------------------------------
/1_Denoising/models/resnet.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from numpy.random import normal
4 | from numpy.linalg import svd
5 | from math import sqrt
6 | import torch.nn.init
7 | from .common import *
8 |
9 | class ResidualSequential(nn.Sequential):
10 | def __init__(self, *args):
11 | super(ResidualSequential, self).__init__(*args)
12 |
13 | def forward(self, x):
14 | out = super(ResidualSequential, self).forward(x)
15 | # print(x.size(), out.size())
16 | x_ = None
17 | if out.size(2) != x.size(2) or out.size(3) != x.size(3):
18 | diff2 = x.size(2) - out.size(2)
19 | diff3 = x.size(3) - out.size(3)
20 | # print(1)
21 | x_ = x[:, :, diff2 /2:out.size(2) + diff2 / 2, diff3 / 2:out.size(3) + diff3 / 2]
22 | else:
23 | x_ = x
24 | return out + x_
25 |
26 | def eval(self):
27 | print(2)
28 | for m in self.modules():
29 | m.eval()
30 | exit()
31 |
32 |
33 | def get_block(num_channels, norm_layer, act_fun):
34 | layers = [
35 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False),
36 | norm_layer(num_channels, affine=True),
37 | act(act_fun),
38 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False),
39 | norm_layer(num_channels, affine=True),
40 | ]
41 | return layers
42 |
43 |
44 | class ResNet(nn.Module):
45 | def __init__(self, num_input_channels, num_output_channels, num_blocks, num_channels, need_residual=True, act_fun='LeakyReLU', need_sigmoid=True, norm_layer=nn.BatchNorm2d, pad='reflection'):
46 | '''
47 | pad = 'start|zero|replication'
48 | '''
49 | super(ResNet, self).__init__()
50 |
51 | if need_residual:
52 | s = ResidualSequential
53 | else:
54 | s = nn.Sequential
55 |
56 | stride = 1
57 | # First layers
58 | layers = [
59 | # nn.ReplicationPad2d(num_blocks * 2 * stride + 3),
60 | conv(num_input_channels, num_channels, 3, stride=1, bias=True, pad=pad),
61 | act(act_fun)
62 | ]
63 | # Residual blocks
64 | # layers_residual = []
65 | for i in range(num_blocks):
66 | layers += [s(*get_block(num_channels, norm_layer, act_fun))]
67 |
68 | layers += [
69 | nn.Conv2d(num_channels, num_channels, 3, 1, 1),
70 | norm_layer(num_channels, affine=True)
71 | ]
72 |
73 | # if need_residual:
74 | # layers += [ResidualSequential(*layers_residual)]
75 | # else:
76 | # layers += [Sequential(*layers_residual)]
77 |
78 | # if factor >= 2:
79 | # # Do upsampling if needed
80 | # layers += [
81 | # nn.Conv2d(num_channels, num_channels *
82 | # factor ** 2, 3, 1),
83 | # nn.PixelShuffle(factor),
84 | # act(act_fun)
85 | # ]
86 | layers += [
87 | conv(num_channels, num_output_channels, 3, 1, bias=True, pad=pad),
88 | nn.Sigmoid()
89 | ]
90 | self.model = nn.Sequential(*layers)
91 |
92 | def forward(self, input):
93 | return self.model(input)
94 |
95 | def eval(self):
96 | self.model.eval()
97 |
--------------------------------------------------------------------------------
/1_Denoising/models/skip.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from .common import *
4 |
5 | def skip(
6 | num_input_channels=2, num_output_channels=3,
7 | num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4],
8 | filter_size_down=3, filter_size_up=3, filter_skip_size=1,
9 | need_sigmoid=True, need_bias=True,
10 | pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU',
11 | need1x1_up=True):
12 | """Assembles encoder-decoder with skip connections.
13 |
14 | Arguments:
15 | act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU)
16 | pad (string): zero|reflection (default: 'zero')
17 | upsample_mode (string): 'nearest|bilinear' (default: 'nearest')
18 | downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride')
19 |
20 | """
21 | assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip)
22 |
23 | n_scales = len(num_channels_down)
24 |
25 | if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) :
26 | upsample_mode = [upsample_mode]*n_scales
27 |
28 | if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)):
29 | downsample_mode = [downsample_mode]*n_scales
30 |
31 | if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) :
32 | filter_size_down = [filter_size_down]*n_scales
33 |
34 | if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) :
35 | filter_size_up = [filter_size_up]*n_scales
36 |
37 | last_scale = n_scales - 1
38 |
39 | cur_depth = None
40 |
41 | model = nn.Sequential()
42 | model_tmp = model
43 |
44 | input_depth = num_input_channels
45 | for i in range(len(num_channels_down)):
46 |
47 | deeper = nn.Sequential()
48 | skip = nn.Sequential()
49 |
50 | if num_channels_skip[i] != 0:
51 | model_tmp.add(Concat(1, skip, deeper))
52 | else:
53 | model_tmp.add(deeper)
54 |
55 | model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i])))
56 |
57 | if num_channels_skip[i] != 0:
58 | skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad))
59 | skip.add(bn(num_channels_skip[i]))
60 | skip.add(act(act_fun))
61 |
62 | # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part))
63 |
64 | deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i]))
65 | deeper.add(bn(num_channels_down[i]))
66 | deeper.add(act(act_fun))
67 |
68 | deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad))
69 | deeper.add(bn(num_channels_down[i]))
70 | deeper.add(act(act_fun))
71 |
72 | deeper_main = nn.Sequential()
73 |
74 | if i == len(num_channels_down) - 1:
75 | # The deepest
76 | k = num_channels_down[i]
77 | else:
78 | deeper.add(deeper_main)
79 | k = num_channels_up[i + 1]
80 |
81 | deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i]))
82 |
83 | model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad))
84 | model_tmp.add(bn(num_channels_up[i]))
85 | model_tmp.add(act(act_fun))
86 |
87 |
88 | if need1x1_up:
89 | model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad))
90 | model_tmp.add(bn(num_channels_up[i]))
91 | model_tmp.add(act(act_fun))
92 |
93 | input_depth = num_channels_down[i]
94 | model_tmp = deeper_main
95 |
96 | model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad))
97 | if need_sigmoid:
98 | model.add(nn.Sigmoid())
99 |
100 | return model
101 |
--------------------------------------------------------------------------------
/1_Denoising/models/texture_nets.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from .common import *
4 |
5 |
6 | normalization = nn.BatchNorm2d
7 |
8 |
9 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero'):
10 | if pad == 'zero':
11 | return nn.Conv2d(in_f, out_f, kernel_size, stride, padding=(kernel_size - 1) / 2, bias=bias)
12 | elif pad == 'reflection':
13 | layers = [nn.ReflectionPad2d((kernel_size - 1) / 2),
14 | nn.Conv2d(in_f, out_f, kernel_size, stride, padding=0, bias=bias)]
15 | return nn.Sequential(*layers)
16 |
17 | def get_texture_nets(inp=3, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False, pad='zero', need_sigmoid=False, conv_num=8, upsample_mode='nearest'):
18 |
19 |
20 | for i in range(len(ratios)):
21 | j = i + 1
22 |
23 | seq = nn.Sequential()
24 |
25 | tmp = nn.AvgPool2d(ratios[i], ratios[i])
26 |
27 | seq.add(tmp)
28 | if fill_noise:
29 | seq.add(GenNoise(inp))
30 |
31 | seq.add(conv(inp, conv_num, 3, pad=pad))
32 | seq.add(normalization(conv_num))
33 | seq.add(act())
34 |
35 | seq.add(conv(conv_num, conv_num, 3, pad=pad))
36 | seq.add(normalization(conv_num))
37 | seq.add(act())
38 |
39 | seq.add(conv(conv_num, conv_num, 1, pad=pad))
40 | seq.add(normalization(conv_num))
41 | seq.add(act())
42 |
43 | if i == 0:
44 | seq.add(nn.Upsample(scale_factor=2, mode=upsample_mode))
45 | cur = seq
46 | else:
47 |
48 | cur_temp = cur
49 |
50 | cur = nn.Sequential()
51 |
52 | # Batch norm before merging
53 | seq.add(normalization(conv_num))
54 | cur_temp.add(normalization(conv_num * (j - 1)))
55 |
56 | cur.add(Concat(1, cur_temp, seq))
57 |
58 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad))
59 | cur.add(normalization(conv_num * j))
60 | cur.add(act())
61 |
62 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad))
63 | cur.add(normalization(conv_num * j))
64 | cur.add(act())
65 |
66 | cur.add(conv(conv_num * j, conv_num * j, 1, pad=pad))
67 | cur.add(normalization(conv_num * j))
68 | cur.add(act())
69 |
70 | if i == len(ratios) - 1:
71 | cur.add(conv(conv_num * j, 3, 1, pad=pad))
72 | else:
73 | cur.add(nn.Upsample(scale_factor=2, mode=upsample_mode))
74 |
75 | model = cur
76 | if need_sigmoid:
77 | model.add(nn.Sigmoid())
78 |
79 | return model
80 |
--------------------------------------------------------------------------------
/1_Denoising/psnr_ssim.py:
--------------------------------------------------------------------------------
1 | ### This script will calcualte psnr and ssim
2 | import numpy as np
3 | from skimage.metrics import peak_signal_noise_ratio
4 | from skimage.measure import compare_ssim
5 |
6 |
7 | def calcualte_PSNR(im_true, im_test):
8 | im_true = np.transpose(im_true,(1,2,0))
9 | im_test = np.transpose(im_test, (1, 2, 0))
10 | psnr_value = peak_signal_noise_ratio(im_true, im_test)
11 | return psnr_value
12 |
13 | def calcualte_SSIM(im_true, im_test, multichannel): # multichannel=True for RGB images
14 | im_true = np.transpose(im_true, (1, 2, 0))
15 | im_test = np.transpose(im_test, (1, 2, 0))
16 | ssim_value = compare_ssim(im_true, im_test, multichannel=multichannel, data_range=im_test.max() - im_test.min())
17 | return ssim_value
18 |
19 | if __name__ == '__main__':
20 | pass
--------------------------------------------------------------------------------
/1_Denoising/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__init__.py
--------------------------------------------------------------------------------
/1_Denoising/utils/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/utils/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/__init__.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/utils/__pycache__/common_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/common_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/utils/__pycache__/common_utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/common_utils.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/utils/__pycache__/denoising_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/denoising_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/1_Denoising/utils/__pycache__/denoising_utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/__pycache__/denoising_utils.cpython-37.pyc
--------------------------------------------------------------------------------
/1_Denoising/utils/common_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision
4 | import sys
5 |
6 | import numpy as np
7 | from PIL import Image
8 | import PIL
9 | import numpy as np
10 |
11 | import matplotlib.pyplot as plt
12 |
13 | def crop_image(img, d=32):
14 | '''Make dimensions divisible by `d`'''
15 |
16 | new_size = (img.size[0] - img.size[0] % d,
17 | img.size[1] - img.size[1] % d)
18 |
19 | bbox = [
20 | int((img.size[0] - new_size[0])/2),
21 | int((img.size[1] - new_size[1])/2),
22 | int((img.size[0] + new_size[0])/2),
23 | int((img.size[1] + new_size[1])/2),
24 | ]
25 |
26 | img_cropped = img.crop(bbox)
27 | return img_cropped
28 |
29 | def get_params(opt_over, net, net_input, downsampler=None):
30 | '''Returns parameters that we want to optimize over.
31 |
32 | Args:
33 | opt_over: comma separated list, e.g. "net,input" or "net"
34 | net: network
35 | net_input: torch.Tensor that stores input `z`
36 | '''
37 | opt_over_list = opt_over.split(',')
38 | params = []
39 |
40 | for opt in opt_over_list:
41 |
42 | if opt == 'net':
43 | params += [x for x in net.parameters() ]
44 | elif opt=='down':
45 | assert downsampler is not None
46 | params = [x for x in downsampler.parameters()]
47 | elif opt == 'input':
48 | net_input.requires_grad = True
49 | params += [net_input]
50 | else:
51 | assert False, 'what is it?'
52 |
53 | return params
54 |
55 | def get_image_grid(images_np, nrow=8):
56 | '''Creates a grid from a list of images by concatenating them.'''
57 | images_torch = [torch.from_numpy(x) for x in images_np]
58 | torch_grid = torchvision.utils.make_grid(images_torch, nrow)
59 |
60 | return torch_grid.numpy()
61 |
62 | def plot_image_grid(images_np, nrow =8, factor=1, interpolation='lanczos'):
63 | """Draws images in a grid
64 |
65 | Args:
66 | images_np: list of images, each image is np.array of size 3xHxW of 1xHxW
67 | nrow: how many images will be in one row
68 | factor: size if the plt.figure
69 | interpolation: interpolation used in plt.imshow
70 | """
71 | n_channels = max(x.shape[0] for x in images_np)
72 | assert (n_channels == 3) or (n_channels == 1), "images should have 1 or 3 channels"
73 |
74 | images_np = [x if (x.shape[0] == n_channels) else np.concatenate([x, x, x], axis=0) for x in images_np]
75 |
76 | grid = get_image_grid(images_np, nrow)
77 |
78 | plt.figure(figsize=(len(images_np) + factor, 12 + factor))
79 |
80 | if images_np[0].shape[0] == 1:
81 | plt.imshow(grid[0], cmap='gray', interpolation=interpolation)
82 | else:
83 | plt.imshow(grid.transpose(1, 2, 0), interpolation=interpolation)
84 |
85 | plt.show()
86 |
87 | return grid
88 |
89 | def load(path):
90 | """Load PIL image."""
91 | img = Image.open(path)
92 | return img
93 |
94 | def get_image(path, imsize=-1):
95 | """Load an image and resize to a cpecific size.
96 |
97 | Args:
98 | path: path to image
99 | imsize: tuple or scalar with dimensions; -1 for `no resize`
100 | """
101 | img = load(path)
102 |
103 | if isinstance(imsize, int):
104 | imsize = (imsize, imsize)
105 |
106 | if imsize[0]!= -1 and img.size != imsize:
107 | if imsize[0] > img.size[0]:
108 | img = img.resize(imsize, Image.BICUBIC)
109 | else:
110 | img = img.resize(imsize, Image.ANTIALIAS)
111 |
112 | img_np = pil_to_np(img)
113 |
114 | return img, img_np
115 |
116 |
117 |
118 | def fill_noise(x, noise_type):
119 | """Fills tensor `x` with noise of type `noise_type`."""
120 | if noise_type == 'u':
121 | x.uniform_()
122 | elif noise_type == 'n':
123 | x.normal_()
124 | else:
125 | assert False
126 |
127 | def get_noise(input_depth, method, spatial_size, noise_type='u', var=1./10):
128 | """Returns a pytorch.Tensor of size (1 x `input_depth` x `spatial_size[0]` x `spatial_size[1]`)
129 | initialized in a specific way.
130 | Args:
131 | input_depth: number of channels in the tensor
132 | method: `noise` for fillting tensor with noise; `meshgrid` for np.meshgrid
133 | spatial_size: spatial size of the tensor to initialize
134 | noise_type: 'u' for uniform; 'n' for normal
135 | var: a factor, a noise will be multiplicated by. Basically it is standard deviation scaler.
136 | """
137 | if isinstance(spatial_size, int):
138 | spatial_size = (spatial_size, spatial_size)
139 | if method == 'noise':
140 | shape = [1, input_depth, spatial_size[0], spatial_size[1]]
141 | net_input = torch.zeros(shape)
142 |
143 | fill_noise(net_input, noise_type)
144 | net_input *= var
145 | elif method == 'meshgrid':
146 | assert input_depth == 2
147 | X, Y = np.meshgrid(np.arange(0, spatial_size[1])/float(spatial_size[1]-1), np.arange(0, spatial_size[0])/float(spatial_size[0]-1))
148 | meshgrid = np.concatenate([X[None,:], Y[None,:]])
149 | net_input= np_to_torch(meshgrid)
150 | else:
151 | assert False
152 |
153 | return net_input
154 |
155 | def pil_to_np(img_PIL):
156 | '''Converts image in PIL format to np.array.
157 |
158 | From W x H x C [0...255] to C x W x H [0..1]
159 | '''
160 | ar = np.array(img_PIL)
161 |
162 | if len(ar.shape) == 3:
163 | ar = ar.transpose(2,0,1)
164 | else:
165 | ar = ar[None, ...]
166 |
167 | return ar.astype(np.float32) / 255.
168 |
169 | def np_to_pil(img_np):
170 | '''Converts image in np.array format to PIL image.
171 |
172 | From C x W x H [0..1] to W x H x C [0...255]
173 | '''
174 | ar = np.clip(img_np*255,0,255).astype(np.uint8)
175 |
176 | if img_np.shape[0] == 1:
177 | ar = ar[0]
178 | else:
179 | ar = ar.transpose(1, 2, 0)
180 |
181 | return Image.fromarray(ar)
182 |
183 | def np_to_torch(img_np):
184 | '''Converts image in numpy.array to torch.Tensor.
185 |
186 | From C x W x H [0..1] to C x W x H [0..1]
187 | '''
188 | return torch.from_numpy(img_np)[None, :]
189 |
190 | def torch_to_np(img_var):
191 | '''Converts an image in torch.Tensor format to np.array.
192 |
193 | From 1 x C x W x H [0..1] to C x W x H [0..1]
194 | '''
195 | return img_var.detach().cpu().numpy()[0]
196 |
197 |
198 | def optimize(optimizer_type, parameters, closure, LR, num_iter):
199 | """Runs optimization loop.
200 |
201 | Args:
202 | optimizer_type: 'LBFGS' of 'adam'
203 | parameters: list of Tensors to optimize over
204 | closure: function, that returns loss variable
205 | LR: learning rate
206 | num_iter: number of iterations
207 | """
208 | if optimizer_type == 'LBFGS':
209 | # Do several steps with adam first
210 | optimizer = torch.optim.Adam(parameters, lr=0.001)
211 | for j in range(100):
212 | optimizer.zero_grad()
213 | closure()
214 | optimizer.step()
215 |
216 | print('Starting optimization with LBFGS')
217 | def closure2():
218 | optimizer.zero_grad()
219 | return closure()
220 | optimizer = torch.optim.LBFGS(parameters, max_iter=num_iter, lr=LR, tolerance_grad=-1, tolerance_change=-1)
221 | optimizer.step(closure2)
222 |
223 | elif optimizer_type == 'adam':
224 | print('Starting optimization with ADAM')
225 | optimizer = torch.optim.Adam(parameters, lr=LR)
226 |
227 | for j in range(num_iter):
228 | optimizer.zero_grad()
229 | closure()
230 | optimizer.step()
231 | else:
232 | assert False
--------------------------------------------------------------------------------
/1_Denoising/utils/denoising_utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | from .common_utils import *
3 |
4 |
5 |
6 | def get_noisy_image(img_np, sigma):
7 | """Adds Gaussian noise to an image.
8 |
9 | Args:
10 | img_np: image, np.array with values from 0 to 1
11 | sigma: std of the noise
12 | """
13 | img_noisy_np = np.clip(img_np + np.random.normal(scale=sigma, size=img_np.shape), 0, 1).astype(np.float32)
14 | img_noisy_pil = np_to_pil(img_noisy_np)
15 |
16 | return img_noisy_pil, img_noisy_np
--------------------------------------------------------------------------------
/1_Denoising/utils/feature_inversion_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision.transforms as transforms
4 | import torchvision.models as models
5 | from .matcher import Matcher
6 | import os
7 | from collections import OrderedDict
8 |
9 | class View(nn.Module):
10 | def __init__(self):
11 | super(View, self).__init__()
12 |
13 | def forward(self, x):
14 | return x.view(-1)
15 |
16 | def get_vanilla_vgg_features(cut_idx=-1):
17 | if not os.path.exists('vgg_features.pth'):
18 | os.system(
19 | 'wget --no-check-certificate -N https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth')
20 | vgg_weights = torch.load('vgg19-d01eb7cb.pth')
21 | # fix compatibility issues
22 | map = {'classifier.6.weight':u'classifier.7.weight', 'classifier.6.bias':u'classifier.7.bias'}
23 | vgg_weights = OrderedDict([(map[k] if k in map else k,v) for k,v in vgg_weights.iteritems()])
24 |
25 |
26 |
27 | model = models.vgg19()
28 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values())
29 |
30 |
31 | model.load_state_dict(vgg_weights)
32 |
33 | torch.save(model.features, 'vgg_features.pth')
34 | torch.save(model.classifier, 'vgg_classifier.pth')
35 |
36 | vgg = torch.load('vgg_features.pth')
37 | if cut_idx > 36:
38 | vgg_classifier = torch.load('vgg_classifier.pth')
39 | vgg = nn.Sequential(*(vgg._modules.values() + vgg_classifier._modules.values()))
40 |
41 | vgg.eval()
42 |
43 | return vgg
44 |
45 |
46 | def get_matcher(net, opt):
47 | idxs = [x for x in opt['layers'].split(',')]
48 | matcher = Matcher(opt['what'])
49 |
50 | def hook(module, input, output):
51 | matcher(module, output)
52 |
53 | for i in idxs:
54 | net._modules[i].register_forward_hook(hook)
55 |
56 | return matcher
57 |
58 |
59 |
60 | def get_vgg(cut_idx=-1):
61 | f = get_vanilla_vgg_features(cut_idx)
62 |
63 | if cut_idx > 0:
64 | num_modules = len(f._modules)
65 | keys_to_delete = [f._modules.keys()[x] for x in range(cut_idx, num_modules)]
66 | for k in keys_to_delete:
67 | del f._modules[k]
68 |
69 | return f
70 |
71 | def vgg_preprocess_var(var):
72 | (r, g, b) = torch.chunk(var, 3, dim=1)
73 | bgr = torch.cat((b, g, r), 1)
74 | out = bgr * 255 - torch.autograd.Variable(vgg_mean[None, ...]).type(var.type()).expand_as(bgr)
75 | return out
76 |
77 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1)
78 |
79 |
80 |
81 | def get_preprocessor(imsize):
82 | def vgg_preprocess(tensor):
83 | (r, g, b) = torch.chunk(tensor, 3, dim=0)
84 | bgr = torch.cat((b, g, r), 0)
85 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr)
86 | return out
87 | preprocess = transforms.Compose([
88 | transforms.Resize(imsize),
89 | transforms.ToTensor(),
90 | transforms.Lambda(vgg_preprocess)
91 | ])
92 |
93 | return preprocess
94 |
95 |
96 | def get_deprocessor():
97 | def vgg_deprocess(tensor):
98 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0
99 | (b, g, r) = torch.chunk(bgr, 3, dim=0)
100 | rgb = torch.cat((r, g, b), 0)
101 | return rgb
102 | deprocess = transforms.Compose([
103 | transforms.Lambda(vgg_deprocess),
104 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)),
105 | transforms.ToPILImage()
106 | ])
107 | return deprocess
108 |
--------------------------------------------------------------------------------
/1_Denoising/utils/inpainting_utils.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from PIL import Image
3 | import PIL.ImageDraw as ImageDraw
4 | import PIL.ImageFont as ImageFont
5 | from .common_utils import *
6 |
7 | def get_text_mask(for_image, sz=20):
8 | font_fname = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf'
9 | font_size = sz
10 | font = ImageFont.truetype(font_fname, font_size)
11 |
12 | img_mask = Image.fromarray(np.array(for_image)*0+255)
13 | draw = ImageDraw.Draw(img_mask)
14 | draw.text((128, 128), "hello world", font=font, fill='rgb(0, 0, 0)')
15 |
16 | return img_mask
17 |
18 | def get_bernoulli_mask(for_image, zero_fraction=0.95):
19 | img_mask_np=(np.random.random_sample(size=pil_to_np(for_image).shape) > zero_fraction).astype(int)
20 | img_mask = np_to_pil(img_mask_np)
21 |
22 | return img_mask
23 |
--------------------------------------------------------------------------------
/1_Denoising/utils/matcher.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | class Matcher:
5 | def __init__(self, how='gram_matrix', loss='mse'):
6 | self.mode = 'store'
7 | self.stored = {}
8 | self.losses = {}
9 |
10 | if how in all_features.keys():
11 | self.get_statistics = all_features[how]
12 | else:
13 | assert False
14 | pass
15 |
16 | if loss in all_losses.keys():
17 | self.loss = all_losses[loss]
18 | else:
19 | assert False
20 |
21 | def __call__(self, module, features):
22 | statistics = self.get_statistics(features)
23 |
24 | self.statistics = statistics
25 | if self.mode == 'store':
26 | self.stored[module] = statistics.detach().clone()
27 | elif self.mode == 'match':
28 | self.losses[module] = self.loss(statistics, self.stored[module])
29 |
30 | def clean(self):
31 | self.losses = {}
32 |
33 | def gram_matrix(x):
34 | (b, ch, h, w) = x.size()
35 | features = x.view(b, ch, w * h)
36 | features_t = features.transpose(1, 2)
37 | gram = features.bmm(features_t) / (ch * h * w)
38 | return gram
39 |
40 |
41 | def features(x):
42 | return x
43 |
44 |
45 | all_features = {
46 | 'gram_matrix': gram_matrix,
47 | 'features': features,
48 | }
49 |
50 | all_losses = {
51 | 'mse': nn.MSELoss(),
52 | 'smoothL1': nn.SmoothL1Loss(),
53 | 'L1': nn.L1Loss(),
54 | }
55 |
--------------------------------------------------------------------------------
/1_Denoising/utils/perceptual_loss/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/1_Denoising/utils/perceptual_loss/__init__.py
--------------------------------------------------------------------------------
/1_Denoising/utils/perceptual_loss/matcher.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | class Matcher:
6 | def __init__(self, how='gram_matrix', loss='mse', map_index=933):
7 | self.mode = 'store'
8 | self.stored = {}
9 | self.losses = {}
10 |
11 | if how in all_features.keys():
12 | self.get_statistics = all_features[how]
13 | else:
14 | assert False
15 | pass
16 |
17 | if loss in all_losses.keys():
18 | self.loss = all_losses[loss]
19 | else:
20 | assert False
21 |
22 | self.map_index = map_index
23 | self.method = 'match'
24 |
25 |
26 | def __call__(self, module, features):
27 | statistics = self.get_statistics(features)
28 |
29 | self.statistics = statistics
30 | if self.mode == 'store':
31 | self.stored[module] = statistics.detach()
32 |
33 | elif self.mode == 'match':
34 |
35 | if statistics.ndimension() == 2:
36 |
37 | if self.method == 'maximize':
38 | self.losses[module] = - statistics[0, self.map_index]
39 | else:
40 | self.losses[module] = torch.abs(300 - statistics[0, self.map_index])
41 |
42 | else:
43 | ws = self.window_size
44 |
45 | t = statistics.detach() * 0
46 |
47 | s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
48 | t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
49 | t_cc[:, self.map_index,...] = 1
50 |
51 | if self.method == 'maximize':
52 | self.losses[module] = -(s_cc * t_cc.contiguous()).sum()
53 | else:
54 | self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum()
55 |
56 |
57 | def clean(self):
58 | self.losses = {}
59 |
60 | def gram_matrix(x):
61 | (b, ch, h, w) = x.size()
62 | features = x.view(b, ch, w * h)
63 | features_t = features.transpose(1, 2)
64 | gram = features.bmm(features_t) / (ch * h * w)
65 | return gram
66 |
67 |
68 | def features(x):
69 | return x
70 |
71 |
72 | all_features = {
73 | 'gram_matrix': gram_matrix,
74 | 'features': features,
75 | }
76 |
77 | all_losses = {
78 | 'mse': nn.MSELoss(),
79 | 'smoothL1': nn.SmoothL1Loss(),
80 | 'L1': nn.L1Loss(),
81 | }
82 |
--------------------------------------------------------------------------------
/1_Denoising/utils/perceptual_loss/perceptual_loss.py:
--------------------------------------------------------------------------------
1 | import os
2 | import torch
3 | import torch.nn as nn
4 | import torchvision.transforms as transforms
5 | import torchvision.models as models
6 | from .matcher import Matcher
7 | from collections import OrderedDict
8 |
9 | from torchvision.models.vgg import model_urls
10 | from torchvision.models import vgg19
11 | from torch.autograd import Variable
12 |
13 | from .vgg_modified import VGGModified
14 |
15 | def get_pretrained_net(name):
16 | """Loads pretrained network"""
17 | if name == 'alexnet_caffe':
18 | if not os.path.exists('alexnet-torch_py3.pth'):
19 | print('Downloading AlexNet')
20 | os.system('wget -O alexnet-torch_py3.pth --no-check-certificate -nc https://box.skoltech.ru/index.php/s/77xSWvrDN0CiQtK/download')
21 | return torch.load('alexnet-torch_py3.pth')
22 | elif name == 'vgg19_caffe':
23 | if not os.path.exists('vgg19-caffe-py3.pth'):
24 | print('Downloading VGG-19')
25 | os.system('wget -O vgg19-caffe-py3.pth --no-check-certificate -nc https://box.skoltech.ru/index.php/s/HPcOFQTjXxbmp4X/download')
26 |
27 | vgg = get_vgg19_caffe()
28 |
29 | return vgg
30 | elif name == 'vgg16_caffe':
31 | if not os.path.exists('vgg16-caffe-py3.pth'):
32 | print('Downloading VGG-16')
33 | os.system('wget -O vgg16-caffe-py3.pth --no-check-certificate -nc https://box.skoltech.ru/index.php/s/TUZ62HnPKWdxyLr/download')
34 |
35 | vgg = get_vgg16_caffe()
36 |
37 | return vgg
38 | elif name == 'vgg19_pytorch_modified':
39 | # os.system('wget -O data/feature_inversion/vgg19-caffe.pth --no-check-certificate -nc https://www.dropbox.com/s/xlbdo688dy4keyk/vgg19-caffe.pth?dl=1')
40 |
41 | model = VGGModified(vgg19(pretrained=False), 0.2)
42 | model.load_state_dict(torch.load('vgg_pytorch_modified.pkl')['state_dict'])
43 |
44 | return model
45 | else:
46 | assert False
47 |
48 |
49 | class PerceputalLoss(nn.modules.loss._Loss):
50 | """
51 | Assumes input image is in range [0,1] if `input_range` is 'sigmoid', [-1, 1] if 'tanh'
52 | """
53 | def __init__(self, input_range='sigmoid',
54 | net_type = 'vgg_torch',
55 | input_preprocessing='corresponding',
56 | match=[{'layers':[11,20,29],'what':'features'}]):
57 |
58 | if input_range not in ['sigmoid', 'tanh']:
59 | assert False
60 |
61 | self.net = get_pretrained_net(net_type).cuda()
62 |
63 | self.matchers = [get_matcher(self.net, match_opts) for match_opts in match]
64 |
65 | preprocessing_correspondence = {
66 | 'vgg19_torch': vgg_preprocess_caffe,
67 | 'vgg16_torch': vgg_preprocess_caffe,
68 | 'vgg19_pytorch': vgg_preprocess_pytorch,
69 | 'vgg19_pytorch_modified': vgg_preprocess_pytorch,
70 | }
71 |
72 | if input_preprocessing == 'corresponding':
73 | self.preprocess_input = preprocessing_correspondence[net_type]
74 | else:
75 | self.preprocessing = preprocessing_correspondence[input_preprocessing]
76 |
77 | def preprocess_input(self, x):
78 | if self.input_range == 'tanh':
79 | x = (x + 1.) / 2.
80 |
81 | return self.preprocess(x)
82 |
83 | def __call__(self, x, y):
84 |
85 | # for
86 | self.matcher_content.mode = 'store'
87 | self.net(self.preprocess_input(y));
88 |
89 | self.matcher_content.mode = 'match'
90 | self.net(self.preprocess_input(x));
91 |
92 | return sum([sum(matcher.losses.values()) for matcher in self.matchers])
93 |
94 |
95 | def get_vgg19_caffe():
96 | model = vgg19()
97 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values())
98 | vgg = model.features
99 | vgg_classifier = model.classifier
100 |
101 | names = ['conv1_1','relu1_1','conv1_2','relu1_2','pool1',
102 | 'conv2_1','relu2_1','conv2_2','relu2_2','pool2',
103 | 'conv3_1','relu3_1','conv3_2','relu3_2','conv3_3','relu3_3','conv3_4','relu3_4','pool3',
104 | 'conv4_1','relu4_1','conv4_2','relu4_2','conv4_3','relu4_3','conv4_4','relu4_4','pool4',
105 | 'conv5_1','relu5_1','conv5_2','relu5_2','conv5_3','relu5_3','conv5_4','relu5_4','pool5',
106 | 'torch_view','fc6','relu6','drop6','fc7','relu7','drop7','fc8']
107 |
108 | model = nn.Sequential()
109 | for n, m in zip(names, list(vgg) + list(vgg_classifier)):
110 | model.add_module(n, m)
111 |
112 | model.load_state_dict(torch.load('vgg19-caffe-py3.pth'))
113 |
114 | return model
115 |
116 | def get_vgg16_caffe():
117 | vgg = torch.load('vgg16-caffe-py3.pth')
118 |
119 | names = ['conv1_1','relu1_1','conv1_2','relu1_2','pool1',
120 | 'conv2_1','relu2_1','conv2_2','relu2_2','pool2',
121 | 'conv3_1','relu3_1','conv3_2','relu3_2','conv3_3','relu3_3','pool3',
122 | 'conv4_1','relu4_1','conv4_2','relu4_2','conv4_3','relu4_3','pool4',
123 | 'conv5_1','relu5_1','conv5_2','relu5_2','conv5_3','relu5_3','pool5',
124 | 'torch_view','fc6','relu6','drop6','fc7','relu7','fc8']
125 |
126 | model = nn.Sequential()
127 | for n, m in zip(names, list(vgg)):
128 | model.add_module(n, m)
129 |
130 | # model.load_state_dict(torch.load('vgg19-caffe-py3.pth'))
131 |
132 | return model
133 |
134 |
135 | class View(nn.Module):
136 | def __init__(self):
137 | super(View, self).__init__()
138 |
139 | def forward(self, x):
140 | return x.view(x.size(0), -1)
141 |
142 |
143 | def get_matcher(vgg, opt):
144 | # idxs = [int(x) for x in opt['layers'].split(',')]
145 | matcher = Matcher(opt['what'], 'mse', opt['map_idx'])
146 |
147 | def hook(module, input, output):
148 | matcher(module, output)
149 |
150 | for layer_name in opt['layers']:
151 | vgg._modules[layer_name].register_forward_hook(hook)
152 |
153 | return matcher
154 |
155 |
156 | def get_vgg(cut_idx=-1, vgg_type='pytorch'):
157 | f = get_vanilla_vgg_features(cut_idx, vgg_type)
158 |
159 | keys = [x for x in cnn._modules.keys()]
160 | max_idx = max(keys.index(x) for x in opt_content['layers'].split(','))
161 | for k in keys[max_idx+1:]:
162 | cnn._modules.pop(k)
163 |
164 | return f
165 |
166 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1)
167 | def vgg_preprocess_caffe(var):
168 | (r, g, b) = torch.chunk(var, 3, dim=1)
169 | bgr = torch.cat((b, g, r), 1)
170 | out = bgr * 255 - torch.autograd.Variable(vgg_mean).type(var.type())
171 | return out
172 |
173 |
174 |
175 | mean_pytorch = Variable(torch.Tensor([0.485, 0.456, 0.406]).view(3, 1, 1))
176 | std_pytorch = Variable(torch.Tensor([0.229, 0.224, 0.225]).view(3, 1, 1))
177 |
178 | def vgg_preprocess_pytorch(var):
179 | return (var - mean_pytorch.type_as(var))/std_pytorch.type_as(var)
180 |
181 |
182 |
183 | def get_preprocessor(imsize):
184 | def vgg_preprocess(tensor):
185 | (r, g, b) = torch.chunk(tensor, 3, dim=0)
186 | bgr = torch.cat((b, g, r), 0)
187 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr)
188 | return out
189 | preprocess = transforms.Compose([
190 | transforms.Resize(imsize),
191 | transforms.ToTensor(),
192 | transforms.Lambda(vgg_preprocess)
193 | ])
194 |
195 | return preprocess
196 |
197 |
198 | def get_deprocessor():
199 | def vgg_deprocess(tensor):
200 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0
201 | (b, g, r) = torch.chunk(bgr, 3, dim=0)
202 | rgb = torch.cat((r, g, b), 0)
203 | return rgb
204 | deprocess = transforms.Compose([
205 | transforms.Lambda(vgg_deprocess),
206 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)),
207 | transforms.ToPILImage()
208 | ])
209 | return deprocess
210 |
211 |
--------------------------------------------------------------------------------
/1_Denoising/utils/perceptual_loss/vgg_modified.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 |
3 | class VGGModified(nn.Module):
4 | def __init__(self, vgg19_orig, slope=0.01):
5 | super(VGGModified, self).__init__()
6 | self.features = nn.Sequential()
7 |
8 | self.features.add_module(str(0), vgg19_orig.features[0])
9 | self.features.add_module(str(1), nn.LeakyReLU(slope, True))
10 | self.features.add_module(str(2), vgg19_orig.features[2])
11 | self.features.add_module(str(3), nn.LeakyReLU(slope, True))
12 | self.features.add_module(str(4), nn.AvgPool2d((2,2), (2,2)))
13 |
14 | self.features.add_module(str(5), vgg19_orig.features[5])
15 | self.features.add_module(str(6), nn.LeakyReLU(slope, True))
16 | self.features.add_module(str(7), vgg19_orig.features[7])
17 | self.features.add_module(str(8), nn.LeakyReLU(slope, True))
18 | self.features.add_module(str(9), nn.AvgPool2d((2,2), (2,2)))
19 |
20 | self.features.add_module(str(10), vgg19_orig.features[10])
21 | self.features.add_module(str(11), nn.LeakyReLU(slope, True))
22 | self.features.add_module(str(12), vgg19_orig.features[12])
23 | self.features.add_module(str(13), nn.LeakyReLU(slope, True))
24 | self.features.add_module(str(14), vgg19_orig.features[14])
25 | self.features.add_module(str(15), nn.LeakyReLU(slope, True))
26 | self.features.add_module(str(16), vgg19_orig.features[16])
27 | self.features.add_module(str(17), nn.LeakyReLU(slope, True))
28 | self.features.add_module(str(18), nn.AvgPool2d((2,2), (2,2)))
29 |
30 | self.features.add_module(str(19), vgg19_orig.features[19])
31 | self.features.add_module(str(20), nn.LeakyReLU(slope, True))
32 | self.features.add_module(str(21), vgg19_orig.features[21])
33 | self.features.add_module(str(22), nn.LeakyReLU(slope, True))
34 | self.features.add_module(str(23), vgg19_orig.features[23])
35 | self.features.add_module(str(24), nn.LeakyReLU(slope, True))
36 | self.features.add_module(str(25), vgg19_orig.features[25])
37 | self.features.add_module(str(26), nn.LeakyReLU(slope, True))
38 | self.features.add_module(str(27), nn.AvgPool2d((2,2), (2,2)))
39 |
40 | self.features.add_module(str(28), vgg19_orig.features[28])
41 | self.features.add_module(str(29), nn.LeakyReLU(slope, True))
42 | self.features.add_module(str(30), vgg19_orig.features[30])
43 | self.features.add_module(str(31), nn.LeakyReLU(slope, True))
44 | self.features.add_module(str(32), vgg19_orig.features[32])
45 | self.features.add_module(str(33), nn.LeakyReLU(slope, True))
46 | self.features.add_module(str(34), vgg19_orig.features[34])
47 | self.features.add_module(str(35), nn.LeakyReLU(slope, True))
48 | self.features.add_module(str(36), nn.AvgPool2d((2,2), (2,2)))
49 |
50 | self.classifier = nn.Sequential()
51 |
52 | self.classifier.add_module(str(0), vgg19_orig.classifier[0])
53 | self.classifier.add_module(str(1), nn.LeakyReLU(slope, True))
54 | self.classifier.add_module(str(2), nn.Dropout2d(p = 0.5))
55 | self.classifier.add_module(str(3), vgg19_orig.classifier[3])
56 | self.classifier.add_module(str(4), nn.LeakyReLU(slope, True))
57 | self.classifier.add_module(str(5), nn.Dropout2d(p = 0.5))
58 | self.classifier.add_module(str(6), vgg19_orig.classifier[6])
59 |
60 | def forward(self, x):
61 | return self.classifier(self.features.forward(x))
--------------------------------------------------------------------------------
/1_Denoising/utils/sr_utils.py:
--------------------------------------------------------------------------------
1 | from .common_utils import *
2 |
3 | def put_in_center(img_np, target_size):
4 | img_out = np.zeros([3, target_size[0], target_size[1]])
5 |
6 | bbox = [
7 | int((target_size[0] - img_np.shape[1]) / 2),
8 | int((target_size[1] - img_np.shape[2]) / 2),
9 | int((target_size[0] + img_np.shape[1]) / 2),
10 | int((target_size[1] + img_np.shape[2]) / 2),
11 | ]
12 |
13 | img_out[:, bbox[0]:bbox[2], bbox[1]:bbox[3]] = img_np
14 |
15 | return img_out
16 |
17 |
18 | def load_LR_HR_imgs_sr(fname, imsize, factor, enforse_div32=None):
19 | '''Loads an image, resizes it, center crops and downscales.
20 |
21 | Args:
22 | fname: path to the image
23 | imsize: new size for the image, -1 for no resizing
24 | factor: downscaling factor
25 | enforse_div32: if 'CROP' center crops an image, so that its dimensions are divisible by 32.
26 | '''
27 | img_orig_pil, img_orig_np = get_image(fname, -1)
28 |
29 | if imsize != -1:
30 | img_orig_pil, img_orig_np = get_image(fname, imsize)
31 |
32 | # For comparison with GT
33 | if enforse_div32 == 'CROP':
34 | new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 32,
35 | img_orig_pil.size[1] - img_orig_pil.size[1] % 32)
36 |
37 | bbox = [
38 | (img_orig_pil.size[0] - new_size[0])/2,
39 | (img_orig_pil.size[1] - new_size[1])/2,
40 | (img_orig_pil.size[0] + new_size[0])/2,
41 | (img_orig_pil.size[1] + new_size[1])/2,
42 | ]
43 |
44 | img_HR_pil = img_orig_pil.crop(bbox)
45 | img_HR_np = pil_to_np(img_HR_pil)
46 | else:
47 | img_HR_pil, img_HR_np = img_orig_pil, img_orig_np
48 |
49 | LR_size = [
50 | img_HR_pil.size[0] // factor,
51 | img_HR_pil.size[1] // factor
52 | ]
53 |
54 | img_LR_pil = img_HR_pil.resize(LR_size, Image.ANTIALIAS)
55 | img_LR_np = pil_to_np(img_LR_pil)
56 |
57 | print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str (img_LR_pil.size)))
58 |
59 | return {
60 | 'orig_pil': img_orig_pil,
61 | 'orig_np': img_orig_np,
62 | 'LR_pil': img_LR_pil,
63 | 'LR_np': img_LR_np,
64 | 'HR_pil': img_HR_pil,
65 | 'HR_np': img_HR_np
66 | }
67 |
68 |
69 | def get_baselines(img_LR_pil, img_HR_pil):
70 | '''Gets `bicubic`, sharpened bicubic and `nearest` baselines.'''
71 | img_bicubic_pil = img_LR_pil.resize(img_HR_pil.size, Image.BICUBIC)
72 | img_bicubic_np = pil_to_np(img_bicubic_pil)
73 |
74 | img_nearest_pil = img_LR_pil.resize(img_HR_pil.size, Image.NEAREST)
75 | img_nearest_np = pil_to_np(img_nearest_pil)
76 |
77 | img_bic_sharp_pil = img_bicubic_pil.filter(PIL.ImageFilter.UnsharpMask())
78 | img_bic_sharp_np = pil_to_np(img_bic_sharp_pil)
79 |
80 | return img_bicubic_np, img_bic_sharp_np, img_nearest_np
81 |
82 |
83 |
84 | def tv_loss(x, beta = 0.5):
85 | '''Calculates TV loss for an image `x`.
86 |
87 | Args:
88 | x: image, torch.Variable of torch.Tensor
89 | beta: See https://arxiv.org/abs/1412.0035 (fig. 2) to see effect of `beta`
90 | '''
91 | dh = torch.pow(x[:,:,:,1:] - x[:,:,:,:-1], 2)
92 | dw = torch.pow(x[:,:,1:,:] - x[:,:,:-1,:], 2)
93 |
94 | return torch.sum(torch.pow(dh[:, :, :-1] + dw[:, :, :, :-1], beta))
95 |
--------------------------------------------------------------------------------
/2_Super_Resolution/BN_Net.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 |
3 |
4 |
5 | #-----------------------------------------------------------
6 | #-----------------------------------------------------------
7 | # BN for the input seed
8 | #-----------------------------------------------------------
9 | #-----------------------------------------------------------
10 | class BNNet(nn.Module):
11 | def __init__(self,num_channel):
12 | super(BNNet, self).__init__()
13 | self.bn = nn.BatchNorm2d(num_channel)
14 |
15 | def forward(self, input_data):
16 | output_data = self.bn(input_data)
17 | return output_data
18 |
--------------------------------------------------------------------------------
/2_Super_Resolution/Main_Start.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from train_sr import *
4 | import time
5 | import datetime
6 | # from include import *
7 | from utils import *
8 | from utils.sr_utils import *
9 | from utils.common_utils import *
10 |
11 |
12 |
13 |
14 | #-----------------------------------------------------------
15 | #-----------------------------------------------------------
16 | # This is the Main of the code
17 | #-----------------------------------------------------------
18 | #-----------------------------------------------------------
19 |
20 |
21 | if __name__ == '__main__':
22 | # Set random seed for reproducibility
23 | manualSeed = 100
24 | manualSeed = random.randint(1, 10000) # use if you want new results
25 | print("Random Seed: ", manualSeed)
26 | random.seed(manualSeed)
27 | torch.manual_seed(manualSeed)
28 |
29 | ##### Set up hyper-parameters
30 | #-----------------------------------------------------------
31 | gpu = 1
32 |
33 | print_step = 1
34 | max_epoch = 5000
35 | max_epoch = int(max_epoch / print_step) * print_step + 1
36 |
37 | #-----------------------------------------------------------
38 | tv_weight = 0.75 #
39 | learning_rate_model = 0.5
40 | OPTIMIZER = 'Adam'
41 | LOSS = "MSE"#'Huber' or "L1"
42 |
43 | factor = 4
44 |
45 | # image_list = ['Peppers']
46 |
47 | # image_list = ['baby', 'bird', 'butterfly', 'head', 'woman',
48 | # 'baboon', 'barbara', 'bridge', 'coastguard', 'comic', 'face', 'flowers',
49 | # 'foreman', 'lenna', 'man', 'monarch', 'pepper', 'ppt3', 'zebra']
50 |
51 | # Set5 = ['baby', 'bird', 'butterfly', 'head', 'woman']
52 |
53 | # Set14 = ['baboon', 'barbara', 'bridge', 'coastguard', 'comic', 'face', 'flowers',
54 | # 'foreman', 'lenna', 'man', 'monarch', 'pepper', 'ppt3', 'zebra']
55 |
56 | Set5 = ['baby']
57 |
58 | Set14 = ['baboon', 'barbara', 'bridge', 'coastguard', 'comic', 'face', 'flowers',
59 | 'foreman', 'lenna', 'man', 'monarch', 'pepper', 'ppt3', 'zebra']
60 |
61 |
62 | image_list = Set5
63 | ###################################### Processing images one by one #################################
64 | for cur_image in image_list:
65 | # set up the path info
66 | if cur_image in Set5:
67 | path = '../0_Dataset/Super_Resolution/Set5/'
68 | elif cur_image in Set14:
69 | path = '../0_Dataset/Super_Resolution/Set14/'
70 | else:
71 | assert False, 'Error: please put your images under Set5 or Set14!'
72 |
73 | #---load image
74 | PLOT = False
75 | imsize = -1
76 | enforse_div32 = 'CROP' # set to be NONE for Set5/14 'CROP' # we usually need the dimensions to be divisible by a power of two (32 in this case)
77 | # Starts here
78 | path_to_image = os.path.join(path, cur_image+'.png')
79 | imgs = load_LR_HR_imgs_sr(path_to_image, imsize, factor, enforse_div32)
80 |
81 | imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np'] = get_baselines(imgs['LR_pil'], imgs['HR_pil'])
82 |
83 | if PLOT:
84 | plot_image_grid([imgs['HR_np'], imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np']], 4, 12)
85 | print('PSNR bicubic: %.4f PSNR nearest: %.4f' % (
86 | compare_psnr(imgs['HR_np'], imgs['bicubic_np']),
87 | compare_psnr(imgs['HR_np'], imgs['nearest_np'])))
88 |
89 | ### set up input noise width, height, and channel
90 | HR_np_shape = imgs['HR_np'].shape
91 | input_h = HR_np_shape[1]
92 | input_w = HR_np_shape[2]
93 | input_c = 1
94 | width = 128
95 |
96 | DIP_train(imgs,
97 | learning_rate_model,
98 | OPTIMIZER,
99 | LOSS,
100 | width,
101 | input_w,
102 | input_h,
103 | input_c,
104 | max_epoch,
105 | print_step,
106 | gpu,
107 | cur_image,
108 | tv_weight,
109 | factor
110 | )
111 |
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__init__.py:
--------------------------------------------------------------------------------
1 | from .skip import skip
2 | from .texture_nets import get_texture_nets
3 | from .resnet import ResNet
4 | from .unet import UNet
5 |
6 | import torch.nn as nn
7 |
8 | def get_net(input_depth, NET_TYPE, pad, upsample_mode, n_channels=3, act_fun='LeakyReLU', skip_n33d=128, skip_n33u=128, skip_n11=4, num_scales=5, downsample_mode='stride'):
9 | if NET_TYPE == 'ResNet':
10 | # TODO
11 | net = ResNet(input_depth, 3, 10, 16, 1, nn.BatchNorm2d, False)
12 | elif NET_TYPE == 'skip':
13 | net = skip(input_depth, n_channels, num_channels_down = [skip_n33d]*num_scales if isinstance(skip_n33d, int) else skip_n33d,
14 | num_channels_up = [skip_n33u]*num_scales if isinstance(skip_n33u, int) else skip_n33u,
15 | num_channels_skip = [skip_n11]*num_scales if isinstance(skip_n11, int) else skip_n11,
16 | upsample_mode=upsample_mode, downsample_mode=downsample_mode,
17 | need_sigmoid=True, need_bias=True, pad=pad, act_fun=act_fun)
18 |
19 | elif NET_TYPE == 'texture_nets':
20 | net = get_texture_nets(inp=input_depth, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False,pad=pad)
21 |
22 | elif NET_TYPE =='UNet':
23 | net = UNet(num_input_channels=input_depth, num_output_channels=3,
24 | feature_scale=4, more_layers=0, concat_x=False,
25 | upsample_mode=upsample_mode, pad=pad, norm_layer=nn.BatchNorm2d, need_sigmoid=True, need_bias=True)
26 | elif NET_TYPE == 'identity':
27 | assert input_depth == 3
28 | net = nn.Sequential()
29 | else:
30 | assert False
31 |
32 | return net
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/__init__.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/common.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/common.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/common.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/common.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/downsampler.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/downsampler.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/downsampler.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/downsampler.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/resnet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/resnet.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/resnet.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/resnet.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/skip.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/skip.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/skip.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/skip.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/texture_nets.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/texture_nets.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/texture_nets.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/texture_nets.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/unet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/unet.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/__pycache__/unet.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/models/__pycache__/unet.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/models/common.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 | from .downsampler import Downsampler
5 |
6 | def add_module(self, module):
7 | self.add_module(str(len(self) + 1), module)
8 |
9 | torch.nn.Module.add = add_module
10 |
11 | class Concat(nn.Module):
12 | def __init__(self, dim, *args):
13 | super(Concat, self).__init__()
14 | self.dim = dim
15 |
16 | for idx, module in enumerate(args):
17 | self.add_module(str(idx), module)
18 |
19 | def forward(self, input):
20 | inputs = []
21 | for module in self._modules.values():
22 | inputs.append(module(input))
23 |
24 | inputs_shapes2 = [x.shape[2] for x in inputs]
25 | inputs_shapes3 = [x.shape[3] for x in inputs]
26 |
27 | if np.all(np.array(inputs_shapes2) == min(inputs_shapes2)) and np.all(np.array(inputs_shapes3) == min(inputs_shapes3)):
28 | inputs_ = inputs
29 | else:
30 | target_shape2 = min(inputs_shapes2)
31 | target_shape3 = min(inputs_shapes3)
32 |
33 | inputs_ = []
34 | for inp in inputs:
35 | diff2 = (inp.size(2) - target_shape2) // 2
36 | diff3 = (inp.size(3) - target_shape3) // 2
37 | inputs_.append(inp[:, :, diff2: diff2 + target_shape2, diff3:diff3 + target_shape3])
38 |
39 | return torch.cat(inputs_, dim=self.dim)
40 |
41 | def __len__(self):
42 | return len(self._modules)
43 |
44 |
45 | class GenNoise(nn.Module):
46 | def __init__(self, dim2):
47 | super(GenNoise, self).__init__()
48 | self.dim2 = dim2
49 |
50 | def forward(self, input):
51 | a = list(input.size())
52 | a[1] = self.dim2
53 | # print (input.data.type())
54 |
55 | b = torch.zeros(a).type_as(input.data)
56 | b.normal_()
57 |
58 | x = torch.autograd.Variable(b)
59 |
60 | return x
61 |
62 |
63 | class Swish(nn.Module):
64 | """
65 | https://arxiv.org/abs/1710.05941
66 | The hype was so huge that I could not help but try it
67 | """
68 | def __init__(self):
69 | super(Swish, self).__init__()
70 | self.s = nn.Sigmoid()
71 |
72 | def forward(self, x):
73 | return x * self.s(x)
74 |
75 |
76 | def act(act_fun = 'LeakyReLU'):
77 | '''
78 | Either string defining an activation function or module (e.g. nn.ReLU)
79 | '''
80 | if isinstance(act_fun, str):
81 | if act_fun == 'LeakyReLU':
82 | return nn.LeakyReLU(0.2, inplace=True)
83 | elif act_fun == 'Swish':
84 | return Swish()
85 | elif act_fun == 'ELU':
86 | return nn.ELU()
87 | elif act_fun == 'none':
88 | return nn.Sequential()
89 | else:
90 | assert False
91 | else:
92 | return act_fun()
93 |
94 |
95 | def bn(num_features):
96 | return nn.BatchNorm2d(num_features)
97 |
98 |
99 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero', downsample_mode='stride'):
100 | downsampler = None
101 | if stride != 1 and downsample_mode != 'stride':
102 |
103 | if downsample_mode == 'avg':
104 | downsampler = nn.AvgPool2d(stride, stride)
105 | elif downsample_mode == 'max':
106 | downsampler = nn.MaxPool2d(stride, stride)
107 | elif downsample_mode in ['lanczos2', 'lanczos3']:
108 | downsampler = Downsampler(n_planes=out_f, factor=stride, kernel_type=downsample_mode, phase=0.5, preserve_size=True)
109 | else:
110 | assert False
111 |
112 | stride = 1
113 |
114 | padder = None
115 | to_pad = int((kernel_size - 1) / 2)
116 | if pad == 'reflection':
117 | padder = nn.ReflectionPad2d(to_pad)
118 | to_pad = 0
119 |
120 | convolver = nn.Conv2d(in_f, out_f, kernel_size, stride, padding=to_pad, bias=bias)
121 |
122 |
123 | layers = filter(lambda x: x is not None, [padder, convolver, downsampler])
124 | return nn.Sequential(*layers)
--------------------------------------------------------------------------------
/2_Super_Resolution/models/dcgan.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | def dcgan(inp=2,
5 | ndf=32,
6 | num_ups=4, need_sigmoid=True, need_bias=True, pad='zero', upsample_mode='nearest', need_convT = True):
7 |
8 | layers= [nn.ConvTranspose2d(inp, ndf, kernel_size=3, stride=1, padding=0, bias=False),
9 | nn.BatchNorm2d(ndf),
10 | nn.LeakyReLU(True)]
11 |
12 | for i in range(num_ups-3):
13 | if need_convT:
14 | layers += [ nn.ConvTranspose2d(ndf, ndf, kernel_size=4, stride=2, padding=1, bias=False),
15 | nn.BatchNorm2d(ndf),
16 | nn.LeakyReLU(True)]
17 | else:
18 | layers += [ nn.Upsample(scale_factor=2, mode=upsample_mode),
19 | nn.Conv2d(ndf, ndf, kernel_size=3, stride=1, padding=1, bias=False),
20 | nn.BatchNorm2d(ndf),
21 | nn.LeakyReLU(True)]
22 |
23 | if need_convT:
24 | layers += [nn.ConvTranspose2d(ndf, 3, 4, 2, 1, bias=False),]
25 | else:
26 | layers += [nn.Upsample(scale_factor=2, mode='bilinear'),
27 | nn.Conv2d(ndf, 3, kernel_size=3, stride=1, padding=1, bias=False)]
28 |
29 |
30 | if need_sigmoid:
31 | layers += [nn.Sigmoid()]
32 |
33 | model =nn.Sequential(*layers)
34 | return model
--------------------------------------------------------------------------------
/2_Super_Resolution/models/downsampler.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 | import torch.nn as nn
4 |
5 | class Downsampler(nn.Module):
6 | '''
7 | http://www.realitypixels.com/turk/computergraphics/ResamplingFilters.pdf
8 | '''
9 | def __init__(self, n_planes, factor, kernel_type, phase=0, kernel_width=None, support=None, sigma=None, preserve_size=False):
10 | super(Downsampler, self).__init__()
11 |
12 | assert phase in [0, 0.5], 'phase should be 0 or 0.5'
13 |
14 | if kernel_type == 'lanczos2':
15 | support = 2
16 | kernel_width = 4 * factor + 1
17 | kernel_type_ = 'lanczos'
18 |
19 | elif kernel_type == 'lanczos3':
20 | support = 3
21 | kernel_width = 6 * factor + 1
22 | kernel_type_ = 'lanczos'
23 |
24 | elif kernel_type == 'gauss12':
25 | kernel_width = 7
26 | sigma = 1/2
27 | kernel_type_ = 'gauss'
28 |
29 | elif kernel_type == 'gauss1sq2':
30 | kernel_width = 9
31 | sigma = 1./np.sqrt(2)
32 | kernel_type_ = 'gauss'
33 |
34 | elif kernel_type in ['lanczos', 'gauss', 'box']:
35 | kernel_type_ = kernel_type
36 |
37 | else:
38 | assert False, 'wrong name kernel'
39 |
40 |
41 | # note that `kernel width` will be different to actual size for phase = 1/2
42 | self.kernel = get_kernel(factor, kernel_type_, phase, kernel_width, support=support, sigma=sigma)
43 |
44 | downsampler = nn.Conv2d(n_planes, n_planes, kernel_size=self.kernel.shape, stride=factor, padding=0)
45 | downsampler.weight.data[:] = 0
46 | downsampler.bias.data[:] = 0
47 |
48 | kernel_torch = torch.from_numpy(self.kernel)
49 | for i in range(n_planes):
50 | downsampler.weight.data[i, i] = kernel_torch
51 |
52 | self.downsampler_ = downsampler
53 |
54 | if preserve_size:
55 |
56 | if self.kernel.shape[0] % 2 == 1:
57 | pad = int((self.kernel.shape[0] - 1) / 2.)
58 | else:
59 | pad = int((self.kernel.shape[0] - factor) / 2.)
60 |
61 | self.padding = nn.ReplicationPad2d(pad)
62 |
63 | self.preserve_size = preserve_size
64 |
65 | def forward(self, input):
66 | if self.preserve_size:
67 | x = self.padding(input)
68 | else:
69 | x= input
70 | self.x = x
71 | return self.downsampler_(x)
72 |
73 | def get_kernel(factor, kernel_type, phase, kernel_width, support=None, sigma=None):
74 | assert kernel_type in ['lanczos', 'gauss', 'box']
75 |
76 | # factor = float(factor)
77 | if phase == 0.5 and kernel_type != 'box':
78 | kernel = np.zeros([kernel_width - 1, kernel_width - 1])
79 | else:
80 | kernel = np.zeros([kernel_width, kernel_width])
81 |
82 |
83 | if kernel_type == 'box':
84 | assert phase == 0.5, 'Box filter is always half-phased'
85 | kernel[:] = 1./(kernel_width * kernel_width)
86 |
87 | elif kernel_type == 'gauss':
88 | assert sigma, 'sigma is not specified'
89 | assert phase != 0.5, 'phase 1/2 for gauss not implemented'
90 |
91 | center = (kernel_width + 1.)/2.
92 | print(center, kernel_width)
93 | sigma_sq = sigma * sigma
94 |
95 | for i in range(1, kernel.shape[0] + 1):
96 | for j in range(1, kernel.shape[1] + 1):
97 | di = (i - center)/2.
98 | dj = (j - center)/2.
99 | kernel[i - 1][j - 1] = np.exp(-(di * di + dj * dj)/(2 * sigma_sq))
100 | kernel[i - 1][j - 1] = kernel[i - 1][j - 1]/(2. * np.pi * sigma_sq)
101 | elif kernel_type == 'lanczos':
102 | assert support, 'support is not specified'
103 | center = (kernel_width + 1) / 2.
104 |
105 | for i in range(1, kernel.shape[0] + 1):
106 | for j in range(1, kernel.shape[1] + 1):
107 |
108 | if phase == 0.5:
109 | di = abs(i + 0.5 - center) / factor
110 | dj = abs(j + 0.5 - center) / factor
111 | else:
112 | di = abs(i - center) / factor
113 | dj = abs(j - center) / factor
114 |
115 |
116 | pi_sq = np.pi * np.pi
117 |
118 | val = 1
119 | if di != 0:
120 | val = val * support * np.sin(np.pi * di) * np.sin(np.pi * di / support)
121 | val = val / (np.pi * np.pi * di * di)
122 |
123 | if dj != 0:
124 | val = val * support * np.sin(np.pi * dj) * np.sin(np.pi * dj / support)
125 | val = val / (np.pi * np.pi * dj * dj)
126 |
127 | kernel[i - 1][j - 1] = val
128 |
129 |
130 | else:
131 | assert False, 'wrong method name'
132 |
133 | kernel /= kernel.sum()
134 |
135 | return kernel
136 |
137 | #a = Downsampler(n_planes=3, factor=2, kernel_type='lanczos2', phase='1', preserve_size=True)
138 |
139 |
140 |
141 |
142 |
143 |
144 | #################
145 | # Learnable downsampler
146 |
147 | # KS = 32
148 | # dow = nn.Sequential(nn.ReplicationPad2d(int((KS - factor) / 2.)), nn.Conv2d(1,1,KS,factor))
149 |
150 | # class Apply(nn.Module):
151 | # def __init__(self, what, dim, *args):
152 | # super(Apply, self).__init__()
153 | # self.dim = dim
154 |
155 | # self.what = what
156 |
157 | # def forward(self, input):
158 | # inputs = []
159 | # for i in range(input.size(self.dim)):
160 | # inputs.append(self.what(input.narrow(self.dim, i, 1)))
161 |
162 | # return torch.cat(inputs, dim=self.dim)
163 |
164 | # def __len__(self):
165 | # return len(self._modules)
166 |
167 | # downs = Apply(dow, 1)
168 | # downs.type(dtype)(net_input.type(dtype)).size()
169 |
--------------------------------------------------------------------------------
/2_Super_Resolution/models/resnet.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from numpy.random import normal
4 | from numpy.linalg import svd
5 | from math import sqrt
6 | import torch.nn.init
7 | from .common import *
8 |
9 | class ResidualSequential(nn.Sequential):
10 | def __init__(self, *args):
11 | super(ResidualSequential, self).__init__(*args)
12 |
13 | def forward(self, x):
14 | out = super(ResidualSequential, self).forward(x)
15 | # print(x.size(), out.size())
16 | x_ = None
17 | if out.size(2) != x.size(2) or out.size(3) != x.size(3):
18 | diff2 = x.size(2) - out.size(2)
19 | diff3 = x.size(3) - out.size(3)
20 | # print(1)
21 | x_ = x[:, :, diff2 /2:out.size(2) + diff2 / 2, diff3 / 2:out.size(3) + diff3 / 2]
22 | else:
23 | x_ = x
24 | return out + x_
25 |
26 | def eval(self):
27 | print(2)
28 | for m in self.modules():
29 | m.eval()
30 | exit()
31 |
32 |
33 | def get_block(num_channels, norm_layer, act_fun):
34 | layers = [
35 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False),
36 | norm_layer(num_channels, affine=True),
37 | act(act_fun),
38 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False),
39 | norm_layer(num_channels, affine=True),
40 | ]
41 | return layers
42 |
43 |
44 | class ResNet(nn.Module):
45 | def __init__(self, num_input_channels, num_output_channels, num_blocks, num_channels, need_residual=True, act_fun='LeakyReLU', need_sigmoid=True, norm_layer=nn.BatchNorm2d, pad='reflection'):
46 | '''
47 | pad = 'start|zero|replication'
48 | '''
49 | super(ResNet, self).__init__()
50 |
51 | if need_residual:
52 | s = ResidualSequential
53 | else:
54 | s = nn.Sequential
55 |
56 | stride = 1
57 | # First layers
58 | layers = [
59 | # nn.ReplicationPad2d(num_blocks * 2 * stride + 3),
60 | conv(num_input_channels, num_channels, 3, stride=1, bias=True, pad=pad),
61 | act(act_fun)
62 | ]
63 | # Residual blocks
64 | # layers_residual = []
65 | for i in range(num_blocks):
66 | layers += [s(*get_block(num_channels, norm_layer, act_fun))]
67 |
68 | layers += [
69 | nn.Conv2d(num_channels, num_channels, 3, 1, 1),
70 | norm_layer(num_channels, affine=True)
71 | ]
72 |
73 | # if need_residual:
74 | # layers += [ResidualSequential(*layers_residual)]
75 | # else:
76 | # layers += [Sequential(*layers_residual)]
77 |
78 | # if factor >= 2:
79 | # # Do upsampling if needed
80 | # layers += [
81 | # nn.Conv2d(num_channels, num_channels *
82 | # factor ** 2, 3, 1),
83 | # nn.PixelShuffle(factor),
84 | # act(act_fun)
85 | # ]
86 | layers += [
87 | conv(num_channels, num_output_channels, 3, 1, bias=True, pad=pad),
88 | nn.Sigmoid()
89 | ]
90 | self.model = nn.Sequential(*layers)
91 |
92 | def forward(self, input):
93 | return self.model(input)
94 |
95 | def eval(self):
96 | self.model.eval()
97 |
--------------------------------------------------------------------------------
/2_Super_Resolution/models/skip.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from .common import *
4 |
5 | def skip(
6 | num_input_channels=2, num_output_channels=3,
7 | num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4],
8 | filter_size_down=3, filter_size_up=3, filter_skip_size=1,
9 | need_sigmoid=True, need_bias=True,
10 | pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU',
11 | need1x1_up=True):
12 | """Assembles encoder-decoder with skip connections.
13 |
14 | Arguments:
15 | act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU)
16 | pad (string): zero|reflection (default: 'zero')
17 | upsample_mode (string): 'nearest|bilinear' (default: 'nearest')
18 | downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride')
19 |
20 | """
21 | assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip)
22 |
23 | n_scales = len(num_channels_down)
24 |
25 | if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) :
26 | upsample_mode = [upsample_mode]*n_scales
27 |
28 | if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)):
29 | downsample_mode = [downsample_mode]*n_scales
30 |
31 | if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) :
32 | filter_size_down = [filter_size_down]*n_scales
33 |
34 | if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) :
35 | filter_size_up = [filter_size_up]*n_scales
36 |
37 | last_scale = n_scales - 1
38 |
39 | cur_depth = None
40 |
41 | model = nn.Sequential()
42 | model_tmp = model
43 |
44 | input_depth = num_input_channels
45 | for i in range(len(num_channels_down)):
46 |
47 | deeper = nn.Sequential()
48 | skip = nn.Sequential()
49 |
50 | if num_channels_skip[i] != 0:
51 | model_tmp.add(Concat(1, skip, deeper))
52 | else:
53 | model_tmp.add(deeper)
54 |
55 | model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i])))
56 |
57 | if num_channels_skip[i] != 0:
58 | skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad))
59 | skip.add(bn(num_channels_skip[i]))
60 | skip.add(act(act_fun))
61 |
62 | # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part))
63 |
64 | deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i]))
65 | deeper.add(bn(num_channels_down[i]))
66 | deeper.add(act(act_fun))
67 |
68 | deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad))
69 | deeper.add(bn(num_channels_down[i]))
70 | deeper.add(act(act_fun))
71 |
72 | deeper_main = nn.Sequential()
73 |
74 | if i == len(num_channels_down) - 1:
75 | # The deepest
76 | k = num_channels_down[i]
77 | else:
78 | deeper.add(deeper_main)
79 | k = num_channels_up[i + 1]
80 |
81 | deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i]))
82 |
83 | model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad))
84 | model_tmp.add(bn(num_channels_up[i]))
85 | model_tmp.add(act(act_fun))
86 |
87 |
88 | if need1x1_up:
89 | model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad))
90 | model_tmp.add(bn(num_channels_up[i]))
91 | model_tmp.add(act(act_fun))
92 |
93 | input_depth = num_channels_down[i]
94 | model_tmp = deeper_main
95 |
96 | model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad))
97 | if need_sigmoid:
98 | model.add(nn.Sigmoid())
99 |
100 | return model
101 |
--------------------------------------------------------------------------------
/2_Super_Resolution/models/texture_nets.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from .common import *
4 |
5 |
6 | normalization = nn.BatchNorm2d
7 |
8 |
9 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero'):
10 | if pad == 'zero':
11 | return nn.Conv2d(in_f, out_f, kernel_size, stride, padding=(kernel_size - 1) / 2, bias=bias)
12 | elif pad == 'reflection':
13 | layers = [nn.ReflectionPad2d((kernel_size - 1) / 2),
14 | nn.Conv2d(in_f, out_f, kernel_size, stride, padding=0, bias=bias)]
15 | return nn.Sequential(*layers)
16 |
17 | def get_texture_nets(inp=3, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False, pad='zero', need_sigmoid=False, conv_num=8, upsample_mode='nearest'):
18 |
19 |
20 | for i in range(len(ratios)):
21 | j = i + 1
22 |
23 | seq = nn.Sequential()
24 |
25 | tmp = nn.AvgPool2d(ratios[i], ratios[i])
26 |
27 | seq.add(tmp)
28 | if fill_noise:
29 | seq.add(GenNoise(inp))
30 |
31 | seq.add(conv(inp, conv_num, 3, pad=pad))
32 | seq.add(normalization(conv_num))
33 | seq.add(act())
34 |
35 | seq.add(conv(conv_num, conv_num, 3, pad=pad))
36 | seq.add(normalization(conv_num))
37 | seq.add(act())
38 |
39 | seq.add(conv(conv_num, conv_num, 1, pad=pad))
40 | seq.add(normalization(conv_num))
41 | seq.add(act())
42 |
43 | if i == 0:
44 | seq.add(nn.Upsample(scale_factor=2, mode=upsample_mode))
45 | cur = seq
46 | else:
47 |
48 | cur_temp = cur
49 |
50 | cur = nn.Sequential()
51 |
52 | # Batch norm before merging
53 | seq.add(normalization(conv_num))
54 | cur_temp.add(normalization(conv_num * (j - 1)))
55 |
56 | cur.add(Concat(1, cur_temp, seq))
57 |
58 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad))
59 | cur.add(normalization(conv_num * j))
60 | cur.add(act())
61 |
62 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad))
63 | cur.add(normalization(conv_num * j))
64 | cur.add(act())
65 |
66 | cur.add(conv(conv_num * j, conv_num * j, 1, pad=pad))
67 | cur.add(normalization(conv_num * j))
68 | cur.add(act())
69 |
70 | if i == len(ratios) - 1:
71 | cur.add(conv(conv_num * j, 3, 1, pad=pad))
72 | else:
73 | cur.add(nn.Upsample(scale_factor=2, mode=upsample_mode))
74 |
75 | model = cur
76 | if need_sigmoid:
77 | model.add(nn.Sigmoid())
78 |
79 | return model
80 |
--------------------------------------------------------------------------------
/2_Super_Resolution/psnr_ssim.py:
--------------------------------------------------------------------------------
1 | ### This script will calcualte psnr and ssim
2 | import numpy as np
3 | from skimage.metrics import peak_signal_noise_ratio
4 | from skimage.measure import compare_ssim
5 |
6 |
7 | def calcualte_PSNR(im_true, im_test):
8 | im_true = np.transpose(im_true,(1,2,0))
9 | im_test = np.transpose(im_test, (1, 2, 0))
10 | psnr_value = peak_signal_noise_ratio(im_true, im_test)
11 | return psnr_value
12 |
13 |
14 | def calcualte_SSIM(im_true, im_test, multichannel): # multichannel=True for RGB images
15 | im_true = np.transpose(im_true, (1, 2, 0))
16 | im_test = np.transpose(im_test, (1, 2, 0))
17 | ssim_value = compare_ssim(im_true, im_test, multichannel=multichannel, data_range=im_test.max() - im_test.min())
18 | return ssim_value
19 |
20 |
21 |
22 |
23 |
24 | if __name__ == '__main__':
25 | pass
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__init__.py
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/__init__.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/common_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/common_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/common_utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/common_utils.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/denoising_utils.cpython-37.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/__pycache__/sr_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/__pycache__/sr_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/common_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision
4 | import sys
5 |
6 | import numpy as np
7 | from PIL import Image
8 | import PIL
9 | import numpy as np
10 |
11 | import matplotlib.pyplot as plt
12 |
13 | def crop_image(img, d=32):
14 | '''Make dimensions divisible by `d`'''
15 |
16 | new_size = (img.size[0] - img.size[0] % d,
17 | img.size[1] - img.size[1] % d)
18 |
19 | bbox = [
20 | int((img.size[0] - new_size[0])/2),
21 | int((img.size[1] - new_size[1])/2),
22 | int((img.size[0] + new_size[0])/2),
23 | int((img.size[1] + new_size[1])/2),
24 | ]
25 |
26 | img_cropped = img.crop(bbox)
27 | return img_cropped
28 |
29 | def get_params(opt_over, net, net_input, downsampler=None):
30 | '''Returns parameters that we want to optimize over.
31 |
32 | Args:
33 | opt_over: comma separated list, e.g. "net,input" or "net"
34 | net: network
35 | net_input: torch.Tensor that stores input `z`
36 | '''
37 | opt_over_list = opt_over.split(',')
38 | params = []
39 |
40 | for opt in opt_over_list:
41 |
42 | if opt == 'net':
43 | params += [x for x in net.parameters() ]
44 | elif opt=='down':
45 | assert downsampler is not None
46 | params = [x for x in downsampler.parameters()]
47 | elif opt == 'input':
48 | net_input.requires_grad = True
49 | params += [net_input]
50 | else:
51 | assert False, 'what is it?'
52 |
53 | return params
54 |
55 | def get_image_grid(images_np, nrow=8):
56 | '''Creates a grid from a list of images by concatenating them.'''
57 | images_torch = [torch.from_numpy(x) for x in images_np]
58 | torch_grid = torchvision.utils.make_grid(images_torch, nrow)
59 |
60 | return torch_grid.numpy()
61 |
62 | def plot_image_grid(images_np, nrow =8, factor=1, interpolation='lanczos'):
63 | """Draws images in a grid
64 |
65 | Args:
66 | images_np: list of images, each image is np.array of size 3xHxW of 1xHxW
67 | nrow: how many images will be in one row
68 | factor: size if the plt.figure
69 | interpolation: interpolation used in plt.imshow
70 | """
71 | n_channels = max(x.shape[0] for x in images_np)
72 | assert (n_channels == 3) or (n_channels == 1), "images should have 1 or 3 channels"
73 |
74 | images_np = [x if (x.shape[0] == n_channels) else np.concatenate([x, x, x], axis=0) for x in images_np]
75 |
76 | grid = get_image_grid(images_np, nrow)
77 |
78 | plt.figure(figsize=(len(images_np) + factor, 12 + factor))
79 |
80 | if images_np[0].shape[0] == 1:
81 | plt.imshow(grid[0], cmap='gray', interpolation=interpolation)
82 | else:
83 | plt.imshow(grid.transpose(1, 2, 0), interpolation=interpolation)
84 |
85 | plt.show()
86 |
87 | return grid
88 |
89 | def load(path):
90 | """Load PIL image."""
91 | img = Image.open(path)
92 | return img
93 |
94 | def get_image(path, imsize=-1):
95 | """Load an image and resize to a cpecific size.
96 |
97 | Args:
98 | path: path to image
99 | imsize: tuple or scalar with dimensions; -1 for `no resize`
100 | """
101 | img = load(path)
102 |
103 | if isinstance(imsize, int):
104 | imsize = (imsize, imsize)
105 |
106 | if imsize[0]!= -1 and img.size != imsize:
107 | if imsize[0] > img.size[0]:
108 | img = img.resize(imsize, Image.BICUBIC)
109 | else:
110 | img = img.resize(imsize, Image.ANTIALIAS)
111 |
112 | img_np = pil_to_np(img)
113 |
114 | return img, img_np
115 |
116 |
117 |
118 | def fill_noise(x, noise_type):
119 | """Fills tensor `x` with noise of type `noise_type`."""
120 | if noise_type == 'u':
121 | x.uniform_()
122 | elif noise_type == 'n':
123 | x.normal_()
124 | else:
125 | assert False
126 |
127 | def get_noise(input_depth, method, spatial_size, noise_type='u', var=1./10):
128 | """Returns a pytorch.Tensor of size (1 x `input_depth` x `spatial_size[0]` x `spatial_size[1]`)
129 | initialized in a specific way.
130 | Args:
131 | input_depth: number of channels in the tensor
132 | method: `noise` for fillting tensor with noise; `meshgrid` for np.meshgrid
133 | spatial_size: spatial size of the tensor to initialize
134 | noise_type: 'u' for uniform; 'n' for normal
135 | var: a factor, a noise will be multiplicated by. Basically it is standard deviation scaler.
136 | """
137 | if isinstance(spatial_size, int):
138 | spatial_size = (spatial_size, spatial_size)
139 | if method == 'noise':
140 | shape = [1, input_depth, spatial_size[0], spatial_size[1]]
141 | net_input = torch.zeros(shape)
142 |
143 | fill_noise(net_input, noise_type)
144 | net_input *= var
145 | elif method == 'meshgrid':
146 | assert input_depth == 2
147 | X, Y = np.meshgrid(np.arange(0, spatial_size[1])/float(spatial_size[1]-1), np.arange(0, spatial_size[0])/float(spatial_size[0]-1))
148 | meshgrid = np.concatenate([X[None,:], Y[None,:]])
149 | net_input= np_to_torch(meshgrid)
150 | else:
151 | assert False
152 |
153 | return net_input
154 |
155 | def pil_to_np(img_PIL):
156 | '''Converts image in PIL format to np.array.
157 |
158 | From W x H x C [0...255] to C x W x H [0..1]
159 | '''
160 | ar = np.array(img_PIL)
161 |
162 | if len(ar.shape) == 3:
163 | ar = ar.transpose(2,0,1)
164 | else:
165 | ar = ar[None, ...]
166 |
167 | return ar.astype(np.float32) / 255.
168 |
169 | def np_to_pil(img_np):
170 | '''Converts image in np.array format to PIL image.
171 |
172 | From C x W x H [0..1] to W x H x C [0...255]
173 | '''
174 | ar = np.clip(img_np*255,0,255).astype(np.uint8)
175 |
176 | if img_np.shape[0] == 1:
177 | ar = ar[0]
178 | else:
179 | ar = ar.transpose(1, 2, 0)
180 |
181 | return Image.fromarray(ar)
182 |
183 | def np_to_torch(img_np):
184 | '''Converts image in numpy.array to torch.Tensor.
185 |
186 | From C x W x H [0..1] to C x W x H [0..1]
187 | '''
188 | return torch.from_numpy(img_np)[None, :]
189 |
190 | def torch_to_np(img_var):
191 | '''Converts an image in torch.Tensor format to np.array.
192 |
193 | From 1 x C x W x H [0..1] to C x W x H [0..1]
194 | '''
195 | return img_var.detach().cpu().numpy()[0]
196 |
197 |
198 | def optimize(optimizer_type, parameters, closure, LR, num_iter):
199 | """Runs optimization loop.
200 |
201 | Args:
202 | optimizer_type: 'LBFGS' of 'adam'
203 | parameters: list of Tensors to optimize over
204 | closure: function, that returns loss variable
205 | LR: learning rate
206 | num_iter: number of iterations
207 | """
208 | if optimizer_type == 'LBFGS':
209 | # Do several steps with adam first
210 | optimizer = torch.optim.Adam(parameters, lr=0.001)
211 | for j in range(100):
212 | optimizer.zero_grad()
213 | closure()
214 | optimizer.step()
215 |
216 | print('Starting optimization with LBFGS')
217 | def closure2():
218 | optimizer.zero_grad()
219 | return closure()
220 | optimizer = torch.optim.LBFGS(parameters, max_iter=num_iter, lr=LR, tolerance_grad=-1, tolerance_change=-1)
221 | optimizer.step(closure2)
222 |
223 | elif optimizer_type == 'adam':
224 | print('Starting optimization with ADAM')
225 | optimizer = torch.optim.Adam(parameters, lr=LR)
226 |
227 | for j in range(num_iter):
228 | optimizer.zero_grad()
229 | closure()
230 | optimizer.step()
231 | else:
232 | assert False
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/denoising_utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | from .common_utils import *
3 |
4 |
5 |
6 | def get_noisy_image(img_np, sigma):
7 | """Adds Gaussian noise to an image.
8 |
9 | Args:
10 | img_np: image, np.array with values from 0 to 1
11 | sigma: std of the noise
12 | """
13 | img_noisy_np = np.clip(img_np + np.random.normal(scale=sigma, size=img_np.shape), 0, 1).astype(np.float32)
14 | img_noisy_pil = np_to_pil(img_noisy_np)
15 |
16 | return img_noisy_pil, img_noisy_np
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/feature_inversion_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision.transforms as transforms
4 | import torchvision.models as models
5 | from .matcher import Matcher
6 | import os
7 | from collections import OrderedDict
8 |
9 | class View(nn.Module):
10 | def __init__(self):
11 | super(View, self).__init__()
12 |
13 | def forward(self, x):
14 | return x.view(-1)
15 |
16 | def get_vanilla_vgg_features(cut_idx=-1):
17 | if not os.path.exists('vgg_features.pth'):
18 | os.system(
19 | 'wget --no-check-certificate -N https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth')
20 | vgg_weights = torch.load('vgg19-d01eb7cb.pth')
21 | # fix compatibility issues
22 | map = {'classifier.6.weight':u'classifier.7.weight', 'classifier.6.bias':u'classifier.7.bias'}
23 | vgg_weights = OrderedDict([(map[k] if k in map else k,v) for k,v in vgg_weights.iteritems()])
24 |
25 |
26 |
27 | model = models.vgg19()
28 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values())
29 |
30 |
31 | model.load_state_dict(vgg_weights)
32 |
33 | torch.save(model.features, 'vgg_features.pth')
34 | torch.save(model.classifier, 'vgg_classifier.pth')
35 |
36 | vgg = torch.load('vgg_features.pth')
37 | if cut_idx > 36:
38 | vgg_classifier = torch.load('vgg_classifier.pth')
39 | vgg = nn.Sequential(*(vgg._modules.values() + vgg_classifier._modules.values()))
40 |
41 | vgg.eval()
42 |
43 | return vgg
44 |
45 |
46 | def get_matcher(net, opt):
47 | idxs = [x for x in opt['layers'].split(',')]
48 | matcher = Matcher(opt['what'])
49 |
50 | def hook(module, input, output):
51 | matcher(module, output)
52 |
53 | for i in idxs:
54 | net._modules[i].register_forward_hook(hook)
55 |
56 | return matcher
57 |
58 |
59 |
60 | def get_vgg(cut_idx=-1):
61 | f = get_vanilla_vgg_features(cut_idx)
62 |
63 | if cut_idx > 0:
64 | num_modules = len(f._modules)
65 | keys_to_delete = [f._modules.keys()[x] for x in range(cut_idx, num_modules)]
66 | for k in keys_to_delete:
67 | del f._modules[k]
68 |
69 | return f
70 |
71 | def vgg_preprocess_var(var):
72 | (r, g, b) = torch.chunk(var, 3, dim=1)
73 | bgr = torch.cat((b, g, r), 1)
74 | out = bgr * 255 - torch.autograd.Variable(vgg_mean[None, ...]).type(var.type()).expand_as(bgr)
75 | return out
76 |
77 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1)
78 |
79 |
80 |
81 | def get_preprocessor(imsize):
82 | def vgg_preprocess(tensor):
83 | (r, g, b) = torch.chunk(tensor, 3, dim=0)
84 | bgr = torch.cat((b, g, r), 0)
85 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr)
86 | return out
87 | preprocess = transforms.Compose([
88 | transforms.Resize(imsize),
89 | transforms.ToTensor(),
90 | transforms.Lambda(vgg_preprocess)
91 | ])
92 |
93 | return preprocess
94 |
95 |
96 | def get_deprocessor():
97 | def vgg_deprocess(tensor):
98 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0
99 | (b, g, r) = torch.chunk(bgr, 3, dim=0)
100 | rgb = torch.cat((r, g, b), 0)
101 | return rgb
102 | deprocess = transforms.Compose([
103 | transforms.Lambda(vgg_deprocess),
104 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)),
105 | transforms.ToPILImage()
106 | ])
107 | return deprocess
108 |
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/inpainting_utils.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from PIL import Image
3 | import PIL.ImageDraw as ImageDraw
4 | import PIL.ImageFont as ImageFont
5 | from .common_utils import *
6 |
7 | def get_text_mask(for_image, sz=20):
8 | font_fname = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf'
9 | font_size = sz
10 | font = ImageFont.truetype(font_fname, font_size)
11 |
12 | img_mask = Image.fromarray(np.array(for_image)*0+255)
13 | draw = ImageDraw.Draw(img_mask)
14 | draw.text((128, 128), "hello world", font=font, fill='rgb(0, 0, 0)')
15 |
16 | return img_mask
17 |
18 | def get_bernoulli_mask(for_image, zero_fraction=0.95):
19 | img_mask_np=(np.random.random_sample(size=pil_to_np(for_image).shape) > zero_fraction).astype(int)
20 | img_mask = np_to_pil(img_mask_np)
21 |
22 | return img_mask
23 |
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/matcher.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | class Matcher:
5 | def __init__(self, how='gram_matrix', loss='mse'):
6 | self.mode = 'store'
7 | self.stored = {}
8 | self.losses = {}
9 |
10 | if how in all_features.keys():
11 | self.get_statistics = all_features[how]
12 | else:
13 | assert False
14 | pass
15 |
16 | if loss in all_losses.keys():
17 | self.loss = all_losses[loss]
18 | else:
19 | assert False
20 |
21 | def __call__(self, module, features):
22 | statistics = self.get_statistics(features)
23 |
24 | self.statistics = statistics
25 | if self.mode == 'store':
26 | self.stored[module] = statistics.detach().clone()
27 | elif self.mode == 'match':
28 | self.losses[module] = self.loss(statistics, self.stored[module])
29 |
30 | def clean(self):
31 | self.losses = {}
32 |
33 | def gram_matrix(x):
34 | (b, ch, h, w) = x.size()
35 | features = x.view(b, ch, w * h)
36 | features_t = features.transpose(1, 2)
37 | gram = features.bmm(features_t) / (ch * h * w)
38 | return gram
39 |
40 |
41 | def features(x):
42 | return x
43 |
44 |
45 | all_features = {
46 | 'gram_matrix': gram_matrix,
47 | 'features': features,
48 | }
49 |
50 | all_losses = {
51 | 'mse': nn.MSELoss(),
52 | 'smoothL1': nn.SmoothL1Loss(),
53 | 'L1': nn.L1Loss(),
54 | }
55 |
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/perceptual_loss/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/2_Super_Resolution/utils/perceptual_loss/__init__.py
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/perceptual_loss/matcher.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | class Matcher:
6 | def __init__(self, how='gram_matrix', loss='mse', map_index=933):
7 | self.mode = 'store'
8 | self.stored = {}
9 | self.losses = {}
10 |
11 | if how in all_features.keys():
12 | self.get_statistics = all_features[how]
13 | else:
14 | assert False
15 | pass
16 |
17 | if loss in all_losses.keys():
18 | self.loss = all_losses[loss]
19 | else:
20 | assert False
21 |
22 | self.map_index = map_index
23 | self.method = 'match'
24 |
25 |
26 | def __call__(self, module, features):
27 | statistics = self.get_statistics(features)
28 |
29 | self.statistics = statistics
30 | if self.mode == 'store':
31 | self.stored[module] = statistics.detach()
32 |
33 | elif self.mode == 'match':
34 |
35 | if statistics.ndimension() == 2:
36 |
37 | if self.method == 'maximize':
38 | self.losses[module] = - statistics[0, self.map_index]
39 | else:
40 | self.losses[module] = torch.abs(300 - statistics[0, self.map_index])
41 |
42 | else:
43 | ws = self.window_size
44 |
45 | t = statistics.detach() * 0
46 |
47 | s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
48 | t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
49 | t_cc[:, self.map_index,...] = 1
50 |
51 | if self.method == 'maximize':
52 | self.losses[module] = -(s_cc * t_cc.contiguous()).sum()
53 | else:
54 | self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum()
55 |
56 |
57 | def clean(self):
58 | self.losses = {}
59 |
60 | def gram_matrix(x):
61 | (b, ch, h, w) = x.size()
62 | features = x.view(b, ch, w * h)
63 | features_t = features.transpose(1, 2)
64 | gram = features.bmm(features_t) / (ch * h * w)
65 | return gram
66 |
67 |
68 | def features(x):
69 | return x
70 |
71 |
72 | all_features = {
73 | 'gram_matrix': gram_matrix,
74 | 'features': features,
75 | }
76 |
77 | all_losses = {
78 | 'mse': nn.MSELoss(),
79 | 'smoothL1': nn.SmoothL1Loss(),
80 | 'L1': nn.L1Loss(),
81 | }
82 |
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/perceptual_loss/vgg_modified.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 |
3 | class VGGModified(nn.Module):
4 | def __init__(self, vgg19_orig, slope=0.01):
5 | super(VGGModified, self).__init__()
6 | self.features = nn.Sequential()
7 |
8 | self.features.add_module(str(0), vgg19_orig.features[0])
9 | self.features.add_module(str(1), nn.LeakyReLU(slope, True))
10 | self.features.add_module(str(2), vgg19_orig.features[2])
11 | self.features.add_module(str(3), nn.LeakyReLU(slope, True))
12 | self.features.add_module(str(4), nn.AvgPool2d((2,2), (2,2)))
13 |
14 | self.features.add_module(str(5), vgg19_orig.features[5])
15 | self.features.add_module(str(6), nn.LeakyReLU(slope, True))
16 | self.features.add_module(str(7), vgg19_orig.features[7])
17 | self.features.add_module(str(8), nn.LeakyReLU(slope, True))
18 | self.features.add_module(str(9), nn.AvgPool2d((2,2), (2,2)))
19 |
20 | self.features.add_module(str(10), vgg19_orig.features[10])
21 | self.features.add_module(str(11), nn.LeakyReLU(slope, True))
22 | self.features.add_module(str(12), vgg19_orig.features[12])
23 | self.features.add_module(str(13), nn.LeakyReLU(slope, True))
24 | self.features.add_module(str(14), vgg19_orig.features[14])
25 | self.features.add_module(str(15), nn.LeakyReLU(slope, True))
26 | self.features.add_module(str(16), vgg19_orig.features[16])
27 | self.features.add_module(str(17), nn.LeakyReLU(slope, True))
28 | self.features.add_module(str(18), nn.AvgPool2d((2,2), (2,2)))
29 |
30 | self.features.add_module(str(19), vgg19_orig.features[19])
31 | self.features.add_module(str(20), nn.LeakyReLU(slope, True))
32 | self.features.add_module(str(21), vgg19_orig.features[21])
33 | self.features.add_module(str(22), nn.LeakyReLU(slope, True))
34 | self.features.add_module(str(23), vgg19_orig.features[23])
35 | self.features.add_module(str(24), nn.LeakyReLU(slope, True))
36 | self.features.add_module(str(25), vgg19_orig.features[25])
37 | self.features.add_module(str(26), nn.LeakyReLU(slope, True))
38 | self.features.add_module(str(27), nn.AvgPool2d((2,2), (2,2)))
39 |
40 | self.features.add_module(str(28), vgg19_orig.features[28])
41 | self.features.add_module(str(29), nn.LeakyReLU(slope, True))
42 | self.features.add_module(str(30), vgg19_orig.features[30])
43 | self.features.add_module(str(31), nn.LeakyReLU(slope, True))
44 | self.features.add_module(str(32), vgg19_orig.features[32])
45 | self.features.add_module(str(33), nn.LeakyReLU(slope, True))
46 | self.features.add_module(str(34), vgg19_orig.features[34])
47 | self.features.add_module(str(35), nn.LeakyReLU(slope, True))
48 | self.features.add_module(str(36), nn.AvgPool2d((2,2), (2,2)))
49 |
50 | self.classifier = nn.Sequential()
51 |
52 | self.classifier.add_module(str(0), vgg19_orig.classifier[0])
53 | self.classifier.add_module(str(1), nn.LeakyReLU(slope, True))
54 | self.classifier.add_module(str(2), nn.Dropout2d(p = 0.5))
55 | self.classifier.add_module(str(3), vgg19_orig.classifier[3])
56 | self.classifier.add_module(str(4), nn.LeakyReLU(slope, True))
57 | self.classifier.add_module(str(5), nn.Dropout2d(p = 0.5))
58 | self.classifier.add_module(str(6), vgg19_orig.classifier[6])
59 |
60 | def forward(self, x):
61 | return self.classifier(self.features.forward(x))
--------------------------------------------------------------------------------
/2_Super_Resolution/utils/sr_utils.py:
--------------------------------------------------------------------------------
1 | from .common_utils import *
2 |
3 | def put_in_center(img_np, target_size):
4 | img_out = np.zeros([3, target_size[0], target_size[1]])
5 |
6 | bbox = [
7 | int((target_size[0] - img_np.shape[1]) / 2),
8 | int((target_size[1] - img_np.shape[2]) / 2),
9 | int((target_size[0] + img_np.shape[1]) / 2),
10 | int((target_size[1] + img_np.shape[2]) / 2),
11 | ]
12 |
13 | img_out[:, bbox[0]:bbox[2], bbox[1]:bbox[3]] = img_np
14 |
15 | return img_out
16 |
17 |
18 | def load_LR_HR_imgs_sr(fname, imsize, factor, enforse_div32=None):
19 | '''Loads an image, resizes it, center crops and downscales.
20 |
21 | Args:
22 | fname: path to the image
23 | imsize: new size for the image, -1 for no resizing
24 | factor: downscaling factor
25 | enforse_div32: if 'CROP' center crops an image, so that its dimensions are divisible by 32.
26 | '''
27 | img_orig_pil, img_orig_np = get_image(fname, -1)
28 |
29 | if imsize != -1:
30 | img_orig_pil, img_orig_np = get_image(fname, imsize)
31 |
32 | # For comparison with GT
33 | if enforse_div32 == 'CROP':
34 | new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 32,
35 | img_orig_pil.size[1] - img_orig_pil.size[1] % 32)
36 |
37 | bbox = [
38 | (img_orig_pil.size[0] - new_size[0])/2,
39 | (img_orig_pil.size[1] - new_size[1])/2,
40 | (img_orig_pil.size[0] + new_size[0])/2,
41 | (img_orig_pil.size[1] + new_size[1])/2,
42 | ]
43 |
44 | img_HR_pil = img_orig_pil.crop(bbox)
45 | img_HR_np = pil_to_np(img_HR_pil)
46 | else:
47 | img_HR_pil, img_HR_np = img_orig_pil, img_orig_np
48 |
49 | LR_size = [
50 | img_HR_pil.size[0] // factor,
51 | img_HR_pil.size[1] // factor
52 | ]
53 |
54 | img_LR_pil = img_HR_pil.resize(LR_size, Image.ANTIALIAS)
55 | img_LR_np = pil_to_np(img_LR_pil)
56 |
57 | print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str (img_LR_pil.size)))
58 |
59 | return {
60 | 'orig_pil': img_orig_pil,
61 | 'orig_np': img_orig_np,
62 | 'LR_pil': img_LR_pil,
63 | 'LR_np': img_LR_np,
64 | 'HR_pil': img_HR_pil,
65 | 'HR_np': img_HR_np
66 | }
67 |
68 |
69 | def get_baselines(img_LR_pil, img_HR_pil):
70 | '''Gets `bicubic`, sharpened bicubic and `nearest` baselines.'''
71 | img_bicubic_pil = img_LR_pil.resize(img_HR_pil.size, Image.BICUBIC)
72 | img_bicubic_np = pil_to_np(img_bicubic_pil)
73 |
74 | img_nearest_pil = img_LR_pil.resize(img_HR_pil.size, Image.NEAREST)
75 | img_nearest_np = pil_to_np(img_nearest_pil)
76 |
77 | img_bic_sharp_pil = img_bicubic_pil.filter(PIL.ImageFilter.UnsharpMask())
78 | img_bic_sharp_np = pil_to_np(img_bic_sharp_pil)
79 |
80 | return img_bicubic_np, img_bic_sharp_np, img_nearest_np
81 |
82 |
83 |
84 | def tv_loss(x, beta = 0.5):
85 | '''Calculates TV loss for an image `x`.
86 |
87 | Args:
88 | x: image, torch.Variable of torch.Tensor
89 | beta: See https://arxiv.org/abs/1412.0035 (fig. 2) to see effect of `beta`
90 | '''
91 | dh = torch.pow(x[:,:,:,1:] - x[:,:,:,:-1], 2)
92 | dw = torch.pow(x[:,:,1:,:] - x[:,:,:-1,:], 2)
93 |
94 | return torch.sum(torch.pow(dh[:, :, :-1] + dw[:, :, :, :-1], beta))
95 |
--------------------------------------------------------------------------------
/3_Inpainting/BN_Net.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 | import torch
3 | import torch.nn.functional as F
4 |
5 |
6 | #-----------------------------------------------------------
7 | #-----------------------------------------------------------
8 | # BN for the input seed
9 | #-----------------------------------------------------------
10 | #-----------------------------------------------------------
11 | class BNNet(nn.Module):
12 | def __init__(self,num_channel):
13 | super(BNNet, self).__init__()
14 | self.bn = nn.BatchNorm2d(num_channel)
15 |
16 | def forward(self, input_data):
17 | output_data = self.bn(input_data)
18 | return output_data
19 |
--------------------------------------------------------------------------------
/3_Inpainting/Main_Start.py:
--------------------------------------------------------------------------------
1 | from train_ip import *
2 | import time
3 | import datetime
4 | from include import *
5 |
6 |
7 |
8 |
9 | #-----------------------------------------------------------
10 | #-----------------------------------------------------------
11 | # This is the Main of the code
12 | #-----------------------------------------------------------
13 | #-----------------------------------------------------------
14 |
15 | if __name__ == '__main__':
16 | # Set random seed for reproducibility
17 | manualSeed = 100
18 | manualSeed = random.randint(1, 10000) # use if you want new results
19 | print("Random Seed: ", manualSeed)
20 | random.seed(manualSeed)
21 | torch.manual_seed(manualSeed)
22 |
23 | ##### Set up hyper-parameters
24 | #-----------------------------------------------------------
25 | gpu = 2
26 |
27 | print_step = 1
28 | max_epoch = 5000
29 | max_epoch = int(max_epoch / print_step) * print_step + 1
30 |
31 |
32 | tv_weight = 1e-2
33 | learning_rate_model = 0.05
34 |
35 | OPTIMIZER = 'Adam'
36 | LOSS = "MSE"#'Huber' or "L1"
37 |
38 | IP_ratio = 0.1
39 |
40 | input_w = 512
41 | input_h = 512
42 | input_c = 2
43 | width = 128
44 | #-----------------------------------------------------------
45 |
46 | # image_list = ['Peppers']
47 |
48 | #image_list = ['couple','fingerprint', 'hill', 'house', 'Lena512','man', 'montage', 'peppers256']
49 |
50 | image_list = ['barbara']
51 |
52 | ###################################### Processing images one by one #################################
53 | for cur_image in image_list:
54 | path = '../0_Dataset/Inpainting/0_IP11/IP_Dataset_{}/'.format(IP_ratio)
55 |
56 | # load clean image
57 | img_path = path + cur_image + "_img_{}.png".format(IP_ratio)
58 | img_pil = Image.open(img_path)
59 | img_np = pil_to_np(img_pil)
60 |
61 | # ----------------------------
62 | # ----- Clean Var-----
63 | # ----------------------------
64 | img_clean_var = np_to_var(img_np).type(dtype)
65 |
66 | output_depth = img_np.shape[0]
67 |
68 | # load its mask
69 | mask_path = path + cur_image + '_mask_{}.png'.format(IP_ratio)
70 | img_mask_pil = Image.open(mask_path)
71 | mask_np = pil_to_np(img_mask_pil)
72 | mask_np = np.array([mask_np[0, :, :] / np.max(mask_np)] * output_depth)
73 |
74 | # ----------------------------
75 | # ----- Mask Var-----
76 | # ----------------------------
77 | mask_var = np_to_var(mask_np).type(dtype)
78 |
79 | ##################Generate inpainted image
80 | # ----------------------------
81 | # ----- Noisy Var-----
82 | # ----------------------------
83 | img_noisy_var = img_clean_var * mask_var
84 | img_noisy_np = var_to_np(img_noisy_var)
85 |
86 |
87 | DIP_train(img_clean_var,
88 | img_noisy_var,
89 | mask_var,
90 | learning_rate_model,
91 | OPTIMIZER,
92 | LOSS,
93 | width,
94 | input_w,
95 | input_h,
96 | input_c,
97 | max_epoch,
98 | print_step,
99 | gpu,
100 | cur_image,
101 | tv_weight
102 | )
103 |
--------------------------------------------------------------------------------
/3_Inpainting/include/__init__.py:
--------------------------------------------------------------------------------
1 | from .wavelet import *
2 | from .decoder import *
3 | from .visualize import *
4 | from .fit import *
5 | from .helpers import *
6 | from .compression import *
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/compression.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/compression.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/decoder.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/decoder.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/fit.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/fit.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/helpers.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/helpers.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/visualize.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/visualize.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/__pycache__/wavelet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/include/__pycache__/wavelet.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/include/compression.py:
--------------------------------------------------------------------------------
1 | from torch.autograd import Variable
2 | import torch
3 | import torch.optim
4 | import copy
5 | import numpy as np
6 |
7 | from .helpers import *
8 | from .decoder import *
9 | from .fit import *
10 | from .wavelet import *
11 |
12 | def rep_error_deep_decoder(img_np,k=128,convert2ycbcr=False):
13 | '''
14 | mse obtained by representing img_np with the deep decoder
15 | '''
16 | output_depth = img_np.shape[0]
17 | if output_depth == 3 and convert2ycbcr:
18 | img = rgb2ycbcr(img_np)
19 | else:
20 | img = img_np
21 | img_var = np_to_var(img).type(dtype)
22 |
23 | num_channels = [k]*5
24 | net = decodernwv2(output_depth,num_channels_up=num_channels,bn_before_act=True).type(dtype)
25 | rnd = 500
26 | numit = 15000
27 | rn = 0.005
28 | mse_n, mse_t, ni, net = fit( num_channels=num_channels,
29 | reg_noise_std=rn,
30 | reg_noise_decayevery = rnd,
31 | num_iter=numit,
32 | LR=0.004,
33 | img_noisy_var=img_var,
34 | net=net,
35 | img_clean_var=img_var,
36 | find_best=True,
37 | )
38 | out_img = net(ni.type(dtype)).data.cpu().numpy()[0]
39 | if output_depth == 3 and convert2ycbcr:
40 | out_img = ycbcr2rgb(out_img)
41 | return psnr(out_img,img_np), out_img, num_param(net)
42 |
43 | def rep_error_wavelet(img_np,ncoeff=300):
44 | '''
45 | mse obtained by representing img_np with wavelet thresholding
46 | ncoff coefficients are retained per color channel
47 | '''
48 | if img_np.shape[0] == 1:
49 | img_np = img_np[0,:,:]
50 | out_img_np = denoise_wavelet(img_np, ncoeff=ncoeff, multichannel=False, convert2ycbcr=True, mode='hard')
51 | else:
52 | img_np = np.transpose(img_np)
53 | out_img_np = denoise_wavelet(img_np, ncoeff=ncoeff, multichannel=True, convert2ycbcr=True, mode='hard')
54 | # img_np = np.array([img_np[:,:,0],img_np[:,:,1],img_np[:,:,2]])
55 | return psnr(out_img_np,img_np), out_img_np
56 |
57 | def myimgshow(plt,img):
58 | if(img.shape[0] == 1):
59 | plt.imshow(np.clip(img[0],0,1),cmap='Greys',interpolation='none')
60 | else:
61 | plt.imshow(np.clip(img.transpose(1, 2, 0),0,1),interpolation='none')
62 |
63 |
--------------------------------------------------------------------------------
/3_Inpainting/include/fit.py:
--------------------------------------------------------------------------------
1 | from torch.autograd import Variable
2 | import torch
3 | import torch.optim
4 | import copy
5 | import numpy as np
6 | from scipy.linalg import hadamard
7 |
8 | from .helpers import *
9 |
10 | dtype = torch.cuda.FloatTensor
11 | #dtype = torch.FloatTensor
12 |
13 |
14 | def exp_lr_scheduler(optimizer, epoch, init_lr=0.001, lr_decay_epoch=500):
15 | """Decay learning rate by a factor of 0.1 every lr_decay_epoch epochs."""
16 | lr = init_lr * (0.65**(epoch // lr_decay_epoch))
17 |
18 | if epoch % lr_decay_epoch == 0:
19 | print('LR is set to {}'.format(lr))
20 |
21 | for param_group in optimizer.param_groups:
22 | param_group['lr'] = lr
23 |
24 | return optimizer
25 |
26 |
27 |
28 | def fit(net,
29 | img_noisy_var,
30 | num_channels,
31 | img_clean_var,
32 | num_iter = 5000,
33 | LR = 0.01,
34 | OPTIMIZER='adam',
35 | opt_input = False,
36 | reg_noise_std = 0,
37 | reg_noise_decayevery = 100000,
38 | mask_var = None,
39 | apply_f = None,
40 | lr_decay_epoch = 0,
41 | net_input = None,
42 | net_input_gen = "random",
43 | find_best=False,
44 | weight_decay=0,
45 | ):
46 |
47 | if net_input is not None:
48 | print("input provided")
49 | else:
50 | # feed uniform noise into the network
51 | totalupsample = 2**len(num_channels)
52 | width = int(img_clean_var.data.shape[2]/totalupsample)
53 | height = int(img_clean_var.data.shape[3]/totalupsample)
54 | shape = [1,num_channels[0], width, height]
55 | print("shape: ", shape)
56 | net_input = Variable(torch.zeros(shape))
57 | net_input.data.uniform_()
58 | net_input.data *= 1./10
59 |
60 | net_input_saved = net_input.data.clone()
61 | noise = net_input.data.clone()
62 | p = [x for x in net.parameters() ]
63 |
64 | if(opt_input == True): # optimizer over the input as well
65 | net_input.requires_grad = True
66 | p += [net_input]
67 |
68 | mse_wrt_noisy = np.zeros(num_iter)
69 | mse_wrt_truth = np.zeros(num_iter)
70 |
71 | if OPTIMIZER == 'SGD':
72 | print("optimize with SGD", LR)
73 | optimizer = torch.optim.SGD(p, lr=LR,momentum=0.9,weight_decay=weight_decay)
74 | elif OPTIMIZER == 'adam':
75 | print("optimize with adam", LR)
76 | optimizer = torch.optim.Adam(p, lr=LR,weight_decay=weight_decay)
77 | elif OPTIMIZER == 'LBFGS':
78 | print("optimize with LBFGS", LR)
79 | optimizer = torch.optim.LBFGS(p, lr=LR)
80 |
81 | mse = torch.nn.MSELoss() #.type(dtype)
82 | noise_energy = mse(img_noisy_var, img_clean_var)
83 |
84 | if find_best:
85 | best_net = copy.deepcopy(net)
86 | best_mse = 1000000.0
87 |
88 | for i in range(num_iter):
89 |
90 | if lr_decay_epoch is not 0:
91 | optimizer = exp_lr_scheduler(optimizer, i, init_lr=LR, lr_decay_epoch=lr_decay_epoch)
92 | if reg_noise_std > 0:
93 | if i % reg_noise_decayevery == 0:
94 | reg_noise_std *= 0.7
95 | net_input = Variable(net_input_saved + (noise.normal_() * reg_noise_std))
96 |
97 | def closure():
98 | optimizer.zero_grad()
99 | out = net(net_input.type(dtype))
100 |
101 | # training loss
102 | if mask_var is not None:
103 | loss = mse( out * mask_var , img_noisy_var * mask_var )
104 | elif apply_f:
105 | loss = mse( apply_f(out) , img_noisy_var )
106 | else:
107 | loss = mse(out, img_noisy_var)
108 |
109 | loss.backward()
110 | mse_wrt_noisy[i] = loss.data.cpu().numpy()
111 |
112 |
113 | # the actual loss
114 | true_loss = mse(Variable(out.data, requires_grad=False), img_clean_var)
115 | mse_wrt_truth[i] = true_loss.data.cpu().numpy()
116 | if i % 10 == 0:
117 | out2 = net(Variable(net_input_saved).type(dtype))
118 | loss2 = mse(out2, img_clean_var)
119 | print ('Iteration %05d Train loss %f Actual loss %f Actual loss orig %f Noise Energy %f' % (i, loss.data,true_loss.data,loss2.data,noise_energy.data), '\r', end='')
120 | return loss
121 |
122 |
123 | #if OPTIMIZER == 'LBFGS':
124 | # if i < 100:
125 | # optimizer = torch.optim.Adam(p, lr=LR)
126 | # else:
127 | # optimizer = torch.optim.LBFGS(p, lr=LR)
128 |
129 |
130 | loss = optimizer.step(closure)
131 |
132 | if find_best:
133 | # if training loss improves by at least one percent, we found a new best net
134 | if best_mse > 1.005*loss.data:
135 | best_mse = loss.data
136 | best_net = copy.deepcopy(net)
137 |
138 |
139 | if find_best:
140 | net = best_net
141 | return mse_wrt_noisy, mse_wrt_truth,net_input_saved, net
142 |
143 |
144 |
145 |
146 |
147 | ### weight regularization
148 | #if orth_reg > 0:
149 | # for name, param in net.named_parameters():
150 | # consider all the conv weights, but the last one which only combines colors
151 | # if '.1.weight' in name and str( len(net)-1 ) not in name:
152 | # param_flat = param.view(param.shape[0], -1)
153 | # sym = torch.mm(param_flat, torch.t(param_flat))
154 | # sym -= Variable(torch.eye(param_flat.shape[0])).type(dtype)
155 | # loss = loss + (orth_reg * sym.sum().type(dtype) )
156 | ###
--------------------------------------------------------------------------------
/3_Inpainting/include/helpers.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision
4 | import sys
5 |
6 | import numpy as np
7 | from PIL import Image
8 | import PIL
9 | import numpy as np
10 |
11 | from torch.autograd import Variable
12 |
13 |
14 |
15 |
16 |
17 | import numpy as np
18 | import torch
19 | import matplotlib.pyplot as plt
20 |
21 | from PIL import Image
22 | import PIL
23 |
24 | from torch.autograd import Variable
25 |
26 | def load_and_crop(imgname,target_width=512,target_height=512):
27 | '''
28 | imgname: string of image location
29 | load an image, and center-crop if the image is large enough, else return none
30 | '''
31 | img = Image.open(imgname)
32 | width, height = img.size
33 | if width <= target_width or height <= target_height:
34 | return None
35 |
36 | left = (width - target_width)/2
37 | top = (height - target_height)/2
38 | right = (width + target_width)/2
39 | bottom = (height + target_height)/2
40 |
41 | return img.crop((left, top, right, bottom))
42 |
43 | def save_np_img(img,filename):
44 | if(img.shape[0] == 1):
45 | plt.imshow(np.clip(img[0],0,1),cmap='Greys',interpolation='nearest')
46 | else:
47 | plt.imshow(np.clip(img.transpose(1, 2, 0),0,1))
48 | plt.axis('off')
49 | plt.savefig(filename, bbox_inches='tight')
50 | plt.close()
51 |
52 | def np_to_tensor(img_np):
53 | '''Converts image in numpy.array to torch.Tensor.
54 |
55 | From C x W x H [0..1] to C x W x H [0..1]
56 | '''
57 | return torch.from_numpy(img_np)
58 |
59 | def np_to_var(img_np, dtype = torch.cuda.FloatTensor):
60 | '''Converts image in numpy.array to torch.Variable.
61 |
62 | From C x W x H [0..1] to 1 x C x W x H [0..1]
63 | '''
64 | return Variable(np_to_tensor(img_np)[None, :])
65 |
66 | def var_to_np(img_var):
67 | '''Converts an image in torch.Variable format to np.array.
68 |
69 | From 1 x C x W x H [0..1] to C x W x H [0..1]
70 | '''
71 | return img_var.data.cpu().numpy()[0]
72 |
73 |
74 | def pil_to_np(img_PIL):
75 | '''Converts image in PIL format to np.array.
76 |
77 | From W x H x C [0...255] to C x W x H [0..1]
78 | '''
79 | ar = np.array(img_PIL)
80 |
81 | if len(ar.shape) == 3:
82 | ar = ar.transpose(2,0,1)
83 | else:
84 | ar = ar[None, ...]
85 |
86 | return ar.astype(np.float32) / 255.
87 |
88 |
89 | def rgb2ycbcr(img):
90 | #out = color.rgb2ycbcr( img.transpose(1, 2, 0) )
91 | #return out.transpose(2,0,1)/256.
92 | r,g,b = img[0],img[1],img[2]
93 | y = 0.299*r+0.587*g+0.114*b
94 | cb = 0.5 - 0.168736*r - 0.331264*g + 0.5*b
95 | cr = 0.5 + 0.5*r - 0.418588*g - 0.081312*b
96 | return np.array([y,cb,cr])
97 |
98 | def ycbcr2rgb(img):
99 | #out = color.ycbcr2rgb( 256.*img.transpose(1, 2, 0) )
100 | #return (out.transpose(2,0,1) - np.min(out))/(np.max(out)-np.min(out))
101 | y,cb,cr = img[0],img[1],img[2]
102 | r = y + 1.402*(cr-0.5)
103 | g = y - 0.344136*(cb-0.5) - 0.714136*(cr-0.5)
104 | b = y + 1.772*(cb - 0.5)
105 | return np.array([r,g,b])
106 |
107 |
108 |
109 | def mse(x_hat,x_true,maxv=1.):
110 | x_hat = x_hat.flatten()
111 | x_true = x_true.flatten()
112 | mse = np.mean(np.square(x_hat-x_true))
113 | energy = np.mean(np.square(x_true))
114 | return mse/energy
115 |
116 | def psnr(x_hat,x_true,maxv=1.):
117 | x_hat = x_hat.flatten()
118 | x_true = x_true.flatten()
119 | mse=np.mean(np.square(x_hat-x_true))
120 | psnr_ = 10.*np.log(maxv**2/mse)/np.log(10.)
121 | return psnr_
122 |
123 | def num_param(net):
124 | s = sum([np.prod(list(p.size())) for p in net.parameters()]);
125 | return s
126 | #print('Number of params: %d' % s)
127 |
128 | def rgb2gray(rgb):
129 | r, g, b = rgb[0,:,:], rgb[1,:,:], rgb[2,:,:]
130 | gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
131 | return np.array([gray])
132 |
133 | def savemtx_for_logplot(A,filename = "exp.dat"):
134 | ind = sorted(list(set([int(i) for i in np.geomspace(1, len(A[0])-1 ,num=700)])))
135 | A = [ [a[i] for i in ind] for a in A]
136 | X = np.array([ind] + A)
137 | np.savetxt(filename, X.T, delimiter=' ')
138 |
--------------------------------------------------------------------------------
/3_Inpainting/include/visualize.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | from torch.autograd import Variable
3 | import torch
4 | import torch.optim
5 | import numpy as np
6 | from collections import Iterable
7 |
8 |
9 | dtype = torch.cuda.FloatTensor
10 | #dtype = torch.FloatTensor
11 |
12 | def save_np_img(img,filename):
13 | if(img.shape[0] == 1):
14 | plt.imshow(np.clip(img[0],0,1))
15 | else:
16 | plt.imshow(np.clip(img.transpose(1, 2, 0),0,1))
17 | plt.axis('off')
18 | plt.savefig(filename, bbox_inches='tight')
19 | plt.close()
20 |
21 | def apply_until(net_input,net,n = 100):
22 | # applies function by funtion of a network
23 | for i,fun in enumerate(net):
24 | if i>=n:
25 | break
26 | if i==0:
27 | out = fun(net_input.type(dtype))
28 | else:
29 | out = fun(out)
30 | print(i, "last func. applied:", net[i-1])
31 | if n == 0:
32 | return net_input
33 | else:
34 | return out
35 |
36 |
37 | from math import ceil
38 |
39 |
40 | # given a lists of images as np-arrays, plot them as a row# given
41 | def plot_image_grid(imgs,nrows=10):
42 | ncols = ceil( len(imgs)/nrows )
43 | nrows = min(nrows,len(imgs))
44 | fig, axes = plt.subplots(nrows=nrows, ncols=ncols, sharex=True, sharey=True,figsize=(ncols,nrows),squeeze=False)
45 | for i, row in enumerate(axes):
46 | for j, ax in enumerate(row):
47 | ax.imshow(imgs[j*nrows+i], cmap='Greys_r', interpolation='none')
48 | ax.get_xaxis().set_visible(False)
49 | ax.get_yaxis().set_visible(False)
50 | fig.tight_layout(pad=0.1)
51 | return fig
52 |
53 | def save_tensor(out,filename,nrows=8):
54 | imgs = [img for img in out.data.cpu().numpy()[0]]
55 | fig = plot_image_grid(imgs,nrows=nrows)
56 | plt.savefig(filename)
57 | plt.close()
58 |
59 |
--------------------------------------------------------------------------------
/3_Inpainting/include/wavelet.py:
--------------------------------------------------------------------------------
1 | #import matplotlib.pyplot as plt
2 | import numpy as np
3 | import numbers
4 | import pywt
5 | import scipy
6 | import skimage.color as color
7 | from skimage.restoration import (denoise_wavelet, estimate_sigma)
8 | from skimage import data, img_as_float
9 | from skimage.util import random_noise
10 | from skimage.measure import compare_psnr
11 | from include import *
12 |
13 | def _wavelet_threshold(image, wavelet, ncoeff = None, threshold=None, mode='soft', wavelet_levels=None):
14 |
15 | wavelet = pywt.Wavelet(wavelet)
16 |
17 | # original_extent is used to workaround PyWavelets issue #80
18 | # odd-sized input results in an image with 1 extra sample after waverecn
19 | original_extent = [slice(s) for s in image.shape]
20 |
21 | # Determine the number of wavelet decomposition levels
22 | if wavelet_levels is None:
23 | # Determine the maximum number of possible levels for image
24 | dlen = wavelet.dec_len
25 | wavelet_levels = np.min(
26 | [pywt.dwt_max_level(s, dlen) for s in image.shape])
27 |
28 | # Skip coarsest wavelet scales (see Notes in docstring).
29 | wavelet_levels = max(wavelet_levels - 3, 1)
30 |
31 | coeffs = pywt.wavedecn(image, wavelet=wavelet, level=wavelet_levels)
32 | # Detail coefficients at each decomposition level
33 | dcoeffs = coeffs[1:]
34 |
35 | a = []
36 | for level in dcoeffs:
37 | for key in level:
38 | a += [np.ndarray.flatten(level[key])]
39 | a = np.concatenate(a)
40 | a = np.sort( np.abs(a) )
41 |
42 | sh = coeffs[0].shape
43 | basecoeffs = sh[0]*sh[1]
44 | threshold = a[- (ncoeff - basecoeffs)]
45 |
46 | # A single threshold for all coefficient arrays
47 | denoised_detail = [{key: pywt.threshold(level[key],value=threshold,
48 | mode=mode) for key in level} for level in dcoeffs]
49 |
50 | denoised_coeffs = [coeffs[0]] + denoised_detail
51 | return pywt.waverecn(denoised_coeffs, wavelet)[original_extent]
52 |
53 |
54 | def denoise_wavelet(image, ncoeff=None, wavelet='db1', mode='hard',
55 | wavelet_levels=None, multichannel=False,
56 | convert2ycbcr=False):
57 |
58 | image = img_as_float(image)
59 |
60 |
61 | if multichannel:
62 | if convert2ycbcr:
63 | out = color.rgb2ycbcr(image)
64 | for i in range(3):
65 | # renormalizing this color channel to live in [0, 1]
66 | min, max = out[..., i].min(), out[..., i].max()
67 | channel = out[..., i] - min
68 | channel /= max - min
69 | out[..., i] = denoise_wavelet(channel, wavelet=wavelet,ncoeff=ncoeff,
70 | mode=mode,
71 | wavelet_levels=wavelet_levels)
72 |
73 | out[..., i] = out[..., i] * (max - min)
74 | out[..., i] += min
75 | out = color.ycbcr2rgb(out)
76 | else:
77 | out = np.empty_like(image)
78 | for c in range(image.shape[-1]):
79 | out[..., c] = _wavelet_threshold(image[..., c],ncoeff=ncoeff,
80 | wavelet=wavelet, mode=mode,
81 | wavelet_levels=wavelet_levels)
82 | else:
83 | out = _wavelet_threshold(image, wavelet=wavelet, mode=mode,ncoeff=ncoeff,
84 | wavelet_levels=wavelet_levels)
85 |
86 | clip_range = (-1, 1) if image.min() < 0 else (0, 1)
87 | return np.clip(out, *clip_range)
88 |
89 |
90 |
--------------------------------------------------------------------------------
/3_Inpainting/models/__init__.py:
--------------------------------------------------------------------------------
1 | from .skip import skip
2 | from .texture_nets import get_texture_nets
3 | from .resnet import ResNet
4 | from .unet import UNet
5 |
6 | import torch.nn as nn
7 |
8 | def get_net(input_depth, NET_TYPE, pad, upsample_mode, n_channels=3, act_fun='LeakyReLU', skip_n33d=128, skip_n33u=128, skip_n11=4, num_scales=5, downsample_mode='stride'):
9 | if NET_TYPE == 'ResNet':
10 | # TODO
11 | net = ResNet(input_depth, 3, 10, 16, 1, nn.BatchNorm2d, False)
12 | elif NET_TYPE == 'skip':
13 | net = skip(input_depth, n_channels, num_channels_down = [skip_n33d]*num_scales if isinstance(skip_n33d, int) else skip_n33d,
14 | num_channels_up = [skip_n33u]*num_scales if isinstance(skip_n33u, int) else skip_n33u,
15 | num_channels_skip = [skip_n11]*num_scales if isinstance(skip_n11, int) else skip_n11,
16 | upsample_mode=upsample_mode, downsample_mode=downsample_mode,
17 | need_sigmoid=True, need_bias=True, pad=pad, act_fun=act_fun)
18 |
19 | elif NET_TYPE == 'texture_nets':
20 | net = get_texture_nets(inp=input_depth, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False,pad=pad)
21 |
22 | elif NET_TYPE =='UNet':
23 | net = UNet(num_input_channels=input_depth, num_output_channels=3,
24 | feature_scale=4, more_layers=0, concat_x=False,
25 | upsample_mode=upsample_mode, pad=pad, norm_layer=nn.BatchNorm2d, need_sigmoid=True, need_bias=True)
26 | elif NET_TYPE == 'identity':
27 | assert input_depth == 3
28 | net = nn.Sequential()
29 | else:
30 | assert False
31 |
32 | return net
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/__init__.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/common.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/common.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/common.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/common.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/downsampler.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/downsampler.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/downsampler.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/downsampler.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/resnet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/resnet.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/resnet.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/resnet.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/skip.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/skip.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/skip.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/skip.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/texture_nets.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/texture_nets.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/texture_nets.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/texture_nets.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/unet.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/unet.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/__pycache__/unet.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/models/__pycache__/unet.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/models/common.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 | from .downsampler import Downsampler
5 |
6 | def add_module(self, module):
7 | self.add_module(str(len(self) + 1), module)
8 |
9 | torch.nn.Module.add = add_module
10 |
11 | class Concat(nn.Module):
12 | def __init__(self, dim, *args):
13 | super(Concat, self).__init__()
14 | self.dim = dim
15 |
16 | for idx, module in enumerate(args):
17 | self.add_module(str(idx), module)
18 |
19 | def forward(self, input):
20 | inputs = []
21 | for module in self._modules.values():
22 | inputs.append(module(input))
23 |
24 | inputs_shapes2 = [x.shape[2] for x in inputs]
25 | inputs_shapes3 = [x.shape[3] for x in inputs]
26 |
27 | if np.all(np.array(inputs_shapes2) == min(inputs_shapes2)) and np.all(np.array(inputs_shapes3) == min(inputs_shapes3)):
28 | inputs_ = inputs
29 | else:
30 | target_shape2 = min(inputs_shapes2)
31 | target_shape3 = min(inputs_shapes3)
32 |
33 | inputs_ = []
34 | for inp in inputs:
35 | diff2 = (inp.size(2) - target_shape2) // 2
36 | diff3 = (inp.size(3) - target_shape3) // 2
37 | inputs_.append(inp[:, :, diff2: diff2 + target_shape2, diff3:diff3 + target_shape3])
38 |
39 | return torch.cat(inputs_, dim=self.dim)
40 |
41 | def __len__(self):
42 | return len(self._modules)
43 |
44 |
45 | class GenNoise(nn.Module):
46 | def __init__(self, dim2):
47 | super(GenNoise, self).__init__()
48 | self.dim2 = dim2
49 |
50 | def forward(self, input):
51 | a = list(input.size())
52 | a[1] = self.dim2
53 | # print (input.data.type())
54 |
55 | b = torch.zeros(a).type_as(input.data)
56 | b.normal_()
57 |
58 | x = torch.autograd.Variable(b)
59 |
60 | return x
61 |
62 |
63 | class Swish(nn.Module):
64 | """
65 | https://arxiv.org/abs/1710.05941
66 | The hype was so huge that I could not help but try it
67 | """
68 | def __init__(self):
69 | super(Swish, self).__init__()
70 | self.s = nn.Sigmoid()
71 |
72 | def forward(self, x):
73 | return x * self.s(x)
74 |
75 |
76 | def act(act_fun = 'LeakyReLU'):
77 | '''
78 | Either string defining an activation function or module (e.g. nn.ReLU)
79 | '''
80 | if isinstance(act_fun, str):
81 | if act_fun == 'LeakyReLU':
82 | return nn.LeakyReLU(0.2, inplace=True)
83 | elif act_fun == 'Swish':
84 | return Swish()
85 | elif act_fun == 'ELU':
86 | return nn.ELU()
87 | elif act_fun == 'none':
88 | return nn.Sequential()
89 | else:
90 | assert False
91 | else:
92 | return act_fun()
93 |
94 |
95 | def bn(num_features):
96 | return nn.BatchNorm2d(num_features)
97 |
98 |
99 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero', downsample_mode='stride'):
100 | downsampler = None
101 | if stride != 1 and downsample_mode != 'stride':
102 |
103 | if downsample_mode == 'avg':
104 | downsampler = nn.AvgPool2d(stride, stride)
105 | elif downsample_mode == 'max':
106 | downsampler = nn.MaxPool2d(stride, stride)
107 | elif downsample_mode in ['lanczos2', 'lanczos3']:
108 | downsampler = Downsampler(n_planes=out_f, factor=stride, kernel_type=downsample_mode, phase=0.5, preserve_size=True)
109 | else:
110 | assert False
111 |
112 | stride = 1
113 |
114 | padder = None
115 | to_pad = int((kernel_size - 1) / 2)
116 | if pad == 'reflection':
117 | padder = nn.ReflectionPad2d(to_pad)
118 | to_pad = 0
119 |
120 | convolver = nn.Conv2d(in_f, out_f, kernel_size, stride, padding=to_pad, bias=bias)
121 |
122 |
123 | layers = filter(lambda x: x is not None, [padder, convolver, downsampler])
124 | return nn.Sequential(*layers)
--------------------------------------------------------------------------------
/3_Inpainting/models/dcgan.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | def dcgan(inp=2,
5 | ndf=32,
6 | num_ups=4, need_sigmoid=True, need_bias=True, pad='zero', upsample_mode='nearest', need_convT = True):
7 |
8 | layers= [nn.ConvTranspose2d(inp, ndf, kernel_size=3, stride=1, padding=0, bias=False),
9 | nn.BatchNorm2d(ndf),
10 | nn.LeakyReLU(True)]
11 |
12 | for i in range(num_ups-3):
13 | if need_convT:
14 | layers += [ nn.ConvTranspose2d(ndf, ndf, kernel_size=4, stride=2, padding=1, bias=False),
15 | nn.BatchNorm2d(ndf),
16 | nn.LeakyReLU(True)]
17 | else:
18 | layers += [ nn.Upsample(scale_factor=2, mode=upsample_mode),
19 | nn.Conv2d(ndf, ndf, kernel_size=3, stride=1, padding=1, bias=False),
20 | nn.BatchNorm2d(ndf),
21 | nn.LeakyReLU(True)]
22 |
23 | if need_convT:
24 | layers += [nn.ConvTranspose2d(ndf, 3, 4, 2, 1, bias=False),]
25 | else:
26 | layers += [nn.Upsample(scale_factor=2, mode='bilinear'),
27 | nn.Conv2d(ndf, 3, kernel_size=3, stride=1, padding=1, bias=False)]
28 |
29 |
30 | if need_sigmoid:
31 | layers += [nn.Sigmoid()]
32 |
33 | model =nn.Sequential(*layers)
34 | return model
--------------------------------------------------------------------------------
/3_Inpainting/models/downsampler.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 | import torch.nn as nn
4 |
5 | class Downsampler(nn.Module):
6 | '''
7 | http://www.realitypixels.com/turk/computergraphics/ResamplingFilters.pdf
8 | '''
9 | def __init__(self, n_planes, factor, kernel_type, phase=0, kernel_width=None, support=None, sigma=None, preserve_size=False):
10 | super(Downsampler, self).__init__()
11 |
12 | assert phase in [0, 0.5], 'phase should be 0 or 0.5'
13 |
14 | if kernel_type == 'lanczos2':
15 | support = 2
16 | kernel_width = 4 * factor + 1
17 | kernel_type_ = 'lanczos'
18 |
19 | elif kernel_type == 'lanczos3':
20 | support = 3
21 | kernel_width = 6 * factor + 1
22 | kernel_type_ = 'lanczos'
23 |
24 | elif kernel_type == 'gauss12':
25 | kernel_width = 7
26 | sigma = 1/2
27 | kernel_type_ = 'gauss'
28 |
29 | elif kernel_type == 'gauss1sq2':
30 | kernel_width = 9
31 | sigma = 1./np.sqrt(2)
32 | kernel_type_ = 'gauss'
33 |
34 | elif kernel_type in ['lanczos', 'gauss', 'box']:
35 | kernel_type_ = kernel_type
36 |
37 | else:
38 | assert False, 'wrong name kernel'
39 |
40 |
41 | # note that `kernel width` will be different to actual size for phase = 1/2
42 | self.kernel = get_kernel(factor, kernel_type_, phase, kernel_width, support=support, sigma=sigma)
43 |
44 | downsampler = nn.Conv2d(n_planes, n_planes, kernel_size=self.kernel.shape, stride=factor, padding=0)
45 | downsampler.weight.data[:] = 0
46 | downsampler.bias.data[:] = 0
47 |
48 | kernel_torch = torch.from_numpy(self.kernel)
49 | for i in range(n_planes):
50 | downsampler.weight.data[i, i] = kernel_torch
51 |
52 | self.downsampler_ = downsampler
53 |
54 | if preserve_size:
55 |
56 | if self.kernel.shape[0] % 2 == 1:
57 | pad = int((self.kernel.shape[0] - 1) / 2.)
58 | else:
59 | pad = int((self.kernel.shape[0] - factor) / 2.)
60 |
61 | self.padding = nn.ReplicationPad2d(pad)
62 |
63 | self.preserve_size = preserve_size
64 |
65 | def forward(self, input):
66 | if self.preserve_size:
67 | x = self.padding(input)
68 | else:
69 | x= input
70 | self.x = x
71 | return self.downsampler_(x)
72 |
73 | def get_kernel(factor, kernel_type, phase, kernel_width, support=None, sigma=None):
74 | assert kernel_type in ['lanczos', 'gauss', 'box']
75 |
76 | # factor = float(factor)
77 | if phase == 0.5 and kernel_type != 'box':
78 | kernel = np.zeros([kernel_width - 1, kernel_width - 1])
79 | else:
80 | kernel = np.zeros([kernel_width, kernel_width])
81 |
82 |
83 | if kernel_type == 'box':
84 | assert phase == 0.5, 'Box filter is always half-phased'
85 | kernel[:] = 1./(kernel_width * kernel_width)
86 |
87 | elif kernel_type == 'gauss':
88 | assert sigma, 'sigma is not specified'
89 | assert phase != 0.5, 'phase 1/2 for gauss not implemented'
90 |
91 | center = (kernel_width + 1.)/2.
92 | print(center, kernel_width)
93 | sigma_sq = sigma * sigma
94 |
95 | for i in range(1, kernel.shape[0] + 1):
96 | for j in range(1, kernel.shape[1] + 1):
97 | di = (i - center)/2.
98 | dj = (j - center)/2.
99 | kernel[i - 1][j - 1] = np.exp(-(di * di + dj * dj)/(2 * sigma_sq))
100 | kernel[i - 1][j - 1] = kernel[i - 1][j - 1]/(2. * np.pi * sigma_sq)
101 | elif kernel_type == 'lanczos':
102 | assert support, 'support is not specified'
103 | center = (kernel_width + 1) / 2.
104 |
105 | for i in range(1, kernel.shape[0] + 1):
106 | for j in range(1, kernel.shape[1] + 1):
107 |
108 | if phase == 0.5:
109 | di = abs(i + 0.5 - center) / factor
110 | dj = abs(j + 0.5 - center) / factor
111 | else:
112 | di = abs(i - center) / factor
113 | dj = abs(j - center) / factor
114 |
115 |
116 | pi_sq = np.pi * np.pi
117 |
118 | val = 1
119 | if di != 0:
120 | val = val * support * np.sin(np.pi * di) * np.sin(np.pi * di / support)
121 | val = val / (np.pi * np.pi * di * di)
122 |
123 | if dj != 0:
124 | val = val * support * np.sin(np.pi * dj) * np.sin(np.pi * dj / support)
125 | val = val / (np.pi * np.pi * dj * dj)
126 |
127 | kernel[i - 1][j - 1] = val
128 |
129 |
130 | else:
131 | assert False, 'wrong method name'
132 |
133 | kernel /= kernel.sum()
134 |
135 | return kernel
136 |
137 | #a = Downsampler(n_planes=3, factor=2, kernel_type='lanczos2', phase='1', preserve_size=True)
138 |
139 |
140 |
141 |
142 |
143 |
144 | #################
145 | # Learnable downsampler
146 |
147 | # KS = 32
148 | # dow = nn.Sequential(nn.ReplicationPad2d(int((KS - factor) / 2.)), nn.Conv2d(1,1,KS,factor))
149 |
150 | # class Apply(nn.Module):
151 | # def __init__(self, what, dim, *args):
152 | # super(Apply, self).__init__()
153 | # self.dim = dim
154 |
155 | # self.what = what
156 |
157 | # def forward(self, input):
158 | # inputs = []
159 | # for i in range(input.size(self.dim)):
160 | # inputs.append(self.what(input.narrow(self.dim, i, 1)))
161 |
162 | # return torch.cat(inputs, dim=self.dim)
163 |
164 | # def __len__(self):
165 | # return len(self._modules)
166 |
167 | # downs = Apply(dow, 1)
168 | # downs.type(dtype)(net_input.type(dtype)).size()
169 |
--------------------------------------------------------------------------------
/3_Inpainting/models/resnet.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from numpy.random import normal
4 | from numpy.linalg import svd
5 | from math import sqrt
6 | import torch.nn.init
7 | from .common import *
8 |
9 | class ResidualSequential(nn.Sequential):
10 | def __init__(self, *args):
11 | super(ResidualSequential, self).__init__(*args)
12 |
13 | def forward(self, x):
14 | out = super(ResidualSequential, self).forward(x)
15 | # print(x.size(), out.size())
16 | x_ = None
17 | if out.size(2) != x.size(2) or out.size(3) != x.size(3):
18 | diff2 = x.size(2) - out.size(2)
19 | diff3 = x.size(3) - out.size(3)
20 | # print(1)
21 | x_ = x[:, :, diff2 /2:out.size(2) + diff2 / 2, diff3 / 2:out.size(3) + diff3 / 2]
22 | else:
23 | x_ = x
24 | return out + x_
25 |
26 | def eval(self):
27 | print(2)
28 | for m in self.modules():
29 | m.eval()
30 | exit()
31 |
32 |
33 | def get_block(num_channels, norm_layer, act_fun):
34 | layers = [
35 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False),
36 | norm_layer(num_channels, affine=True),
37 | act(act_fun),
38 | nn.Conv2d(num_channels, num_channels, 3, 1, 1, bias=False),
39 | norm_layer(num_channels, affine=True),
40 | ]
41 | return layers
42 |
43 |
44 | class ResNet(nn.Module):
45 | def __init__(self, num_input_channels, num_output_channels, num_blocks, num_channels, need_residual=True, act_fun='LeakyReLU', need_sigmoid=True, norm_layer=nn.BatchNorm2d, pad='reflection'):
46 | '''
47 | pad = 'start|zero|replication'
48 | '''
49 | super(ResNet, self).__init__()
50 |
51 | if need_residual:
52 | s = ResidualSequential
53 | else:
54 | s = nn.Sequential
55 |
56 | stride = 1
57 | # First layers
58 | layers = [
59 | # nn.ReplicationPad2d(num_blocks * 2 * stride + 3),
60 | conv(num_input_channels, num_channels, 3, stride=1, bias=True, pad=pad),
61 | act(act_fun)
62 | ]
63 | # Residual blocks
64 | # layers_residual = []
65 | for i in range(num_blocks):
66 | layers += [s(*get_block(num_channels, norm_layer, act_fun))]
67 |
68 | layers += [
69 | nn.Conv2d(num_channels, num_channels, 3, 1, 1),
70 | norm_layer(num_channels, affine=True)
71 | ]
72 |
73 | # if need_residual:
74 | # layers += [ResidualSequential(*layers_residual)]
75 | # else:
76 | # layers += [Sequential(*layers_residual)]
77 |
78 | # if factor >= 2:
79 | # # Do upsampling if needed
80 | # layers += [
81 | # nn.Conv2d(num_channels, num_channels *
82 | # factor ** 2, 3, 1),
83 | # nn.PixelShuffle(factor),
84 | # act(act_fun)
85 | # ]
86 | layers += [
87 | conv(num_channels, num_output_channels, 3, 1, bias=True, pad=pad),
88 | nn.Sigmoid()
89 | ]
90 | self.model = nn.Sequential(*layers)
91 |
92 | def forward(self, input):
93 | return self.model(input)
94 |
95 | def eval(self):
96 | self.model.eval()
97 |
--------------------------------------------------------------------------------
/3_Inpainting/models/skip.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from .common import *
4 |
5 | def skip(
6 | num_input_channels=2, num_output_channels=3,
7 | num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4],
8 | filter_size_down=3, filter_size_up=3, filter_skip_size=1,
9 | need_sigmoid=True, need_bias=True,
10 | pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU',
11 | need1x1_up=True):
12 | """Assembles encoder-decoder with skip connections.
13 |
14 | Arguments:
15 | act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU)
16 | pad (string): zero|reflection (default: 'zero')
17 | upsample_mode (string): 'nearest|bilinear' (default: 'nearest')
18 | downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride')
19 |
20 | """
21 | assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip)
22 |
23 | n_scales = len(num_channels_down)
24 |
25 | if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) :
26 | upsample_mode = [upsample_mode]*n_scales
27 |
28 | if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)):
29 | downsample_mode = [downsample_mode]*n_scales
30 |
31 | if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) :
32 | filter_size_down = [filter_size_down]*n_scales
33 |
34 | if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) :
35 | filter_size_up = [filter_size_up]*n_scales
36 |
37 | last_scale = n_scales - 1
38 |
39 | cur_depth = None
40 |
41 | model = nn.Sequential()
42 | model_tmp = model
43 |
44 | input_depth = num_input_channels
45 | for i in range(len(num_channels_down)):
46 |
47 | deeper = nn.Sequential()
48 | skip = nn.Sequential()
49 |
50 | if num_channels_skip[i] != 0:
51 | model_tmp.add(Concat(1, skip, deeper))
52 | else:
53 | model_tmp.add(deeper)
54 |
55 | model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i])))
56 |
57 | if num_channels_skip[i] != 0:
58 | skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad))
59 | skip.add(bn(num_channels_skip[i]))
60 | skip.add(act(act_fun))
61 |
62 | # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part))
63 |
64 | deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i]))
65 | deeper.add(bn(num_channels_down[i]))
66 | deeper.add(act(act_fun))
67 |
68 | deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad))
69 | deeper.add(bn(num_channels_down[i]))
70 | deeper.add(act(act_fun))
71 |
72 | deeper_main = nn.Sequential()
73 |
74 | if i == len(num_channels_down) - 1:
75 | # The deepest
76 | k = num_channels_down[i]
77 | else:
78 | deeper.add(deeper_main)
79 | k = num_channels_up[i + 1]
80 |
81 | deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i]))
82 |
83 | model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad))
84 | model_tmp.add(bn(num_channels_up[i]))
85 | model_tmp.add(act(act_fun))
86 |
87 |
88 | if need1x1_up:
89 | model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad))
90 | model_tmp.add(bn(num_channels_up[i]))
91 | model_tmp.add(act(act_fun))
92 |
93 | input_depth = num_channels_down[i]
94 | model_tmp = deeper_main
95 |
96 | model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad))
97 | if need_sigmoid:
98 | model.add(nn.Sigmoid())
99 |
100 | return model
101 |
--------------------------------------------------------------------------------
/3_Inpainting/models/texture_nets.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from .common import *
4 |
5 |
6 | normalization = nn.BatchNorm2d
7 |
8 |
9 | def conv(in_f, out_f, kernel_size, stride=1, bias=True, pad='zero'):
10 | if pad == 'zero':
11 | return nn.Conv2d(in_f, out_f, kernel_size, stride, padding=(kernel_size - 1) / 2, bias=bias)
12 | elif pad == 'reflection':
13 | layers = [nn.ReflectionPad2d((kernel_size - 1) / 2),
14 | nn.Conv2d(in_f, out_f, kernel_size, stride, padding=0, bias=bias)]
15 | return nn.Sequential(*layers)
16 |
17 | def get_texture_nets(inp=3, ratios = [32, 16, 8, 4, 2, 1], fill_noise=False, pad='zero', need_sigmoid=False, conv_num=8, upsample_mode='nearest'):
18 |
19 |
20 | for i in range(len(ratios)):
21 | j = i + 1
22 |
23 | seq = nn.Sequential()
24 |
25 | tmp = nn.AvgPool2d(ratios[i], ratios[i])
26 |
27 | seq.add(tmp)
28 | if fill_noise:
29 | seq.add(GenNoise(inp))
30 |
31 | seq.add(conv(inp, conv_num, 3, pad=pad))
32 | seq.add(normalization(conv_num))
33 | seq.add(act())
34 |
35 | seq.add(conv(conv_num, conv_num, 3, pad=pad))
36 | seq.add(normalization(conv_num))
37 | seq.add(act())
38 |
39 | seq.add(conv(conv_num, conv_num, 1, pad=pad))
40 | seq.add(normalization(conv_num))
41 | seq.add(act())
42 |
43 | if i == 0:
44 | seq.add(nn.Upsample(scale_factor=2, mode=upsample_mode))
45 | cur = seq
46 | else:
47 |
48 | cur_temp = cur
49 |
50 | cur = nn.Sequential()
51 |
52 | # Batch norm before merging
53 | seq.add(normalization(conv_num))
54 | cur_temp.add(normalization(conv_num * (j - 1)))
55 |
56 | cur.add(Concat(1, cur_temp, seq))
57 |
58 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad))
59 | cur.add(normalization(conv_num * j))
60 | cur.add(act())
61 |
62 | cur.add(conv(conv_num * j, conv_num * j, 3, pad=pad))
63 | cur.add(normalization(conv_num * j))
64 | cur.add(act())
65 |
66 | cur.add(conv(conv_num * j, conv_num * j, 1, pad=pad))
67 | cur.add(normalization(conv_num * j))
68 | cur.add(act())
69 |
70 | if i == len(ratios) - 1:
71 | cur.add(conv(conv_num * j, 3, 1, pad=pad))
72 | else:
73 | cur.add(nn.Upsample(scale_factor=2, mode=upsample_mode))
74 |
75 | model = cur
76 | if need_sigmoid:
77 | model.add(nn.Sigmoid())
78 |
79 | return model
80 |
--------------------------------------------------------------------------------
/3_Inpainting/psnr_ssim.py:
--------------------------------------------------------------------------------
1 | ### This script will calcualte psnr and ssim
2 | import numpy as np
3 | from skimage.metrics import peak_signal_noise_ratio
4 | from skimage.measure import compare_ssim
5 |
6 |
7 | def calcualte_PSNR(im_true, im_test):
8 | im_true = np.transpose(im_true,(1,2,0))
9 | im_test = np.transpose(im_test, (1, 2, 0))
10 | psnr_value = peak_signal_noise_ratio(im_true, im_test)
11 | return psnr_value
12 |
13 |
14 | def calcualte_SSIM(im_true, im_test, multichannel): # multichannel=True for RGB images
15 | im_true = np.transpose(im_true, (1, 2, 0))
16 | im_test = np.transpose(im_test, (1, 2, 0))
17 | ssim_value = compare_ssim(im_true, im_test, multichannel=multichannel, data_range=im_test.max() - im_test.min())
18 | return ssim_value
19 |
20 |
21 |
22 |
23 |
24 | if __name__ == '__main__':
25 | pass
--------------------------------------------------------------------------------
/3_Inpainting/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__init__.py
--------------------------------------------------------------------------------
/3_Inpainting/utils/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/utils/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/__init__.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/utils/__pycache__/common_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/common_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/utils/__pycache__/common_utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/common_utils.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/utils/__pycache__/denoising_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/denoising_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/3_Inpainting/utils/__pycache__/denoising_utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/__pycache__/denoising_utils.cpython-37.pyc
--------------------------------------------------------------------------------
/3_Inpainting/utils/common_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision
4 | import sys
5 |
6 | import numpy as np
7 | from PIL import Image
8 | import PIL
9 | import numpy as np
10 |
11 | import matplotlib.pyplot as plt
12 |
13 | def crop_image(img, d=32):
14 | '''Make dimensions divisible by `d`'''
15 |
16 | new_size = (img.size[0] - img.size[0] % d,
17 | img.size[1] - img.size[1] % d)
18 |
19 | bbox = [
20 | int((img.size[0] - new_size[0])/2),
21 | int((img.size[1] - new_size[1])/2),
22 | int((img.size[0] + new_size[0])/2),
23 | int((img.size[1] + new_size[1])/2),
24 | ]
25 |
26 | img_cropped = img.crop(bbox)
27 | return img_cropped
28 |
29 | def get_params(opt_over, net, net_input, downsampler=None):
30 | '''Returns parameters that we want to optimize over.
31 |
32 | Args:
33 | opt_over: comma separated list, e.g. "net,input" or "net"
34 | net: network
35 | net_input: torch.Tensor that stores input `z`
36 | '''
37 | opt_over_list = opt_over.split(',')
38 | params = []
39 |
40 | for opt in opt_over_list:
41 |
42 | if opt == 'net':
43 | params += [x for x in net.parameters() ]
44 | elif opt=='down':
45 | assert downsampler is not None
46 | params = [x for x in downsampler.parameters()]
47 | elif opt == 'input':
48 | net_input.requires_grad = True
49 | params += [net_input]
50 | else:
51 | assert False, 'what is it?'
52 |
53 | return params
54 |
55 | def get_image_grid(images_np, nrow=8):
56 | '''Creates a grid from a list of images by concatenating them.'''
57 | images_torch = [torch.from_numpy(x) for x in images_np]
58 | torch_grid = torchvision.utils.make_grid(images_torch, nrow)
59 |
60 | return torch_grid.numpy()
61 |
62 | def plot_image_grid(images_np, nrow =8, factor=1, interpolation='lanczos'):
63 | """Draws images in a grid
64 |
65 | Args:
66 | images_np: list of images, each image is np.array of size 3xHxW of 1xHxW
67 | nrow: how many images will be in one row
68 | factor: size if the plt.figure
69 | interpolation: interpolation used in plt.imshow
70 | """
71 | n_channels = max(x.shape[0] for x in images_np)
72 | assert (n_channels == 3) or (n_channels == 1), "images should have 1 or 3 channels"
73 |
74 | images_np = [x if (x.shape[0] == n_channels) else np.concatenate([x, x, x], axis=0) for x in images_np]
75 |
76 | grid = get_image_grid(images_np, nrow)
77 |
78 | plt.figure(figsize=(len(images_np) + factor, 12 + factor))
79 |
80 | if images_np[0].shape[0] == 1:
81 | plt.imshow(grid[0], cmap='gray', interpolation=interpolation)
82 | else:
83 | plt.imshow(grid.transpose(1, 2, 0), interpolation=interpolation)
84 |
85 | plt.show()
86 |
87 | return grid
88 |
89 | def load(path):
90 | """Load PIL image."""
91 | img = Image.open(path)
92 | return img
93 |
94 | def get_image(path, imsize=-1):
95 | """Load an image and resize to a cpecific size.
96 |
97 | Args:
98 | path: path to image
99 | imsize: tuple or scalar with dimensions; -1 for `no resize`
100 | """
101 | img = load(path)
102 |
103 | if isinstance(imsize, int):
104 | imsize = (imsize, imsize)
105 |
106 | if imsize[0]!= -1 and img.size != imsize:
107 | if imsize[0] > img.size[0]:
108 | img = img.resize(imsize, Image.BICUBIC)
109 | else:
110 | img = img.resize(imsize, Image.ANTIALIAS)
111 |
112 | img_np = pil_to_np(img)
113 |
114 | return img, img_np
115 |
116 |
117 |
118 | def fill_noise(x, noise_type):
119 | """Fills tensor `x` with noise of type `noise_type`."""
120 | if noise_type == 'u':
121 | x.uniform_()
122 | elif noise_type == 'n':
123 | x.normal_()
124 | else:
125 | assert False
126 |
127 | def get_noise(input_depth, method, spatial_size, noise_type='u', var=1./10):
128 | """Returns a pytorch.Tensor of size (1 x `input_depth` x `spatial_size[0]` x `spatial_size[1]`)
129 | initialized in a specific way.
130 | Args:
131 | input_depth: number of channels in the tensor
132 | method: `noise` for fillting tensor with noise; `meshgrid` for np.meshgrid
133 | spatial_size: spatial size of the tensor to initialize
134 | noise_type: 'u' for uniform; 'n' for normal
135 | var: a factor, a noise will be multiplicated by. Basically it is standard deviation scaler.
136 | """
137 | if isinstance(spatial_size, int):
138 | spatial_size = (spatial_size, spatial_size)
139 | if method == 'noise':
140 | shape = [1, input_depth, spatial_size[0], spatial_size[1]]
141 | net_input = torch.zeros(shape)
142 |
143 | fill_noise(net_input, noise_type)
144 | net_input *= var
145 | elif method == 'meshgrid':
146 | assert input_depth == 2
147 | X, Y = np.meshgrid(np.arange(0, spatial_size[1])/float(spatial_size[1]-1), np.arange(0, spatial_size[0])/float(spatial_size[0]-1))
148 | meshgrid = np.concatenate([X[None,:], Y[None,:]])
149 | net_input= np_to_torch(meshgrid)
150 | else:
151 | assert False
152 |
153 | return net_input
154 |
155 | def pil_to_np(img_PIL):
156 | '''Converts image in PIL format to np.array.
157 |
158 | From W x H x C [0...255] to C x W x H [0..1]
159 | '''
160 | ar = np.array(img_PIL)
161 |
162 | if len(ar.shape) == 3:
163 | ar = ar.transpose(2,0,1)
164 | else:
165 | ar = ar[None, ...]
166 |
167 | return ar.astype(np.float32) / 255.
168 |
169 | def np_to_pil(img_np):
170 | '''Converts image in np.array format to PIL image.
171 |
172 | From C x W x H [0..1] to W x H x C [0...255]
173 | '''
174 | ar = np.clip(img_np*255,0,255).astype(np.uint8)
175 |
176 | if img_np.shape[0] == 1:
177 | ar = ar[0]
178 | else:
179 | ar = ar.transpose(1, 2, 0)
180 |
181 | return Image.fromarray(ar)
182 |
183 | def np_to_torch(img_np):
184 | '''Converts image in numpy.array to torch.Tensor.
185 |
186 | From C x W x H [0..1] to C x W x H [0..1]
187 | '''
188 | return torch.from_numpy(img_np)[None, :]
189 |
190 | def torch_to_np(img_var):
191 | '''Converts an image in torch.Tensor format to np.array.
192 |
193 | From 1 x C x W x H [0..1] to C x W x H [0..1]
194 | '''
195 | return img_var.detach().cpu().numpy()[0]
196 |
197 |
198 | def optimize(optimizer_type, parameters, closure, LR, num_iter):
199 | """Runs optimization loop.
200 |
201 | Args:
202 | optimizer_type: 'LBFGS' of 'adam'
203 | parameters: list of Tensors to optimize over
204 | closure: function, that returns loss variable
205 | LR: learning rate
206 | num_iter: number of iterations
207 | """
208 | if optimizer_type == 'LBFGS':
209 | # Do several steps with adam first
210 | optimizer = torch.optim.Adam(parameters, lr=0.001)
211 | for j in range(100):
212 | optimizer.zero_grad()
213 | closure()
214 | optimizer.step()
215 |
216 | print('Starting optimization with LBFGS')
217 | def closure2():
218 | optimizer.zero_grad()
219 | return closure()
220 | optimizer = torch.optim.LBFGS(parameters, max_iter=num_iter, lr=LR, tolerance_grad=-1, tolerance_change=-1)
221 | optimizer.step(closure2)
222 |
223 | elif optimizer_type == 'adam':
224 | print('Starting optimization with ADAM')
225 | optimizer = torch.optim.Adam(parameters, lr=LR)
226 |
227 | for j in range(num_iter):
228 | optimizer.zero_grad()
229 | closure()
230 | optimizer.step()
231 | else:
232 | assert False
--------------------------------------------------------------------------------
/3_Inpainting/utils/denoising_utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | from .common_utils import *
3 |
4 |
5 |
6 | def get_noisy_image(img_np, sigma):
7 | """Adds Gaussian noise to an image.
8 |
9 | Args:
10 | img_np: image, np.array with values from 0 to 1
11 | sigma: std of the noise
12 | """
13 | img_noisy_np = np.clip(img_np + np.random.normal(scale=sigma, size=img_np.shape), 0, 1).astype(np.float32)
14 | img_noisy_pil = np_to_pil(img_noisy_np)
15 |
16 | return img_noisy_pil, img_noisy_np
--------------------------------------------------------------------------------
/3_Inpainting/utils/feature_inversion_utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torchvision.transforms as transforms
4 | import torchvision.models as models
5 | from .matcher import Matcher
6 | import os
7 | from collections import OrderedDict
8 |
9 | class View(nn.Module):
10 | def __init__(self):
11 | super(View, self).__init__()
12 |
13 | def forward(self, x):
14 | return x.view(-1)
15 |
16 | def get_vanilla_vgg_features(cut_idx=-1):
17 | if not os.path.exists('vgg_features.pth'):
18 | os.system(
19 | 'wget --no-check-certificate -N https://s3-us-west-2.amazonaws.com/jcjohns-models/vgg19-d01eb7cb.pth')
20 | vgg_weights = torch.load('vgg19-d01eb7cb.pth')
21 | # fix compatibility issues
22 | map = {'classifier.6.weight':u'classifier.7.weight', 'classifier.6.bias':u'classifier.7.bias'}
23 | vgg_weights = OrderedDict([(map[k] if k in map else k,v) for k,v in vgg_weights.iteritems()])
24 |
25 |
26 |
27 | model = models.vgg19()
28 | model.classifier = nn.Sequential(View(), *model.classifier._modules.values())
29 |
30 |
31 | model.load_state_dict(vgg_weights)
32 |
33 | torch.save(model.features, 'vgg_features.pth')
34 | torch.save(model.classifier, 'vgg_classifier.pth')
35 |
36 | vgg = torch.load('vgg_features.pth')
37 | if cut_idx > 36:
38 | vgg_classifier = torch.load('vgg_classifier.pth')
39 | vgg = nn.Sequential(*(vgg._modules.values() + vgg_classifier._modules.values()))
40 |
41 | vgg.eval()
42 |
43 | return vgg
44 |
45 |
46 | def get_matcher(net, opt):
47 | idxs = [x for x in opt['layers'].split(',')]
48 | matcher = Matcher(opt['what'])
49 |
50 | def hook(module, input, output):
51 | matcher(module, output)
52 |
53 | for i in idxs:
54 | net._modules[i].register_forward_hook(hook)
55 |
56 | return matcher
57 |
58 |
59 |
60 | def get_vgg(cut_idx=-1):
61 | f = get_vanilla_vgg_features(cut_idx)
62 |
63 | if cut_idx > 0:
64 | num_modules = len(f._modules)
65 | keys_to_delete = [f._modules.keys()[x] for x in range(cut_idx, num_modules)]
66 | for k in keys_to_delete:
67 | del f._modules[k]
68 |
69 | return f
70 |
71 | def vgg_preprocess_var(var):
72 | (r, g, b) = torch.chunk(var, 3, dim=1)
73 | bgr = torch.cat((b, g, r), 1)
74 | out = bgr * 255 - torch.autograd.Variable(vgg_mean[None, ...]).type(var.type()).expand_as(bgr)
75 | return out
76 |
77 | vgg_mean = torch.FloatTensor([103.939, 116.779, 123.680]).view(3, 1, 1)
78 |
79 |
80 |
81 | def get_preprocessor(imsize):
82 | def vgg_preprocess(tensor):
83 | (r, g, b) = torch.chunk(tensor, 3, dim=0)
84 | bgr = torch.cat((b, g, r), 0)
85 | out = bgr * 255 - vgg_mean.type(tensor.type()).expand_as(bgr)
86 | return out
87 | preprocess = transforms.Compose([
88 | transforms.Resize(imsize),
89 | transforms.ToTensor(),
90 | transforms.Lambda(vgg_preprocess)
91 | ])
92 |
93 | return preprocess
94 |
95 |
96 | def get_deprocessor():
97 | def vgg_deprocess(tensor):
98 | bgr = (tensor + vgg_mean.expand_as(tensor)) / 255.0
99 | (b, g, r) = torch.chunk(bgr, 3, dim=0)
100 | rgb = torch.cat((r, g, b), 0)
101 | return rgb
102 | deprocess = transforms.Compose([
103 | transforms.Lambda(vgg_deprocess),
104 | transforms.Lambda(lambda x: torch.clamp(x, 0, 1)),
105 | transforms.ToPILImage()
106 | ])
107 | return deprocess
108 |
--------------------------------------------------------------------------------
/3_Inpainting/utils/inpainting_utils.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from PIL import Image
3 | import PIL.ImageDraw as ImageDraw
4 | import PIL.ImageFont as ImageFont
5 | from .common_utils import *
6 |
7 | def get_text_mask(for_image, sz=20):
8 | font_fname = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf'
9 | font_size = sz
10 | font = ImageFont.truetype(font_fname, font_size)
11 |
12 | img_mask = Image.fromarray(np.array(for_image)*0+255)
13 | draw = ImageDraw.Draw(img_mask)
14 | draw.text((128, 128), "hello world", font=font, fill='rgb(0, 0, 0)')
15 |
16 | return img_mask
17 |
18 | def get_bernoulli_mask(for_image, zero_fraction=0.95):
19 | img_mask_np=(np.random.random_sample(size=pil_to_np(for_image).shape) > zero_fraction).astype(int)
20 | img_mask = np_to_pil(img_mask_np)
21 |
22 | return img_mask
23 |
--------------------------------------------------------------------------------
/3_Inpainting/utils/matcher.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | class Matcher:
5 | def __init__(self, how='gram_matrix', loss='mse'):
6 | self.mode = 'store'
7 | self.stored = {}
8 | self.losses = {}
9 |
10 | if how in all_features.keys():
11 | self.get_statistics = all_features[how]
12 | else:
13 | assert False
14 | pass
15 |
16 | if loss in all_losses.keys():
17 | self.loss = all_losses[loss]
18 | else:
19 | assert False
20 |
21 | def __call__(self, module, features):
22 | statistics = self.get_statistics(features)
23 |
24 | self.statistics = statistics
25 | if self.mode == 'store':
26 | self.stored[module] = statistics.detach().clone()
27 | elif self.mode == 'match':
28 | self.losses[module] = self.loss(statistics, self.stored[module])
29 |
30 | def clean(self):
31 | self.losses = {}
32 |
33 | def gram_matrix(x):
34 | (b, ch, h, w) = x.size()
35 | features = x.view(b, ch, w * h)
36 | features_t = features.transpose(1, 2)
37 | gram = features.bmm(features_t) / (ch * h * w)
38 | return gram
39 |
40 |
41 | def features(x):
42 | return x
43 |
44 |
45 | all_features = {
46 | 'gram_matrix': gram_matrix,
47 | 'features': features,
48 | }
49 |
50 | all_losses = {
51 | 'mse': nn.MSELoss(),
52 | 'smoothL1': nn.SmoothL1Loss(),
53 | 'L1': nn.L1Loss(),
54 | }
55 |
--------------------------------------------------------------------------------
/3_Inpainting/utils/perceptual_loss/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sun-umn/Deep-Random-Projector/cdddfb1e69b4e50d67f5ca6350bffa8c9fd60856/3_Inpainting/utils/perceptual_loss/__init__.py
--------------------------------------------------------------------------------
/3_Inpainting/utils/perceptual_loss/matcher.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | class Matcher:
6 | def __init__(self, how='gram_matrix', loss='mse', map_index=933):
7 | self.mode = 'store'
8 | self.stored = {}
9 | self.losses = {}
10 |
11 | if how in all_features.keys():
12 | self.get_statistics = all_features[how]
13 | else:
14 | assert False
15 | pass
16 |
17 | if loss in all_losses.keys():
18 | self.loss = all_losses[loss]
19 | else:
20 | assert False
21 |
22 | self.map_index = map_index
23 | self.method = 'match'
24 |
25 |
26 | def __call__(self, module, features):
27 | statistics = self.get_statistics(features)
28 |
29 | self.statistics = statistics
30 | if self.mode == 'store':
31 | self.stored[module] = statistics.detach()
32 |
33 | elif self.mode == 'match':
34 |
35 | if statistics.ndimension() == 2:
36 |
37 | if self.method == 'maximize':
38 | self.losses[module] = - statistics[0, self.map_index]
39 | else:
40 | self.losses[module] = torch.abs(300 - statistics[0, self.map_index])
41 |
42 | else:
43 | ws = self.window_size
44 |
45 | t = statistics.detach() * 0
46 |
47 | s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
48 | t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
49 | t_cc[:, self.map_index,...] = 1
50 |
51 | if self.method == 'maximize':
52 | self.losses[module] = -(s_cc * t_cc.contiguous()).sum()
53 | else:
54 | self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum()
55 |
56 |
57 | def clean(self):
58 | self.losses = {}
59 |
60 | def gram_matrix(x):
61 | (b, ch, h, w) = x.size()
62 | features = x.view(b, ch, w * h)
63 | features_t = features.transpose(1, 2)
64 | gram = features.bmm(features_t) / (ch * h * w)
65 | return gram
66 |
67 |
68 | def features(x):
69 | return x
70 |
71 |
72 | all_features = {
73 | 'gram_matrix': gram_matrix,
74 | 'features': features,
75 | }
76 |
77 | all_losses = {
78 | 'mse': nn.MSELoss(),
79 | 'smoothL1': nn.SmoothL1Loss(),
80 | 'L1': nn.L1Loss(),
81 | }
82 |
--------------------------------------------------------------------------------
/3_Inpainting/utils/perceptual_loss/vgg_modified.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 |
3 | class VGGModified(nn.Module):
4 | def __init__(self, vgg19_orig, slope=0.01):
5 | super(VGGModified, self).__init__()
6 | self.features = nn.Sequential()
7 |
8 | self.features.add_module(str(0), vgg19_orig.features[0])
9 | self.features.add_module(str(1), nn.LeakyReLU(slope, True))
10 | self.features.add_module(str(2), vgg19_orig.features[2])
11 | self.features.add_module(str(3), nn.LeakyReLU(slope, True))
12 | self.features.add_module(str(4), nn.AvgPool2d((2,2), (2,2)))
13 |
14 | self.features.add_module(str(5), vgg19_orig.features[5])
15 | self.features.add_module(str(6), nn.LeakyReLU(slope, True))
16 | self.features.add_module(str(7), vgg19_orig.features[7])
17 | self.features.add_module(str(8), nn.LeakyReLU(slope, True))
18 | self.features.add_module(str(9), nn.AvgPool2d((2,2), (2,2)))
19 |
20 | self.features.add_module(str(10), vgg19_orig.features[10])
21 | self.features.add_module(str(11), nn.LeakyReLU(slope, True))
22 | self.features.add_module(str(12), vgg19_orig.features[12])
23 | self.features.add_module(str(13), nn.LeakyReLU(slope, True))
24 | self.features.add_module(str(14), vgg19_orig.features[14])
25 | self.features.add_module(str(15), nn.LeakyReLU(slope, True))
26 | self.features.add_module(str(16), vgg19_orig.features[16])
27 | self.features.add_module(str(17), nn.LeakyReLU(slope, True))
28 | self.features.add_module(str(18), nn.AvgPool2d((2,2), (2,2)))
29 |
30 | self.features.add_module(str(19), vgg19_orig.features[19])
31 | self.features.add_module(str(20), nn.LeakyReLU(slope, True))
32 | self.features.add_module(str(21), vgg19_orig.features[21])
33 | self.features.add_module(str(22), nn.LeakyReLU(slope, True))
34 | self.features.add_module(str(23), vgg19_orig.features[23])
35 | self.features.add_module(str(24), nn.LeakyReLU(slope, True))
36 | self.features.add_module(str(25), vgg19_orig.features[25])
37 | self.features.add_module(str(26), nn.LeakyReLU(slope, True))
38 | self.features.add_module(str(27), nn.AvgPool2d((2,2), (2,2)))
39 |
40 | self.features.add_module(str(28), vgg19_orig.features[28])
41 | self.features.add_module(str(29), nn.LeakyReLU(slope, True))
42 | self.features.add_module(str(30), vgg19_orig.features[30])
43 | self.features.add_module(str(31), nn.LeakyReLU(slope, True))
44 | self.features.add_module(str(32), vgg19_orig.features[32])
45 | self.features.add_module(str(33), nn.LeakyReLU(slope, True))
46 | self.features.add_module(str(34), vgg19_orig.features[34])
47 | self.features.add_module(str(35), nn.LeakyReLU(slope, True))
48 | self.features.add_module(str(36), nn.AvgPool2d((2,2), (2,2)))
49 |
50 | self.classifier = nn.Sequential()
51 |
52 | self.classifier.add_module(str(0), vgg19_orig.classifier[0])
53 | self.classifier.add_module(str(1), nn.LeakyReLU(slope, True))
54 | self.classifier.add_module(str(2), nn.Dropout2d(p = 0.5))
55 | self.classifier.add_module(str(3), vgg19_orig.classifier[3])
56 | self.classifier.add_module(str(4), nn.LeakyReLU(slope, True))
57 | self.classifier.add_module(str(5), nn.Dropout2d(p = 0.5))
58 | self.classifier.add_module(str(6), vgg19_orig.classifier[6])
59 |
60 | def forward(self, x):
61 | return self.classifier(self.features.forward(x))
--------------------------------------------------------------------------------
/3_Inpainting/utils/sr_utils.py:
--------------------------------------------------------------------------------
1 | from .common_utils import *
2 |
3 | def put_in_center(img_np, target_size):
4 | img_out = np.zeros([3, target_size[0], target_size[1]])
5 |
6 | bbox = [
7 | int((target_size[0] - img_np.shape[1]) / 2),
8 | int((target_size[1] - img_np.shape[2]) / 2),
9 | int((target_size[0] + img_np.shape[1]) / 2),
10 | int((target_size[1] + img_np.shape[2]) / 2),
11 | ]
12 |
13 | img_out[:, bbox[0]:bbox[2], bbox[1]:bbox[3]] = img_np
14 |
15 | return img_out
16 |
17 |
18 | def load_LR_HR_imgs_sr(fname, imsize, factor, enforse_div32=None):
19 | '''Loads an image, resizes it, center crops and downscales.
20 |
21 | Args:
22 | fname: path to the image
23 | imsize: new size for the image, -1 for no resizing
24 | factor: downscaling factor
25 | enforse_div32: if 'CROP' center crops an image, so that its dimensions are divisible by 32.
26 | '''
27 | img_orig_pil, img_orig_np = get_image(fname, -1)
28 |
29 | if imsize != -1:
30 | img_orig_pil, img_orig_np = get_image(fname, imsize)
31 |
32 | # For comparison with GT
33 | if enforse_div32 == 'CROP':
34 | new_size = (img_orig_pil.size[0] - img_orig_pil.size[0] % 32,
35 | img_orig_pil.size[1] - img_orig_pil.size[1] % 32)
36 |
37 | bbox = [
38 | (img_orig_pil.size[0] - new_size[0])/2,
39 | (img_orig_pil.size[1] - new_size[1])/2,
40 | (img_orig_pil.size[0] + new_size[0])/2,
41 | (img_orig_pil.size[1] + new_size[1])/2,
42 | ]
43 |
44 | img_HR_pil = img_orig_pil.crop(bbox)
45 | img_HR_np = pil_to_np(img_HR_pil)
46 | else:
47 | img_HR_pil, img_HR_np = img_orig_pil, img_orig_np
48 |
49 | LR_size = [
50 | img_HR_pil.size[0] // factor,
51 | img_HR_pil.size[1] // factor
52 | ]
53 |
54 | img_LR_pil = img_HR_pil.resize(LR_size, Image.ANTIALIAS)
55 | img_LR_np = pil_to_np(img_LR_pil)
56 |
57 | print('HR and LR resolutions: %s, %s' % (str(img_HR_pil.size), str (img_LR_pil.size)))
58 |
59 | return {
60 | 'orig_pil': img_orig_pil,
61 | 'orig_np': img_orig_np,
62 | 'LR_pil': img_LR_pil,
63 | 'LR_np': img_LR_np,
64 | 'HR_pil': img_HR_pil,
65 | 'HR_np': img_HR_np
66 | }
67 |
68 |
69 | def get_baselines(img_LR_pil, img_HR_pil):
70 | '''Gets `bicubic`, sharpened bicubic and `nearest` baselines.'''
71 | img_bicubic_pil = img_LR_pil.resize(img_HR_pil.size, Image.BICUBIC)
72 | img_bicubic_np = pil_to_np(img_bicubic_pil)
73 |
74 | img_nearest_pil = img_LR_pil.resize(img_HR_pil.size, Image.NEAREST)
75 | img_nearest_np = pil_to_np(img_nearest_pil)
76 |
77 | img_bic_sharp_pil = img_bicubic_pil.filter(PIL.ImageFilter.UnsharpMask())
78 | img_bic_sharp_np = pil_to_np(img_bic_sharp_pil)
79 |
80 | return img_bicubic_np, img_bic_sharp_np, img_nearest_np
81 |
82 |
83 |
84 | def tv_loss(x, beta = 0.5):
85 | '''Calculates TV loss for an image `x`.
86 |
87 | Args:
88 | x: image, torch.Variable of torch.Tensor
89 | beta: See https://arxiv.org/abs/1412.0035 (fig. 2) to see effect of `beta`
90 | '''
91 | dh = torch.pow(x[:,:,:,1:] - x[:,:,:,:-1], 2)
92 | dw = torch.pow(x[:,:,1:,:] - x[:,:,:-1,:], 2)
93 |
94 | return torch.sum(torch.pow(dh[:, :, :-1] + dw[:, :, :, :-1], beta))
95 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 GLOVEX @ UMN
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Deep-Random-Projector
2 | ## Deep Random Projector: Accelerated Deep Image Prior
3 |
4 | This is the official implementation of our paper *Deep Random Projector: Accelerated Deep Image Prior* which has been accepted to the [CVPR2023](https://cvpr2023.thecvf.com/). You can find our paper via [this link](https://openaccess.thecvf.com/content/CVPR2023/html/Li_Deep_Random_Projector_Accelerated_Deep_Image_Prior_CVPR_2023_paper.html).
5 |
6 |
7 | ## Set Up the Environment
8 |
9 | 1. Get and clone the github repository:
10 |
11 | `git clone https://github.com/sun-umn/Deep-Random-Projector/`
12 |
13 | 2. Switch to `Deep-Random-Projector` :
14 |
15 | `cd XXX/Deep-Random-Projector`
16 | (*Note*: `XXX` here indicates the upper directory of `Deep-Random-Projector`. For example, if you clone `Deep-Random-Projector` under `/home/Download`, then you should replace `XXX` with `/home/Download`.)
17 |
18 | 3. Create a new conda environment with the YML file we provide:
19 |
20 | `conda env create -f environment.yml`
21 |
22 | 4. Activate conda environment and you are now ready to explpre the codes/models!
23 |
24 | `conda activate pytorch_py3.6`
25 |
26 |
27 | ## Explore the Codes/Models
28 |
29 | - **0_Dataset**: we provide a ready-to-use *example* dataset.
30 |
31 | - **1_Denoising**: the code for image denoising.
32 |
33 | - **2_Super_Resolution**: the code for image super-solution.
34 |
35 | - **3_Inpainting**: the code for image inpainting.
36 |
37 | We have provided the detailed inline comments in each Python file. You can modify any parameters you want to explore the models. Or, you can try our method by simply running the command below:
38 |
39 | ```
40 | python Main_Start.py
41 | ```
42 |
43 |
44 | ## Citation/BibTex
45 |
46 | More technical details and experimental results can be found in our paper:
47 |
48 | Taihui Li, Hengkang Wang, Zhong Zhuang, and Ju Sun. "Deep Random Projector: Accelerated Deep Image Prior." In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition, pp. 18176-18185. 2023.
49 |
50 | ```
51 | @inproceedings{li2023deep,
52 | title={Deep Random Projector: Accelerated Deep Image Prior},
53 | author={Li, Taihui and Wang, Hengkang and Zhuang, Zhong and Sun, Ju},
54 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
55 | pages={18176--18185},
56 | year={2023}
57 | }
58 | ```
59 |
60 |
61 | ## Acknowledgements
62 | Our code is based on the [deep image prior (DIP)](https://github.com/DmitryUlyanov/deep-image-prior) and [deep decoder (DD)](https://github.com/reinhardh/supplement_deep_decoder).
63 |
64 |
65 | ## Contact
66 | - Taihui Li, lixx5027@umn.edu, [https://taihui.github.io/](https://taihui.github.io/)
67 | - Hengkang Wang, wang9881@umn.edu, [https://www.linkedin.com/in/hengkang-henry-wang-a1b293104/](https://www.linkedin.com/in/hengkang-henry-wang-a1b293104/)
68 | - Zhong Zhuang, zhuan143@umn.edu, [https://scholar.google.com/citations?user=rGGxUQEAAAAJ](https://scholar.google.com/citations?user=rGGxUQEAAAAJ)
69 | - Ju Sun, jusun@umn.edu, [https://sunju.org/](https://sunju.org/)
70 |
--------------------------------------------------------------------------------
/environment.yml:
--------------------------------------------------------------------------------
1 | name: pytorch_py3.6
2 | channels:
3 | - pytorch
4 | - defaults
5 | dependencies:
6 | - _libgcc_mutex=0.1=main
7 | - blas=1.0=mkl
8 | - bzip2=1.0.8=h7b6447c_0
9 | - ca-certificates=2021.1.19=h06a4308_1
10 | - certifi=2020.12.5=py36h06a4308_0
11 | - cudatoolkit=10.2.89=hfd86e86_1
12 | - dataclasses=0.8=pyh4f3eec9_6
13 | - ffmpeg=4.3=hf484d3e_0
14 | - freetype=2.10.4=h5ab3b9f_0
15 | - gmp=6.2.1=h2531618_2
16 | - gnutls=3.6.5=h71b1129_1002
17 | - intel-openmp=2020.2=254
18 | - jpeg=9b=h024ee3a_2
19 | - lame=3.100=h7b6447c_0
20 | - lcms2=2.11=h396b838_0
21 | - ld_impl_linux-64=2.33.1=h53a641e_7
22 | - libffi=3.3=he6710b0_2
23 | - libgcc-ng=9.1.0=hdf63c60_0
24 | - libiconv=1.15=h63c8f33_5
25 | - libpng=1.6.37=hbc83047_0
26 | - libstdcxx-ng=9.1.0=hdf63c60_0
27 | - libtiff=4.2.0=h3942068_0
28 | - libuv=1.40.0=h7b6447c_0
29 | - libwebp-base=1.2.0=h27cfd23_0
30 | - lz4-c=1.9.3=h2531618_0
31 | - mkl=2020.2=256
32 | - mkl-service=2.3.0=py36he8ac12f_0
33 | - mkl_fft=1.3.0=py36h54f3939_0
34 | - mkl_random=1.1.1=py36h0573a6f_0
35 | - ncurses=6.2=he6710b0_1
36 | - nettle=3.4.1=hbb512f6_0
37 | - ninja=1.10.2=py36hff7bd54_0
38 | - numpy=1.19.2=py36h54aff64_0
39 | - numpy-base=1.19.2=py36hfa32c7d_0
40 | - olefile=0.46=py36_0
41 | - openh264=2.1.0=hd408876_0
42 | - openssl=1.1.1j=h27cfd23_0
43 | - pillow=8.1.2=py36he98fc37_0
44 | - pip=21.0.1=py36h06a4308_0
45 | - python=3.6.13=hdb3f193_0
46 | - pytorch=1.8.0=py3.6_cuda10.2_cudnn7.6.5_0
47 | - readline=8.1=h27cfd23_0
48 | - setuptools=52.0.0=py36h06a4308_0
49 | - six=1.15.0=py36h06a4308_0
50 | - sqlite=3.35.1=hdfb4753_0
51 | - tk=8.6.10=hbc83047_0
52 | - torchaudio=0.8.0=py36
53 | - torchvision=0.9.0=py36_cu102
54 | - typing_extensions=3.7.4.3=pyha847dfd_0
55 | - wheel=0.36.2=pyhd3eb1b0_0
56 | - xz=5.2.5=h7b6447c_0
57 | - zlib=1.2.11=h7b6447c_3
58 | - zstd=1.4.5=h9ceee32_0
59 | - pip:
60 | - backcall==0.2.0
61 | - charset-normalizer==2.0.4
62 | - cycler==0.10.0
63 | - decorator==4.4.2
64 | - idna==3.2
65 | - imageio==2.9.0
66 | - ipython==7.16.3
67 | - ipython-genutils==0.2.0
68 | - jedi==0.17.2
69 | - joblib==1.0.1
70 | - kiwisolver==1.3.1
71 | - matplotlib==3.3.4
72 | - networkx==2.5
73 | - opencv-python==4.5.1.48
74 | - pandas==1.1.5
75 | - parso==0.7.1
76 | - pexpect==4.8.0
77 | - pickleshare==0.7.5
78 | - prompt-toolkit==3.0.29
79 | - ptyprocess==0.7.0
80 | - pygments==2.12.0
81 | - pyparsing==2.4.7
82 | - python-dateutil==2.8.1
83 | - pytz==2021.1
84 | - pywavelets==1.1.1
85 | - requests==2.26.0
86 | - scikit-image==0.17.2
87 | - scikit-learn==0.24.1
88 | - scipy==1.5.4
89 | - seaborn==0.11.1
90 | - tdqm==0.0.1
91 | - threadpoolctl==2.1.0
92 | - tifffile==2020.9.3
93 | - tqdm==4.59.0
94 | - traitlets==4.3.3
95 | - urllib3==1.26.6
96 | - wand==0.6.6
97 | - wcwidth==0.2.5
98 | prefix: /home/taihui/anaconda3/envs/pytorch_py3.6
99 |
--------------------------------------------------------------------------------