├── Dataset ├── ILSVRC2012_val_00000328.png ├── ILSVRC2012_val_00002437.png └── ILSVRC2012_val_00030569.png ├── EdgeFoolExamples ├── ILSVRC2012_val_00000328.png ├── ILSVRC2012_val_00000328_EdgeFool.png ├── ILSVRC2012_val_00002437.png ├── ILSVRC2012_val_00002437_EdgeFool.png ├── ILSVRC2012_val_00030569.png ├── ILSVRC2012_val_00030569_EdgeFool.png ├── Places365_val_00000702.png └── Places365_val_00000702_EdgeFool.png ├── README.md ├── Smoothing ├── L0_helpers.py ├── L0_serial.py └── script.sh ├── Train ├── box_filter.py ├── dataset.py ├── guided_filter.py ├── misc_function.py ├── module.py ├── rgb_lab_formulation_pytorch.py ├── script.sh ├── train_base.py ├── train_hr.py ├── utils.py └── vis_utils.py └── requirements.txt /Dataset/ILSVRC2012_val_00000328.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/Dataset/ILSVRC2012_val_00000328.png -------------------------------------------------------------------------------- /Dataset/ILSVRC2012_val_00002437.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/Dataset/ILSVRC2012_val_00002437.png -------------------------------------------------------------------------------- /Dataset/ILSVRC2012_val_00030569.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/Dataset/ILSVRC2012_val_00030569.png -------------------------------------------------------------------------------- /EdgeFoolExamples/ILSVRC2012_val_00000328.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/ILSVRC2012_val_00000328.png -------------------------------------------------------------------------------- /EdgeFoolExamples/ILSVRC2012_val_00000328_EdgeFool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/ILSVRC2012_val_00000328_EdgeFool.png -------------------------------------------------------------------------------- /EdgeFoolExamples/ILSVRC2012_val_00002437.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/ILSVRC2012_val_00002437.png -------------------------------------------------------------------------------- /EdgeFoolExamples/ILSVRC2012_val_00002437_EdgeFool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/ILSVRC2012_val_00002437_EdgeFool.png -------------------------------------------------------------------------------- /EdgeFoolExamples/ILSVRC2012_val_00030569.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/ILSVRC2012_val_00030569.png -------------------------------------------------------------------------------- /EdgeFoolExamples/ILSVRC2012_val_00030569_EdgeFool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/ILSVRC2012_val_00030569_EdgeFool.png -------------------------------------------------------------------------------- /EdgeFoolExamples/Places365_val_00000702.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/Places365_val_00000702.png -------------------------------------------------------------------------------- /EdgeFoolExamples/Places365_val_00000702_EdgeFool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcameras/EdgeFool/6c7fc86a1e825d487d64af09e55438aa67b4b720/EdgeFoolExamples/Places365_val_00000702_EdgeFool.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EdgeFool 2 | 3 | This is the official repository of [EDGEFOOL: AN ADVERSARIAL IMAGE ENHANCEMENT FILTER](https://arxiv.org/pdf/1910.12227.pdf), a work published in the Proc. of the 45th IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP), Barcelona, Spain, May 4-8, 2020.
4 | 5 | 6 | Example of results 7 | 8 | | Original Image | Adversarial Image | Original Image | Adversarial Image | 9 | |---|---|---|---| 10 | | ![Original Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/ILSVRC2012_val_00000328.png) | ![Adversarial Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/ILSVRC2012_val_00000328_EdgeFool.png) |![Original Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/ILSVRC2012_val_00030569.png) | ![Adversarial Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/ILSVRC2012_val_00030569_EdgeFool.png) | 11 | | ![Original Image](https://github.com/smartcameras/EdgeFool/blob/master/Dataset/ILSVRC2012_val_00002437.png) | ![Adversarial Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/ILSVRC2012_val_00002437_EdgeFool.png) |![Original Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/Places365_val_00000702.png) | ![Adversarial Image](https://github.com/smartcameras/EdgeFool/blob/master/EdgeFoolExamples/Places365_val_00000702_EdgeFool.png) | 12 | 13 | 14 | ## Setup 15 | 1. Create [conda](https://docs.conda.io/en/latest/miniconda.html) virtual-environment 16 | ``` 17 | module load python2/anaconda 18 | conda create --name EdgeFool python=2.7.15 19 | ``` 20 | 2. Activate conda environment 21 | ``` 22 | source activate EdgeFool 23 | ``` 24 | 3. Download source code from GitHub 25 | ``` 26 | git clone https://github.com/smartcameras/EdgeFool.git 27 | ``` 28 | 4. Install requirements 29 | ``` 30 | pip install -r requirements.txt 31 | ``` 32 | 33 | 34 | ## Description 35 | The code first locates all the images in Dataset folder and then generates the enhanced adversarial images in two steps: 36 | 1. Image smoothing with l0 smoothing filters 37 | 2. Generate the enhanced adversarial images after training of a Fully Convolutional Neural Network 38 | 39 | 40 | ### Image Smoothing 41 | 42 | Image smoothing is performed with the Python implementation of [Image Smoothing via L0 Gradient Minimization](http://www.cse.cuhk.edu.hk/~leojia/papers/L0smooth_Siggraph_Asia2011.pdf) provided by [Kevin Zhang](https://github.com/kjzhang/kzhang-cs205-l0-smoothing), as follows: 43 | 44 | 1. Go to Smoothing directory 45 | ``` 46 | cd Smoothing 47 | ``` 48 | 2. Smooth the original images 49 | ``` 50 | bash script.sh 51 | ``` 52 | 3. The l0 smoothed images will be saved in the SmoothImgs directory (within the 'root' directory) with the same name as their corresponding original images 53 | 54 | ### Generate the enhanced adversarial images 55 | 56 | A Fully Convolutional Neural Network (FCNN) is first trained end-to-end with a multi-task loss function which includes smoothing and adversarial losses. The architecture of the FCNN is instantiated from [Fast Image Processing with Fully-Convolutional Networks](https://arxiv.org/pdf/1709.00643.pdf) implemented in PyTorch by [Wu Huikai](https://github.com/wuhuikai/DeepGuidedFilter/tree/master/ImageProcessing/DeepGuidedFilteringNetwork). We enhance the image details of the L image channel only, after conversion to the Lab colour space without changing the colours of the image. In order to do this, we provided a differentiable PyTorch implementation of RGB-to-Lab and Lab-to-RGB. The enhanced adversarial images are then generated 57 | 58 | 59 | 1. Go to Train directory 60 | ``` 61 | cd Train 62 | ``` 63 | 2. In the script.sh set the paths of 64 | (i) directory of the original images, 65 | (ii) directory of the smoothed images, and 66 | (iii) classifier under attack. The current implementation supports three classifiers Resnet18, Resnet50 and Alexnet, however other classifiers can be employed by changing the lines (80-88) in train_base.py. 67 | 3. Generate the enhanced adversarial images 68 | ``` 69 | bash script.sh 70 | ``` 71 | 4. The enhanced adversarial images are saved in the EnhancedAdvImgsfor_{classifier} (within the 'root' directory) with the same name as their corresponding original images 72 | 73 | 74 | ## Authors 75 | * [Ali Shahin Shamsabadi](mailto:a.shahinshamsabadi@qmul.ac.uk) 76 | * [Changjae Oh](mailto:c.oh@qmul.ac.uk) 77 | * [Andrea Cavallaro](mailto:a.cavallaro@qmul.ac.uk) 78 | 79 | 80 | ## References 81 | If you use our code, please cite the following paper: 82 | 83 | @InProceedings{shamsabadi2020edgefool, 84 | title = {EdgeFool: An Adversarial Image Enhancement Filter}, 85 | author = {Shamsabadi, Ali Shahin and Oh, Changjae and Cavallaro, Andrea}, 86 | booktitle = {Proceedings of the 45th IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, 87 | year = {2020}, 88 | address = {Barcelona, Spain}, 89 | month = May 90 | } 91 | ## License 92 | The content of this project itself is licensed under the [Creative Commons Non-Commercial (CC BY-NC)](https://creativecommons.org/licenses/by-nc/2.0/uk/legalcode). 93 | -------------------------------------------------------------------------------- /Smoothing/L0_helpers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | 4 | # Convert point-spread function to optical transfer function 5 | def psf2otf(psf, outSize=None): 6 | # Prepare psf for conversion 7 | data = prepare_psf(psf, outSize) 8 | 9 | # Compute the OTF 10 | otf = np.fft.fftn(data) 11 | 12 | return np.complex64(otf) 13 | 14 | def prepare_psf(psf, outSize=None, dtype=None): 15 | if not dtype: 16 | dtype=np.float32 17 | 18 | psf = np.float32(psf) 19 | 20 | # Determine PSF / OTF shapes 21 | psfSize = np.int32(psf.shape) 22 | if not outSize: 23 | outSize = psfSize 24 | outSize = np.int32(outSize) 25 | 26 | # Pad the PSF to outSize 27 | new_psf = np.zeros(outSize, dtype=dtype) 28 | new_psf[:psfSize[0],:psfSize[1]] = psf[:,:] 29 | psf = new_psf 30 | 31 | # Circularly shift the OTF so that PSF center is at (0,0) 32 | shift = -(psfSize / 2) 33 | psf = circshift(psf, shift) 34 | 35 | return psf 36 | 37 | # Circularly shift array 38 | def circshift(A, shift): 39 | for i in xrange(shift.size): 40 | A = np.roll(A, shift[i], axis=i) 41 | return A 42 | -------------------------------------------------------------------------------- /Smoothing/L0_serial.py: -------------------------------------------------------------------------------- 1 | # Import Libraries 2 | import numpy as np 3 | import cv2 4 | import argparse 5 | import time 6 | import fnmatch 7 | import os 8 | from tqdm import tqdm 9 | 10 | 11 | # Import User Libraries 12 | import L0_helpers 13 | 14 | 15 | # L0 minimization parameters 16 | kappa = 2.0; 17 | _lambda = 2e-2; 18 | 19 | # Verbose output 20 | verbose = False; 21 | 22 | def find_recursive(root_dir, ext='.jpg'): 23 | files = [] 24 | for root, dirnames, filenames in os.walk(root_dir): 25 | for filename in fnmatch.filter(filenames, '*' + ext): 26 | files.append(os.path.join(root, filename)) 27 | return files 28 | 29 | if __name__ == '__main__': 30 | # Parse arguments 31 | parser = argparse.ArgumentParser( 32 | description="Serial implementation of image smoothing via L0 gradient minimization") 33 | parser.add_argument('--path_imgs', required=True, nargs='+', type=str, 34 | help='a list of image paths, or a directory name') 35 | parser.add_argument('-k', type=float, default=2.0, 36 | metavar='kappa', help='updating weight (default 2.0)') 37 | parser.add_argument('-l', type=float, default=2e-2, 38 | metavar='lambda', help='smoothing weight (default 2e-2)') 39 | parser.add_argument('-v', '--verbose', action='store_true', 40 | help='enable verbose logging for each iteration') 41 | args = parser.parse_args() 42 | 43 | # Set parameters 44 | kappa = args.k 45 | _lambda = args.l 46 | 47 | 48 | verbose = args.verbose 49 | dataset_path = args.path_imgs[0] 50 | 51 | 52 | image_list = find_recursive(dataset_path, ext='.png') 53 | NumImg=len(image_list) 54 | 55 | smooth_path = '../SmoothImgs/' 56 | if not os.path.exists(smooth_path): 57 | os.makedirs(smooth_path) 58 | 59 | for idx in tqdm(range(NumImg)): 60 | 61 | img_name = image_list[idx].split('/')[-1] 62 | # Read image I 63 | image = cv2.imread(dataset_path+img_name) 64 | 65 | # Timers 66 | step_1 = 0.0 67 | step_2 = 0.0 68 | step_2_fft = 0.0 69 | 70 | # Start time 71 | start_time = time.time() 72 | 73 | # Validate image format 74 | N, M, D = np.int32(image.shape) 75 | assert D == 3, "Error: input must be 3-channel RGB image" 76 | print "Processing %d x %d RGB image" % (M, N) 77 | 78 | # Initialize S as I 79 | S = np.float32(image) / 256 80 | 81 | # Compute image OTF 82 | size_2D = [N, M] 83 | fx = np.int32([[1, -1]]) 84 | fy = np.int32([[1], [-1]]) 85 | otfFx = L0_helpers.psf2otf(fx, size_2D) 86 | otfFy = L0_helpers.psf2otf(fy, size_2D) 87 | 88 | # Compute F(I) 89 | FI = np.complex64(np.zeros((N, M, D))) 90 | FI[:,:,0] = np.fft.fft2(S[:,:,0]) 91 | FI[:,:,1] = np.fft.fft2(S[:,:,1]) 92 | FI[:,:,2] = np.fft.fft2(S[:,:,2]) 93 | 94 | # Compute MTF 95 | MTF = np.power(np.abs(otfFx), 2) + np.power(np.abs(otfFy), 2) 96 | MTF = np.tile(MTF[:, :, np.newaxis], (1, 1, D)) 97 | 98 | # Initialize buffers 99 | h = np.float32(np.zeros((N, M, D))) 100 | v = np.float32(np.zeros((N, M, D))) 101 | dxhp = np.float32(np.zeros((N, M, D))) 102 | dyvp = np.float32(np.zeros((N, M, D))) 103 | FS = np.complex64(np.zeros((N, M, D))) 104 | 105 | # Iteration settings 106 | beta_max = 1e5; 107 | beta = 2 * _lambda 108 | iteration = 0 109 | 110 | # Done initializing 111 | init_time = time.time() 112 | 113 | # Iterate until desired convergence in similarity 114 | while beta < beta_max: 115 | 116 | if verbose: 117 | print "ITERATION %i" % iteration 118 | 119 | ### Step 1: estimate (h, v) subproblem 120 | 121 | # subproblem 1 start time 122 | s_time = time.time() 123 | 124 | # compute dxSp 125 | h[:,0:M-1,:] = np.diff(S, 1, 1) 126 | h[:,M-1:M,:] = S[:,0:1,:] - S[:,M-1:M,:] 127 | 128 | # compute dySp 129 | v[0:N-1,:,:] = np.diff(S, 1, 0) 130 | v[N-1:N,:,:] = S[0:1,:,:] - S[N-1:N,:,:] 131 | 132 | # compute minimum energy E = dxSp^2 + dySp^2 <= _lambda/beta 133 | t = np.sum(np.power(h, 2) + np.power(v, 2), axis=2) < _lambda / beta 134 | t = np.tile(t[:, :, np.newaxis], (1, 1, 3)) 135 | 136 | # compute piecewise solution for hp, vp 137 | h[t] = 0 138 | v[t] = 0 139 | 140 | # subproblem 1 end time 141 | e_time = time.time() 142 | step_1 = step_1 + e_time - s_time 143 | if verbose: 144 | print "-subproblem 1: estimate (h,v)" 145 | print "--time: %f (s)" % (e_time - s_time) 146 | 147 | ### Step 2: estimate S subproblem 148 | 149 | # subproblem 2 start time 150 | s_time = time.time() 151 | 152 | # compute dxhp + dyvp 153 | dxhp[:,0:1,:] = h[:,M-1:M,:] - h[:,0:1,:] 154 | dxhp[:,1:M,:] = -(np.diff(h, 1, 1)) 155 | dyvp[0:1,:,:] = v[N-1:N,:,:] - v[0:1,:,:] 156 | dyvp[1:N,:,:] = -(np.diff(v, 1, 0)) 157 | normin = dxhp + dyvp 158 | 159 | fft_s = time.time() 160 | FS[:,:,0] = np.fft.fft2(normin[:,:,0]) 161 | FS[:,:,1] = np.fft.fft2(normin[:,:,1]) 162 | FS[:,:,2] = np.fft.fft2(normin[:,:,2]) 163 | fft_e = time.time() 164 | step_2_fft += fft_e - fft_s 165 | 166 | # solve for S + 1 in Fourier domain 167 | denorm = 1 + beta * MTF; 168 | FS[:,:,:] = (FI + beta * FS) / denorm 169 | 170 | # inverse FFT to compute S + 1 171 | fft_s = time.time() 172 | S[:,:,0] = np.float32((np.fft.ifft2(FS[:,:,0])).real) 173 | S[:,:,1] = np.float32((np.fft.ifft2(FS[:,:,1])).real) 174 | S[:,:,2] = np.float32((np.fft.ifft2(FS[:,:,2])).real) 175 | fft_e = time.time() 176 | step_2_fft += fft_e - fft_s 177 | 178 | # subproblem 2 end time 179 | e_time = time.time() 180 | step_2 = step_2 + e_time - s_time 181 | if verbose: 182 | print "-subproblem 2: estimate S + 1" 183 | print "--time: %f (s)" % (e_time - s_time) 184 | print "" 185 | 186 | # update beta for next iteration 187 | beta *= kappa 188 | iteration += 1 189 | 190 | # Rescale image 191 | S = S * 256 192 | 193 | # Total end time 194 | final_time = time.time() 195 | 196 | print "Total Time: %f (s)" % (final_time - start_time) 197 | print "Setup: %f (s)" % (init_time - start_time) 198 | print "Step 1: %f (s)" % (step_1) 199 | print "Step 2: %f (s)" % (step_2) 200 | print "Step 2 (FFT): %f (s)" % (step_2_fft) 201 | print "Iterations: %d" % (iteration) 202 | 203 | cv2.imwrite(smooth_path+'{}.png'.format(img_name.split('.')[0]), S) 204 | -------------------------------------------------------------------------------- /Smoothing/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | #PATH_IMGS=(/jmain01/home/JAD007/txk02/axs14-txk02/ICASSP19/dataset/2Ali/original/Original/Dataset_resized/test/) 5 | PATH_IMGS=(../Dataset/) 6 | clear 7 | python -W ignore L0_serial.py --path_imgs=$PATH_IMGS 8 | 9 | -------------------------------------------------------------------------------- /Train/box_filter.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | def diff_x(input, r): 5 | assert input.dim() == 4 6 | 7 | left = input[:, :, r:2 * r + 1] 8 | middle = input[:, :, 2 * r + 1: ] - input[:, :, :-2 * r - 1] 9 | right = input[:, :, -1: ] - input[:, :, -2 * r - 1: -r - 1] 10 | 11 | output = torch.cat([left, middle, right], dim=2) 12 | 13 | return output 14 | 15 | def diff_y(input, r): 16 | assert input.dim() == 4 17 | 18 | left = input[:, :, :, r:2 * r + 1] 19 | middle = input[:, :, :, 2 * r + 1: ] - input[:, :, :, :-2 * r - 1] 20 | right = input[:, :, :, -1: ] - input[:, :, :, -2 * r - 1: -r - 1] 21 | 22 | output = torch.cat([left, middle, right], dim=3) 23 | 24 | return output 25 | 26 | class BoxFilter(nn.Module): 27 | def __init__(self, r): 28 | super(BoxFilter, self).__init__() 29 | 30 | self.r = r 31 | 32 | def forward(self, x): 33 | assert x.dim() == 4 34 | 35 | return diff_y(diff_x(x.cumsum(dim=2), self.r).cumsum(dim=3), self.r) -------------------------------------------------------------------------------- /Train/dataset.py: -------------------------------------------------------------------------------- 1 | import os, csv 2 | import random 3 | 4 | from PIL import Image 5 | 6 | import torch.utils.data as data 7 | from torchvision import transforms 8 | 9 | 10 | def default_loader(path): 11 | return Image.open(path).convert('RGB') 12 | 13 | class Transforms(object): 14 | def __init__(self, transformer): 15 | self.transformer = transformer 16 | 17 | def __call__(self, imgs): 18 | return [self.transformer(img) for img in imgs] 19 | 20 | class RandomTransforms(object): 21 | def __init__(self, transformer): 22 | self.transformer = transformer 23 | 24 | def __call__(self, imgs): 25 | if random.random() < 0.5: 26 | return imgs 27 | 28 | return [self.transformer(img) for img in imgs] 29 | 30 | class RandomCrop(object): 31 | def __init__(self, size): 32 | self.size = size 33 | 34 | def __call__(self, imgs): 35 | w, h = imgs[0].size 36 | if w == self.size and h == self.size: 37 | return imgs 38 | 39 | x1 = random.randint(0, w - self.size) 40 | y1 = random.randint(0, h - self.size) 41 | 42 | return [img.crop(x1, y1, x1+self.size, y1+self.size) for img in imgs] 43 | 44 | class RandomRotate(object): 45 | def __call__(self, imgs): 46 | angle = random.randrange(4) 47 | if angle == 0: 48 | return imgs 49 | 50 | return [im.rotate(90*angle) for im in imgs] 51 | 52 | class Compose(object): 53 | def __init__(self, transforms): 54 | self.transforms = [t for t in transforms if t is not None] 55 | 56 | def __call__(self, imgs): 57 | for t in self.transforms: 58 | imgs = t(imgs) 59 | return imgs 60 | 61 | class SuDataset(data.Dataset): 62 | def __init__(self, root, list_path, low_size=64, fine_size=-1, loader=default_loader): 63 | super(SuDataset, self).__init__() 64 | 65 | gt_root = '/jmain01/home/JAD007/txk02/axs14-txk02/image_enhance/kzhang-cs205-l0-smoothing/ImageNet_val_smooth/' 66 | image_list = [] 67 | 68 | with open(list_path) as class_file: 69 | csv_reader = csv.reader(class_file, delimiter=',') 70 | for line in csv_reader: 71 | name = line[0] 72 | inp_path = root+name[:-4]+'.png' 73 | gt_path = gt_root+name[:-4]+'.png' 74 | image_list.append([inp_path,gt_path]) 75 | 76 | imgs = image_list 77 | self.imgs = imgs 78 | self.loader = loader 79 | 80 | ''' 81 | def append(imgs): 82 | imgs.append(transforms.Scale(low_size, interpolation=Image.NEAREST)(imgs[0])) 83 | imgs.append(transforms.Scale(low_size, interpolation=Image.NEAREST)(imgs[1])) 84 | return imgs 85 | ''' 86 | self.transform = Compose([ 87 | RandomCrop(fine_size) if fine_size > 0 else None, 88 | #transforms.Lambda(append), 89 | Transforms(transforms.ToTensor()) 90 | ]) 91 | 92 | def get_path(self, idx): 93 | return self.imgs[idx][0] 94 | 95 | def __getitem__(self, index): 96 | # input_img, gt_img 97 | imgs = [self.loader(path) for path in self.imgs[index]] 98 | 99 | # input_img, gt_img, low_res_input_img, low_res_gt_img 100 | if self.transform is not None: 101 | imgs = self.transform(imgs) 102 | 103 | return imgs 104 | 105 | def __len__(self): 106 | return len(self.imgs) 107 | 108 | class PreSuDataset(data.Dataset): 109 | def __init__(self, img_list, low_size=64, loader=default_loader): 110 | super(PreSuDataset, self).__init__() 111 | 112 | self.imgs = list(img_list) 113 | self.loader = loader 114 | 115 | def append(imgs): 116 | imgs.append(transforms.Scale(low_size, interpolation=Image.NEAREST)(imgs[0])) 117 | return imgs 118 | 119 | self.transform = Compose([ 120 | transforms.Lambda(append), 121 | Transforms(transforms.ToTensor()) 122 | ]) 123 | 124 | def get_path(self, idx): 125 | return self.imgs[idx] 126 | 127 | def __getitem__(self, index): 128 | # input_img 129 | imgs = [self.loader(self.imgs[index])] 130 | 131 | # input_img, low_res_input_img 132 | if self.transform is not None: 133 | imgs = self.transform(imgs) 134 | 135 | return imgs 136 | 137 | def __len__(self): 138 | return len(self.imgs) -------------------------------------------------------------------------------- /Train/guided_filter.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | from torch.nn import functional as F 3 | from torch.autograd import Variable 4 | 5 | from box_filter import BoxFilter 6 | 7 | class FastGuidedFilter(nn.Module): 8 | def __init__(self, r, eps=1e-8): 9 | super(FastGuidedFilter, self).__init__() 10 | 11 | self.r = r 12 | self.eps = eps 13 | self.boxfilter = BoxFilter(r) 14 | 15 | 16 | def forward(self, lr_x, lr_y, hr_x): 17 | n_lrx, c_lrx, h_lrx, w_lrx = lr_x.size() 18 | n_lry, c_lry, h_lry, w_lry = lr_y.size() 19 | n_hrx, c_hrx, h_hrx, w_hrx = hr_x.size() 20 | 21 | assert n_lrx == n_lry and n_lry == n_hrx 22 | assert c_lrx == c_hrx and (c_lrx == 1 or c_lrx == c_lry) 23 | assert h_lrx == h_lry and w_lrx == w_lry 24 | assert h_lrx > 2*self.r+1 and w_lrx > 2*self.r+1 25 | 26 | ## N 27 | N = self.boxfilter(Variable(lr_x.data.new().resize_((1, 1, h_lrx, w_lrx)).fill_(1.0))) 28 | 29 | ## mean_x 30 | mean_x = self.boxfilter(lr_x) / N 31 | ## mean_y 32 | mean_y = self.boxfilter(lr_y) / N 33 | ## cov_xy 34 | cov_xy = self.boxfilter(lr_x * lr_y) / N - mean_x * mean_y 35 | ## var_x 36 | var_x = self.boxfilter(lr_x * lr_x) / N - mean_x * mean_x 37 | 38 | ## A 39 | A = cov_xy / (var_x + self.eps) 40 | ## b 41 | b = mean_y - A * mean_x 42 | 43 | ## mean_A; mean_b 44 | mean_A = F.upsample(A, (h_hrx, w_hrx), mode='bilinear') 45 | mean_b = F.upsample(b, (h_hrx, w_hrx), mode='bilinear') 46 | 47 | return mean_A*hr_x+mean_b 48 | 49 | 50 | class GuidedFilter(nn.Module): 51 | def __init__(self, r, eps=1e-8): 52 | super(GuidedFilter, self).__init__() 53 | 54 | self.r = r 55 | self.eps = eps 56 | self.boxfilter = BoxFilter(r) 57 | 58 | 59 | def forward(self, x, y): 60 | n_x, c_x, h_x, w_x = x.size() 61 | n_y, c_y, h_y, w_y = y.size() 62 | 63 | assert n_x == n_y 64 | assert c_x == 1 or c_x == c_y 65 | assert h_x == h_y and w_x == w_y 66 | assert h_x > 2 * self.r + 1 and w_x > 2 * self.r + 1 67 | 68 | # N 69 | N = self.boxfilter(Variable(x.data.new().resize_((1, 1, h_x, w_x)).fill_(1.0))) 70 | 71 | # mean_x 72 | mean_x = self.boxfilter(x) / N 73 | # mean_y 74 | mean_y = self.boxfilter(y) / N 75 | # cov_xy 76 | cov_xy = self.boxfilter(x * y) / N - mean_x * mean_y 77 | # var_x 78 | var_x = self.boxfilter(x * x) / N - mean_x * mean_x 79 | 80 | # A 81 | A = cov_xy / (var_x + self.eps) 82 | # b 83 | b = mean_y - A * mean_x 84 | 85 | # mean_A; mean_b 86 | mean_A = self.boxfilter(A) / N 87 | mean_b = self.boxfilter(b) / N 88 | 89 | return mean_A * x + mean_b -------------------------------------------------------------------------------- /Train/misc_function.py: -------------------------------------------------------------------------------- 1 | from rgb_lab_formulation_pytorch import * 2 | import cv2 3 | import torch 4 | import numpy as np 5 | from torch.autograd import Variable 6 | from torch.nn import functional as F 7 | import copy 8 | 9 | 10 | def processImage(dataset_path,img_name): 11 | 12 | x = cv2.imread(dataset_path+img_name, 1)/255.0 13 | # Have RGB images 14 | x = x[:, :, (2, 1, 0)] 15 | x = x.transpose(2, 0, 1) # Convert array to C,W,H 16 | x = torch.from_numpy(x).float() 17 | # Add one more channel to the beginning. Tensor shape = 1,3,224,224 18 | x.unsqueeze_(0) 19 | # Convert to Pytorch variable 20 | x = Variable(x.cuda()) 21 | return x 22 | 23 | def detail_enhance_lab(img, smooth_img): 24 | #mean = [0.485, 0.456, 0.406] 25 | #std = [0.229, 0.224, 0.225] 26 | 27 | val0 = 15 28 | val2 = 1 29 | exposure = 1.0 30 | saturation = 1.0 31 | gamma = 1.0 32 | 33 | #for c in range(3): 34 | # img[:,:,c] = img[:,:,c] * std[c] 35 | # img[:,:,c] = img[:,:,c] + mean[c] 36 | #img[img > 1] = 1 37 | #img[img < 0] = 0 38 | 39 | 40 | # convert 1,C,W,H --> W,H,C 41 | img = img.squeeze().permute(1,2,0)#(2,1,0) 42 | smooth_img = smooth_img.squeeze().permute(1,2,0) 43 | 44 | # Convert image and smooth_img from rgb to lab 45 | img_lab=rgb_to_lab(img) 46 | smooth_img_lab=rgb_to_lab(smooth_img) 47 | # do the enhancement 48 | img_l, img_a, img_b =torch.unbind(img_lab,dim=2) 49 | smooth_l, smooth_a, smooth_b =torch.unbind(smooth_img_lab,dim=2) 50 | diff = my_sig((img_l-smooth_l)/100.0,val0)*100.0 51 | base = (my_sig((exposure*smooth_l-56.0)/100.0,val2)*100.0)+56.0 52 | res = base + diff 53 | img_l = res 54 | img_a = img_a * saturation 55 | img_b = img_b * saturation 56 | img_lab = torch.stack([img_l, img_a, img_b], dim=2) 57 | 58 | L_chan, a_chan, b_chan = preprocess_lab(img_lab) 59 | img_lab = deprocess_lab(L_chan, a_chan, b_chan) 60 | #img = color.lab2rgb(img_lab) 61 | img_final = lab_to_rgb(img_lab) 62 | 63 | #img_final = (img_final - mean) / std 64 | 65 | return img_final 66 | def my_sig(x,a): 67 | 68 | # Applies a sigmoid function on the data x in [0-1] range. Then rescales 69 | # the result so 0.5 will be mapped to itself. 70 | 71 | # Apply Sigmoid 72 | y = 1./(1+torch.exp(-a*x)) - 0.5 73 | 74 | # Re-scale 75 | y05 = 1./(1+torch.exp(-torch.tensor(a*0.5,dtype=torch.float32))) - 0.5 76 | y = y*(0.5/y05) 77 | 78 | return y 79 | 80 | 81 | def recreate_image(im_as_var): 82 | 83 | if im_as_var.shape[0] == 1: 84 | recreated_im = copy.copy(im_as_var.cpu().data.numpy()[0]).transpose(1,2,0) 85 | else: 86 | recreated_im = copy.copy(im_as_var.cpu().data.numpy()) 87 | recreated_im[recreated_im > 1] = 1 88 | recreated_im[recreated_im < 0] = 0 89 | recreated_im = np.round(recreated_im * 255) 90 | # Convert RBG to GBR 91 | recreated_im = recreated_im[..., ::-1] 92 | return recreated_im 93 | 94 | 95 | 96 | def PreidictLabel(x, classifier): 97 | 98 | mean = torch.zeros(x.shape).float().cuda() 99 | mean[:,0,:,:]=0.485 100 | mean[:,1,:,:]=0.456 101 | mean[:,2,:,:]=0.406 102 | 103 | std = torch.zeros(x.shape).float().cuda() 104 | std[:,0,:,:]=0.229 105 | std[:,1,:,:]=0.224 106 | std[:,2,:,:]=0.225 107 | 108 | 109 | # Standarise 110 | x = (x - mean) / std 111 | 112 | logit_x = classifier.forward(x) 113 | h_x = F.softmax(logit_x).data.squeeze() 114 | probs_x, idx_x = h_x.sort(0, True) 115 | class_x = idx_x[0] 116 | class_x_prob = probs_x[0] 117 | ############### 118 | target_class = idx_x[1] 119 | return class_x, class_x_prob, probs_x, logit_x,target_class 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | def AdvLoss(logits, target, is_targeted, num_classes=1000, kappa=0): 131 | # inputs to the softmax function are called logits. 132 | # https://arxiv.org/pdf/1608.04644.pdf 133 | target_one_hot = torch.eye(num_classes).type(logits.type())[target.long()] 134 | 135 | # workaround here. 136 | # subtract large value from target class to find other max value 137 | # https://github.com/carlini/nn_robust_attacks/blob/master/l2_attack.py 138 | real = torch.sum(target_one_hot*logits, 1) 139 | other = torch.max((1-target_one_hot)*logits - (target_one_hot*10000), 1)[0] 140 | kappa = torch.zeros_like(other).fill_(kappa) 141 | 142 | if is_targeted: 143 | return torch.sum(torch.max(other-real, kappa)) 144 | return torch.sum(torch.max(real-other, kappa)) 145 | -------------------------------------------------------------------------------- /Train/module.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from torch.nn import init 5 | 6 | from guided_filter import FastGuidedFilter 7 | 8 | def weights_init_identity(m): 9 | classname = m.__class__.__name__ 10 | if classname.find('Conv') != -1: 11 | n_out, n_in, h, w = m.weight.data.size() 12 | # Last Layer 13 | if n_out < n_in: 14 | init.xavier_uniform(m.weight.data) 15 | return 16 | 17 | # Except Last Layer 18 | m.weight.data.zero_() 19 | ch, cw = h // 2, w // 2 20 | for i in range(n_in): 21 | m.weight.data[i, i, ch, cw] = 1.0 22 | 23 | elif classname.find('BatchNorm2d') != -1: 24 | init.constant(m.weight.data, 1.0) 25 | init.constant(m.bias.data, 0.0) 26 | 27 | class AdaptiveNorm(nn.Module): 28 | def __init__(self, n): 29 | super(AdaptiveNorm, self).__init__() 30 | 31 | self.w_0 = nn.Parameter(torch.Tensor([1.0])) 32 | self.w_1 = nn.Parameter(torch.Tensor([0.0])) 33 | 34 | self.bn = nn.BatchNorm2d(n, momentum=0.999, eps=0.001) 35 | 36 | def forward(self, x): 37 | return self.w_0 * x + self.w_1 * self.bn(x) 38 | 39 | def build_lr_net(norm=AdaptiveNorm, layer=5): 40 | layers = [ 41 | nn.Conv2d(3, 24, kernel_size=3, stride=1, padding=1, dilation=1, bias=False), 42 | norm(24), 43 | nn.LeakyReLU(0.2, inplace=True), 44 | ] 45 | 46 | for l in range(1, layer): 47 | layers += [nn.Conv2d(24, 24, kernel_size=3, stride=1, padding=2**l, dilation=2**l, bias=False), 48 | norm(24), 49 | nn.LeakyReLU(0.2, inplace=True)] 50 | 51 | layers += [ 52 | nn.Conv2d(24, 24, kernel_size=3, stride=1, padding=1, dilation=1, bias=False), 53 | norm(24), 54 | nn.LeakyReLU(0.2, inplace=True), 55 | 56 | nn.Conv2d(24, 3, kernel_size=1, stride=1, padding=0, dilation=1) 57 | ] 58 | 59 | net = nn.Sequential(*layers) 60 | 61 | net.apply(weights_init_identity) 62 | 63 | return net 64 | 65 | class DeepGuidedFilter(nn.Module): 66 | def __init__(self, radius=1, eps=1e-8): 67 | super(DeepGuidedFilter, self).__init__() 68 | self.lr = build_lr_net() 69 | self.gf = FastGuidedFilter(radius, eps) 70 | 71 | def forward(self, x_lr, x_hr): 72 | return self.gf(x_lr, self.lr(x_lr), x_hr).clamp(0, 1) 73 | 74 | def init_lr(self, path): 75 | self.lr.load_state_dict(torch.load(path), strict=False) 76 | 77 | 78 | -------------------------------------------------------------------------------- /Train/rgb_lab_formulation_pytorch.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import torch 3 | import numpy as np 4 | from scipy import misc 5 | from PIL import Image 6 | 7 | def preprocess_lab(lab): 8 | L_chan, a_chan, b_chan =torch.unbind(lab,dim=2) 9 | # L_chan: black and white with input range [0, 100] 10 | # a_chan/b_chan: color channels with input range ~[-110, 110], not exact 11 | # [0, 100] => [-1, 1], ~[-110, 110] => [-1, 1] 12 | return [L_chan / 50.0 - 1.0, a_chan / 110.0, b_chan / 110.0] 13 | 14 | 15 | def deprocess_lab(L_chan, a_chan, b_chan): 16 | #TODO This is axis=3 instead of axis=2 when deprocessing batch of images 17 | # ( we process individual images but deprocess batches) 18 | #return tf.stack([(L_chan + 1) / 2 * 100, a_chan * 110, b_chan * 110], axis=3) 19 | return torch.stack([(L_chan + 1) / 2.0 * 100.0, a_chan * 110.0, b_chan * 110.0], dim=2) 20 | 21 | 22 | def rgb_to_lab(srgb): 23 | 24 | srgb_pixels = torch.reshape(srgb, [-1, 3]) 25 | 26 | linear_mask = (srgb_pixels <= 0.04045).type(torch.FloatTensor).cuda() 27 | exponential_mask = (srgb_pixels > 0.04045).type(torch.FloatTensor).cuda() 28 | rgb_pixels = (srgb_pixels / 12.92 * linear_mask) + (((srgb_pixels + 0.055) / 1.055) ** 2.4) * exponential_mask 29 | 30 | rgb_to_xyz = torch.tensor([ 31 | # X Y Z 32 | [0.412453, 0.212671, 0.019334], # R 33 | [0.357580, 0.715160, 0.119193], # G 34 | [0.180423, 0.072169, 0.950227], # B 35 | ]).type(torch.FloatTensor).cuda() 36 | 37 | xyz_pixels = torch.mm(rgb_pixels, rgb_to_xyz) 38 | 39 | 40 | # XYZ to Lab 41 | xyz_normalized_pixels = torch.mul(xyz_pixels, torch.tensor([1/0.950456, 1.0, 1/1.088754]).type(torch.FloatTensor).cuda()) 42 | 43 | epsilon = 6.0/29.0 44 | 45 | linear_mask = (xyz_normalized_pixels <= (epsilon**3)).type(torch.FloatTensor).cuda() 46 | 47 | exponential_mask = (xyz_normalized_pixels > (epsilon**3)).type(torch.FloatTensor).cuda() 48 | 49 | fxfyfz_pixels = (xyz_normalized_pixels / (3 * epsilon**2) + 4.0/29.0) * linear_mask + ((xyz_normalized_pixels+0.000001) ** (1.0/3.0)) * exponential_mask 50 | # convert to lab 51 | fxfyfz_to_lab = torch.tensor([ 52 | # l a b 53 | [ 0.0, 500.0, 0.0], # fx 54 | [116.0, -500.0, 200.0], # fy 55 | [ 0.0, 0.0, -200.0], # fz 56 | ]).type(torch.FloatTensor).cuda() 57 | lab_pixels = torch.mm(fxfyfz_pixels, fxfyfz_to_lab) + torch.tensor([-16.0, 0.0, 0.0]).type(torch.FloatTensor).cuda() 58 | #return tf.reshape(lab_pixels, tf.shape(srgb)) 59 | return torch.reshape(lab_pixels, srgb.shape) 60 | 61 | def lab_to_rgb(lab): 62 | lab_pixels = torch.reshape(lab, [-1, 3]) 63 | # convert to fxfyfz 64 | lab_to_fxfyfz = torch.tensor([ 65 | # fx fy fz 66 | [1/116.0, 1/116.0, 1/116.0], # l 67 | [1/500.0, 0.0, 0.0], # a 68 | [ 0.0, 0.0, -1/200.0], # b 69 | ]).type(torch.FloatTensor).cuda() 70 | fxfyfz_pixels = torch.mm(lab_pixels + torch.tensor([16.0, 0.0, 0.0]).type(torch.FloatTensor).cuda(), lab_to_fxfyfz) 71 | 72 | # convert to xyz 73 | epsilon = 6.0/29.0 74 | linear_mask = (fxfyfz_pixels <= epsilon).type(torch.FloatTensor).cuda() 75 | exponential_mask = (fxfyfz_pixels > epsilon).type(torch.FloatTensor).cuda() 76 | 77 | 78 | xyz_pixels = (3 * epsilon**2 * (fxfyfz_pixels - 4/29.0)) * linear_mask + ((fxfyfz_pixels+0.000001) ** 3) * exponential_mask 79 | 80 | # denormalize for D65 white point 81 | xyz_pixels = torch.mul(xyz_pixels, torch.tensor([0.950456, 1.0, 1.088754]).type(torch.FloatTensor).cuda()) 82 | 83 | 84 | xyz_to_rgb = torch.tensor([ 85 | # r g b 86 | [ 3.2404542, -0.9692660, 0.0556434], # x 87 | [-1.5371385, 1.8760108, -0.2040259], # y 88 | [-0.4985314, 0.0415560, 1.0572252], # z 89 | ]).type(torch.FloatTensor).cuda() 90 | 91 | rgb_pixels = torch.mm(xyz_pixels, xyz_to_rgb) 92 | # avoid a slightly negative number messing up the conversion 93 | #clip 94 | rgb_pixels[rgb_pixels > 1] = 1 95 | rgb_pixels[rgb_pixels < 0] = 0 96 | 97 | linear_mask = (rgb_pixels <= 0.0031308).type(torch.FloatTensor).cuda() 98 | exponential_mask = (rgb_pixels > 0.0031308).type(torch.FloatTensor).cuda() 99 | srgb_pixels = (rgb_pixels * 12.92 * linear_mask) + (((rgb_pixels+0.000001) ** (1/2.4) * 1.055) - 0.055) * exponential_mask 100 | 101 | return torch.reshape(srgb_pixels, lab.shape) 102 | 103 | 104 | # test 105 | ''' 106 | img = cv2.imread('data/test_rgb.jpg',1)/ 255.0 107 | img = img[:, :, (2, 1, 0)] 108 | #img = misc.imread('data/test_rgb.jpg')/255.0 109 | img = torch.from_numpy(img).cuda() 110 | lab = rgb_to_lab(img) 111 | L_chan, a_chan, b_chan = preprocess_lab(lab) 112 | 113 | lab = deprocess_lab(L_chan, a_chan, b_chan) 114 | true_image = lab_to_rgb(lab) 115 | true_image = np.round(true_image.cpu()* 255.0) 116 | true_image = np.uint8(true_image) 117 | #np.save('torch.npy',np.array(img.cpu())) 118 | #conv_img = Image.fromarray(true_image, 'RGB') 119 | #conv_img.save('converted_test_pytorch.jpg') 120 | true_image = true_image[:, :, (2, 1, 0)] 121 | cv2.imwrite('pytorch.jpg',true_image) 122 | #import pdb; pdb.set_trace() 123 | ''' -------------------------------------------------------------------------------- /Train/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | clear 4 | 5 | # Classifier that is under attack 6 | ADVMODEl=(resnet50) 7 | 8 | # Path to the directory that contains images 9 | PATH_IMGS=(../Dataset/) 10 | 11 | # Path to the directory that contains smoothed images 12 | PATH_SMOOTHS=(../SmoothImgs/) 13 | 14 | for adv_model in "${ADVMODEl[@]}" 15 | do 16 | echo Attacking $adv_model via EdgeFool 17 | python -W ignore train_hr.py --adv_model=$adv_model --path_imgs=$PATH_IMGS --path_smooth=$PATH_SMOOTHS 18 | done 19 | -------------------------------------------------------------------------------- /Train/train_base.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import copy 4 | import torch 5 | from os.path import join,isfile 6 | from tqdm import tqdm 7 | from os import listdir 8 | import random 9 | from torch import nn 10 | from torch import optim 11 | from torch.autograd import Variable 12 | from torch.utils.data import DataLoader 13 | from torch import autograd 14 | from utils import Config 15 | from dataset import SuDataset 16 | from vis_utils import VisUtils 17 | 18 | import numpy as np 19 | import cv2 20 | import torchvision.transforms as T 21 | 22 | from torchvision import models 23 | from torch.nn import functional as F 24 | 25 | from misc_function import processImage, detail_enhance_lab, recreate_image, PreidictLabel, AdvLoss 26 | 27 | default_config = Config( 28 | N_START = 0, 29 | N_EPOCH = None, 30 | FINE_SIZE = -1, 31 | #################### CONSTANT ##################### 32 | IMG = None, 33 | SAVE = 'checkpoints', 34 | BATCH = 1l, 35 | GPU = 0, 36 | LR = 0.001, 37 | # clip 38 | clip = None, 39 | # model 40 | model = None, 41 | # forward 42 | forward = None, 43 | # img size 44 | exceed_limit = None, 45 | # vis 46 | vis = None 47 | ) 48 | 49 | 50 | def run(config, dataset_path, dataset_smooth_path, image_list, idx, adv_model): 51 | 52 | # Ceate a directory for saving the trained models 53 | save_path = config.SAVE 54 | path = os.path.join(save_path, 'snapshots') 55 | if not os.path.isdir(path): 56 | os.makedirs(path) 57 | 58 | # Create a directory for saving adversarial images 59 | adv_path = '../EnhancedAdvImgsfor_{}/'.format(adv_model) 60 | if not os.path.isdir(adv_path): 61 | os.makedirs(adv_path) 62 | 63 | 64 | # Smoothing loss function 65 | criterion = nn.MSELoss() 66 | 67 | 68 | # Using GPU 69 | if config.GPU >= 0: 70 | with torch.cuda.device(config.GPU): 71 | config.model.cuda() 72 | criterion.cuda() 73 | 74 | 75 | # Setup optimizer 76 | optimizer = optim.Adam(config.model.parameters(), lr=config.LR) 77 | 78 | 79 | 80 | # Load the classifier for attacking 81 | if adv_model == 'resnet18': 82 | classifier = models.resnet18(pretrained=True) 83 | elif adv_model == 'resnet50': 84 | classifier = models.resnet50(pretrained=True) 85 | elif adv_model == 'alexnet': 86 | classifier = models.alexnet(pretrained=True) 87 | classifier.cuda() 88 | classifier.eval() 89 | 90 | # Freeze the parameters of the classifeir under attack to not be updated 91 | for param in classifier.parameters(): 92 | param.requires_grad = False 93 | 94 | 95 | 96 | 97 | # The name of the chosen image 98 | img_name = image_list[idx].split('/')[-1] 99 | 100 | 101 | 102 | # Pre-processing the original image and ground truth L_0 smooth image 103 | x= processImage(dataset_path,img_name) 104 | gt_smooth = processImage(dataset_smooth_path,img_name) 105 | 106 | 107 | # Prediction of the original image using the classifier chosen for attacking 108 | class_x, prob_class_x, prob_x, logit_x, target_class = PreidictLabel(x, classifier) 109 | 110 | 111 | # Initilize number of misclassification and maximum number of iterations for updating FCNN using total_loss 112 | misclassified = 0 113 | maxIters = 5000 114 | 115 | 116 | for it in range(maxIters): 117 | t = time.time() 118 | 119 | 120 | with autograd.detect_anomaly(): 121 | # Smooth images 122 | x_smooth= config.forward(x,gt_smooth, config) 123 | 124 | # Enhance adversarial image 125 | enh = detail_enhance_lab(x,x_smooth) 126 | 127 | 128 | # Prediction of the adversarial image using the classifier chosen for attacking 129 | class_enh, prob_class_enh, prob_enh, logit_enh, _ = PreidictLabel(enh.permute(2,0,1).unsqueeze(dim=0), classifier) 130 | 131 | 132 | # Computing smoothing and adversarial losses 133 | loss1 = criterion(x_smooth, gt_smooth) 134 | loss2 = AdvLoss(logit_enh, class_x, is_targeted=False) 135 | 136 | 137 | # Combining the smoothing and adversarial losses 138 | loss = 10*loss1 + loss2 139 | 140 | 141 | # backward 142 | optimizer.zero_grad() 143 | loss.backward() 144 | if config.clip is not None: 145 | torch.nn.utils.clip_grad_norm(config.model.parameters(), config.clip) 146 | optimizer.step() 147 | 148 | 149 | 150 | # Save the adversarial image when the classifier is fooled and smoothing loss is less than a threshold 151 | if (class_x != class_enh): 152 | misclassified=1 153 | cv2.imwrite('{}{}'.format(adv_path,img_name), recreate_image(enh)) 154 | if (loss1< 0.0005): 155 | break 156 | 157 | 158 | # Save the FCNN 159 | torch.save(config.model.state_dict(), os.path.join(save_path, 'snapshots', '{}_latest.pth'.format(adv_model))) 160 | -------------------------------------------------------------------------------- /Train/train_hr.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import argparse 3 | 4 | from train_base import * 5 | 6 | from module import DeepGuidedFilter 7 | 8 | parser = argparse.ArgumentParser(description='Train FCNN to Generate Enhanced Adversarial Images ') 9 | parser.add_argument('--adv_model', type=str, help='adversarial model') 10 | parser.add_argument('--path_imgs', required=True, nargs='+', type=str, 11 | help='a list of image paths, or a directory name') 12 | parser.add_argument('--path_smooth', required=True, nargs='+', type=str, 13 | help='a list of smoothed image paths, or a directory name') 14 | args = parser.parse_args() 15 | 16 | 17 | def forward(imgs,gt, config): 18 | x_hr= imgs 19 | gt_hr=gt 20 | return config.model(x_hr, x_hr) 21 | 22 | dataset_path = args.path_imgs[0] 23 | dataset_smooth_path = args.path_smooth[0] 24 | 25 | 26 | # List of the name of all the images in the dataset_path 27 | image_list = [f for f in listdir(dataset_path) if isfile(join(dataset_path,f))] 28 | NumImg=len(image_list) 29 | 30 | 31 | # Configuration 32 | config = copy.deepcopy(default_config) 33 | config.N_EPOCH = 100 34 | # model 35 | config.model = DeepGuidedFilter() 36 | config.forward = forward 37 | config.clip = 0.01 38 | 39 | 40 | # Run the attack for each image 41 | for idx in tqdm(range(NumImg)): 42 | run(config, dataset_path, dataset_smooth_path, image_list, idx, args.adv_model) 43 | 44 | 45 | -------------------------------------------------------------------------------- /Train/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | import warnings 5 | warnings.simplefilter('ignore') 6 | 7 | from multiprocessing import Pool 8 | 9 | from skimage import img_as_ubyte 10 | from skimage.io import imread 11 | from skimage.color import grey2rgb 12 | from skimage.transform import resize 13 | from skimage.measure import compare_mse, compare_psnr, compare_ssim 14 | 15 | class Config(object): 16 | def __init__(self, **params): 17 | for k, v in params.items(): 18 | self.__dict__[k] = v 19 | 20 | def tensor_to_img(tensor, transpose=False): 21 | im = np.asarray(np.clip(np.squeeze(tensor.numpy()) * 255, 0, 255), dtype=np.uint8) 22 | if transpose: 23 | im = im.transpose((1, 2, 0)) 24 | 25 | return im 26 | 27 | def calc_metric_with_np(pre_im, gt_im, multichannel=True): 28 | return compare_mse(pre_im, gt_im),\ 29 | compare_psnr(pre_im, gt_im),\ 30 | compare_ssim(pre_im, gt_im, multichannel=multichannel) 31 | 32 | def calc_metric_per_img(im_name, pre_path, gt_path, opts={}): 33 | pre_im_path = os.path.join(pre_path, im_name) 34 | gt_im_path = os.path.join(gt_path, im_name) 35 | 36 | assert os.path.isfile(pre_im_path) 37 | assert os.path.isfile(gt_im_path) or os.path.islink(gt_im_path) 38 | 39 | pre = img_as_ubyte(imread(pre_im_path)) 40 | gt = img_as_ubyte(imread(gt_im_path)) 41 | if gt.ndim == 2: 42 | gt = grey2rgb(gt) 43 | if pre.shape != gt.shape: 44 | gt = img_as_ubyte(resize(gt, pre.shape[:2], mode='reflect')) 45 | 46 | return calc_metric_with_np(pre, gt, **opts) 47 | 48 | def calc_metric(pre_path, gt_path, n_process=8): 49 | params = [(im_name, pre_path, gt_path) for im_name in os.listdir(pre_path)] 50 | return np.asarray(Pool(n_process).starmap(calc_metric_per_img, params)) -------------------------------------------------------------------------------- /Train/vis_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from tqdm import tqdm 4 | 5 | from tensorboardX import SummaryWriter 6 | 7 | class VisUtils(object): 8 | def __init__(self, name, n_iter, n_epoch, log_dir='tensorboard_logs', stat_dir='tensorboard_stats'): 9 | self.n_iter = n_iter 10 | self.stat_dir = os.path.join(stat_dir, name) 11 | if not os.path.isdir(self.stat_dir): 12 | os.makedirs(self.stat_dir) 13 | 14 | self.e_bar = tqdm(total=n_epoch, desc='#Epoch') 15 | self.i_bar = tqdm(total=n_iter, desc=' #Iter') 16 | 17 | self.writer = SummaryWriter(log_dir=os.path.join(log_dir, name)) 18 | 19 | def reset(self, n_iter, n_epoch): 20 | self.e_bar.close() 21 | self.i_bar.close() 22 | self.e_bar = tqdm(total=n_epoch, desc='#Epoch') 23 | self.i_bar = tqdm(total=n_iter, desc=' #Iter') 24 | 25 | 26 | def update(self, post_fix): 27 | self.i_bar.set_postfix(**post_fix) 28 | self.i_bar.update() 29 | 30 | def next_epoch(self): 31 | self.e_bar.update() 32 | 33 | self.i_bar.close() 34 | self.i_bar = tqdm(total=self.n_iter, desc=' #Iter') 35 | 36 | def close(self): 37 | self.writer.export_scalars_to_json(os.path.join(self.stat_dir, 'scalars.json')) 38 | self.writer.close() -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | torch 3 | torchvision 4 | opencv-python 5 | tqdm 6 | future 7 | scikit-image 8 | tensorboardX 9 | --------------------------------------------------------------------------------