├── .gitignore ├── requirements.txt ├── example_images └── turtles │ ├── turtle_1.png │ ├── turtle_2.jpg │ ├── turtle_3.jpg │ ├── turtle_4.jpg │ └── turtle_5.jpg ├── lib ├── augment.py ├── attack.py ├── __init__.py ├── criterions.py ├── models.py └── bias_field_attack.py ├── README.md ├── LICENSE └── example.py /.gitignore: -------------------------------------------------------------------------------- 1 | /env 2 | 3 | .DS_Store 4 | __pycache__/ 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | torchvision 3 | numpy 4 | scipy 5 | pytorchcv 6 | pillow 7 | -------------------------------------------------------------------------------- /example_images/turtles/turtle_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tsingqguo/jadena/HEAD/example_images/turtles/turtle_1.png -------------------------------------------------------------------------------- /example_images/turtles/turtle_2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tsingqguo/jadena/HEAD/example_images/turtles/turtle_2.jpg -------------------------------------------------------------------------------- /example_images/turtles/turtle_3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tsingqguo/jadena/HEAD/example_images/turtles/turtle_3.jpg -------------------------------------------------------------------------------- /example_images/turtles/turtle_4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tsingqguo/jadena/HEAD/example_images/turtles/turtle_4.jpg -------------------------------------------------------------------------------- /example_images/turtles/turtle_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tsingqguo/jadena/HEAD/example_images/turtles/turtle_5.jpg -------------------------------------------------------------------------------- /lib/augment.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def augment(image_tensor: torch.Tensor) -> list[torch.Tensor]: 4 | return [ 5 | image_tensor, 6 | image_tensor.flip(-1), 7 | image_tensor.flip(-2), 8 | image_tensor.rot90(dims=(-2, -1)), 9 | image_tensor.rot90(dims=(-1, -2)), 10 | ] 11 | -------------------------------------------------------------------------------- /lib/attack.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | class Attack(metaclass=ABCMeta): 8 | def __init__(self, model: nn.Module): 9 | self.model = model 10 | 11 | @abstractmethod 12 | def __call__(self, tensor: torch.Tensor, *args, **kwargs): 13 | pass 14 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | import imp 2 | from .attack import Attack 3 | from .bias_field_attack import BiasFieldAttack 4 | from .criterions import Criterion, l1_criterion, l2_criterion, bce_criterion, fcon_criterion 5 | from .models import ClassifierModel 6 | from .augment import augment 7 | 8 | __all__ = [ 9 | 'Attack', 'IterativeAttack', 'BiasFieldAttack', 10 | 'Criterion', 'l1_criterion', 'l2_criterion', 'bce_criterion', 'fcon_criterion', 11 | 'ClassifierModel', 'augment' 12 | ] 13 | -------------------------------------------------------------------------------- /lib/criterions.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, TypeVar 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | 6 | 7 | Criterion = TypeVar( 8 | 'Criterion', 9 | Callable[[torch.Tensor], torch.Tensor], 10 | Callable[[torch.Tensor, torch.Tensor], torch.Tensor] 11 | ) 12 | 13 | 14 | l1_criterion = lambda r, gt: F.l1_loss(r, 1 - gt) 15 | l2_criterion = lambda r, gt: F.mse_loss(r, 1 - gt) 16 | bce_criterion = lambda r, gt: F.binary_cross_entropy(r, 1 - gt) 17 | fcon_criterion = lambda r: sum(f.transpose(0, 1).flatten(1).std(-1).mean() for f in r).div_(len(r)) 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jadena 2 | 3 | Official implementation of "Can You Spot the Chameleon? Adversarially Camouflaging Images from Co-Salient Object Detection" in CVPR 2022. [arXiv](https://arxiv.org/pdf/2009.09258.pdf) 4 | 5 | ## Usage 6 | 7 | 1. Clone this repo. 8 | 2. Create and activate a virtual environment with Python >= 3.9 (or you need to do some workarounds when having a lower Python version). 9 | 3. Install the dependencies via the command "pip install -r requirements.txt" or any other way you prefer. You should pay attention to the installation of `torch` due to the difference between cpu and gpu. 10 | 4. Run the script `example.py`. It will perturb the image `example_images/turtles/turtle_1.png` and save the result perturbed images in the same folder. 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Qing Guo and Ruijun Gao 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /lib/models.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | from pytorchcv.model_provider import get_model as ptcv_get_model 7 | 8 | 9 | class ClassifierModel(nn.Module): 10 | def __init__(self, classifier_name: str, stages: Sequence[int]): 11 | super(ClassifierModel, self).__init__() 12 | if isinstance(stages, int): 13 | stages = [stages] 14 | self.classifier_name = classifier_name 15 | self.stages = stages 16 | self.max_stage = max(stages) 17 | self.features = ptcv_get_model(classifier_name, pretrained=True).features 18 | 19 | # See https://github.com/osmr/imgclsmob/blob/a5c5bf8d2f3777d16d0898e2cd6572e32de17f2c/pytorch/datasets/imagenet1k_cls_dataset.py#L66 20 | self.register_buffer('image_mean', torch.tensor((0.485, 0.456, 0.406))) 21 | self.register_buffer('image_std', torch.tensor((0.229, 0.224, 0.225))) 22 | 23 | def forward(self, x): 24 | # centerization 25 | x = x.clone() 26 | x = x.sub_(self.image_mean[:, None, None]).div_(self.image_std[:, None, None]) 27 | 28 | feat = [] 29 | last = x 30 | for stage, layer in enumerate(self.features): 31 | last = layer(last) 32 | if stage in self.stages: 33 | feat.append(last) 34 | if stage >= self.max_stage: 35 | break 36 | 37 | return feat -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from pathlib import Path 4 | 5 | import torch 6 | import torchvision.transforms.functional as TF 7 | 8 | from lib import BiasFieldAttack, ClassifierModel, fcon_criterion, augment 9 | from PIL import Image 10 | 11 | 12 | if __name__ == '__main__': 13 | # prepare the attack 14 | model = ClassifierModel('resneta50b', (1, 2, 3)) 15 | attack = BiasFieldAttack( 16 | model, 17 | fcon_criterion, 18 | step=20, 19 | noise_mode='add', 20 | bias_mode='same', 21 | spatial_mode='optical_flow', 22 | noise_lr=1. / 255., 23 | bias_lr=1e-1, 24 | spatial_lr=1e-2, 25 | lambda_b=1e-2, 26 | lambda_s=1e-2, 27 | momentum_decay=1.0, 28 | epsilon_n=16. / 255., 29 | degree=10, 30 | ) 31 | 32 | # prepare images 33 | image_folder = Path(__file__).parent / 'example_images' / 'turtles' 34 | images = ['turtle_1.png', 'turtle_2.jpg', 'turtle_3.jpg', 'turtle_4.jpg', 'turtle_5.jpg'] 35 | images = [Image.open(image_folder / image) for image in images] 36 | images = [TF.to_tensor(image) for image in images] 37 | 38 | # for "group" variant 39 | pert, _ = attack(torch.stack(images)) 40 | TF.to_pil_image(pert[0]).save(image_folder / 'result_group.png') 41 | 42 | # for "augment" variant 43 | pert, _ = attack(torch.stack(augment(images[0]))) 44 | TF.to_pil_image(pert[0]).save(image_folder / 'result_augment.png') 45 | -------------------------------------------------------------------------------- /lib/bias_field_attack.py: -------------------------------------------------------------------------------- 1 | from math import inf 2 | 3 | import torch 4 | import torch.autograd as ag 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | import numpy as np 9 | import scipy.stats as st 10 | 11 | from .attack import Attack 12 | from .criterions import Criterion 13 | 14 | def _gaussian_kernel(kernel_size, nsig=3): 15 | """Returns a 2D Gaussian kernel array.""" 16 | x = np.linspace(-nsig, nsig, kernel_size) 17 | kern1d = st.norm.pdf(x) 18 | kernel_raw = np.outer(kern1d, kern1d) 19 | kernel = kernel_raw / kernel_raw.sum() 20 | return kernel 21 | 22 | class BiasFieldAttack(Attack): 23 | def __init__( 24 | self, 25 | model: nn.Module, 26 | criterion: Criterion, 27 | step: int, 28 | attack_mode: str = 'first', 29 | noise_mode: str = 'none', 30 | bias_mode: str = 'none', 31 | spatial_mode: str = 'none', 32 | noise_lr: float = 0., 33 | bias_lr: float = 0., 34 | spatial_lr: float = 0., 35 | lambda_l: float = 1., 36 | lambda_n: float = 0., 37 | lambda_b: float = 0., 38 | lambda_s: float = 0., 39 | momentum_decay: float = 0., 40 | epsilon: float = inf, 41 | epsilon_n: float = inf, 42 | noise_ti: bool = False, 43 | degree: int = 10, 44 | tune_scale: int = 8, 45 | ti_size: int = 21, 46 | ): 47 | super(BiasFieldAttack, self).__init__(model) 48 | assert step >= 0, f'step should be non-negative integer, got {step}' 49 | assert attack_mode in ('first', 'all') 50 | assert noise_mode in ('none', 'add') 51 | assert bias_mode in ('none', 'rgb', 'same') 52 | assert spatial_mode in ('none', 'spatial_weight', 'super_pixel', 'optical_flow') 53 | assert noise_lr >= 0, f'noise_lr should be non-negative floats, got {noise_lr}' 54 | assert bias_lr >= 0, f'bias_lr should be non-negative floats, got {bias_lr}' 55 | assert spatial_lr >= 0, f'spatial_lr should be non-negative floats, got {spatial_lr}' 56 | assert lambda_l >= 0, f'lambda_l should be non-negative float, got {lambda_l}' 57 | assert lambda_n >= 0, f'lambda_b should be non-negative float, got {lambda_n}' 58 | assert lambda_b >= 0, f'lambda_b should be non-negative float, got {lambda_b}' 59 | assert lambda_s >= 0, f'lambda_s should be non-negative float, got {lambda_s}' 60 | assert momentum_decay >= 0, f'momentum_decay should be non-negative float, got {momentum_decay}' 61 | assert epsilon >= 0, f'epsilon should be non-negative float, got {epsilon}' 62 | assert epsilon_n >= 0, f'epsilon_n should be non-negative float, got {epsilon_n}' 63 | assert degree > 0, f'degree should be positive integer, got {degree}' 64 | assert tune_scale > 0, f'tune_scale should be positive integer, got {tune_scale}' 65 | assert ti_size > 0, f'ti_size should be positive integer, got {ti_size}' 66 | if noise_ti: 67 | assert noise_mode == 'add' 68 | self.criterion = criterion 69 | self.step = step 70 | self.attack_mode = attack_mode 71 | self.noise_mode = noise_mode 72 | self.bias_mode = bias_mode 73 | self.spatial_mode = spatial_mode 74 | self.noise_lr = noise_lr 75 | self.bias_lr = bias_lr 76 | self.spatial_lr = spatial_lr 77 | self.lambda_l = lambda_l 78 | self.lambda_n = lambda_n 79 | self.lambda_b = lambda_b 80 | self.lambda_s = lambda_s 81 | self.momentum_decay = momentum_decay 82 | self.epsilon = epsilon 83 | self.epsilon_n = epsilon_n 84 | self.noise_ti = noise_ti 85 | self.degree = degree 86 | self.tune_scale = tune_scale 87 | self.ti_size = ti_size 88 | 89 | def __call__(self, tensor: torch.Tensor, target: torch.Tensor = None, vis=False): 90 | # identity bias field 91 | n, c, h, w = tensor.size() 92 | assert c == 3, f'tensor should be batched RGB images, got {c} channels' 93 | 94 | params = [] 95 | lrs = [] 96 | 97 | if self.noise_mode == 'add': 98 | noise = torch.zeros_like(tensor).requires_grad_() 99 | params.append(noise) 100 | lrs.append(self.noise_lr) 101 | elif self.noise_mode == 'none': 102 | noise = None 103 | 104 | num_coef = (self.degree + 1) * (self.degree + 2) // 2 105 | if self.bias_mode in ('rgb', 'same'): 106 | coef = torch.zeros(n, 1 if self.bias_mode == 'same' else 3, num_coef).to(tensor).requires_grad_() 107 | params.append(coef) 108 | lrs.append(self.bias_lr) 109 | elif self.bias_mode == 'none': 110 | coef = None 111 | 112 | if self.spatial_mode == 'spatial_weight': 113 | coef_spatial = torch.zeros(n, num_coef).to(tensor).requires_grad_() 114 | params.append(coef_spatial) 115 | lrs.append(self.spatial_lr) 116 | elif self.spatial_mode == 'optical_flow': 117 | optical_flow = torch.zeros(n, h // self.tune_scale, w // self.tune_scale, 2).to(tensor).requires_grad_() 118 | params.append(optical_flow) 119 | lrs.append(self.spatial_lr) 120 | 121 | if self.noise_ti: 122 | ti_kernel = _gaussian_kernel(self.ti_size, 3) 123 | ti_kernel = torch.as_tensor(ti_kernel).to(tensor).expand(c, 1, -1, -1) 124 | ti_padding = (self.ti_size - 1) // 2 125 | 126 | # create coord base 127 | coord_x = torch.linspace(-1, 1, w).to(tensor)[None, :] 128 | coord_y = torch.linspace(-1, 1, h).to(tensor)[:, None] 129 | coord = torch.stack((coord_x.expand(h, -1), coord_y.expand(-1, w)), dim=-1) 130 | base = torch.zeros(num_coef, h, w).to(tensor) 131 | i = 0 132 | for t in range(self.degree + 1): 133 | for l in range(self.degree - t + 1): 134 | base[i, :, :].add_(coord_x ** t).mul_(coord_y ** l) 135 | if vis and (t <= 1 and l <= 1): 136 | base[i, :, :] = 0 137 | i += 1 138 | del i 139 | 140 | if vis: 141 | step_perts_wo_noise = [] 142 | step_perts = [] 143 | step_exposures_wo_tuning = [] 144 | step_exposures = [] 145 | step_noises = [] 146 | 147 | for n_iter in range(self.step + 1): 148 | pert = tensor.clone() 149 | # apply bias field 150 | if self.bias_mode in ('rgb', 'same'): 151 | bias_field = (base[None, None, :, :, :] * coef[:, :, :, None, None]).sum(dim=2) 152 | 153 | if vis and self.bias_mode != 'none': 154 | step_exposures_wo_tuning.append(bias_field.detach().clone()) 155 | 156 | if self.spatial_mode == 'spatial_weight': 157 | spatial_tuning = (base[None, None, :, :, :] * coef_spatial[:, None, :, None, None]).sum(dim=2).sigmoid_() 158 | bias_field = bias_field.mul_(spatial_tuning) 159 | elif self.spatial_mode == 'optical_flow': 160 | upsampled_optical_flow = F.interpolate( 161 | optical_flow.permute(0, 3, 1, 2), 162 | align_corners=False, 163 | mode='bilinear', 164 | size=(h, w), 165 | ).permute(0, 2, 3, 1) 166 | spatial_tuning = (coord + upsampled_optical_flow).clamp_(-1, 1) 167 | bias_field = F.grid_sample(bias_field, spatial_tuning, align_corners=True) 168 | elif self.spatial_mode == 'none': 169 | spatial_tuning = None 170 | 171 | pert = pert.log_().add_(bias_field).exp_() 172 | 173 | if vis: 174 | if self.bias_mode != 'none': 175 | step_exposures.append(bias_field.detach().clone()) 176 | if self.noise_mode != 'none': 177 | step_noises.append(noise.detach().clone()) 178 | step_perts_wo_noise.append(pert.detach().clone()) 179 | 180 | if self.noise_mode == 'add': 181 | pert = pert + noise 182 | 183 | # optimized clamp 184 | if self.epsilon != inf: 185 | pert = torch.min(pert, tensor + self.epsilon) 186 | pert = torch.max(pert, tensor - self.epsilon) 187 | pert = pert.clamp(0, 1) 188 | 189 | if vis: 190 | step_perts.append(pert.detach().clone()) 191 | 192 | pred = self.model(pert) 193 | 194 | # calculate loss and sparsity constraint terms 195 | loss = torch.zeros(1).to(tensor) 196 | 197 | if self.lambda_l > 0: 198 | crit = self.criterion(pred) if target is None else self.criterion(pred, target) 199 | loss.add_(crit, alpha=self.lambda_l) 200 | 201 | if self.lambda_n > 0 and self.noise_mode != 'none': 202 | sparsity_n = noise.pow(2).sum() 203 | loss.add_(sparsity_n, alpha=self.lambda_n) 204 | 205 | if self.lambda_b > 0 and self.bias_mode != 'none': 206 | sparsity_b = bias_field.pow(2).sum().div_(h * w) 207 | loss.add_(sparsity_b, alpha=self.lambda_b) 208 | 209 | if self.lambda_s > 0 and self.spatial_mode == 'optical_flow': 210 | diff_s_h = (optical_flow[:, :, 1:, :] - optical_flow[:, :, :-1, :]) 211 | diff_s_v = (optical_flow[:, 1:, :, :] - optical_flow[:, :-1, :, :]) 212 | sparsity_s = diff_s_h.pow(2).sum() + diff_s_v.pow(2).sum() 213 | loss.add_(sparsity_s, alpha=self.lambda_s / (self.tune_scale * self.tune_scale)) 214 | 215 | # grad and update 216 | if n_iter < self.step: 217 | with torch.no_grad(): 218 | grads = ag.grad(loss, params) 219 | 220 | # apply ti on noise grad 221 | if self.noise_ti: 222 | grads = list(grads) 223 | grads[0] = F.conv2d(grads[0], ti_kernel, padding=ti_padding, groups=c) 224 | 225 | if self.momentum_decay > 0: 226 | grad_norms = [grad.flatten(1).norm(p=1, dim=1) for grad in grads] 227 | grad_norms = [norm.where(norm > 0, torch.ones(1).to(norm)) for norm in grad_norms] 228 | grads = [ 229 | grad.div_(norm.view((-1, *((1,) * (grad.dim() - 1))))) 230 | for grad, norm in zip(grads, grad_norms) 231 | ] 232 | if n_iter == 0: 233 | cum_grads = grads 234 | else: 235 | cum_grads = [ 236 | cum_grad.mul_(self.momentum_decay).add_(grad) 237 | for cum_grad, grad in zip(cum_grads, grads) 238 | ] 239 | else: 240 | cum_grads = grads 241 | 242 | for param, cum_grad, lr in zip(params, cum_grads, lrs): 243 | if self.attack_mode == 'first': 244 | param[0].sub_(cum_grad[0].sign(), alpha=lr) 245 | elif self.attack_mode == 'all': 246 | param.sub_(cum_grad.sign(), alpha=lr) 247 | if self.noise_mode != 'none' and self.epsilon_n != inf: 248 | noise = noise.clamp_(-self.epsilon_n, self.epsilon_n) 249 | 250 | extra = {'pred': pred} 251 | if self.noise_mode != 'none': 252 | extra['noise'] = noise 253 | if self.bias_mode != 'none': 254 | extra['bias_field'] = bias_field 255 | if self.spatial_mode != 'none': 256 | extra['spatial_tuning'] = spatial_tuning 257 | if vis: 258 | extra['step_perts_wo_noise'] = step_perts_wo_noise 259 | extra['step_perts'] = step_perts 260 | extra['step_exposures_wo_tuning'] = step_exposures_wo_tuning 261 | extra['step_exposures'] = step_exposures 262 | extra['step_noises'] = step_noises 263 | return pert, extra 264 | --------------------------------------------------------------------------------