├── data └── input │ ├── flower.jpg │ ├── sanjuan.jpg │ └── notredame.jpg ├── LICENSE ├── README.md ├── utils.py ├── test.py ├── attack_queries.py ├── run_exp_fig5.sh ├── run_exp_tab1.sh ├── run_exp_tab2.sh ├── eval_retrieval.py ├── tma.py └── cirtorchclone └── imageretrievalnet.py /data/input/flower.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gtolias/tma/HEAD/data/input/flower.jpg -------------------------------------------------------------------------------- /data/input/sanjuan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gtolias/tma/HEAD/data/input/sanjuan.jpg -------------------------------------------------------------------------------- /data/input/notredame.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gtolias/tma/HEAD/data/input/notredame.jpg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Visual Recognition Group, Czech Technical University in Prague 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Targeted mismatch adversarial attack (TMA) 2 | 3 | This is a Python package that uses Pytorch to implement our paper: 4 | 5 | ``` 6 | @conference{TRC19, 7 | title = {Targeted Mismatch Adversarial Attack: Query with a Flower to Retrieve the Tower}, 8 | author = {Tolias, Giorgos and Radenovi{\'c}, Filip and Chum, Ondrej} 9 | booktitle = {International Conference on Computer Vision (ICCV)}, 10 | year = {2019} 11 | } 12 | ``` 13 | 14 | It implements targeted mismatch attacks and reproduces the main experiments of the paper. 15 | 16 | ## Prerequisites 17 | 18 | 1. Python3 (tested with Python 3.5.3 on Debian 8.1) 19 | 1. PyTorch deep learning framework (tested with version 1.0.1.post2) 20 | 1. Package [cnnimageretrieval-pytorch](https://github.com/filipradenovic/cnnimageretrieval-pytorch). The code is developed with [release v1.1](https://github.com/filipradenovic/cnnimageretrieval-pytorch/archive/v1.1.tar.gz). The root folder of cnnimageretrieval-pytorch should be added to the python path 21 | 22 | ``` 23 | export PYTHONPATH="${PYTHONPATH}:cnnimageretrieval_pytorch_1.1_rootfolder/" 24 | ``` 25 | 26 | ## Usage 27 | 28 | A simple TMA on a single image is performed by running 29 | 30 | ``` 31 | python test.py 32 | ``` 33 | 34 | All results of Table 1 in the paper are reproduced by running 35 | 36 | ``` 37 | bash run_exp_tab1.sh 38 | ``` 39 | 40 | All results of Table 2 in the paper are reproduced by running 41 | 42 | ``` 43 | bash run_exp_tab2.sh 44 | ``` 45 | 46 | All results of Figure 5 in the paper are reproduced by running 47 | 48 | ``` 49 | bash run_exp_fig5.sh 50 | ``` 51 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import random, pdb 2 | import numpy as np 3 | from PIL import Image 4 | 5 | import torch 6 | from torch import nn 7 | from torch.autograd import Variable 8 | import torchvision.transforms.functional as TF 9 | 10 | from cirtorch.datasets.datahelpers import imresize 11 | 12 | 13 | def center_crop(query_img, carrier_img): 14 | q_i, q_c, q_w, q_h = query_img.size() 15 | c_i, c_c, c_w, c_h = carrier_img.size() 16 | left = (c_w - q_w)//2 17 | top = (c_h - q_h)//2 18 | return carrier_img[0,:,left:left+query_img.size(2),top:top+query_img.size(3)].unsqueeze(0) 19 | 20 | 21 | # load image, optionally crop, and resize 22 | def img_loader(image_name, imsize, bbx = None): 23 | img = Image.open(image_name) 24 | imsize_ = np.max((img.height,img.width)) 25 | if bbx: img = img.crop(bbx) 26 | img = imresize(img, np.floor(1.0*np.max((img.height,img.width))*imsize/imsize_)) 27 | return Variable(TF.to_tensor(img)).unsqueeze(0) 28 | 29 | 30 | def reproduce(seed): 31 | torch.cuda.manual_seed(seed) 32 | torch.cuda.manual_seed_all(seed) 33 | torch.manual_seed(seed) 34 | np.random.seed(seed) 35 | random.seed(seed) 36 | torch.backends.cudnn.enabled = False 37 | torch.backends.cudnn.benchmark = False 38 | torch.backends.cudnn.deterministic = True 39 | 40 | 41 | def eval_sim(networks, scales, target_img, carrier_img, attack_img): 42 | for network in networks: 43 | network.cuda() 44 | m = torch.FloatTensor(network.meta['mean']).cuda().unsqueeze(0).unsqueeze(2).unsqueeze(3) 45 | s = torch.FloatTensor(network.meta['std']).cuda().unsqueeze(0).unsqueeze(2).unsqueeze(3) 46 | for scale in scales: 47 | print(network.meta['architecture']+"-"+network.meta['pooling']+" scale "+str(scale)) 48 | target = network(nn.functional.interpolate((target_img-m)/s, scale_factor=scale, mode='bilinear', align_corners=False)).squeeze() 49 | attack = network(nn.functional.interpolate((attack_img-m)/s, scale_factor=scale, mode='bilinear', align_corners=False)).squeeze() 50 | carrier = network(nn.functional.interpolate((carrier_img-m)/s, scale_factor=scale, mode='bilinear', align_corners=False)).squeeze() 51 | 52 | print("T2A: {:.3f} C2A: {:.3f} C2T: {:.3f}".format(target.dot(attack).item(), carrier.dot(attack).item(), carrier.dot(target).item())) 53 | network.cpu() -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys, time, copy, os, pdb, argparse, ast 3 | from scipy.misc import imsave 4 | import numpy as np 5 | from PIL import Image 6 | import matplotlib.pyplot as plt 7 | 8 | import torch 9 | import torch.nn as nn 10 | from torch.autograd import Variable 11 | import torch.nn.functional as F 12 | import torchvision.transforms as transforms 13 | import torchvision.transforms.functional as TF 14 | import torchvision.models as models 15 | from torch.utils.model_zoo import load_url 16 | 17 | from cirtorch.layers import pooling 18 | from cirtorch.layers import normalization 19 | from cirtorch.datasets.datahelpers import imresize 20 | 21 | from cirtorchclone.imageretrievalnet import init_network 22 | 23 | from tma import tma 24 | from utils import * 25 | 26 | gpu_id = '0' 27 | 28 | train_scales = [300,350,400,450,500,550,600,650,700,750,800,850,900,950,1024] 29 | test_scales = [1024] 30 | iters = 100 31 | lr = 0.01 32 | lam = 0 33 | sigma_blur = 0.3 34 | 35 | carrier_fn = 'data/input/flower.jpg' 36 | # carrier_fn = 'data/input/sanjuan.jpg' 37 | target_fn = 'data/input/notredame.jpg' 38 | 39 | mode = 'hist' 40 | pool = mode 41 | # mode = 'global' 42 | # pool = 'gem' 43 | arch = 'alexnet' 44 | modellist = arch+"-"+pool 45 | 46 | testarch = 'alexnet' 47 | testpool = 'gem' 48 | testmodellist = testarch+"-"+testpool 49 | 50 | os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" 51 | os.environ["CUDA_VISIBLE_DEVICES"]=gpu_id 52 | 53 | output_folder = 'data/' 54 | 55 | # list of training networks / pooling 56 | net_params = {'local_whitening':False,'regional':False,'whitening':False,'pretrained':True} # default cir network params 57 | 58 | test_networks = [] 59 | for s in testmodellist.split("+"): 60 | net_params['architecture'] = s.split("-")[0] 61 | for s2 in testmodellist.split("-")[1:]: 62 | net_params['pooling'] = s2 63 | test_networks.append(init_network(net_params)) 64 | 65 | train_networks = [] 66 | for s in modellist.split("+"): 67 | net_params['architecture'] = s.split("-")[0] 68 | train_networks.append(init_network(net_params)) 69 | if mode == 'global': train_networks[-1].poolattack = s.split("-")[1:] 70 | 71 | for n in train_networks: n.eval(); n.cuda(); 72 | 73 | imsize = 1024 74 | train_scale_factors = [x / imsize for x in train_scales] 75 | test_scale_factors = [x / imsize for x in test_scales] 76 | 77 | use_cuda = torch.cuda.is_available() 78 | dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 79 | for n in train_networks: n.cuda(); n.eval() 80 | for n in test_networks: n.eval() 81 | 82 | def loader(image_name, im_size): 83 | return Variable(TF.to_tensor(imresize(Image.open(image_name), im_size))).unsqueeze(0) 84 | 85 | target_img = img_loader(target_fn, imsize).type(dtype) 86 | carrier_img = img_loader(carrier_fn, imsize).type(dtype) 87 | 88 | carrier_img = center_crop(target_img, carrier_img) 89 | carrier_img_org = carrier_img.clone().clamp_(0, 1) 90 | 91 | print("TARGET "+target_fn+" CARRIER " + carrier_fn) 92 | 93 | t = time.time() 94 | attack_img = tma(train_networks, train_scale_factors, target_img, carrier_img, mode = mode, num_steps = iters, lr = lr, lam = lam, sigma_blur = sigma_blur, verbose = True)[0] 95 | print("Elapsed time {:.4f}\n".format(time.time()-t)) 96 | 97 | # save to disk 98 | img2save = np.transpose(attack_img.cpu().numpy(), (2,3,1,0)).squeeze() 99 | imsave(output_folder+'/attack_image.png',img2save) 100 | img_df = (attack_img-carrier_img_org) 101 | img_df = (img_df-img_df.min()) / (img_df.max()-img_df.min()) 102 | img2save = np.transpose(img_df.cpu().numpy(), (2,3,1,0)).squeeze() 103 | imsave(output_folder+'/attack_carrier_diff.png',img2save) 104 | 105 | print("Evaluate descriptor similarity") 106 | eval_sim(test_networks, test_scale_factors, target_img, carrier_img_org, attack_img) 107 | -------------------------------------------------------------------------------- /attack_queries.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os, sys, argparse, ast, time, pdb 3 | import numpy as np 4 | from PIL import Image 5 | from scipy.misc import imsave 6 | 7 | import torch 8 | 9 | from cirtorch.datasets.testdataset import configdataset 10 | from cirtorch.utils.general import get_data_root 11 | 12 | from cirtorchclone.imageretrievalnet import init_network 13 | from tma import tma 14 | from utils import img_loader, center_crop 15 | 16 | # cmd arguments 17 | parser = argparse.ArgumentParser() 18 | # parser.add_argument('--dataset', type=str, default="roxford5k", help = 'roxford5k | rparis6k | holidays | copydays') # holidays and copydays are not yet supported 19 | parser.add_argument('--dataset', type=str, default="roxford5k", help = 'roxford5k | rparis6k') 20 | parser.add_argument('--carrier', type=str, default="flower", help = ' flower | sanjuan') 21 | parser.add_argument('--mode', type=str, default="global", help = "global | tensor | hist") 22 | parser.add_argument('--modellist', type=str, default="alexnet-mac") 23 | parser.add_argument('--scales', type=str, default="[1024]", help = "[1024], [300,400,500,600,700,800,900,1024], [300,350,400,450,500,550,600,650,700,750,800,850,900,950,1024]") 24 | parser.add_argument('--iters', type=int, default=100) 25 | parser.add_argument('--lr', type=float, default=0.01) 26 | parser.add_argument('--lam', type=float, default=.0) 27 | parser.add_argument('--sigma-blur', type=float, default=0.3, help = "no blur if 0.0") 28 | parser.add_argument('--gpu-id', type=str, default="0") 29 | parser.add_argument('--variant', type=str, default="") 30 | 31 | args = parser.parse_args() 32 | args.scales = ast.literal_eval(args.scales) 33 | 34 | # choose gpu 35 | os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" 36 | os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_id 37 | 38 | # copy arguments 39 | dataset, train_scales, iters, lr, lam, sigma_blur, mode, variant, carrier_fn = args.dataset, args.scales, args.iters, args.lr, args.lam, args.sigma_blur, args.mode, args.variant, args.carrier 40 | 41 | # project folders 42 | output_folder = "data/" 43 | datasets_folder = get_data_root()+"/test/" 44 | 45 | # attack-FCN / attack-pooling list 46 | net_params = {'local_whitening':False,'regional':False,'whitening':False,'pretrained':True} # default cir network params 47 | train_networks = [] 48 | for s in args.modellist.split("+"): 49 | net_params['architecture'] = s.split("-")[0] 50 | train_networks.append(init_network(net_params)) 51 | if mode == 'global': train_networks[-1].poolattack = s.split("-")[1:] 52 | for n in train_networks: n.eval(); n.cuda(); 53 | 54 | # output name 55 | if mode == 'global': exp_name = dataset+"_"+("+".join([n.meta['architecture']+"-"+"-".join(n.poolattack) for n in train_networks])) 56 | else: exp_name = dataset+"_"+("+".join([n.meta['architecture']+"-"+mode for n in train_networks])) 57 | if len(variant): variant= "+"+variant 58 | exp_name+= "+"+str(train_scales).replace(" ","")+"+iter"+str(iters)+"+lr"+str(lr)+"+lam"+str(lam)+"+sigmablur"+str(sigma_blur)+"_"+carrier_fn+variant 59 | 60 | # dataset config 61 | cfg = configdataset(dataset, datasets_folder) 62 | if dataset.startswith('holidays') or dataset.startswith('copydays'): cfg['nq'] = 50 # hard code holidays and copydays queries to first 50 63 | if 'bbx' in cfg['gnd'][0].keys(): bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] # bounding boxes for roxford5k and rparis6k datasets 64 | else: bbxs = None 65 | im_size = {'roxford5k':1024, 'rparis6k':1024, 'holidays':1024, 'copydays':1024} # original image size 66 | scale_factors = [x / im_size[dataset] for x in train_scales] # compute relative re-scaling factors 67 | 68 | # log file 69 | log = open(output_folder+"/log_"+exp_name+".txt", 'a') 70 | 71 | # save folder for attacks 72 | if not os.path.exists(output_folder+"/"+exp_name): 73 | os.makedirs(output_folder+"/"+exp_name) 74 | 75 | # params for rerun if no converge 76 | max_trials, multiply_rate_iters, divide_rate_lr = 10, 2, 5 77 | 78 | total_time = 0 79 | for i in range(cfg['nq']): 80 | query_fn = cfg['qimlist'][i] 81 | print(str(i)+" QUERY "+query_fn); log.write(str(i)+" QUERY "+query_fn+"\n"); log.flush() 82 | 83 | # load target (query) image 84 | if bbxs is not None: 85 | target_img = img_loader(cfg['qim_fname'](cfg,i), im_size[dataset], cfg['gnd'][i]['bbx']).type(torch.cuda.FloatTensor) 86 | else: 87 | target_img = img_loader(cfg['qim_fname'](cfg,i), im_size[dataset]).type(torch.cuda.FloatTensor) 88 | 89 | # attack 90 | t = time.time() 91 | trials = 0 92 | converged = False 93 | while not converged and trials < max_trials: 94 | carrier_img = img_loader("data/input/"+carrier_fn+".jpg", im_size[dataset]).type(torch.cuda.FloatTensor) 95 | carrier_img = center_crop(target_img, carrier_img) 96 | alr = lr / divide_rate_lr**trials # reducing lr after every failure 97 | aiters = int(iters * multiply_rate_iters**trials) # increase iterations after every failure 98 | attack_img, loss_perf, loss_distort, converged = tma(train_networks, scale_factors, target_img, carrier_img, mode = mode, num_steps = aiters, lr = alr, lam = lam, sigma_blur = sigma_blur, verbose = True) 99 | trials += 1 100 | 101 | # time and log 102 | total_time += time.time()-t 103 | log.write("performance loss {:6f} distortion loss {:6f} total loss {:6f}\n".format(loss_perf.item(), (loss_distort).item(), (loss_distort+loss_perf).item())); log.flush() 104 | if trials == max_trials: print("Failed...") 105 | 106 | # save the attack in png file 107 | savefn = output_folder+exp_name+"/"+query_fn+'.png' 108 | if not os.path.exists(savefn[0:savefn.rfind('/')]): os.makedirs(savefn[0:savefn.rfind('/')]) 109 | imsave(savefn,np.transpose(attack_img.cpu().numpy(), (2,3,1,0)).squeeze()) 110 | print("Attack saved in "+savefn+"\n"); sys.stdout.flush() 111 | 112 | log.write("Average time per image {:6f}\n".format(total_time / cfg['nq'])); log.flush() 113 | print("Average time per image {:6f}\n".format(total_time / cfg['nq'])) 114 | 115 | log.close() 116 | -------------------------------------------------------------------------------- /run_exp_fig5.sh: -------------------------------------------------------------------------------- 1 | gpuid=0 2 | dataset='rparis6k' 3 | 4 | dir_cache='data/' 5 | log='log_fig5_'$dataset'.txt' 6 | 7 | # attack params 8 | scales_s0='[1024]' 9 | scales_s1='[300,400,500,600,700,800,900,1024]' 10 | scales_s2='[300,350,400,450,500,550,600,650,700,750,800,850,900,950,1024]' 11 | scales_s3='[262,289,319,351,387,427,470,518,571,630,694,765,843,929,1024]' 12 | carrier='flower' 13 | lam=0.0 14 | 15 | 16 | # attack params 17 | scales=$scales_s0 18 | arch='alexnet' 19 | sigmablur=0.0 20 | iter=100 21 | mode='hist' 22 | modellist=$arch"-"$mode 23 | #attack 24 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 25 | 26 | # test params 27 | testpool='gem' 28 | for testscale in $(seq 250 10 1020) 1024; 29 | do 30 | echo $testscale; 31 | # evaluate retrieval performance 32 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 33 | done 34 | 35 | 36 | # attack params 37 | scales=$scales_s0 38 | arch='alexnet' 39 | sigmablur=0.3 40 | iter=100 41 | mode='hist' 42 | modellist=$arch"-"$mode 43 | #attack 44 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 45 | 46 | # test params 47 | testpool='gem' 48 | for testscale in $(seq 250 10 1020) 1024; 49 | do 50 | echo $testscale; 51 | # evaluate retrieval performance 52 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 53 | done 54 | 55 | 56 | # attack params 57 | scales=$scales_s1 58 | arch='alexnet' 59 | sigmablur=0.0 60 | iter=100 61 | mode='hist' 62 | modellist=$arch"-"$mode 63 | #attack 64 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 65 | 66 | # test params 67 | testpool='gem' 68 | for testscale in $(seq 250 10 1020) 1024; 69 | do 70 | echo $testscale; 71 | # evaluate retrieval performance 72 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 73 | done 74 | 75 | 76 | # attack params 77 | scales=$scales_s2 78 | arch='alexnet' 79 | sigmablur=0.0 80 | iter=100 81 | mode='hist' 82 | modellist=$arch"-"$mode 83 | #attack 84 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 85 | 86 | # test params 87 | testpool='gem' 88 | for testscale in $(seq 250 10 1020) 1024; 89 | do 90 | echo $testscale; 91 | # evaluate retrieval performance 92 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 93 | done 94 | 95 | 96 | # attack params 97 | scales=$scales_s1 98 | arch='alexnet' 99 | sigmablur=0.3 100 | iter=100 101 | mode='hist' 102 | modellist=$arch"-"$mode 103 | #attack 104 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 105 | 106 | # test params 107 | testpool='gem' 108 | for testscale in $(seq 250 10 1020) 1024; 109 | do 110 | echo $testscale; 111 | # evaluate retrieval performance 112 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 113 | done 114 | 115 | 116 | # attack params 117 | scales=$scales_s2 118 | arch='alexnet' 119 | sigmablur=0.3 120 | iter=100 121 | mode='hist' 122 | modellist=$arch"-"$mode 123 | #attack 124 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 125 | 126 | # test params 127 | testpool='gem' 128 | for testscale in $(seq 250 10 1020) 1024; 129 | do 130 | echo $testscale; 131 | # evaluate retrieval performance 132 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 133 | done 134 | 135 | 136 | # attack params 137 | scales=$scales_s3 138 | arch='alexnet' 139 | sigmablur=0.3 140 | iter=100 141 | mode='hist' 142 | modellist=$arch"-"$mode 143 | #attack 144 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 145 | 146 | # test params 147 | testpool='gem' 148 | for testscale in $(seq 250 10 1020) 1024; 149 | do 150 | echo $testscale; 151 | # evaluate retrieval performance 152 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 153 | done 154 | -------------------------------------------------------------------------------- /run_exp_tab1.sh: -------------------------------------------------------------------------------- 1 | gpuid=0 2 | 3 | dir_cache='data2/' 4 | log='data/log_tab1.txt' 5 | 6 | # attack params 7 | dataset='rparis6k' 8 | carrier='flower' 9 | arch='alexnet' 10 | sigmablur=0.0 11 | scales='[1024]' 12 | lam=0.0 13 | 14 | # test params 15 | testscale=1024 16 | 17 | 18 | # attack params 19 | iter=100 20 | mode='global' 21 | pool='gem' 22 | modellist=$arch"-"$pool 23 | # run attack 24 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 25 | # evaluate retrieval performance 26 | testpool='gem' 27 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 28 | testpool='mac' 29 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 30 | testpool='spoc' 31 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 32 | testpool='rmac' 33 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 34 | testpool='crow' 35 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 36 | 37 | 38 | # attack params 39 | iter=100 40 | mode='global' 41 | pool='gem-mac-spoc' 42 | modellist=$arch"-"$pool 43 | # run attack 44 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$arch"-"$pool --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 45 | # evaluate retrieval performance 46 | testpool='gem' 47 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 48 | testpool='mac' 49 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 50 | testpool='spoc' 51 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 52 | testpool='rmac' 53 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 54 | testpool='crow' 55 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 56 | 57 | 58 | # attack params 59 | iter=100 60 | mode='hist' 61 | pool=$mode 62 | modellist=$arch"-"$pool 63 | # run attack 64 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$arch"-"$pool --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 65 | # evaluate retrieval performance 66 | testpool='gem' 67 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 68 | testpool='mac' 69 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 70 | testpool='spoc' 71 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 72 | testpool='rmac' 73 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 74 | testpool='crow' 75 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 76 | 77 | 78 | # attack params 79 | iter=1000 80 | mode='tensor' 81 | pool=$mode 82 | modellist=$arch"-"$pool 83 | # run attack 84 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$arch"-"$pool --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 85 | # evaluate retrieval performance 86 | testpool='gem' 87 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 88 | testpool='mac' 89 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 90 | testpool='spoc' 91 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 92 | testpool='rmac' 93 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 94 | testpool='crow' 95 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 96 | -------------------------------------------------------------------------------- /run_exp_tab2.sh: -------------------------------------------------------------------------------- 1 | gpuid=0 2 | 3 | dataset='roxford5k' 4 | # dataset='rparis6k' 5 | # dataset='copydays' 6 | # dataset='holidays' 7 | 8 | dir_cache='data/' 9 | log='data/log_tab2_'$dataset'.txt' 10 | 11 | # attack params 12 | scales='[300,350,400,450,500,550,600,650,700,750,800,850,900,950,1024]' 13 | carrier='flower' 14 | lam=0.0 15 | 16 | 17 | # attack params 18 | arch='alexnet' 19 | sigmablur=0.3 20 | iter=100 21 | mode='hist' 22 | modellist=$arch"-"$mode 23 | #attack 24 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 25 | # test params 26 | testscale=1024 27 | testpool='gem' 28 | # evaluate retrieval performance 29 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 30 | 31 | 32 | # attack params 33 | arch='resnet18' 34 | sigmablur=0.3 35 | iter=100 36 | mode='global' 37 | pool='gem' 38 | modellist=$arch"-"$pool 39 | #attack 40 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 41 | # test params 42 | testscale=1024 43 | testpool='gem' 44 | # evaluate retrieval performance 45 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 46 | # test params 47 | testscale=768 48 | testpool='gem' 49 | # evaluate retrieval performance 50 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 51 | # test params 52 | testscale=512 53 | testpool='gem' 54 | # evaluate retrieval performance 55 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 56 | 57 | 58 | 59 | # attack params 60 | arch='resnet18' 61 | sigmablur=0.0 62 | iter=100 63 | mode='hist' 64 | modellist=$arch"-"$mode 65 | #attack 66 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 67 | # test params 68 | testscale=1024 69 | testpool='gem' 70 | # evaluate retrieval performance 71 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 72 | # test params 73 | testscale=768 74 | testpool='gem' 75 | # evaluate retrieval performance 76 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 77 | # test params 78 | testscale=512 79 | testpool='gem' 80 | # evaluate retrieval performance 81 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 82 | 83 | 84 | # attack params 85 | arch='resnet18' 86 | sigmablur=0.3 87 | iter=100 88 | mode='hist' 89 | modellist=$arch"-"$mode 90 | #attack 91 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 92 | # test params 93 | testscale=1024 94 | testpool='gem' 95 | # evaluate retrieval performance 96 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 97 | # test params 98 | testscale=768 99 | testpool='gem' 100 | # evaluate retrieval performance 101 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 102 | # test params 103 | testscale=512 104 | testpool='gem' 105 | # evaluate retrieval performance 106 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 107 | 108 | 109 | 110 | 111 | 112 | # attack params 113 | arch='resnet18' 114 | sigmablur=0.3 115 | iter=100 116 | mode='global' 117 | pool='gem-mac-spoc' 118 | modellist=$arch"-"$pool 119 | #attack 120 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 121 | # test params 122 | testscale=1024 123 | testpool='crow' 124 | # evaluate retrieval performance 125 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 126 | 127 | 128 | # attack params 129 | arch='resnet18' 130 | sigmablur=0.3 131 | iter=100 132 | mode='hist' 133 | modellist=$arch"-"$mode 134 | #attack 135 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 136 | # test params 137 | testscale=1024 138 | testpool='crow' 139 | # evaluate retrieval performance 140 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 141 | 142 | 143 | # attack params 144 | arch='resnet18' 145 | sigmablur=0.3 146 | iter=1000 147 | mode='tensor' 148 | modellist=$arch"-"$mode 149 | #attack 150 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 151 | # test params 152 | testscale=1024 153 | testpool='crow' 154 | # evaluate retrieval performance 155 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 156 | 157 | 158 | #attack params 159 | sigmablur=0.3 160 | iter=100 161 | mode='hist' 162 | modellist="alexnet-"$mode"+resnet18-"$mode 163 | #attack 164 | python3 attack_queries.py --dataset=$dataset --carrier=$carrier --mode=$mode --modellist=$modellist --scales=$scales --iters=$iter --lam=$lam --sigma-blur=$sigmablur --gpu-id=$gpuid 165 | # test params 166 | testscale=1024 167 | testpool='gem' 168 | arch='alexnet' 169 | # evaluate retrieval performance 170 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 171 | # test params 172 | testscale=1024 173 | testpool='crow' 174 | arch='resnet18' 175 | # evaluate retrieval performance 176 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 177 | # test params 178 | testscale=1024 179 | testpool='gem' 180 | arch='vgg16' 181 | # evaluate retrieval performance 182 | python3 eval_retrieval.py --gpu-id $gpuid --network-offtheshelf $arch"-"$testpool --dataset $dataset --image-size 1024 --image-resize $testscale --dir-attack "data/"$dataset"_"$modellist"+"$scales"+iter"$iter"+lr0.01+lam"$lam"+sigmablur"$sigmablur"_"$carrier"/" --ext-attack '.png' --dir-cache $dir_cache --log=$log 183 | -------------------------------------------------------------------------------- /eval_retrieval.py: -------------------------------------------------------------------------------- 1 | import argparse, os, sys, pdb 2 | 3 | import numpy as np 4 | import scipy as sp 5 | import scipy.stats 6 | from PIL import Image 7 | 8 | import torch 9 | import torch.nn as nn 10 | from torchvision import transforms 11 | 12 | from cirtorch.datasets.testdataset import configdataset 13 | from cirtorch.datasets.genericdataset import ImagesFromList 14 | from cirtorch.utils.download import download_test 15 | from cirtorch.utils.evaluate import compute_map 16 | from cirtorch.utils.general import get_data_root 17 | 18 | from cirtorchclone.imageretrievalnet import init_network 19 | from utils import img_loader 20 | 21 | 22 | datasets_names = ['roxford5k', 'rparis6k'] 23 | # datasets_names = ['roxford5k', 'rparis6k', 'holidays', 'copydays'] # holidays and copydays are not yet supported 24 | 25 | parser = argparse.ArgumentParser(description='PyTorch CNN Image Retrieval Testing') 26 | 27 | # network 28 | group = parser.add_mutually_exclusive_group(required=True) 29 | group.add_argument('--network-offtheshelf', '-noff', metavar='NETWORK', help="network off-the-shelf, in the format 'ARCHITECTURE-POOLING, eg: resnet18-gem or alexnet-gem or alexnet-mac or alexnet-rmac etc...") 30 | 31 | # test options 32 | parser.add_argument('--dataset', '-d', metavar='DATASETS', default='roxford5k', help="test dataset name: | ".join(datasets_names) + " (default: 'roxford5k')") 33 | parser.add_argument('--image-size', '-imsize', default=1024, type=int, metavar='N', help="maximum size of longer image side used for testing (default: 1024)") 34 | parser.add_argument('--image-resize', '-imresize', default=1024, type=int, metavar='N', help="maximum size of longer image side used for testing (default: 1024)") 35 | 36 | # attack options 37 | parser.add_argument('--dir-attack', metavar='DIR_ATTACK', default=None, help="directory where the attack images are saved") 38 | parser.add_argument('--ext-attack', metavar='EXT_ATTACK', default=None, help="extension in which the attack images are saved") 39 | parser.add_argument('--dir-cache', metavar='DIR_CACHE', default=None, help="directory where extracted vectors are cached") 40 | 41 | # GPU ID 42 | parser.add_argument('--gpu-id', '-g', default='0', metavar='N', help="gpu id used for testing (default: '0')") 43 | 44 | # output text file 45 | parser.add_argument('--log', default=None, help="text file saving results for the paper") 46 | 47 | def main(): 48 | print(">> Retrieval evaluation of attacks\n") 49 | 50 | args = parser.parse_args() 51 | 52 | # check if unknown dataset 53 | if args.dataset not in datasets_names: 54 | raise ValueError('Unsupported or unknown dataset: {}!'.format(args.dataset)) 55 | 56 | # check if test dataset are downloaded and download if they are not 57 | download_test(get_data_root()) 58 | 59 | # setting up the visible GPU 60 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id 61 | 62 | # parse off-the-shelf parameters 63 | offtheshelf = args.network_offtheshelf.split('-') 64 | net_params = {'architecture':offtheshelf[0], 'pooling':offtheshelf[1],'local_whitening':False,'regional':False,'whitening':False,'pretrained':True} 65 | 66 | # load off-the-shelf network 67 | print(">> Loading off-the-shelf network: '{}'".format(args.network_offtheshelf)) 68 | net = init_network(net_params) 69 | # print(">>>> loaded network: \n'{}'".format(net.meta_repr())) 70 | 71 | # moving network to gpu and eval mode 72 | net.cuda() 73 | net.eval() 74 | 75 | # set up the transform 76 | normalize = transforms.Normalize(mean=net.meta['mean'],std=net.meta['std']) 77 | transform = transforms.Compose([transforms.ToTensor(),normalize]) 78 | 79 | # evaluate on test dataset 80 | dataset=args.dataset 81 | print('>> {}: Extracting...'.format(dataset)) 82 | 83 | # prepare config structure for the test dataset 84 | cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) 85 | cfg.update({'qext_a':args.ext_attack}) 86 | cfg.update({'dir_data_a':args.dir_attack}) 87 | cfg.update({'dir_images_a':cfg['dir_data_a']}) 88 | cfg.update({'qim_fname_a':config_qimname_a}) 89 | 90 | # reduce number of queries for holidays and copydays 91 | if dataset.startswith('holidays') or dataset.startswith('copydays'): 92 | cfg['nq'] = 50 93 | cfg['gnd'] = cfg['gnd'][:cfg['nq']] 94 | 95 | images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])] 96 | qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])] 97 | qimages_a = [cfg['qim_fname_a'](cfg,i) for i in range(cfg['nq'])] 98 | 99 | try: 100 | bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] 101 | except: 102 | bbxs = None # for holidays and copydays 103 | 104 | # extract descriptors and cache or load cached ones 105 | print('>> {}: database images...'.format(dataset)) 106 | network_fn = args.network_offtheshelf 107 | if args.dir_cache is not None: 108 | vecs_fn = os.path.join(args.dir_cache, '{}_{}_{}_{}_vecs.pth'.format(dataset, network_fn, args.image_size, args.image_resize)) 109 | if os.path.isfile(vecs_fn): 110 | vecs = torch.load(vecs_fn) 111 | print('>> loaded cached descriptors from {}'.format(vecs_fn)) 112 | else: 113 | vecs = extract_vectors_a(net, images, args.image_size, args.image_resize, transform) 114 | torch.save(vecs, vecs_fn) 115 | 116 | print('>> {}: standard query images...'.format(dataset)) 117 | qvecs = extract_vectors_a(net, qimages, args.image_size, args.image_resize, transform, bbxs=bbxs) 118 | print('>> {}: attack query images...'.format(dataset)) 119 | qvecs_a = extract_vectors_a(net, qimages_a, args.image_size, args.image_resize, transform) 120 | 121 | print('>> {}: evaluating for image resolution {}'.format(dataset, args.image_resize)) 122 | # convert to numpy 123 | vecs = vecs.numpy() 124 | qvecs = qvecs.numpy() 125 | qvecs_a = qvecs_a.numpy() 126 | 127 | qip = (qvecs*qvecs_a).sum(axis=0) 128 | qip_mean, qip_std = qip.mean(), qip.std() 129 | print('>> {}: inner product (target,attack) mean: {:.3f}, std: {:.3f}'.format(dataset, qip_mean, qip_std)) 130 | 131 | # search, rank, and print 132 | scores = np.dot(vecs.T, qvecs) 133 | ranks = np.argsort(-scores, axis=0) 134 | maps, mprs, aps = compute_map_and_print(dataset, ranks, cfg['gnd']) 135 | 136 | # attack search, rank, and print 137 | scores = np.dot(vecs.T, qvecs_a) 138 | ranks = np.argsort(-scores, axis=0) 139 | maps_a, mprs_a, aps_a = compute_map_and_print(dataset + ' attack ({})'.format(args.ext_attack), ranks, cfg['gnd']) 140 | 141 | if dataset.startswith('roxford5k') or dataset.startswith('rparis6k'): 142 | r1, r2, r3 = 100*maps[1], 100*maps_a[1], 100*(maps_a[1]-maps[1]) # medium protocol 143 | else: 144 | r1, r2, r3 = 100*maps[0], 100*maps_a[0], 100*(maps_a[0]-maps[0]) 145 | 146 | print('\n*** Summary ***\n attack: {}\n test: {}-{}-{} \n mean ip (target,attack): {:.3f}\n mAP: org {:.2f} att {:.2f} dif {:.2f}\n'.format(args.dir_attack.split('/')[-2], dataset, args.network_offtheshelf, args.image_resize, qip_mean, r1, r2, r3)) 147 | 148 | if args.log is not None: 149 | with open(args.log, 'a') as f: 150 | f.write('\n attack: {}\n test: {}-{}-{} \n mean ip (target,attack): {:.3f}\n mAP: org {:.2f} att {:.2f} dif {:.2f}\n\n'.format(args.dir_attack.split('/')[-2], dataset, args.network_offtheshelf, args.image_resize, qip_mean, r1, r2, r3)) 151 | 152 | 153 | 154 | def extract_vectors_a(net, images, image_size, image_resize, transform, bbxs=None, print_freq=10): 155 | # moving network to gpu and eval mode 156 | net.cuda() 157 | net.eval() 158 | 159 | # creating dataset loader 160 | loader = torch.utils.data.DataLoader( 161 | ImagesFromList(root='', images=images, imsize=image_size, bbxs=bbxs, transform=transform), 162 | batch_size=1, shuffle=False, num_workers=8, pin_memory=True 163 | ) 164 | 165 | # extracting vectors 166 | with torch.no_grad(): 167 | vecs = torch.zeros(net.meta['outputdim'], len(images)) 168 | for i, input in enumerate(loader): 169 | input = input.cuda() 170 | input_t = nn.functional.interpolate(input, scale_factor=image_resize/image_size, mode='bilinear', align_corners=False) 171 | vecs[:, i] = net(input_t).cpu().data.squeeze() 172 | if (i+1) % print_freq == 0 or (i+1) == len(images): 173 | print('\r>>>> {}/{} done...'.format((i+1), len(images)), end='') 174 | print('') 175 | 176 | return vecs 177 | 178 | 179 | def compute_map_and_print(dataset, ranks, gnd, kappas=[1, 5, 10], doprint = True): 180 | 181 | # old evaluation protocol 182 | if dataset.startswith('oxford5k') or dataset.startswith('paris6k') or dataset.startswith('instre') or dataset.startswith('holidays') or dataset.startswith('copydays'): 183 | map, aps, _, _ = compute_map(ranks, gnd) 184 | if doprint: 185 | print('>> {}: mAP {:.2f}'.format(dataset, np.around(map*100, decimals=2))) 186 | return [map], [-1], aps 187 | 188 | # new evaluation protocol 189 | elif dataset.startswith('roxford5k') or dataset.startswith('rparis6k'): 190 | 191 | gnd_t = [] 192 | for i in range(len(gnd)): 193 | g = {} 194 | g['ok'] = np.concatenate([gnd[i]['easy']]) 195 | g['junk'] = np.concatenate([gnd[i]['junk'], gnd[i]['hard']]) 196 | gnd_t.append(g) 197 | mapE, apsE, mprE, prsE = compute_map(ranks, gnd_t, kappas) 198 | 199 | gnd_t = [] 200 | for i in range(len(gnd)): 201 | g = {} 202 | g['ok'] = np.concatenate([gnd[i]['easy'], gnd[i]['hard']]) 203 | g['junk'] = np.concatenate([gnd[i]['junk']]) 204 | gnd_t.append(g) 205 | mapM, apsM, mprM, prsM = compute_map(ranks, gnd_t, kappas) 206 | 207 | gnd_t = [] 208 | for i in range(len(gnd)): 209 | g = {} 210 | g['ok'] = np.concatenate([gnd[i]['hard']]) 211 | g['junk'] = np.concatenate([gnd[i]['junk'], gnd[i]['easy']]) 212 | gnd_t.append(g) 213 | mapH, apsH, mprH, prsH = compute_map(ranks, gnd_t, kappas) 214 | 215 | if doprint: 216 | print('>> {}: mAP E: {}, M: {}, H: {}'.format(dataset, np.around(mapE*100, decimals=2), np.around(mapM*100, decimals=2), np.around(mapH*100, decimals=2))) 217 | print('>> {}: mP@k{} E: {}, M: {}, H: {}'.format(dataset, kappas, np.around(mprE*100, decimals=2), np.around(mprM*100, decimals=2), np.around(mprH*100, decimals=2))) 218 | return [mapE, mapM, mapH], [mprE, mprM, mprH], apsM 219 | 220 | 221 | # query filenames for attacks 222 | def config_qimname_a(cfg, i): 223 | return os.path.join(cfg['dir_images_a'], cfg['qimlist'][i] + cfg['qext_a']) 224 | 225 | if __name__ == '__main__': 226 | main() -------------------------------------------------------------------------------- /tma.py: -------------------------------------------------------------------------------- 1 | import math, numbers, pdb 2 | import numpy as np 3 | from skimage import filters 4 | from PIL import Image 5 | 6 | import torch 7 | from torch import nn 8 | import torch.optim as optim 9 | from torch.nn import functional as F 10 | 11 | from cirtorch.layers.functional import mac, spoc, gem, rmac 12 | 13 | from utils import reproduce 14 | 15 | POOLING = {'mac' : mac,'spoc' : spoc,'gem' : gem} 16 | 17 | 18 | # targeted mismatch attack 19 | def tma(networks, scales, target_img, carrier_img, mode = 'normal', num_steps=100, lr = 1.0, lam = 0.0, sigma_blur = 0.0, verbose = True, seed = 155): 20 | # if seed is not None: # uncomment to reproduce the results of the ICCV19 paper - still some randomness though 21 | # reproduce(seed) 22 | 23 | carrier_optim = nn.Parameter(carrier_img.data) # parameters to be learned are the carrier's pixels values 24 | carrier_org = carrier_img.clone() # to compute distortion 25 | optimizer = optim.Adam([carrier_optim], lr = lr) 26 | 27 | bin_centers_fixed = torch.DoubleTensor(np.arange(0,1.001,0.05)).cuda() # for histograms only 28 | scales = np.array(scales) 29 | sigma_blur_all = sigma_blur / np.array(scales) 30 | kernel_size_all = 2*np.floor((np.ceil(6*sigma_blur_all)/2))+1 31 | 32 | # pre-compute all target global-descriptors / histograms / tensors 33 | targets, norm_factors = {}, {} 34 | for network in networks: # optimize for all networks 35 | network.eval() 36 | network.cuda() 37 | 38 | m = torch.FloatTensor(network.meta['mean']).cuda().unsqueeze(0).unsqueeze(2).unsqueeze(3) 39 | s = torch.FloatTensor(network.meta['std']).cuda().unsqueeze(0).unsqueeze(2).unsqueeze(3) 40 | for scale in scales: # optimize for all scales 41 | si = (scales==scale).nonzero()[0].item() 42 | if sigma_blur > 0.0: 43 | GS = GaussianSmoothing(channels = 3, kernel_size = kernel_size_all[si], sigma = sigma_blur_all[si]).cuda() 44 | else: 45 | GS = nn.Sequential() # identity function 46 | 47 | # normalize (mean-std), re-scale, and feed to the network 48 | xl = network.features(nn.functional.interpolate((GS(target_img) - m )/ s, scale_factor=scale, mode='bilinear', align_corners=False)) 49 | 50 | if not isinstance(xl, list): xl = [xl] # to support optimization for internal layers too 51 | for l in range(len(xl)): 52 | x = xl[l] 53 | if mode == 'global': 54 | for pool in network.poolattack: # global descriptors 55 | targets[network.meta['architecture'],str(scale), pool, 'layer'+str(l)] = network.norm(POOLING[pool](x)).squeeze().detach() 56 | elif mode == 'hist': # activation histogram 57 | nf = x.max().detach() 58 | norm_factors[network.meta['architecture'],str(scale), 'layer'+str(l)] = nf 59 | targets[network.meta['architecture'],str(scale), 'layer'+str(l)] = hist_per_channel((x / nf).clamp(0,1), bin_centers_fixed).detach() 60 | elif mode == 'tensor': # activation tensor 61 | nf = (0.1*x.max()).detach() # 0.1 ??? 62 | norm_factors[network.meta['architecture'],str(scale), 'layer'+str(l)] = nf 63 | targets[network.meta['architecture'],str(scale), 'layer'+str(l)] = (x / nf).detach() 64 | 65 | 66 | # for convergence checks 67 | globals()['converged'] = True; globals()['loss_perf_min'] = 1e+9; globals()['loss_perf_converged'] = 1e-4; globals()['convergence_safe'] = False; 68 | 69 | print('Optimizing..') 70 | itr = [0] 71 | while itr[0] <= num_steps: 72 | 73 | def closure(): 74 | carrier_optim.data.clamp_(0, 1) # correct pixels values 75 | optimizer.zero_grad() 76 | loss_perf = torch.Tensor(1).cuda()*0.0; 77 | n = 0 # counter for loss summands 78 | for network in networks: # optimize for all networks 79 | network.eval() 80 | network.cuda() 81 | m = torch.FloatTensor(network.meta['mean']).cuda().unsqueeze(0).unsqueeze(2).unsqueeze(3) 82 | s = torch.FloatTensor(network.meta['std']).cuda().unsqueeze(0).unsqueeze(2).unsqueeze(3) 83 | for scale in scales: # optimize for all scales 84 | si = (scales==scale).nonzero()[0].item() 85 | if sigma_blur > 0.0: 86 | GS = GaussianSmoothing(channels = 3, kernel_size = kernel_size_all[si], sigma = sigma_blur_all[si]).cuda() 87 | else: 88 | GS = nn.Sequential() # identity function 89 | 90 | # normalize (mean-std), re-scale, and feed to the network 91 | xl = network.features(nn.functional.interpolate((GS(carrier_optim) - m )/ s, scale_factor=scale, mode='bilinear', align_corners=False)) 92 | 93 | if not isinstance(xl, list): xl = [xl] 94 | for l in range(len(xl)): 95 | x = xl[l] 96 | if mode == 'global': # global descriptors 97 | for pool in network.poolattack: 98 | ref = network.norm(POOLING[pool](x)).squeeze() 99 | target = targets[network.meta['architecture'],str(scale), pool, 'layer'+str(l)] 100 | loss_perf += 1 - (ref).dot(target) # add loss over networks and scales 101 | n+= 1 102 | elif mode == 'hist': # activation histogram 103 | nf = norm_factors[network.meta['architecture'],str(scale), 'layer'+str(l)] # similar normalization to the target image 104 | hists = hist_per_channel((x / nf).clamp(0,1), bin_centers_fixed) 105 | loss_perf += (targets[network.meta['architecture'],str(scale), 'layer'+str(l)]-hists).pow(2.0).sum(1).sqrt().mean() 106 | n+= 1 107 | elif mode == 'tensor': # activation tensor 108 | nf = norm_factors[network.meta['architecture'],str(scale), 'layer'+str(l)] # similar normalization to the target image 109 | x_norm = x / nf 110 | loss_perf += (targets[network.meta['architecture'],str(scale), 'layer'+str(l)]-x_norm).pow(2).mean() 111 | n += 1 112 | 113 | # compute loss 114 | if lam > 0: loss_distort = (carrier_optim-carrier_org).pow(2.0).sum() / (carrier_optim.size(-1)*carrier_optim.size(-2)) 115 | else: loss_distort = torch.Tensor(1).cuda()*0.0 116 | loss_perf = loss_perf / n # divide by number of summands (networks, scales, poolings) 117 | total_loss = loss_perf + lam * loss_distort 118 | 119 | # check for convergence (hacky!) 120 | if loss_perf < globals()['loss_perf_min']: globals()['loss_perf_min'] = loss_perf.clone() 121 | if loss_perf < globals()['loss_perf_converged']: globals()['convergence_safe'] = True 122 | if globals()['converged'] and (loss_perf-globals()['loss_perf_min']) > 1*globals()['loss_perf_min'] and globals()['convergence_safe'] == False: 123 | globals()['converged'] = False 124 | print("Iter {:5d}, Loss_perf = {:6f} Loss_distort = {:6f} Loss_total = {:6f}".format(itr[0], loss_perf.item(), loss_distort.item(), total_loss.item())) 125 | print('Did not converge') 126 | 127 | total_loss.backward() 128 | 129 | if verbose == True and itr[0] % 5 == 0: 130 | print("Iter {:5d}, Loss_perf = {:6f} Loss_distort = {:6f}, Loss_total = {:6f}".format(itr[0], loss_perf.item(), loss_distort.item(), total_loss.item())) 131 | globals()['loss_perf'] = loss_perf; globals()['loss_distort'] = loss_distort 132 | itr[0] += 1 133 | return total_loss 134 | 135 | 136 | if not globals()['converged']: return carrier_optim.data, 0, 0, False 137 | optimizer.step(closure) 138 | 139 | carrier_optim.data.clamp_(0, 1) # pixel value correction 140 | return carrier_optim.data, globals()['loss_perf'], globals()['loss_distort'], globals()['converged'] 141 | 142 | 143 | def hist_per_channel(x, bin_centers, sigma = 0.1): 144 | x = x.squeeze(0) 145 | N = x.size()[1]*x.size()[2] 146 | xflat = x.flatten().unsqueeze(1) 147 | expx = torch.exp(-torch.add(xflat.type(torch.cuda.DoubleTensor),-1.0*bin_centers.unsqueeze(0)).pow(2.0) / (2*sigma**2) ).type(torch.cuda.FloatTensor) 148 | nf = expx.sum(1).unsqueeze(1) 149 | nf[nf==0] = 1 150 | xh = torch.div(expx, nf) 151 | xh = xh.reshape(x.size(0),N,xh.size(-1)) 152 | hists = xh.sum(1) / (x.size(1)*x.size(2)) 153 | 154 | return hists 155 | 156 | 157 | class GaussianSmoothing(nn.Module): 158 | """ 159 | Apply gaussian smoothing on a 160 | 1d, 2d or 3d tensor. Filtering is performed seperately for each channel 161 | in the input using a depthwise convolution. 162 | Arguments: 163 | channels (int, sequence): Number of channels of the input tensors. Output will 164 | have this number of channels as well. 165 | kernel_size (int, sequence): Size of the gaussian kernel. 166 | sigma (float, sequence): Standard deviation of the gaussian kernel. 167 | dim (int, optional): The number of dimensions of the data. 168 | Default value is 2 (spatial). 169 | 170 | function implemented by Adrian Sahlman https://tinyurl.com/y2w8ktp5 171 | """ 172 | def __init__(self, channels, kernel_size, sigma, dim=2): 173 | super(GaussianSmoothing, self).__init__() 174 | if isinstance(kernel_size, numbers.Number): 175 | kernel_size = [kernel_size] * dim 176 | if isinstance(sigma, numbers.Number): 177 | sigma = [sigma] * dim 178 | 179 | # The gaussian kernel is the product of the 180 | # gaussian function of each dimension. 181 | kernel = 1 182 | meshgrids = torch.meshgrid( 183 | [ 184 | torch.arange(size, dtype=torch.float32) 185 | for size in kernel_size 186 | ] 187 | ) 188 | for size, std, mgrid in zip(kernel_size, sigma, meshgrids): 189 | mean = (size - 1) / 2 190 | kernel *= 1 / (std * math.sqrt(2 * math.pi)) * \ 191 | torch.exp(-((mgrid - mean) / (2 * std)) ** 2) 192 | 193 | # Make sure sum of values in gaussian kernel equals 1. 194 | kernel = kernel / torch.sum(kernel) 195 | 196 | # Reshape to depthwise convolutional weight 197 | kernel = kernel.view(1, 1, *kernel.size()) 198 | kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1)) 199 | 200 | self.register_buffer('weight', kernel) 201 | self.groups = channels 202 | self.kernel_size = kernel_size 203 | 204 | if dim == 1: 205 | self.conv = F.conv1d 206 | elif dim == 2: 207 | self.conv = F.conv2d 208 | elif dim == 3: 209 | self.conv = F.conv3d 210 | else: 211 | raise RuntimeError( 212 | 'Only 1, 2 and 3 dimensions are supported. Received {}.'.format(dim) 213 | ) 214 | 215 | def forward(self, input): 216 | """ 217 | Apply gaussian filter to input. 218 | Arguments: 219 | input (torch.Tensor): Input to apply gaussian filter on. 220 | Returns: 221 | filtered (torch.Tensor): Filtered output. 222 | """ 223 | return self.conv(input, weight=self.weight, groups=self.groups, padding=(int((self.kernel_size[0]-1)/2),int((self.kernel_size[0]-1)/2))) 224 | -------------------------------------------------------------------------------- /cirtorchclone/imageretrievalnet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pdb 3 | import numpy as np 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.utils.model_zoo as model_zoo 8 | 9 | import torchvision 10 | 11 | from cirtorch.layers.pooling import MAC, SPoC, GeM, RMAC, Rpool 12 | from cirtorch.layers.normalization import L2N, PowerLaw 13 | from cirtorch.datasets.genericdataset import ImagesFromList 14 | from cirtorch.utils.general import get_data_root 15 | 16 | # clone file from CIR with added CroW support 17 | # https://github.com/filipradenovic/cnnimageretrieval-pytorch 18 | class CroW(nn.Module): 19 | 20 | def __init__(self): 21 | super(CroW,self).__init__() 22 | 23 | def forward(self, x): 24 | return crow(x) 25 | 26 | def __repr__(self): 27 | return self.__class__.__name__ + '()' 28 | 29 | def crow(x): 30 | # transfer to numpy 31 | X = x.cpu().data.squeeze(0).numpy() 32 | 33 | # spatial weights 34 | a, b = 2, 2 35 | S = X.sum(axis=0) 36 | z = (S**a).sum()**(1./a) 37 | S = (S / z)**(1./b) if b != 1 else (S / z) 38 | 39 | # channel weights 40 | K, w, h = X.shape 41 | area = float(w * h) 42 | nonzeros = np.zeros(K, dtype=np.float32) 43 | for i, x in enumerate(X): 44 | nonzeros[i] = np.count_nonzero(x) / area 45 | nzsum = nonzeros.sum() 46 | for i, d in enumerate(nonzeros): 47 | nonzeros[i] = np.log(nzsum / d) if d > 0. else 0. 48 | 49 | # apply weights 50 | X = X * S 51 | X = X.sum(axis=(1, 2)) 52 | X = X * nonzeros 53 | 54 | # transfer to torch and return 55 | return torch.from_numpy(X).unsqueeze(0).cuda() 56 | 57 | # for some models, we have imported features (convolutions) from caffe because the image retrieval performance is higher for them 58 | FEATURES = { 59 | 'vgg16' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/networks/imagenet/imagenet-caffe-vgg16-features-d369c8e.pth', 60 | 'resnet50' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/networks/imagenet/imagenet-caffe-resnet50-features-ac468af.pth', 61 | 'resnet101' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/networks/imagenet/imagenet-caffe-resnet101-features-10a101d.pth', 62 | 'resnet152' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/networks/imagenet/imagenet-caffe-resnet152-features-1011020.pth', 63 | } 64 | 65 | # TODO: pre-compute for more architectures and properly test variations (pre l2norm, post l2norm) 66 | # pre-computed local pca whitening that can be applied before the pooling layer 67 | L_WHITENING = { 68 | 'resnet101' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-lwhiten-9f830ef.pth', # no pre l2 norm 69 | # 'resnet101' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-lwhiten-da5c935.pth', # with pre l2 norm 70 | } 71 | 72 | # possible global pooling layers, each on of these can be made regional 73 | POOLING = { 74 | 'mac' : MAC, 75 | 'spoc' : SPoC, 76 | 'gem' : GeM, 77 | 'rmac' : RMAC, 78 | 'crow' : CroW, 79 | } 80 | 81 | # TODO: pre-compute for: resnet50-gem-r, resnet50-mac-r, vgg16-mac-r, alexnet-mac-r 82 | # pre-computed regional whitening, for most commonly used architectures and pooling methods 83 | R_WHITENING = { 84 | 'alexnet-gem-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-alexnet-gem-r-rwhiten-c8cf7e2.pth', 85 | 'vgg16-gem-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-vgg16-gem-r-rwhiten-19b204e.pth', 86 | 'resnet101-mac-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-mac-r-rwhiten-7f1ed8c.pth', 87 | 'resnet101-gem-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-gem-r-rwhiten-adace84.pth', 88 | } 89 | 90 | # TODO: pre-compute for more architectures 91 | # pre-computed final (global) whitening, for most commonly used architectures and pooling methods 92 | WHITENING = { 93 | 'alexnet-gem' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-alexnet-gem-whiten-454ad53.pth', 94 | 'alexnet-gem-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-alexnet-gem-r-whiten-4c9126b.pth', 95 | 'vgg16-gem' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-vgg16-gem-whiten-eaa6695.pth', 96 | 'vgg16-gem-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-vgg16-gem-r-whiten-83582df.pth', 97 | 'resnet101-mac-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-mac-r-whiten-9df41d3.pth', 98 | 'resnet101-gem' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-gem-whiten-22ab0c1.pth', 99 | 'resnet101-gem-r' : 'http://cmp.felk.cvut.cz/cnnimageretrieval/data/whiten/retrieval-SfM-120k/retrieval-SfM-120k-resnet101-gem-r-whiten-b379c0a.pth', 100 | } 101 | 102 | # output dimensionality for supported architectures 103 | OUTPUT_DIM = { 104 | 'alexnet' : 256, 105 | 'vgg11' : 512, 106 | 'vgg13' : 512, 107 | 'vgg16' : 512, 108 | 'vgg19' : 512, 109 | 'resnet18' : 512, 110 | 'resnet34' : 512, 111 | 'resnet50' : 2048, 112 | 'resnet101' : 2048, 113 | 'resnet152' : 2048, 114 | 'densenet121' : 1024, 115 | 'densenet161' : 2208, 116 | 'densenet169' : 1664, 117 | 'densenet201' : 1920, 118 | 'squeezenet1_0' : 512, 119 | 'squeezenet1_1' : 512, 120 | } 121 | 122 | 123 | class ImageRetrievalNet(nn.Module): 124 | 125 | def __init__(self, features, lwhiten, pool, whiten, meta): 126 | super(ImageRetrievalNet, self).__init__() 127 | self.features = nn.Sequential(*features) 128 | self.lwhiten = lwhiten 129 | self.pool = pool 130 | self.whiten = whiten 131 | self.norm = L2N() 132 | self.meta = meta 133 | 134 | def forward(self, x): 135 | # x -> features 136 | o = self.features(x) 137 | 138 | # TODO: properly test (with pre-l2norm and/or post-l2norm) 139 | # if lwhiten exist: features -> local whiten 140 | if self.lwhiten is not None: 141 | # o = self.norm(o) 142 | s = o.size() 143 | o = o.permute(0,2,3,1).contiguous().view(-1, s[1]) 144 | o = self.lwhiten(o) 145 | o = o.view(s[0],s[2],s[3],self.lwhiten.out_features).permute(0,3,1,2) 146 | # o = self.norm(o) 147 | 148 | # features -> pool -> norm 149 | o = self.norm(self.pool(o)).squeeze(-1).squeeze(-1) 150 | 151 | # if whiten exist: pooled features -> whiten -> norm 152 | if self.whiten is not None: 153 | o = self.norm(self.whiten(o)) 154 | 155 | # permute so that it is Dx1 column vector per image (DxN if many images) 156 | return o.permute(1,0) 157 | 158 | def __repr__(self): 159 | tmpstr = super(ImageRetrievalNet, self).__repr__()[:-1] 160 | tmpstr += self.meta_repr() 161 | tmpstr = tmpstr + ')' 162 | return tmpstr 163 | 164 | def meta_repr(self): 165 | tmpstr = ' (' + 'meta' + '): dict( \n' # + self.meta.__repr__() + '\n' 166 | tmpstr += ' architecture: {}\n'.format(self.meta['architecture']) 167 | tmpstr += ' local_whitening: {}\n'.format(self.meta['local_whitening']) 168 | tmpstr += ' pooling: {}\n'.format(self.meta['pooling']) 169 | tmpstr += ' regional: {}\n'.format(self.meta['regional']) 170 | tmpstr += ' whitening: {}\n'.format(self.meta['whitening']) 171 | tmpstr += ' outputdim: {}\n'.format(self.meta['outputdim']) 172 | tmpstr += ' mean: {}\n'.format(self.meta['mean']) 173 | tmpstr += ' std: {}\n'.format(self.meta['std']) 174 | tmpstr = tmpstr + ' )\n' 175 | return tmpstr 176 | 177 | 178 | def init_network(params): 179 | 180 | # parse params with default values 181 | architecture = params.get('architecture', 'resnet101') 182 | local_whitening = params.get('local_whitening', False) 183 | pooling = params.get('pooling', 'gem') 184 | regional = params.get('regional', False) 185 | whitening = params.get('whitening', False) 186 | mean = params.get('mean', [0.485, 0.456, 0.406]) 187 | std = params.get('std', [0.229, 0.224, 0.225]) 188 | pretrained = params.get('pretrained', True) 189 | 190 | # get output dimensionality size 191 | dim = OUTPUT_DIM[architecture] 192 | 193 | # loading network from torchvision 194 | if pretrained: 195 | if architecture not in FEATURES: 196 | # initialize with network pretrained on imagenet in pytorch 197 | net_in = getattr(torchvision.models, architecture)(pretrained=True) 198 | else: 199 | # initialize with random weights, later on we will fill features with custom pretrained network 200 | net_in = getattr(torchvision.models, architecture)(pretrained=False) 201 | else: 202 | # initialize with random weights 203 | net_in = getattr(torchvision.models, architecture)(pretrained=False) 204 | 205 | # initialize features 206 | # take only convolutions for features, 207 | # always ends with ReLU to make last activations non-negative 208 | if architecture.startswith('alexnet'): 209 | features = list(net_in.features.children())[:-1] 210 | elif architecture.startswith('vgg'): 211 | features = list(net_in.features.children())[:-1] 212 | elif architecture.startswith('resnet'): 213 | features = list(net_in.children())[:-2] 214 | elif architecture.startswith('densenet'): 215 | features = list(net_in.features.children()) 216 | features.append(nn.ReLU(inplace=True)) 217 | elif architecture.startswith('squeezenet'): 218 | features = list(net_in.features.children()) 219 | else: 220 | raise ValueError('Unsupported or unknown architecture: {}!'.format(architecture)) 221 | 222 | # initialize local whitening 223 | if local_whitening: 224 | lwhiten = nn.Linear(dim, dim, bias=True) 225 | # TODO: lwhiten with possible dimensionality reduce 226 | 227 | if pretrained: 228 | lw = architecture 229 | if lw in L_WHITENING: 230 | print(">> {}: for '{}' custom computed local whitening '{}' is used" 231 | .format(os.path.basename(__file__), lw, os.path.basename(L_WHITENING[lw]))) 232 | whiten_dir = os.path.join(get_data_root(), 'whiten') 233 | lwhiten.load_state_dict(model_zoo.load_url(L_WHITENING[lw], model_dir=whiten_dir)) 234 | else: 235 | print(">> {}: for '{}' there is no local whitening computed, random weights are used" 236 | .format(os.path.basename(__file__), lw)) 237 | 238 | else: 239 | lwhiten = None 240 | 241 | # initialize pooling 242 | pool = POOLING[pooling]() 243 | 244 | # initialize regional pooling 245 | if regional: 246 | rpool = pool 247 | rwhiten = nn.Linear(dim, dim, bias=True) 248 | # TODO: rwhiten with possible dimensionality reduce 249 | 250 | if pretrained: 251 | rw = '{}-{}-r'.format(architecture, pooling) 252 | if rw in R_WHITENING: 253 | print(">> {}: for '{}' custom computed regional whitening '{}' is used" 254 | .format(os.path.basename(__file__), rw, os.path.basename(R_WHITENING[rw]))) 255 | whiten_dir = os.path.join(get_data_root(), 'whiten') 256 | rwhiten.load_state_dict(model_zoo.load_url(R_WHITENING[rw], model_dir=whiten_dir)) 257 | else: 258 | print(">> {}: for '{}' there is no regional whitening computed, random weights are used" 259 | .format(os.path.basename(__file__), rw)) 260 | 261 | pool = Rpool(rpool, rwhiten) 262 | 263 | # initialize whitening 264 | if whitening: 265 | whiten = nn.Linear(dim, dim, bias=True) 266 | # TODO: whiten with possible dimensionality reduce 267 | 268 | if pretrained: 269 | w = architecture 270 | if local_whitening: 271 | w += '-lw' 272 | w += '-' + pooling 273 | if regional: 274 | w += '-r' 275 | if w in WHITENING: 276 | print(">> {}: for '{}' custom computed whitening '{}' is used" 277 | .format(os.path.basename(__file__), w, os.path.basename(WHITENING[w]))) 278 | whiten_dir = os.path.join(get_data_root(), 'whiten') 279 | whiten.load_state_dict(model_zoo.load_url(WHITENING[w], model_dir=whiten_dir)) 280 | else: 281 | print(">> {}: for '{}' there is no whitening computed, random weights are used" 282 | .format(os.path.basename(__file__), w)) 283 | else: 284 | whiten = None 285 | 286 | # create meta information to be stored in the network 287 | meta = { 288 | 'architecture' : architecture, 289 | 'local_whitening' : local_whitening, 290 | 'pooling' : pooling, 291 | 'regional' : regional, 292 | 'whitening' : whitening, 293 | 'mean' : mean, 294 | 'std' : std, 295 | 'outputdim' : dim, 296 | } 297 | 298 | # create a generic image retrieval network 299 | net = ImageRetrievalNet(features, lwhiten, pool, whiten, meta) 300 | 301 | # initialize features with custom pretrained network if needed 302 | if pretrained and architecture in FEATURES: 303 | print(">> {}: for '{}' custom pretrained features '{}' are used" 304 | .format(os.path.basename(__file__), architecture, os.path.basename(FEATURES[architecture]))) 305 | model_dir = os.path.join(get_data_root(), 'networks') 306 | net.features.load_state_dict(model_zoo.load_url(FEATURES[architecture], model_dir=model_dir)) 307 | 308 | return net 309 | 310 | 311 | def extract_vectors(net, images, image_size, transform, bbxs=None, ms=[1], msp=1, print_freq=10): 312 | # moving network to gpu and eval mode 313 | net.cuda() 314 | net.eval() 315 | 316 | # creating dataset loader 317 | loader = torch.utils.data.DataLoader( 318 | ImagesFromList(root='', images=images, imsize=image_size, bbxs=bbxs, transform=transform), 319 | batch_size=1, shuffle=False, num_workers=8, pin_memory=True 320 | ) 321 | 322 | # extracting vectors 323 | with torch.no_grad(): 324 | vecs = torch.zeros(net.meta['outputdim'], len(images)) 325 | for i, input in enumerate(loader): 326 | input = input.cuda() 327 | 328 | if len(ms) == 1: 329 | vecs[:, i] = extract_ss(net, input) 330 | else: 331 | vecs[:, i] = extract_ms(net, input, ms, msp) 332 | 333 | if (i+1) % print_freq == 0 or (i+1) == len(images): 334 | print('\r>>>> {}/{} done...'.format((i+1), len(images)), end='') 335 | print('') 336 | 337 | return vecs 338 | 339 | def extract_ss(net, input): 340 | return net(input).cpu().data.squeeze() 341 | 342 | def extract_ms(net, input, ms, msp): 343 | 344 | v = torch.zeros(net.meta['outputdim']) 345 | 346 | for s in ms: 347 | if s == 1: 348 | input_t = input.clone() 349 | else: 350 | input_t = nn.functional.interpolate(input, scale_factor=s, mode='bilinear', align_corners=False) 351 | v += net(input_t).pow(msp).cpu().data.squeeze() 352 | 353 | v /= len(ms) 354 | v = v.pow(1./msp) 355 | v /= v.norm() 356 | 357 | return v 358 | 359 | 360 | def extract_regional_vectors(net, images, image_size, transform, bbxs=None, ms=[1], msp=1, print_freq=10): 361 | # moving network to gpu and eval mode 362 | net.cuda() 363 | net.eval() 364 | 365 | # creating dataset loader 366 | loader = torch.utils.data.DataLoader( 367 | ImagesFromList(root='', images=images, imsize=image_size, bbxs=bbxs, transform=transform), 368 | batch_size=1, shuffle=False, num_workers=8, pin_memory=True 369 | ) 370 | 371 | # extracting vectors 372 | with torch.no_grad(): 373 | vecs = [] 374 | for i, input in enumerate(loader): 375 | input = input.cuda() 376 | 377 | if len(ms) == 1: 378 | vecs.append(extract_ssr(net, input)) 379 | else: 380 | # TODO: not implemented yet 381 | # vecs.append(extract_msr(net, input, ms, msp)) 382 | raise NotImplementedError 383 | 384 | if (i+1) % print_freq == 0 or (i+1) == len(images): 385 | print('\r>>>> {}/{} done...'.format((i+1), len(images)), end='') 386 | print('') 387 | 388 | return vecs 389 | 390 | def extract_ssr(net, input): 391 | return net.pool(net.features(input), aggregate=False).squeeze(0).squeeze(-1).squeeze(-1).permute(1,0).cpu().data 392 | 393 | 394 | def extract_local_vectors(net, images, image_size, transform, bbxs=None, ms=[1], msp=1, print_freq=10): 395 | # moving network to gpu and eval mode 396 | net.cuda() 397 | net.eval() 398 | 399 | # creating dataset loader 400 | loader = torch.utils.data.DataLoader( 401 | ImagesFromList(root='', images=images, imsize=image_size, bbxs=bbxs, transform=transform), 402 | batch_size=1, shuffle=False, num_workers=8, pin_memory=True 403 | ) 404 | 405 | # extracting vectors 406 | with torch.no_grad(): 407 | vecs = [] 408 | for i, input in enumerate(loader): 409 | input = input.cuda() 410 | 411 | if len(ms) == 1: 412 | vecs.append(extract_ssl(net, input)) 413 | else: 414 | # TODO: not implemented yet 415 | # vecs.append(extract_msl(net, input, ms, msp)) 416 | raise NotImplementedError 417 | 418 | if (i+1) % print_freq == 0 or (i+1) == len(images): 419 | print('\r>>>> {}/{} done...'.format((i+1), len(images)), end='') 420 | print('') 421 | 422 | return vecs 423 | 424 | def extract_ssl(net, input): 425 | return net.norm(net.features(input)).squeeze(0).view(net.meta['outputdim'], -1).cpu().data 426 | --------------------------------------------------------------------------------