├── .gitignore ├── README.md ├── _static └── assets │ └── logos │ ├── FundedbytheEU.png │ ├── elsa.jpg │ └── sec4AI4sec.png ├── analysis ├── __init__.py ├── compile.py ├── compile_jsons.py ├── plot.py ├── plot_distances.py ├── print_tables.py ├── read.py └── utils.py ├── attack_evaluation ├── __init__.py ├── attacks.json ├── attacks │ ├── README.md │ ├── __init__.py │ ├── adv_lib │ │ ├── __init__.py │ │ ├── configs.py │ │ └── wrapper.py │ ├── art │ │ ├── __init__.py │ │ ├── configs.py │ │ └── wrapper.py │ ├── cleverhans │ │ ├── __init__.py │ │ ├── configs.py │ │ └── wrapper.py │ ├── deeprobust │ │ ├── __init__.py │ │ ├── configs.py │ │ └── wrapper.py │ ├── foolbox │ │ ├── __init__.py │ │ ├── bb_adv_init.py │ │ ├── configs.py │ │ └── wrapper.py │ ├── ingredient.py │ ├── original │ │ ├── __init__.py │ │ ├── auto_pgd.py │ │ ├── configs.py │ │ ├── deepfool.py │ │ ├── fast_adaptive_boundary.py │ │ ├── fast_minimum_norm.py │ │ ├── pgd_lzero.py │ │ ├── sigma_zero.py │ │ └── trust_region.py │ └── torchattacks │ │ ├── __init__.py │ │ ├── configs.py │ │ └── wrapper.py ├── datasets │ ├── __init__.py │ ├── imagenet.py │ ├── ingredient.py │ └── subsets │ │ ├── __init__.py │ │ └── imagenet-5000-val.txt ├── experiments.sh ├── models │ ├── __init__.py │ ├── benchmodel_wrapper.py │ ├── ingredient.py │ ├── mnist.py │ └── original │ │ ├── __init__.py │ │ ├── stutz2020 │ │ ├── __init__.py │ │ ├── ccat.py │ │ ├── classifier.py │ │ ├── resnet.py │ │ ├── resnet_block.py │ │ └── torch.py │ │ ├── utils.py │ │ ├── wang2023 │ │ ├── __init__.py │ │ ├── dm_adv_training.py │ │ └── wideresnetwithswish.py │ │ ├── xiao2020 │ │ ├── __init__.py │ │ ├── kwta.py │ │ └── models.py │ │ └── zhang2020 │ │ ├── __init__.py │ │ ├── crown.py │ │ ├── model_defs_gowal.py │ │ └── utils.py ├── run.py └── utils.py ├── compiled ├── cifar10 │ ├── standard_l0.json │ ├── standard_l1.json │ ├── standard_l2.json │ ├── standard_linf.json │ ├── stutz_2020_l0.json │ ├── stutz_2020_l1.json │ ├── stutz_2020_l2.json │ ├── stutz_2020_linf.json │ ├── wang_2023_small_l0.json │ ├── wang_2023_small_l1.json │ ├── wang_2023_small_l2.json │ ├── wang_2023_small_linf.json │ ├── xiao_2020_l0.json │ ├── xiao_2020_l1.json │ ├── xiao_2020_l2.json │ ├── xiao_2020_linf.json │ ├── zhang_2020_small_l0.json │ ├── zhang_2020_small_l1.json │ ├── zhang_2020_small_l2.json │ └── zhang_2020_small_linf.json └── imagenet │ ├── debenedetti_2022_l0.json │ ├── debenedetti_2022_l1.json │ ├── debenedetti_2022_l2.json │ ├── debenedetti_2022_linf.json │ ├── salman_2020_l0.json │ ├── salman_2020_l1.json │ ├── salman_2020_l2.json │ ├── salman_2020_linf.json │ ├── standard_imagenet_l0.json │ ├── standard_imagenet_l1.json │ ├── standard_imagenet_l2.json │ ├── standard_imagenet_linf.json │ ├── wong_2020_l0.json │ ├── wong_2020_l1.json │ ├── wong_2020_l2.json │ └── wong_2020_linf.json ├── environment.yml ├── exp_configs ├── cifar10_l0_1000_bs128.json ├── cifar10_l1_1000_bs128.json ├── cifar10_l2_1000_bs128.json ├── cifar10_linf_1000.json ├── cifar10_linf_1000_bs1.json ├── cifar10_linf_1000_bs128.json ├── cifar10_models.json ├── cifar10_models_key.json ├── imagenet_l0_1000.json ├── imagenet_l1_1000.json ├── imagenet_l2_1000.json ├── imagenet_linf_1000.json ├── imagenet_models.json ├── imagenet_models_key.json ├── l0_1000_attacks.json ├── l1_1000_attacks.json ├── l2_1000_attacks.json ├── linf_1000_attacks.json ├── mnist_linf_1000.json └── mnist_models.json ├── job_runner_cc.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | .DS_Store 7 | # Pycharm 8 | .idea/ 9 | 10 | # Model Zoo 11 | *.pth 12 | 13 | # project 14 | figs/ 15 | 16 | # C extensions 17 | *.so 18 | *.pdf 19 | 20 | # Distribution / packaging 21 | .Python 22 | build/ 23 | develop-eggs/ 24 | dist/ 25 | downloads/ 26 | eggs/ 27 | .eggs/ 28 | lib/ 29 | lib64/ 30 | parts/ 31 | sdist/ 32 | var/ 33 | wheels/ 34 | pip-wheel-metadata/ 35 | share/python-wheels/ 36 | *.egg-info/ 37 | .installed.cfg 38 | *.egg 39 | MANIFEST 40 | 41 | # PyInstaller 42 | # Usually these files are written by a python script from a template 43 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 44 | *.manifest 45 | *.spec 46 | 47 | # Installer logs 48 | pip-log.txt 49 | pip-delete-this-directory.txt 50 | 51 | # Unit test / coverage reports 52 | htmlcov/ 53 | .tox/ 54 | .nox/ 55 | .coverage 56 | .coverage.* 57 | .cache 58 | nosetests.xml 59 | coverage.xml 60 | *.cover 61 | *.py,cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | local_settings.py 72 | db.sqlite3 73 | db.sqlite3-journal 74 | 75 | # Flask stuff: 76 | instance/ 77 | .webassets-cache 78 | 79 | # Scrapy stuff: 80 | .scrapy 81 | 82 | # Sphinx documentation 83 | docs/_build/ 84 | 85 | # PyBuilder 86 | target/ 87 | 88 | # Jupyter Notebook 89 | .ipynb_checkpoints 90 | 91 | # IPython 92 | profile_default/ 93 | ipython_config.py 94 | 95 | # pyenv 96 | .python-version 97 | 98 | # pipenv 99 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 100 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 101 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 102 | # install all needed dependencies. 103 | #Pipfile.lock 104 | 105 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 106 | __pypackages__/ 107 | 108 | # Celery stuff 109 | celerybeat-schedule 110 | celerybeat.pid 111 | 112 | # SageMath parsed files 113 | *.sage.py 114 | 115 | # Environments 116 | .env 117 | .venv 118 | env/ 119 | venv/ 120 | ENV/ 121 | env.bak/ 122 | venv.bak/ 123 | 124 | # Spyder project settings 125 | .spyderproject 126 | .spyproject 127 | 128 | # Rope project settings 129 | .ropeproject 130 | 131 | # mkdocs documentation 132 | /site 133 | 134 | # mypy 135 | .mypy_cache/ 136 | .dmypy.json 137 | dmypy.json 138 | 139 | # Pyre type checker 140 | .pyre/ 141 | 142 | # resources folders 143 | /results/ 144 | /data/ 145 | /models/ 146 | -------------------------------------------------------------------------------- /_static/assets/logos/FundedbytheEU.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/_static/assets/logos/FundedbytheEU.png -------------------------------------------------------------------------------- /_static/assets/logos/elsa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/_static/assets/logos/elsa.jpg -------------------------------------------------------------------------------- /_static/assets/logos/sec4AI4sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/_static/assets/logos/sec4AI4sec.png -------------------------------------------------------------------------------- /analysis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/analysis/__init__.py -------------------------------------------------------------------------------- /analysis/compile.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import pathlib 5 | 6 | from read import read_distances 7 | from utils import Scenario 8 | 9 | 10 | def compile_scenario(path: pathlib.Path, scenario: Scenario, distance_type: str, recompile_all: bool = False) -> None: 11 | # find completed experiment results 12 | scenario_path = path / scenario.dataset / scenario.threat_model / scenario.model.lower() 13 | info_files = list(scenario_path.glob(os.path.join('**', 'info.json'))) 14 | if len(info_files) == 0: 15 | return 16 | 17 | # open previous file if already existing 18 | best_distances_path = path / f'{scenario.dataset}-{scenario.threat_model}-{scenario.model}-{scenario.batch_size}.json' 19 | best_distances = {} 20 | if best_distances_path.exists() and not recompile_all: 21 | with open(best_distances_path, 'r') as f: 22 | best_distances = json.load(f) 23 | 24 | # compile best distances 25 | for info_file in info_files: 26 | hash_distances = read_distances(info_file=info_file, distance_type=distance_type)[1] 27 | for hash, distance in hash_distances.items(): 28 | best_distance = best_distances.get(hash, float('inf')) 29 | if distance < best_distance: 30 | best_distances[hash] = distance 31 | 32 | with open(best_distances_path, 'w') as f: 33 | json.dump(best_distances, f, indent=4) 34 | 35 | 36 | if __name__ == '__main__': 37 | parser = argparse.ArgumentParser('Compile results from several attacks') 38 | 39 | parser.add_argument('--dir', '-d', type=str, default='results', help='Directory used to store experiment results') 40 | parser.add_argument('--dataset', type=str, default=None, help='Dataset for which to plot results') 41 | parser.add_argument('--threat-model', '--tm', type=str, default=None, help='Threat model for which to plot results') 42 | parser.add_argument('--model', '-m', type=str, default=None, help='Model for which to plot results') 43 | parser.add_argument('--recompile-all', '--ra', action='store_true', 44 | help='Ignores previous best distance file and recompile it from scratch.') 45 | 46 | args = parser.parse_args() 47 | 48 | # check that result directory exists 49 | result_path = pathlib.Path(args.dir) 50 | assert result_path.exists() # find info files corresponding to finished experiments 51 | 52 | dataset = args.dataset or '*' 53 | threat_model = args.threat_model or '*' 54 | model = args.model or '*' 55 | 56 | scenarios = result_path.glob(os.sep.join((dataset, threat_model, model))) 57 | for scenario_dir in scenarios: 58 | scenario_dataset, scenario_threat_model, scenario_model = scenario_dir.parts[-3:] 59 | scenario = Scenario(dataset=scenario_dataset, threat_model=scenario_threat_model, model=scenario_model) 60 | compile_scenario(path=result_path, scenario=scenario, recompile_all=args.recompile_all) 61 | -------------------------------------------------------------------------------- /analysis/compile_jsons.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import pathlib 5 | import warnings 6 | from collections import defaultdict 7 | from scipy.stats import wilcoxon, mannwhitneyu, ks_2samp 8 | from pathlib import Path 9 | import numpy as np 10 | from matplotlib import pyplot as plt, ticker 11 | from itertools import combinations 12 | from compile import compile_scenario 13 | from read import read_distances, read_info 14 | from utils import top_k_attacks, eval_optimality, ensemble_distances 15 | import pandas as pd 16 | from tabulate import tabulate 17 | from itertools import product 18 | from utils import rename_attack 19 | 20 | threat_model_labels = { 21 | 'l0': r'$\ell_0$', 22 | 'l1': r'$\ell_1$', 23 | 'l2': r'$\ell_2$', 24 | 'linf': r'$\ell_{\infty}$', 25 | } 26 | 27 | ROUND = lambda x: np.around(x, 4) 28 | TOLERANCE = 1e-04 29 | TEST_SIGNIFICANCE = 0.01 30 | 31 | if __name__ == '__main__': 32 | parser = argparse.ArgumentParser('Plot results') 33 | 34 | parser.add_argument('--dir', '-d', type=str, default='results', help='Directory used to store experiment results') 35 | parser.add_argument('--cdir', '-cd', type=str, default='compiled', help='Directory used to store compiled results') 36 | parser.add_argument('--dataset', type=str, default=None, help='Dataset for which to plot results') 37 | parser.add_argument('--threat-model', '--tm', type=str, default=None, help='Threat model for which to plot results') 38 | parser.add_argument('--model', '-m', type=str, default=None, help='Model for which to plot results') 39 | parser.add_argument('--library', '-l', type=str, default=None, help='Library for which to plot results') 40 | parser.add_argument('--K', '-k', type=int, default=None, help='Top K attacks to show') 41 | parser.add_argument('--LE', '-le', type=int, default=1, help='Levels of ensemble') 42 | parser.add_argument('--KE', '-ke', type=int, default=1, help='Top K ensembles for next level') 43 | parser.add_argument('--batch_size', '-bs', type=int, default=None, help="Batch size for which to plot the results") 44 | parser.add_argument('--info-files', '--if', type=str, nargs='+', default=None, 45 | help='List of info files to plot from.') 46 | parser.add_argument('--distance_type', '-dist', type=str, default='best', choices=['best', 'actual'], 47 | help='Define distances to plot results') 48 | parser.add_argument('--suffix', '-s', type=str, default=None, help='Suffix for the name of the plot') 49 | 50 | args = parser.parse_args() 51 | 52 | # check that result directory exists 53 | result_path = pathlib.Path(args.dir) 54 | assert result_path.exists() 55 | 56 | distance_type = args.distance_type 57 | 58 | to_plot = defaultdict(list) 59 | if args.info_files is not None: 60 | info_files_paths = args.info_files 61 | info_files = [pathlib.Path(info_file) for info_file in info_files_paths] 62 | assert all(info_file.exists() for info_file in info_files) 63 | else: 64 | dataset = args.dataset or '*' 65 | threat_model = args.threat_model or '*' 66 | threat_model_lst = threat_model_labels.keys() if threat_model == '*' else [threat_model] 67 | model = args.model or '*' 68 | library = f'{args.library}_*/**' if args.library else '**' 69 | batch_size = f'batch_size_{args.batch_size}' if args.batch_size else '*' 70 | info_files_paths = os.sep.join((dataset, threat_model, model, batch_size, library, 'info.json')) 71 | info_files = result_path.glob(info_files_paths) 72 | 73 | for info_file in info_files: 74 | scenario, info = read_info(info_file) 75 | to_plot[scenario].append((info_file.parent, info)) 76 | 77 | df = pd.DataFrame() 78 | for scenario in to_plot.keys(): 79 | best_distances_file = result_path / f'{scenario.dataset}-{scenario.threat_model}-{scenario.model}-{scenario.batch_size}.json' 80 | if not best_distances_file.exists(): 81 | # print("Compiling ", scenario) 82 | warnings.warn(f'Best distances files {best_distances_file} does not exist for scenario {scenario}.') 83 | warnings.warn(f'Compiling best distances file for scenario {scenario}') 84 | compile_scenario(path=result_path, scenario=scenario, distance_type=distance_type) 85 | 86 | with open(best_distances_file, 'r') as f: 87 | data = json.load(f) 88 | best_distances = list(data.values()) 89 | 90 | attacks_to_plot = {} 91 | for attack_folder, info in sorted(to_plot[scenario]): 92 | adv_distances = info['best_optim_distances'] if distance_type == 'best' else info['distances'] 93 | optimality = eval_optimality(adv_distances=adv_distances, best_distances=best_distances) 94 | 95 | attack_label = attack_folder.relative_to(attack_folder.parents[1]).as_posix() 96 | 97 | avg_n_forwards = int(np.mean(info['num_forwards'])) 98 | avg_n_backwards = int(np.mean(info['num_backwards'])) 99 | avg_exec_time = np.mean(info['times']) 100 | 101 | library, attack = rename_attack(attack_label).split(' ', 1) 102 | df = df.append({ 103 | 'dataset': scenario.dataset, 104 | 'attack': attack, 105 | 'library': library, 106 | 'model': scenario.model, 107 | 'threat_model': scenario.threat_model, 108 | 'ASR': info['ASR'], 109 | 'optimality': optimality, 110 | 'n_forwards': avg_n_forwards, 111 | 'n_backwards': avg_n_backwards, 112 | 'time': avg_exec_time 113 | }, ignore_index=True) 114 | 115 | for (dataset, model, threat), sub_df in df.groupby(['dataset', 'model', 'threat_model']): 116 | compile_dir = Path(args.cdir or 'compiled') / Path(dataset) 117 | compile_dir.mkdir(exist_ok=True, parents=True) 118 | sub_df.to_json(path_or_buf=compile_dir/f'{model}_{threat}.json', orient='records') -------------------------------------------------------------------------------- /analysis/plot.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import pathlib 5 | import warnings 6 | from collections import defaultdict 7 | 8 | import numpy as np 9 | from matplotlib import pyplot as plt, ticker 10 | 11 | from read import read_results 12 | from utils import ensemble_gain, _MAX_GAIN 13 | 14 | threat_model_labels = { 15 | 'l0': r'$\ell_0$', 16 | 'l1': r'$\ell_1$', 17 | 'l2': r'$\ell_2$', 18 | 'linf': r'$\ell_{\infty}$', 19 | } 20 | 21 | if __name__ == '__main__': 22 | parser = argparse.ArgumentParser('Plot results') 23 | 24 | parser.add_argument('--dir', '-d', type=str, default='results', help='Directory used to store experiment results') 25 | parser.add_argument('--dataset', type=str, default=None, help='Dataset for which to plot results') 26 | parser.add_argument('--threat-model', '--tm', type=str, default=None, help='Threat model for which to plot results') 27 | parser.add_argument('--model', '-m', type=str, default=None, help='Model for which to plot results') 28 | parser.add_argument('--library', '-l', type=str, default=None, help='Library for which to plot results') 29 | parser.add_argument('--batch_size', '-bs', type=int, default=None, help="Batch size for which to plot the results") 30 | parser.add_argument('--info-files', '--if', type=str, nargs='+', default=None, 31 | help='List of info files to plot from.') 32 | parser.add_argument('--suffix', '-s', type=str, default=None, help='Suffix for the name of the plot') 33 | 34 | args = parser.parse_args() 35 | 36 | # check that result directory exists 37 | result_path = pathlib.Path(args.dir) 38 | assert result_path.exists() 39 | 40 | to_plot = defaultdict(dict) 41 | if args.info_files is not None: 42 | info_files_paths = args.info_files 43 | info_files = [pathlib.Path(info_file) for info_file in info_files_paths] 44 | assert all(info_file.exists() for info_file in info_files) 45 | else: 46 | dataset = args.dataset or '*' 47 | threat_model = args.threat_model or '*' 48 | model = args.model or '*' 49 | library = f'{args.library}_*/**' if args.library else '**' 50 | batch_size = f'batch_size_{args.batch_size}' if args.batch_size else '*' 51 | info_files_paths = os.sep.join((dataset, threat_model, model, batch_size, library, 'info.json')) 52 | info_files = result_path.glob(info_files_paths) 53 | 54 | for info_file in info_files: 55 | scenario, info_attack = read_results(info_file) 56 | to_plot[scenario] = info_attack 57 | 58 | atks_gain = np.zeros((len(to_plot), len(to_plot))) 59 | 60 | for i, scenario_atk1 in enumerate(to_plot.keys()): 61 | atk1_info = to_plot[scenario_atk1] 62 | for j, scenario_atk2 in enumerate(to_plot.keys()): 63 | atk2_info = to_plot[scenario_atk2] 64 | atks_gain[i, j] = ensemble_gain(atk1_info['adv_success'], atk2_info['adv_success']) 65 | atks_gain[j, i] = ensemble_gain(atk2_info['adv_success'], atk1_info['adv_success']) 66 | 67 | fig, ax = plt.subplots(figsize=(7, 7), layout="constrained") 68 | #ax.set_title(f'{args.library} ensemble gain', pad=5) 69 | 70 | ax.matshow(atks_gain.transpose(), origin='upper', cmap=plt.cm.Blues, alpha=0.75) 71 | for i in range(len(atks_gain)): 72 | for j in range(len(atks_gain)): 73 | c = '$\infty$' if atks_gain[i, j] == _MAX_GAIN else '%.3f' % atks_gain[i, j] 74 | ax.text(i, j, c, va='center', ha='center') 75 | 76 | atk_names = [f"{scenario.attack}" for scenario in to_plot.keys()] 77 | atk_names_asr = [f"{atk_names[i]}-{to_plot[scenario]['ASR']:.3f}" for i, scenario in enumerate(to_plot.keys())] 78 | 79 | ax.set_xticklabels([''] + atk_names, rotation=90) 80 | ax.set_yticklabels([''] + atk_names_asr) 81 | ax.xaxis.set_major_locator(ticker.MultipleLocator(1)) 82 | ax.yaxis.set_major_locator(ticker.MultipleLocator(1)) 83 | plt.show() -------------------------------------------------------------------------------- /analysis/read.py: -------------------------------------------------------------------------------- 1 | import json 2 | import warnings 3 | from pathlib import Path 4 | from typing import Mapping, Tuple, Union 5 | 6 | import numpy as np 7 | 8 | from utils import Scenario, get_model_key 9 | 10 | 11 | def read_distances(info_file: Union[Path, str], 12 | distance_type: str = 'best', 13 | already_adv_distance: float = 0, 14 | worst_case_distance: float = float('inf')) -> Tuple[Scenario, Mapping[str, float]]: 15 | info_file = Path(info_file) 16 | assert info_file.exists(), f'No info.json found in {dir}' 17 | 18 | # read config file 19 | with open(info_file.parent / 'config.json', 'r') as f: 20 | config = json.load(f) 21 | 22 | # extract main configs 23 | dataset = config['model']['dataset'] 24 | batch_size = str(config['dataset']['batch_size']) 25 | threat_model = config['attack']['threat_model'] 26 | model = get_model_key(config['model']['name']) 27 | 28 | 29 | # read info file 30 | with open(info_file, 'r') as f: 31 | info = json.load(f) 32 | 33 | # get hashes and distances for the adversarial examples 34 | hashes = info['hashes'] 35 | distances = np.array(info['best_optim_distances' if distance_type == 'best' else 'distances'][threat_model]) 36 | ori_success = np.array(info['ori_success']) 37 | adv_success = np.array(info['adv_success']) 38 | 39 | # check that adversarial examples have 0 distance for adversarial clean samples 40 | if (n := np.count_nonzero(distances[ori_success])): 41 | warnings.warn(f'{n} already adversarial clean samples have non zero perturbations in {info_file}.') 42 | 43 | # replace distances with inf for failed attacks and 0 for already adv clean samples 44 | distances[~adv_success] = worst_case_distance 45 | distances[ori_success] = already_adv_distance 46 | 47 | # store results 48 | #scenario = Scenario(dataset=dataset, batch_size=batch_size, attack=attack, library=lib, threat_model=threat_model, 49 | # model=model) 50 | scenario = Scenario(dataset=dataset, batch_size=batch_size, threat_model=threat_model, model=model) 51 | hash_distances = {hash: distance for (hash, distance) in zip(hashes, distances)} 52 | return scenario, hash_distances 53 | 54 | 55 | def read_info(info_file: Union[Path, str], 56 | already_adv_distance: float = 0, 57 | worst_case_distance: float = float('inf')) -> Tuple[Scenario, Mapping[str, float]]: 58 | info_file = Path(info_file) 59 | assert info_file.exists(), f'No info.json found in {dir}' 60 | 61 | # read config file 62 | with open(info_file.parent / 'config.json', 'r') as f: 63 | config = json.load(f) 64 | 65 | # extract main configs 66 | dataset = config['model']['dataset'] 67 | batch_size = str(config['dataset']['batch_size']) 68 | model = get_model_key(config['model']['name']) 69 | threat_model = config['attack']['threat_model'] 70 | 71 | # read info file 72 | with open(info_file, 'r') as f: 73 | info = json.load(f) 74 | 75 | # get distances for the adversarial examples wrt the given threat model 76 | for key in info.keys(): 77 | if isinstance(info[key], list): 78 | info[key] = np.array(info[key]) 79 | 80 | ori_success = info['ori_success'] 81 | adv_success = info['adv_success'] 82 | 83 | # check that adversarial examples have 0 distance for adversarial clean samples 84 | #if (n := np.count_nonzero(info['distances'][info['ori_success']])): 85 | # warnings.warn(f'{n} already adversarial clean samples have non zero perturbations in {info_file}.') 86 | 87 | # replace distances with inf for failed attacks and 0 for already adv clean samples 88 | for distance_type in ['distances', 'best_optim_distances']: 89 | info[distance_type] = np.array(info[distance_type][threat_model]) 90 | info[distance_type][~adv_success] = worst_case_distance 91 | info[distance_type][ori_success] = already_adv_distance 92 | info['adv_valid_success'] = adv_success & (~ori_success) 93 | 94 | # store results 95 | scenario = Scenario(dataset=dataset, batch_size=batch_size, threat_model=threat_model, model=model) 96 | return scenario, info 97 | -------------------------------------------------------------------------------- /analysis/utils.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | import numpy as np 4 | import json 5 | from itertools import product 6 | 7 | Scenario = namedtuple('Scenario', ['dataset', 'batch_size', 'threat_model', 'model']) 8 | 9 | _MAX_GAIN = 2.1 10 | 11 | 12 | def top_k_attacks(attacks_to_plot: dict, k: int = None, only_distinct: bool = False): 13 | if only_distinct: 14 | return top_k_unique_attacks(attacks_to_plot, k) 15 | #attack_names = [k for k, v in sorted(attacks_to_plot.items(), reverse=False, key=lambda item: item[1]['area'])] 16 | attack_names = [k for k, v in sorted(attacks_to_plot.items(), reverse=True, key=lambda item: item[1]['optimality'])] 17 | return zip(attack_names, [rename_attack(atk) for atk in attack_names][:k]) 18 | 19 | def top_k_unique_attacks(attacks_to_plot: dict, k: int = None): 20 | # {k: v['area'] for k, v in sorted(attacks_to_plot.items(), reverse=True, key=lambda item: item[1]['area'])} 21 | attack_names = np.array([k for k, v in sorted(attacks_to_plot.items(), reverse=False, key=lambda item: item[1]['area'])]) 22 | attack_no_lib = [rename_attack(atk).split(' ')[1] for atk in attack_names] 23 | names, index = np.unique(attack_no_lib, return_index=True) 24 | return zip(attack_names[index][:k], names[:k]) 25 | 26 | 27 | def rename_attack(atk_name): 28 | return ((atk_name[:-2].replace('_minimal', '').replace('apgd_t', 'APGD$_t$').replace('l1', '$\ell_1$')) 29 | .replace('linf', '$\ell_\infty$').replace('l2', '$\ell_\infty$').replace('_$\ell','-$\ell').replace('apgd', 'APGD').replace('fmn', 'FMN').replace('pdpgd', 'PDPGD') 30 | .replace('sigma_zero', '$\sigma$-zero').replace('bb', 'BB').replace('pgd0', 'PGD-$\ell_0$').replace('ead', 'EAD') 31 | .replace('vfga', 'VFGA').replace('ddn_NQ', 'DDN').replace('ddn', 'DDN').replace('bim', 'BIM').replace('alma', 'ALMA') 32 | .replace('adv_lib_', 'AdvLib ').replace('original_', 'Original ').replace('fb_', 'Foolbox ').replace('ta_', 'TorchAttack ') 33 | .replace('art_', 'Art ').replace('dr_', 'DeepRobust ').replace('ch_', 'Cleverhans ') 34 | .replace('fab', 'FAB').replace('pgd', 'PGD').replace('cw', 'CW').replace('pdgd', 'PDGD') 35 | ) 36 | 37 | def get_model_key(model): 38 | with open('exp_configs/cifar10_models_key.json', 'r') as f: 39 | models_dict = json.load(f) 40 | return models_dict[model] 41 | 42 | 43 | def complementarity(atk1: np.ndarray, atk2: np.ndarray) -> float: 44 | diversity = (atk1 ^ atk2).sum() 45 | double_fault = ((atk1 + atk2) == 0).sum() 46 | available = diversity + double_fault 47 | if available == 0: 48 | return 0 49 | return diversity / available 50 | 51 | 52 | def ensemble_gain_(atk1: np.ndarray, atk2: np.ndarray) -> float: 53 | assert len(atk1) == len(atk2), "Invalid shape error when evaluating attacks gain." 54 | c = complementarity(atk1, atk2) 55 | if c == 0: 56 | return 0 57 | 58 | e = 1 / (len(atk1) - max(atk1.sum(), atk2.sum())) 59 | if e == float('inf'): 60 | return _MAX_GAIN 61 | 62 | return min(c * e, _MAX_GAIN) 63 | 64 | 65 | def ensemble_gain(atk1: np.ndarray, atk2: np.ndarray) -> float: 66 | n = len(atk1) 67 | return ((atk2 == 0) & (atk1 == 1)).sum() / n 68 | 69 | 70 | def ensemble_distances(atk1_distances: np.ndarray, atk2_distances: np.ndarray) -> np.ndarray: 71 | return np.minimum(atk1_distances, atk2_distances) 72 | 73 | def eval_optimality(adv_distances: np.ndarray, best_distances: list) -> np.ndarray: 74 | distances, counts = np.unique(best_distances, return_counts=True) 75 | robust_acc = 1 - counts.cumsum() / len(best_distances) 76 | 77 | # get quantities for optimality calculation 78 | clean_acc = np.count_nonzero(best_distances) / len(best_distances) 79 | max_dist = np.amax(distances) 80 | best_area = np.trapz(robust_acc, distances) 81 | 82 | distances, counts = np.unique(adv_distances, return_counts=True) 83 | robust_acc = 1 - counts.cumsum() / len(adv_distances) 84 | 85 | distances_clipped, counts = np.unique(adv_distances.clip(min=None, max=max_dist), return_counts=True) 86 | robust_acc_clipped = 1 - counts.cumsum() / len(adv_distances) 87 | 88 | area = np.trapz(robust_acc_clipped, distances_clipped) 89 | optimality = 1 - (area - best_area) / (clean_acc * max_dist - best_area) 90 | 91 | return optimality 92 | 93 | -------------------------------------------------------------------------------- /attack_evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks.json: -------------------------------------------------------------------------------- 1 | { 2 | "l0": { 3 | "adversarial_lib": { 4 | "prefix": "adv_lib", 5 | "attacks": ["fmn", "pdpgd_l0", "vfga"] 6 | }, 7 | "art": { 8 | "prefix": "art", 9 | "attacks": ["bb", "jsma"] 10 | }, 11 | "cleverhans":{ 12 | "prefix": "ch", 13 | "attacks": [] 14 | }, 15 | "deeprobust":{ 16 | "prefix": "dr", 17 | "attacks": [] 18 | }, 19 | "foolbox": { 20 | "prefix": "fb", 21 | "attacks": ["bb", "fmn"] 22 | }, 23 | "original": { 24 | "prefix": "original", 25 | "attacks": ["fmn"] 26 | }, 27 | "torch_attacks": { 28 | "prefix": "ta", 29 | "attacks": ["sparsefool"] 30 | } 31 | }, 32 | "l1": { 33 | "adversarial_lib": { 34 | "prefix": "adv_lib", 35 | "attacks": ["alma_l1", "apgd_minimal_l1", "fab", "fmn", "pdpgd"] 36 | }, 37 | "art": { 38 | "prefix": "art", 39 | "attacks": ["apgd_minimal", "fgm_minimal", "bb", "ead", "pgd_minimal"] 40 | }, 41 | "cleverhans":{ 42 | "prefix": "ch", 43 | "attacks": ["fgm_minimal", "pgd_minimal"] 44 | }, 45 | "deeprobust":{ 46 | "prefix": "dr", 47 | "attacks": [] 48 | }, 49 | "foolbox": { 50 | "prefix": "fb", 51 | "attacks": ["bim_minimal", "bb", "ead", "fgm_minimal", "fmn", "pgd_minimal"] 52 | }, 53 | "original": { 54 | "prefix": "original", 55 | "attacks": ["apgd_minimal_l1", "apgd_t_minimal_l1", "fab", "fmn"] 56 | }, 57 | "torch_attacks": { 58 | "prefix": "ta", 59 | "attacks": ["fab"] 60 | } 61 | }, 62 | "l2": { 63 | "adversarial_lib": { 64 | "prefix": "adv_lib", 65 | "attacks": ["alma_l2", "apgd_minimal", "cw_l2", "ddn", "fab", "fmn", "pdgd", "pdpgd", "tr", "tr_adaptive"] 66 | }, 67 | "art": { 68 | "prefix": "art", 69 | "attacks": ["apgd_minimal", "bb", "cw_l2", "deepfool", "fgm_minimal", "pgd_minimal"] 70 | }, 71 | "cleverhans":{ 72 | "prefix": "ch", 73 | "attacks": ["cw_l2", "fgm_minimal", "pgd_minimal"] 74 | }, 75 | "deeprobust":{ 76 | "prefix": "dr", 77 | "attacks": ["cw_l2", "deepfool", "fgm_minimal", "pgd_minimal"] 78 | }, 79 | "foolbox": { 80 | "prefix": "fb", 81 | "attacks": ["bb", "cw_l2", "dataset", "ddn", "deepfool", "fmn", "pgd_minimal", "fgm_minimal", "bim_minimal"] 82 | }, 83 | "original": { 84 | "prefix": "original", 85 | "attacks": ["apgd_minimal", "apgd_t_minimal", "deepfool", "fab", "fmn", "tr", "tr_adaptive"] 86 | }, 87 | "torch_attacks": { 88 | "prefix": "ta", 89 | "attacks": ["cw_l2", "apgd_minimal", "deepfool", "fab", "pgd_l2_minimal"] 90 | } 91 | }, 92 | "linf": { 93 | "adversarial_lib": { 94 | "prefix": "adv_lib", 95 | "attacks": ["apgd_minimal", "cw_linf", "fab", "fmn", "pdpgd", "pgd_minimal", "tr", "tr_adaptive"] 96 | }, 97 | "art": { 98 | "prefix": "art", 99 | "attacks": ["apgd_minimal", "bim_minimal", "bb", "cw_linf", "fgm_minimal", "pgd_minimal"] 100 | }, 101 | "cleverhans":{ 102 | "prefix": "ch", 103 | "attacks": ["fgm_minimal", "pgd_minimal"] 104 | }, 105 | "deeprobust":{ 106 | "prefix": "dr", 107 | "attacks": ["fgm_minimal", "pgd_minimal"] 108 | }, 109 | "foolbox": { 110 | "prefix": "fb", 111 | "attacks": ["bb", "bim_minimal", "deepfool", "fmn", "pgd_minimal", "fgm_minimal"] 112 | }, 113 | "original": { 114 | "prefix": "original", 115 | "attacks": ["apgd_minimal", "apgd_t_minimal", "fab", "fmn_linf", "tr", "tr_adaptive"] 116 | }, 117 | "torch_attacks": { 118 | "prefix": "ta", 119 | "attacks": ["fgsm", "pgd", "apgd"] 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /attack_evaluation/attacks/README.md: -------------------------------------------------------------------------------- 1 | # Adding an attack 2 | 3 | To add an attack that can be called through the sacred CLI, you need to implement two functions. 4 | 5 | ### Config function 6 | 7 | The first function will be the named config and should follow the template: 8 | 9 | ```python 10 | def _(): 11 | name = '' 12 | source = '' 13 | threat_model = '' 14 | option1 = 0.01 15 | ``` 16 | 17 | The library prefix corresponds to a shorter version of the library name; for instance, Foolbox's prefix is `fb`. 18 | All other variables of the config function will correspond to the attack's option. 19 | This function should be placed in the `/configs.py` file. 20 | 21 | ### Getter function 22 | 23 | The second function to implemented is the getter function, which will return the attack as a callable. This getter 24 | function should follow the template: 25 | 26 | ```python 27 | def get__(option1: float) -> Callable: 28 | return ... 29 | ``` 30 | 31 | The `` in the name of the getter function should match exactly the name of the attack in the config 32 | function: `name = `. This is necessary to determine which getter function to call when calling a named 33 | config. 34 | 35 | ### Example 36 | 37 | In `foolbox/configs.py`, the DDN attack is added with the two functions: 38 | 39 | ```python 40 | def fb_ddn(): 41 | name = 'ddn' 42 | source = 'foolbox' 43 | threat_model = 'l2' 44 | init_epsilon = 1 45 | num_steps = 100 46 | gamma = 0.05 47 | 48 | 49 | def get_fb_ddn(init_epsilon: float, num_steps: int, gamma: float) -> Callable: 50 | return partial(DDNAttack, init_epsilon=init_epsilon, steps=num_steps, gamma=gamma) 51 | ``` 52 | 53 | Additionally, one could add a second named config for DDN by simply implementing a second config function: 54 | 55 | ```python 56 | def fb_ddn_large_gamma(): 57 | name = 'ddn' 58 | source = 'foolbox' 59 | threat_model = 'l2' 60 | init_epsilon = 1 61 | num_steps = 100 62 | gamma = 0.5 63 | ``` 64 | 65 | This second named config would point to the same getter function. -------------------------------------------------------------------------------- /attack_evaluation/attacks/__init__.py: -------------------------------------------------------------------------------- 1 | minimal_search_steps = 20 2 | minimal_init_eps = { 3 | 'l0': 100, 4 | 'l1': 10, 5 | 'l2': 1, 6 | 'linf': 1 / 255, 7 | } 8 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/adv_lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/adv_lib/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/adv_lib/wrapper.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from typing import Callable, Optional 3 | 4 | import torch 5 | from torch import Tensor, nn 6 | 7 | 8 | def adv_lib_wrapper(attack: Callable, 9 | model: nn.Module, 10 | inputs: Tensor, 11 | labels: Tensor, 12 | targets: Optional[Tensor] = None, 13 | targeted: bool = False) -> Tensor: 14 | attack_labels = targets if targeted else labels 15 | return attack(model=model, inputs=inputs, labels=attack_labels, targeted=targeted) 16 | 17 | 18 | def adv_lib_minimal_wrapper(model: nn.Module, 19 | inputs: Tensor, 20 | labels: Tensor, 21 | attack: partial, 22 | init_eps: float, 23 | eps_name: str = 'eps', 24 | targeted: bool = False, 25 | max_eps: Optional[float] = None, 26 | search_steps: int = 20) -> Tensor: 27 | device = inputs.device 28 | batch_size = len(inputs) 29 | 30 | adv_inputs = inputs.clone() 31 | eps_low = torch.full((batch_size,), 0, dtype=inputs.dtype, device=device) 32 | best_eps = torch.full_like(eps_low, float('inf') if max_eps is None else 2 * max_eps) 33 | found_high = torch.full_like(eps_low, False, dtype=torch.bool) 34 | 35 | eps = torch.full_like(eps_low, init_eps) 36 | for i in range(search_steps): 37 | adv_inputs_run = attack(model=model, inputs=inputs, labels=labels, targeted=targeted, **{eps_name: eps}) 38 | 39 | logits = model(adv_inputs_run) 40 | preds = logits.argmax(dim=1) 41 | is_adv = (preds == labels) if targeted else (preds != labels) 42 | 43 | better_adv = is_adv & (eps < best_eps) 44 | adv_inputs[better_adv] = adv_inputs_run[better_adv] 45 | 46 | found_high.logical_or_(better_adv) 47 | eps_low = torch.where(better_adv, eps_low, eps) 48 | best_eps = torch.where(better_adv, eps, best_eps) 49 | 50 | eps = torch.where(found_high | ((2 * eps_low) >= best_eps), (eps_low + best_eps) / 2, 2 * eps_low) 51 | 52 | return adv_inputs 53 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/art/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/art/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/art/wrapper.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from functools import partial 3 | from typing import Callable, Optional 4 | 5 | import numpy as np 6 | from art.estimators.classification import PyTorchClassifier 7 | from torch import Tensor, from_numpy, nn 8 | 9 | 10 | def art_wrapper(attack: Callable, 11 | model: nn.Module, 12 | inputs: Tensor, 13 | labels: Tensor, 14 | targets: Optional[Tensor] = None, 15 | targeted: bool = False) -> Tensor: 16 | loss = nn.CrossEntropyLoss() 17 | input_shape = inputs.shape[1:] 18 | output_shape = [module for module in model.modules()][-1].out_features 19 | 20 | art_model = PyTorchClassifier(model=model, clip_values=(0, 1), loss=loss, input_shape=input_shape, 21 | nb_classes=output_shape) 22 | model_arg = next(iter(inspect.signature(attack.func).parameters)) 23 | attack_kwargs = {model_arg: art_model} 24 | 25 | if 'targeted' in attack.func.attack_params: # not all attacks have the targeted arg 26 | attack_kwargs['targeted'] = targeted 27 | 28 | attack = attack(batch_size=len(inputs), **attack_kwargs) 29 | y = targets if targeted else labels 30 | adv_inputs = attack.generate(x=inputs.cpu().numpy(), y=y.cpu().numpy()) 31 | 32 | return from_numpy(adv_inputs).to(inputs.device) 33 | 34 | 35 | class ArtMinimalWrapper: 36 | def __init__(self, attack: partial, init_eps: float, max_eps: Optional[float] = None, search_steps: int = 20, 37 | batched: bool = False): 38 | self.attack_partial = attack 39 | self.init_eps = init_eps 40 | self.max_eps = max_eps 41 | self.search_steps = search_steps 42 | self.batched = batched 43 | 44 | @property 45 | def func(self): 46 | return self.attack_partial.func 47 | 48 | def __call__(self, **kwargs): 49 | self.attack_kwargs = kwargs 50 | return self 51 | 52 | def generate(self, x: np.ndarray, y: np.ndarray) -> np.ndarray: 53 | adv_inputs = x.copy() 54 | eps_low = np.full_like(y, 0, dtype=x.dtype) 55 | best_eps = np.full_like(eps_low, float('inf') if self.max_eps is None else 2 * self.max_eps) 56 | found_high = np.full_like(eps_low, False, dtype=bool) 57 | 58 | eps = np.full_like(eps_low, self.init_eps) 59 | for i in range(self.search_steps): 60 | if self.batched: 61 | eps_step = np.full_like(eps, self.attack_partial.keywords['eps_step']) 62 | attack = self.attack_partial(eps=eps, **self.attack_kwargs, eps_step=eps_step) 63 | adv_inputs_run = attack.generate(x=x, y=y) 64 | else: 65 | # run attacks by batching based on unique epsilons 66 | adv_inputs_run = x.copy() 67 | unique_eps = np.unique(eps) 68 | for eps_ in unique_eps: 69 | mask = eps == eps_ 70 | attack = self.attack_partial(eps=float(eps_), **self.attack_kwargs) 71 | adv_inputs_run[mask] = attack.generate(x=x[mask], y=y[mask]) 72 | 73 | logits = attack.estimator.predict(x=adv_inputs_run, batch_size=len(adv_inputs_run)) 74 | preds = np.argmax(logits, axis=1) 75 | is_adv = (preds == y) if attack.targeted else (preds != y) 76 | 77 | better_adv = is_adv & (eps < best_eps) 78 | adv_inputs[better_adv] = adv_inputs_run[better_adv] 79 | 80 | found_high |= better_adv 81 | eps_low = np.where(better_adv, eps_low, eps) 82 | best_eps = np.where(better_adv, eps, best_eps) 83 | 84 | eps = np.where(found_high | ((2 * eps_low) >= best_eps), (eps_low + best_eps) / 2, 2 * eps_low) 85 | 86 | return adv_inputs 87 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/cleverhans/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/cleverhans/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/cleverhans/configs.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from typing import Callable, Optional 3 | 4 | from cleverhans.torch.attacks.carlini_wagner_l2 import carlini_wagner_l2 5 | from cleverhans.torch.attacks.fast_gradient_method import fast_gradient_method 6 | from cleverhans.torch.attacks.hop_skip_jump_attack import hop_skip_jump_attack 7 | from cleverhans.torch.attacks.projected_gradient_descent import projected_gradient_descent 8 | from cleverhans.torch.attacks.spsa import spsa 9 | 10 | from .wrapper import cleverhans_minimal_wrapper, cleverhans_wrapper 11 | from .. import minimal_init_eps, minimal_search_steps 12 | 13 | _prefix = 'ch' 14 | _wrapper = cleverhans_wrapper 15 | _norms = { 16 | 'l0': 0, 17 | 'l1': 1, 18 | 'l2': 2, 19 | 'linf': float('inf'), 20 | } 21 | 22 | 23 | def ch_cw_l2(): 24 | name = 'cw_l2' 25 | source = 'cleverhans' 26 | threat_model = 'l2' 27 | lr = 5e-03 28 | confidence = 0 29 | clip_min = 0 30 | clip_max = 1 31 | initial_const = 1e-02 32 | binary_search_steps = 5 33 | num_steps = 100 # 1000 34 | 35 | 36 | def get_ch_cw_l2(lr: float, confidence: float, clip_min: float, clip_max: float, initial_const: float, 37 | binary_search_steps: int, num_steps: int) -> Callable: 38 | return partial(carlini_wagner_l2, max_iterations=num_steps, lr=lr, confidence=confidence, clip_max=clip_max, 39 | clip_min=clip_min, initial_const=initial_const, binary_search_steps=binary_search_steps) 40 | 41 | 42 | def ch_fgm(): 43 | name = 'fgm' 44 | source = 'cleverhans' 45 | threat_model = 'l2' # available: l1, l2, linf 46 | eps = 0.3 47 | 48 | 49 | def get_ch_fgm(threat_model: str, eps: float) -> Callable: 50 | return partial(fast_gradient_method, norm=_norms[threat_model], eps=eps, clip_min=0, clip_max=1) 51 | 52 | 53 | def ch_fgm_minimal(): 54 | name = 'fgm_minimal' 55 | source = 'cleverhans' 56 | threat_model = 'linf' # available: l1, l2, linf 57 | 58 | 59 | def get_ch_fgm_minimal(threat_model: str, 60 | init_eps: Optional[float] = None, search_steps: int = minimal_search_steps) -> Callable: 61 | attack = partial(fast_gradient_method, norm=_norms[threat_model], clip_min=0, clip_max=1) 62 | init_eps = minimal_init_eps[threat_model] if init_eps is None else init_eps 63 | max_eps = 1 if threat_model == 'linf' else None 64 | return partial(cleverhans_minimal_wrapper, attack=attack, init_eps=init_eps, max_eps=max_eps, 65 | search_steps=search_steps) 66 | 67 | 68 | def ch_hsja(): 69 | name = 'hsja' 70 | source = 'cleverhans' 71 | threat_model = 'l2' # available: l2, linf 72 | num_steps = 64 73 | initial_num_evals = 100 74 | max_num_evals = 10000 75 | stepsize_search = "geometric_progression" 76 | gamma = 1.0 77 | constraint = 2 78 | batch_size = 128 79 | 80 | 81 | def get_ch_hsja(threat_model: str, num_steps: int, initial_num_evals: int, max_num_evals: int, stepsize_search: int, 82 | gamma: float, constraint: int, batch_size: int) -> Callable: 83 | return partial(hop_skip_jump_attack, norm=_norms[threat_model], initial_num_evals=initial_num_evals, 84 | max_num_evals=max_num_evals, stepsize_search=stepsize_search, num_iterations=num_steps, 85 | gamma=gamma, constraint=constraint, batch_size=batch_size, clip_min=0, clip_max=1) 86 | 87 | 88 | def ch_spsa(): 89 | name = 'spsa' 90 | source = 'cleverhans' 91 | num_steps = 100 92 | threat_model = 'linf' 93 | eps = 0.3 94 | early_stop_loss_threshold = None 95 | lr = 0.01 96 | delta = 0.01 97 | spsa_samples = 128 98 | spsa_iters = 1 99 | 100 | 101 | def get_ch_spsa(threat_model: str, num_steps: int, eps: float, early_stop_loss_threshold: float, lr: float, 102 | delta: float, spsa_samples: int, spsa_iters: int) -> Callable: 103 | return partial(spsa, norm=_norms[threat_model], nb_iter=num_steps, eps=eps, learning_rate=lr, delta=delta, 104 | spsa_iters=spsa_iters, early_stop_loss_threshold=early_stop_loss_threshold, 105 | spsa_samples=spsa_samples, clip_min=0, clip_max=1, sanity_checks=False) 106 | 107 | 108 | def ch_pgd(): 109 | name = 'pgd' 110 | source = 'cleverhans' 111 | threat_model = 'l2' # available: np.inf, 1 or 2. 112 | eps = 10.0 113 | eps_iter = 1.0 114 | num_steps = 20 115 | 116 | 117 | def get_ch_pgd(threat_model: str, eps: float, eps_iter: float, num_steps: int) -> Callable: 118 | return partial(projected_gradient_descent, norm=_norms[threat_model], nb_iter=num_steps, eps=eps, eps_iter=eps_iter, 119 | clip_min=0, clip_max=1, sanity_checks=False) 120 | 121 | 122 | def ch_pgd_minimal(): 123 | name = 'pgd_minimal' 124 | source = 'cleverhans' 125 | threat_model = 'linf' # available: np.inf, 1 or 2. 126 | eps_iter = 1.0 127 | num_steps = 40 # default was 20 128 | 129 | 130 | def get_ch_pgd_minimal(threat_model: str, eps_iter: float, num_steps: int, 131 | init_eps: Optional[float] = None, search_steps: int = minimal_search_steps) -> Callable: 132 | attack = partial(projected_gradient_descent, norm=_norms[threat_model], nb_iter=num_steps, eps_iter=eps_iter, 133 | clip_min=0, clip_max=1, sanity_checks=False) 134 | init_eps = minimal_init_eps[threat_model] if init_eps is None else init_eps 135 | max_eps = 1 if threat_model == 'linf' else None 136 | return partial(cleverhans_minimal_wrapper, attack=attack, init_eps=init_eps, max_eps=max_eps, 137 | search_steps=search_steps) 138 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/cleverhans/wrapper.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from typing import Callable, Optional 3 | 4 | import torch 5 | from torch import Tensor, nn 6 | 7 | 8 | def cleverhans_wrapper(attack: Callable, 9 | model: nn.Module, 10 | inputs: Tensor, 11 | labels: Tensor, 12 | targets: Optional[Tensor] = None, 13 | targeted: bool = False) -> Tensor: 14 | if attack.func == cleverhans_minimal_wrapper: 15 | cleverhans_attack = attack.keywords['attack'].func 16 | else: 17 | cleverhans_attack = attack.func 18 | attack_parameters = inspect.signature(cleverhans_attack).parameters 19 | attack_kwargs = {} 20 | if 'n_classes' in attack_parameters: # specify the number of classes for some attacks 21 | attack_kwargs['n_classes'] = [module for module in model.modules()][-1].out_features 22 | if 'targeted' in attack_parameters: 23 | attack_kwargs['targeted'] = targeted 24 | if 'y' in attack_parameters: 25 | attack_kwargs['y'] = targets if targeted else labels 26 | if 'y_target' in attack_parameters and targeted: 27 | attack_kwargs['y_target'] = targets 28 | 29 | adv_examples = attack(model_fn=model, x=inputs, **attack_kwargs) 30 | return adv_examples 31 | 32 | 33 | def cleverhans_minimal_wrapper(attack: Callable, 34 | model_fn: nn.Module, 35 | x: Tensor, 36 | init_eps: float, 37 | max_eps: Optional[float] = None, 38 | search_steps: int = 20, **kwargs) -> Tensor: 39 | device = x.device 40 | batch_size = len(x) 41 | 42 | labels = kwargs.get('y', model_fn(x).argmax(dim=1)) 43 | targeted = kwargs.get('targeted', False) 44 | 45 | adv_inputs = x.clone() 46 | eps_low = torch.full((batch_size,), 0, dtype=x.dtype, device=device) 47 | best_eps = torch.full_like(eps_low, float('inf') if max_eps is None else 2 * max_eps) 48 | found_high = torch.full_like(eps_low, False, dtype=torch.bool) 49 | 50 | eps = torch.full_like(eps_low, init_eps) 51 | for i in range(search_steps): 52 | adv_inputs_run = x.clone() 53 | for eps_ in torch.unique(eps): 54 | mask = eps == eps_ 55 | eps_kwargs = {k: v for k, v in kwargs.items() if k in ['n_classes', 'targeted']} 56 | if 'y' in kwargs: 57 | eps_kwargs['y'] = kwargs['y'][mask] 58 | if 'y_target' in kwargs: 59 | eps_kwargs['y_target'] = kwargs['y_target'][mask] 60 | adv_inputs_run[mask] = attack(model_fn=model_fn, x=x[mask], eps=eps_, **eps_kwargs) 61 | 62 | logits = model_fn(adv_inputs_run) 63 | preds = logits.argmax(dim=1) 64 | is_adv = (preds == labels) if targeted else (preds != labels) 65 | 66 | better_adv = is_adv & (eps < best_eps) 67 | adv_inputs[better_adv] = adv_inputs_run[better_adv] 68 | 69 | found_high.logical_or_(better_adv) 70 | eps_low = torch.where(better_adv, eps_low, eps) 71 | best_eps = torch.where(better_adv, eps, best_eps) 72 | 73 | eps = torch.where(found_high | ((2 * eps_low) >= best_eps), (eps_low + best_eps) / 2, 2 * eps_low) 74 | 75 | return adv_inputs 76 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/deeprobust/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/deeprobust/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/deeprobust/configs.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from functools import partial 3 | from typing import Dict, Optional 4 | 5 | from deeprobust.image.attack.cw import CarliniWagner 6 | from deeprobust.image.attack.deepfool import DeepFool 7 | from deeprobust.image.attack.fgsm import FGSM 8 | from deeprobust.image.attack.pgd import PGD 9 | 10 | from .wrapper import DeepRobustMinimalWrapper, deeprobust_wrapper 11 | from .. import minimal_init_eps, minimal_search_steps 12 | 13 | _prefix = 'dr' 14 | _wrapper = deeprobust_wrapper 15 | 16 | 17 | def dr_cw_l2(): 18 | name = 'cw_l2' 19 | source = 'deeprobust' 20 | threat_model = 'l2' 21 | confidence = 0.0001 22 | num_binary_search_steps = 5 23 | num_steps = 100 # default was 1000 24 | initial_const = 0.01 25 | step_size = 0.00001 26 | abort_early = True 27 | 28 | 29 | def get_dr_cw_l2(confidence: float, num_binary_search_steps: int, num_steps: int, initial_const: float, 30 | step_size: float, abort_early: bool) -> Dict: 31 | warnings.warn('C&W L2 is not functional with DeepRobust.') 32 | # Several issues: 33 | # - cannot import _status_message from scipy.optimize.optimize with scipy>=1.8.0 ==> use scipy<1.8.0 34 | # - does not work in batches 35 | # - error in line 181 of deeprobust.image.attack.cw which tries to format the loss tensor 36 | return dict( 37 | attack=CarliniWagner, 38 | attack_params=dict(confidence=confidence, binary_search_steps=num_binary_search_steps, abort_early=abort_early, 39 | max_iterations=num_steps, initial_const=initial_const, learning_rate=step_size) 40 | ) 41 | 42 | 43 | def dr_deepfool(): 44 | name = 'deepfool' 45 | source = 'deeprobust' 46 | threat_model = 'l2' 47 | overshoot = 0.02 48 | num_steps = 50 49 | num_classes = 10 50 | 51 | 52 | def get_dr_deepfool(overshoot: float, num_steps: int, num_classes: int) -> Dict: 53 | warnings.warn('DeepFool does not support batches in DeepRobust.') 54 | return dict(attack=DeepFool, 55 | attack_params=dict(overshoot=overshoot, max_iteration=num_steps, num_classes=num_classes)) 56 | 57 | 58 | def dr_pgd(): 59 | name = 'pgd' 60 | source = 'deeprobust' 61 | threat_model = 'linf' # available: 'l2', 'linf' 62 | epsilon = 0.3 63 | num_steps = 40 64 | step_size = 0.01 65 | 66 | 67 | def get_dr_pgd(threat_model: str, epsilon: float, num_steps: int, step_size: float) -> Dict: 68 | return dict(attack=PGD, 69 | attack_params=dict(bound=threat_model, epsilon=epsilon, num_steps=num_steps, step_size=step_size)) 70 | 71 | 72 | def dr_pgd_minimal(): 73 | name = 'pgd_minimal' 74 | source = 'deeprobust' 75 | threat_model = 'linf' # available: 'l2', 'linf' 76 | num_steps = 40 77 | step_size = 0.01 78 | 79 | 80 | def get_dr_pgd_minimal(threat_model: str, num_steps: int, step_size: float, 81 | init_eps: Optional[float] = None, search_steps: int = minimal_search_steps) -> Dict: 82 | init_eps = minimal_init_eps[threat_model] if init_eps is None else init_eps 83 | max_eps = 1 if threat_model == 'linf' else None 84 | attack = partial(DeepRobustMinimalWrapper, attack=PGD, init_eps=init_eps, search_steps=search_steps, 85 | max_eps=max_eps) 86 | return dict(attack=attack, attack_params=dict(bound=threat_model, num_steps=num_steps, step_size=step_size)) 87 | 88 | 89 | def dr_fgm(): 90 | name = 'fgm' 91 | source = 'deeprobust' 92 | threat_model = 'linf' # available: 'l2', 'linf' 93 | epsilon = 0.2 94 | 95 | 96 | def get_dr_fgm(threat_model: str, epsilon: float) -> Dict: 97 | order = {'linf': float('inf'), 'l2': 2} 98 | return dict(attack=FGSM, attack_params=dict(order=order[threat_model], epsilon=epsilon, clip_max=1, clip_min=0)) 99 | 100 | 101 | def dr_fgm_minimal(): 102 | name = 'fgm_minimal' 103 | source = 'deeprobust' 104 | threat_model = 'linf' # available: 'l2', 'linf' 105 | 106 | 107 | def get_dr_fgm_minimal(threat_model: str, 108 | init_eps: Optional[float] = None, search_steps: int = minimal_search_steps) -> Dict: 109 | order = {'linf': float('inf'), 'l2': 2} 110 | init_eps = minimal_init_eps[threat_model] if init_eps is None else init_eps 111 | max_eps = 1 if threat_model == 'linf' else None 112 | attack = partial(DeepRobustMinimalWrapper, attack=FGSM, init_eps=init_eps, search_steps=search_steps, 113 | max_eps=max_eps) 114 | return dict(attack=attack, attack_params=dict(order=order[threat_model], clip_min=0, clip_max=1)) 115 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/deeprobust/wrapper.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from typing import Optional 3 | 4 | import torch 5 | from deeprobust.image.attack.base_attack import BaseAttack 6 | from torch import Tensor, nn 7 | 8 | 9 | class DeepRobustModel(nn.Module): 10 | def __init__(self, model: nn.Module): 11 | super(DeepRobustModel, self).__init__() 12 | self.model = model 13 | 14 | def forward(self, x: Tensor) -> Tensor: 15 | return self.model(x) 16 | 17 | def get_logits(self, x: Tensor) -> Tensor: 18 | return self.forward(x) 19 | 20 | 21 | def deeprobust_wrapper(attack: type[BaseAttack], 22 | attack_params: dict, 23 | model: nn.Module, 24 | inputs: Tensor, 25 | labels: Tensor, 26 | targets: Optional[Tensor] = None, 27 | targeted: bool = False) -> Tensor: 28 | attack = attack(model=DeepRobustModel(model), device=inputs.device.type) 29 | 30 | attack_param_names = inspect.signature(attack.parse_params).parameters 31 | if 'classnum' in attack_param_names: # specify the number of classes for some attacks 32 | attack_params['classnum'] = [module for module in model.modules()][-1].out_features 33 | 34 | if 'target_label' in inspect.signature(attack.generate).parameters: 35 | adv_examples = attack.generate(image=inputs, label=labels, target_label=0, **attack_params) 36 | else: 37 | adv_examples = attack.generate(image=inputs, label=labels, **attack_params) 38 | 39 | return adv_examples 40 | 41 | 42 | class DeepRobustMinimalWrapper: 43 | def __init__(self, model: DeepRobustModel, device: str, attack: type[BaseAttack], init_eps: float, 44 | search_steps: int, max_eps: Optional[float] = None): 45 | self.model = model 46 | self.attack = attack(model=model, device=device) 47 | self.init_eps = init_eps 48 | self.search_steps = search_steps 49 | self.max_eps = max_eps 50 | 51 | @property 52 | def parse_params(self): 53 | return self.attack.parse_params 54 | 55 | def generate(self, image: Tensor, label: Tensor, **attack_params) -> Tensor: 56 | adv_inputs = image.clone() 57 | eps_low = torch.full_like(label, 0, dtype=image.dtype) 58 | best_eps = torch.full_like(eps_low, float('inf') if self.max_eps is None else 2 * self.max_eps) 59 | found_high = torch.full_like(eps_low, False, dtype=torch.bool) 60 | 61 | eps = torch.full_like(eps_low, self.init_eps) 62 | for i in range(self.search_steps): 63 | adv_inputs_run = image.clone() 64 | for eps_ in torch.unique(eps): 65 | mask = eps == eps_ 66 | adv_inputs_run[mask] = self.attack.generate(image=image[mask], label=label[mask], epsilon=eps_.item(), 67 | **attack_params) 68 | 69 | logits = self.model(adv_inputs_run) 70 | preds = logits.argmax(dim=1) 71 | is_adv = preds != label 72 | 73 | better_adv = is_adv & (eps < best_eps) 74 | adv_inputs[better_adv] = adv_inputs_run[better_adv] 75 | 76 | found_high.logical_or_(better_adv) 77 | eps_low = torch.where(better_adv, eps_low, eps) 78 | best_eps = torch.where(better_adv, eps, best_eps) 79 | 80 | eps = torch.where(found_high | ((2 * eps_low) >= best_eps), (eps_low + best_eps) / 2, 2 * eps_low) 81 | 82 | return adv_inputs 83 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/foolbox/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/foolbox/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/foolbox/bb_adv_init.py: -------------------------------------------------------------------------------- 1 | from foolbox.attacks import L0BrendelBethgeAttack, L1BrendelBethgeAttack, L2BrendelBethgeAttack, LinfinityBrendelBethgeAttack 2 | from torch import Tensor, nn 3 | from typing import Callable 4 | from foolbox.attacks.dataset_attack import DatasetAttack 5 | import foolbox as fb 6 | from foolbox import PyTorchModel 7 | 8 | _bb_attacks = { 9 | 'l0': L0BrendelBethgeAttack, 10 | 'l1': L1BrendelBethgeAttack, 11 | 'l2': L2BrendelBethgeAttack, 12 | 'linf': LinfinityBrendelBethgeAttack, 13 | } 14 | 15 | def dataset_BB_attack(bb_attack: Callable, model: PyTorchModel, inputs: Tensor, labels: Tensor, **kwargs): 16 | dataset_atk = DatasetAttack() 17 | dataset_atk.feed(model, inputs) 18 | atk = bb_attack(init_attack=dataset_atk, **kwargs) 19 | _, advs, success = atk(model, inputs, labels, epsilons=None) 20 | return advs 21 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/foolbox/wrapper.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from typing import Optional, Union 3 | 4 | import torch 5 | from foolbox import Misclassification, PyTorchModel, TargetedMisclassification 6 | from foolbox.attacks.base import FixedEpsilonAttack 7 | from torch import Tensor, nn 8 | 9 | 10 | def foolbox_wrapper(attack: partial, 11 | model: nn.Module, 12 | inputs: Tensor, 13 | labels: Tensor, 14 | targets: Optional[Tensor] = None, 15 | targeted: bool = False) -> Tensor: 16 | # check if attack is distance constrained, and extract epsilon 17 | epsilon_kwarg = {} 18 | attack_kwargs = attack.keywords.copy() 19 | if issubclass(attack.func, FixedEpsilonAttack): 20 | epsilon_kwarg['epsilon'] = attack_kwargs.pop('epsilon') 21 | 22 | fb_model = PyTorchModel(model=model, bounds=(0, 1)) 23 | fb_attack = partial(attack.func, **attack_kwargs)() 24 | 25 | if hasattr(fb_attack, 'feed'): 26 | fb_attack.feed(model=fb_model, inputs=inputs) 27 | 28 | if targeted: 29 | criterion = TargetedMisclassification(targets) 30 | else: 31 | criterion = Misclassification(labels) 32 | 33 | adv_inputs = fb_attack.run(model=fb_model, inputs=inputs, criterion=criterion, **epsilon_kwarg) 34 | return adv_inputs 35 | 36 | 37 | class FoolboxMinimalWrapper: 38 | _label_args = { 39 | Misclassification: 'labels', 40 | TargetedMisclassification: 'target_classes', 41 | } 42 | 43 | def __init__(self, attack: partial, init_eps: float, search_steps: int, max_eps: Optional[float] = None): 44 | self.attack = attack() 45 | self.init_eps = init_eps 46 | self.search_steps = search_steps 47 | self.max_eps = max_eps 48 | 49 | def run(self, model: PyTorchModel, inputs: Tensor, 50 | criterion: Union[TargetedMisclassification, Misclassification]) -> Tensor: 51 | 52 | # fetch labels / targets from criterion 53 | crit_class = criterion.__class__ 54 | label_arg = self._label_args[crit_class] 55 | labels = getattr(criterion, label_arg) 56 | 57 | adv_inputs = inputs.clone() 58 | eps_low = inputs.new_zeros(len(inputs)) 59 | best_eps = torch.full_like(eps_low, float('inf') if self.max_eps is None else 2 * self.max_eps) 60 | found_high = torch.full_like(eps_low, False, dtype=torch.bool) 61 | 62 | eps = torch.full_like(eps_low, self.init_eps) 63 | for i in range(self.search_steps): 64 | adv_inputs_run = inputs.clone() 65 | for eps_ in torch.unique(eps): 66 | mask = eps == eps_ 67 | crit_eps = crit_class(labels[mask]) 68 | adv_inputs_run[mask] = self.attack.run(model=model, inputs=inputs[mask], criterion=crit_eps, 69 | epsilon=eps_.item()) 70 | 71 | logits = model(adv_inputs_run) 72 | is_adv = criterion(adv_inputs_run, logits) 73 | 74 | better_adv = is_adv & (eps < best_eps) 75 | adv_inputs[better_adv] = adv_inputs_run[better_adv] 76 | 77 | found_high.logical_or_(better_adv) 78 | eps_low = torch.where(better_adv, eps_low, eps) 79 | best_eps = torch.where(better_adv, eps, best_eps) 80 | 81 | eps = torch.where(found_high | ((2 * eps_low) >= best_eps), (eps_low + best_eps) / 2, 2 * eps_low) 82 | 83 | return adv_inputs 84 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/ingredient.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import sys 3 | from collections import defaultdict 4 | from functools import partial 5 | from typing import Callable 6 | 7 | from sacred import Ingredient 8 | 9 | from .adv_lib import configs as adv_lib_configs 10 | from .art import configs as art_configs 11 | from .cleverhans import configs as cleverhans_configs 12 | from .deeprobust import configs as deeprobust_configs 13 | from .foolbox import configs as foolbox_configs 14 | from .original import configs as original_configs 15 | from .torchattacks import configs as torchattacks_configs 16 | 17 | attack_ingredient = Ingredient('attack') 18 | 19 | library_modules = { 20 | 'adv_lib': adv_lib_configs, 21 | 'art': art_configs, 22 | 'cleverhans': cleverhans_configs, 23 | 'deeprobust': deeprobust_configs, 24 | 'foolbox': foolbox_configs, 25 | 'original': original_configs, 26 | 'torchattacks': torchattacks_configs, 27 | } 28 | library_getters = defaultdict(dict) 29 | 30 | for module_name, module in library_modules.items(): 31 | # gather function defined in .configs modules 32 | module_funcs = inspect.getmembers(sys.modules[module.__name__], 33 | predicate=lambda f: inspect.isfunction(f) and f.__module__ == module.__name__) 34 | 35 | for name, func in module_funcs: # search for functions that are configs or getters 36 | config_prefix = module._prefix + '_' 37 | getter_prefix = 'get_' + config_prefix 38 | 39 | if name.startswith(config_prefix): # add function as named config 40 | attack_ingredient.named_config(func) 41 | 42 | elif name.startswith(getter_prefix): # capture function and add it to the getters 43 | library_getters[module_name][name.removeprefix(getter_prefix)] = attack_ingredient.capture(func) 44 | 45 | 46 | @attack_ingredient.capture 47 | def get_attack(source: str, name: str, threat_model: str) -> Callable: 48 | attack = library_getters[source][name]() 49 | wrapper = library_modules[source]._wrapper 50 | 51 | if isinstance(attack, dict): 52 | return partial(wrapper, **attack) 53 | else: 54 | return partial(wrapper, attack=attack) 55 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/original/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/original/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/original/deepfool.py: -------------------------------------------------------------------------------- 1 | # Adapted from https://github.com/LTS4/DeepFool/blob/master/Python/deepfool.py 2 | import copy 3 | from typing import Optional 4 | 5 | import numpy as np 6 | import torch as torch 7 | from torch import Tensor, nn 8 | from torch.autograd import Variable 9 | 10 | 11 | def deepfool(image, net, num_classes=10, overshoot=0.02, max_iter=50): 12 | """ 13 | :param image: Image of size HxWx3 14 | :param net: network (input: images, output: values of activation **BEFORE** softmax). 15 | :param num_classes: num_classes (limits the number of classes to test against, by default = 10) 16 | :param overshoot: used as a termination criterion to prevent vanishing updates (default = 0.02). 17 | :param max_iter: maximum number of iterations for deepfool (default = 50) 18 | :return: minimal perturbation that fools the classifier, number of iterations that it required, new estimated_label and perturbed image 19 | """ 20 | 21 | # COMMENTED 22 | # is_cuda = torch.cuda.is_available() 23 | # 24 | # if is_cuda: 25 | # print("Using GPU") 26 | # image = image.cuda() 27 | # net = net.cuda() 28 | # else: 29 | # print("Using CPU") 30 | device = image.device 31 | 32 | f_image = net.forward(Variable(image[None, :, :, :], requires_grad=True)).data.cpu().numpy().flatten() 33 | I = (np.array(f_image)).flatten().argsort()[::-1] 34 | 35 | I = I[0:num_classes] 36 | label = I[0] 37 | 38 | input_shape = image.cpu().numpy().shape 39 | pert_image = copy.deepcopy(image) 40 | w = np.zeros(input_shape) 41 | r_tot = np.zeros(input_shape) 42 | 43 | loop_i = 0 44 | 45 | x = Variable(pert_image[None, :], requires_grad=True) 46 | fs = net.forward(x) 47 | fs_list = [fs[0, I[k]] for k in range(num_classes)] 48 | k_i = label 49 | 50 | while k_i == label and loop_i < max_iter: 51 | 52 | pert = np.inf 53 | fs[0, I[0]].backward(retain_graph=True) 54 | grad_orig = x.grad.data.cpu().numpy().copy() 55 | 56 | for k in range(1, num_classes): 57 | x.grad = None # MODIFIED: zero_gradients(x) 58 | 59 | fs[0, I[k]].backward(retain_graph=True) 60 | cur_grad = x.grad.data.cpu().numpy().copy() 61 | 62 | # set new w_k and new f_k 63 | w_k = cur_grad - grad_orig 64 | f_k = (fs[0, I[k]] - fs[0, I[0]]).data.cpu().numpy() 65 | 66 | pert_k = abs(f_k) / np.linalg.norm(w_k.flatten()) 67 | 68 | # determine which w_k to use 69 | if pert_k < pert: 70 | pert = pert_k 71 | w = w_k 72 | 73 | # compute r_i and r_tot 74 | # Added 1e-4 for numerical stability 75 | r_i = (pert + 1e-4) * w / np.linalg.norm(w) 76 | r_tot = np.float32(r_tot + r_i) 77 | 78 | pert_image = image + (1 + overshoot) * torch.from_numpy(r_tot).to(device) # MODIFIED:... 79 | # if is_cuda: 80 | # pert_image = image + (1 + overshoot) * torch.from_numpy(r_tot).cuda() 81 | # else: 82 | # pert_image = image + (1 + overshoot) * torch.from_numpy(r_tot) 83 | 84 | x = Variable(pert_image, requires_grad=True) 85 | fs = net.forward(x) 86 | k_i = np.argmax(fs.data.cpu().numpy().flatten()) 87 | 88 | loop_i += 1 89 | 90 | r_tot = (1 + overshoot) * r_tot 91 | 92 | return r_tot, loop_i, label, k_i, pert_image 93 | 94 | 95 | def deepfool_attack(model: nn.Module, 96 | inputs: Tensor, 97 | labels: Tensor, 98 | targets: Optional[Tensor] = None, 99 | targeted: bool = False, **kwargs) -> Tensor: 100 | adv_inputs = [] 101 | for input in inputs: 102 | adv_inputs.append(deepfool(image=input, net=model, **kwargs)[-1]) 103 | return torch.cat(adv_inputs, dim=0) 104 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/original/sigma_zero.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | import torch.optim.lr_scheduler as lr_scheduler 5 | from adv_lib.utils.losses import difference_of_logits 6 | from foolbox import PyTorchModel 7 | from foolbox.attacks.dataset_attack import DatasetAttack 8 | from torch import Tensor 9 | 10 | 11 | def l0_mid_points( 12 | x0: Tensor, 13 | x1: Tensor, 14 | ε: Tensor, 15 | bounds, 16 | ): 17 | # returns a point between x0 and x1 where 18 | # epsilon = 0 returns x0 and epsilon = 1 19 | # returns x1 20 | 21 | # get ε in right shape for broadcasting 22 | ε = ε.reshape(ε.shape + (1,) * (x0.ndim - 1)) 23 | 24 | threshold = (bounds[1] - bounds[0]) * ε 25 | mask = (x1 - x0).abs() < threshold 26 | new_x = torch.where(mask, x1, x0) 27 | return new_x 28 | 29 | 30 | def delta_init(model, inputs, labels, device, starting_point=None, binary_search_steps=10, targeted=False): 31 | batch_size, max_size = inputs.shape[0], torch.prod(torch.tensor(inputs.shape[1:])) 32 | 33 | if starting_point is None: 34 | delta = torch.zeros_like(inputs, requires_grad=True, device=device) 35 | elif starting_point == 'adversarial': 36 | fmodel = PyTorchModel(model, bounds=(0, 1)) 37 | dataset_atk = DatasetAttack() 38 | 39 | dataset_atk.feed(fmodel, inputs) 40 | _, starting_points, success = dataset_atk(fmodel, inputs, labels, epsilons=None) 41 | is_adv = model(starting_points).argmax(dim=1) != labels 42 | if not is_adv.all(): 43 | raise ValueError('Starting points are not all adversarial.') 44 | lower_bound = torch.zeros(batch_size, device=device) 45 | upper_bound = torch.ones(batch_size, device=device) 46 | for _ in range(binary_search_steps): 47 | ε = (lower_bound + upper_bound) / 2 48 | mid_points = l0_mid_points(x0=inputs, x1=starting_points, ε=ε, bounds=[0, 1]) 49 | mid_points = mid_points.reshape(inputs.shape) 50 | pred_labels = model(mid_points).argmax(dim=1) 51 | is_adv = (pred_labels == labels) if targeted else (pred_labels != labels) 52 | lower_bound = torch.where(is_adv, lower_bound, ε) 53 | upper_bound = torch.where(is_adv, ε, upper_bound) 54 | 55 | mid_points = l0_mid_points(x0=inputs, x1=starting_points, ε=upper_bound, bounds=[0, 1]) 56 | mid_points = mid_points.reshape(inputs.shape) 57 | delta = mid_points - inputs 58 | delta.requires_grad_() 59 | else: 60 | raise "Not implemented error, only None or 'adversarial' are considered" 61 | 62 | return delta 63 | 64 | 65 | def sigma_zero(model: torch.nn.Module, 66 | inputs: Tensor, 67 | labels: Tensor, 68 | steps: int = 1000, 69 | lr: float = 1.0, 70 | sigma: float = 1e-3, 71 | thr_0: float = 0.3, 72 | thr_lr: float = 0.01, 73 | verbose: bool = False, 74 | starting_point=None, 75 | binary_search_steps: int = 10, 76 | targets: Optional[Tensor] = None, 77 | targeted: bool = False, 78 | grad_norm=torch.inf) -> Tensor: 79 | clamp = lambda tensor: tensor.data.add_(inputs.data).clamp_(min=0, max=1).sub_(inputs.data) 80 | l0_approximation = lambda tensor, sigma: tensor.square().div(tensor.square().add(sigma)).sum(dim=1) 81 | batch_view = lambda tensor: tensor.view(batch_size, *[1] * (inputs.ndim - 1)) 82 | normalize = lambda tensor: ( 83 | tensor.flatten(1) / tensor.flatten(1).norm(p=grad_norm, dim=1, keepdim=True).clamp_(min=1e-12)).view( 84 | tensor.shape) 85 | 86 | device = next(model.parameters()).device 87 | batch_size, max_size = inputs.shape[0], torch.prod(torch.tensor(inputs.shape[1:])) 88 | 89 | delta = delta_init(model, inputs, labels, device, starting_point=starting_point, 90 | binary_search_steps=binary_search_steps) 91 | optimizer = torch.optim.Adam([delta], lr=lr) 92 | scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=steps, eta_min=lr / 10) 93 | best_l0 = torch.full((batch_size,), max_size, device=device) 94 | best_delta = delta.clone() 95 | 96 | # th = threshold * torch.ones(size=(batch_size,)) 97 | th = torch.ones(size=inputs.shape, device=device) * thr_0 98 | 99 | for i in range(steps): 100 | adv_inputs = inputs + delta 101 | 102 | # compute loss 103 | logits = model(adv_inputs) 104 | dl_loss = difference_of_logits(logits, labels).clip(0) 105 | l0_approx = l0_approximation(delta.flatten(1), sigma) 106 | l0_approx_normalized = l0_approx / delta.data.flatten(1).shape[1] 107 | # keep best solutions 108 | predicted_classes = (logits).argmax(1) 109 | true_l0 = delta.data.flatten(1).ne(0).sum(dim=1) 110 | is_not_adv = (predicted_classes == labels) 111 | is_adv = (predicted_classes != labels) 112 | is_smaller = (true_l0 < best_l0) 113 | is_both = is_adv & is_smaller 114 | best_l0 = torch.where(is_both, true_l0.detach(), best_l0) 115 | best_delta = torch.where(batch_view(is_both), delta.data.clone().detach(), best_delta) 116 | 117 | # update step 118 | adv_loss = (is_not_adv + dl_loss + l0_approx_normalized).mean() 119 | 120 | if verbose and i % 100 == 0: 121 | print(th.flatten(1).mean(dim=1), th.flatten(1).mean(dim=1).shape) 122 | print(is_not_adv) 123 | print( 124 | f"iter: {i}, dl loss: {dl_loss.mean().item():.4f}, l0 normalized loss: {l0_approx_normalized.mean().item():.4f}, current median norm: {delta.data.flatten(1).ne(0).sum(dim=1).median()}") 125 | 126 | optimizer.zero_grad() 127 | adv_loss.backward() 128 | delta.grad.data = normalize(delta.grad.data) 129 | optimizer.step() 130 | scheduler.step() 131 | clamp(delta.data) 132 | 133 | # dynamic thresholding 134 | th[is_not_adv, :, :, :] -= thr_lr * scheduler.get_last_lr()[0] 135 | th[~is_not_adv, :, :, :] += thr_lr * scheduler.get_last_lr()[0] 136 | th.clamp_(0, 1) 137 | 138 | # filter components 139 | delta.data[delta.data.abs() < th] = 0 140 | 141 | return (inputs + best_delta) 142 | -------------------------------------------------------------------------------- /attack_evaluation/attacks/torchattacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/attacks/torchattacks/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/attacks/torchattacks/wrapper.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from typing import Optional 3 | 4 | import torch 5 | from torch import Tensor, nn 6 | from torchattacks.attack import Attack 7 | 8 | 9 | def torchattacks_wrapper(attack: Attack, 10 | model: nn.Module, 11 | inputs: Tensor, 12 | labels: Tensor, 13 | targets: Optional[Tensor] = None, 14 | targeted: bool = False) -> Tensor: 15 | attack = attack(model=model) 16 | if targeted: 17 | attack.set_mode_targeted_by_function() 18 | adv_examples = attack(inputs, labels=targets if targeted else labels) 19 | return adv_examples 20 | 21 | 22 | class TorchattacksMinimalWrapper: 23 | def __init__(self, model: nn.Module, attack: partial, init_eps: float, search_steps: int, 24 | max_eps: Optional[float] = None, batched: bool = False): 25 | self.attack = attack(model=model) 26 | self.init_eps = init_eps 27 | self.search_steps = search_steps 28 | self.max_eps = max_eps 29 | self.batched = batched 30 | self.targeted = False 31 | 32 | def set_mode_targeted_by_function(self): 33 | self.attack.set_mode_targeted_by_function() 34 | self.targeted = True 35 | 36 | def __call__(self, inputs: Tensor, labels: Tensor) -> Tensor: 37 | batch_size = len(inputs) 38 | batch_view = lambda tensor: tensor.view(batch_size, *[1] * (inputs.ndim - 1)) 39 | adv_inputs = inputs.clone() 40 | eps_low = inputs.new_zeros(batch_size) 41 | best_eps = torch.full_like(eps_low, float('inf') if self.max_eps is None else 2 * self.max_eps) 42 | found_high = torch.full_like(eps_low, False, dtype=torch.bool) 43 | 44 | eps = torch.full_like(eps_low, self.init_eps) 45 | for i in range(self.search_steps): 46 | if self.batched: 47 | self.attack.eps = batch_view(eps) 48 | adv_inputs_run = self.attack(inputs, labels=labels) 49 | else: 50 | adv_inputs_run = inputs.clone() 51 | for eps_ in torch.unique(eps): 52 | mask = eps == eps_ 53 | self.attack.eps = eps_.item() 54 | adv_inputs_run[mask] = self.attack(inputs[mask], labels=labels[mask]) 55 | 56 | logits = self.attack.model(adv_inputs_run) 57 | preds = logits.argmax(dim=1) 58 | is_adv = (preds == labels) if self.targeted else (preds != labels) 59 | 60 | better_adv = is_adv & (eps < best_eps) 61 | adv_inputs[better_adv] = adv_inputs_run[better_adv] 62 | 63 | found_high.logical_or_(better_adv) 64 | eps_low = torch.where(better_adv, eps_low, eps) 65 | best_eps = torch.where(better_adv, eps, best_eps) 66 | 67 | eps = torch.where(found_high | ((2 * eps_low) >= best_eps), (eps_low + best_eps) / 2, 2 * eps_low) 68 | 69 | return adv_inputs 70 | -------------------------------------------------------------------------------- /attack_evaluation/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/datasets/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/datasets/imagenet.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from pathlib import Path 4 | from typing import Optional, Union 5 | 6 | import numpy as np 7 | from PIL import Image 8 | from torch.utils.data import Dataset, Subset 9 | 10 | from . import subsets 11 | 12 | 13 | class ImageNetKaggle(Dataset): 14 | def __init__(self, root, split='val', transform=None): 15 | self.samples = [] 16 | self.targets = [] 17 | self.transform = transform 18 | self.syn_to_class = {} 19 | self.split = split 20 | 21 | with open(os.path.join(root, "imagenet_class_index.json"), "rb") as f: 22 | json_file = json.load(f) 23 | for class_id, v in json_file.items(): 24 | self.syn_to_class[v[0]] = int(class_id) 25 | with open(os.path.join(root, "ILSVRC2012_val_labels.json"), "rb") as f: 26 | self.val_to_syn = json.load(f) 27 | samples_dir = os.path.join(root, "ILSVRC/Data/CLS-LOC", split) 28 | self.samples_lst = os.listdir(samples_dir) 29 | 30 | for entry in self.samples_lst: 31 | if split == "train": 32 | syn_id = entry 33 | target = self.syn_to_class[syn_id] 34 | syn_folder = os.path.join(samples_dir, syn_id) 35 | for sample in os.listdir(syn_folder): 36 | sample_path = os.path.join(syn_folder, sample) 37 | self.samples.append(sample_path) 38 | self.targets.append(target) 39 | elif split == "val": 40 | syn_id = self.val_to_syn[entry] 41 | target = self.syn_to_class[syn_id] 42 | sample_path = os.path.join(samples_dir, entry) 43 | self.samples.append(sample_path) 44 | self.targets.append(target) 45 | 46 | def __len__(self): 47 | return len(self.samples) 48 | 49 | def __getitem__(self, idx): 50 | x = Image.open(self.samples[idx]).convert("RGB") 51 | if self.transform: 52 | x = self.transform(x) 53 | return x, self.targets[idx] 54 | 55 | 56 | def prepare_imagenet_subset(root, split='val', n_samples: int = 5000): 57 | samples_dir = os.path.join(root, "ILSVRC/Data/CLS-LOC", split) 58 | data_list = np.array(os.listdir(samples_dir)) 59 | 60 | np.random.seed(0) 61 | subset = np.random.choice(data_list, replace=False, size=n_samples) 62 | subset_dir = Path(os.path.dirname(subsets.__file__)) 63 | np.savetxt(subset_dir / f'imagenet-{n_samples}-{split}.txt', subset, fmt='%s') 64 | 65 | 66 | def load_imagenet(root: Union[str, Path], 67 | split: str = 'val', 68 | transform=None, 69 | n_samples: Optional[int] = 5000) -> Dataset: 70 | data = ImageNetKaggle(root=root, split='val', transform=transform) 71 | if n_samples is None: 72 | return data 73 | 74 | subset_names_file = Path(os.path.dirname(subsets.__file__)) / f'imagenet-{n_samples}-{split}.txt' 75 | if not subset_names_file.exists(): 76 | prepare_imagenet_subset(root, split=data.split, n_samples=n_samples) 77 | subset_names = np.loadtxt(subset_names_file, dtype=str) 78 | subset_indices = [i for i, file in enumerate(data.samples_lst) if file in subset_names] 79 | return Subset(dataset=data, indices=subset_indices) 80 | -------------------------------------------------------------------------------- /attack_evaluation/datasets/ingredient.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Optional 3 | 4 | import numpy as np 5 | from sacred import Ingredient 6 | from torch.utils.data import DataLoader, Dataset, Subset 7 | from torchvision import transforms 8 | from torchvision.datasets import CIFAR10, MNIST 9 | 10 | from .imagenet import load_imagenet 11 | 12 | dataset_ingredient = Ingredient('dataset') 13 | 14 | 15 | @dataset_ingredient.config 16 | def config(): 17 | root = 'data' 18 | num_samples = None # number of samples to attack, None for all 19 | random_subset = True # True for random subset. False for sequential data in Dataset 20 | batch_size = 128 21 | 22 | 23 | @dataset_ingredient.capture 24 | def get_mnist(root: str) -> Dataset: 25 | transform = transforms.ToTensor() 26 | dataset = MNIST(root=root, train=False, transform=transform, download=True) 27 | return dataset 28 | 29 | 30 | @dataset_ingredient.capture 31 | def get_cifar10(root: str) -> Dataset: 32 | transform = transforms.ToTensor() 33 | dataset = CIFAR10(root=root, train=False, transform=transform, download=True) 34 | return dataset 35 | 36 | 37 | @dataset_ingredient.capture 38 | def get_imagenet(root: str, num_samples: Optional[int] = None) -> Dataset: 39 | transform = transforms.Compose([ 40 | transforms.Resize(256), 41 | transforms.CenterCrop(224), 42 | transforms.ToTensor() 43 | ]) 44 | data_path = Path(root) / 'imagenet-data' 45 | dataset = load_imagenet(root=data_path, split='val', transform=transform, n_samples=num_samples) 46 | print("Dataset size: ", len(dataset)) 47 | return dataset 48 | 49 | 50 | _datasets = { 51 | 'mnist': get_mnist, 52 | 'cifar10': get_cifar10, 53 | 'imagenet': get_imagenet, 54 | } 55 | 56 | 57 | @dataset_ingredient.capture 58 | def get_dataset(dataset: str): 59 | return _datasets[dataset]() 60 | 61 | 62 | @dataset_ingredient.capture 63 | def get_loader(dataset: str, batch_size: int, num_samples: Optional[int] = None, 64 | random_subset: bool = True) -> DataLoader: 65 | data = get_dataset(dataset=dataset) 66 | 67 | if num_samples is not None and num_samples < len(data): 68 | if not random_subset: 69 | data = Subset(data, indices=list(range(num_samples))) 70 | else: 71 | indices = np.random.choice(len(data), replace=False, size=num_samples) 72 | data = Subset(data, indices=indices) 73 | loader = DataLoader(dataset=data, batch_size=batch_size) 74 | return loader 75 | -------------------------------------------------------------------------------- /attack_evaluation/datasets/subsets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/datasets/subsets/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/experiments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #python job_runner.py --model='carmon_2019' # cifar10 4 | #python job_runner.py --model='wideresnet_28_10' # cifar10 5 | #python job_runner.py --dataset="mnist" --model='mnist_smallcnn' # mnist 6 | 7 | # Test attacks with their default parameters on a subset of the test set (1000) on standard model. 8 | # Note: num_steps is the only parameter changed to have fair comparison between different implementation of the 9 | # same attack algorithm. 10 | python job_runner.py --exp_dir=results5000 --threat_model=l2 --model=standard --batch_size=128 --num_samples=1000 --seed=4444 11 | python job_runner.py --exp_dir=results5000 --threat_model=linf --model=standard --batch_size=128 --num_samples=1000 --seed=4444 12 | python job_runner.py --exp_dir=results1000 --threat_model=l0 --model=standard --batch_size=128 --num_samples=1000 --seed=4444 13 | python job_runner.py --exp_dir=results1000 --threat_model=l1 --model=standard --batch_size=128 --num_samples=1000 --seed=4444 14 | 15 | python job_runner.py --exp_dir=results5000 --threat_model=l2 --model=standard --batch_size=1 --num_samples=1000 --seed=4444 16 | python job_runner.py --exp_dir=results5000 --threat_model=linf --model=standard --batch_size=1 --num_samples=1000 --seed=4444 17 | python job_runner.py --exp_dir=results1000 --threat_model=l0 --model=standard --batch_size=1 --num_samples=1000 --seed=4444 18 | python job_runner.py --exp_dir=results1000 --threat_model=l1 --model=standard --batch_size=1 --num_samples=1000 --seed=4444 19 | 20 | # Test attacks with their default parameters on a subset of the test set (1000) on carmon model. 21 | # Note: num_steps is the only parameter changed to have fair comparison between different implementation of the 22 | # same attack algorithm. 23 | python job_runner.py --exp_dir=results5000 --threat_model=l2 --model=carmon_2019 --batch_size=128 --num_samples=1000 --seed=4444 24 | python job_runner.py --exp_dir=results5000 --threat_model=linf --model=carmon_2019 --batch_size=128 --num_samples=1000 --seed=4444 25 | python job_runner.py --exp_dir=results1000 --threat_model=l0 --model=carmon_2019 --batch_size=128 --num_samples=1000 --seed=4444 26 | python job_runner.py --exp_dir=results1000 --threat_model=l1 --model=carmon_2019 --batch_size=128 --num_samples=1000 --seed=4444 27 | 28 | python job_runner.py --exp_dir=results5000 --threat_model=l2 --model=carmon_2019 --batch_size=1 --num_samples=1000 --seed=4444 29 | python job_runner.py --exp_dir=results5000 --threat_model=linf --model=carmon_2019 --batch_size=1 --num_samples=1000 --seed=4444 30 | python job_runner.py --exp_dir=results1000 --threat_model=l0 --model=carmon_2019 --batch_size=1 --num_samples=1000 --seed=4444 31 | python job_runner.py --exp_dir=results1000 --threat_model=l1 --model=carmon_2019 --batch_size=1 --num_samples=1000 --seed=4444 -------------------------------------------------------------------------------- /attack_evaluation/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/models/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/models/mnist.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from torch import nn 4 | 5 | 6 | class SmallCNN(nn.Module): 7 | def __init__(self, drop=0.5): 8 | super(SmallCNN, self).__init__() 9 | 10 | self.num_channels = 1 11 | self.num_labels = 10 12 | 13 | activ = nn.ReLU(True) 14 | 15 | self.feature_extractor = nn.Sequential(OrderedDict([ 16 | ('conv1', nn.Conv2d(self.num_channels, 32, 3)), 17 | ('relu1', activ), 18 | ('conv2', nn.Conv2d(32, 32, 3)), 19 | ('relu2', activ), 20 | ('maxpool1', nn.MaxPool2d(2, 2)), 21 | ('conv3', nn.Conv2d(32, 64, 3)), 22 | ('relu3', activ), 23 | ('conv4', nn.Conv2d(64, 64, 3)), 24 | ('relu4', activ), 25 | ('maxpool2', nn.MaxPool2d(2, 2)), 26 | ])) 27 | 28 | self.classifier = nn.Sequential(OrderedDict([ 29 | ('fc1', nn.Linear(64 * 4 * 4, 200)), 30 | ('relu1', activ), 31 | ('drop', nn.Dropout(drop)), 32 | ('fc2', nn.Linear(200, 200)), 33 | ('relu2', activ), 34 | ('fc3', nn.Linear(200, self.num_labels)), 35 | ])) 36 | 37 | for m in self.modules(): 38 | if isinstance(m, (nn.Conv2d)): 39 | nn.init.kaiming_normal_(m.weight) 40 | if m.bias is not None: 41 | nn.init.constant_(m.bias, 0) 42 | elif isinstance(m, nn.BatchNorm2d): 43 | nn.init.constant_(m.weight, 1) 44 | nn.init.constant_(m.bias, 0) 45 | nn.init.constant_(self.classifier.fc3.weight, 0) 46 | nn.init.constant_(self.classifier.fc3.bias, 0) 47 | 48 | def forward(self, input): 49 | features = self.feature_extractor(input) 50 | logits = self.classifier(features.view(-1, 64 * 4 * 4)) 51 | return logits 52 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/models/original/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/models/original/stutz2020/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/models/original/stutz2020/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/models/original/stutz2020/ccat.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code adapted from: https://github.com/davidstutz/confidence-calibrated-adversarial-training 3 | """ 4 | 5 | import wget 6 | import zipfile 7 | import os 8 | import torch 9 | from .resnet import ResNet 10 | 11 | _pretrained_model_names = {'Stutz2020CCAT': 'ccat'} 12 | 13 | _model_name = 'stutz_2020_ccat.pth.tar' 14 | 15 | 16 | def download_model(dataset='cifar10', model='stutz_2020'): 17 | model_name = _pretrained_model_names[model] 18 | 19 | url = 'https://datasets.d2.mpi-inf.mpg.de/arxiv2019-ccat/%s_%s.zip' % (dataset, model_name) 20 | filename = wget.download(url) 21 | 22 | # Directory to extract the model to. 23 | model_dir = './models/checkpoints/' 24 | with zipfile.ZipFile(filename, 'r') as zip_ref: 25 | zip_ref.extractall(model_dir) 26 | os.remove(filename) 27 | os.rename(model_dir + zip_ref.infolist()[0].filename, model_dir + _model_name) 28 | 29 | 30 | def load_ccat_model(dataset='cifar10', model='stutz2020', threat_model='Linf'): 31 | assert dataset in ['cifar10'] # , 'svhn', 'mnist'] 32 | # assert model in ['normal', 'at', 'stutz_2020', 'msd'] 33 | 34 | model_file = 'models/checkpoints/%s' % _model_name 35 | if not os.path.exists(model_file): 36 | download_model(dataset, model) 37 | assert os.path.exists(model_file) 38 | 39 | checkpoint = torch.load(model_file, map_location=lambda storage, loc: storage) 40 | arguments = {**checkpoint['arguments'], **checkpoint['kwargs']} 41 | model = ResNet(**arguments) 42 | model.load_state_dict(checkpoint['model']) 43 | return model 44 | 45 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/stutz2020/classifier.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from .resnet_block import ResNetBlock 3 | from .torch import Clamp, Normalize, Scale 4 | 5 | 6 | class Classifier(torch.nn.Module): 7 | """ 8 | Base classifier. 9 | """ 10 | 11 | def __init__(self, N_class, resolution, **kwargs): 12 | """ 13 | Initialize classifier. 14 | The keyword arguments, resolution, number of classes and other architecture parameters 15 | from subclasses are saved as attributes. This allows to easily save and load the model 16 | using common.state without knowing the exact architecture in advance. 17 | :param N_class: number of classes to classify 18 | :type N_class: int 19 | :param resolution: resolution 20 | :type resolution: [int] 21 | """ 22 | 23 | super(Classifier, self).__init__() 24 | 25 | assert N_class > 0, 'positive N_class expected' 26 | assert len(resolution) <= 3 27 | 28 | self.N_class = int( 29 | N_class) # Having strange bug where torch complaints about numpy.in64 being passed to nn.Linear. 30 | """ (int) Number of classes. """ 31 | 32 | self.resolution = list(resolution) 33 | """ ([int]) Resolution as (channels, height, width) """ 34 | 35 | self.kwargs = kwargs 36 | """ (dict) Kwargs. """ 37 | 38 | self.include_clamp = self.kwargs_get('clamp', True) 39 | """ (bool) Whether to apply input clamping. """ 40 | 41 | self.include_whiten = self.kwargs_get('whiten', False) 42 | """ (bool) Whether to apply input whitening/normalization. """ 43 | 44 | self.include_scale = self.kwargs_get('scale', False) 45 | """ (bool) Whether to apply input scaling. """ 46 | 47 | # __ attributes are private, which is important for the State to work properly. 48 | self.__layers = [] 49 | """ ([str]) Will hold layer names. """ 50 | 51 | self._N_output = self.N_class if self.N_class > 2 else 1 52 | """ (int) Number of outputs. """ 53 | 54 | if self.include_clamp: 55 | self.append_layer('clamp', Clamp()) 56 | 57 | assert not (self.include_whiten and self.include_scale) 58 | 59 | if self.include_whiten: 60 | # Note that the weight and bias needs to set manually corresponding to mean and std! 61 | whiten = Normalize(resolution[0]) 62 | self.append_layer('whiten', whiten) 63 | 64 | if self.include_scale: 65 | # Note that the weight and bias needs to set manually! 66 | scale = Scale(1) 67 | scale.weight.data[0] = -1 68 | scale.bias.data[0] = 1 69 | self.append_layer('scale', scale) 70 | 71 | def kwargs_get(self, key, default): 72 | """ 73 | Get argument if not None. 74 | :param key: key 75 | :type key: str 76 | :param default: default value 77 | :type default: mixed 78 | :return: value 79 | :rtype: mixed 80 | """ 81 | 82 | value = self.kwargs.get(key, default) 83 | if value is None: 84 | value = default 85 | return value 86 | 87 | def append_layer(self, name, layer): 88 | """ 89 | Add a layer. 90 | :param name: layer name 91 | :type name: str 92 | :param layer: layer 93 | :type layer: torch.nn.Module 94 | """ 95 | 96 | setattr(self, name, layer) 97 | self.__layers.append(name) 98 | 99 | def prepend_layer(self, name, layer): 100 | """ 101 | Add a layer. 102 | :param name: layer name 103 | :type name: str 104 | :param layer: layer 105 | :type layer: torch.nn.Module 106 | """ 107 | 108 | self.insert_layer(0, name, layer) 109 | 110 | def insert_layer(self, index, name, layer): 111 | """ 112 | Add a layer. 113 | :param index: index 114 | :type index: int 115 | :param name: layer name 116 | :type name: str 117 | :param layer: layer 118 | :type layer: torch.nn.Module 119 | """ 120 | 121 | setattr(self, name, layer) 122 | self.__layers.insert(index, name) 123 | 124 | def forward(self, image, return_features=False): 125 | """ 126 | Forward pass, takes an image and outputs the predictions. 127 | :param image: input image 128 | :type image: torch.autograd.Variable 129 | :param return_features: whether to also return representation layer 130 | :type return_features: bool 131 | :return: logits 132 | :rtype: torch.autograd.Variable 133 | """ 134 | 135 | features = [] 136 | output = image 137 | 138 | # separate loops for memory constraints 139 | if return_features: 140 | for name in self.__layers: 141 | output = getattr(self, name)(output) 142 | features.append(output) 143 | return output, features 144 | else: 145 | for name in self.__layers: 146 | output = getattr(self, name)(output) 147 | return output 148 | 149 | def layers(self): 150 | """ 151 | Get layer names. 152 | :return: layer names 153 | :rtype: [str] 154 | """ 155 | 156 | return self.__layers 157 | 158 | def __str__(self): 159 | """ 160 | Print network. 161 | """ 162 | 163 | string = '' 164 | for name in self.__layers: 165 | string += '(' + name + ', ' + getattr(self, name).__class__.__name__ + ')\n' 166 | if isinstance(getattr(self, name), torch.nn.Sequential) or isinstance(getattr(self, name), ResNetBlock): 167 | for module in getattr(self, name).modules(): 168 | string += '\t(' + module.__class__.__name__ + ')\n' 169 | return string 170 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/stutz2020/resnet.py: -------------------------------------------------------------------------------- 1 | """ 2 | ResNet. 3 | Take from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py. 4 | """ 5 | import torch 6 | from .torch import View 7 | from .classifier import Classifier 8 | from .resnet_block import ResNetBlock 9 | 10 | class ResNet(Classifier): 11 | """ 12 | Simple classifier. 13 | """ 14 | 15 | def __init__(self, N_class, resolution=(1, 32, 32), blocks=[3, 3, 3], normalization=True, channels=64, **kwargs): 16 | """ 17 | Initialize classifier. 18 | :param N_class: number of classes to classify 19 | :type N_class: int 20 | :param resolution: resolution (assumed to be square) 21 | :type resolution: int 22 | :param blocks: layers per block 23 | :type blocks: [int] 24 | :param normalization: normalization to use 25 | :type normalization: None or torch.nn.Module 26 | :param channels: channels to start with 27 | :type channels: int 28 | """ 29 | 30 | super(ResNet, self).__init__(N_class, resolution, **kwargs) 31 | 32 | self.blocks = blocks 33 | """ ([int]) Blocks. """ 34 | 35 | self.channels = channels 36 | """ (int) Channels. """ 37 | 38 | self.normalization = normalization 39 | """ (callable) Normalization. """ 40 | 41 | self.inplace = False 42 | """ (bool) Inplace. """ 43 | 44 | conv1 = torch.nn.Conv2d(self.resolution[0], self.channels, kernel_size=3, stride=1, padding=1, bias=False) 45 | torch.nn.init.kaiming_normal_(conv1.weight, mode='fan_out', nonlinearity='relu') 46 | self.append_layer('conv1', conv1) 47 | 48 | if self.normalization: 49 | norm1 = torch.nn.BatchNorm2d(self.channels) 50 | torch.nn.init.constant_(norm1.weight, 1) 51 | torch.nn.init.constant_(norm1.bias, 0) 52 | self.append_layer('norm1', norm1) 53 | 54 | relu = torch.nn.ReLU(inplace=self.inplace) 55 | self.append_layer('relu1', relu) 56 | 57 | downsampled = 1 58 | for i in range(len(self.blocks)): 59 | in_planes = (2 ** max(0, i - 1)) * self.channels 60 | out_planes = (2 ** i) * self.channels 61 | layers = self.blocks[i] 62 | stride = 2 if i > 0 else 1 63 | 64 | downsample = None 65 | if stride != 1 or in_planes != out_planes: 66 | conv = torch.nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) 67 | torch.nn.init.kaiming_normal_(conv.weight, mode='fan_out', nonlinearity='relu') 68 | 69 | if self.normalization: 70 | bn = torch.nn.BatchNorm2d(out_planes) 71 | torch.nn.init.constant_(bn.weight, 1) 72 | torch.nn.init.constant_(bn.bias, 0) 73 | downsample = torch.nn.Sequential(*[conv, bn]) 74 | else: 75 | downsample = torch.nn.Sequential(*[conv]) 76 | 77 | sequence = [] 78 | sequence.append(ResNetBlock(in_planes, out_planes, stride=stride, downsample=downsample, normalization=self.normalization)) 79 | for _ in range(1, layers): 80 | sequence.append(ResNetBlock(out_planes, out_planes, stride=1, downsample=None, normalization=self.normalization)) 81 | 82 | self.append_layer('block%d' % i, torch.nn.Sequential(*sequence)) 83 | downsampled *= stride 84 | 85 | representation = out_planes 86 | pool = torch.nn.AvgPool2d((self.resolution[1] // downsampled, self.resolution[2] // downsampled), stride=1) 87 | self.append_layer('avgpool', pool) 88 | 89 | view = View(-1, representation) 90 | self.append_layer('view', view) 91 | 92 | gain = torch.nn.init.calculate_gain('relu') 93 | logits = torch.nn.Linear(representation, self._N_output) 94 | torch.nn.init.kaiming_normal_(logits.weight, gain) 95 | torch.nn.init.constant_(logits.bias, 0) 96 | self.append_layer('logits', logits) -------------------------------------------------------------------------------- /attack_evaluation/models/original/stutz2020/resnet_block.py: -------------------------------------------------------------------------------- 1 | """ 2 | ResNet block. 3 | Take from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py. 4 | """ 5 | import torch 6 | 7 | class ResNetBlock(torch.nn.Module): 8 | """ 9 | ResNet block. 10 | """ 11 | 12 | def __init__(self, inplanes, planes, stride=1, downsample=None, normalization=True): 13 | """ 14 | Constructor. 15 | :param inplanes: input channels 16 | :type inplanes: int 17 | :param planes: output channels 18 | :type planes: int 19 | :param stride: stride 20 | :type stride: int 21 | :param downsample: whether to downsample 22 | :type downsample: bool 23 | :param normalization: whether to use normalization 24 | :type normalization: bool 25 | """ 26 | 27 | super(ResNetBlock, self).__init__() 28 | 29 | self.inplace = False 30 | """ (bool) Inplace. """ 31 | 32 | self.conv1 = torch.nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 33 | torch.nn.init.kaiming_normal_(self.conv1.weight, mode='fan_out', nonlinearity='relu') 34 | 35 | self.normalization = normalization 36 | if self.normalization: 37 | self.norm1 = torch.nn.BatchNorm2d(planes) 38 | torch.nn.init.constant_(self.norm1.weight, 1) 39 | torch.nn.init.constant_(self.norm1.bias, 0) 40 | 41 | self.relu = torch.nn.ReLU(inplace=self.inplace) 42 | self.conv2 = torch.nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 43 | torch.nn.init.kaiming_normal_(self.conv2.weight, mode='fan_out', nonlinearity='relu') 44 | 45 | if self.normalization: 46 | self.norm2 = torch.nn.BatchNorm2d(planes) 47 | torch.nn.init.constant_(self.norm2.weight, 1) 48 | torch.nn.init.constant_(self.norm2.bias, 0) 49 | 50 | self.downsample = downsample 51 | self.stride = stride 52 | 53 | def forward(self, x): 54 | """ 55 | Forward pass. 56 | :param x: input 57 | :type x: torch.autograd.Variable 58 | :return: output 59 | :rtype: torch.autograd.Variable 60 | """ 61 | 62 | out = self.conv1(x) 63 | if self.normalization: 64 | out = self.norm1(out) 65 | 66 | out = self.relu(out) 67 | out = self.conv2(out) 68 | if self.normalization: 69 | out = self.norm2(out) 70 | 71 | if self.downsample is not None: 72 | identity = self.downsample(x) 73 | else: 74 | identity = x 75 | 76 | out += identity 77 | out = self.relu(out) 78 | 79 | return out 80 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/stutz2020/torch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def expand_as(tensor, tensor_as): 5 | """ 6 | Expands the tensor using view to allow broadcasting. 7 | :param tensor: input tensor 8 | :type tensor: torch.Tensor or torch.autograd.Variable 9 | :param tensor_as: reference tensor 10 | :type tensor_as: torch.Tensor or torch.autograd.Variable 11 | :return: tensor expanded with singelton dimensions as tensor_as 12 | :rtype: torch.Tensor or torch.autograd.Variable 13 | """ 14 | 15 | view = list(tensor.size()) 16 | for i in range(len(tensor.size()), len(tensor_as.size())): 17 | view.append(1) 18 | 19 | return tensor.view(view) 20 | 21 | 22 | class View(torch.nn.Module): 23 | """ 24 | Simple view layer. 25 | """ 26 | 27 | def __init__(self, *args): 28 | """ 29 | Constructor. 30 | :param args: shape 31 | :type args: [int] 32 | """ 33 | 34 | super(View, self).__init__() 35 | 36 | self.shape = args 37 | 38 | def forward(self, input): 39 | """ 40 | Forward pass. 41 | :param input: input 42 | :type input: torch.autograd.Variable 43 | :return: output 44 | :rtype: torch.autograd.Variable 45 | """ 46 | 47 | return input.view(self.shape) 48 | 49 | 50 | class Flatten(torch.nn.Module): 51 | """ 52 | Flatten module. 53 | """ 54 | 55 | def forward(self, input): 56 | """ 57 | Forward pass. 58 | :param input: input 59 | :type input: torch.autograd.Variable 60 | :return: output 61 | :rtype: torch.autograd.Variable 62 | """ 63 | 64 | return input.view(input.shape[0], -1) 65 | 66 | 67 | class Clamp(torch.nn.Module): 68 | """ 69 | Wrapper for clamp. 70 | """ 71 | 72 | def __init__(self, min=0, max=1): 73 | """ 74 | Constructor. 75 | """ 76 | 77 | super(Clamp, self).__init__() 78 | 79 | self.min = min 80 | """ (float) Min value. """ 81 | 82 | self.max = max 83 | """ (float) Max value. """ 84 | 85 | def forward(self, input): 86 | """ 87 | Forward pass. 88 | :param input: input 89 | :type input: torch.autograd.Variable 90 | :return: output 91 | :rtype: torch.autograd.Variable 92 | """ 93 | 94 | return torch.clamp(torch.clamp(input, min=self.min), max=self.max) 95 | 96 | 97 | class Scale(torch.nn.Module): 98 | """ 99 | Simply scaling layer, mainly to allow simple saving and loading. 100 | """ 101 | 102 | def __init__(self, shape): 103 | """ 104 | Constructor. 105 | :param shape: shape 106 | :type shape: [int] 107 | """ 108 | 109 | super(Scale, self).__init__() 110 | 111 | self.weight = torch.nn.Parameter(torch.zeros(shape)) # min 112 | self.bias = torch.nn.Parameter(torch.ones(shape)) # max 113 | 114 | def forward(self, input): 115 | """ 116 | Forward pass. 117 | :param input: input 118 | :type input: torch.autograd.Variable 119 | :return: output 120 | :rtype: torch.autograd.Variable 121 | """ 122 | 123 | return expand_as(self.weight, input) + torch.mul(expand_as(self.bias, input) - expand_as(self.weight, input), 124 | input) 125 | 126 | 127 | class Normalize(torch.nn.Module): 128 | """ 129 | Normalization layer to be learned. 130 | """ 131 | 132 | def __init__(self, n_channels): 133 | """ 134 | Constructor. 135 | :param n_channels: number of channels 136 | :type n_channels: int 137 | """ 138 | 139 | super(Normalize, self).__init__() 140 | 141 | # nn.Parameter is a special kind of Tensor, that will get 142 | # automatically registered as Module's parameter once it's assigned 143 | # as an attribute. Parameters and buffers need to be registered, or 144 | # they won't appear in .parameters() (doesn't apply to buffers), and 145 | # won't be converted when e.g. .cuda() is called. You can use 146 | # .register_buffer() to register buffers. 147 | # nn.Parameters require gradients by default. 148 | self.weight = torch.nn.Parameter(torch.ones(n_channels)) 149 | self.bias = torch.nn.Parameter(torch.zeros(n_channels)) 150 | 151 | def forward(self, input): 152 | """ 153 | Forward pass. 154 | :param input: input 155 | :type input: torch.autograd.Variable 156 | :return: output 157 | :rtype: torch.autograd.Variable 158 | """ 159 | 160 | return (input - self.bias.view(1, -1, 1, 1)) / self.weight.view(1, -1, 1, 1) 161 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/utils.py: -------------------------------------------------------------------------------- 1 | from .stutz2020.ccat import load_ccat_model 2 | from .wang2023.dm_adv_training import load_dm_adv_model 3 | from .xiao2020.kwta import load_kwta_model 4 | from .zhang2020.crown import load_crown_model 5 | 6 | _available_defenses = {'Stutz2020CCAT': load_ccat_model, 7 | 'Zhang2020CrownLarge': load_crown_model, 8 | 'Zhang2020CrownSmall': load_crown_model, 9 | 'Xiao2020KWTA': load_kwta_model, 10 | 'Wang2023DMAdvSmall': load_dm_adv_model, 11 | 'Wang2023DMAdvLarge': load_dm_adv_model} 12 | 13 | 14 | def load_original_model(model_name: str, dataset: str, threat_model: str): 15 | return _available_defenses[model_name](model=model_name, dataset=dataset, threat_model=threat_model) 16 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/wang2023/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/models/original/wang2023/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/models/original/wang2023/dm_adv_training.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from .wideresnetwithswish import wideresnetwithswish 3 | import os 4 | import urllib.request 5 | 6 | _pretrained_model_info = {'cifar10': { 7 | 'Wang2023DMAdvSmall': { 8 | 'name': 'wrn-28-10-swish', 9 | 'constructor': wideresnetwithswish, 10 | 'checkpoint_url': dict( 11 | Linf='https://huggingface.co/wzekai99/DM-Improves-AT/resolve/main/checkpoint/cifar10_linf_wrn28-10.pt', 12 | L2='https://huggingface.co/wzekai99/DM-Improves-AT/resolve/main/checkpoint/cifar10_l2_wrn28-10.pt')}, 13 | 14 | 'Wang2023DMAdvLarge': { 15 | 'name': 'wrn-70-16-swish', 16 | 'constructor': wideresnetwithswish, 17 | 'checkpoint_url': dict( 18 | Linf='https://huggingface.co/wzekai99/DM-Improves-AT/resolve/main/checkpoint/cifar10_linf_wrn70-16.pt', 19 | L2='https://huggingface.co/wzekai99/DM-Improves-AT/resolve/main/checkpoint/cifar10_l2_wrn70-16.pt') 20 | }}} 21 | 22 | 23 | def convert_state_dict(state_dict): 24 | new_state_dict = {} 25 | for key, value in state_dict.items(): 26 | if key.startswith('module.'): 27 | # Remove 'module.' prefix 28 | new_key = key[7:] 29 | else: 30 | new_key = key 31 | new_state_dict[new_key] = value 32 | return new_state_dict 33 | 34 | def download_model(dataset='cifar10', model='stutz_2020', threat_model='LInf'): 35 | model_info = _pretrained_model_info[dataset][model] 36 | 37 | url = model_info['checkpoint_url'][threat_model] 38 | model_file = f'models/checkpoints/%s' % url.split('/')[-1] 39 | 40 | urllib.request.urlretrieve(url, model_file) 41 | assert os.path.exists(model_file) 42 | 43 | 44 | def create_model(checkpoint_path, model_info): 45 | checkpoint = torch.load(checkpoint_path, map_location='cpu') 46 | model = model_info['constructor'](model_info['name']) 47 | 48 | # checkpoints were stored inside a dataparallel and a sequential. 49 | model = torch.nn.Sequential(model) 50 | model.load_state_dict(convert_state_dict(checkpoint['model_state_dict'])) 51 | return model 52 | 53 | 54 | def load_dm_adv_model(dataset='cifar10', model='stutz2020', threat_model='Linf'): 55 | assert dataset in _pretrained_model_info.keys() 56 | assert model in _pretrained_model_info[dataset].keys() 57 | assert threat_model in _pretrained_model_info[dataset][model]['checkpoint_url'].keys() 58 | 59 | model_info = _pretrained_model_info[dataset][model] 60 | 61 | url = model_info['checkpoint_url'][threat_model] 62 | model_file = f'models/checkpoints/%s' % url.split('/')[-1] 63 | 64 | if not os.path.exists(model_file): 65 | print('Downloading pretrained weights.') 66 | download_model(dataset, model, threat_model) 67 | 68 | model = create_model(model_file, model_info) 69 | return model 70 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/xiao2020/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/models/original/xiao2020/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/models/original/xiao2020/kwta.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import urllib.request 4 | from .models import SparseResNet18 5 | 6 | MODEL_ID = '1Af_owmMvg1LxjITLE1gFUmPx5idogeTP' 7 | 8 | 9 | def load_kwta_model(dataset='cifar10', model='kwta_spresnet18', threat_model='Linf'): 10 | gamma = 0.1 11 | net_name = f'kwta_spresnet18_{gamma}_cifar_adv.pth' 12 | model_file = f'models/checkpoints/%s' % net_name 13 | url = f'https://github.com/wielandbrendel/robustness_workshop/releases/download/v0.0.1/{net_name}' 14 | 15 | if not os.path.exists(model_file): 16 | print('Downloading pretrained weights.') 17 | urllib.request.urlretrieve(url, model_file) 18 | assert os.path.exists(model_file) 19 | 20 | model = SparseResNet18(sparsities=[gamma, gamma, gamma, gamma], sparse_func='vol') 21 | state_dict = torch.load(model_file, map_location='cpu') 22 | model.load_state_dict(state_dict) 23 | return model 24 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/zhang2020/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attackbench/AttackBench/f928a378043c9d3994f9c2e81942f5f3def3c464/attack_evaluation/models/original/zhang2020/__init__.py -------------------------------------------------------------------------------- /attack_evaluation/models/original/zhang2020/crown.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tarfile 3 | 4 | import requests 5 | from adv_lib.utils import normalize_model 6 | from torch import nn 7 | 8 | from .model_defs_gowal import IBP_large, model_cnn_4layer 9 | from .utils import load_crown_dict 10 | 11 | _pretrained_model_configs = { 12 | 'Zhang2020CrownLarge': dict(prefix='_dm-large', 13 | model_path="cifar_dm-large_8_255/IBP_large_best.pth", 14 | model_class="IBP_large", 15 | model_params={"in_ch": 3, "in_dim": 32, "linear_size": 512}), 16 | 'Zhang2020CrownSmall': dict(prefix="", 17 | model_path="cifar_crown_0.03137/cifar_8_small/cnn_4layer_linear_512_width_1_best.pth", 18 | model_class="model_cnn_4layer", 19 | model_params={"in_ch": 3, "in_dim": 32, "width": 1, "linear_size": 512}) 20 | } 21 | 22 | 23 | def download_model(model: str, dataset: str = 'cifar10') -> None: 24 | pretrained_config = _pretrained_model_configs[model] 25 | 26 | url = f"https://download.huan-zhang.com/models/crown-ibp/models_crown-ibp{pretrained_config['prefix']}.tar.gz" 27 | response = requests.get(url, stream=True) 28 | file = tarfile.open(fileobj=response.raw, mode="r|gz") 29 | file.extractall(path="./models/checkpoints/") 30 | 31 | 32 | def load_crown_model(model: str, dataset: str = 'cifar10', threat_model: str = 'Linf') -> nn.Module: 33 | assert dataset in ['cifar10'] # available ['mnist'] 34 | 35 | config = _pretrained_model_configs[model] 36 | if model == 'Zhang2020CrownLarge': 37 | net = IBP_large(**config['model_params']) 38 | model_file = "./models/checkpoints/models_crown-ibp_dm-large/" 39 | else: 40 | net = model_cnn_4layer(**config['model_params']) 41 | model_file = "./models/checkpoints/crown-ibp_models/" 42 | 43 | if not os.path.exists(model_file): 44 | download_model(model=model, dataset=dataset) 45 | assert os.path.exists(model_file) 46 | 47 | model_dict = load_crown_dict(model_file + config['model_path']) 48 | net.load_state_dict(model_dict) 49 | return normalize_model(net, mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010)) 50 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/zhang2020/model_defs_gowal.py: -------------------------------------------------------------------------------- 1 | """ 2 | This code has been taken from the authors' repository. 3 | https://github.com/huanzhang12/CROWN-IBP 4 | """ 5 | import torch.nn as nn 6 | 7 | 8 | def IBP_large(in_ch, in_dim, linear_size=512): 9 | model = nn.Sequential( 10 | nn.Conv2d(in_ch, 64, 3, stride=1, padding=1), 11 | nn.ReLU(), 12 | nn.Conv2d(64, 64, 3, stride=1, padding=1), 13 | nn.ReLU(), 14 | nn.Conv2d(64, 128, 3, stride=2, padding=1), 15 | nn.ReLU(), 16 | nn.Conv2d(128, 128, 3, stride=1, padding=1), 17 | nn.ReLU(), 18 | nn.Conv2d(128, 128, 3, stride=1, padding=1), 19 | nn.ReLU(), 20 | nn.Flatten(), 21 | nn.Linear((in_dim // 2) * (in_dim // 2) * 128, linear_size), 22 | nn.ReLU(), 23 | nn.Linear(linear_size, 10) 24 | ) 25 | return model 26 | 27 | 28 | def IBP_debug(in_ch, in_dim, linear_size=512): 29 | model = nn.Sequential( 30 | nn.Conv2d(1, 1, 3, stride=2, padding=1), 31 | nn.ReLU(), 32 | nn.Conv2d(1, 1, 3, stride=2, padding=1), 33 | nn.ReLU(), 34 | nn.Flatten(), 35 | nn.Linear((in_dim // 4) * (in_dim // 4) * 1, 10), 36 | ) 37 | return model 38 | 39 | 40 | def model_cnn_10layer(in_ch, in_dim, width): 41 | model = nn.Sequential( 42 | # input 32*32*3 43 | nn.Conv2d(in_ch, 4 * width, 3, stride=1, padding=1), 44 | nn.ReLU(), 45 | # input 32*32*4 46 | nn.Conv2d(4 * width, 8 * width, 2, stride=2, padding=0), 47 | nn.ReLU(), 48 | # input 16*16*8 49 | nn.Conv2d(8 * width, 8 * width, 3, stride=1, padding=1), 50 | nn.ReLU(), 51 | # input 16*16*8 52 | nn.Conv2d(8 * width, 16 * width, 2, stride=2, padding=0), 53 | nn.ReLU(), 54 | # input 8*8*16 55 | nn.Conv2d(16 * width, 16 * width, 3, stride=1, padding=1), 56 | nn.ReLU(), 57 | # input 8*8*16 58 | nn.Conv2d(16 * width, 32 * width, 2, stride=2, padding=0), 59 | nn.ReLU(), 60 | # input 4*4*32 61 | nn.Conv2d(32 * width, 32 * width, 3, stride=1, padding=1), 62 | nn.ReLU(), 63 | # input 4*4*32 64 | nn.Conv2d(32 * width, 64 * width, 2, stride=2, padding=0), 65 | nn.ReLU(), 66 | # input 2*2*64 67 | nn.Flatten(), 68 | nn.Linear(2 * 2 * 64 * width, 10) 69 | ) 70 | return model 71 | 72 | 73 | def model_cnn_4layer(in_ch, in_dim, width, linear_size): 74 | model = nn.Sequential( 75 | nn.Conv2d(in_ch, 4 * width, 3, stride=1, padding=1), 76 | nn.ReLU(), 77 | nn.Conv2d(4 * width, 4 * width, 4, stride=2, padding=1), 78 | nn.ReLU(), 79 | nn.Conv2d(4 * width, 8 * width, 3, stride=1, padding=1), 80 | nn.ReLU(), 81 | nn.Conv2d(8 * width, 8 * width, 4, stride=2, padding=1), 82 | nn.ReLU(), 83 | nn.Flatten(), 84 | nn.Linear(8 * width * (in_dim // 4) * (in_dim // 4), linear_size), 85 | nn.ReLU(), 86 | nn.Linear(linear_size, linear_size), 87 | nn.ReLU(), 88 | nn.Linear(linear_size, 10) 89 | ) 90 | return model 91 | -------------------------------------------------------------------------------- /attack_evaluation/models/original/zhang2020/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def load_crown_dict(model_file): 5 | checkpoint = torch.load(model_file, map_location='cpu') 6 | if isinstance(checkpoint["state_dict"], list): 7 | checkpoint["state_dict"] = checkpoint["state_dict"][0] 8 | new_state_dict = {} 9 | for k in checkpoint["state_dict"].keys(): 10 | if "prev" in k: 11 | pass 12 | else: 13 | new_state_dict[k] = checkpoint["state_dict"][k] 14 | checkpoint["state_dict"] = new_state_dict 15 | 16 | """ 17 | state_dict = m.state_dict() 18 | state_dict.update(checkpoint["state_dict"]) 19 | m.load_state_dict(state_dict) 20 | print(checkpoint["state_dict"]["__mask_layer.weight"]) 21 | """ 22 | 23 | return checkpoint["state_dict"] 24 | -------------------------------------------------------------------------------- /attack_evaluation/run.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from pathlib import Path 3 | from pprint import pprint 4 | 5 | import torch 6 | from adv_lib.distances.lp_norms import l0_distances, l1_distances, l2_distances, linf_distances 7 | from sacred import Experiment 8 | from sacred.observers import FileStorageObserver 9 | 10 | from .attacks.ingredient import attack_ingredient, get_attack 11 | from .datasets.ingredient import dataset_ingredient, get_loader 12 | from .models.ingredient import get_model, model_ingredient 13 | from .utils import run_attack, set_seed 14 | 15 | ex = Experiment('attack_evaluation', ingredients=[dataset_ingredient, model_ingredient, attack_ingredient]) 16 | 17 | 18 | @ex.config 19 | def config(): 20 | cpu = False # force experiment to run on CPU 21 | save_adv = False # save the inputs and perturbed inputs; not to be used with large datasets 22 | cudnn_flag = 'deterministic' # choose between "deterministic" and "benchmark" 23 | 24 | 25 | @ex.named_config 26 | def save_adv(): # act as a flag 27 | save_adv = True 28 | 29 | 30 | @ex.option_hook 31 | def modify_filestorage(options): 32 | if (file_storage := options['--file_storage']) is None: 33 | return 34 | 35 | update = options['UPDATE'] 36 | 37 | # find dataset, model and attack names from CLI 38 | names = {} 39 | for ingredient in (model_ingredient, attack_ingredient): 40 | ingredient_name = ingredient.path 41 | prefix = ingredient_name + '.' 42 | ingredient_updates = list(filter(lambda s: s.startswith(prefix) and '=' not in s, update)) 43 | if (n := len(ingredient_updates)) != 1: 44 | raise ValueError(f'Incorrect {ingredient_name} configuration: {n} (!=1) named configs specified.') 45 | named_config = ingredient_updates[0].removeprefix(prefix) 46 | # names.append(ingredient.named_configs[named_config]()['name']) 47 | names[ingredient_name] = named_config 48 | 49 | # get dataset from model named config 50 | dataset = model_ingredient.named_configs[names['model']]()['dataset'] 51 | 52 | # find threat model 53 | attack_updates = list(filter(lambda s: s.startswith('attack.') and 'threat_model=' in s, update)) 54 | if len(attack_updates): 55 | threat_model = attack_updates[-1].split('=')[-1] 56 | else: 57 | threat_model = ingredient.named_configs[named_config]()['threat_model'] 58 | 59 | batch_size_update = list(filter(lambda s: "batch_size" in s, update)) 60 | if len(batch_size_update): 61 | batch_size = batch_size_update[-1].split('=')[-1] 62 | else: 63 | batch_size = dataset_ingredient.configurations[0]()['batch_size'] 64 | batch_name = f'batch_size_{batch_size}' 65 | 66 | # insert threat model and batch_size at desired position for folder structure 67 | subdirs = [dataset, threat_model, names['model'], batch_name, names['attack']] 68 | options['--file_storage'] = Path(file_storage).joinpath(*subdirs).as_posix() 69 | 70 | 71 | metrics = OrderedDict([ 72 | ('linf', linf_distances), 73 | ('l0', l0_distances), 74 | ('l1', l1_distances), 75 | ('l2', l2_distances), 76 | ]) 77 | 78 | 79 | @ex.automain 80 | def main(cpu: bool, 81 | cudnn_flag: str, 82 | save_adv: bool, 83 | _config, _run, _log, _seed): 84 | device = torch.device('cuda' if torch.cuda.is_available() and not cpu else 'cpu') 85 | setattr(torch.backends.cudnn, cudnn_flag, True) 86 | 87 | set_seed(_seed) 88 | print(f'Running experiments with seed {_seed}') 89 | 90 | threat_model = _config['attack']['threat_model'] 91 | loader = get_loader(dataset=_config['model']['dataset']) 92 | attack = get_attack() 93 | model = get_model() 94 | model.to(device) 95 | 96 | if len(loader) == 0: # end experiment if there are no inputs to attack 97 | return 98 | 99 | # find the current folder where the artifacts are saved 100 | file_observers = [obs for obs in _run.observers if isinstance(obs, FileStorageObserver)] 101 | save_dir = file_observers[0].dir if len(file_observers) else None 102 | 103 | attack_data = run_attack(model=model, loader=loader, attack=attack, metrics=metrics, threat_model=threat_model, 104 | return_adv=save_adv and save_dir is not None, debug=_run.debug) 105 | 106 | if save_adv and save_dir is not None: 107 | torch.save(attack_data, Path(save_dir) / f'attack_data.pt') 108 | 109 | if 'inputs' in attack_data.keys(): 110 | del attack_data['inputs'], attack_data['adv_inputs'] 111 | _run.info = attack_data 112 | 113 | if _run.debug: 114 | pprint(_run.info) 115 | 116 | -------------------------------------------------------------------------------- /attack_evaluation/utils.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import random 3 | import traceback 4 | import warnings 5 | from collections import defaultdict 6 | from typing import Callable, Dict, Optional, Union 7 | 8 | import numpy as np 9 | import torch 10 | from adv_lib.utils.attack_utils import _default_metrics 11 | from torch import Tensor 12 | from torch.utils.data import DataLoader 13 | from tqdm import tqdm 14 | 15 | from .models.benchmodel_wrapper import BenchModel 16 | 17 | 18 | def run_attack(model: BenchModel, 19 | loader: DataLoader, 20 | attack: Callable, 21 | targets: Optional[Union[int, Tensor]] = None, 22 | metrics: Dict[str, Callable] = _default_metrics, 23 | threat_model: str = 'l2', 24 | return_adv: bool = False, 25 | debug: bool = False) -> dict: 26 | device = next(model.parameters()).device 27 | targeted = True if targets is not None else False 28 | loader_length = len(loader) 29 | 30 | accuracies, ori_success, adv_success, hashes, box_failures, batch_failures = [], [], [], [], [], [] 31 | forwards, backwards, times = [], [], [] 32 | distances, best_optim_distances = defaultdict(list), defaultdict(list) 33 | 34 | if return_adv: 35 | all_inputs, all_adv_inputs = [], [] 36 | 37 | for inputs, labels in tqdm(loader, ncols=80, total=loader_length): 38 | if return_adv: 39 | all_inputs.append(inputs.clone()) 40 | 41 | # compute hashes to ensure that input samples are identical 42 | for input in inputs: 43 | input_hash = hashlib.sha512(np.ascontiguousarray(input.numpy())).hexdigest() 44 | hashes.append(input_hash) 45 | 46 | inputs, labels = inputs.to(device), labels.to(device) # move data to device 47 | attack_inputs, attack_labels = inputs.clone(), labels.clone() # ensure no in-place modification 48 | # start tracking of the batch 49 | model.start_tracking(inputs=inputs, labels=labels, targeted=targeted, targets=targets, 50 | tracking_metric=_default_metrics[threat_model], tracking_threat_model=threat_model) 51 | 52 | if debug: 53 | adv_inputs = attack(model=model, inputs=attack_inputs, labels=attack_labels, 54 | targeted=targeted, targets=targets) 55 | else: 56 | try: 57 | adv_inputs = attack(model=model, inputs=attack_inputs, labels=attack_labels, 58 | targeted=targeted, targets=targets) 59 | batch_failures.append(False) 60 | except Exception: 61 | warnings.warn(f'Error running batch for {attack}') 62 | traceback.print_exc() 63 | batch_failures.append(True) 64 | adv_inputs = inputs 65 | 66 | model.end_tracking() 67 | adv_inputs.detach_() 68 | times.append(model.elapsed_time) 69 | forwards.extend(model.num_forwards.cpu().tolist()) 70 | backwards.extend(model.num_backwards.cpu().tolist()) 71 | 72 | # original inputs 73 | accuracies.extend(model.correct.cpu().tolist()) 74 | ori_success.extend(model.ori_success.cpu().tolist()) 75 | 76 | # checking box constraint 77 | batch_box_failures = ((adv_inputs < 0) | (adv_inputs > 1)).flatten(1).any(1) 78 | box_failures.extend(batch_box_failures.cpu().tolist()) 79 | 80 | if batch_box_failures.any(): 81 | warnings.warn('Values of produced adversarials are not in the [0, 1] range -> Clipping to [0, 1].') 82 | adv_inputs.clamp_(min=0, max=1) 83 | 84 | if return_adv: 85 | all_adv_inputs.append(adv_inputs.cpu().clone()) 86 | 87 | adv_logits = model(adv_inputs) 88 | adv_pred = adv_logits.argmax(dim=1) 89 | 90 | success = (adv_pred == targets) if targeted else (adv_pred != labels) 91 | adv_success.extend(success.cpu().tolist()) 92 | 93 | for metric, metric_func in metrics.items(): 94 | distances[metric].extend(metric_func(adv_inputs, inputs).cpu().tolist()) 95 | best_optim_distances[metric].extend(model.min_dist[metric].cpu().tolist()) 96 | 97 | data = { 98 | 'hashes': hashes, 99 | 'targeted': targeted, 100 | 'accuracy': sum(accuracies) / len(accuracies), 101 | 'ori_success': ori_success, 102 | 'adv_success': adv_success, 103 | 'ASR': sum(adv_success) / len(adv_success), 104 | 'times': times, 105 | 'num_forwards': forwards, 106 | 'num_backwards': backwards, 107 | 'distances': dict(distances), 108 | 'best_optim_distances': dict(best_optim_distances), 109 | 'box_failures': box_failures, 110 | 'batch_failures': batch_failures, 111 | } 112 | 113 | if return_adv: 114 | # shapes = [img.shape for img in all_inputs] 115 | # if len(set(shapes)) == 1: 116 | if len(all_inputs) > 1: 117 | all_inputs = torch.cat(all_inputs, dim=0) 118 | all_adv_inputs = torch.cat(all_adv_inputs, dim=0) 119 | data['inputs'] = all_inputs 120 | data['adv_inputs'] = all_adv_inputs 121 | 122 | return data 123 | 124 | 125 | def set_seed(seed: int = None) -> None: 126 | """Random seed (int) generation for PyTorch. See https://pytorch.org/docs/stable/notes/randomness.html for further 127 | details.""" 128 | if seed is None: 129 | return 130 | 131 | random.seed(seed) 132 | np.random.seed(seed) 133 | torch.manual_seed(seed) 134 | 135 | -------------------------------------------------------------------------------- /compiled/cifar10/standard_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"standard","threat_model":"l0","ASR":1.0,"optimality":0.8754903488,"n_forwards":1000.0,"n_backwards":1000.0,"time":266.6437317567},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"standard","threat_model":"l0","ASR":1.0,"optimality":0.2839547706,"n_forwards":839.0,"n_backwards":838.0,"time":264.5754539911},{"dataset":"cifar10","attack":"VFGA","library":"AdvLib","model":"standard","threat_model":"l0","ASR":0.987,"optimality":0.6692084852,"n_forwards":359.0,"n_backwards":17.0,"time":50.885500953},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"standard","threat_model":"l0","ASR":1.0,"optimality":0.8535817006,"n_forwards":892.0,"n_backwards":892.0,"time":264.5156097878},{"dataset":"cifar10","attack":"PGD-$\\ell_0$","library":"Original","model":"standard","threat_model":"l0","ASR":1.0,"optimality":0.5098505851,"n_forwards":974.0,"n_backwards":955.0,"time":423.4975453732},{"dataset":"cifar10","attack":"$\\sigma$-zero","library":"Original","model":"standard","threat_model":"l0","ASR":1.0,"optimality":0.9721915916,"n_forwards":999.0,"n_backwards":999.0,"time":269.6820253298}] -------------------------------------------------------------------------------- /compiled/cifar10/standard_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.9726467439,"n_forwards":993.0,"n_backwards":993.0,"time":262.8869639886},{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.9317732908,"n_forwards":1000.0,"n_backwards":990.0,"time":274.9118527629},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"standard","threat_model":"l1","ASR":0.9998,"optimality":0.8691386442,"n_forwards":590.0,"n_backwards":570.0,"time":160.7699799029},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"standard","threat_model":"l1","ASR":0.9473,"optimality":0.8342482562,"n_forwards":361.0,"n_backwards":1620.0,"time":267.4834114626},{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"standard","threat_model":"l1","ASR":0.9924,"optimality":0.966827413,"n_forwards":1000.0,"n_backwards":1000.0,"time":268.6907379768},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.9557600897,"n_forwards":992.0,"n_backwards":992.0,"time":263.2809479714},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.1197435391,"n_forwards":1044.0,"n_backwards":457.0,"time":456.8828615549},{"dataset":"cifar10","attack":"EAD","library":"Art","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.7262125707,"n_forwards":334.0,"n_backwards":1665.0,"time":275.161293912},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"standard","threat_model":"l1","ASR":0.9866,"optimality":0.2563180859,"n_forwards":40.0,"n_backwards":20.0,"time":31.4860464974},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"standard","threat_model":"l1","ASR":0.8404,"optimality":0.4836524221,"n_forwards":1010.0,"n_backwards":980.0,"time":579.3126300753},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"standard","threat_model":"l1","ASR":0.0537,"optimality":0.2354402316,"n_forwards":634.0,"n_backwards":0.0,"time":96.6516876565},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.4586664075,"n_forwards":220.0,"n_backwards":200.0,"time":284.3668990106},{"dataset":"cifar10","attack":"EAD","library":"Foolbox","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.8192615284,"n_forwards":405.0,"n_backwards":207.0,"time":80.5415762119},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"standard","threat_model":"l1","ASR":0.9865,"optimality":0.2563830661,"n_forwards":40.0,"n_backwards":20.0,"time":30.3241901859},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"standard","threat_model":"l1","ASR":0.9924,"optimality":0.9667088855,"n_forwards":1000.0,"n_backwards":1000.0,"time":265.7503164948},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.4721633278,"n_forwards":1000.0,"n_backwards":990.0,"time":640.9025351491},{"dataset":"cifar10","attack":"APGD-$\\ell_1$","library":"Original","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.959369504,"n_forwards":958.0,"n_backwards":938.0,"time":523.0360655787},{"dataset":"cifar10","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"standard","threat_model":"l1","ASR":1.0,"optimality":0.8823634047,"n_forwards":667.0,"n_backwards":622.0,"time":546.5681665378},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"standard","threat_model":"l1","ASR":0.9413,"optimality":0.8257388419,"n_forwards":311.0,"n_backwards":1544.0,"time":255.765985129},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"standard","threat_model":"l1","ASR":0.9924,"optimality":0.9667088855,"n_forwards":1000.0,"n_backwards":1000.0,"time":264.3066793094},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"standard","threat_model":"l1","ASR":0.0522,"optimality":0.6127032867,"n_forwards":311.0,"n_backwards":1544.0,"time":256.941042286}] -------------------------------------------------------------------------------- /compiled/cifar10/standard_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9789355215,"n_forwards":1000.0,"n_backwards":990.0,"time":263.7414422257},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"standard","threat_model":"linf","ASR":0.9967,"optimality":0.993964042,"n_forwards":641.0,"n_backwards":621.0,"time":169.0567646318},{"dataset":"cifar10","attack":"CW-$\\ell_\\infty$","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.6101079812,"n_forwards":758.0,"n_backwards":756.0,"time":206.577217438},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"standard","threat_model":"linf","ASR":0.9999,"optimality":0.9769843699,"n_forwards":363.0,"n_backwards":1629.0,"time":268.084213606},{"dataset":"cifar10","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9693960964,"n_forwards":1000.0,"n_backwards":1000.0,"time":271.1564847581},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9634771544,"n_forwards":970.0,"n_backwards":970.0,"time":269.7126066181},{"dataset":"cifar10","attack":"PGD","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9686020753,"n_forwards":1000.0,"n_backwards":990.0,"time":260.8865639157},{"dataset":"cifar10","attack":"tr","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.849256131,"n_forwards":10.0,"n_backwards":14.0,"time":3.4742501009},{"dataset":"cifar10","attack":"tr_adaptive","library":"AdvLib","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.8380777728,"n_forwards":12.0,"n_backwards":13.0,"time":3.5326667101},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"standard","threat_model":"linf","ASR":0.9966,"optimality":0.9321203815,"n_forwards":1274.0,"n_backwards":623.0,"time":389.9829526149},{"dataset":"cifar10","attack":"BB","library":"Art","model":"standard","threat_model":"linf","ASR":0.1032,"optimality":0.0050114565,"n_forwards":24.0,"n_backwards":5.0,"time":29.0395714088},{"dataset":"cifar10","attack":"CW-$\\ell_\\infty$","library":"Art","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.423270699,"n_forwards":1346.0,"n_backwards":651.0,"time":2269.622471352},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"standard","threat_model":"linf","ASR":0.9979,"optimality":0.5619569079,"n_forwards":40.0,"n_backwards":20.0,"time":31.6908615416},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.7742933883,"n_forwards":1010.0,"n_backwards":980.0,"time":798.5723020488},{"dataset":"cifar10","attack":"fgm","library":"Cleverhans","model":"standard","threat_model":"linf","ASR":0.9979,"optimality":0.5619569079,"n_forwards":41.0,"n_backwards":20.0,"time":30.5000192954},{"dataset":"cifar10","attack":"fgm","library":"DeepRobust","model":"standard","threat_model":"linf","ASR":0.9979,"optimality":0.5619569079,"n_forwards":40.0,"n_backwards":20.0,"time":31.920620163},{"dataset":"cifar10","attack":"PGD","library":"DeepRobust","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.8603298509,"n_forwards":1010.0,"n_backwards":990.0,"time":707.0463129866},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"standard","threat_model":"linf","ASR":0.0773,"optimality":0.0390963814,"n_forwards":751.0,"n_backwards":17.0,"time":110.9280726358},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9838332744,"n_forwards":1000.0,"n_backwards":990.0,"time":692.2857320573},{"dataset":"cifar10","attack":"deepfool","library":"Foolbox","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.6697374959,"n_forwards":48.0,"n_backwards":47.0,"time":12.9158129156},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"standard","threat_model":"linf","ASR":0.9979,"optimality":0.5619569079,"n_forwards":40.0,"n_backwards":20.0,"time":30.2174484923},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9695105471,"n_forwards":1000.0,"n_backwards":1000.0,"time":262.7987280162},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9543299121,"n_forwards":1000.0,"n_backwards":990.0,"time":727.3190632787},{"dataset":"cifar10","attack":"APGD","library":"Original","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9900128038,"n_forwards":958.0,"n_backwards":938.0,"time":711.4984119006},{"dataset":"cifar10","attack":"APGD$_t$","library":"Original","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9952576445,"n_forwards":707.0,"n_backwards":659.0,"time":626.1366574338},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"standard","threat_model":"linf","ASR":0.9999,"optimality":0.9767281564,"n_forwards":317.0,"n_backwards":1573.0,"time":262.067695145},{"dataset":"cifar10","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.9695105471,"n_forwards":1000.0,"n_backwards":1000.0,"time":264.431109639},{"dataset":"cifar10","attack":"tr","library":"Original","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.849256131,"n_forwards":24.0,"n_backwards":14.0,"time":5.234910014},{"dataset":"cifar10","attack":"tr_adaptive","library":"Original","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.8380777728,"n_forwards":19.0,"n_backwards":13.0,"time":4.4300177375},{"dataset":"cifar10","attack":"APGD","library":"TorchAttack","model":"standard","threat_model":"linf","ASR":0.0522,"optimality":0.6601181035,"n_forwards":1.0,"n_backwards":0.0,"time":0.1566745683},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"standard","threat_model":"linf","ASR":0.9999,"optimality":0.9767281564,"n_forwards":317.0,"n_backwards":1573.0,"time":257.8745368253},{"dataset":"cifar10","attack":"fgsm","library":"TorchAttack","model":"standard","threat_model":"linf","ASR":0.9979,"optimality":0.5619581292,"n_forwards":40.0,"n_backwards":20.0,"time":7.7605358245},{"dataset":"cifar10","attack":"PGD","library":"TorchAttack","model":"standard","threat_model":"linf","ASR":1.0,"optimality":0.885900142,"n_forwards":1000.0,"n_backwards":990.0,"time":696.4475099182}] -------------------------------------------------------------------------------- /compiled/cifar10/stutz_2020_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"stutz_2020","threat_model":"l0","ASR":1.0,"optimality":0.8337056996,"n_forwards":1000.0,"n_backwards":1000.0,"time":37.6204900431},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"stutz_2020","threat_model":"l0","ASR":0.9999,"optimality":0.144708582,"n_forwards":848.0,"n_backwards":848.0,"time":39.9815066849},{"dataset":"cifar10","attack":"VFGA","library":"AdvLib","model":"stutz_2020","threat_model":"l0","ASR":1.0,"optimality":0.7422894258,"n_forwards":136.0,"n_backwards":6.0,"time":3.1348209957},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"stutz_2020","threat_model":"l0","ASR":1.0,"optimality":0.8046308821,"n_forwards":842.0,"n_backwards":842.0,"time":40.8737640872},{"dataset":"cifar10","attack":"PGD-$\\ell_0$","library":"Original","model":"stutz_2020","threat_model":"l0","ASR":1.0,"optimality":0.4264940385,"n_forwards":953.0,"n_backwards":934.0,"time":134.6153771637},{"dataset":"cifar10","attack":"$\\sigma$-zero","library":"Original","model":"stutz_2020","threat_model":"l0","ASR":1.0,"optimality":0.9915558571,"n_forwards":999.0,"n_backwards":999.0,"time":38.4523698034}] -------------------------------------------------------------------------------- /compiled/cifar10/stutz_2020_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.8656519097,"n_forwards":987.0,"n_backwards":987.0,"time":39.4876167107},{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.7235690756,"n_forwards":1000.0,"n_backwards":990.0,"time":44.8083662464},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"stutz_2020","threat_model":"l1","ASR":0.9992,"optimality":0.4810248995,"n_forwards":572.0,"n_backwards":552.0,"time":32.7470679048},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"stutz_2020","threat_model":"l1","ASR":0.9743,"optimality":0.8998973223,"n_forwards":361.0,"n_backwards":1620.0,"time":38.8450781173},{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"stutz_2020","threat_model":"l1","ASR":0.9942,"optimality":0.9442675582,"n_forwards":1000.0,"n_backwards":1000.0,"time":38.8480580294},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.9272757096,"n_forwards":992.0,"n_backwards":992.0,"time":38.9619677668},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.4231913745,"n_forwards":752.0,"n_backwards":321.0,"time":253.5695459561},{"dataset":"cifar10","attack":"EAD","library":"Art","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.5740822831,"n_forwards":334.0,"n_backwards":1665.0,"time":42.6333366211},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.1089060791,"n_forwards":40.0,"n_backwards":20.0,"time":18.992693023},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"stutz_2020","threat_model":"l1","ASR":0.9998,"optimality":0.6854976964,"n_forwards":1010.0,"n_backwards":980.0,"time":127.182218382},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"stutz_2020","threat_model":"l1","ASR":0.8047,"optimality":0.6639467372,"n_forwards":567.0,"n_backwards":176.0,"time":51.6880372666},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.5558867891,"n_forwards":220.0,"n_backwards":200.0,"time":137.1502497783},{"dataset":"cifar10","attack":"EAD","library":"Foolbox","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.476958616,"n_forwards":389.0,"n_backwards":199.0,"time":10.9317464277},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.1089060212,"n_forwards":40.0,"n_backwards":20.0,"time":17.5391520947},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"stutz_2020","threat_model":"l1","ASR":0.9941,"optimality":0.9435650626,"n_forwards":1000.0,"n_backwards":1000.0,"time":39.0383000273},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.581803605,"n_forwards":1000.0,"n_backwards":990.0,"time":135.0994982351},{"dataset":"cifar10","attack":"APGD-$\\ell_1$","library":"Original","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.8845263187,"n_forwards":910.0,"n_backwards":890.0,"time":133.935252139},{"dataset":"cifar10","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"stutz_2020","threat_model":"l1","ASR":1.0,"optimality":0.679076186,"n_forwards":628.0,"n_backwards":584.0,"time":231.1564917663},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"stutz_2020","threat_model":"l1","ASR":0.9691,"optimality":0.8925234446,"n_forwards":295.0,"n_backwards":1465.0,"time":33.7738351001},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"stutz_2020","threat_model":"l1","ASR":0.9941,"optimality":0.9435650626,"n_forwards":1000.0,"n_backwards":1000.0,"time":39.5095614043},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"stutz_2020","threat_model":"l1","ASR":0.1008,"optimality":0.6016942599,"n_forwards":295.0,"n_backwards":1465.0,"time":33.165961624}] -------------------------------------------------------------------------------- /compiled/cifar10/stutz_2020_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9756146599,"n_forwards":1000.0,"n_backwards":990.0,"time":37.8372195462},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":0.9899,"optimality":0.9545314692,"n_forwards":644.0,"n_backwards":624.0,"time":27.9351787818},{"dataset":"cifar10","attack":"CW-$\\ell_\\infty$","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.0060739769,"n_forwards":307.0,"n_backwards":306.0,"time":17.6280338819},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9834568153,"n_forwards":363.0,"n_backwards":1629.0,"time":39.2204238495},{"dataset":"cifar10","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9508153808,"n_forwards":1000.0,"n_backwards":1000.0,"time":37.7143186538},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.877946779,"n_forwards":872.0,"n_backwards":872.0,"time":42.6281349145},{"dataset":"cifar10","attack":"PGD","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9731193904,"n_forwards":1000.0,"n_backwards":990.0,"time":37.0750455279},{"dataset":"cifar10","attack":"tr","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.7747217323,"n_forwards":6.0,"n_backwards":12.0,"time":0.4039631172},{"dataset":"cifar10","attack":"tr_adaptive","library":"AdvLib","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.7757743637,"n_forwards":8.0,"n_backwards":12.0,"time":0.4670626291},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9259321316,"n_forwards":1212.0,"n_backwards":590.0,"time":122.7880404241},{"dataset":"cifar10","attack":"BB","library":"Art","model":"stutz_2020","threat_model":"linf","ASR":0.1368,"optimality":0.0065879154,"n_forwards":20.0,"n_backwards":3.0,"time":11.1896012896},{"dataset":"cifar10","attack":"CW-$\\ell_\\infty$","library":"Art","model":"stutz_2020","threat_model":"linf","ASR":0.997,"optimality":0.6165925794,"n_forwards":1291.0,"n_backwards":622.0,"time":1393.1882290082},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.6213395769,"n_forwards":40.0,"n_backwards":20.0,"time":17.9985027418},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.8534122674,"n_forwards":1010.0,"n_backwards":980.0,"time":326.1730099225},{"dataset":"cifar10","attack":"fgm","library":"Cleverhans","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.6213395769,"n_forwards":41.0,"n_backwards":20.0,"time":17.3024708617},{"dataset":"cifar10","attack":"fgm","library":"DeepRobust","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.6213395769,"n_forwards":40.0,"n_backwards":20.0,"time":18.3992749231},{"dataset":"cifar10","attack":"PGD","library":"DeepRobust","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.8536153416,"n_forwards":1010.0,"n_backwards":990.0,"time":319.5648903807},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"stutz_2020","threat_model":"linf","ASR":0.9203,"optimality":0.9071611897,"n_forwards":1038.0,"n_backwards":634.0,"time":132.9615622274},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9828039099,"n_forwards":1000.0,"n_backwards":990.0,"time":210.8213104401},{"dataset":"cifar10","attack":"deepfool","library":"Foolbox","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.269647928,"n_forwards":37.0,"n_backwards":36.0,"time":1.3744534879},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.6213395769,"n_forwards":40.0,"n_backwards":20.0,"time":16.8867378567},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9505122731,"n_forwards":1000.0,"n_backwards":1000.0,"time":38.4317317337},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9650262824,"n_forwards":1000.0,"n_backwards":990.0,"time":202.5737616068},{"dataset":"cifar10","attack":"APGD","library":"Original","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9898092212,"n_forwards":910.0,"n_backwards":890.0,"time":197.7080969885},{"dataset":"cifar10","attack":"APGD$_t$","library":"Original","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9623389841,"n_forwards":680.0,"n_backwards":633.0,"time":260.5282984611},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"stutz_2020","threat_model":"linf","ASR":0.9988,"optimality":0.981321911,"n_forwards":301.0,"n_backwards":1492.0,"time":33.6192540409},{"dataset":"cifar10","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.9505122731,"n_forwards":1000.0,"n_backwards":1000.0,"time":38.4970835106},{"dataset":"cifar10","attack":"tr","library":"Original","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.7747218462,"n_forwards":10.0,"n_backwards":12.0,"time":0.4794204197},{"dataset":"cifar10","attack":"tr_adaptive","library":"Original","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.7757749672,"n_forwards":13.0,"n_backwards":12.0,"time":0.544000877},{"dataset":"cifar10","attack":"APGD","library":"TorchAttack","model":"stutz_2020","threat_model":"linf","ASR":0.1008,"optimality":0.6165925794,"n_forwards":1.0,"n_backwards":0.0,"time":0.0283032037},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"stutz_2020","threat_model":"linf","ASR":0.9988,"optimality":0.981321911,"n_forwards":301.0,"n_backwards":1492.0,"time":33.9916679317},{"dataset":"cifar10","attack":"fgsm","library":"TorchAttack","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.6213471781,"n_forwards":40.0,"n_backwards":20.0,"time":1.090870292},{"dataset":"cifar10","attack":"PGD","library":"TorchAttack","model":"stutz_2020","threat_model":"linf","ASR":1.0,"optimality":0.8510828594,"n_forwards":1000.0,"n_backwards":990.0,"time":284.6067210218}] -------------------------------------------------------------------------------- /compiled/cifar10/wang_2023_small_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"wang_2023_small","threat_model":"l0","ASR":1.0,"optimality":0.7844016128,"n_forwards":999.0,"n_backwards":999.0,"time":278.8025530052},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"wang_2023_small","threat_model":"l0","ASR":0.9991,"optimality":0.4424898373,"n_forwards":962.0,"n_backwards":962.0,"time":280.3719427637},{"dataset":"cifar10","attack":"VFGA","library":"AdvLib","model":"wang_2023_small","threat_model":"l0","ASR":0.8883,"optimality":0.7824364762,"n_forwards":768.0,"n_backwards":36.0,"time":106.1816265056},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"wang_2023_small","threat_model":"l0","ASR":0.9998,"optimality":0.7804391003,"n_forwards":911.0,"n_backwards":911.0,"time":277.2554864449},{"dataset":"cifar10","attack":"PGD-$\\ell_0$","library":"Original","model":"wang_2023_small","threat_model":"l0","ASR":1.0,"optimality":0.618474335,"n_forwards":978.0,"n_backwards":959.0,"time":544.9969057578},{"dataset":"cifar10","attack":"$\\sigma$-zero","library":"Original","model":"wang_2023_small","threat_model":"l0","ASR":1.0,"optimality":0.9984339479,"n_forwards":999.0,"n_backwards":999.0,"time":292.1994489043}] -------------------------------------------------------------------------------- /compiled/cifar10/wang_2023_small_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.9581735955,"n_forwards":989.0,"n_backwards":989.0,"time":276.723927881},{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.9639195963,"n_forwards":1000.0,"n_backwards":990.0,"time":284.2357060724},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"wang_2023_small","threat_model":"l1","ASR":0.9995,"optimality":0.9639860553,"n_forwards":657.0,"n_backwards":637.0,"time":190.4608361708},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"wang_2023_small","threat_model":"l1","ASR":0.9833,"optimality":0.8755468203,"n_forwards":360.0,"n_backwards":1618.0,"time":288.106110232},{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"wang_2023_small","threat_model":"l1","ASR":0.999,"optimality":0.9455871116,"n_forwards":1000.0,"n_backwards":1000.0,"time":276.7007690153},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.966451624,"n_forwards":998.0,"n_backwards":998.0,"time":279.6005687943},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"wang_2023_small","threat_model":"l1","ASR":0.9429,"optimality":0.0310068718,"n_forwards":1098.0,"n_backwards":482.0,"time":303.1720839146},{"dataset":"cifar10","attack":"EAD","library":"Art","model":"wang_2023_small","threat_model":"l1","ASR":0.9951,"optimality":0.5811061254,"n_forwards":334.0,"n_backwards":1665.0,"time":295.6934379186},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"wang_2023_small","threat_model":"l1","ASR":0.925,"optimality":0.2220831223,"n_forwards":40.0,"n_backwards":20.0,"time":29.526432499},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"wang_2023_small","threat_model":"l1","ASR":0.2433,"optimality":0.5480584973,"n_forwards":1010.0,"n_backwards":980.0,"time":429.6663938255},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"wang_2023_small","threat_model":"l1","ASR":0.0862,"optimality":0.0517672481,"n_forwards":647.0,"n_backwards":2.0,"time":119.4244203309},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.493913756,"n_forwards":220.0,"n_backwards":200.0,"time":279.1598967424},{"dataset":"cifar10","attack":"EAD","library":"Foolbox","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.8600027141,"n_forwards":482.0,"n_backwards":245.0,"time":96.5035757779},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"wang_2023_small","threat_model":"l1","ASR":0.925,"optimality":0.2213522415,"n_forwards":40.0,"n_backwards":20.0,"time":27.5098903887},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"wang_2023_small","threat_model":"l1","ASR":0.999,"optimality":0.9456017119,"n_forwards":1000.0,"n_backwards":1000.0,"time":281.2072094068},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.5229609701,"n_forwards":1000.0,"n_backwards":990.0,"time":714.964925237},{"dataset":"cifar10","attack":"APGD-$\\ell_1$","library":"Original","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.9651039174,"n_forwards":935.0,"n_backwards":915.0,"time":892.3625879321},{"dataset":"cifar10","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"wang_2023_small","threat_model":"l1","ASR":1.0,"optimality":0.9688912818,"n_forwards":726.0,"n_backwards":679.0,"time":860.5935260093},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"wang_2023_small","threat_model":"l1","ASR":0.9569,"optimality":0.8454113047,"n_forwards":303.0,"n_backwards":1505.0,"time":268.2862605252},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"wang_2023_small","threat_model":"l1","ASR":0.999,"optimality":0.9456017119,"n_forwards":1000.0,"n_backwards":1000.0,"time":275.9847838358},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"wang_2023_small","threat_model":"l1","ASR":0.0756,"optimality":0.5910912076,"n_forwards":303.0,"n_backwards":1505.0,"time":275.5629674094}] -------------------------------------------------------------------------------- /compiled/cifar10/xiao_2020_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"xiao_2020","threat_model":"l0","ASR":1.0,"optimality":0.9275940961,"n_forwards":999.0,"n_backwards":999.0,"time":45.4927299812},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"xiao_2020","threat_model":"l0","ASR":0.9973,"optimality":0.508584488,"n_forwards":919.0,"n_backwards":919.0,"time":47.1681128111},{"dataset":"cifar10","attack":"VFGA","library":"AdvLib","model":"xiao_2020","threat_model":"l0","ASR":0.8667,"optimality":0.8749039748,"n_forwards":496.0,"n_backwards":23.0,"time":15.5487521784},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"xiao_2020","threat_model":"l0","ASR":0.998,"optimality":0.9160041204,"n_forwards":690.0,"n_backwards":690.0,"time":47.2757052862},{"dataset":"cifar10","attack":"PGD-$\\ell_0$","library":"Original","model":"xiao_2020","threat_model":"l0","ASR":1.0,"optimality":0.9101090768,"n_forwards":880.0,"n_backwards":861.0,"time":286.1116586787},{"dataset":"cifar10","attack":"$\\sigma$-zero","library":"Original","model":"xiao_2020","threat_model":"l0","ASR":1.0,"optimality":0.9673510578,"n_forwards":999.0,"n_backwards":999.0,"time":46.3349770818}] -------------------------------------------------------------------------------- /compiled/cifar10/xiao_2020_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"xiao_2020","threat_model":"l1","ASR":0.9994,"optimality":0.1740275793,"n_forwards":805.0,"n_backwards":805.0,"time":47.8833541931},{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.8435285435,"n_forwards":998.0,"n_backwards":988.0,"time":52.9566180325},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"xiao_2020","threat_model":"l1","ASR":0.9971,"optimality":0.8914780108,"n_forwards":690.0,"n_backwards":670.0,"time":43.812346308},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"xiao_2020","threat_model":"l1","ASR":0.9601,"optimality":0.0164966836,"n_forwards":216.0,"n_backwards":1012.0,"time":41.3471714724},{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.9101955899,"n_forwards":1000.0,"n_backwards":1000.0,"time":46.1342817484},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.8992105654,"n_forwards":998.0,"n_backwards":998.0,"time":46.9025774154},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"xiao_2020","threat_model":"l1","ASR":0.999,"optimality":0.3710168154,"n_forwards":773.0,"n_backwards":331.0,"time":228.7820287618},{"dataset":"cifar10","attack":"EAD","library":"Art","model":"xiao_2020","threat_model":"l1","ASR":0.474,"optimality":0.3795743263,"n_forwards":334.0,"n_backwards":1665.0,"time":44.5048624761},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"xiao_2020","threat_model":"l1","ASR":0.9901,"optimality":0.3367514384,"n_forwards":40.0,"n_backwards":20.0,"time":18.3881221359},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"xiao_2020","threat_model":"l1","ASR":0.4203,"optimality":0.5785726559,"n_forwards":1010.0,"n_backwards":980.0,"time":169.8268568607},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"xiao_2020","threat_model":"l1","ASR":0.3463,"optimality":0.0660767162,"n_forwards":629.0,"n_backwards":0.0,"time":20.8356102768},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.6394983762,"n_forwards":220.0,"n_backwards":200.0,"time":160.0733207569},{"dataset":"cifar10","attack":"EAD","library":"Foolbox","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.8338922936,"n_forwards":292.0,"n_backwards":150.0,"time":10.6203713159},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"xiao_2020","threat_model":"l1","ASR":0.9903,"optimality":0.3345855093,"n_forwards":40.0,"n_backwards":20.0,"time":16.7609595438},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.9105155871,"n_forwards":999.0,"n_backwards":999.0,"time":47.9783709971},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"xiao_2020","threat_model":"l1","ASR":0.9997,"optimality":0.6058469547,"n_forwards":999.0,"n_backwards":989.0,"time":317.4080947389},{"dataset":"cifar10","attack":"APGD-$\\ell_1$","library":"Original","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.9071969125,"n_forwards":668.0,"n_backwards":648.0,"time":547.4885996127},{"dataset":"cifar10","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.9289001663,"n_forwards":558.0,"n_backwards":518.0,"time":526.8210488949},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"xiao_2020","threat_model":"l1","ASR":0.9183,"optimality":0.0991468532,"n_forwards":155.0,"n_backwards":790.0,"time":32.2343334151},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"xiao_2020","threat_model":"l1","ASR":1.0,"optimality":0.9105155871,"n_forwards":999.0,"n_backwards":999.0,"time":46.6873221321},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"xiao_2020","threat_model":"l1","ASR":0.3449,"optimality":0.5421612129,"n_forwards":155.0,"n_backwards":790.0,"time":31.849613204}] -------------------------------------------------------------------------------- /compiled/cifar10/xiao_2020_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.9577441739,"n_forwards":1000.0,"n_backwards":990.0,"time":45.6090359492},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":0.9855,"optimality":0.9269651497,"n_forwards":723.0,"n_backwards":703.0,"time":37.3134202189},{"dataset":"cifar10","attack":"CW-$\\ell_\\infty$","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":0.9975,"optimality":0.8141004752,"n_forwards":594.0,"n_backwards":594.0,"time":36.8254741504},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":0.9988,"optimality":0.005729934,"n_forwards":126.0,"n_backwards":726.0,"time":41.9809830605},{"dataset":"cifar10","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.8106230457,"n_forwards":996.0,"n_backwards":996.0,"time":45.4983423281},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.8670872859,"n_forwards":979.0,"n_backwards":979.0,"time":50.7495358119},{"dataset":"cifar10","attack":"PGD","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.8186487957,"n_forwards":999.0,"n_backwards":989.0,"time":44.5417190085},{"dataset":"cifar10","attack":"tr","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":0.993,"optimality":0.6413788174,"n_forwards":191.0,"n_backwards":104.0,"time":18.0554325838},{"dataset":"cifar10","attack":"tr_adaptive","library":"AdvLib","model":"xiao_2020","threat_model":"linf","ASR":0.9694,"optimality":0.6211308125,"n_forwards":239.0,"n_backwards":89.0,"time":20.8562981508},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"xiao_2020","threat_model":"linf","ASR":0.8895,"optimality":0.7594507137,"n_forwards":902.0,"n_backwards":434.0,"time":184.9918291344},{"dataset":"cifar10","attack":"BB","library":"Art","model":"xiao_2020","threat_model":"linf","ASR":0.3449,"optimality":0.5638866222,"n_forwards":13.0,"n_backwards":0.0,"time":12.1020370307},{"dataset":"cifar10","attack":"CW-$\\ell_\\infty$","library":"Art","model":"xiao_2020","threat_model":"linf","ASR":0.6344,"optimality":0.5794352802,"n_forwards":1324.0,"n_backwards":645.0,"time":1893.6699292692},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"xiao_2020","threat_model":"linf","ASR":0.974,"optimality":0.5182765446,"n_forwards":40.0,"n_backwards":20.0,"time":16.3391370805},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"xiao_2020","threat_model":"linf","ASR":0.9949,"optimality":0.8654779332,"n_forwards":1006.0,"n_backwards":976.0,"time":272.3489933405},{"dataset":"cifar10","attack":"fgm","library":"Cleverhans","model":"xiao_2020","threat_model":"linf","ASR":0.974,"optimality":0.5182765446,"n_forwards":41.0,"n_backwards":20.0,"time":14.4565517569},{"dataset":"cifar10","attack":"fgm","library":"DeepRobust","model":"xiao_2020","threat_model":"linf","ASR":0.974,"optimality":0.5182765446,"n_forwards":40.0,"n_backwards":20.0,"time":18.3588108669},{"dataset":"cifar10","attack":"PGD","library":"DeepRobust","model":"xiao_2020","threat_model":"linf","ASR":0.9945,"optimality":0.8173715923,"n_forwards":1007.0,"n_backwards":987.0,"time":255.5108982209},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"xiao_2020","threat_model":"linf","ASR":0.4276,"optimality":0.0661014756,"n_forwards":717.0,"n_backwards":0.0,"time":26.0670157661},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"xiao_2020","threat_model":"linf","ASR":0.9966,"optimality":0.8353025975,"n_forwards":996.0,"n_backwards":986.0,"time":273.6509238717},{"dataset":"cifar10","attack":"deepfool","library":"Foolbox","model":"xiao_2020","threat_model":"linf","ASR":0.9179,"optimality":0.0093544079,"n_forwards":396.0,"n_backwards":394.0,"time":64.1355937409},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"xiao_2020","threat_model":"linf","ASR":0.974,"optimality":0.518265375,"n_forwards":40.0,"n_backwards":20.0,"time":15.5728739261},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.8113748493,"n_forwards":996.0,"n_backwards":996.0,"time":46.2467077178},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"xiao_2020","threat_model":"linf","ASR":0.9962,"optimality":0.7603617142,"n_forwards":997.0,"n_backwards":987.0,"time":271.0859351225},{"dataset":"cifar10","attack":"APGD","library":"Original","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.9315686196,"n_forwards":668.0,"n_backwards":648.0,"time":306.054089669},{"dataset":"cifar10","attack":"APGD$_t$","library":"Original","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.9471703206,"n_forwards":572.0,"n_backwards":529.0,"time":295.4248649646},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"xiao_2020","threat_model":"linf","ASR":0.9739,"optimality":0.0076250163,"n_forwards":80.0,"n_backwards":501.0,"time":32.9417186017},{"dataset":"cifar10","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"xiao_2020","threat_model":"linf","ASR":1.0,"optimality":0.8113748493,"n_forwards":996.0,"n_backwards":996.0,"time":46.4148923842},{"dataset":"cifar10","attack":"tr","library":"Original","model":"xiao_2020","threat_model":"linf","ASR":0.9931,"optimality":0.6414722496,"n_forwards":944.0,"n_backwards":104.0,"time":35.6249942277},{"dataset":"cifar10","attack":"tr_adaptive","library":"Original","model":"xiao_2020","threat_model":"linf","ASR":0.9697,"optimality":0.6215726442,"n_forwards":1127.0,"n_backwards":89.0,"time":42.1939577252},{"dataset":"cifar10","attack":"APGD","library":"TorchAttack","model":"xiao_2020","threat_model":"linf","ASR":0.3449,"optimality":0.5638866222,"n_forwards":1.0,"n_backwards":0.0,"time":0.0361870824},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"xiao_2020","threat_model":"linf","ASR":0.3507,"optimality":0.043388038,"n_forwards":80.0,"n_backwards":501.0,"time":32.644738526},{"dataset":"cifar10","attack":"fgsm","library":"TorchAttack","model":"xiao_2020","threat_model":"linf","ASR":0.9739,"optimality":0.5182816788,"n_forwards":40.0,"n_backwards":20.0,"time":1.4207888265},{"dataset":"cifar10","attack":"PGD","library":"TorchAttack","model":"xiao_2020","threat_model":"linf","ASR":0.9939,"optimality":0.8093200446,"n_forwards":996.0,"n_backwards":986.0,"time":253.3254305105}] -------------------------------------------------------------------------------- /compiled/cifar10/zhang_2020_small_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"zhang_2020_small","threat_model":"l0","ASR":0.9365,"optimality":0.8452789518,"n_forwards":1000.0,"n_backwards":1000.0,"time":3.4153744289},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"zhang_2020_small","threat_model":"l0","ASR":0.9791,"optimality":0.5847767816,"n_forwards":997.0,"n_backwards":997.0,"time":4.9836243324},{"dataset":"cifar10","attack":"VFGA","library":"AdvLib","model":"zhang_2020_small","threat_model":"l0","ASR":0.9803,"optimality":0.9431476932,"n_forwards":183.0,"n_backwards":8.0,"time":1.2691021074},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"zhang_2020_small","threat_model":"l0","ASR":0.9365,"optimality":0.8452779417,"n_forwards":999.0,"n_backwards":999.0,"time":4.829095198},{"dataset":"cifar10","attack":"PGD-$\\ell_0$","library":"Original","model":"zhang_2020_small","threat_model":"l0","ASR":1.0,"optimality":0.8692686166,"n_forwards":812.0,"n_backwards":794.0,"time":69.4236781607},{"dataset":"cifar10","attack":"$\\sigma$-zero","library":"Original","model":"zhang_2020_small","threat_model":"l0","ASR":0.9999,"optimality":0.9928862647,"n_forwards":998.0,"n_backwards":998.0,"time":4.3377118686}] -------------------------------------------------------------------------------- /compiled/cifar10/zhang_2020_small_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"cifar10","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"zhang_2020_small","threat_model":"l1","ASR":0.9991,"optimality":0.5504789172,"n_forwards":841.0,"n_backwards":841.0,"time":5.3666238487},{"dataset":"cifar10","attack":"APGD","library":"AdvLib","model":"zhang_2020_small","threat_model":"l1","ASR":1.0,"optimality":0.5902310297,"n_forwards":999.0,"n_backwards":989.0,"time":10.9587832261},{"dataset":"cifar10","attack":"APGD$_t$","library":"AdvLib","model":"zhang_2020_small","threat_model":"l1","ASR":0.9953,"optimality":0.772043721,"n_forwards":637.0,"n_backwards":617.0,"time":12.6904437023},{"dataset":"cifar10","attack":"FAB","library":"AdvLib","model":"zhang_2020_small","threat_model":"l1","ASR":0.984,"optimality":0.7357665887,"n_forwards":360.0,"n_backwards":1619.0,"time":3.6599164356},{"dataset":"cifar10","attack":"FMN","library":"AdvLib","model":"zhang_2020_small","threat_model":"l1","ASR":0.9123,"optimality":0.752784449,"n_forwards":1000.0,"n_backwards":1000.0,"time":4.0290882281},{"dataset":"cifar10","attack":"PDPGD","library":"AdvLib","model":"zhang_2020_small","threat_model":"l1","ASR":0.9875,"optimality":0.9099407192,"n_forwards":997.0,"n_backwards":997.0,"time":5.2023629262},{"dataset":"cifar10","attack":"APGD","library":"Art","model":"zhang_2020_small","threat_model":"l1","ASR":0.9959,"optimality":0.3342241372,"n_forwards":441.0,"n_backwards":179.0,"time":38.0240247929},{"dataset":"cifar10","attack":"EAD","library":"Art","model":"zhang_2020_small","threat_model":"l1","ASR":0.7925,"optimality":0.4048791033,"n_forwards":334.0,"n_backwards":1665.0,"time":6.3916848832},{"dataset":"cifar10","attack":"fgm","library":"Art","model":"zhang_2020_small","threat_model":"l1","ASR":0.9813,"optimality":0.4760048979,"n_forwards":40.0,"n_backwards":20.0,"time":2.2513111702},{"dataset":"cifar10","attack":"PGD","library":"Art","model":"zhang_2020_small","threat_model":"l1","ASR":0.8404,"optimality":0.6561374819,"n_forwards":1010.0,"n_backwards":980.0,"time":35.4253693964},{"dataset":"cifar10","attack":"BB","library":"Foolbox","model":"zhang_2020_small","threat_model":"l1","ASR":0.6109,"optimality":0.1727321278,"n_forwards":637.0,"n_backwards":0.0,"time":1.908094632},{"dataset":"cifar10","attack":"BIM","library":"Foolbox","model":"zhang_2020_small","threat_model":"l1","ASR":0.9998,"optimality":0.6677950538,"n_forwards":220.0,"n_backwards":200.0,"time":21.2157963403},{"dataset":"cifar10","attack":"EAD","library":"Foolbox","model":"zhang_2020_small","threat_model":"l1","ASR":0.9963,"optimality":0.6071031959,"n_forwards":355.0,"n_backwards":182.0,"time":1.0106098666},{"dataset":"cifar10","attack":"fgm","library":"Foolbox","model":"zhang_2020_small","threat_model":"l1","ASR":0.9813,"optimality":0.4760048947,"n_forwards":40.0,"n_backwards":20.0,"time":2.208171073},{"dataset":"cifar10","attack":"FMN","library":"Foolbox","model":"zhang_2020_small","threat_model":"l1","ASR":0.9122,"optimality":0.7525264609,"n_forwards":1000.0,"n_backwards":1000.0,"time":5.0863315192},{"dataset":"cifar10","attack":"PGD","library":"Foolbox","model":"zhang_2020_small","threat_model":"l1","ASR":0.9999,"optimality":0.595276927,"n_forwards":999.0,"n_backwards":989.0,"time":56.9627358361},{"dataset":"cifar10","attack":"APGD-$\\ell_1$","library":"Original","model":"zhang_2020_small","threat_model":"l1","ASR":1.0,"optimality":0.827393554,"n_forwards":406.0,"n_backwards":386.0,"time":143.9676255215},{"dataset":"cifar10","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"zhang_2020_small","threat_model":"l1","ASR":1.0,"optimality":0.8094836497,"n_forwards":306.0,"n_backwards":275.0,"time":163.3765035659},{"dataset":"cifar10","attack":"FAB","library":"Original","model":"zhang_2020_small","threat_model":"l1","ASR":0.9743,"optimality":0.7109921774,"n_forwards":128.0,"n_backwards":635.0,"time":2.8510193712},{"dataset":"cifar10","attack":"FMN","library":"Original","model":"zhang_2020_small","threat_model":"l1","ASR":0.9122,"optimality":0.7525264609,"n_forwards":1000.0,"n_backwards":1000.0,"time":5.3244167722},{"dataset":"cifar10","attack":"FAB","library":"TorchAttack","model":"zhang_2020_small","threat_model":"l1","ASR":0.61,"optimality":0.5379606859,"n_forwards":128.0,"n_backwards":635.0,"time":2.8530700186}] -------------------------------------------------------------------------------- /compiled/imagenet/debenedetti_2022_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"debenedetti_2022","threat_model":"l0","ASR":1.0,"optimality":0.8948001722,"n_forwards":1000.0,"n_backwards":1000.0,"time":358.7625801835},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"l0","ASR":1.0,"optimality":0.5837782665,"n_forwards":974.0,"n_backwards":974.0,"time":345.0454482451},{"dataset":"imagenet","attack":"VFGA","library":"AdvLib","model":"debenedetti_2022","threat_model":"l0","ASR":0.4752,"optimality":0.6616019008,"n_forwards":1171.0,"n_backwards":55.0,"time":75.5515185409},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"debenedetti_2022","threat_model":"l0","ASR":0.9996,"optimality":0.8922599652,"n_forwards":763.0,"n_backwards":763.0,"time":357.1527706547},{"dataset":"imagenet","attack":"PGD-$\\ell_0$","library":"Original","model":"debenedetti_2022","threat_model":"l0","ASR":1.0,"optimality":0.4991493523,"n_forwards":926.0,"n_backwards":907.0,"time":1680.0993884368},{"dataset":"imagenet","attack":"$\\sigma$-zero","library":"Original","model":"debenedetti_2022","threat_model":"l0","ASR":1.0,"optimality":0.9986056115,"n_forwards":999.0,"n_backwards":999.0,"time":345.1680462227}] -------------------------------------------------------------------------------- /compiled/imagenet/debenedetti_2022_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.9361876336,"n_forwards":973.0,"n_backwards":973.0,"time":359.2634376041},{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.9590018539,"n_forwards":1000.0,"n_backwards":990.0,"time":381.3173845494},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.9650394605,"n_forwards":816.0,"n_backwards":796.0,"time":389.679456558},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"debenedetti_2022","threat_model":"l1","ASR":0.9952,"optimality":0.9597308328,"n_forwards":1000.0,"n_backwards":1000.0,"time":346.3860099015},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.9718677235,"n_forwards":1000.0,"n_backwards":1000.0,"time":339.2396149093},{"dataset":"imagenet","attack":"EAD","library":"Foolbox","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.8957418305,"n_forwards":399.0,"n_backwards":204.0,"time":77.5390999273},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"debenedetti_2022","threat_model":"l1","ASR":0.9952,"optimality":0.9597304456,"n_forwards":1000.0,"n_backwards":1000.0,"time":374.0054689746},{"dataset":"imagenet","attack":"APGD-$\\ell_1$","library":"Original","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.9883449779,"n_forwards":738.0,"n_backwards":718.0,"time":1085.9324655331},{"dataset":"imagenet","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"debenedetti_2022","threat_model":"l1","ASR":1.0,"optimality":0.9724928656,"n_forwards":663.0,"n_backwards":618.0,"time":990.1339801594},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"debenedetti_2022","threat_model":"l1","ASR":0.9952,"optimality":0.9597304456,"n_forwards":1000.0,"n_backwards":1000.0,"time":346.3454845707}] -------------------------------------------------------------------------------- /compiled/imagenet/debenedetti_2022_l2.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"l2","ASR":1.0,"optimality":0.9454759581,"n_forwards":1000.0,"n_backwards":990.0,"time":333.5351760921},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"debenedetti_2022","threat_model":"l2","ASR":0.9966,"optimality":0.9757434663,"n_forwards":744.0,"n_backwards":724.0,"time":325.0832068308},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"debenedetti_2022","threat_model":"l2","ASR":1.0,"optimality":0.9628120893,"n_forwards":1000.0,"n_backwards":1000.0,"time":334.7091551269},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"debenedetti_2022","threat_model":"l2","ASR":1.0,"optimality":0.9819826811,"n_forwards":1000.0,"n_backwards":1000.0,"time":340.1784783842},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"debenedetti_2022","threat_model":"l2","ASR":0.9998,"optimality":0.9677941872,"n_forwards":999.0,"n_backwards":999.0,"time":362.7478817749},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"debenedetti_2022","threat_model":"l2","ASR":1.0,"optimality":0.9679492945,"n_forwards":1000.0,"n_backwards":990.0,"time":1252.4270311121},{"dataset":"imagenet","attack":"DDN","library":"Foolbox","model":"debenedetti_2022","threat_model":"l2","ASR":1.0,"optimality":0.9819829014,"n_forwards":1000.0,"n_backwards":1000.0,"time":335.9942522784},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"debenedetti_2022","threat_model":"l2","ASR":0.2756,"optimality":0.3973486334,"n_forwards":532.0,"n_backwards":518.0,"time":696.8776280126},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"debenedetti_2022","threat_model":"l2","ASR":0.327,"optimality":0.221567568,"n_forwards":538.0,"n_backwards":499.0,"time":784.5366724637},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"debenedetti_2022","threat_model":"l2","ASR":0.9998,"optimality":0.9685922332,"n_forwards":1000.0,"n_backwards":1000.0,"time":366.8100480472}] -------------------------------------------------------------------------------- /compiled/imagenet/debenedetti_2022_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.9627730412,"n_forwards":1000.0,"n_backwards":990.0,"time":333.9778958178},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"debenedetti_2022","threat_model":"linf","ASR":0.997,"optimality":0.9928883529,"n_forwards":770.0,"n_backwards":750.0,"time":358.65209614},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.9768208397,"n_forwards":999.0,"n_backwards":999.0,"time":340.900167442},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.975890159,"n_forwards":986.0,"n_backwards":986.0,"time":409.0709145864},{"dataset":"imagenet","attack":"PGD","library":"AdvLib","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.9549192658,"n_forwards":1000.0,"n_backwards":990.0,"time":361.3099150001},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.9823541075,"n_forwards":1000.0,"n_backwards":990.0,"time":1293.7300784939},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.9768286931,"n_forwards":999.0,"n_backwards":999.0,"time":363.7482343116},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"debenedetti_2022","threat_model":"linf","ASR":0.2756,"optimality":0.3923538944,"n_forwards":540.0,"n_backwards":526.0,"time":608.0954879085},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"debenedetti_2022","threat_model":"linf","ASR":0.3276,"optimality":0.3062403452,"n_forwards":546.0,"n_backwards":507.0,"time":745.8417199173},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"debenedetti_2022","threat_model":"linf","ASR":1.0,"optimality":0.9768286931,"n_forwards":999.0,"n_backwards":999.0,"time":335.1464726262}] -------------------------------------------------------------------------------- /compiled/imagenet/salman_2020_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"salman_2020","threat_model":"l0","ASR":1.0,"optimality":0.8403924489,"n_forwards":1000.0,"n_backwards":1000.0,"time":227.1847570817},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"salman_2020","threat_model":"l0","ASR":1.0,"optimality":0.4680367361,"n_forwards":986.0,"n_backwards":986.0,"time":230.0193684537},{"dataset":"imagenet","attack":"VFGA","library":"AdvLib","model":"salman_2020","threat_model":"l0","ASR":0.6064,"optimality":0.6951409297,"n_forwards":968.0,"n_backwards":46.0,"time":76.3568978458},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"salman_2020","threat_model":"l0","ASR":1.0,"optimality":0.8393204615,"n_forwards":734.0,"n_backwards":734.0,"time":230.2361246141},{"dataset":"imagenet","attack":"PGD-$\\ell_0$","library":"Original","model":"salman_2020","threat_model":"l0","ASR":1.0,"optimality":0.4082365952,"n_forwards":914.0,"n_backwards":895.0,"time":1010.759180582},{"dataset":"imagenet","attack":"$\\sigma$-zero","library":"Original","model":"salman_2020","threat_model":"l0","ASR":1.0,"optimality":0.9992617759,"n_forwards":999.0,"n_backwards":999.0,"time":232.1163579288}] -------------------------------------------------------------------------------- /compiled/imagenet/salman_2020_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.8965090012,"n_forwards":968.0,"n_backwards":968.0,"time":231.9585782198},{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.9685950437,"n_forwards":1000.0,"n_backwards":990.0,"time":268.1084899809},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.9664910902,"n_forwards":735.0,"n_backwards":715.0,"time":212.9277082433},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"salman_2020","threat_model":"l1","ASR":0.9378,"optimality":0.8742956435,"n_forwards":1000.0,"n_backwards":1000.0,"time":238.4505212241},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.9745682758,"n_forwards":999.0,"n_backwards":999.0,"time":234.3428351064},{"dataset":"imagenet","attack":"EAD","library":"Foolbox","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.9261950328,"n_forwards":427.0,"n_backwards":218.0,"time":65.9371724736},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"salman_2020","threat_model":"l1","ASR":0.9362,"optimality":0.8728310292,"n_forwards":1000.0,"n_backwards":1000.0,"time":235.7348480211},{"dataset":"imagenet","attack":"APGD-$\\ell_1$","library":"Original","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.991496313,"n_forwards":699.0,"n_backwards":679.0,"time":527.1989533702},{"dataset":"imagenet","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"salman_2020","threat_model":"l1","ASR":1.0,"optimality":0.9727535422,"n_forwards":584.0,"n_backwards":543.0,"time":465.8880014278},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"salman_2020","threat_model":"l1","ASR":0.9362,"optimality":0.8728310292,"n_forwards":1000.0,"n_backwards":1000.0,"time":237.0899781579}] -------------------------------------------------------------------------------- /compiled/imagenet/salman_2020_l2.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"salman_2020","threat_model":"l2","ASR":1.0,"optimality":0.9516409863,"n_forwards":1000.0,"n_backwards":990.0,"time":227.8644419077},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"salman_2020","threat_model":"l2","ASR":0.9994,"optimality":0.9779219509,"n_forwards":706.0,"n_backwards":686.0,"time":162.6032894485},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"salman_2020","threat_model":"l2","ASR":1.0,"optimality":0.9638763556,"n_forwards":1000.0,"n_backwards":1000.0,"time":226.5027057171},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"salman_2020","threat_model":"l2","ASR":1.0,"optimality":0.9837049117,"n_forwards":1000.0,"n_backwards":1000.0,"time":225.806622026},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"salman_2020","threat_model":"l2","ASR":1.0,"optimality":0.9733736198,"n_forwards":1000.0,"n_backwards":1000.0,"time":228.3273237794},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"salman_2020","threat_model":"l2","ASR":1.0,"optimality":0.9698902899,"n_forwards":1000.0,"n_backwards":990.0,"time":439.9069163914},{"dataset":"imagenet","attack":"DDN","library":"Foolbox","model":"salman_2020","threat_model":"l2","ASR":1.0,"optimality":0.9837296656,"n_forwards":1000.0,"n_backwards":1000.0,"time":227.9149510606},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"salman_2020","threat_model":"l2","ASR":0.3148,"optimality":0.4109892547,"n_forwards":444.0,"n_backwards":432.0,"time":186.1154647064},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"salman_2020","threat_model":"l2","ASR":0.3336,"optimality":0.2578981762,"n_forwards":456.0,"n_backwards":421.0,"time":246.9928954041},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"salman_2020","threat_model":"l2","ASR":0.9998,"optimality":0.9709650223,"n_forwards":1000.0,"n_backwards":1000.0,"time":230.0157931481}] -------------------------------------------------------------------------------- /compiled/imagenet/salman_2020_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.9701347343,"n_forwards":1000.0,"n_backwards":990.0,"time":227.6067117263},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"salman_2020","threat_model":"linf","ASR":0.9992,"optimality":0.9932895505,"n_forwards":777.0,"n_backwards":757.0,"time":177.4519474555},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.980881038,"n_forwards":1000.0,"n_backwards":1000.0,"time":226.8804382204},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.980324896,"n_forwards":980.0,"n_backwards":980.0,"time":269.8037357728},{"dataset":"imagenet","attack":"PGD","library":"AdvLib","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.9676654634,"n_forwards":1000.0,"n_backwards":990.0,"time":225.2309721038},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.9810642321,"n_forwards":1000.0,"n_backwards":990.0,"time":389.5145786864},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.9810215128,"n_forwards":1000.0,"n_backwards":1000.0,"time":230.2418067265},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"salman_2020","threat_model":"linf","ASR":0.3148,"optimality":0.4215361054,"n_forwards":440.0,"n_backwards":428.0,"time":157.5026127729},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"salman_2020","threat_model":"linf","ASR":0.3174,"optimality":0.3890424883,"n_forwards":463.0,"n_backwards":430.0,"time":197.561286451},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"salman_2020","threat_model":"linf","ASR":1.0,"optimality":0.9810215128,"n_forwards":1000.0,"n_backwards":1000.0,"time":233.5262035958}] -------------------------------------------------------------------------------- /compiled/imagenet/standard_imagenet_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"standard_imagenet","threat_model":"l0","ASR":1.0,"optimality":0.8347791772,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.3795864971},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"standard_imagenet","threat_model":"l0","ASR":1.0,"optimality":0.500125532,"n_forwards":982.0,"n_backwards":982.0,"time":148.206446024},{"dataset":"imagenet","attack":"VFGA","library":"AdvLib","model":"standard_imagenet","threat_model":"l0","ASR":0.6912,"optimality":0.7120362494,"n_forwards":916.0,"n_backwards":43.0,"time":34.9839489499},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"standard_imagenet","threat_model":"l0","ASR":1.0,"optimality":0.8291275659,"n_forwards":790.0,"n_backwards":790.0,"time":147.2706875719},{"dataset":"imagenet","attack":"PGD-$\\ell_0$","library":"Original","model":"standard_imagenet","threat_model":"l0","ASR":1.0,"optimality":0.1641397786,"n_forwards":937.0,"n_backwards":917.0,"time":993.9800508603},{"dataset":"imagenet","attack":"$\\sigma$-zero","library":"Original","model":"standard_imagenet","threat_model":"l0","ASR":1.0,"optimality":0.9956698134,"n_forwards":999.0,"n_backwards":999.0,"time":145.0861523744}] -------------------------------------------------------------------------------- /compiled/imagenet/standard_imagenet_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.9525340594,"n_forwards":990.0,"n_backwards":990.0,"time":145.5594216024},{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.8980508765,"n_forwards":1000.0,"n_backwards":990.0,"time":184.931186369},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.8573296029,"n_forwards":680.0,"n_backwards":660.0,"time":143.482071063},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"standard_imagenet","threat_model":"l1","ASR":0.9576,"optimality":0.9331767587,"n_forwards":1000.0,"n_backwards":1000.0,"time":152.6926271615},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.9833288248,"n_forwards":999.0,"n_backwards":999.0,"time":147.839566801},{"dataset":"imagenet","attack":"EAD","library":"Foolbox","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.8569524445,"n_forwards":366.0,"n_backwards":187.0,"time":33.0370135493},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"standard_imagenet","threat_model":"l1","ASR":0.9574,"optimality":0.9341308001,"n_forwards":1000.0,"n_backwards":1000.0,"time":153.8189095922},{"dataset":"imagenet","attack":"APGD-$\\ell_1$","library":"Original","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.9456374247,"n_forwards":773.0,"n_backwards":753.0,"time":545.0514399898},{"dataset":"imagenet","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"standard_imagenet","threat_model":"l1","ASR":1.0,"optimality":0.8674231044,"n_forwards":615.0,"n_backwards":573.0,"time":494.4296830157},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"standard_imagenet","threat_model":"l1","ASR":0.9574,"optimality":0.9341308001,"n_forwards":1000.0,"n_backwards":1000.0,"time":153.8598954828}] -------------------------------------------------------------------------------- /compiled/imagenet/standard_imagenet_l2.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"standard_imagenet","threat_model":"l2","ASR":1.0,"optimality":0.9730474511,"n_forwards":1000.0,"n_backwards":990.0,"time":144.6217384052},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"standard_imagenet","threat_model":"l2","ASR":0.9998,"optimality":0.9863117669,"n_forwards":533.0,"n_backwards":513.0,"time":80.7040409911},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"standard_imagenet","threat_model":"l2","ASR":1.0,"optimality":0.6140674894,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.2880511878},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"standard_imagenet","threat_model":"l2","ASR":1.0,"optimality":0.9936675542,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.9062364503},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"standard_imagenet","threat_model":"l2","ASR":0.989,"optimality":0.9704153173,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.3573123487},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"standard_imagenet","threat_model":"l2","ASR":1.0,"optimality":0.9788302335,"n_forwards":1000.0,"n_backwards":990.0,"time":304.7110878126},{"dataset":"imagenet","attack":"DDN","library":"Foolbox","model":"standard_imagenet","threat_model":"l2","ASR":1.0,"optimality":0.993674327,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.6966863943},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"standard_imagenet","threat_model":"l2","ASR":0.2402,"optimality":0.4030738203,"n_forwards":545.0,"n_backwards":532.0,"time":127.5481465115},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"standard_imagenet","threat_model":"l2","ASR":0.7154,"optimality":0.6848957913,"n_forwards":489.0,"n_backwards":449.0,"time":238.0313186885},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"standard_imagenet","threat_model":"l2","ASR":0.9908,"optimality":0.9726315222,"n_forwards":1000.0,"n_backwards":1000.0,"time":146.0405833781}] -------------------------------------------------------------------------------- /compiled/imagenet/standard_imagenet_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.979640402,"n_forwards":1000.0,"n_backwards":990.0,"time":143.6847012786},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"standard_imagenet","threat_model":"linf","ASR":0.9996,"optimality":0.9943596819,"n_forwards":536.0,"n_backwards":516.0,"time":81.3875016601},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.9839651765,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.4073218527},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.9877714086,"n_forwards":997.0,"n_backwards":997.0,"time":188.0713850683},{"dataset":"imagenet","attack":"PGD","library":"AdvLib","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.9737399258,"n_forwards":1000.0,"n_backwards":990.0,"time":141.4884557067},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.9833394214,"n_forwards":1000.0,"n_backwards":990.0,"time":347.0779804563},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.9841095718,"n_forwards":1000.0,"n_backwards":1000.0,"time":145.0196993804},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"standard_imagenet","threat_model":"linf","ASR":0.2402,"optimality":0.3992592588,"n_forwards":543.0,"n_backwards":530.0,"time":134.4497613249},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"standard_imagenet","threat_model":"linf","ASR":0.7128,"optimality":0.6720255474,"n_forwards":490.0,"n_backwards":451.0,"time":233.8420754009},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"standard_imagenet","threat_model":"linf","ASR":1.0,"optimality":0.9841095718,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.8435949303}] -------------------------------------------------------------------------------- /compiled/imagenet/wong_2020_l0.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"wong_2020","threat_model":"l0","ASR":1.0,"optimality":0.9301593571,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.5235404851},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"wong_2020","threat_model":"l0","ASR":1.0,"optimality":0.5928366286,"n_forwards":989.0,"n_backwards":989.0,"time":148.2542727611},{"dataset":"imagenet","attack":"VFGA","library":"AdvLib","model":"wong_2020","threat_model":"l0","ASR":0.77,"optimality":0.7769645194,"n_forwards":656.0,"n_backwards":31.0,"time":26.9318339288},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"wong_2020","threat_model":"l0","ASR":0.9986,"optimality":0.926501517,"n_forwards":618.0,"n_backwards":618.0,"time":147.6637061148},{"dataset":"imagenet","attack":"PGD-$\\ell_0$","library":"Original","model":"wong_2020","threat_model":"l0","ASR":1.0,"optimality":0.4958497896,"n_forwards":873.0,"n_backwards":854.0,"time":906.938387563},{"dataset":"imagenet","attack":"$\\sigma$-zero","library":"Original","model":"wong_2020","threat_model":"l0","ASR":1.0,"optimality":0.9990385511,"n_forwards":999.0,"n_backwards":999.0,"time":145.7710039469}] -------------------------------------------------------------------------------- /compiled/imagenet/wong_2020_l1.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"ALMA-$\\ell_1$","library":"AdvLib","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9153510666,"n_forwards":959.0,"n_backwards":959.0,"time":146.5033623408},{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9728707143,"n_forwards":1000.0,"n_backwards":990.0,"time":184.962229569},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9724979126,"n_forwards":721.0,"n_backwards":701.0,"time":152.0439866825},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"wong_2020","threat_model":"l1","ASR":0.888,"optimality":0.7868440995,"n_forwards":1000.0,"n_backwards":1000.0,"time":152.6892654549},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9602760894,"n_forwards":999.0,"n_backwards":999.0,"time":147.937349674},{"dataset":"imagenet","attack":"EAD","library":"Foolbox","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9491534077,"n_forwards":420.0,"n_backwards":214.0,"time":37.8202483305},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"wong_2020","threat_model":"l1","ASR":0.8862,"optimality":0.7830130202,"n_forwards":1000.0,"n_backwards":1000.0,"time":154.0011101579},{"dataset":"imagenet","attack":"APGD-$\\ell_1$","library":"Original","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9940243079,"n_forwards":562.0,"n_backwards":542.0,"time":418.1881578825},{"dataset":"imagenet","attack":"APGD$_t$-$\\ell_1$","library":"Original","model":"wong_2020","threat_model":"l1","ASR":1.0,"optimality":0.9774878932,"n_forwards":455.0,"n_backwards":419.0,"time":405.1188796804},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"wong_2020","threat_model":"l1","ASR":0.8862,"optimality":0.7830130202,"n_forwards":1000.0,"n_backwards":1000.0,"time":154.3459050371}] -------------------------------------------------------------------------------- /compiled/imagenet/wong_2020_l2.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"wong_2020","threat_model":"l2","ASR":1.0,"optimality":0.9623023077,"n_forwards":1000.0,"n_backwards":990.0,"time":143.9423899714},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"wong_2020","threat_model":"l2","ASR":0.9996,"optimality":0.9837289908,"n_forwards":698.0,"n_backwards":678.0,"time":103.5438655585},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"wong_2020","threat_model":"l2","ASR":1.0,"optimality":0.9608868667,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.5135975574},{"dataset":"imagenet","attack":"DDN","library":"AdvLib","model":"wong_2020","threat_model":"l2","ASR":1.0,"optimality":0.9852192899,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.4095577939},{"dataset":"imagenet","attack":"FMN","library":"AdvLib","model":"wong_2020","threat_model":"l2","ASR":0.9954,"optimality":0.9684692089,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.5182526739},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"wong_2020","threat_model":"l2","ASR":1.0,"optimality":0.9747911816,"n_forwards":1000.0,"n_backwards":990.0,"time":368.7410658009},{"dataset":"imagenet","attack":"DDN","library":"Foolbox","model":"wong_2020","threat_model":"l2","ASR":1.0,"optimality":0.9850000896,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.2580250671},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"wong_2020","threat_model":"l2","ASR":0.5358,"optimality":0.2323650506,"n_forwards":498.0,"n_backwards":481.0,"time":258.8794676693},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"wong_2020","threat_model":"l2","ASR":0.8918,"optimality":0.7940247711,"n_forwards":430.0,"n_backwards":393.0,"time":266.2826384803},{"dataset":"imagenet","attack":"FMN","library":"Original","model":"wong_2020","threat_model":"l2","ASR":0.9942,"optimality":0.9648092846,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.5742689653}] -------------------------------------------------------------------------------- /compiled/imagenet/wong_2020_linf.json: -------------------------------------------------------------------------------- 1 | [{"dataset":"imagenet","attack":"APGD","library":"AdvLib","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.965524378,"n_forwards":1000.0,"n_backwards":990.0,"time":143.7602953323},{"dataset":"imagenet","attack":"APGD$_t$","library":"AdvLib","model":"wong_2020","threat_model":"linf","ASR":0.997,"optimality":0.9866963206,"n_forwards":787.0,"n_backwards":767.0,"time":115.798405346},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"AdvLib","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.9846051587,"n_forwards":1000.0,"n_backwards":1000.0,"time":143.6372293564},{"dataset":"imagenet","attack":"PDPGD","library":"AdvLib","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.985593579,"n_forwards":986.0,"n_backwards":986.0,"time":185.0423487737},{"dataset":"imagenet","attack":"PGD","library":"AdvLib","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.968460603,"n_forwards":1000.0,"n_backwards":990.0,"time":141.8431305581},{"dataset":"imagenet","attack":"BIM","library":"Foolbox","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.9737428155,"n_forwards":1000.0,"n_backwards":990.0,"time":312.4031933732},{"dataset":"imagenet","attack":"FMN","library":"Foolbox","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.9845306536,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.7665351734},{"dataset":"imagenet","attack":"APGD","library":"Original","model":"wong_2020","threat_model":"linf","ASR":0.5478,"optimality":0.3503839328,"n_forwards":503.0,"n_backwards":485.0,"time":215.3366097382},{"dataset":"imagenet","attack":"APGD$_t$","library":"Original","model":"wong_2020","threat_model":"linf","ASR":0.8268,"optimality":0.7097604494,"n_forwards":455.0,"n_backwards":417.0,"time":223.3192178813},{"dataset":"imagenet","attack":"FMN-$\\ell_\\infty$","library":"Original","model":"wong_2020","threat_model":"linf","ASR":1.0,"optimality":0.9845306536,"n_forwards":1000.0,"n_backwards":1000.0,"time":144.4676572579}] -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: atkbench 2 | channels: 3 | - pytorch 4 | - defaults 5 | - conda-forge 6 | dependencies: 7 | - matplotlib=3.5.1 8 | - pip=21.2.4 9 | - python=3.9 10 | - pytorch=1.12.1 11 | - torchvision=0.13.1 12 | - cleverhans=4.0.0 13 | - pip: 14 | - adversarial-robustness-toolbox==1.14.0 15 | - foolbox==3.3.3 16 | - git+https://github.com/RobustBench/robustbench.git 17 | - git+https://github.com/jeromerony/adversarial-library.git 18 | - deeprobust==0.2.8 19 | - pandas==1.3.5 20 | - sacred==0.8.4 21 | - scipy<1.8.0 22 | - torchattacks==3.4.0 23 | - tqdm==4.56.2 24 | - wget==3.2 25 | 26 | -------------------------------------------------------------------------------- /exp_configs/cifar10_l0_1000_bs128.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "l0" 6 | }, 7 | "dataset": { 8 | "batch_size": 128 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "l0_1000_attacks.json", 17 | "model": "cifar10_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/cifar10_l1_1000_bs128.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "l1" 6 | }, 7 | "dataset": { 8 | "batch_size": 128 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "l1_1000_attacks.json", 17 | "model": "cifar10_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/cifar10_l2_1000_bs128.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "l2" 6 | }, 7 | "dataset": { 8 | "batch_size": 128 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "l2_1000_attacks.json", 17 | "model": "cifar10_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/cifar10_linf_1000.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "linf" 6 | }, 7 | "dataset": { 8 | "batch_size": 128 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "linf_1000_attacks.json", 17 | "model": "cifar10_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/cifar10_linf_1000_bs1.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "linf" 6 | }, 7 | "dataset": { 8 | "batch_size": 1 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "linf_1000_attacks.json", 17 | "model": "cifar10_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/cifar10_linf_1000_bs128.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "linf" 6 | }, 7 | "dataset": { 8 | "batch_size": 128 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "linf_1000_attacks.json", 17 | "model": "cifar10_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/cifar10_models.json: -------------------------------------------------------------------------------- 1 | [ 2 | "standard", 3 | "stutz_2020", 4 | "zhang_2020_small", 5 | "xiao_2020", 6 | "wang_2023_small" 7 | ] -------------------------------------------------------------------------------- /exp_configs/cifar10_models_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "Standard": "standard", 3 | "Stutz2020CCAT": "stutz_2020", 4 | "Zhang2020CrownSmall": "zhang_2020_small", 5 | "Xiao2020KWTA": "xiao_2020", 6 | "Wang2023DMAdvSmall": "wang_2023_small" 7 | } -------------------------------------------------------------------------------- /exp_configs/imagenet_l0_1000.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "l0" 6 | }, 7 | "dataset": { 8 | "batch_size": 32, 9 | "num_samples": 5000 10 | }, 11 | "model": { 12 | "num_max_propagations": 2000 13 | }, 14 | "seed": 42 15 | }, 16 | "cartesian": { 17 | "attack": "l0_1000_attacks.json", 18 | "model": "imagenet_models.json" 19 | } 20 | } -------------------------------------------------------------------------------- /exp_configs/imagenet_l1_1000.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "l1" 6 | }, 7 | "dataset": { 8 | "batch_size": 32, 9 | "num_samples": 5000 10 | }, 11 | "model": { 12 | "num_max_propagations": 2000 13 | }, 14 | "seed": 42 15 | }, 16 | "cartesian": { 17 | "attack": { 18 | "adv_lib_alma_l1": { 19 | "num_steps": 1000 20 | }, 21 | "adv_lib_apgd_minimal": { 22 | "num_steps": 96, 23 | "search_steps": 10, 24 | "use_large_reps": true 25 | }, 26 | "adv_lib_apgd_t_minimal": { 27 | "num_steps": 19, 28 | "search_steps": 10, 29 | "num_targets": 5, 30 | "use_large_reps": true 31 | }, 32 | "adv_lib_fmn": { 33 | "num_steps": 1000 34 | }, 35 | "adv_lib_pdpgd": { 36 | "num_steps": 1000 37 | }, 38 | "fb_ead": { 39 | "num_steps": 74 40 | }, 41 | "fb_fmn": { 42 | "num_steps": 1000 43 | }, 44 | "original_apgd_minimal_l1": { 45 | "num_steps": 96, 46 | "search_steps": 10 47 | }, 48 | "original_apgd_t_minimal_l1": { 49 | "num_steps": 22, 50 | "search_steps": 10, 51 | "num_target_classes": 5 52 | }, 53 | "original_fmn": { 54 | "num_steps": 1000 55 | } 56 | }, 57 | "model": "imagenet_models.json" 58 | } 59 | } -------------------------------------------------------------------------------- /exp_configs/imagenet_l2_1000.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "l2" 6 | }, 7 | "dataset": { 8 | "batch_size": 32, 9 | "num_samples": 5000 10 | }, 11 | "model": { 12 | "num_max_propagations": 2000 13 | }, 14 | "seed": 42 15 | }, 16 | "cartesian": { 17 | "attack": { 18 | "adv_lib_apgd_minimal": { 19 | "num_steps": 98, 20 | "search_steps": 10 21 | }, 22 | "adv_lib_apgd_t_minimal": { 23 | "num_steps": 20, 24 | "search_steps": 10, 25 | "num_targets": 5 26 | }, 27 | "adv_lib_ddn": { 28 | "num_steps": 1000 29 | }, 30 | "adv_lib_fmn": { 31 | "num_steps": 1000 32 | }, 33 | "fb_ddn": { 34 | "num_steps": 1000 35 | }, 36 | "fb_bim_minimal": { 37 | "num_steps": 99, 38 | "search_steps": 10 39 | }, 40 | "original_apgd_minimal": { 41 | "num_steps": 98, 42 | "search_steps": 10 43 | }, 44 | "original_apgd_t_minimal": { 45 | "num_steps": 22, 46 | "search_steps": 10, 47 | "num_target_classes": 5 48 | }, 49 | "original_fmn": { 50 | "num_steps": 1000 51 | } 52 | }, 53 | "model": "imagenet_models.json" 54 | } 55 | } -------------------------------------------------------------------------------- /exp_configs/imagenet_linf_1000.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "linf" 6 | }, 7 | "dataset": { 8 | "batch_size": 32, 9 | "num_samples": 5000 10 | }, 11 | "model": { 12 | "num_max_propagations": 2000 13 | }, 14 | "seed": 42 15 | }, 16 | "cartesian": { 17 | "attack": { 18 | "original_apgd_t_minimal": { 19 | "num_steps": 22, 20 | "search_steps": 10, 21 | "num_target_classes": 5 22 | }, 23 | "original_apgd_minimal": { 24 | "num_steps": 98, 25 | "search_steps": 10 26 | }, 27 | "adv_lib_apgd_minimal": { 28 | "num_steps": 98, 29 | "search_steps": 10 30 | }, 31 | "adv_lib_apgd_t_minimal": { 32 | "num_steps": 20, 33 | "search_steps": 10, 34 | "num_targets": 5 35 | }, 36 | "fb_bim_minimal": { 37 | "num_steps": 99, 38 | "search_steps": 10 39 | }, 40 | "adv_lib_pdpgd": { 41 | "num_steps": 1000 42 | }, 43 | "adv_lib_pgd_minimal": { 44 | "num_steps": 99, 45 | "search_steps": 10 46 | }, 47 | "original_fmn_linf": { 48 | "num_steps": 1000 49 | }, 50 | "adv_lib_fmn_linf": { 51 | "num_steps": 1000 52 | }, 53 | "fb_fmn": { 54 | "num_steps": 1000, 55 | "max_stepsize": 10 56 | } 57 | }, 58 | "model": "imagenet_models.json" 59 | } 60 | } -------------------------------------------------------------------------------- /exp_configs/imagenet_models.json: -------------------------------------------------------------------------------- 1 | [ 2 | "standard_imagenet", 3 | "debenedetti_2022", 4 | "salman_2020", 5 | "wong_2020" 6 | ] -------------------------------------------------------------------------------- /exp_configs/imagenet_models_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "Standard_R50": "standard_imagenet", 3 | "Wong2020Fast": "wong_2020", 4 | "Salman2020Do_50_2": "salman_2020", 5 | "Debenedetti2022Light_XCiT-S12": "debenedetti_2022" 6 | } -------------------------------------------------------------------------------- /exp_configs/l0_1000_attacks.json: -------------------------------------------------------------------------------- 1 | { 2 | "adv_lib_fmn": { 3 | "num_steps": 1000 4 | }, 5 | "adv_lib_pdpgd": { 6 | "num_steps": 1000 7 | }, 8 | "adv_lib_vfga": { 9 | "num_steps": 90 10 | }, 11 | "original_fmn": { 12 | "num_steps": 1000 13 | } 14 | } -------------------------------------------------------------------------------- /exp_configs/l1_1000_attacks.json: -------------------------------------------------------------------------------- 1 | { 2 | "adv_lib_alma_l1": { 3 | "num_steps": 1000 4 | }, 5 | "adv_lib_apgd_minimal": { 6 | "num_steps": 96, 7 | "search_steps": 10, 8 | "use_large_reps": true 9 | }, 10 | "adv_lib_apgd_t_minimal": { 11 | "num_steps": 19, 12 | "search_steps": 10, 13 | "num_targets": 5, 14 | "use_large_reps": true 15 | }, 16 | "adv_lib_fab": { 17 | "num_steps": 180 18 | }, 19 | "adv_lib_fmn": { 20 | "num_steps": 1000 21 | }, 22 | "adv_lib_pdpgd": { 23 | "num_steps": 1000 24 | }, 25 | "art_apgd_minimal": { 26 | "num_steps": 13, 27 | "search_steps": 10 28 | }, 29 | "art_ead": { 30 | "num_steps": 13 31 | }, 32 | "art_fgm_minimal": {}, 33 | "art_pgd_minimal": { 34 | "num_steps": 98, 35 | "search_steps": 10 36 | }, 37 | "fb_bim_minimal": {}, 38 | "fb_bb": { 39 | "num_steps": 225 40 | }, 41 | "fb_ead": { 42 | "num_steps": 74 43 | }, 44 | "fb_fgm_minimal": {}, 45 | "fb_fmn": { 46 | "num_steps": 1000 47 | }, 48 | "fb_pgd_minimal": { 49 | "num_steps": 99, 50 | "search_steps": 10 51 | }, 52 | "original_apgd_minimal_l1": { 53 | "num_steps": 96, 54 | "search_steps": 10 55 | }, 56 | "original_apgd_t_minimal_l1": { 57 | "num_steps": 22, 58 | "search_steps": 10, 59 | "num_target_classes": 5 60 | }, 61 | "original_fab": { 62 | "num_steps": 163 63 | }, 64 | "original_fmn": { 65 | "num_steps": 1000 66 | }, 67 | "ta_fab": { 68 | "num_steps": 163 69 | } 70 | } -------------------------------------------------------------------------------- /exp_configs/l2_1000_attacks.json: -------------------------------------------------------------------------------- 1 | { 2 | "adv_lib_alma_l2": { 3 | "num_steps": 1000 4 | }, 5 | "adv_lib_apgd_minimal": { 6 | "num_steps": 98, 7 | "search_steps": 10 8 | }, 9 | "adv_lib_apgd_t_minimal": { 10 | "num_steps": 20, 11 | "search_steps": 10, 12 | "num_targets": 5 13 | }, 14 | "adv_lib_cw_l2": { 15 | "num_steps": 111 16 | }, 17 | "adv_lib_ddn": { 18 | "num_steps": 1000 19 | }, 20 | "adv_lib_fab": { 21 | "num_steps": 180 22 | }, 23 | "adv_lib_fmn": { 24 | "num_steps": 1000 25 | }, 26 | "adv_lib_pdgd": { 27 | "num_steps": 1000 28 | }, 29 | "adv_lib_pdpgd": { 30 | "num_steps": 1000 31 | }, 32 | "adv_lib_pgd_minimal": { 33 | "num_steps": 99, 34 | "search_steps": 10 35 | }, 36 | "adv_lib_tr": { 37 | "num_steps": 663 38 | }, 39 | "adv_lib_tr_adaptive": { 40 | "num_steps": 497 41 | }, 42 | "art_apgd_minimal": { 43 | "num_steps": 13, 44 | "search_steps": 10 45 | }, 46 | "art_cw_l2": { 47 | "num_steps": 10 48 | }, 49 | "art_deepfool": { 50 | "num_steps": 1000 51 | }, 52 | "art_fgm_minimal": {}, 53 | "art_pgd_minimal": { 54 | "num_steps": 98, 55 | "search_steps": 10 56 | }, 57 | "ch_cw_l2": { 58 | "num_steps": 200 59 | }, 60 | "ch_fgm_minimal": {}, 61 | "dr_pgd_minimal": { 62 | "num_steps": 39, 63 | "search_steps": 10 64 | }, 65 | "dr_fgm_minimal": {}, 66 | "fb_bb": { 67 | "num_steps": 225 68 | }, 69 | "fb_cw_l2": { 70 | "num_steps": 111 71 | }, 72 | "fb_ddn": { 73 | "num_steps": 1000 74 | }, 75 | "fb_deepfool": { 76 | "num_steps": 111 77 | }, 78 | "fb_fmn": { 79 | "num_steps": 1000 80 | }, 81 | "fb_pgd_minimal": { 82 | "num_steps": 99, 83 | "search_steps": 10 84 | }, 85 | "fb_fgm_minimal": {}, 86 | "fb_bim_minimal": { 87 | "num_steps": 99, 88 | "search_steps": 10 89 | }, 90 | "original_apgd_minimal": { 91 | "num_steps": 98, 92 | "search_steps": 10 93 | }, 94 | "original_apgd_t_minimal": { 95 | "num_steps": 22, 96 | "search_steps": 10, 97 | "num_target_classes": 5 98 | }, 99 | "original_deepfool": { 100 | "num_steps": 1000 101 | }, 102 | "original_fab": { 103 | "num_steps": 163 104 | }, 105 | "original_fmn": { 106 | "num_steps": 1000 107 | }, 108 | "original_tr": { 109 | "num_steps": 663 110 | }, 111 | "original_tr_adaptive": { 112 | "num_steps": 497 113 | }, 114 | "ta_apgd_minimal": { 115 | "num_steps": 98, 116 | "search_steps": 10 117 | }, 118 | "ta_cw_l2": { 119 | "num_steps": 1000 120 | }, 121 | "ta_fab": { 122 | "num_steps": 163 123 | }, 124 | "ta_pgd_l2_minimal": { 125 | "num_steps": 99, 126 | "search_steps": 10 127 | } 128 | } -------------------------------------------------------------------------------- /exp_configs/linf_1000_attacks.json: -------------------------------------------------------------------------------- 1 | { 2 | "adv_lib_apgd_minimal": { 3 | "num_steps": 98, 4 | "search_steps": 10 5 | }, 6 | "adv_lib_apgd_t_minimal": { 7 | "num_steps": 20, 8 | "search_steps": 10, 9 | "num_targets": 5 10 | }, 11 | "adv_lib_cw_linf": { 12 | "num_steps": 40 13 | }, 14 | "adv_lib_fab": { 15 | "num_steps": 181 16 | }, 17 | "adv_lib_fmn_linf": { 18 | "num_steps": 1000 19 | }, 20 | "adv_lib_pdpgd": { 21 | "num_steps": 1000 22 | }, 23 | "adv_lib_pgd_minimal": { 24 | "num_steps": 99, 25 | "search_steps": 10 26 | }, 27 | "adv_lib_tr": { 28 | "num_steps": 1000 29 | }, 30 | "adv_lib_tr_adaptive": { 31 | "num_steps": 1000 32 | }, 33 | "art_apgd_minimal": { 34 | "num_steps": 100, 35 | "search_steps": 10 36 | }, 37 | "art_bb": { 38 | "num_steps": 658 39 | }, 40 | "art_cw_linf": { 41 | "num_steps": 29 42 | }, 43 | "art_fgm_minimal": {}, 44 | "art_pgd_minimal": { 45 | "num_steps": 98, 46 | "search_steps": 10 47 | }, 48 | "ch_fgm_minimal": {}, 49 | "dr_pgd_minimal": { 50 | "num_steps": 99, 51 | "search_steps": 10 52 | }, 53 | "dr_fgm_minimal": {}, 54 | "fb_bb": { 55 | "num_steps": 700 56 | }, 57 | "fb_deepfool": { 58 | "num_steps": 1000 59 | }, 60 | "fb_fmn": { 61 | "num_steps": 1000, 62 | "max_stepsize": 10 63 | }, 64 | "fb_pgd_minimal": { 65 | "num_steps": 99, 66 | "search_steps": 10 67 | }, 68 | "fb_fgm_minimal": {}, 69 | "fb_bim_minimal": { 70 | "num_steps": 99, 71 | "search_steps": 10 72 | }, 73 | "original_apgd_minimal": { 74 | "num_steps": 98, 75 | "search_steps": 10 76 | }, 77 | "original_apgd_t_minimal": { 78 | "num_steps": 22, 79 | "search_steps": 10, 80 | "num_target_classes": 5 81 | }, 82 | "original_fab": { 83 | "num_steps": 166 84 | }, 85 | "original_fmn_linf": { 86 | "num_steps": 1000 87 | }, 88 | "original_tr": { 89 | "num_steps": 1000 90 | }, 91 | "original_tr_adaptive": { 92 | "num_steps": 1000 93 | }, 94 | "ta_apgd_minimal": { 95 | "num_steps": 98, 96 | "search_steps": 10 97 | }, 98 | "ta_fab": { 99 | "num_steps": 166 100 | }, 101 | "ta_fgsm_minimal": {}, 102 | "ta_pgd_minimal": { 103 | "num_steps": 99, 104 | "search_steps": 10 105 | } 106 | } -------------------------------------------------------------------------------- /exp_configs/mnist_linf_1000.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "named_configs": [], 4 | "attack": { 5 | "threat_model": "linf" 6 | }, 7 | "dataset": { 8 | "batch_size": 128 9 | }, 10 | "model": { 11 | "num_max_propagations": 2000 12 | }, 13 | "seed": 42 14 | }, 15 | "cartesian": { 16 | "attack": "linf_1000_attacks.json", 17 | "model": "mnist_models.json" 18 | } 19 | } -------------------------------------------------------------------------------- /exp_configs/mnist_models.json: -------------------------------------------------------------------------------- 1 | [ 2 | "mnist_smallcnn", 3 | "mnist_smallcnn_dnn", 4 | "mnist_smallcnn_trades" 5 | ] -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | adv_lib @ git+https://github.com/jeromerony/adversarial-library 2 | foolbox 3 | robustbench @ git+https://github.com/RobustBench/robustbench 4 | sacred --------------------------------------------------------------------------------