├── .gitignore ├── .gitmodules ├── MANIFEST.in ├── README.md ├── demo_scoring.ipynb ├── lib_demo.ipynb ├── pyproject.toml ├── requirements.txt ├── res ├── amifgsm_maniqa_test.csv ├── cumulative-uap_maniqa_COCO.npy ├── cumulative-uap_maniqa_COCO.png ├── cumulative-uap_maniqa_VOC2012.npy ├── cumulative-uap_maniqa_VOC2012.png ├── cumulative-uap_maniqa_test.csv ├── generative-uap_maniqa_COCO.npy ├── generative-uap_maniqa_COCO.png ├── generative-uap_maniqa_VOC2012.npy ├── generative-uap_maniqa_VOC2012.png ├── generative-uap_maniqa_test.csv ├── ifgsm_maniqa_test.csv ├── korhonen-et-al_maniqa_test.csv ├── madc_maniqa_test.csv ├── mifgsm_maniqa_test.csv ├── std-fgsm_maniqa_test.csv ├── uap_maniqa_COCO.npy ├── uap_maniqa_COCO.png ├── uap_maniqa_VOC2012.npy ├── uap_maniqa_VOC2012.png └── uap_maniqa_test.csv ├── robustness_benchmark ├── NOT.py ├── __init__.py ├── interface.py ├── methods │ ├── __init__.py │ ├── amifgsm │ │ └── run.py │ ├── cumulative-uap │ │ ├── run.py │ │ └── train.py │ ├── generative-uap │ │ ├── run.py │ │ └── train.py │ ├── ifgsm │ │ └── run.py │ ├── korhonen-et-al │ │ ├── .gitkeep │ │ └── run.py │ ├── madc │ │ └── run.py │ ├── mifgsm │ │ └── run.py │ ├── std-fgsm │ │ └── run.py │ ├── uap │ │ ├── run.py │ │ └── train.py │ └── utils │ │ ├── __init__.py │ │ ├── bounds.json │ │ ├── evaluate.py │ │ ├── metrics.py │ │ └── read_dataset.py ├── models │ ├── .gitkeep │ └── models_to_mdtvsfa_1d.pth ├── requirements.txt └── score_methods.py ├── setup.cfg ├── subjects ├── _init │ └── Dockerfile └── maniqa │ ├── Dockerfile │ ├── config.json │ ├── model.py │ ├── patches │ └── maniqa.patch │ └── test.py ├── test_dataset ├── 000b7d55b6184b08.png ├── 00c3cd597f1ee96f.png ├── 0aebe24fc257286e.png ├── 0af0a5dfee6b84ff.png └── 0b1d45bd9ab1064e.png └── test_results ├── amifgsm └── maniqa.csv ├── cumulative-uap └── maniqa.csv ├── generative-uap └── maniqa.csv ├── ifgsm └── maniqa.csv ├── korhonen-et-al └── maniqa.csv ├── madc └── maniqa.csv ├── mifgsm └── maniqa.csv ├── std-fgsm └── maniqa.csv └── uap └── maniqa.csv /.gitignore: -------------------------------------------------------------------------------- 1 | # Python compiled 2 | *.pyc 3 | 4 | # Model checkpoints 5 | *.pt 6 | *.pth.1 7 | *.mat 8 | data/ 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | share/python-wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .nox/ 52 | .coverage 53 | .coverage.* 54 | .cache 55 | nosetests.xml 56 | coverage.xml 57 | *.cover 58 | *.py,cover 59 | .hypothesis/ 60 | .pytest_cache/ 61 | cover/ 62 | 63 | # Translations 64 | *.mo 65 | *.pot 66 | 67 | # Django stuff: 68 | *.log 69 | local_settings.py 70 | db.sqlite3 71 | db.sqlite3-journal 72 | 73 | # Flask stuff: 74 | instance/ 75 | .webassets-cache 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | .pybuilder/ 85 | target/ 86 | 87 | # Jupyter Notebook 88 | .ipynb_checkpoints 89 | 90 | # IPython 91 | profile_default/ 92 | ipython_config.py 93 | 94 | # pyenv 95 | # For a library or package, you might want to ignore these files since the code is 96 | # intended to run in multiple environments; otherwise, check them in: 97 | # .python-version 98 | 99 | # pipenv 100 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 101 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 102 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 103 | # install all needed dependencies. 104 | #Pipfile.lock 105 | 106 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 107 | __pypackages__/ 108 | 109 | # Celery stuff 110 | celerybeat-schedule 111 | celerybeat.pid 112 | 113 | # SageMath parsed files 114 | *.sage.py 115 | 116 | # Environments 117 | .env 118 | .venv 119 | env/ 120 | venv/ 121 | ENV/ 122 | env.bak/ 123 | venv.bak/ 124 | 125 | # Spyder project settings 126 | .spyderproject 127 | .spyproject 128 | 129 | # Rope project settings 130 | .ropeproject 131 | 132 | # mkdocs documentation 133 | /site 134 | 135 | # mypy 136 | .mypy_cache/ 137 | .dmypy.json 138 | dmypy.json 139 | 140 | # Pyre type checker 141 | .pyre/ 142 | 143 | # pytype static type analyzer 144 | .pytype/ 145 | 146 | # Cython debug symbols 147 | cython_debug/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "subjects/maniqa/src"] 2 | path = subjects/maniqa/src 3 | url = https://github.com/IIGROUP/MANIQA.git 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include robustness_benchmark/models/models_to_mdtvsfa_1d.pth 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Metrics Robustness Benchmark 2 | Repository for "Comparing the robustness of modern image- and video-quality metrics to adversarial attacks" paper 3 | 4 | 5 | ## General requirements 6 | Python3, Jypyter, GPU (with CUDA), Docker (if you want to launch adversarial attacks) 7 | 8 | ## Repository structure 9 | - ```robustness_benchmark/methods/``` - adversarial attacks and utils 10 | - ```robustness_benchmark/methods/utils/``` - supportive functions for attacks 11 | - ```robustness_benchmark/models/``` - domain transformation model weights 12 | - ```robustness_benchmark/interface.py``` - benchmark module interface. Main functions are ```run_attacks(), collect_results(), domain_transform(), evaluate_robustness(), run_full_pipeline()```. More details on usage in functions' docstrings and demo Notebook ```lib_demo.ipynb```. 13 | - ```subjects/``` - metrics code (only MANIQA metric for demo) 14 | - ```res/``` - precomputed results (only MANIQA metric for demo) 15 | - ```test_dataset/``` - small test set of images to test module functionality 16 | - ```test_results/``` - results for ```test_dataset``` (MANIQA) 17 | 18 | Demo code: 19 | - ```lib_demo.ipynb``` - benchmark module usage 20 | - ```demo_scoring.ipynb``` - calculate robustness scores for MANIQA (using precomputed results from /res/) 21 | 22 | 23 | Supplementary code: 24 | - ```robustness_benchmark/score_methods.py``` - functions to calculate attack efficiency scores (described in paper in "Methodology" section) 25 | - ```robustness_benchmark/NOT.py``` - functions to perform Neural Optimal Transport for mapping metrics values to one domain 26 | 27 | ## Running the demo code 28 | ### Robusness Benchmark pip module 29 | #### Module installation 30 | Note: It is recommended to install PyTorch version suitable for your Python/CUDA installation from [official website](https://pytorch.org/) before installing the library.\ 31 | Direct install via ```pip install robustness_benchmark``` will be available soon. To install the latest version of the module you can clone the repo and pip install it:\ 32 | ```git clone https://github.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/tree/main```\ 33 | ```pip install -r requirements.txt```\ 34 | ```pip install -e .``` 35 | 36 | #### Demo metric setup 37 | To install demo metric MANIQA you can use following commands in benchmark's root directory: 38 | 39 | ```git submodule update --init --recursive```\ 40 | ```cd subjects/maniqa/src && git apply ../patches/maniqa.patch -v```\ 41 | ```cd subjects/maniqa && cp model.py src``` 42 | 43 | #### Launch adversarial attacks 44 | Example usage of module can be found in ```lib_demo.ipynb``` Notebook. To run the attacks and evaluate metric's robustness you can run it cell-by-cell. You can also test other metrics and attacks if they follow the module interface. Main library funtions are listed below, for more details check functions' docstrings. 45 | #### robustness_benchmark.interface functions 46 | 1. ```run_attacks()``` - Run given attacks on metric on specified datasets. 47 | 2. ```collect_results()``` - Given the path to directory with raw results produced by ```run_attack()```, collects them into single DataFrame. 48 | 3. ```domain_transform()``` - Apply domain transformation to collected attack results from ```collect_results()```. Works only with supported metrics. 49 | 4. ```evaluate_robustness()``` - Evaluate metric's robustness to attacks and return table with results on each type of attack. 50 | 5. ```run_full_pipeline()``` - Run full benchmark pipeline: run attacks, save results, collect them, apply domain transform (if needed), evaluate. 51 | 52 | ### Using precomputed results 53 | 1. Download precomputed results used in article from [here](https://calypso.gml-team.ru:5001/sharing/NFLRz05g9) (password: 'neurips_benchmark_2023') 54 | 2. Clone this repo: ```git clone https://github.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/tree/main``` 55 | 3. Install requirements: ```pip install -r requirements.txt``` 56 | 4. Launch Jupyter noteboor or jupyter lab 57 | 5. Launch demo_scoring.ipynb cell-by-cell 58 | 59 | 61 | 62 | 74 | 75 | 81 | 82 | ## Cite us 83 | ``` 84 | @article{Antsiferova_Abud_Gushchin_Shumitskaya_Lavrushkin_Vatolin_2024, 85 | title={Comparing the Robustness of Modern No-Reference Image- and Video-Quality Metrics to Adversarial Attacks}, 86 | author={Antsiferova, Anastasia and Abud, Khaled and Gushchin, Aleksandr and Shumitskaya, Ekaterina and Lavrushkin, Sergey and Vatolin, Dmitriy}, 87 | journal={Proceedings of the AAAI Conference on Artificial Intelligence}, 88 | volume={38}, 89 | url={https://ojs.aaai.org/index.php/AAAI/article/view/27827}, 90 | DOI={10.1609/aaai.v38i2.27827}, 91 | number={2}, 92 | year={2024}, 93 | month={Mar.}, 94 | pages={700-708} 95 | } 96 | ``` 97 | -------------------------------------------------------------------------------- /demo_scoring.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%load_ext autoreload\n", 10 | "import numpy as np\n", 11 | "import pandas as pd\n", 12 | "#import seaborn as sns\n", 13 | "#from matplotlib import pyplot as plt\n", 14 | "#from tqdm import tqdm\n", 15 | "import scipy.stats as st\n", 16 | "import os\n", 17 | "from pathlib import Path \n", 18 | "import re" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "from robustness_benchmark.NOT import transform_full_df, load_models\n", 28 | "from robustness_benchmark.score_methods import *" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 5, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "# get attack name from filename\n", 38 | "def get_atk(filename, metric):\n", 39 | " pattern = f\"(.+)_{metric}\"\n", 40 | " match = re.search(pattern, filename)\n", 41 | " if match:\n", 42 | " return match.group(1)\n", 43 | " else:\n", 44 | " return None\n", 45 | "\n", 46 | "\n", 47 | "# parse results for one metric\n", 48 | "def parse_results(path='./res', metric='maniqa'):\n", 49 | " required_cols = ['clear', 'attacked', 'ssim', 'psnr', 'mse']\n", 50 | " uap_amps = [0.2, 0.4, 0.8]\n", 51 | " uap_train_dsets = ['VOC2012', 'COCO']\n", 52 | " result_df = pd.DataFrame()\n", 53 | " for file in Path(path).iterdir():\n", 54 | " if not str(file).endswith('test.csv'):\n", 55 | " continue\n", 56 | " atk_name = get_atk(str(file.stem), metric)\n", 57 | " if atk_name is None:\n", 58 | " continue\n", 59 | " \n", 60 | " #print(atk_name)\n", 61 | " cur_df = pd.read_csv(file).rename(columns={'test_dataset': 'dataset'})\n", 62 | "\n", 63 | " if 'uap' not in str(file.stem):\n", 64 | " data_to_add = cur_df[['dataset'] + required_cols].copy()\n", 65 | " data_to_add = data_to_add.rename(columns={x: f'{metric}_{x}' for x in required_cols})\n", 66 | " data_to_add['attack'] = atk_name\n", 67 | " data_to_add.dataset.replace(to_replace={'NIPS2017': 'NIPS'}, inplace=True)\n", 68 | " result_df = pd.concat([result_df, data_to_add], axis=0)\n", 69 | " else:\n", 70 | " for amp in uap_amps:\n", 71 | " for train_set in uap_train_dsets:\n", 72 | " cur_atk_name = atk_name\n", 73 | " if atk_name == 'uap':\n", 74 | " cur_atk_name = 'default-uap'\n", 75 | " cur_atk_name += f'_{train_set}_amp{str(amp)}'\n", 76 | " data_to_add = cur_df[cur_df['amplitude'] == amp][cur_df['train_dataset'] == train_set][['dataset'] + required_cols].copy()\n", 77 | " data_to_add = data_to_add.rename(columns={x: f'{metric}_{x}' for x in required_cols})\n", 78 | " data_to_add['attack'] = cur_atk_name\n", 79 | " data_to_add.dataset.replace(to_replace={'NIPS2017': 'NIPS'}, inplace=True)\n", 80 | " result_df = pd.concat([result_df, data_to_add], axis=0)\n", 81 | " return result_df\n" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 9, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "metric_to_evaluate = 'maniqa'\n", 91 | "\n", 92 | "# parse results from demo directory after running attacks\n", 93 | "#data = parse_results(path='./res', metric=metric_to_evaluate)\n", 94 | "\n", 95 | "# results as of the time of writing\n", 96 | "# available at https://calypso.gml-team.ru:5001/sharing/NFLRz05g9 (password: 'neurips_benchmark_2023')\n", 97 | "path_to_results = './results.ft'\n", 98 | "data = pd.read_feather(path_to_results)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 10, 104 | "metadata": {}, 105 | "outputs": [ 106 | { 107 | "data": { 108 | "text/html": [ 109 | "
\n", 110 | "\n", 123 | "\n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | "
datasetattackmaniqa_clearmaniqa_attackedmaniqa_ssimmaniqa_psnrmaniqa_mse
0NIPSamifgsm0.4975641.1093800.81034230.2223430.000950
1NIPSamifgsm0.5310791.0777010.87549729.7262150.001065
2NIPSamifgsm0.6687031.1143590.85574229.7287920.001064
3NIPSamifgsm0.5684881.0903260.90685129.7793490.001052
4NIPSamifgsm0.6597131.0739180.74734030.4071430.000911
........................
77275VIMEOdefault-uap_COCO_amp0.80.5206850.1494980.38437424.4418530.003596
77276VIMEOdefault-uap_COCO_amp0.80.4118540.2045670.54667324.4418530.003596
77277VIMEOdefault-uap_COCO_amp0.80.3703960.1785190.50635224.4418530.003596
77278VIMEOdefault-uap_COCO_amp0.80.4200290.2031370.39106024.4418530.003596
77279VIMEOdefault-uap_COCO_amp0.80.4041970.1686230.50605124.4418530.003596
\n", 249 | "

77280 rows × 7 columns

\n", 250 | "
" 251 | ], 252 | "text/plain": [ 253 | " dataset attack maniqa_clear maniqa_attacked \\\n", 254 | "0 NIPS amifgsm 0.497564 1.109380 \n", 255 | "1 NIPS amifgsm 0.531079 1.077701 \n", 256 | "2 NIPS amifgsm 0.668703 1.114359 \n", 257 | "3 NIPS amifgsm 0.568488 1.090326 \n", 258 | "4 NIPS amifgsm 0.659713 1.073918 \n", 259 | "... ... ... ... ... \n", 260 | "77275 VIMEO default-uap_COCO_amp0.8 0.520685 0.149498 \n", 261 | "77276 VIMEO default-uap_COCO_amp0.8 0.411854 0.204567 \n", 262 | "77277 VIMEO default-uap_COCO_amp0.8 0.370396 0.178519 \n", 263 | "77278 VIMEO default-uap_COCO_amp0.8 0.420029 0.203137 \n", 264 | "77279 VIMEO default-uap_COCO_amp0.8 0.404197 0.168623 \n", 265 | "\n", 266 | " maniqa_ssim maniqa_psnr maniqa_mse \n", 267 | "0 0.810342 30.222343 0.000950 \n", 268 | "1 0.875497 29.726215 0.001065 \n", 269 | "2 0.855742 29.728792 0.001064 \n", 270 | "3 0.906851 29.779349 0.001052 \n", 271 | "4 0.747340 30.407143 0.000911 \n", 272 | "... ... ... ... \n", 273 | "77275 0.384374 24.441853 0.003596 \n", 274 | "77276 0.546673 24.441853 0.003596 \n", 275 | "77277 0.506352 24.441853 0.003596 \n", 276 | "77278 0.391060 24.441853 0.003596 \n", 277 | "77279 0.506051 24.441853 0.003596 \n", 278 | "\n", 279 | "[77280 rows x 7 columns]" 280 | ] 281 | }, 282 | "execution_count": 10, 283 | "metadata": {}, 284 | "output_type": "execute_result" 285 | } 286 | ], 287 | "source": [ 288 | "data_to_evaluate = data\\\n", 289 | " [[\n", 290 | " 'dataset',\n", 291 | " 'attack',\n", 292 | " f'{metric_to_evaluate}_clear',\n", 293 | " f'{metric_to_evaluate}_attacked',\n", 294 | " f'{metric_to_evaluate}_ssim',\n", 295 | " f'{metric_to_evaluate}_psnr',\n", 296 | " f'{metric_to_evaluate}_mse'\n", 297 | " ]].reset_index(drop=True)\n", 298 | "data_to_evaluate" 299 | ] 300 | }, 301 | { 302 | "attachments": {}, 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "Domain transformation" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 11, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stderr", 316 | "output_type": "stream", 317 | "text": [ 318 | "100%|██████████| 78/78 [00:13<00:00, 5.99it/s]\n" 319 | ] 320 | }, 321 | { 322 | "data": { 323 | "text/html": [ 324 | "
\n", 325 | "\n", 338 | "\n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | "
datasetattackmaniqa_clearmaniqa_attackedmaniqa_ssimmaniqa_psnrmaniqa_mse
0NIPSamifgsm0.5920091.3852880.81034230.2223430.000950
1NIPSamifgsm0.6413241.3462770.87549729.7262150.001065
2NIPSamifgsm0.8380711.3899770.85574229.7287920.001064
3NIPSamifgsm0.6984921.3623700.90685129.7793490.001052
4NIPSamifgsm0.8296411.3407570.74734030.4071430.000911
........................
77275VIMEOdefault-uap_COCO_amp0.80.6245940.0996420.38437424.4418530.003596
77276VIMEOdefault-uap_COCO_amp0.80.4407770.1161130.54667324.4418530.003596
77277VIMEOdefault-uap_COCO_amp0.80.3719810.1104580.50635224.4418530.003596
77278VIMEOdefault-uap_COCO_amp0.80.4563080.1171330.39106024.4418530.003596
77279VIMEOdefault-uap_COCO_amp0.80.4347880.1099970.50605124.4418530.003596
\n", 464 | "

77280 rows × 7 columns

\n", 465 | "
" 466 | ], 467 | "text/plain": [ 468 | " dataset attack maniqa_clear maniqa_attacked \\\n", 469 | "0 NIPS amifgsm 0.592009 1.385288 \n", 470 | "1 NIPS amifgsm 0.641324 1.346277 \n", 471 | "2 NIPS amifgsm 0.838071 1.389977 \n", 472 | "3 NIPS amifgsm 0.698492 1.362370 \n", 473 | "4 NIPS amifgsm 0.829641 1.340757 \n", 474 | "... ... ... ... ... \n", 475 | "77275 VIMEO default-uap_COCO_amp0.8 0.624594 0.099642 \n", 476 | "77276 VIMEO default-uap_COCO_amp0.8 0.440777 0.116113 \n", 477 | "77277 VIMEO default-uap_COCO_amp0.8 0.371981 0.110458 \n", 478 | "77278 VIMEO default-uap_COCO_amp0.8 0.456308 0.117133 \n", 479 | "77279 VIMEO default-uap_COCO_amp0.8 0.434788 0.109997 \n", 480 | "\n", 481 | " maniqa_ssim maniqa_psnr maniqa_mse \n", 482 | "0 0.810342 30.222343 0.000950 \n", 483 | "1 0.875497 29.726215 0.001065 \n", 484 | "2 0.855742 29.728792 0.001064 \n", 485 | "3 0.906851 29.779349 0.001052 \n", 486 | "4 0.747340 30.407143 0.000911 \n", 487 | "... ... ... ... \n", 488 | "77275 0.384374 24.441853 0.003596 \n", 489 | "77276 0.546673 24.441853 0.003596 \n", 490 | "77277 0.506352 24.441853 0.003596 \n", 491 | "77278 0.391060 24.441853 0.003596 \n", 492 | "77279 0.506051 24.441853 0.003596 \n", 493 | "\n", 494 | "[77280 rows x 7 columns]" 495 | ] 496 | }, 497 | "execution_count": 11, 498 | "metadata": {}, 499 | "output_type": "execute_result" 500 | } 501 | ], 502 | "source": [ 503 | "from tqdm import tqdm\n", 504 | "\n", 505 | "# affects VRAM usage if run on GPU\n", 506 | "CHUNK_SIZE = 1000\n", 507 | "def chunker(df, size):\n", 508 | " return [df.iloc[pos:pos + size] for pos in range(0, len(df), size)]\n", 509 | "\n", 510 | "\n", 511 | "# Run domain transform by chunks\n", 512 | "data_transformed = pd.DataFrame()\n", 513 | "for cur_df in tqdm(chunker(data_to_evaluate, 1000)):\n", 514 | " cur_data_transformed = transform_full_df(df=cur_df, metrics=[metric_to_evaluate], path_to_models='./models', domain='mdtvsfa', dev='cuda:0')\n", 515 | " data_transformed = pd.concat([data_transformed, cur_data_transformed])\n", 516 | "data_transformed " 517 | ] 518 | }, 519 | { 520 | "attachments": {}, 521 | "cell_type": "markdown", 522 | "metadata": {}, 523 | "source": [ 524 | "Evaluate results" 525 | ] 526 | }, 527 | { 528 | "attachments": {}, 529 | "cell_type": "markdown", 530 | "metadata": {}, 531 | "source": [ 532 | "Overall scores (on all datasets and attacks)" 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": 12, 538 | "metadata": {}, 539 | "outputs": [ 540 | { 541 | "name": "stdout", 542 | "output_type": "stream", 543 | "text": [ 544 | "Absolute gain: 0.094 (0.091, 0.097)\n", 545 | "Relative gain: 0.073 (0.070, 0.075)\n", 546 | "Robustness score: 0.645 (0.641, 0.650)\n", 547 | "Wasserstein score: 0.203\n", 548 | "Energy score: 0.238\n" 549 | ] 550 | } 551 | ], 552 | "source": [ 553 | "eval_df = data_transformed.copy()\n", 554 | "\n", 555 | "# W/o domain transform:\n", 556 | "# eval_df = data.copy()\n", 557 | "\n", 558 | "abs_gain_scores = normalized_absolute_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n", 559 | "low, high = st.t.interval(0.95, len(abs_gain_scores)-1, loc=np.mean(abs_gain_scores), scale=st.sem(abs_gain_scores))\n", 560 | "print('Absolute gain: {:.3f} ({:.3f}, {:.3f})'.format(abs_gain_scores.mean(), low, high))\n", 561 | "\n", 562 | "rel_gain_scores = normalized_relative_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n", 563 | "low, high = st.t.interval(0.95, len(rel_gain_scores)-1, loc=np.mean(rel_gain_scores), scale=st.sem(rel_gain_scores))\n", 564 | "print('Relative gain: {:.3f} ({:.3f}, {:.3f})'.format(rel_gain_scores.mean(), low, high))\n", 565 | "\n", 566 | "rob_scores = robustness_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n", 567 | "low, high = st.t.interval(0.95, len(rob_scores)-1, loc=np.mean(rob_scores), scale=st.sem(rob_scores))\n", 568 | "print('Robustness score: {:.3f} ({:.3f}, {:.3f})'.format(rob_scores.mean(), low, high))\n", 569 | "\n", 570 | "print('Wasserstein score: {:.3f}'.format(calc_wasserstein_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))\n", 571 | "print('Energy score: {:.3f}'.format(energy_distance_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))" 572 | ] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": 13, 577 | "metadata": {}, 578 | "outputs": [ 579 | { 580 | "data": { 581 | "text/plain": [ 582 | "array(['amifgsm', 'cumulative-uap_VOC2012_amp0.2',\n", 583 | " 'cumulative-uap_COCO_amp0.2', 'cumulative-uap_VOC2012_amp0.4',\n", 584 | " 'cumulative-uap_COCO_amp0.4', 'cumulative-uap_VOC2012_amp0.8',\n", 585 | " 'cumulative-uap_COCO_amp0.8', 'generative-uap_VOC2012_amp0.2',\n", 586 | " 'generative-uap_COCO_amp0.2', 'generative-uap_VOC2012_amp0.4',\n", 587 | " 'generative-uap_COCO_amp0.4', 'generative-uap_VOC2012_amp0.8',\n", 588 | " 'generative-uap_COCO_amp0.8', 'ifgsm', 'korhonen-et-al', 'madc',\n", 589 | " 'mifgsm', 'std-fgsm', 'default-uap_VOC2012_amp0.2',\n", 590 | " 'default-uap_COCO_amp0.2', 'default-uap_VOC2012_amp0.4',\n", 591 | " 'default-uap_COCO_amp0.4', 'default-uap_VOC2012_amp0.8',\n", 592 | " 'default-uap_COCO_amp0.8'], dtype=object)" 593 | ] 594 | }, 595 | "execution_count": 13, 596 | "metadata": {}, 597 | "output_type": "execute_result" 598 | } 599 | ], 600 | "source": [ 601 | "# Available attacks\n", 602 | "data.attack.unique()" 603 | ] 604 | }, 605 | { 606 | "cell_type": "code", 607 | "execution_count": 14, 608 | "metadata": {}, 609 | "outputs": [ 610 | { 611 | "data": { 612 | "text/plain": [ 613 | "array(['NIPS', 'DERF', 'VIMEO'], dtype=object)" 614 | ] 615 | }, 616 | "execution_count": 14, 617 | "metadata": {}, 618 | "output_type": "execute_result" 619 | } 620 | ], 621 | "source": [ 622 | "# Available datasets\n", 623 | "data.dataset.unique()" 624 | ] 625 | }, 626 | { 627 | "attachments": {}, 628 | "cell_type": "markdown", 629 | "metadata": {}, 630 | "source": [ 631 | "Scores on selected dataset\\attack" 632 | ] 633 | }, 634 | { 635 | "cell_type": "code", 636 | "execution_count": 20, 637 | "metadata": {}, 638 | "outputs": [ 639 | { 640 | "name": "stdout", 641 | "output_type": "stream", 642 | "text": [ 643 | "Absolute gain: 0.297 (0.288, 0.305)\n", 644 | "Relative gain: 0.200 (0.193, 0.207)\n", 645 | "Robustness score: 0.399 (0.380, 0.419)\n", 646 | "Wasserstein score: 0.297\n", 647 | "Energy score: 0.530\n" 648 | ] 649 | } 650 | ], 651 | "source": [ 652 | "dataset = 'NIPS' # \"NIPS\", \"DERF\", \"VIMEO\" or \"all\"\n", 653 | "attack = 'std-fgsm' # attack name or \"all\" or \"uap-based\" or \"iterative\"\n", 654 | "uap_attacks = [ x for x in data_transformed.attack.unique() if 'uap' in x]\n", 655 | "\n", 656 | "eval_df = data_transformed.copy()\n", 657 | "\n", 658 | "# W/o domain transform\n", 659 | "# eval_df = data.copy()\n", 660 | "\n", 661 | "\n", 662 | "if dataset != 'all':\n", 663 | " eval_df = eval_df[eval_df.dataset == dataset]\n", 664 | "if attack != 'all' and attack != 'uap-based' and attack != 'iterative':\n", 665 | " eval_df = eval_df[eval_df.attack == attack]\n", 666 | "elif attack == 'uap-based':\n", 667 | " eval_df = eval_df[eval_df.attack.isin(uap_attacks)]\n", 668 | "elif attack == 'iterative':\n", 669 | " eval_df = eval_df[~eval_df.attack.isin(uap_attacks)]\n", 670 | "\n", 671 | "abs_gain_scores = normalized_absolute_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n", 672 | "low, high = st.t.interval(0.95, len(abs_gain_scores)-1, loc=np.mean(abs_gain_scores), scale=st.sem(abs_gain_scores))\n", 673 | "print('Absolute gain: {:.3f} ({:.3f}, {:.3f})'.format(abs_gain_scores.mean(), low, high))\n", 674 | "\n", 675 | "rel_gain_scores = normalized_relative_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n", 676 | "low, high = st.t.interval(0.95, len(rel_gain_scores)-1, loc=np.mean(rel_gain_scores), scale=st.sem(rel_gain_scores))\n", 677 | "print('Relative gain: {:.3f} ({:.3f}, {:.3f})'.format(rel_gain_scores.mean(), low, high))\n", 678 | "\n", 679 | "rob_scores = robustness_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n", 680 | "low, high = st.t.interval(0.95, len(rob_scores)-1, loc=np.mean(rob_scores), scale=st.sem(rob_scores))\n", 681 | "print('Robustness score: {:.3f} ({:.3f}, {:.3f})'.format(rob_scores.mean(), low, high))\n", 682 | "\n", 683 | "print('Wasserstein score: {:.3f}'.format(calc_wasserstein_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))\n", 684 | "print('Energy score: {:.3f}'.format(energy_distance_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))" 685 | ] 686 | } 687 | ], 688 | "metadata": { 689 | "kernelspec": { 690 | "display_name": "base", 691 | "language": "python", 692 | "name": "python3" 693 | }, 694 | "language_info": { 695 | "codemirror_mode": { 696 | "name": "ipython", 697 | "version": 3 698 | }, 699 | "file_extension": ".py", 700 | "mimetype": "text/x-python", 701 | "name": "python", 702 | "nbconvert_exporter": "python", 703 | "pygments_lexer": "ipython3", 704 | "version": "3.9.12" 705 | }, 706 | "orig_nbformat": 4 707 | }, 708 | "nbformat": 4, 709 | "nbformat_minor": 2 710 | } 711 | -------------------------------------------------------------------------------- /lib_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "Demo for MANIQA metric setup" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# Run theese commands in benchmark's root directory to setup MANIQA metric from it's source directory.\n", 17 | "# You can also use your custom metric if it matches framework's call interface. An example implementation can be found in the model.py file in subjects/maniqa/\n", 18 | "\n", 19 | "# !git submodule update --init --recursive\n", 20 | "# !cd subjects/maniqa/src && git apply ../patches/maniqa.patch -v\n", 21 | "# !cd subjects/maniqa && cp model.py src" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "# Load MANIQA weights\n", 31 | "# !wget https://github.com/IIGROUP/MANIQA/releases/download/Koniq10k/ckpt_koniq10k.pt -P subjects/maniqa/\n", 32 | "# Install MANIQA requirements\n", 33 | "# !pip install -r subjects/maniqa/src/requirements.txt" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 5, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "# Change these parameters if you are using other metric. \n", 43 | "# 'bounds' in metric_cfg specify minimum and maximum possible metric values if such limits exist, approximate range of metric's values otherwise.\n", 44 | "metric_path = './subjects/maniqa/src/'\n", 45 | "path_to_weights = './subjects/maniqa/ckpt_koniq10k.pt'\n", 46 | "metric_cfg = {'is_fr': False, 'bounds':{'low':0, 'high':1}, 'name':'maniqa'}\n", 47 | "device = 'cuda:0'" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 6, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "import sys\n", 57 | "sys.path.append(metric_path)\n", 58 | "from model import MetricModel" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 7, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "model = MetricModel(model_path=path_to_weights, device=device)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "Run benchmark attacks" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 8, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "import robustness_benchmark as rb" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 9, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/plain": [ 94 | "dict_keys(['ifgsm', 'mifgsm', 'amifgsm', 'std-fgsm', 'korhonen-et-al', 'madc', 'cumulative-uap', 'generative-uap', 'uap'])" 95 | ] 96 | }, 97 | "execution_count": 9, 98 | "metadata": {}, 99 | "output_type": "execute_result" 100 | } 101 | ], 102 | "source": [ 103 | "# Specify path to dataset and it's name. It must be a directory containing only images (or only videos)\n", 104 | "dataset_path = './test_dataset/'\n", 105 | "dataset_name = 'TEST'\n", 106 | "\n", 107 | "# Specify directory where results of attacks will be stored. \n", 108 | "# Results will be organized as directories with names similar to the attacks.\n", 109 | "# Files with names *metric_name*.csv will be saved in these directories.\n", 110 | "result_save_dir = './test_results/'\n", 111 | "\n", 112 | "# attacks_to_run = rb.interface.iterative_attacks\n", 113 | "# attacks_to_run = rb.interface.uap_attacks\n", 114 | "attacks_to_run = rb.interface.all_default_attacks\n", 115 | "rb.interface.all_default_attacks.keys()" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "# You can also add your custom attack \n", 125 | "# Make sure that it follows benchmark call interface.\n", 126 | "# You can find default attacks implementation in robustness_benchmark.methods.\n", 127 | "# attacks_to_run['attack name'] = " 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "# Results saved in this notebook's cell outputs are not representative, attacks were launched on small debug set of only 5 images.\n", 137 | "rb.interface.run_attacks(\n", 138 | " attacks_dict=attacks_to_run,\n", 139 | " device=device, \n", 140 | " metric_model=model, \n", 141 | " metric_cfg=metric_cfg, \n", 142 | " dataset_names=[dataset_name], \n", 143 | " dataset_paths=[dataset_path], \n", 144 | " save_dir=result_save_dir)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "Collect results from csv files" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 11, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "data": { 161 | "text/html": [ 162 | "
\n", 163 | "\n", 176 | "\n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | "
datasetmaniqa_clearmaniqa_attackedmaniqa_ssimmaniqa_psnrmaniqa_mseattack
0TEST0.4975641.1086950.80992430.2140030.000952amifgsm
1TEST0.5684881.0902780.90682929.7789930.001052amifgsm
2TEST0.3356491.0608560.81297730.1849570.000958amifgsm
3TEST0.5106991.0918310.79363830.1288860.000971amifgsm
4TEST0.4858680.9565750.90538329.4162440.001144amifgsm
........................
47TEST0.4975640.3137440.56795724.8127810.003302default-uap_VOC2012_amp0.8
50TEST0.5684880.3530590.78325324.8127810.003302default-uap_VOC2012_amp0.8
53TEST0.3356490.2025280.57862424.8127810.003302default-uap_VOC2012_amp0.8
56TEST0.5106990.2890650.57221524.8127810.003302default-uap_VOC2012_amp0.8
59TEST0.4858680.3380820.79944124.8127810.003302default-uap_VOC2012_amp0.8
\n", 302 | "

285 rows × 7 columns

\n", 303 | "
" 304 | ], 305 | "text/plain": [ 306 | " dataset maniqa_clear maniqa_attacked maniqa_ssim maniqa_psnr \\\n", 307 | "0 TEST 0.497564 1.108695 0.809924 30.214003 \n", 308 | "1 TEST 0.568488 1.090278 0.906829 29.778993 \n", 309 | "2 TEST 0.335649 1.060856 0.812977 30.184957 \n", 310 | "3 TEST 0.510699 1.091831 0.793638 30.128886 \n", 311 | "4 TEST 0.485868 0.956575 0.905383 29.416244 \n", 312 | ".. ... ... ... ... ... \n", 313 | "47 TEST 0.497564 0.313744 0.567957 24.812781 \n", 314 | "50 TEST 0.568488 0.353059 0.783253 24.812781 \n", 315 | "53 TEST 0.335649 0.202528 0.578624 24.812781 \n", 316 | "56 TEST 0.510699 0.289065 0.572215 24.812781 \n", 317 | "59 TEST 0.485868 0.338082 0.799441 24.812781 \n", 318 | "\n", 319 | " maniqa_mse attack \n", 320 | "0 0.000952 amifgsm \n", 321 | "1 0.001052 amifgsm \n", 322 | "2 0.000958 amifgsm \n", 323 | "3 0.000971 amifgsm \n", 324 | "4 0.001144 amifgsm \n", 325 | ".. ... ... \n", 326 | "47 0.003302 default-uap_VOC2012_amp0.8 \n", 327 | "50 0.003302 default-uap_VOC2012_amp0.8 \n", 328 | "53 0.003302 default-uap_VOC2012_amp0.8 \n", 329 | "56 0.003302 default-uap_VOC2012_amp0.8 \n", 330 | "59 0.003302 default-uap_VOC2012_amp0.8 \n", 331 | "\n", 332 | "[285 rows x 7 columns]" 333 | ] 334 | }, 335 | "execution_count": 11, 336 | "metadata": {}, 337 | "output_type": "execute_result" 338 | } 339 | ], 340 | "source": [ 341 | "results_df = rb.interface.collect_results(result_save_dir, metric_cfg=metric_cfg)\n", 342 | "results_df" 343 | ] 344 | }, 345 | { 346 | "cell_type": "markdown", 347 | "metadata": {}, 348 | "source": [ 349 | "Domain transformation" 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": {}, 355 | "source": [ 356 | "Note: domain transformation (based on Neural Optimal Transport, described ) currently only works with metrics used in benchmark, for which domain transformation is pretrained and stored in ```robustness_benchmark/models/models_to_mdtvsfa.pth```.\\\n", 357 | "For other metrics, you can still use ```evaluate_robustness()``` on results_df." 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "execution_count": 31, 363 | "metadata": {}, 364 | "outputs": [ 365 | { 366 | "name": "stderr", 367 | "output_type": "stream", 368 | "text": [ 369 | "100%|██████████| 1/1 [00:00<00:00, 12.58it/s]" 370 | ] 371 | }, 372 | { 373 | "name": "stdout", 374 | "output_type": "stream", 375 | "text": [ 376 | "current metric: maniqa\n" 377 | ] 378 | }, 379 | { 380 | "name": "stderr", 381 | "output_type": "stream", 382 | "text": [ 383 | "\n" 384 | ] 385 | }, 386 | { 387 | "data": { 388 | "text/html": [ 389 | "
\n", 390 | "\n", 403 | "\n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | "
datasetmaniqa_clearmaniqa_attackedmaniqa_ssimmaniqa_psnrmaniqa_mseattack
0TEST0.5930201.3856260.80992430.2140030.000952amifgsm
1TEST0.6917311.3600140.90682929.7789930.001052amifgsm
2TEST0.3166371.3268890.81297730.1849570.000958amifgsm
3TEST0.6059281.3639780.79363830.1288860.000971amifgsm
4TEST0.5710771.1995760.90538329.4162440.001144amifgsm
........................
47TEST0.5930670.2861160.56795724.8127810.003302default-uap_VOC2012_amp0.8
50TEST0.6993350.3395400.78325324.8127810.003302default-uap_VOC2012_amp0.8
53TEST0.3211930.1158630.57862424.8127810.003302default-uap_VOC2012_amp0.8
56TEST0.6161400.2311540.57221524.8127810.003302default-uap_VOC2012_amp0.8
59TEST0.5721590.3191310.79944124.8127810.003302default-uap_VOC2012_amp0.8
\n", 529 | "

285 rows × 7 columns

\n", 530 | "
" 531 | ], 532 | "text/plain": [ 533 | " dataset maniqa_clear maniqa_attacked maniqa_ssim maniqa_psnr \\\n", 534 | "0 TEST 0.593020 1.385626 0.809924 30.214003 \n", 535 | "1 TEST 0.691731 1.360014 0.906829 29.778993 \n", 536 | "2 TEST 0.316637 1.326889 0.812977 30.184957 \n", 537 | "3 TEST 0.605928 1.363978 0.793638 30.128886 \n", 538 | "4 TEST 0.571077 1.199576 0.905383 29.416244 \n", 539 | ".. ... ... ... ... ... \n", 540 | "47 TEST 0.593067 0.286116 0.567957 24.812781 \n", 541 | "50 TEST 0.699335 0.339540 0.783253 24.812781 \n", 542 | "53 TEST 0.321193 0.115863 0.578624 24.812781 \n", 543 | "56 TEST 0.616140 0.231154 0.572215 24.812781 \n", 544 | "59 TEST 0.572159 0.319131 0.799441 24.812781 \n", 545 | "\n", 546 | " maniqa_mse attack \n", 547 | "0 0.000952 amifgsm \n", 548 | "1 0.001052 amifgsm \n", 549 | "2 0.000958 amifgsm \n", 550 | "3 0.000971 amifgsm \n", 551 | "4 0.001144 amifgsm \n", 552 | ".. ... ... \n", 553 | "47 0.003302 default-uap_VOC2012_amp0.8 \n", 554 | "50 0.003302 default-uap_VOC2012_amp0.8 \n", 555 | "53 0.003302 default-uap_VOC2012_amp0.8 \n", 556 | "56 0.003302 default-uap_VOC2012_amp0.8 \n", 557 | "59 0.003302 default-uap_VOC2012_amp0.8 \n", 558 | "\n", 559 | "[285 rows x 7 columns]" 560 | ] 561 | }, 562 | "execution_count": 31, 563 | "metadata": {}, 564 | "output_type": "execute_result" 565 | } 566 | ], 567 | "source": [ 568 | "transformed_results_df = rb.interface.domain_transform(results_df, metrics=[metric_cfg['name']], batch_size=1000, device=device)\n", 569 | "transformed_results_df" 570 | ] 571 | }, 572 | { 573 | "cell_type": "markdown", 574 | "metadata": {}, 575 | "source": [ 576 | "Metric robustness evaluation" 577 | ] 578 | }, 579 | { 580 | "cell_type": "code", 581 | "execution_count": 32, 582 | "metadata": {}, 583 | "outputs": [ 584 | { 585 | "data": { 586 | "text/html": [ 587 | "
\n", 588 | "\n", 601 | "\n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | " \n", 875 | " \n", 876 | " \n", 877 | " \n", 878 | " \n", 879 | " \n", 880 | " \n", 881 | " \n", 882 | " \n", 883 | " \n", 884 | " \n", 885 | " \n", 886 | " \n", 887 | " \n", 888 | " \n", 889 | " \n", 890 | " \n", 891 | " \n", 892 | " \n", 893 | " \n", 894 | " \n", 895 | " \n", 896 | " \n", 897 | " \n", 898 | " \n", 899 | " \n", 900 | " \n", 901 | " \n", 902 | " \n", 903 | " \n", 904 | " \n", 905 | " \n", 906 | " \n", 907 | " \n", 908 | " \n", 909 | " \n", 910 | " \n", 911 | " \n", 912 | " \n", 913 | " \n", 914 | "
metricattackenergy_distance_scorenormalized_absolute_gainnormalized_relative_gainrelative_gain_classicrobustness_scorewasserstein_score
0maniqaamifgsm1.8792.016 (1.848, 2.184)1.364 (1.055, 1.674)1.563 (1.171, 1.956)-0.387 (-0.422, -0.351)2.016
1maniqacumulative-uap_COCO_amp0.2-0.063-0.011 (-0.021, -0.001)-0.007 (-0.012, -0.001)-0.007 (-0.013, -0.001)2.101 (1.594, 2.609)-0.012
2maniqacumulative-uap_VOC2012_amp0.2-0.325-0.163 (-0.229, -0.096)-0.099 (-0.134, -0.064)-0.112 (-0.150, -0.074)0.766 (0.567, 0.966)-0.163
3maniqacumulative-uap_COCO_amp0.4-0.187-0.057 (-0.093, -0.022)-0.034 (-0.053, -0.014)-0.038 (-0.059, -0.016)1.289 (1.010, 1.567)-0.057
4maniqacumulative-uap_VOC2012_amp0.4-0.654-0.440 (-0.532, -0.349)-0.276 (-0.328, -0.224)-0.315 (-0.379, -0.252)0.284 (0.183, 0.385)-0.440
5maniqacumulative-uap_COCO_amp0.8-0.253-0.051 (-0.138, 0.037)-0.016 (-0.082, 0.050)-0.015 (-0.093, 0.063)0.960 (0.760, 1.160)-0.108
6maniqacumulative-uap_VOC2012_amp0.8-1.083-0.856 (-1.003, -0.709)-0.525 (-0.564, -0.486)-0.589 (-0.633, -0.545)-0.007 (-0.126, 0.112)-0.856
7maniqagenerative-uap_COCO_amp0.2-0.112-0.034 (-0.050, -0.018)-0.022 (-0.034, -0.011)-0.025 (-0.038, -0.012)1.518 (1.231, 1.806)-0.034
8maniqagenerative-uap_VOC2012_amp0.2-0.107-0.034 (-0.060, -0.008)-0.023 (-0.039, -0.007)-0.026 (-0.045, -0.008)1.413 (1.130, 1.696)-0.034
9maniqagenerative-uap_COCO_amp0.4-0.241-0.106 (-0.156, -0.057)-0.066 (-0.093, -0.039)-0.074 (-0.105, -0.043)1.043 (0.706, 1.379)-0.106
10maniqagenerative-uap_VOC2012_amp0.4-0.300-0.142 (-0.210, -0.074)-0.087 (-0.125, -0.050)-0.099 (-0.140, -0.057)0.872 (0.605, 1.139)-0.142
11maniqagenerative-uap_COCO_amp0.8-0.437-0.272 (-0.405, -0.138)-0.166 (-0.238, -0.095)-0.185 (-0.265, -0.106)0.735 (0.235, 1.236)-0.272
12maniqagenerative-uap_VOC2012_amp0.8-0.581-0.340 (-0.458, -0.223)-0.198 (-0.255, -0.140)-0.220 (-0.281, -0.159)0.455 (0.210, 0.700)-0.340
13maniqaifgsm2.1512.965 (2.470, 3.459)2.011 (1.484, 2.538)2.308 (1.656, 2.960)-0.535 (-0.604, -0.465)2.965
14maniqakorhonen-et-al1.8872.040 (1.849, 2.231)1.374 (1.016, 1.733)1.599 (1.131, 2.068)-0.392 (-0.436, -0.349)2.040
15maniqamadc0.9880.776 (0.671, 0.882)0.529 (0.369, 0.689)0.611 (0.408, 0.813)0.035 (-0.017, 0.086)0.776
16maniqamifgsm1.8752.006 (1.838, 2.174)1.360 (1.050, 1.670)1.565 (1.169, 1.960)-0.386 (-0.423, -0.350)2.006
17maniqastd-fgsm1.1420.935 (0.793, 1.078)0.640 (0.445, 0.836)0.739 (0.492, 0.985)-0.042 (-0.120, 0.037)0.935
18maniqadefault-uap_COCO_amp0.2-0.267-0.128 (-0.190, -0.067)-0.081 (-0.116, -0.045)-0.091 (-0.131, -0.051)0.927 (0.650, 1.203)-0.128
19maniqadefault-uap_VOC2012_amp0.2-0.303-0.151 (-0.215, -0.087)-0.092 (-0.125, -0.058)-0.104 (-0.141, -0.067)0.797 (0.611, 0.984)-0.151
20maniqadefault-uap_COCO_amp0.4-0.585-0.389 (-0.491, -0.286)-0.247 (-0.316, -0.179)-0.284 (-0.368, -0.200)0.348 (0.249, 0.448)-0.389
21maniqadefault-uap_VOC2012_amp0.4-0.627-0.421 (-0.523, -0.319)-0.265 (-0.328, -0.203)-0.302 (-0.377, -0.226)0.310 (0.211, 0.409)-0.421
22maniqadefault-uap_COCO_amp0.8-0.985-0.761 (-0.888, -0.635)-0.470 (-0.517, -0.422)-0.531 (-0.592, -0.470)0.045 (-0.049, 0.140)-0.761
23maniqadefault-uap_VOC2012_amp0.8-1.005-0.781 (-0.913, -0.649)-0.482 (-0.527, -0.437)-0.545 (-0.602, -0.487)0.032 (-0.066, 0.130)-0.781
24maniqaall0.5740.502 (0.364, 0.640)0.351 (0.250, 0.451)0.412 (0.292, 0.532)0.379 (0.291, 0.466)0.691
25maniqaiterative1.5811.846 (1.673, 2.019)1.251 (1.089, 1.412)1.457 (1.256, 1.658)-0.303 (-0.347, -0.260)1.846
26maniqauap-0.390-0.281 (-0.323, -0.239)-0.173 (-0.198, -0.147)-0.198 (-0.227, -0.169)0.776 (0.679, 0.873)-0.281
\n", 915 | "
" 916 | ], 917 | "text/plain": [ 918 | " metric attack energy_distance_score \\\n", 919 | "0 maniqa amifgsm 1.879 \n", 920 | "1 maniqa cumulative-uap_COCO_amp0.2 -0.063 \n", 921 | "2 maniqa cumulative-uap_VOC2012_amp0.2 -0.325 \n", 922 | "3 maniqa cumulative-uap_COCO_amp0.4 -0.187 \n", 923 | "4 maniqa cumulative-uap_VOC2012_amp0.4 -0.654 \n", 924 | "5 maniqa cumulative-uap_COCO_amp0.8 -0.253 \n", 925 | "6 maniqa cumulative-uap_VOC2012_amp0.8 -1.083 \n", 926 | "7 maniqa generative-uap_COCO_amp0.2 -0.112 \n", 927 | "8 maniqa generative-uap_VOC2012_amp0.2 -0.107 \n", 928 | "9 maniqa generative-uap_COCO_amp0.4 -0.241 \n", 929 | "10 maniqa generative-uap_VOC2012_amp0.4 -0.300 \n", 930 | "11 maniqa generative-uap_COCO_amp0.8 -0.437 \n", 931 | "12 maniqa generative-uap_VOC2012_amp0.8 -0.581 \n", 932 | "13 maniqa ifgsm 2.151 \n", 933 | "14 maniqa korhonen-et-al 1.887 \n", 934 | "15 maniqa madc 0.988 \n", 935 | "16 maniqa mifgsm 1.875 \n", 936 | "17 maniqa std-fgsm 1.142 \n", 937 | "18 maniqa default-uap_COCO_amp0.2 -0.267 \n", 938 | "19 maniqa default-uap_VOC2012_amp0.2 -0.303 \n", 939 | "20 maniqa default-uap_COCO_amp0.4 -0.585 \n", 940 | "21 maniqa default-uap_VOC2012_amp0.4 -0.627 \n", 941 | "22 maniqa default-uap_COCO_amp0.8 -0.985 \n", 942 | "23 maniqa default-uap_VOC2012_amp0.8 -1.005 \n", 943 | "24 maniqa all 0.574 \n", 944 | "25 maniqa iterative 1.581 \n", 945 | "26 maniqa uap -0.390 \n", 946 | "\n", 947 | " normalized_absolute_gain normalized_relative_gain relative_gain_classic \\\n", 948 | "0 2.016 (1.848, 2.184) 1.364 (1.055, 1.674) 1.563 (1.171, 1.956) \n", 949 | "1 -0.011 (-0.021, -0.001) -0.007 (-0.012, -0.001) -0.007 (-0.013, -0.001) \n", 950 | "2 -0.163 (-0.229, -0.096) -0.099 (-0.134, -0.064) -0.112 (-0.150, -0.074) \n", 951 | "3 -0.057 (-0.093, -0.022) -0.034 (-0.053, -0.014) -0.038 (-0.059, -0.016) \n", 952 | "4 -0.440 (-0.532, -0.349) -0.276 (-0.328, -0.224) -0.315 (-0.379, -0.252) \n", 953 | "5 -0.051 (-0.138, 0.037) -0.016 (-0.082, 0.050) -0.015 (-0.093, 0.063) \n", 954 | "6 -0.856 (-1.003, -0.709) -0.525 (-0.564, -0.486) -0.589 (-0.633, -0.545) \n", 955 | "7 -0.034 (-0.050, -0.018) -0.022 (-0.034, -0.011) -0.025 (-0.038, -0.012) \n", 956 | "8 -0.034 (-0.060, -0.008) -0.023 (-0.039, -0.007) -0.026 (-0.045, -0.008) \n", 957 | "9 -0.106 (-0.156, -0.057) -0.066 (-0.093, -0.039) -0.074 (-0.105, -0.043) \n", 958 | "10 -0.142 (-0.210, -0.074) -0.087 (-0.125, -0.050) -0.099 (-0.140, -0.057) \n", 959 | "11 -0.272 (-0.405, -0.138) -0.166 (-0.238, -0.095) -0.185 (-0.265, -0.106) \n", 960 | "12 -0.340 (-0.458, -0.223) -0.198 (-0.255, -0.140) -0.220 (-0.281, -0.159) \n", 961 | "13 2.965 (2.470, 3.459) 2.011 (1.484, 2.538) 2.308 (1.656, 2.960) \n", 962 | "14 2.040 (1.849, 2.231) 1.374 (1.016, 1.733) 1.599 (1.131, 2.068) \n", 963 | "15 0.776 (0.671, 0.882) 0.529 (0.369, 0.689) 0.611 (0.408, 0.813) \n", 964 | "16 2.006 (1.838, 2.174) 1.360 (1.050, 1.670) 1.565 (1.169, 1.960) \n", 965 | "17 0.935 (0.793, 1.078) 0.640 (0.445, 0.836) 0.739 (0.492, 0.985) \n", 966 | "18 -0.128 (-0.190, -0.067) -0.081 (-0.116, -0.045) -0.091 (-0.131, -0.051) \n", 967 | "19 -0.151 (-0.215, -0.087) -0.092 (-0.125, -0.058) -0.104 (-0.141, -0.067) \n", 968 | "20 -0.389 (-0.491, -0.286) -0.247 (-0.316, -0.179) -0.284 (-0.368, -0.200) \n", 969 | "21 -0.421 (-0.523, -0.319) -0.265 (-0.328, -0.203) -0.302 (-0.377, -0.226) \n", 970 | "22 -0.761 (-0.888, -0.635) -0.470 (-0.517, -0.422) -0.531 (-0.592, -0.470) \n", 971 | "23 -0.781 (-0.913, -0.649) -0.482 (-0.527, -0.437) -0.545 (-0.602, -0.487) \n", 972 | "24 0.502 (0.364, 0.640) 0.351 (0.250, 0.451) 0.412 (0.292, 0.532) \n", 973 | "25 1.846 (1.673, 2.019) 1.251 (1.089, 1.412) 1.457 (1.256, 1.658) \n", 974 | "26 -0.281 (-0.323, -0.239) -0.173 (-0.198, -0.147) -0.198 (-0.227, -0.169) \n", 975 | "\n", 976 | " robustness_score wasserstein_score \n", 977 | "0 -0.387 (-0.422, -0.351) 2.016 \n", 978 | "1 2.101 (1.594, 2.609) -0.012 \n", 979 | "2 0.766 (0.567, 0.966) -0.163 \n", 980 | "3 1.289 (1.010, 1.567) -0.057 \n", 981 | "4 0.284 (0.183, 0.385) -0.440 \n", 982 | "5 0.960 (0.760, 1.160) -0.108 \n", 983 | "6 -0.007 (-0.126, 0.112) -0.856 \n", 984 | "7 1.518 (1.231, 1.806) -0.034 \n", 985 | "8 1.413 (1.130, 1.696) -0.034 \n", 986 | "9 1.043 (0.706, 1.379) -0.106 \n", 987 | "10 0.872 (0.605, 1.139) -0.142 \n", 988 | "11 0.735 (0.235, 1.236) -0.272 \n", 989 | "12 0.455 (0.210, 0.700) -0.340 \n", 990 | "13 -0.535 (-0.604, -0.465) 2.965 \n", 991 | "14 -0.392 (-0.436, -0.349) 2.040 \n", 992 | "15 0.035 (-0.017, 0.086) 0.776 \n", 993 | "16 -0.386 (-0.423, -0.350) 2.006 \n", 994 | "17 -0.042 (-0.120, 0.037) 0.935 \n", 995 | "18 0.927 (0.650, 1.203) -0.128 \n", 996 | "19 0.797 (0.611, 0.984) -0.151 \n", 997 | "20 0.348 (0.249, 0.448) -0.389 \n", 998 | "21 0.310 (0.211, 0.409) -0.421 \n", 999 | "22 0.045 (-0.049, 0.140) -0.761 \n", 1000 | "23 0.032 (-0.066, 0.130) -0.781 \n", 1001 | "24 0.379 (0.291, 0.466) 0.691 \n", 1002 | "25 -0.303 (-0.347, -0.260) 1.846 \n", 1003 | "26 0.776 (0.679, 0.873) -0.281 " 1004 | ] 1005 | }, 1006 | "execution_count": 32, 1007 | "metadata": {}, 1008 | "output_type": "execute_result" 1009 | } 1010 | ], 1011 | "source": [ 1012 | "# W/o domain transformation\n", 1013 | "# rb.interface.evaluate_robustness(results_df, attacks=list(transformed_results_df.attack.unique()) + ['all', 'iterative', 'uap'])\n", 1014 | "\n", 1015 | "rb.interface.evaluate_robustness(transformed_results_df, attacks=list(transformed_results_df.attack.unique()) + ['all', 'iterative', 'uap'])" 1016 | ] 1017 | } 1018 | ], 1019 | "metadata": { 1020 | "kernelspec": { 1021 | "display_name": "Python 3 (ipykernel)", 1022 | "language": "python", 1023 | "name": "python3" 1024 | }, 1025 | "language_info": { 1026 | "codemirror_mode": { 1027 | "name": "ipython", 1028 | "version": 3 1029 | }, 1030 | "file_extension": ".py", 1031 | "mimetype": "text/x-python", 1032 | "name": "python", 1033 | "nbconvert_exporter": "python", 1034 | "pygments_lexer": "ipython3", 1035 | "version": "3.8.8" 1036 | } 1037 | }, 1038 | "nbformat": 4, 1039 | "nbformat_minor": 4 1040 | } 1041 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | av==10.0.0 2 | numpy==1.19.5 3 | pandas==1.1.5 4 | Pillow==10.0.0 5 | scikit_image==0.17.2 6 | scikit_learn==1.3.0 7 | scipy==1.4.1 8 | setuptools==59.5.0 9 | torchvision==0.11.3 10 | tqdm==4.57.0 11 | torch>=1.10.2 12 | opencv_python==4.2.0.32 13 | 14 | -------------------------------------------------------------------------------- /res/cumulative-uap_maniqa_COCO.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/cumulative-uap_maniqa_COCO.npy -------------------------------------------------------------------------------- /res/cumulative-uap_maniqa_COCO.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/cumulative-uap_maniqa_COCO.png -------------------------------------------------------------------------------- /res/cumulative-uap_maniqa_VOC2012.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/cumulative-uap_maniqa_VOC2012.npy -------------------------------------------------------------------------------- /res/cumulative-uap_maniqa_VOC2012.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/cumulative-uap_maniqa_VOC2012.png -------------------------------------------------------------------------------- /res/generative-uap_maniqa_COCO.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/generative-uap_maniqa_COCO.npy -------------------------------------------------------------------------------- /res/generative-uap_maniqa_COCO.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/generative-uap_maniqa_COCO.png -------------------------------------------------------------------------------- /res/generative-uap_maniqa_VOC2012.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/generative-uap_maniqa_VOC2012.npy -------------------------------------------------------------------------------- /res/generative-uap_maniqa_VOC2012.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/generative-uap_maniqa_VOC2012.png -------------------------------------------------------------------------------- /res/uap_maniqa_COCO.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/uap_maniqa_COCO.npy -------------------------------------------------------------------------------- /res/uap_maniqa_COCO.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/uap_maniqa_COCO.png -------------------------------------------------------------------------------- /res/uap_maniqa_VOC2012.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/uap_maniqa_VOC2012.npy -------------------------------------------------------------------------------- /res/uap_maniqa_VOC2012.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/res/uap_maniqa_VOC2012.png -------------------------------------------------------------------------------- /robustness_benchmark/NOT.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import warnings 3 | import numpy as np 4 | import os 5 | from pathlib import Path 6 | import pickle 7 | warnings.filterwarnings('ignore') 8 | H = 100 9 | DEVICE_IDS = [0] 10 | T_ITERS = 10 11 | f_LR, T_LR = 1e-4, 1e-4 12 | Z_STD = 0.1 13 | BATCH_SIZE = 128 14 | PLOT_INTERVAL = 200 15 | SEED = 0x000123 16 | Z_SIZE = 512 # sampling size 17 | DIM = 1 18 | dev = 'cuda:0' 19 | 20 | # Models are saved as dictionary containing 2 subdictionaries (i.e. {"models": dict1, "scalers": dict2} - one for torch models, one for scalers) with entries in format: 21 | # "metric_name": torch model in first dict; 22 | # "metric_name": {"X": MinMaxScaler for input distribution, "Y": MinMaxScaler for output distribution(used only during training)} in second. 23 | def load_models(target_metric, corrs_needed=False, DIM=1, folder='./models', custom_path=None): 24 | filename = 'models_to_{}_{}d.pth'.format(target_metric, DIM) 25 | filepath = os.path.join(folder, filename) 26 | if custom_path is not None: 27 | filepath = custom_path 28 | if not os.path.exists(filepath): 29 | raise ValueError('File with models: {} not found'.format(filepath)) 30 | with open(filepath, 'rb') as fd: 31 | res = pickle.load(fd) 32 | if corrs_needed: 33 | return res['models'], res['scalers'], res['corrs'] 34 | else: 35 | return res['models'], res['scalers'] 36 | 37 | 38 | # apply Neural OT model 39 | def transform_domain(vals, metric, models, scalers, dev='cuda:0'): 40 | 41 | cur_scaler_x = scalers[metric]['X'] 42 | transform_data = cur_scaler_x.transform(np.array(vals).reshape(-1, DIM)) 43 | 44 | X = torch.Tensor(np.array(transform_data).reshape(-1, DIM)).to(dev) 45 | T = models[metric].to(dev) 46 | ZD = DIM 47 | with torch.no_grad(): 48 | X = X.reshape(-1, 1, DIM).repeat(1, Z_SIZE, 1) 49 | Z = torch.randn(X.size(0), Z_SIZE, ZD, device=dev) * Z_STD 50 | XZ = torch.cat([X, Z], dim=2) 51 | if DIM == 1: 52 | T_XZ = T( 53 | XZ.flatten(start_dim=0, end_dim=1) 54 | ).reshape(-1, Z_SIZE, 1) 55 | else: 56 | T_XZ = T( 57 | XZ.flatten(start_dim=0, end_dim=1) 58 | ).permute(1, 0).reshape(DIM, -1, Z_SIZE).permute(1, 2, 0) 59 | X_np = X[:, 0].cpu().numpy() 60 | T_bar_np = T_XZ.mean(dim=1).cpu().numpy() 61 | X_T_bar = np.concatenate([X_np, T_bar_np], axis=1) 62 | if DIM == 1: 63 | y_pred = X_T_bar[:, 1] 64 | del T, X, Z 65 | torch.cuda.empty_cache() 66 | return y_pred.reshape(-1) 67 | 68 | 69 | def transform_full_df(df, metrics, path_to_models='./models', domain='mdtvsfa', dev='cuda:0'): 70 | models, scalers = load_models(domain, folder=path_to_models) 71 | result_df = df.copy() 72 | for metric in metrics: 73 | print('current metric: ', metric) 74 | if metric not in models.keys(): 75 | raise ValueError(f'No pretrained domain transformation for {metric} metric found in {path_to_models}.') 76 | result_df[f'{metric}_clear'] = transform_domain(df[f'{metric}_clear'].to_numpy(), metric=metric, models=models, scalers=scalers, dev=dev) 77 | result_df[f'{metric}_attacked'] = transform_domain(df[f'{metric}_attacked'].to_numpy(), metric=metric, models=models, scalers=scalers, dev=dev) 78 | return result_df 79 | -------------------------------------------------------------------------------- /robustness_benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | #from .interface import * 2 | from . import interface 3 | from . import methods -------------------------------------------------------------------------------- /robustness_benchmark/interface.py: -------------------------------------------------------------------------------- 1 | from torchvision import transforms 2 | import torch 3 | import cv2 4 | import os, sys 5 | import csv 6 | import json 7 | import importlib 8 | from pathlib import Path 9 | import numpy as np 10 | import pandas as pd 11 | from tqdm import tqdm 12 | import scipy.stats as st 13 | from .methods.utils.read_dataset import to_numpy, to_torch, iter_images 14 | from .methods.utils.metrics import PSNR, SSIM, MSE 15 | from .methods.utils.evaluate import run, run_uap 16 | 17 | 18 | from .NOT import * 19 | from .score_methods import * 20 | 21 | from pkg_resources import resource_filename 22 | models_filepath = resource_filename('robustness_benchmark', 'models') 23 | DEFAULT_DEVICE = 'cpu' 24 | 25 | # default attacks 26 | from .methods.amifgsm.run import attack as amifgsm_attack 27 | from .methods.mifgsm.run import attack as mifgsm_attack 28 | from .methods.ifgsm.run import attack as ifgsm_attack 29 | from .methods.madc.run import attack as madc_attack 30 | import importlib 31 | stdfgsm_file = importlib.import_module(".methods.std-fgsm.run", package="robustness_benchmark") 32 | stdfgsm_attack = stdfgsm_file.attack 33 | korhonen_file = importlib.import_module(".methods.korhonen-et-al.run", package="robustness_benchmark") 34 | korhonen_attack = korhonen_file.attack 35 | 36 | iterative_attacks = { 37 | 'ifgsm':ifgsm_attack, 38 | 'mifgsm':mifgsm_attack, 39 | 'amifgsm':amifgsm_attack, 40 | 'std-fgsm':stdfgsm_attack, 41 | 'korhonen-et-al':korhonen_attack, 42 | 'madc':madc_attack, 43 | } 44 | uap_attacks = { 45 | 'cumulative-uap':None, 46 | 'generative-uap':None, 47 | 'uap':None, 48 | } 49 | uap_default_config = {'uap_path':'./res/','amplitudes':[0.2, 0.4, 0.8], 'train_datasets':['COCO', 'VOC2012']} 50 | all_default_attacks = dict(list(iterative_attacks.items()) + list(uap_attacks.items())) 51 | 52 | 53 | # run attack on metric on datasets and save results as csv 54 | def test_main(attack_callback, device, metric_model, metric_cfg, test_datasets, dataset_paths, save_path, jpeg_quality=None): 55 | bounds_metric = metric_cfg.get('bounds', None) 56 | metric_range = 100 if bounds_metric is None else bounds_metric['high'] - bounds_metric['low'] 57 | is_fr = metric_cfg.get('is_fr', None) 58 | is_fr = False if is_fr is None else is_fr 59 | metric_model.eval() 60 | for test_dataset, dataset_path in zip(test_datasets, dataset_paths): 61 | run( 62 | metric_model, 63 | dataset_path, 64 | test_dataset, 65 | attack_callback=attack_callback, 66 | save_path=save_path, 67 | is_fr=is_fr, 68 | jpeg_quality=jpeg_quality, 69 | metric_range=metric_range, 70 | device=device 71 | ) 72 | print(f'Attack run successfully on {test_datasets} datasets. Results saved to {save_path}') 73 | 74 | 75 | def test_main_uap(device, metric_model, metric_cfg, test_datasets, dataset_paths, save_path, train_datasets, uap_paths, amplitudes, jpeg_quality=None): 76 | bounds_metric = metric_cfg.get('bounds', None) 77 | metric_range = 100 if bounds_metric is None else bounds_metric['high'] - bounds_metric['low'] 78 | is_fr = metric_cfg.get('is_fr', None) 79 | is_fr = False if is_fr is None else is_fr 80 | #model = module.MetricModel(args.device, *metric_model) 81 | # model = MetricModel(device, *metric_model) 82 | metric_model.eval() 83 | for train_dataset, uap_path in zip(train_datasets, uap_paths): 84 | for test_dataset, dataset_path in zip(test_datasets, dataset_paths): 85 | run_uap( 86 | metric_model, 87 | uap_path, 88 | dataset_path, 89 | train_dataset, 90 | test_dataset, 91 | amplitude=amplitudes, 92 | is_fr=is_fr, 93 | jpeg_quality=jpeg_quality, 94 | save_path=save_path, 95 | device=device 96 | ) 97 | print(f'UAP attack run successfully on {test_datasets} datasets. Results saved to {save_path}') 98 | 99 | 100 | def run_attacks(metric_model, metric_cfg, save_dir, dataset_names, dataset_paths, attacks_dict=all_default_attacks, jpeg_quality=None, 101 | device=DEFAULT_DEVICE, uap_cfg=uap_default_config): 102 | """ 103 | Run attacks from attacks_dict on metric on datasets specified in dataset_names/paths.\n 104 | Args: 105 | metric_model (PyTorch model): Metric model to be attacked. Should be an object of a class that inherits torch.nn.module and has a forward method.\n 106 | metric_cfg (dict): Dictionary containing following items:\n 107 | 'is_fr': (bool) Whether metric is Full-Reference\n 108 | 'name': (str) Name of the metric\n 109 | 'bounds': (dict) Dict with keys 'low' and 'high' specifing minimum and maximum possible metric values if such limits exist, approximate range of metric's typical values otherwise.\n 110 | save_dir (str): Path to directory where attack results will be stored\n 111 | dataset_names (list of strings): Names of used test datasets\n 112 | dataset_paths (list of strings): Paths to test datasets (should be in the same order as in dataset_names)\n 113 | attacks_dict (dict): Dictionary containing attack callables, with keys being attack names\n 114 | (default is all_default_attacks - dict with all attacks used in framework)\n 115 | device (str or torch.device): device to use during computations\n 116 | (default is DEFAULT_DEVICE constant, which is "cpu")\n 117 | jpeg_quality (int/None): Compress image/ video frame after attack. Can be used to assess attack efficiency after compression. 118 | (default is None (no compression))\n 119 | uap_cfg (dict): Configuration of UAP-based attacks.\n 120 | (default is uap_default_config)\n 121 | Returns:\n 122 | Nothing. All results are saved in save_dir directory. 123 | """ 124 | for atk_name, atk_callable in attacks_dict.items(): 125 | cur_csv_dir = Path(save_dir) / atk_name 126 | cur_csv_dir.mkdir(exist_ok=True) 127 | cur_csv_path = str(Path(cur_csv_dir) / f'{metric_cfg["name"]}.csv') 128 | print(cur_csv_path) 129 | if atk_name in uap_attacks.keys(): 130 | uap_paths = [] 131 | for dset in uap_cfg['train_datasets']: 132 | uap_paths.append(str(Path(uap_cfg['uap_path']) / f'{atk_name}_{metric_cfg["name"]}_{dset}.npy')) 133 | test_main_uap(device=device, metric_model=metric_model, 134 | metric_cfg=metric_cfg, test_datasets=dataset_names, dataset_paths=dataset_paths, 135 | save_path=cur_csv_path,jpeg_quality=jpeg_quality, train_datasets=uap_cfg['train_datasets'], uap_paths=uap_paths, amplitudes=uap_cfg['amplitudes']) 136 | else: 137 | test_main(attack_callback=atk_callable, device=device, metric_model=metric_model, 138 | metric_cfg=metric_cfg, test_datasets=dataset_names, dataset_paths=dataset_paths, 139 | save_path=cur_csv_path,jpeg_quality=jpeg_quality) 140 | 141 | 142 | def collect_results(save_dir, metric_cfg, uap_cfg=uap_default_config): 143 | """ 144 | Collect results of attacks produced by run_attacks().\n 145 | Args: 146 | metric_cfg (dict): Dictionary containing following items:\n 147 | 'is_fr': (bool) Whether metric is Full-Reference\n 148 | 'name': (str) Name of the metric\n 149 | 'bounds': (dict) Dict with keys 'low' and 'high' specifing minimum and maximum possible metric values if such limits exist, approximate range of metric's typical values otherwise.\n 150 | save_dir (str): Path to directory where attack results are stored\n 151 | uap_cfg: (dict) Configuration of UAP-based attacks. Should contain following items:\n 152 | 'uap_path': (str) path to directory with pretrained universal additives.\n 153 | 'amplitudes': (list of floats) strength of additive attack. should be in range [0,1].\n 154 | 'train_datasets': (list of strings) List of train datasets, on which UAP was trained. Current options: 'COCO', 'VOC2012'.\n 155 | (default is uap_default_config: {'uap_path':'./res/','amplitudes':[0.2, 0.4, 0.8], 'train_datasets':['COCO', 'VOC2012']})\n 156 | Returns:\n 157 | pandas.DataFrame: DataFrame with columns [dataset, attack, *metric_name*_clear/attacked/ssim/psnr/mse/rel_gain]. It contains merged raw results for all attacks present in save_dir. 158 | """ 159 | required_cols = ['clear', 'attacked', 'ssim', 'psnr', 'mse'] 160 | metric_name = metric_cfg['name'] 161 | result_df = pd.DataFrame() 162 | for atk_dir in Path(save_dir).iterdir(): 163 | atk_name = str(Path(atk_dir).stem) 164 | for csv_file in Path(atk_dir).iterdir(): 165 | if not str(csv_file).endswith('.csv'): 166 | continue 167 | cur_df = pd.read_csv(csv_file).rename(columns={'test_dataset': 'dataset'}) 168 | if 'uap' not in atk_name: 169 | data_to_add = cur_df[['dataset'] + required_cols].copy() 170 | data_to_add = data_to_add.rename(columns={x: f'{metric_name}_{x}' for x in required_cols}) 171 | data_to_add['attack'] = atk_name 172 | #data_to_add.dataset.replace(to_replace={'NIPS2017': 'NIPS'}, inplace=True) 173 | result_df = pd.concat([result_df, data_to_add], axis=0) 174 | else: 175 | for amp in uap_cfg['amplitudes']: 176 | for train_set in uap_cfg['train_datasets']: 177 | cur_atk_name = atk_name 178 | if atk_name == 'uap': 179 | cur_atk_name = 'default-uap' 180 | cur_atk_name += f'_{train_set}_amp{str(amp)}' 181 | data_to_add = cur_df[cur_df['amplitude'] == amp][cur_df['train_dataset'] == train_set][['dataset'] + required_cols].copy() 182 | data_to_add = data_to_add.rename(columns={x: f'{metric_name}_{x}' for x in required_cols}) 183 | data_to_add['attack'] = cur_atk_name 184 | #data_to_add.dataset.replace(to_replace={'NIPS2017': 'NIPS'}, inplace=True) 185 | result_df = pd.concat([result_df, data_to_add], axis=0) 186 | return result_df 187 | 188 | 189 | def domain_transform(result_df_wide, metrics, batch_size=1000, device=DEFAULT_DEVICE, models_path=models_filepath): 190 | """ 191 | Apply domain transformation to collected attack results from collect_results().\n 192 | Args: 193 | result_df_wide (pd.DataFrame): DataFrame with values to be transformed. Should contain columns *metric_name*_clear/attacked for all metrics in metrics argument. 194 | models_to_mdtvsfa.pth file in models_path directory should contain pretrained model for each metric in metrics. 195 | metrics (list of strings): List of metric names to be transformed. If other metrics are present in result_df_wide, they won't be affacted.\n 196 | batch_size (int): Batch size, used during transformation process. Higher batch size results in higher VRAM/RAM usage.\n 197 | (default is 1000)\n 198 | models_path (str): Path to domain transformation model's weights directory, which should contain models_to_mdtvsfa.pth.\n 199 | (default is models_filepath (robustness_benchmark/models/), containing pretrained transformation models to framework metrics (preloaded with pip module))\n 200 | device (str or torch.device): device to use during computations\n 201 | (default is DEFAULT_DEVICE constant, which is "cpu")\n 202 | Returns:\n 203 | pandas.DataFrame: DataFrame similar to input(result_df_wide), but with modified *metric_name*_clear/attacked columns. 204 | """ 205 | def chunker(df, size): 206 | return [df.iloc[pos:pos + size] for pos in range(0, len(df), size)] 207 | data_transformed = pd.DataFrame() 208 | for cur_df in tqdm(chunker(result_df_wide, 1000)): 209 | cur_data_transformed = transform_full_df(df=cur_df, metrics=metrics, path_to_models=models_path, domain='mdtvsfa', dev=device) 210 | data_transformed = pd.concat([data_transformed, cur_data_transformed]) 211 | return data_transformed 212 | 213 | 214 | def evaluate_robustness(result_df_wide, attacks=None, metrics=['maniqa'], add_conf_intervals=True, methods=list(method_name_to_func.keys()), raw_values=False): 215 | """ 216 | Evaluate metric's robustness to attacks and return table with results on each type of attack.\n 217 | Args: 218 | result_df_wide (pd.DataFrame): Input data to evaluate. Should contain columns 'attack' and *metric_name*_clear/attacked/ssim for all metrics in metrics argument.\n 219 | attacks (list or None): List of attacks to evaluate at. 220 | Can also contain special words 'all' (evaluate on all data regardless of attack name), 'iterative' (all iterative attacks), 'uap' (all uap-based attacks). None (default) equal to ['all'].\n 221 | (default is None)\n 222 | metrics (list of strings): list of metrics to evaluate. All should be present in result_df_wide. 223 | (default is ['maniqa'], demo metric) 224 | add_conf_intervals (bool): Whether to provide confidence intervals for methods that involve averaging across dataset. Defaults to True\n 225 | methods (list of strings): Evaluation methods' names. Defaults to all methods used in article.\n 226 | Available options: 'robustness_score', 'normalized_relative_gain', 'normalized_absolute_gain', 'wasserstein_score', 'energy_distance_score'. 227 | raw_values (bool): If set to True, returned DataFrame will contain evaluations in float format, with additional columns for lower and upper conf. intervals' bounds (not very human readable). 228 | If False, DataFrame cells will contain formatted strings with evaluations. Defaults to False.\n 229 | Returns:\n 230 | pandas.DataFrame: results DataFrame with columns corresponding to evaluation methods and rows to metric/attack pairs. 231 | """ 232 | results_df = pd.DataFrame(columns=['metric']) 233 | for metric_to_evaluate in metrics: 234 | eval_df = result_df_wide[['attack', f'{metric_to_evaluate}_clear', f'{metric_to_evaluate}_attacked', f'{metric_to_evaluate}_ssim']].copy() 235 | if attacks is None: 236 | attacks = ['all'] 237 | for atk in attacks: 238 | cur_row = {'metric': metric_to_evaluate, 'attack': atk} 239 | 240 | if atk == 'all': 241 | cur_eval_df = eval_df.copy() 242 | elif atk == 'iterative': 243 | cur_eval_df = eval_df[~eval_df.attack.str.contains('uap', regex=False)].copy() 244 | elif atk == 'uap': 245 | cur_eval_df = eval_df[eval_df.attack.str.contains('uap', regex=False)].copy() 246 | else: 247 | cur_eval_df = eval_df[eval_df.attack == atk].copy() 248 | 249 | 250 | for method in methods: 251 | scores = method_name_to_func[method](cur_eval_df[f'{metric_to_evaluate}_clear'], cur_eval_df[f'{metric_to_evaluate}_attacked']) 252 | scores = np.array(scores) 253 | score = np.mean(scores) 254 | cur_row[method] = score 255 | if not raw_values: 256 | cur_row[method] = '{:.3f}'.format(score) 257 | if add_conf_intervals and method not in ['wasserstein_score', 'energy_distance_score']: 258 | low, high = st.t.interval(0.95, len(scores)-1, loc=np.mean(scores), scale=st.sem(scores)) 259 | if not raw_values: 260 | cur_row[method] += ' ({:.3f}, {:.3f})'.format(low, high) 261 | else: 262 | cur_row[f'{method}_conf_interval_low'] = low 263 | cur_row[f'{method}_conf_interval_high'] = high 264 | results_df = results_df.append(cur_row, ignore_index=True) 265 | return results_df 266 | 267 | 268 | def run_full_pipeline(metric_model, metric_cfg, save_dir, dataset_names, dataset_paths, attacks_dict=all_default_attacks, 269 | device=DEFAULT_DEVICE, run_params={}, use_domain_transform=False, domain_transform_params={}, eval_params={}): 270 | """ 271 | Run full benchmark pipeline: run attacks, save results, collect them, apply domain transform, evaluate.\n 272 | Args: 273 | metric_model (PyTorch model): Metric model to be tested. Should be an object of a class that inherits torch.nn.module and has a forward method.\n 274 | metric_cfg (dict): Dictionary containing following items:\n 275 | 'is_fr': (bool) Whether metric is Full-Reference\n 276 | 'name': (str) Name of the metric\n 277 | 'bounds': (dict) Dict with keys 'low' and 'high' specifing minimum and maximum possible metric values if such limits exist, approximate range of metric's typical values otherwise.\n 278 | save_dir (str): Path to directory where attack results will be stored\n 279 | dataset_names (list of strings): Names of used test datasets\n 280 | dataset_paths (list of strings): Paths to test datasets (should be in the same order as in dataset_names)\n 281 | attacks_dict (dict): Dictionary containing attack callables, with keys being attack names\n 282 | (default is all_default_attacks - dict with all attacks used in framework)\n 283 | device (str or torch.device): device to use during computations\n 284 | (default is DEFAULT_DEVICE constant, which is "cpu")\n 285 | run_params (dict): additional params passed to run_attacks(). Can contain following items: \n 286 | 'jpeg_quality': (int/None) Compress image/ video frame after attack. Can be used to assess attack efficiency after compression. Defaults to None (no compression).\n 287 | 'uap_cfg': (dict) Configuration of UAP-based attacks. Defaults to uap_default_config.\n 288 | (default is empty dict)\n 289 | use_domain_transform (bool): Whether to use domain transformation before the evaluation. If set to True, metric should be one of those used in the framework\n 290 | (default is False)\n 291 | domain_transform_params (dict): additional params passed to domain_transform(). Can contain following items: \n 292 | 'batch_size': (int) Batch size, used during transformation process. Higher batch size results in higher VRAM/RAM usage. Defaults to 1000.\n 293 | 'models_path': (str) Path to domain transformation model's weights. Defaults to models_filepath, containing pretrained transformation models to framework metrics.\n 294 | (default is empty dict)\n 295 | eval_params (dict): additional params passed to evaluate_robustness(). Can contain following items: \n 296 | 'add_conf_intervals': (bool) Whether to provide confidence intervals for methods that involve averaging across dataset. Defaults to True\n 297 | 'methods': (list of strings) Evaluation methods' names. Defaults to all methods used in article.\n 298 | 'raw_values': (bool) If set to True, returned DataFrame will contain evaluations in float format, with additional columns for lower and upper conf. intervals' bounds (not very human readable). 299 | If False, DataFrame cells will contain formatted strings with evaluations. Defaults to False.\n 300 | (default is empty dict) 301 | Returns:\n 302 | pandas.DataFrame: results DataFrame with columns corresponding to evaluation methods and rows to metric/attack pairs. 303 | """ 304 | print('Running attacks...') 305 | run_attacks(metric_model=metric_model, metric_cfg=metric_cfg, save_dir=save_dir, 306 | dataset_names=dataset_names, dataset_paths=dataset_paths, 307 | attacks_dict=attacks_dict,device=device, **run_params) 308 | print('Collecting results...') 309 | collect_params = {} 310 | if 'uap_cfg' in run_params.keys(): 311 | collect_params['uap_cfg'] = run_params['uap_cfg'] 312 | results_df = collect_results(save_dir=save_dir, metric_cfg=metric_cfg, **collect_params) 313 | if use_domain_transform: 314 | print('Applying domain transformation...') 315 | results_df_transformed = domain_transform(result_df_wide=results_df, metrics=[metric_cfg['name']], device=device, **domain_transform_params) 316 | results_df = results_df_transformed.copy() 317 | print('Evaluating...') 318 | return evaluate_robustness(result_df_wide=results_df, attacks=attacks_dict, metrics=[metric_cfg['name']], **eval_params) 319 | 320 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/__init__.py: -------------------------------------------------------------------------------- 1 | from . import utils 2 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/amifgsm/run.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch.autograd import Variable 4 | #from evaluate import test_main 5 | 6 | 7 | def attack(compress_image, ref_image=None, model=None, metric_range=100, device='cpu'): 8 | eps = 10 / 255 9 | iters = 10 10 | nu = 1.0 11 | compress_image = Variable(compress_image.clone().to(device), requires_grad=True) 12 | 13 | g_prev = 0 14 | lr = eps / iters 15 | for i in range(iters): 16 | score = model(ref_image.to(device), compress_image.to(device)) if ref_image is not None else model(compress_image.to(device)) 17 | score = score.mean() 18 | loss = 1 - score / metric_range 19 | loss.backward() 20 | g = compress_image.grad + g_prev * nu 21 | g_prev = g 22 | g = torch.sign(g) 23 | compress_image.data -= lr * g 24 | compress_image.data.clamp_(0., 1.) 25 | compress_image.grad.zero_() 26 | 27 | res_image = (compress_image).data.clamp_(min=0, max=1) 28 | return res_image 29 | 30 | # if __name__ == "__main__": 31 | # test_main(attack) 32 | 33 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/cumulative-uap/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from evaluate import test_main_uap 5 | 6 | 7 | if __name__ == "__main__": 8 | test_main_uap() -------------------------------------------------------------------------------- /robustness_benchmark/methods/cumulative-uap/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | from torch.autograd import Variable 6 | from torch.utils.data import DataLoader 7 | 8 | 9 | from tqdm import tqdm 10 | 11 | from read_dataset import MyCustomDataset 12 | from evaluate import jpeg_generator 13 | 14 | 15 | from evaluate import train_main_uap 16 | 17 | 18 | def train(metric_model, path_train, batch_size=8, is_fr=False, jpeg_quality=None, metric_range=100, device='cpu'): 19 | ds_train = MyCustomDataset(path_gt=path_train, device=device) 20 | dl_train = DataLoader(ds_train, batch_size=batch_size, shuffle=True) 21 | eps = 0.1 22 | n_epoch = 5 23 | universal_noise = torch.zeros((1, 3, 256, 256)).to(device) 24 | universal_noise += 0.0001 25 | universal_noise = Variable(universal_noise, requires_grad=True) 26 | for epoch in range(n_epoch): 27 | loss = [] 28 | for u in tqdm(range(len(dl_train))): 29 | y = next(iter(dl_train)) 30 | 31 | for orig_image, jpeg_image in jpeg_generator(y, jpeg_quality): 32 | if is_fr: 33 | res = (jpeg_image.to(device) + universal_noise).type(torch.FloatTensor) 34 | a = 1 - metric_model(y, res).mean() / metric_range 35 | else: 36 | res = y + universal_noise 37 | a = 1 - metric_model(res).mean() / metric_range 38 | loss.append(a.item()) 39 | a.backward() 40 | delta = universal_noise.grad.clone() 41 | universal_noise.grad.data.zero_() 42 | universal_noise = universal_noise - delta 43 | universal_noise.data.clamp_(min=-eps, max=eps) 44 | universal_noise = Variable(universal_noise, requires_grad=True) 45 | 46 | return universal_noise.squeeze().data.cpu().numpy().transpose(1, 2, 0) 47 | 48 | 49 | if __name__ == "__main__": 50 | train_main_uap(train) -------------------------------------------------------------------------------- /robustness_benchmark/methods/generative-uap/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from evaluate import test_main_uap 5 | 6 | 7 | if __name__ == "__main__": 8 | test_main_uap() -------------------------------------------------------------------------------- /robustness_benchmark/methods/generative-uap/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | 6 | from torch import nn 7 | 8 | import torch 9 | from torch.autograd import Variable 10 | from torch.utils.data import DataLoader 11 | 12 | from tqdm import tqdm 13 | 14 | from read_dataset import MyCustomDataset 15 | from evaluate import jpeg_generator 16 | 17 | from evaluate import train_main_uap 18 | 19 | 20 | class UnetGenerator(nn.Module): 21 | def __init__(self, input_nc, output_nc, ngf, norm_type='batch', act_type='selu'): 22 | super(UnetGenerator, self).__init__() 23 | self.name = 'unet' 24 | self.conv1 = nn.Conv2d(input_nc, ngf, 4, 2, 1) 25 | self.conv2 = nn.Conv2d(ngf, ngf * 2, 4, 2, 1) 26 | self.conv3 = nn.Conv2d(ngf * 2, ngf * 4, 4, 2, 1) 27 | self.conv4 = nn.Conv2d(ngf * 4, ngf * 8, 4, 2, 1) 28 | self.conv5 = nn.Conv2d(ngf * 8, ngf * 8, 4, 2, 1) 29 | self.conv6 = nn.Conv2d(ngf * 8, ngf * 8, 4, 2, 1) 30 | self.conv7 = nn.Conv2d(ngf * 8, ngf * 8, 4, 2, 1) 31 | self.conv8 = nn.Conv2d(ngf * 8, ngf * 8, 4, 2, 1) 32 | self.dconv1 = nn.ConvTranspose2d(ngf * 8, ngf * 8, 4, 2, 1) 33 | self.dconv2 = nn.ConvTranspose2d(ngf * 8 * 2, ngf * 8, 4, 2, 1) 34 | self.dconv3 = nn.ConvTranspose2d(ngf * 8 * 2, ngf * 8, 4, 2, 1) 35 | self.dconv4 = nn.ConvTranspose2d(ngf * 8 * 2, ngf * 8, 4, 2, 1) 36 | self.dconv5 = nn.ConvTranspose2d(ngf * 8 * 2, ngf * 4, 4, 2, 1) 37 | self.dconv6 = nn.ConvTranspose2d(ngf * 4 * 2, ngf * 2, 4, 2, 1) 38 | self.dconv7 = nn.ConvTranspose2d(ngf * 2 * 2, ngf, 4, 2, 1) 39 | self.dconv8 = nn.ConvTranspose2d(ngf * 2, output_nc, 4, 2, 1) 40 | 41 | if norm_type == 'batch': 42 | self.norm = nn.BatchNorm2d(ngf) 43 | self.norm2 = nn.BatchNorm2d(ngf * 2) 44 | self.norm4 = nn.BatchNorm2d(ngf * 4) 45 | self.norm8 = nn.BatchNorm2d(ngf * 8) 46 | elif norm_type == 'instance': 47 | self.norm = nn.InstanceNorm2d(ngf) 48 | self.norm2 = nn.InstanceNorm2d(ngf * 2) 49 | self.norm4 = nn.InstanceNorm2d(ngf * 4) 50 | self.norm8 = nn.InstanceNorm2d(ngf * 8) 51 | self.leaky_relu = nn.LeakyReLU(0.2, True) 52 | 53 | if act_type == 'selu': 54 | self.act = nn.SELU(True) 55 | else: 56 | self.act = nn.ReLU(True) 57 | 58 | self.dropout = nn.Dropout(0.5) 59 | 60 | self.tanh = nn.Tanh() 61 | 62 | def forward(self, input): 63 | # Encoder 64 | # Convolution layers: 65 | # input is (nc) x 512 x 1024 66 | e1 = self.conv1(input) 67 | # state size is (ngf) x 256 x 512 68 | e2 = self.norm2(self.conv2(self.leaky_relu(e1))) 69 | # state size is (ngf x 2) x 128 x 256 70 | e3 = self.norm4(self.conv3(self.leaky_relu(e2))) 71 | # state size is (ngf x 4) x 64 x 128 72 | e4 = self.norm8(self.conv4(self.leaky_relu(e3))) 73 | # state size is (ngf x 8) x 32 x 64 74 | e5 = self.norm8(self.conv5(self.leaky_relu(e4))) 75 | # state size is (ngf x 8) x 16 x 32 76 | e6 = self.norm8(self.conv6(self.leaky_relu(e5))) 77 | # state size is (ngf x 8) x 8 x 16 78 | e7 = self.norm8(self.conv7(self.leaky_relu(e6))) 79 | # state size is (ngf x 8) x 4 x 8 80 | # No batch norm on output of Encoder 81 | e8 = self.conv8(self.leaky_relu(e7)) 82 | 83 | # Decoder 84 | # Deconvolution layers: 85 | # state size is (ngf x 8) x 2 x 4 86 | d1_ = self.dropout(self.norm8(self.dconv1(self.act(e8)))) 87 | # state size is (ngf x 8) x 4 x 8 88 | d1 = torch.cat((d1_, e7), 1) 89 | d2_ = self.dropout(self.norm8(self.dconv2(self.act(d1)))) 90 | # state size is (ngf x 8) x 8 x 16 91 | d2 = torch.cat((d2_, e6), 1) 92 | d3_ = self.dropout(self.norm8(self.dconv3(self.act(d2)))) 93 | # state size is (ngf x 8) x 16 x 32 94 | d3 = torch.cat((d3_, e5), 1) 95 | d4_ = self.norm8(self.dconv4(self.act(d3))) 96 | # state size is (ngf x 8) x 32 x 64 97 | d4 = torch.cat((d4_, e4), 1) 98 | d5_ = self.norm4(self.dconv5(self.act(d4))) 99 | # state size is (ngf x 4) x 64 x 128 100 | d5 = torch.cat((d5_, e3), 1) 101 | d6_ = self.norm2(self.dconv6(self.act(d5))) 102 | # state size is (ngf x 2) x 128 x 256 103 | d6 = torch.cat((d6_, e2), 1) 104 | d7_ = self.norm(self.dconv7(self.act(d6))) 105 | # state size is (ngf) x 256 x 512 106 | d7 = torch.cat((d7_, e1), 1) 107 | d8 = self.dconv8(self.act(d7)) 108 | # state size is (nc) x 512 x 1024 109 | output = self.tanh(d8) 110 | return output 111 | 112 | 113 | 114 | 115 | def normalize_and_scale(delta_im, mode='train'): 116 | delta_im = (delta_im) * 10.0/255.0 117 | return delta_im 118 | 119 | 120 | 121 | 122 | def train(metric_model, path_train, batch_size=8, is_fr=False, jpeg_quality=None, metric_range=100, device='cpu'): 123 | ds_train = MyCustomDataset(path_gt=path_train, device=device) 124 | dl_train = DataLoader(ds_train, batch_size=batch_size, shuffle=True) 125 | eps = 10.0/255. 126 | n_epoch = 1 127 | lr = 0.001 128 | universal_noise = torch.rand((1, 3, 256, 256)).to(device) 129 | universal_noise = Variable(universal_noise, requires_grad=True) 130 | netG = UnetGenerator(3, 3, 64, norm_type='instance', act_type='relu').to(device) 131 | optimizer = torch.optim.Adam(netG.parameters(), lr=lr) 132 | for epoch in range(n_epoch): 133 | for u in tqdm(range(len(dl_train))): 134 | y = next(iter(dl_train)) 135 | 136 | for orig_image, jpeg_image in jpeg_generator(y, jpeg_quality): 137 | delta_im = netG(universal_noise) 138 | delta_im = normalize_and_scale(delta_im, 'train') 139 | if is_fr: 140 | res = (jpeg_image.to(device) + delta_im).type(torch.FloatTensor) 141 | res.data.clamp_(min=0.,max=1.) 142 | a = 1 - metric_model(y, res).mean() / metric_range 143 | else: 144 | res = y + delta_im 145 | res.data.clamp_(min=0.,max=1.) 146 | a = 1 - metric_model(res).mean() / metric_range 147 | optimizer.zero_grad() 148 | a.backward() 149 | optimizer.step() 150 | #universal_noise.data = universal_noise.data * min(1, eps/(universal_noise.data**2).mean()**0.5) 151 | universal_noise.data.clamp_(min=-eps, max=eps) 152 | delta_im = netG(universal_noise) 153 | delta_im = normalize_and_scale(delta_im, 'train') 154 | return delta_im.squeeze().data.cpu().numpy().transpose(1, 2, 0) 155 | 156 | 157 | if __name__ == "__main__": 158 | train_main_uap(train) -------------------------------------------------------------------------------- /robustness_benchmark/methods/ifgsm/run.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch.autograd import Variable 4 | #from evaluate import test_main 5 | 6 | 7 | 8 | def attack(compress_image, ref_image=None, model=None, metric_range=100, device='cpu'): 9 | eps = 10 / 255 10 | iters = 10 11 | alpha = 1/255 12 | compress_image = Variable(compress_image.clone().to(device), requires_grad=True) 13 | 14 | p = torch.zeros_like(compress_image).to(device) 15 | p = Variable(p, requires_grad=True) 16 | for i in range(iters): 17 | res = compress_image + p 18 | res.data.clamp_(0., 1.) 19 | score = model(ref_image.to(device), res.to(device)) if ref_image is not None else model(res.to(device)) 20 | loss = 1 - score / metric_range 21 | loss.backward() 22 | g = p.grad 23 | g = torch.sign(g) 24 | p.data -= alpha * g 25 | p.data.clamp_(-eps, +eps) 26 | p.grad.zero_() 27 | res_image = compress_image + p 28 | 29 | res_image = (res_image).data.clamp_(min=0, max=1) 30 | return res_image 31 | 32 | 33 | # if __name__ == "__main__": 34 | # test_main(attack) 35 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/korhonen-et-al/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/robustness_benchmark/methods/korhonen-et-al/.gitkeep -------------------------------------------------------------------------------- /robustness_benchmark/methods/korhonen-et-al/run.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch.autograd import Variable 4 | #from evaluate import test_main 5 | 6 | import numpy as np 7 | 8 | import cv2 9 | from scipy import ndimage 10 | from torchvision import transforms 11 | 12 | def rgb2ycbcr(im_rgb): 13 | im_rgb = im_rgb.astype(np.float32) 14 | im_ycrcb = cv2.cvtColor(im_rgb, cv2.COLOR_RGB2YCR_CB) 15 | im_ycbcr = im_ycrcb[:,:,(0,2,1)].astype(np.float32) 16 | im_ycbcr[:,:,0] = (im_ycbcr[:,:,0]*(235-16)+16)/255.0 17 | im_ycbcr[:,:,1:] = (im_ycbcr[:,:,1:]*(240-16)+16)/255.0 18 | return im_ycbcr 19 | 20 | def makeSpatialActivityMap(im): 21 | im = im.cpu().detach().permute(0, 2, 3, 1).numpy()[0] 22 | #H = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) / 8 23 | im = rgb2ycbcr(im) 24 | im_sob = ndimage.sobel(im[:,:,0]) 25 | im_zero = np.zeros_like(im_sob) 26 | im_zero[1:-1, 1:-1] = im_sob[1:-1, 1:-1] 27 | 28 | maxval = im_zero.max() 29 | 30 | if maxval == 0: 31 | im_zero = im_zero + 1 32 | maxval = 1 33 | 34 | im_sob = im_zero /maxval 35 | 36 | DF = np.array([[0, 1, 1, 1, 0], 37 | [1, 1, 1, 1, 1], 38 | [1, 1, 1, 1, 1], 39 | [1, 1, 1, 1, 1], 40 | [0, 1, 1, 1, 0]]).astype('uint8') 41 | 42 | out_im = cv2.dilate(im_sob, DF) 43 | return out_im 44 | 45 | 46 | def attack(compress_image, ref_image=None, model=None, metric_range=100, device='cpu'): 47 | iters = 10 48 | lr = 0.005 49 | 50 | sp_map = makeSpatialActivityMap(compress_image * 255) 51 | sp_map = sp_map / 255 52 | sp_map = transforms.ToTensor()(sp_map.astype(np.float32)) 53 | sp_map = sp_map.unsqueeze_(0) 54 | sp_map = sp_map.to(device) 55 | 56 | 57 | compress_image = Variable(compress_image, requires_grad=True) 58 | opt = torch.optim.Adam([compress_image], lr = lr) 59 | 60 | for i in range(iters): 61 | score = model(ref_image.to(device), compress_image.to(device)) if ref_image is not None else model(compress_image.to(device)) 62 | loss = 1 - score / metric_range 63 | loss.backward() 64 | compress_image.grad *= sp_map 65 | opt.step() 66 | compress_image.data.clamp_(0., 1.) 67 | opt.zero_grad() 68 | 69 | res_image = (compress_image).data.clamp_(min=0, max=1) 70 | 71 | return res_image 72 | 73 | # if __name__ == "__main__": 74 | # test_main(attack) 75 | 76 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/madc/run.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch.autograd import Variable 4 | #from evaluate import test_main 5 | 6 | 7 | def attack(compress_image, ref_image=None, model=None, metric_range=100, device='cpu'): 8 | init_image = compress_image.clone() 9 | compress_image = Variable(compress_image, requires_grad=True).to(device) 10 | init_image = Variable(init_image, requires_grad=False).to(device) 11 | lr = 0.001 12 | eps = 10 / 255 13 | iters = 8 14 | for i in range(iters): 15 | score = model(ref_image.to(device), compress_image.to(device)) if ref_image is not None else model(compress_image.to(device)) 16 | score = score.mean() 17 | loss = 1 - score / metric_range 18 | loss.backward() 19 | g2 = compress_image.grad.clone() 20 | compress_image.grad.zero_() 21 | 22 | if (i < 1): 23 | pg = g2.clone() 24 | else: 25 | loss = ((compress_image - init_image)**2).mean()**0.5 26 | loss.backward() 27 | g1 = compress_image.grad.clone() 28 | compress_image.grad.zero_() 29 | pg = g2 - (g1*g2).sum() / (g1*g1).sum() * g1 30 | 31 | 32 | pg = torch.sign(pg) 33 | compress_image.data -= lr * pg 34 | compress_image.grad.zero_() 35 | 36 | cur_score = ((compress_image - init_image)**2).mean()**0.5 37 | while cur_score > eps: 38 | cur_score.backward() 39 | g2 = torch.sign(compress_image.grad) 40 | compress_image.data -= 0.0005 * g2 41 | compress_image.grad.zero_() 42 | compress_image.data.clamp_(0., 1) 43 | cur_score = ((compress_image - init_image)**2).mean()**0.5 44 | 45 | compress_image.data.clamp_(0., 1.) 46 | compress_image.grad.zero_() 47 | res_image = (compress_image).data.clamp_(min=0, max=1) 48 | 49 | return res_image 50 | 51 | 52 | # if __name__ == "__main__": 53 | # test_main(attack) 54 | 55 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/mifgsm/run.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch.autograd import Variable 4 | #from evaluate import test_main 5 | 6 | 7 | def attack(compress_image, ref_image=None, model=None, metric_range=100, device='cpu'): 8 | eps = 10 / 255 9 | iters = 10 10 | alpha = 1/255 11 | nu = 1.0 12 | compress_image = Variable(compress_image.clone().to(device), requires_grad=True) 13 | 14 | p = torch.zeros_like(compress_image).to(device) 15 | p = Variable(p, requires_grad=True) 16 | g_prev = 0 17 | for i in range(iters): 18 | res = compress_image + p 19 | res.data.clamp_(0., 1.) 20 | score = model(ref_image.to(device), res.to(device)) if ref_image is not None else model(res.to(device)) 21 | loss = 1 - score / metric_range 22 | loss.backward() 23 | g = p.grad + g_prev * nu 24 | g_prev = g 25 | g = torch.sign(g) 26 | p.data -= alpha * g 27 | p.data.clamp_(-eps, +eps) 28 | p.grad.zero_() 29 | res_image = compress_image + p 30 | 31 | res_image = (res_image).data.clamp_(min=0, max=1) 32 | return res_image 33 | 34 | 35 | 36 | # if __name__ == "__main__": 37 | # test_main(attack) 38 | 39 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/std-fgsm/run.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from torch.autograd import Variable 4 | # from evaluate import test_main 5 | 6 | def attack(compress_image, ref_image=None, model=None, metric_range=100, device='cpu'): 7 | eps = 10 / 255 8 | compress_image = Variable(compress_image.clone().to(device), requires_grad=True) 9 | score = model(ref_image.to(device), compress_image.to(device)) if ref_image is not None else model(compress_image.to(device)) 10 | loss = 1 - score / metric_range 11 | loss.backward() 12 | g = compress_image.grad 13 | g = torch.sign(g) 14 | compress_image.data -= eps * g 15 | compress_image.data.clamp_(0., 1.) 16 | compress_image.grad.zero_() 17 | 18 | res_image = (compress_image).data.clamp_(min=0, max=1) 19 | return res_image 20 | 21 | # if __name__ == "__main__": 22 | # test_main(attack) 23 | 24 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/uap/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from evaluate import test_main_uap 5 | 6 | 7 | if __name__ == "__main__": 8 | test_main_uap() -------------------------------------------------------------------------------- /robustness_benchmark/methods/uap/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | from torch.autograd import Variable 6 | from torch.utils.data import DataLoader 7 | 8 | 9 | from tqdm import tqdm 10 | 11 | from read_dataset import MyCustomDataset 12 | from evaluate import jpeg_generator 13 | 14 | 15 | from evaluate import train_main_uap 16 | 17 | 18 | def train(metric_model, path_train, batch_size=8, is_fr=False, jpeg_quality=None, metric_range=100, device='cpu'): 19 | ds_train = MyCustomDataset(path_gt=path_train, device=device) 20 | dl_train = DataLoader(ds_train, batch_size=batch_size, shuffle=True) 21 | eps = 0.1 22 | lr = 0.001 23 | n_epoch = 5 24 | universal_noise = torch.zeros((1, 3, 256, 256)).to(device) 25 | universal_noise += 0.0001 26 | universal_noise = Variable(universal_noise, requires_grad=True) 27 | optimizer = torch.optim.Adam([universal_noise], lr=lr) 28 | for epoch in range(n_epoch): 29 | loss = [] 30 | for u in tqdm(range(len(dl_train))): 31 | y = next(iter(dl_train)) 32 | 33 | for orig_image, jpeg_image in jpeg_generator(y, jpeg_quality): 34 | if is_fr: 35 | res = (jpeg_image.to(device) + universal_noise).type(torch.FloatTensor) 36 | res.data.clamp_(min=0.,max=1.) 37 | loss = 1 - metric_model(y, res).mean() / metric_range 38 | else: 39 | res = y + universal_noise 40 | res.data.clamp_(min=0.,max=1.) 41 | loss = 1 - metric_model(res).mean() / metric_range 42 | 43 | optimizer.zero_grad() 44 | universal_noise.retain_grad() 45 | loss.backward() 46 | optimizer.step() 47 | universal_noise.data.clamp_(min=-eps, max=eps) 48 | return universal_noise.squeeze().data.cpu().numpy().transpose(1, 2, 0) 49 | 50 | 51 | 52 | if __name__ == "__main__": 53 | train_main_uap(train) -------------------------------------------------------------------------------- /robustness_benchmark/methods/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .evaluate import * 2 | from .read_dataset import * 3 | -------------------------------------------------------------------------------- /robustness_benchmark/methods/utils/bounds.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /robustness_benchmark/methods/utils/evaluate.py: -------------------------------------------------------------------------------- 1 | from torchvision import transforms 2 | import torch 3 | import cv2 4 | import os 5 | import csv 6 | import json 7 | import importlib 8 | import numpy as np 9 | from .read_dataset import to_numpy, to_torch, iter_images 10 | from .metrics import PSNR, SSIM, MSE 11 | 12 | def predict(img1, img2=None, model=None, device='cpu'): 13 | model.to(device) 14 | if not torch.is_tensor(img1): 15 | img1 = transforms.ToTensor()(img1).unsqueeze(0) 16 | img1 = img1.type(torch.FloatTensor).to(device) 17 | if img2 is not None: 18 | if not torch.is_tensor(img2): 19 | img2 = transforms.ToTensor()(img2).unsqueeze(0) 20 | img2 = img2.to(device) 21 | img2 = img2.type(torch.FloatTensor).to(device) 22 | 23 | return model(img1, img2).item() 24 | else: 25 | return model(img1).item() 26 | 27 | 28 | 29 | 30 | def compress(img, q, return_torch=False): 31 | encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), q] 32 | np_batch = to_numpy(img) 33 | if len(np_batch.shape) == 3: 34 | np_batch = np_batch[np.newaxis] 35 | jpeg_batch = np.empty(np_batch.shape) 36 | for i in range(len(np_batch)): 37 | result, encimg = cv2.imencode('.jpg', np_batch[i] * 255, encode_param) 38 | jpeg_batch[i] = cv2.imdecode(encimg, 1) / 255 39 | return to_torch(jpeg_batch) if return_torch else jpeg_batch[0] 40 | 41 | 42 | def jpeg_generator(img_gen, jpeg_quality): 43 | if jpeg_quality is None: 44 | yield img_gen, None 45 | else: 46 | for q in jpeg_quality: 47 | jpeg_image = compress(img_gen, q, return_torch=True) 48 | yield img_gen, jpeg_image 49 | 50 | 51 | def run(model, dataset_path, test_dataset, attack_callback, save_path='res.csv', is_fr=False, jpeg_quality=None, metric_range=100, device='cpu'): 52 | 53 | with open(save_path, 'a', newline='') as csvfile: 54 | fieldnames = ['image_name', 'test_dataset', 'clear', 'attacked', 'rel_gain', 'psnr', 'ssim', 'mse'] 55 | writer = csv.DictWriter(csvfile, fieldnames=fieldnames) 56 | if csvfile.tell() == 0: 57 | writer.writeheader() 58 | for image, fn in iter_images(dataset_path): 59 | orig_image = image 60 | h, w = image.shape[:2] 61 | image = transforms.ToTensor()(image.astype(np.float32)) 62 | image = image.unsqueeze_(0) 63 | image = image.to(device) 64 | 65 | 66 | 67 | if is_fr: 68 | for q in jpeg_quality: 69 | jpeg_image = compress(orig_image, q, return_torch=True) 70 | clear_metric = predict(orig_image, jpeg_image, model=model, device=device) 71 | attacked_image = attack_callback(jpeg_image, image.clone().detach(), model=model, metric_range=metric_range, device=device) 72 | attacked_metric = predict(orig_image, attacked_image, model=model, device=device) 73 | 74 | writer.writerow({ 75 | 'image_name': f'{fn}-jpeg{q}', 76 | 'clear': clear_metric, 77 | 'attacked': attacked_metric, 78 | 'rel_gain': (attacked_metric / clear_metric) if abs(clear_metric) >= 1e-3 else float('inf'), 79 | 'test_dataset': test_dataset, 80 | 'psnr' : PSNR(jpeg_image, attacked_image), 81 | 'ssim' : SSIM(jpeg_image, attacked_image), 82 | 'mse' : MSE(jpeg_image, attacked_image) 83 | }) 84 | else: 85 | clear_metric = predict(orig_image, model=model, device=device) 86 | attacked_image = attack_callback(image.clone().detach(), model=model, metric_range=metric_range, device=device) 87 | attacked_metric = predict(attacked_image, model=model, device=device) 88 | 89 | writer.writerow({ 90 | 'image_name': fn, 91 | 'clear': clear_metric, 92 | 'attacked': attacked_metric, 93 | 'rel_gain': (attacked_metric / clear_metric) if abs(clear_metric) >= 1e-3 else float('inf'), 94 | 'test_dataset': test_dataset, 95 | 'psnr' : PSNR(image, attacked_image), 96 | 'ssim' : SSIM(image, attacked_image), 97 | 'mse' : MSE(image, attacked_image) 98 | }) 99 | 100 | def test_main(attack_callback): 101 | import argparse 102 | parser = argparse.ArgumentParser() 103 | parser.add_argument("--device", type=str, default='cpu') 104 | parser.add_argument("--metric", type=str, required=True) 105 | parser.add_argument("--test-dataset", type=str, nargs='+') 106 | parser.add_argument("--dataset-path", type=str, nargs='+') 107 | parser.add_argument("--jpeg-quality", type=int, default=None, nargs='*') 108 | parser.add_argument("--save-path", type=str, default='res.csv') 109 | args = parser.parse_args() 110 | with open('src/config.json') as json_file: 111 | config = json.load(json_file) 112 | metric_model = config['weight'] 113 | module = config['module'] 114 | is_fr = config['is_fr'] 115 | with open('bounds.json') as json_file: 116 | bounds = json.load(json_file) 117 | bounds_metric = bounds.get(args.metric, None) 118 | metric_range = 100 if bounds_metric is None else bounds_metric['high'] - bounds_metric['low'] 119 | module = importlib.import_module(f'src.{module}') 120 | model = module.MetricModel(args.device, *metric_model) 121 | model.eval() 122 | for test_dataset, dataset_path in zip(args.test_dataset, args.dataset_path): 123 | run( 124 | model, 125 | dataset_path, 126 | test_dataset, 127 | attack_callback=attack_callback, 128 | save_path=args.save_path, 129 | is_fr=is_fr, 130 | jpeg_quality=args.jpeg_quality, 131 | metric_range=metric_range, 132 | device=args.device 133 | ) 134 | 135 | 136 | def train_main_uap(train_callback): 137 | import importlib 138 | import argparse 139 | parser = argparse.ArgumentParser() 140 | parser.add_argument("--path-train", type=str, nargs='+') 141 | parser.add_argument("--metric", type=str, required=True) 142 | parser.add_argument("--train-dataset", type=str, nargs='+') 143 | parser.add_argument("--save-dir", type=str, default="./") 144 | parser.add_argument("--batch-size", type=int, default=8) 145 | parser.add_argument("--jpeg-quality", type=int, default=None, nargs='*') 146 | parser.add_argument("--device", type=str, default='cpu') 147 | args = parser.parse_args() 148 | with open('src/config.json') as json_file: 149 | config = json.load(json_file) 150 | metric_model = config['weight'] 151 | module = config['module'] 152 | is_fr = config['is_fr'] 153 | with open('bounds.json') as json_file: 154 | bounds = json.load(json_file) 155 | bounds_metric = bounds.get(args.metric, None) 156 | metric_range = 100 if bounds_metric is None else bounds_metric['high'] - bounds_metric['low'] 157 | module = importlib.import_module(f'src.{module}') 158 | model = module.MetricModel(args.device, *metric_model) 159 | model.eval() 160 | for train_dataset, path_train in zip(args.train_dataset, args.path_train): 161 | uap = train_callback(model, path_train, batch_size=args.batch_size, is_fr=is_fr, jpeg_quality=args.jpeg_quality, metric_range=metric_range, device=args.device) 162 | cv2.imwrite(os.path.join(args.save_dir, f'{train_dataset}.png'), (uap + 0.1) * 255) 163 | np.save(os.path.join(args.save_dir, f'{train_dataset}.npy'), uap) 164 | 165 | 166 | def run_uap(model, uap, dataset_path, train_dataset, test_dataset, amplitude=[0.2], is_fr=False, jpeg_quality=None, save_path='res.csv', device='cpu'): 167 | if isinstance(uap, str): 168 | uap = np.load(uap) 169 | 170 | with open(save_path, 'a', newline='') as csvfile: 171 | fieldnames = ['image_name', 'train_dataset', 'test_dataset', 'clear', 'attacked', 'amplitude', 'rel_gain', 'psnr', 'ssim', 'mse'] 172 | writer = csv.DictWriter(csvfile, fieldnames=fieldnames) 173 | if csvfile.tell() == 0: 174 | writer.writeheader() 175 | for image, fn in iter_images(dataset_path): 176 | 177 | orig_image = image 178 | h, w = orig_image.shape[:2] 179 | 180 | uap_h, uap_w = uap.shape[0], uap.shape[1] 181 | uap_resized = np.tile(uap,(h // uap_h + 1, w // uap_w + 1, 1))[:h, :w, :] 182 | 183 | if is_fr: 184 | for q in jpeg_quality: 185 | jpeg_image = compress(orig_image, q) 186 | clear_metric = predict(orig_image, jpeg_image, model=model, device=device) 187 | for k in amplitude: 188 | attacked_image = jpeg_image + uap_resized * k 189 | attacked_metric = predict(orig_image, attacked_image, model=model, device=device) 190 | 191 | writer.writerow({ 192 | 'image_name': f'{fn}-jpeg{q}', 193 | 'clear': clear_metric, 194 | 'attacked': attacked_metric, 195 | 'rel_gain': (attacked_metric / clear_metric) if abs(clear_metric) >= 1e-3 else float('inf'), 196 | 'train_dataset': train_dataset, 197 | 'test_dataset': test_dataset, 198 | 'amplitude': k, 199 | 'psnr' : PSNR(jpeg_image, attacked_image), 200 | 'ssim' : SSIM(jpeg_image, attacked_image), 201 | 'mse' : MSE(jpeg_image, attacked_image) 202 | }) 203 | else: 204 | clear_metric = predict(orig_image, model=model, device=device) 205 | for k in amplitude: 206 | attacked_image = orig_image + uap_resized * k 207 | attacked_metric= predict(attacked_image, model=model, device=device) 208 | 209 | writer.writerow({ 210 | 'image_name': fn, 211 | 'clear': clear_metric, 212 | 'attacked': attacked_metric, 213 | 'rel_gain': (attacked_metric / clear_metric) if abs(clear_metric) >= 1e-3 else float('inf'), 214 | 'train_dataset': train_dataset, 215 | 'test_dataset': test_dataset, 216 | 'amplitude': k, 217 | 'psnr' : PSNR(orig_image, attacked_image), 218 | 'ssim' : SSIM(orig_image, attacked_image), 219 | 'mse' : MSE(orig_image, attacked_image) 220 | }) 221 | 222 | def test_main_uap(): 223 | import argparse 224 | parser = argparse.ArgumentParser() 225 | parser.add_argument("--device", type=str, default='cpu') 226 | parser.add_argument("--uap-path", type=str, nargs='+') 227 | parser.add_argument("--metric", type=str, required=True) 228 | parser.add_argument("--train-dataset", type=str, nargs='+') 229 | parser.add_argument("--test-dataset", type=str, nargs='+') 230 | parser.add_argument("--dataset-path", type=str, nargs='+') 231 | parser.add_argument("--save-path", type=str, default='res.csv') 232 | parser.add_argument("--jpeg-quality", type=int, default=None, nargs='*') 233 | parser.add_argument("--amplitude", type=float, default=[0.2], nargs='+') 234 | args = parser.parse_args() 235 | with open('src/config.json') as json_file: 236 | config = json.load(json_file) 237 | metric_model = config['weight'] 238 | module = config['module'] 239 | is_fr = config['is_fr'] 240 | module = importlib.import_module(f'src.{module}') 241 | model = module.MetricModel(args.device, *metric_model) 242 | model.eval() 243 | for train_dataset, uap_path in zip(args.train_dataset, args.uap_path): 244 | for test_dataset, dataset_path in zip(args.test_dataset, args.dataset_path): 245 | run_uap( 246 | model, 247 | uap_path, 248 | dataset_path, 249 | train_dataset, 250 | test_dataset, 251 | amplitude=args.amplitude, 252 | is_fr=is_fr, 253 | jpeg_quality=args.jpeg_quality, 254 | save_path=args.save_path, 255 | device=args.device 256 | ) -------------------------------------------------------------------------------- /robustness_benchmark/methods/utils/metrics.py: -------------------------------------------------------------------------------- 1 | from skimage.metrics import peak_signal_noise_ratio, structural_similarity, mean_squared_error 2 | 3 | from .read_dataset import to_numpy 4 | 5 | 6 | def PSNR(x, y): 7 | return peak_signal_noise_ratio(to_numpy(x)[0], to_numpy(y)[0]) 8 | 9 | 10 | def SSIM(x, y): 11 | return structural_similarity(to_numpy(x)[0], to_numpy(y)[0], channel_axis=2, data_range=1) 12 | 13 | def MSE(x, y): 14 | return mean_squared_error(to_numpy(x)[0], to_numpy(y)[0]) -------------------------------------------------------------------------------- /robustness_benchmark/methods/utils/read_dataset.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from pathlib import Path 3 | import os 4 | import av 5 | from tqdm import tqdm 6 | from torch.utils.data import Dataset 7 | import torch 8 | import random 9 | from PIL import Image 10 | import numpy as np 11 | from torchvision import transforms 12 | 13 | 14 | def to_torch(x, device='cpu'): 15 | if isinstance(x, np.ndarray): 16 | x = torch.from_numpy(x) 17 | if len(x.shape) == 3: 18 | x = x.permute(2, 0, 1).unsqueeze(0) 19 | else: 20 | x = x.permute(0, 3, 1, 2) 21 | x = x.type(torch.FloatTensor).to(device) 22 | return x 23 | 24 | def to_numpy(x): 25 | if torch.is_tensor(x): 26 | x = x.cpu().detach().permute(0, 2, 3, 1).numpy() 27 | return x if len(x.shape) == 4 else x[np.newaxis] 28 | 29 | def iter_images(path): 30 | def iter_file(fn): 31 | print(fn) 32 | if Path(fn).suffix in ('.png', '.jpg'): 33 | image = cv2.imread(fn) 34 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 35 | image = image / 255 36 | yield image, os.path.basename(fn) 37 | elif Path(fn).suffix in ('.y4m'): 38 | container = av.open(fn) 39 | video_stream = container.streams.video[0] 40 | for i, frame in enumerate(container.decode(video_stream)): 41 | image = frame.to_rgb().to_ndarray() 42 | image = image / 255. 43 | yield image, f'{os.path.basename(fn)}-{i}' 44 | container.close() 45 | else: 46 | print('Error path:', fn) 47 | yield None 48 | if os.path.isdir(path): 49 | for fn in tqdm(os.listdir(path)): 50 | print(fn) 51 | for image in iter_file(os.path.join(path, fn)): 52 | yield image 53 | else: 54 | for image in iter_file(path): 55 | yield image 56 | 57 | 58 | def center_crop(image): 59 | center = image.shape[0] / 2, image.shape[1] / 2 60 | if center[1] < 256 or center[0] < 256: 61 | return cv2.resize(image, (256, 256)) 62 | x = center[1] - 128 63 | y = center[0] - 128 64 | 65 | return image[int(y):int(y+256), int(x):int(x+256)] 66 | 67 | class MyCustomDataset(Dataset): 68 | def __init__(self, 69 | path_gt, 70 | device='cpu' 71 | ): 72 | 73 | self._items = [] 74 | self._index = 0 75 | self.device = device 76 | dir_img = sorted(os.listdir(path_gt)) 77 | img_pathes = dir_img 78 | 79 | for img_path in img_pathes: 80 | self._items.append(( 81 | os.path.join(path_gt, img_path) 82 | )) 83 | random.shuffle(self._items) 84 | 85 | def __len__(self): 86 | return len(self._items) 87 | 88 | def next_data(self): 89 | gt_path = self._items[self._index] 90 | self._index += 1 91 | if self._index == len(self._items): 92 | self._index = 0 93 | random.shuffle(self._items) 94 | 95 | image = Image.open(gt_path).convert('RGB') 96 | image = np.array(image).astype(np.float32) 97 | image = center_crop(image) 98 | 99 | image = image / 255. 100 | image = transforms.ToTensor()(image) 101 | y = image.to(self.device) 102 | return y 103 | 104 | def __getitem__(self, index): 105 | gt_path = self._items[index] 106 | image = Image.open(gt_path).convert('RGB') 107 | image = np.array(image).astype(np.float32) 108 | 109 | image = center_crop(image) 110 | 111 | image = image / 255. 112 | image = transforms.ToTensor()(image) 113 | y = image.to(self.device) 114 | return y 115 | -------------------------------------------------------------------------------- /robustness_benchmark/models/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /robustness_benchmark/models/models_to_mdtvsfa_1d.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/robustness_benchmark/models/models_to_mdtvsfa_1d.pth -------------------------------------------------------------------------------- /robustness_benchmark/requirements.txt: -------------------------------------------------------------------------------- 1 | av>=10.0.0 2 | numpy>=1.19.5 3 | pandas>=1.1.5 4 | Pillow>=10.0.0 5 | scikit_image>=0.17.2 6 | scikit_learn>=1.3.0 7 | scipy>=1.4.1 8 | setuptools>=59.5.0 9 | torchvision>=0.11.3 10 | tqdm>=4.57.0 11 | torch>=1.10.2 12 | opencv_python>=4.2.0.32 13 | -------------------------------------------------------------------------------- /robustness_benchmark/score_methods.py: -------------------------------------------------------------------------------- 1 | from scipy.stats import wasserstein_distance, energy_distance 2 | from sklearn.preprocessing import MinMaxScaler 3 | import numpy as np 4 | 5 | 6 | def calc_wasserstein_score(source_vals, proc_vals, inverse_sign=False, custom_scaler=None): 7 | if custom_scaler is None: 8 | scaler = MinMaxScaler() 9 | source_vals_scaled = scaler.fit_transform(np.array(source_vals.values).reshape(-1, 1)) 10 | proc_vals_scaled = scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 11 | else: 12 | source_vals_scaled = custom_scaler.transform(np.array(source_vals.values).reshape(-1, 1)) 13 | proc_vals_scaled = custom_scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 14 | k = 1. 15 | if inverse_sign: 16 | k = -1. 17 | return k * wasserstein_distance(source_vals_scaled.squeeze(), proc_vals_scaled.squeeze()) * np.sign(np.mean(proc_vals_scaled) - np.mean(source_vals_scaled)) 18 | 19 | 20 | def energy_distance_score(source_vals, proc_vals, inverse_sign=False, custom_scaler=None): 21 | if custom_scaler is None: 22 | scaler = MinMaxScaler() 23 | source_vals_scaled = scaler.fit_transform(np.array(source_vals.values).reshape(-1, 1)) 24 | proc_vals_scaled = scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 25 | else: 26 | source_vals_scaled = custom_scaler.transform(np.array(source_vals.values).reshape(-1, 1)) 27 | proc_vals_scaled = custom_scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 28 | 29 | k = 1. 30 | if inverse_sign: 31 | k = -1. 32 | return k * energy_distance(source_vals_scaled.squeeze(), proc_vals_scaled.squeeze()) * np.sign(np.mean(proc_vals_scaled) - np.mean(source_vals_scaled)) 33 | 34 | 35 | def normalized_absolute_gain(source_vals, proc_vals, inverse_sign=False, custom_scaler=None): 36 | if custom_scaler is None: 37 | scaler = MinMaxScaler() 38 | source_vals_scaled = scaler.fit_transform(np.array(source_vals.values).reshape(-1, 1)) 39 | proc_vals_scaled = scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 40 | else: 41 | source_vals_scaled = custom_scaler.transform(np.array(source_vals.values).reshape(-1, 1)) 42 | proc_vals_scaled = custom_scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 43 | 44 | k = 1. 45 | if inverse_sign: 46 | k = -1. 47 | return k * (proc_vals_scaled.squeeze() - source_vals_scaled.squeeze()) 48 | 49 | 50 | def normalized_relative_gain(source_vals, proc_vals, inverse_sign=False, custom_scaler=None): 51 | if custom_scaler is None: 52 | scaler = MinMaxScaler() 53 | source_vals_scaled = scaler.fit_transform(np.array(source_vals.values).reshape(-1, 1)) + 1.0 54 | proc_vals_scaled = scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) + 1.0 55 | else: 56 | source_vals_scaled = custom_scaler.transform(np.array(source_vals.values).reshape(-1, 1)) + 1.0 57 | proc_vals_scaled = custom_scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) + 1.0 58 | 59 | k = 1. 60 | if inverse_sign: 61 | k = -1. 62 | return (k * (proc_vals_scaled - source_vals_scaled) / (source_vals_scaled)).reshape(-1) 63 | 64 | 65 | def robustness_score(source_vals, proc_vals, inverse_sign=False, beta1=1., beta2=0., eps=1e-6, normalize=True, custom_scaler=None): 66 | if normalize: 67 | if custom_scaler is None: 68 | scaler = MinMaxScaler() 69 | source_vals_scaled = scaler.fit_transform(np.array(source_vals.values).reshape(-1, 1)) 70 | proc_vals_scaled = scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 71 | else: 72 | source_vals_scaled = custom_scaler.transform(np.array(source_vals.values).reshape(-1, 1)) 73 | proc_vals_scaled = custom_scaler.transform(np.array(proc_vals.values).reshape(-1, 1)) 74 | else: 75 | source_vals_scaled = source_vals 76 | proc_vals_scaled = proc_vals 77 | denom = np.abs(source_vals_scaled - proc_vals_scaled) + eps 78 | numer = np.maximum(beta1 - source_vals_scaled, source_vals_scaled - beta2) 79 | scores = np.log10(numer / denom) 80 | return scores.reshape(-1) 81 | 82 | 83 | def relative_gain_classic(source_vals, proc_vals, inverse_sign=False, eps=1e-6, custom_scaler=None): 84 | return (proc_vals - source_vals) / (source_vals + eps) 85 | 86 | 87 | method_name_to_func = { 88 | 'robustness_score': robustness_score, 89 | 'relative_gain_classic': relative_gain_classic, 90 | 'normalized_relative_gain': normalized_relative_gain, 91 | 'normalized_absolute_gain': normalized_absolute_gain, 92 | 'wasserstein_score': calc_wasserstein_score, 93 | 'energy_distance_score': energy_distance_score, 94 | } 95 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = robustness_benchmark 3 | version = 0.0.1 4 | author = Khaled Abud 5 | author_email = khaled.abud@graphics.cs.msu.ru 6 | description = Demo module for MSU IQA/VQA Metrics Robustness Benchmark. 7 | long_description = file: README.md 8 | long_description_content_type = text/markdown 9 | url = https://github.com/msu-video-group/MSU_Metrics_Robustness_Benchmark 10 | project_urls = 11 | Bug Tracker = https://github.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/issues 12 | classifiers = 13 | Programming Language :: Python :: 3 14 | License :: OSI Approved :: MIT License 15 | Operating System :: OS Independent 16 | 17 | [options] 18 | packages = robustness_benchmark 19 | python_requires = >=3.7 20 | install_requires = 21 | setuptools==59.5.0 22 | av==10.0.0 23 | numpy==1.19.5 24 | pandas==1.1.5 25 | Pillow==10.0.0 26 | scikit_image==0.17.2 27 | scikit_learn==1.3.0 28 | scipy==1.4.1 29 | setuptools==59.5.0 30 | torchvision==0.11.3 31 | tqdm==4.57.0 32 | torch>=1.10.2 33 | opencv_python==4.2.0.32 -------------------------------------------------------------------------------- /subjects/_init/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stable 2 | 3 | ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DEBIAN_FRONTEND=noninteractive 4 | 5 | RUN apt-get update -qq && apt-get upgrade -qqy \ 6 | && apt-get install -qqy --no-install-recommends ffmpeg docker.io wget ca-certificates imagemagick 7 | -------------------------------------------------------------------------------- /subjects/maniqa/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM cnstark/pytorch:1.13.1-py3.9.12-cuda11.7.1-devel-ubuntu20.04 2 | 3 | WORKDIR / 4 | 5 | RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -qqy gcc wget git dos2unix libgl1 libglib2.0-0 6 | 7 | RUN wget https://github.com/IIGROUP/MANIQA/releases/download/Koniq10k/ckpt_koniq10k.pt 8 | 9 | COPY src /src 10 | 11 | COPY patches/maniqa.patch /src 12 | 13 | RUN cd /src && dos2unix maniqa.patch && dos2unix models/maniqa.py && patch -Np1 < maniqa.patch 14 | 15 | RUN pip install einops 16 | 17 | COPY test.py / 18 | 19 | COPY model.py /src 20 | COPY config.json /src 21 | -------------------------------------------------------------------------------- /subjects/maniqa/config.json: -------------------------------------------------------------------------------- 1 | {"weight": ["ckpt_koniq10k.pt"], "module": "model", "is_fr": false} -------------------------------------------------------------------------------- /subjects/maniqa/model.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import sys 3 | sys.path.append(str(Path(__file__).parent)) 4 | import torch 5 | from torchvision import transforms 6 | from maniqa_models.maniqa import MANIQA 7 | from config import Config 8 | sys.path.remove(str(Path(__file__).parent)) 9 | 10 | 11 | class MetricModel(torch.nn.Module): 12 | def __init__(self, device, model_path): 13 | super().__init__() 14 | self.device = device 15 | 16 | config = Config({ 17 | # model 18 | "patch_size": 8, 19 | "img_size": 224, 20 | "embed_dim": 768, 21 | "dim_mlp": 768, 22 | "num_heads": [4, 4], 23 | "window_size": 4, 24 | "depths": [2, 2], 25 | "num_outputs": 1, 26 | "num_tab": 2, 27 | "scale": 0.8, 28 | 29 | }) 30 | 31 | model = MANIQA( 32 | embed_dim=config.embed_dim, num_outputs=config.num_outputs, dim_mlp=config.dim_mlp, 33 | patch_size=config.patch_size, img_size=config.img_size, window_size=config.window_size, 34 | depths=config.depths, num_heads=config.num_heads, num_tab=config.num_tab, scale=config.scale 35 | ) 36 | 37 | model.load_state_dict(torch.load(model_path), strict=False) 38 | model.eval().to(device) 39 | 40 | self.model = model 41 | 42 | def forward(self, image, inference=False): 43 | out = self.model( 44 | transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])(transforms.Resize([224, 224])(image)) 45 | ) 46 | if inference: 47 | return out.detach().cpu().numpy()[0][0].item() 48 | else: 49 | return out 50 | 51 | -------------------------------------------------------------------------------- /subjects/maniqa/patches/maniqa.patch: -------------------------------------------------------------------------------- 1 | {models => maniqa_models}/maniqa.py | 4 ++-- 2 | {models => maniqa_models}/swin.py | 0 3 | 2 files changed, 2 insertions(+), 2 deletions(-) 4 | rename {models => maniqa_models}/maniqa.py (98%) 5 | rename {models => maniqa_models}/swin.py (100%) 6 | 7 | diff --git a/models/maniqa.py b/maniqa_models/maniqa.py 8 | similarity index 98% 9 | rename from models/maniqa.py 10 | rename to maniqa_models/maniqa.py 11 | index aa6d012..ab6aa6b 100644 12 | --- a/models/maniqa.py 13 | +++ b/maniqa_models/maniqa.py 14 | @@ -3,7 +3,7 @@ import torch.nn as nn 15 | import timm 16 | 17 | from timm.models.vision_transformer import Block 18 | -from models.swin import SwinTransformer 19 | +from maniqa_models.swin import SwinTransformer 20 | from torch import nn 21 | from einops import rearrange 22 | 23 | @@ -54,7 +54,7 @@ class MANIQA(nn.Module): 24 | self.input_size = img_size // patch_size 25 | self.patches_resolution = (img_size // patch_size, img_size // patch_size) 26 | 27 | - self.vit = timm.create_model('vit_base_patch8_224', pretrained=True) 28 | + self.vit = timm.create_model('vit_base_patch8_224', pretrained=False) 29 | self.save_output = SaveOutput() 30 | hook_handles = [] 31 | for layer in self.vit.modules(): 32 | diff --git a/models/swin.py b/maniqa_models/swin.py 33 | similarity index 100% 34 | rename from models/swin.py 35 | rename to maniqa_models/swin.py 36 | -------------------------------------------------------------------------------- /subjects/maniqa/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | from torchvision import transforms 6 | import numpy as np 7 | from src.model import MetricModel 8 | 9 | 10 | def main(): 11 | import argparse 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("--dist_path", type=str) 14 | parser.add_argument("--width", type=int) 15 | parser.add_argument("--height", type=int) 16 | args = parser.parse_args() 17 | bps = 3 18 | if args.width * args.height <= 0: 19 | raise RuntimeError("unsupported resolution") 20 | 21 | 22 | model = MetricModel('cuda:0', 'ckpt_koniq10k.pt') 23 | 24 | transform = transforms.Compose([ 25 | transforms.ToTensor() 26 | ]) 27 | 28 | print("value") 29 | with open(args.dist_path, 'rb') as dist_rgb24, torch.no_grad(): 30 | while True: 31 | dist = dist_rgb24.read(args.width * args.height * bps) 32 | if len(dist) == 0: 33 | break 34 | if len(dist) != args.width * args.height * bps: 35 | raise RuntimeError("unexpected end of stream dist_path") 36 | 37 | dist = np.frombuffer(dist, dtype='uint8').reshape((args.height,args.width,bps)) / 255. 38 | score = model(torch.unsqueeze(transform(dist), 0).type(torch.FloatTensor).to('cuda:0')).item() 39 | print(score) 40 | 41 | if __name__ == "__main__": 42 | main() 43 | -------------------------------------------------------------------------------- /test_dataset/000b7d55b6184b08.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/test_dataset/000b7d55b6184b08.png -------------------------------------------------------------------------------- /test_dataset/00c3cd597f1ee96f.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/test_dataset/00c3cd597f1ee96f.png -------------------------------------------------------------------------------- /test_dataset/0aebe24fc257286e.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/test_dataset/0aebe24fc257286e.png -------------------------------------------------------------------------------- /test_dataset/0af0a5dfee6b84ff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/test_dataset/0af0a5dfee6b84ff.png -------------------------------------------------------------------------------- /test_dataset/0b1d45bd9ab1064e.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msu-video-group/MSU_Metrics_Robustness_Benchmark/375969e7f1fa054b95f45ad1504bfac182ddabe7/test_dataset/0b1d45bd9ab1064e.png -------------------------------------------------------------------------------- /test_results/amifgsm/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,test_dataset,clear,attacked,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.108695387840271,2.2282473817902613,30.214003078712757,0.8099236,0.0009519183355173187 3 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.0902782678604126,1.9178548433246627,29.778992505029183,0.9068289,0.0010522059407604943 4 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0608562231063843,3.1606156999983486,30.18495664884191,0.81297666,0.0009583062820461024 5 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0918309688568115,2.137912948180551,30.128886353807292,0.7936383,0.000970758863641919 6 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9565749168395996,1.9687964561167195,29.416243527478102,0.9053827,0.0011438673062120872 7 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.108695387840271,2.2282473817902613,30.214003078712757,0.8099236,0.0009519183355173187 8 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.0902782678604126,1.9178548433246627,29.778992505029183,0.9068289,0.0010522059407604943 9 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.061284065246582,3.161890372810721,30.172040147583758,0.8124275,0.0009611606561955041 10 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0918309688568115,2.137912948180551,30.128886353807292,0.7936383,0.000970758863641919 11 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9565749168395996,1.9687964561167195,29.416243527478102,0.9053827,0.0011438673062120872 12 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.108695387840271,2.2282473817902613,30.214003078712757,0.8099236,0.0009519183355173187 13 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.0902782678604126,1.9178548433246627,29.778992505029183,0.9068289,0.0010522059407604943 14 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0608562231063843,3.1606156999983486,30.18495664884191,0.81297666,0.0009583062820461024 15 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.1020655632019043,2.1579532954447975,30.131209805053505,0.7938822,0.0009702396519663914 16 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9563784599304199,1.9683921138537523,29.416695284912663,0.9054168,0.001143748326186841 17 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.108695387840271,2.2282473817902613,30.214003078712757,0.8099236,0.0009519183355173187 18 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.0902782678604126,1.9178548433246627,29.778992505029183,0.9068289,0.0010522059407604943 19 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0645169019699097,3.1715219838440816,30.16466090836287,0.812048,0.0009627951838685644 20 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0918309688568115,2.137912948180551,30.128886353807292,0.7936383,0.000970758863641919 21 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9563784599304199,1.9683921138537523,29.416695284912663,0.9054168,0.001143748326186841 22 | -------------------------------------------------------------------------------- /test_results/cumulative-uap/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,train_dataset,test_dataset,clear,attacked,amplitude,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4946455955505371,0.2,0.9941348771610125,47.65332046951042,0.9952990458409495,1.7165954341796153e-05 3 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.48507142066955566,0.4,0.9748927748258465,41.632720556230794,0.982269950424862,6.866381736718461e-05 4 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4838493764400482,0.8,0.9724367198221937,35.612120642951176,0.9401847071748731,0.0002746552694687385 5 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5670455098152161,0.2,0.997461849366925,47.65332046951042,0.9979328357919209,1.7165954341796157e-05 6 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5608012676239014,0.4,0.9864779102364072,41.632720556230794,0.9921318855109128,6.866381736718463e-05 7 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5588101148605347,0.8,0.9829753714042647,35.612120642951176,0.9728591416740668,0.0002746552694687385 8 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.3338219225406647,0.2,0.994557779277741,47.65332046951042,0.9945786500474307,1.7165954341796153e-05 9 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.33055031299591064,0.4,0.9848106521306375,41.632720556230794,0.9800536597130258,6.866381736718461e-05 10 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.36963626742362976,0.8,1.101259685018368,35.612120642951176,0.9362683571211715,0.00027465526946873845 11 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.500960111618042,0.2,0.9809294109614541,47.65332046951042,0.9950018111468356,1.7165954341796157e-05 12 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.47768133878707886,0.4,0.9353472729999539,41.632720556230794,0.9812671180987539,6.866381736718461e-05 13 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.46581336855888367,0.8,0.9121086143218199,35.612120642951176,0.9378609261305737,0.00027465526946873845 14 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.48360034823417664,0.2,0.9953330732588195,47.65332046951042,0.9984024368705443,1.7165954341796157e-05 15 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.4790230989456177,0.4,0.9859123033646526,41.632720556230794,0.9937892356685766,6.866381736718463e-05 16 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.4730798304080963,0.8,0.973680029834963,35.612120642951176,0.9774333219984209,0.0002746552694687385 17 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.48383012413978577,0.2,0.9723980266986146,36.295110364216995,0.9290926751036293,0.00023468696225735632 18 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.4190899431705475,0.4,0.8422837136749585,30.27451045093737,0.7879219948168167,0.0009387478490294253 19 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.2958766520023346,0.8,0.5946505977997748,24.253910537657745,0.5448045738031873,0.003754991396117701 20 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5289449691772461,0.2,0.9304410634356272,36.295110364216995,0.9738788997964681,0.00023468696225735632 21 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.4608010947704315,0.4,0.8105725276438553,30.27451045093737,0.913147523231268,0.0009387478490294253 22 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.3514421582221985,0.8,0.6182046044241701,24.253910537657745,0.7679798858424242,0.003754991396117701 23 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.31204724311828613,0.2,0.9296843382347326,36.295110364216995,0.9328547557928116,0.00023468696225735632 24 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.26785242557525635,0.4,0.7980144369393006,30.27451045093737,0.7952437530725809,0.0009387478490294253 25 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.21097581088542938,0.8,0.6285615766591035,24.253910537657745,0.5543148536120782,0.003754991396117701 26 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.44085627794265747,0.2,0.8632401642601669,36.295110364216995,0.924139861752391,0.00023468696225735632 27 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.36228200793266296,0.4,0.7093839776893751,30.27451045093737,0.7817024077477366,0.0009387478490294253 28 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.27194643020629883,0.8,0.5324979881805975,24.253910537657745,0.5544408559123455,0.003754991396117701 29 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.45070987939834595,0.2,0.9276388055709923,36.295110364216995,0.9754365727316975,0.00023468696225735632 30 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.4004955589771271,0.4,0.8242890581008917,30.27451045093737,0.9179956647158725,0.0009387478490294253 31 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.30118998885154724,0.8,0.6199010367404322,24.253910537657745,0.7861440022321221,0.003754991396117701 32 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4946455955505371,0.2,0.9941348771610125,47.65332046951042,0.9952990458409495,1.7165954341796153e-05 33 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.48507142066955566,0.4,0.9748927748258465,41.632720556230794,0.982269950424862,6.866381736718461e-05 34 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4838493764400482,0.8,0.9724367198221937,35.612120642951176,0.9401847071748731,0.0002746552694687385 35 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5670455098152161,0.2,0.997461849366925,47.65332046951042,0.9979328357919209,1.7165954341796157e-05 36 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5608012676239014,0.4,0.9864779102364072,41.632720556230794,0.9921318855109128,6.866381736718463e-05 37 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5588101148605347,0.8,0.9829753714042647,35.612120642951176,0.9728591416740668,0.0002746552694687385 38 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.3338219225406647,0.2,0.994557779277741,47.65332046951042,0.9945786500474307,1.7165954341796153e-05 39 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.33055031299591064,0.4,0.9848106521306375,41.632720556230794,0.9800536597130258,6.866381736718461e-05 40 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.36963626742362976,0.8,1.101259685018368,35.612120642951176,0.9362683571211715,0.00027465526946873845 41 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.500960111618042,0.2,0.9809294109614541,47.65332046951042,0.9950018111468356,1.7165954341796157e-05 42 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.47768133878707886,0.4,0.9353472729999539,41.632720556230794,0.9812671180987539,6.866381736718461e-05 43 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.46581336855888367,0.8,0.9121086143218199,35.612120642951176,0.9378609261305737,0.00027465526946873845 44 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.48360034823417664,0.2,0.9953330732588195,47.65332046951042,0.9984024368705443,1.7165954341796157e-05 45 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.4790230989456177,0.4,0.9859123033646526,41.632720556230794,0.9937892356685766,6.866381736718463e-05 46 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.4730798304080963,0.8,0.973680029834963,35.612120642951176,0.9774333219984209,0.0002746552694687385 47 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.48383012413978577,0.2,0.9723980266986146,36.295110364216995,0.9290926751036293,0.00023468696225735632 48 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.4190899431705475,0.4,0.8422837136749585,30.27451045093737,0.7879219948168167,0.0009387478490294253 49 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.2958766520023346,0.8,0.5946505977997748,24.253910537657745,0.5448045738031873,0.003754991396117701 50 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5289449691772461,0.2,0.9304410634356272,36.295110364216995,0.9738788997964681,0.00023468696225735632 51 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.4608010947704315,0.4,0.8105725276438553,30.27451045093737,0.913147523231268,0.0009387478490294253 52 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.3514421582221985,0.8,0.6182046044241701,24.253910537657745,0.7679798858424242,0.003754991396117701 53 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.31204724311828613,0.2,0.9296843382347326,36.295110364216995,0.9328547557928116,0.00023468696225735632 54 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.26785242557525635,0.4,0.7980144369393006,30.27451045093737,0.7952437530725809,0.0009387478490294253 55 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.21097581088542938,0.8,0.6285615766591035,24.253910537657745,0.5543148536120782,0.003754991396117701 56 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.44085627794265747,0.2,0.8632401642601669,36.295110364216995,0.924139861752391,0.00023468696225735632 57 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.36228200793266296,0.4,0.7093839776893751,30.27451045093737,0.7817024077477366,0.0009387478490294253 58 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.27194643020629883,0.8,0.5324979881805975,24.253910537657745,0.5544408559123455,0.003754991396117701 59 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.45070987939834595,0.2,0.9276388055709923,36.295110364216995,0.9754365727316975,0.00023468696225735632 60 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.4004955589771271,0.4,0.8242890581008917,30.27451045093737,0.9179956647158725,0.0009387478490294253 61 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.30118998885154724,0.8,0.6199010367404322,24.253910537657745,0.7861440022321221,0.003754991396117701 62 | -------------------------------------------------------------------------------- /test_results/generative-uap/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,train_dataset,test_dataset,clear,attacked,amplitude,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.49720802903175354,0.2,0.9992848360750246,42.54624984068154,0.9810851152142263,5.563844912970703e-05 3 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.49570852518081665,0.4,0.9962711448786147,36.525649927401915,0.9309707827981976,0.00022255379651882814 4 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4991096258163452,0.8,1.0031066505273616,30.505050014122293,0.7921568801287836,0.0008902151860753126 5 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5599467754364014,0.2,0.984974815082914,42.54624984068154,0.9933637097498211,5.563844912970703e-05 6 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5379979014396667,0.4,0.9463656310415152,36.525649927401915,0.9748422278694115,0.00022255379651882814 7 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.47047290205955505,0.8,0.8275857278514955,30.505050014122293,0.9151497729368115,0.0008902151860753126 8 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.3292046785354614,0.2,0.9808015948149336,42.54624984068154,0.9823976074127185,5.5638449129707036e-05 9 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.31604817509651184,0.4,0.9416043403514922,36.525649927401915,0.935020225074371,0.00022255379651882812 10 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.2961620092391968,0.8,0.8823573597970894,30.505050014122293,0.7993477155088614,0.0008902151860753125 11 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.4934127926826477,0.2,0.9661510145463734,42.54624984068154,0.9790179083539794,5.5638449129707036e-05 12 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.46466487646102905,0.4,0.9098597532829567,36.525649927401915,0.9247920606254544,0.00022255379651882814 13 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.40070101618766785,0.8,0.7846121929969427,30.505050014122293,0.7826558180385988,0.0008902151860753126 14 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.48063457012176514,0.2,0.9892289894755696,42.54624984068154,0.9936513707493192,5.563844912970703e-05 15 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.4651477038860321,0.4,0.9573543429376888,36.525649927401915,0.9759066023432932,0.00022255379651882812 16 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.42865070700645447,0.8,0.882237217398985,30.505050014122293,0.9189178362848471,0.0008902151860753126 17 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.5025396943092346,0.2,1.0100003755509053,42.354344452070414,0.9802704677157571,5.815212036864112e-05 18 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.48961111903190613,0.4,0.984016625345086,36.333744538790796,0.9282446686131878,0.0002326084814745645 19 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.44125792384147644,0.8,0.8868367490995912,30.31314462551117,0.786155577298711,0.000930433925898258 20 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5644716024398804,0.2,0.992934215576935,42.354344452070414,0.9929390921812966,5.815212036864112e-05 21 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5392996668815613,0.4,0.948655502564415,36.333744538790796,0.9734440774457785,0.0002326084814745645 22 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.47009900212287903,0.8,0.826928018874245,30.31314462551117,0.9115813830572197,0.000930433925898258 23 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.32587534189224243,0.2,0.9708824809558235,42.354344452070414,0.9817110201263931,5.815212036864112e-05 24 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.31223711371421814,0.4,0.9302500209101036,36.333744538790796,0.9327982321211117,0.0002326084814745645 25 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.3124200105667114,0.8,0.9307949266672456,30.31314462551117,0.7945039933981596,0.000930433925898258 26 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.49191683530807495,0.2,0.963221782153951,42.354344452070414,0.9781487556674229,5.815212036864112e-05 27 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.44877707958221436,0.4,0.8787498767231766,36.333744538790796,0.9219336992205197,0.0002326084814745645 28 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.38331443071365356,0.8,0.7505675402349291,30.31314462551117,0.7762166382697108,0.0009304339258982581 29 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.472390353679657,0.2,0.972260967599868,42.354344452070414,0.9933395791612359,5.815212036864112e-05 30 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.44559600949287415,0.4,0.9171135777297703,36.333744538790796,0.9748202425116202,0.0002326084814745645 31 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.40230095386505127,0.8,0.8280048727168341,30.31314462551117,0.9159807871664518,0.000930433925898258 32 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.49720802903175354,0.2,0.9992848360750246,42.54624984068154,0.9810851152142263,5.563844912970703e-05 33 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.49570852518081665,0.4,0.9962711448786147,36.525649927401915,0.9309707827981976,0.00022255379651882814 34 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4991096258163452,0.8,1.0031066505273616,30.505050014122293,0.7921568801287836,0.0008902151860753126 35 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5599467754364014,0.2,0.984974815082914,42.54624984068154,0.9933637097498211,5.563844912970703e-05 36 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5379979014396667,0.4,0.9463656310415152,36.525649927401915,0.9748422278694115,0.00022255379651882814 37 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.47047290205955505,0.8,0.8275857278514955,30.505050014122293,0.9151497729368115,0.0008902151860753126 38 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.3292046785354614,0.2,0.9808015948149336,42.54624984068154,0.9823976074127185,5.5638449129707036e-05 39 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.31604817509651184,0.4,0.9416043403514922,36.525649927401915,0.935020225074371,0.00022255379651882812 40 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.2961620092391968,0.8,0.8823573597970894,30.505050014122293,0.7993477155088614,0.0008902151860753125 41 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.4934127926826477,0.2,0.9661510145463734,42.54624984068154,0.9790179083539794,5.5638449129707036e-05 42 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.46466487646102905,0.4,0.9098597532829567,36.525649927401915,0.9247920606254544,0.00022255379651882814 43 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.40070101618766785,0.8,0.7846121929969427,30.505050014122293,0.7826558180385988,0.0008902151860753126 44 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.48063457012176514,0.2,0.9892289894755696,42.54624984068154,0.9936513707493192,5.563844912970703e-05 45 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.4651477038860321,0.4,0.9573543429376888,36.525649927401915,0.9759066023432932,0.00022255379651882812 46 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.42865070700645447,0.8,0.882237217398985,30.505050014122293,0.9189178362848471,0.0008902151860753126 47 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.5025396943092346,0.2,1.0100003755509053,42.354344452070414,0.9802704677157571,5.815212036864112e-05 48 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.48961111903190613,0.4,0.984016625345086,36.333744538790796,0.9282446686131878,0.0002326084814745645 49 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.44125792384147644,0.8,0.8868367490995912,30.31314462551117,0.786155577298711,0.000930433925898258 50 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5644716024398804,0.2,0.992934215576935,42.354344452070414,0.9929390921812966,5.815212036864112e-05 51 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5392996668815613,0.4,0.948655502564415,36.333744538790796,0.9734440774457785,0.0002326084814745645 52 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.47009900212287903,0.8,0.826928018874245,30.31314462551117,0.9115813830572197,0.000930433925898258 53 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.32587534189224243,0.2,0.9708824809558235,42.354344452070414,0.9817110201263931,5.815212036864112e-05 54 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.31223711371421814,0.4,0.9302500209101036,36.333744538790796,0.9327982321211117,0.0002326084814745645 55 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.3124200105667114,0.8,0.9307949266672456,30.31314462551117,0.7945039933981596,0.000930433925898258 56 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.49191683530807495,0.2,0.963221782153951,42.354344452070414,0.9781487556674229,5.815212036864112e-05 57 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.44877707958221436,0.4,0.8787498767231766,36.333744538790796,0.9219336992205197,0.0002326084814745645 58 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.38331443071365356,0.8,0.7505675402349291,30.31314462551117,0.7762166382697108,0.0009304339258982581 59 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.472390353679657,0.2,0.972260967599868,42.354344452070414,0.9933395791612359,5.815212036864112e-05 60 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.44559600949287415,0.4,0.9171135777297703,36.333744538790796,0.9748202425116202,0.0002326084814745645 61 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.40230095386505127,0.8,0.8280048727168341,30.31314462551117,0.9159807871664518,0.000930433925898258 62 | -------------------------------------------------------------------------------- /test_results/ifgsm/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,test_dataset,clear,attacked,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.199542760848999,2.410831726660275,34.78487402823948,0.9287117,0.00033228642339447974 3 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.1964014768600464,2.1045309574588216,33.720211725490394,0.9630051,0.0004245988635695555 4 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.5516899824142456,4.6229598442548,34.91094524806152,0.9353847,0.0003227791511881055 5 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.8499687910079956,3.6224217345355427,35.39909888729071,0.93103224,0.00028846299696472114 6 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,1.066835641860962,2.1957320790871875,32.11971773289229,0.95643073,0.0006138018975565548 7 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.199542760848999,2.410831726660275,34.78487402823948,0.9287117,0.00033228642339447974 8 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.2124444246292114,2.132751317331423,33.684346730221215,0.9628795,0.00042811981292148585 9 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.4660953283309937,4.367947146361313,34.88045644979514,0.9353938,0.00032505313203474317 10 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.8499687910079956,3.6224217345355427,35.39909888729071,0.93103224,0.00028846299696472114 11 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,1.0656147003173828,2.1932191704359068,32.113074908693676,0.9561958,0.0006147414669431926 12 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.199542760848999,2.410831726660275,34.78487402823948,0.9287117,0.00033228642339447974 13 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.19640052318573,2.104529279897266,33.72025395310533,0.96300644,0.0004245947351012252 14 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.38870370388031,4.137373786881027,34.73148767808355,0.9336245,0.000336396316897439 15 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.8540611267089844,3.6304349323042464,35.39750815839977,0.9309872,0.00028856867421579366 16 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,1.0668352842330933,2.195731343027243,32.119716110066314,0.9564306,0.0006138021269156397 17 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.199542760848999,2.410831726660275,34.78487402823948,0.9287117,0.00033228642339447974 18 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.2121866941452026,2.13229795632112,33.68391399103065,0.9628894,0.00042816247371027657 19 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.5688945055007935,4.674217389428171,34.86946227689546,0.9349179,0.00032587704707524244 20 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.8499687910079956,3.6224217345355427,35.39909888729071,0.93103224,0.00028846299696472114 21 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,1.066835641860962,2.1957320790871875,32.11971773289229,0.95643073,0.0006138018975565548 22 | -------------------------------------------------------------------------------- /test_results/korhonen-et-al/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,test_dataset,clear,attacked,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.128896713256836,2.2688478487551684,33.453810709693684,0.9123718,0.0004514596380241752 3 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.1037675142288208,1.941583112742726,34.34191664885873,0.969666,0.00036796654507484817 4 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.063884973526001,3.1696392754076403,33.7690165375542,0.92845917,0.0004198540494268176 5 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.1306735277175903,2.2139705174358655,34.26320083212311,0.93229574,0.0003746967418979378 6 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9692351818084717,1.9948534688665045,32.697465298720275,0.9657219,0.0005373453192315832 7 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.1290019750595093,2.2690594031088547,33.45419798424611,0.9123871,0.0004514193816909051 8 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.1037405729293823,1.9415357216287907,34.34205740638562,0.96966845,0.00036795461924486614 9 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.062071681022644,3.1642369215071113,33.7688827855049,0.92846173,0.0004198669801009584 10 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.1303596496582031,2.213355913173434,34.263675207393064,0.932313,0.0003746558164042237 11 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9692368507385254,1.9948569038129131,32.697462035467204,0.96572137,0.0005373457229885943 12 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.1291472911834717,2.2693514583264576,33.45400741823339,0.9123799,0.00045143919015788205 13 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.1037538051605225,1.9415589977953696,34.342016178815214,0.96966743,0.0003679581122542428 14 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0636675357818604,3.168991461752091,33.76925227709591,0.92845696,0.0004198312599327983 15 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.130079984664917,2.212808301475879,34.263959581360496,0.9323136,0.0003746312849221814 16 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9692307114601135,1.9948442681171954,32.69746197670587,0.9657212,0.0005373457302590416 17 | -------------------------------------------------------------------------------- /test_results/madc/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,test_dataset,clear,attacked,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,TEST,0.4975638687610626,0.6986206769943237,1.4040824120406772,55.67663380170349,0.99888724,2.706055003328163e-06 3 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,0.7519692182540894,1.3227520439252718,56.138953934913765,0.99958706,2.4327899133938665e-06 4 | 0aebe24fc257286e.png,TEST,0.3356485962867737,0.5885117650032043,1.7533568485428366,55.53760849502933,0.9989169,2.7940820191020524e-06 5 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,0.7541601657867432,1.4767201420615854,54.90249775139518,0.99881476,3.2340760249868115e-06 6 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.6200047731399536,1.2760769477066212,55.67713132678806,0.9996087,2.7057450170893174e-06 7 | 000b7d55b6184b08.png,TEST,0.4975638687610626,0.6986206769943237,1.4040824120406772,55.67663380170349,0.99888724,2.706055003328163e-06 8 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,0.7519726753234863,1.3227581250859095,56.13879419911113,0.99958706,2.4328793943364665e-06 9 | 0aebe24fc257286e.png,TEST,0.3356485962867737,0.5853063464164734,1.7438069245384105,55.63346551248764,0.99892926,2.733086951650008e-06 10 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,0.7541601657867432,1.4767201420615854,54.90249775139518,0.99881476,3.2340760249868115e-06 11 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.6200047135353088,1.2760768250299637,55.67710738914166,0.9996087,2.705759930782367e-06 12 | 000b7d55b6184b08.png,TEST,0.4975638687610626,0.6985185742378235,1.4038772067134606,55.67725977624149,0.99888736,2.705664991589708e-06 13 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,0.7519692182540894,1.3227520439252718,56.138953934913765,0.99958706,2.4327899133938665e-06 14 | 0aebe24fc257286e.png,TEST,0.3356485962867737,0.5868899822235107,1.748525060781365,55.60023051324703,0.9989328,2.754082519267427e-06 15 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,0.7541601657867432,1.4767201420615854,54.90249775139518,0.99881476,3.2340760249868115e-06 16 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.6200047135353088,1.2760768250299637,55.67710738914166,0.9996087,2.705759930782367e-06 17 | -------------------------------------------------------------------------------- /test_results/mifgsm/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,test_dataset,clear,attacked,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.106445074081421,2.223724718670744,30.232735185481108,0.81071943,0.0009478213376618465 3 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.0894734859466553,1.9164391910672363,29.77551504647067,0.9068682,0.0010530487946316988 4 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0643751621246338,3.17109969742059,30.171316787153852,0.81235677,0.0009613207603458785 5 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0811896324157715,2.1170761596920675,30.13154056612853,0.7939778,0.0009701657607926908 6 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9565118551254272,1.9686666642131334,29.417064079915352,0.90539426,0.001143651205275685 7 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.106445074081421,2.223724718670744,30.232735185481108,0.81071943,0.0009478213376618465 8 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.0894734859466553,1.9164391910672363,29.77551504647067,0.9068682,0.0010530487946316988 9 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0643751621246338,3.17109969742059,30.171316787153852,0.81235677,0.0009613207603458785 10 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0811550617218018,2.117008466856479,30.112300317872638,0.79229784,0.0009744733538773159 11 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9565118551254272,1.9686666642131334,29.417064079915352,0.90539426,0.001143651205275685 12 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.106445074081421,2.223724718670744,30.232735185481108,0.81071943,0.0009478213376618465 13 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.089492678642273,1.9164729519935355,29.77522182288258,0.9068621,0.0010531198959657108 14 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0643751621246338,3.17109969742059,30.171316787153852,0.81235677,0.0009613207603458785 15 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0808717012405396,2.116453619028223,30.13456173012498,0.7940981,0.0009694911009151277 16 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9565118551254272,1.9686666642131334,29.417064079915352,0.90539426,0.001143651205275685 17 | 000b7d55b6184b08.png,TEST,0.4975638687610626,1.106445074081421,2.223724718670744,30.232735185481108,0.81071943,0.0009478213376618465 18 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,1.089492678642273,1.9164729519935355,29.77522182288258,0.9068621,0.0010531198959657108 19 | 0aebe24fc257286e.png,TEST,0.3356485962867737,1.0584819316864014,3.153541958453622,30.18029205373336,0.8128299,0.0009593361160484796 20 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,1.0732100009918213,2.101451252696772,30.11714639599395,0.79322815,0.0009733865935236424 21 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.9565118551254272,1.9686666642131334,29.417064079915352,0.90539426,0.001143651205275685 22 | -------------------------------------------------------------------------------- /test_results/std-fgsm/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,test_dataset,clear,attacked,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,TEST,0.4975638687610626,0.8044134378433228,1.6167038813455599,28.724052798230154,0.75125366,0.0013415124890753234 3 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,0.7775722742080688,1.3677891196083565,28.18005472791362,0.8708213,0.001520528368570426 4 | 0aebe24fc257286e.png,TEST,0.3356485962867737,0.6357231140136719,1.8940139212455354,28.398665086444666,0.73543984,0.001445884131381423 5 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,0.6657206416130066,1.3035468128053838,28.355143464934997,0.7200904,0.001460446504222719 6 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.7472332119941711,1.5379350574310773,28.15332662932026,0.8752274,0.0015299151211541292 7 | 000b7d55b6184b08.png,TEST,0.4975638687610626,0.8044134378433228,1.6167038813455599,28.724052798230154,0.75125366,0.0013415124890753234 8 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,0.7775722742080688,1.3677891196083565,28.18005472791362,0.8708213,0.001520528368570426 9 | 0aebe24fc257286e.png,TEST,0.3356485962867737,0.6357231140136719,1.8940139212455354,28.398665086444666,0.73543984,0.001445884131381423 10 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,0.6657206416130066,1.3035468128053838,28.355143464934997,0.7200904,0.001460446504222719 11 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.7472332119941711,1.5379350574310773,28.15332662932026,0.8752274,0.0015299151211541292 12 | 000b7d55b6184b08.png,TEST,0.4975638687610626,0.8044134378433228,1.6167038813455599,28.724052798230154,0.75125366,0.0013415124890753234 13 | 00c3cd597f1ee96f.png,TEST,0.5684884190559387,0.7775722742080688,1.3677891196083565,28.18005472791362,0.8708213,0.001520528368570426 14 | 0aebe24fc257286e.png,TEST,0.3356485962867737,0.6357231140136719,1.8940139212455354,28.398665086444666,0.73543984,0.001445884131381423 15 | 0af0a5dfee6b84ff.png,TEST,0.510699450969696,0.6657206416130066,1.3035468128053838,28.355143464934997,0.7200904,0.001460446504222719 16 | 0b1d45bd9ab1064e.png,TEST,0.48586785793304443,0.7472332119941711,1.5379350574310773,28.15332662932026,0.8752274,0.0015299151211541292 17 | -------------------------------------------------------------------------------- /test_results/uap/maniqa.csv: -------------------------------------------------------------------------------- 1 | image_name,train_dataset,test_dataset,clear,attacked,amplitude,rel_gain,psnr,ssim,mse 2 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4914886951446533,0.2,0.9877901632376633,36.5418698398599,0.9313803062879876,0.00022172415885737552 3 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4407700002193451,0.4,0.885856123992414,30.521269926580274,0.7933819215665067,0.0008868966354295021 4 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.33427882194519043,0.8,0.6718309807694577,24.500670013300653,0.5546279232215959,0.0035475865417180074 5 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5374159216880798,0.2,0.9453418991024312,36.5418698398599,0.9752983864972218,0.00022172415885737552 6 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.4688839316368103,0.4,0.8247906481814761,30.521269926580274,0.9163702648148123,0.0008868966354295018 7 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.35927659273147583,0.8,0.6319857726004501,24.500670013300653,0.7734411319330426,0.0035475865417180074 8 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.31037864089012146,0.2,0.9247130609923305,36.5418698398599,0.936078419017051,0.00022172415885737546 9 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.2620271146297455,0.4,0.780659050949443,30.521269926580274,0.8031118447675586,0.0008868966354295018 10 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.20150156319141388,0.8,0.6003348901815565,24.500670013300653,0.5648118168509279,0.0035475865417180074 11 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.45112672448158264,0.2,0.8833507136633904,36.5418698398599,0.9265595538720954,0.00022172415885737552 12 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.37092381715774536,0.4,0.7263054942656582,30.521269926580274,0.7873521867007472,0.0008868966354295021 13 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.2915290296077728,0.8,0.5708426532557044,24.50067001330065,0.5624771838909064,0.0035475865417180083 14 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.46526920795440674,0.2,0.9576044193039082,36.5418698398599,0.9764901303887209,0.00022172415885737552 15 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.424939900636673,0.4,0.8745997367358931,30.521269926580274,0.9208151916379806,0.0008868966354295021 16 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.343079149723053,0.8,0.7061161674340092,24.500670013300653,0.7907642365527786,0.0035475865417180074 17 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.48370397090911865,0.2,0.9721444849151624,36.85398126427166,0.9354851720787959,0.0002063487648145328 18 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.4233352541923523,0.4,0.8508159068029998,30.833381350992042,0.8037633726607808,0.0008253950592581312 19 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.3137436807155609,0.8,0.6305596133754342,24.812781437712417,0.567957052476208,0.0033015802370325244 20 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5313611626625061,0.2,0.9346912704834198,36.85398126427166,0.976770461069724,0.0002063487648145328 21 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.46519967913627625,0.4,0.8183098609270016,30.833381350992042,0.9208608571835897,0.0008253950592581312 22 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.35305917263031006,0.8,0.6210490148886734,24.812781437712417,0.7832531869226166,0.0033015802370325244 23 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.31319335103034973,0.2,0.9330989448344408,36.85398126427167,0.9396692230380709,0.00020634876481453277 24 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.26470744609832764,0.4,0.7886445795595258,30.833381350992042,0.811979302089752,0.0008253950592581311 25 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.2025275081396103,0.8,0.6033914944979346,24.812781437712417,0.5786238659830758,0.003301580237032524 26 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.4436640441417694,0.2,0.8687380479837163,36.85398126427166,0.9298944355982379,0.0002063487648145328 27 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.3628431558609009,0.4,0.7104827607939436,30.833381350992042,0.7945968518586973,0.0008253950592581312 28 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.28906500339508057,0.8,0.5660178463991205,24.812781437712417,0.5722146215908079,0.003301580237032525 29 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.4596312642097473,0.2,0.9460005569520248,36.85398126427166,0.9777695357480921,0.0002063487648145328 30 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.41623377799987793,0.4,0.8566810321032545,30.833381350992042,0.9247886755856124,0.0008253950592581312 31 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.3380816578865051,0.8,0.6958304657664653,24.812781437712417,0.7994405567381188,0.003301580237032525 32 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4914886951446533,0.2,0.9877901632376633,36.5418698398599,0.9313803062879876,0.00022172415885737552 33 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.4407700002193451,0.4,0.885856123992414,30.521269926580274,0.7933819215665067,0.0008868966354295021 34 | 000b7d55b6184b08.png,COCO,TEST,0.4975638687610626,0.33427882194519043,0.8,0.6718309807694577,24.500670013300653,0.5546279232215959,0.0035475865417180074 35 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.5374159216880798,0.2,0.9453418991024312,36.5418698398599,0.9752983864972218,0.00022172415885737552 36 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.4688839316368103,0.4,0.8247906481814761,30.521269926580274,0.9163702648148123,0.0008868966354295018 37 | 00c3cd597f1ee96f.png,COCO,TEST,0.5684884190559387,0.35927659273147583,0.8,0.6319857726004501,24.500670013300653,0.7734411319330426,0.0035475865417180074 38 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.31037864089012146,0.2,0.9247130609923305,36.5418698398599,0.936078419017051,0.00022172415885737546 39 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.2620271146297455,0.4,0.780659050949443,30.521269926580274,0.8031118447675586,0.0008868966354295018 40 | 0aebe24fc257286e.png,COCO,TEST,0.3356485962867737,0.20150156319141388,0.8,0.6003348901815565,24.500670013300653,0.5648118168509279,0.0035475865417180074 41 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.45112672448158264,0.2,0.8833507136633904,36.5418698398599,0.9265595538720954,0.00022172415885737552 42 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.37092381715774536,0.4,0.7263054942656582,30.521269926580274,0.7873521867007472,0.0008868966354295021 43 | 0af0a5dfee6b84ff.png,COCO,TEST,0.510699450969696,0.2915290296077728,0.8,0.5708426532557044,24.50067001330065,0.5624771838909064,0.0035475865417180083 44 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.46526920795440674,0.2,0.9576044193039082,36.5418698398599,0.9764901303887209,0.00022172415885737552 45 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.424939900636673,0.4,0.8745997367358931,30.521269926580274,0.9208151916379806,0.0008868966354295021 46 | 0b1d45bd9ab1064e.png,COCO,TEST,0.48586785793304443,0.343079149723053,0.8,0.7061161674340092,24.500670013300653,0.7907642365527786,0.0035475865417180074 47 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.48370397090911865,0.2,0.9721444849151624,36.85398126427166,0.9354851720787959,0.0002063487648145328 48 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.4233352541923523,0.4,0.8508159068029998,30.833381350992042,0.8037633726607808,0.0008253950592581312 49 | 000b7d55b6184b08.png,VOC2012,TEST,0.4975638687610626,0.3137436807155609,0.8,0.6305596133754342,24.812781437712417,0.567957052476208,0.0033015802370325244 50 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.5313611626625061,0.2,0.9346912704834198,36.85398126427166,0.976770461069724,0.0002063487648145328 51 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.46519967913627625,0.4,0.8183098609270016,30.833381350992042,0.9208608571835897,0.0008253950592581312 52 | 00c3cd597f1ee96f.png,VOC2012,TEST,0.5684884190559387,0.35305917263031006,0.8,0.6210490148886734,24.812781437712417,0.7832531869226166,0.0033015802370325244 53 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.31319335103034973,0.2,0.9330989448344408,36.85398126427167,0.9396692230380709,0.00020634876481453277 54 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.26470744609832764,0.4,0.7886445795595258,30.833381350992042,0.811979302089752,0.0008253950592581311 55 | 0aebe24fc257286e.png,VOC2012,TEST,0.3356485962867737,0.2025275081396103,0.8,0.6033914944979346,24.812781437712417,0.5786238659830758,0.003301580237032524 56 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.4436640441417694,0.2,0.8687380479837163,36.85398126427166,0.9298944355982379,0.0002063487648145328 57 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.3628431558609009,0.4,0.7104827607939436,30.833381350992042,0.7945968518586973,0.0008253950592581312 58 | 0af0a5dfee6b84ff.png,VOC2012,TEST,0.510699450969696,0.28906500339508057,0.8,0.5660178463991205,24.812781437712417,0.5722146215908079,0.003301580237032525 59 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.4596312642097473,0.2,0.9460005569520248,36.85398126427166,0.9777695357480921,0.0002063487648145328 60 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.41623377799987793,0.4,0.8566810321032545,30.833381350992042,0.9247886755856124,0.0008253950592581312 61 | 0b1d45bd9ab1064e.png,VOC2012,TEST,0.48586785793304443,0.3380816578865051,0.8,0.6958304657664653,24.812781437712417,0.7994405567381188,0.003301580237032525 62 | --------------------------------------------------------------------------------