├── .gitignore ├── README.md ├── datasets └── hosting_blacklists.txt ├── experiments ├── ablation_study │ ├── adapt_to_cryptocurrency_phishing.py │ ├── cost_benchmarking.py │ ├── domain_alias.py │ ├── test_on_middle_ranked_benign.py │ └── test_on_public_phishing.py ├── componentwise_evaluation │ ├── brand_recognition.py │ ├── brand_recognition_adversary.py │ ├── crp_prediction.py │ ├── crp_prediction_adversary.py │ ├── crp_transition.py │ ├── crp_transition_adversary.py │ └── torchattacks │ │ ├── README.md │ │ ├── __init__.py │ │ ├── attack.py │ │ ├── attacks │ │ ├── __init__.py │ │ ├── _differential_evolution.py │ │ ├── apgd.py │ │ ├── apgdt.py │ │ ├── autoattack.py │ │ ├── bim.py │ │ ├── cw.py │ │ ├── deepfool.py │ │ ├── difgsm.py │ │ ├── eaden.py │ │ ├── eadl1.py │ │ ├── eotpgd.py │ │ ├── fab.py │ │ ├── ffgsm.py │ │ ├── fgsm.py │ │ ├── gn.py │ │ ├── jitter.py │ │ ├── jsma.py │ │ ├── mifgsm.py │ │ ├── nifgsm.py │ │ ├── onepixel.py │ │ ├── pgd.py │ │ ├── pgdl2.py │ │ ├── pgdrs.py │ │ ├── pgdrsl2.py │ │ ├── pifgsm.py │ │ ├── pifgsmplusplus.py │ │ ├── pixle.py │ │ ├── protect.py │ │ ├── rfgsm.py │ │ ├── sinifgsm.py │ │ ├── sparsefool.py │ │ ├── spsa.py │ │ ├── square.py │ │ ├── tifgsm.py │ │ ├── tpgd.py │ │ ├── upgd.py │ │ ├── vanila.py │ │ ├── vmifgsm.py │ │ └── vnifgsm.py │ │ └── wrappers │ │ ├── __init__.py │ │ ├── lgv.py │ │ └── multiattack.py └── field_study │ ├── draw_utils.py │ ├── fonts │ ├── arabic.ttf │ ├── arialbd.ttf │ ├── chinese_cht.ttf │ ├── cyrillic.ttf │ ├── french.ttf │ ├── german.ttf │ ├── hindi.ttf │ ├── japan.ttc │ ├── kannada.ttf │ ├── korean.ttf │ ├── latin.ttf │ ├── marathi.ttf │ ├── nepali.ttf │ ├── persian.ttf │ ├── simfang.ttf │ ├── spanish.ttf │ ├── tamil.ttf │ ├── telugu.ttf │ ├── urdu.ttf │ └── uyghur.ttf │ ├── monitor_url.py │ ├── plots │ ├── brand_freq.png │ ├── brand_sector.png │ ├── campaign.png │ ├── domain_age.png │ ├── geo.png │ ├── num_phish.png │ └── phishllm_cost.png │ ├── results_statistics.py │ ├── tele │ ├── gsheets.py │ └── tele.py │ ├── test.py │ └── test_baseline.py ├── figures ├── phishllm.pdf └── phishllm.png ├── param_dict.yaml ├── prompts ├── brand_recog_prompt.json └── crp_pred_prompt.json ├── requirements.txt ├── scripts ├── data │ ├── data_utils.py │ └── dom_utils.py ├── infer │ └── test.py ├── pipeline │ └── test_llm.py ├── train │ └── train.py └── utils │ ├── PhishIntentionWrapper.py │ ├── dynaphish │ ├── brand_knowledge_utils.py │ ├── configs.yaml │ ├── google_safebrowsing.py │ └── utils.py │ ├── logger_utils.py │ ├── utils.py │ └── web_utils │ ├── collect_html_screenshots_benign.py │ ├── download_github_phishing_feed.sh │ ├── web_utils.py │ └── web_utils_scripts.js ├── server ├── .gitignore ├── README.md ├── __init__.py ├── announcer.py ├── server.py ├── static │ ├── facebookchatone.mp3 │ └── index.js └── templates │ └── index.html └── setup.sh /.gitignore: -------------------------------------------------------------------------------- 1 | ./datasets/ 2 | ./field_study/tele/cred.json 3 | ./checkpoints/ 4 | ./crawling_scripts/ 5 | /__pycache__/ 6 | debug.py 7 | *.zip 8 | *.pkl 9 | *.npy 10 | *.pt 11 | expand_targetlist/ 12 | *baseline*.py 13 | *adv.py 14 | *_benign*.py 15 | *dynaphish*.py 16 | -------------------------------------------------------------------------------- /datasets/hosting_blacklists.txt: -------------------------------------------------------------------------------- 1 | lws.fr 2 | ionos.com 3 | escrow.com 4 | networksolutions.com 5 | zone.ee 6 | domain.com 7 | virualmin.com 8 | reg.ru 9 | epik.com 10 | dan.com 11 | phpmyadmin.net 12 | phpbb.com 13 | php.net 14 | api.platform 15 | tumblr.com 16 | bigbluebutton.org 17 | nginx.com 18 | tautulli.com 19 | caddyserver.com 20 | rebrandly.com 21 | keycloak.org 22 | zabbix.com 23 | djangoproject.com 24 | control-webpanel.com 25 | appwrite.io 26 | afihost.co.za 27 | afrihost.com 28 | primehost.ca 29 | parallels.com 30 | zyro.com 31 | hostinger.com 32 | diviproject.org 33 | ispmanager.com 34 | ovhcloud.com 35 | directadmin.com 36 | ispconfig.org 37 | zoner.com 38 | nextcloud.com 39 | plesk.com 40 | fornex.com 41 | alfahosting.de 42 | dhosting.pl 43 | drupal.org 44 | ininet.hu 45 | joomla.org 46 | swagger.io 47 | roundcube.net 48 | vistaprint.com 49 | vtiger.com 50 | webmin.com 51 | e-shot.net 52 | mailchimp.com 53 | portainer.io 54 | storybook.js.org 55 | redmine.org 56 | freshrss.org 57 | namecheap.com 58 | debian.org 59 | strapi.io 60 | traefik.io 61 | bitwarden.com 62 | wordpress.org 63 | hostnet.nl 64 | nuxtjs.org 65 | horde.org 66 | hoststar.ch 67 | cloudns.com 68 | domaindiscount24.com 69 | raiolanetworks.es 70 | netsite.dk 71 | webgo.de 72 | hostingwedos.cz 73 | beget.ru 74 | register.com 75 | blogger.com 76 | hostpoint.ch 77 | 1jabber.com 78 | strato.de 79 | zimbra.com 80 | mailrelay.com 81 | onlyoffice.org 82 | onlyoffice.com 83 | yunohost.org 84 | vodien.com 85 | myjobous 86 | domainname.de 87 | laravel.com 88 | autods.com 89 | codingest.net 90 | vhlcentral.com 91 | windsorbrokers.com 92 | logmein.com 93 | xng88.xyz 94 | cpalead.com 95 | profreehost.com 96 | mekari.com 97 | rediffmailpro.com 98 | wix.com 99 | whmcs.com 100 | draytek.com 101 | zyxel.com 102 | adminlte.io 103 | emby.media 104 | invoiceninja.com 105 | jxt.com.au 106 | mantisbt.org 107 | seafile.com 108 | bookstackapp.com 109 | apache.org 110 | liveconfig 111 | firezone.com 112 | wordpress.com 113 | dolibarr.org 114 | cpanel.net 115 | godaddy.com 116 | zimbra.com 117 | owncloud.com 118 | phpmyadmin.net 119 | wordpress.com 120 | draytek.com 121 | afrihost.co.za 122 | 15.vip 123 | okta.com 124 | prohoster.info 125 | matomo.org 126 | hostneel.com 127 | o2switch.fr 128 | froxlor.com 129 | contabo.com 130 | freshrss.org 131 | harmonweb.com 132 | modoboa.org 133 | antagonist.nl 134 | oderland.com 135 | strato 136 | luno 137 | luno.com 138 | bet365 139 | bet365.com 140 | manhattantrust.com 141 | nginxproxymanager.com 142 | bitrix24.com 143 | owncloud.org 144 | websitehosting.ca 145 | fiberhost.pl 146 | smartlife.com 147 | freescout.net 148 | froxlor.com 149 | gestsup.com 150 | cloudflare.com 151 | sonicpanel.com 152 | material.io 153 | invoices.com 154 | inmotionhosting.com 155 | radarr.video 156 | eshop.it 157 | horde.net 158 | moodle.org 159 | mautic.org 160 | pritunl.com 161 | chatwoot.com 162 | sentry.io 163 | pfsense.org 164 | bluehost.com 165 | siteground.com 166 | a2hosting.com 167 | digitalocean.com 168 | linode.com 169 | heroku.com 170 | vultr.com 171 | openshift.com 172 | kubernetes.io 173 | jitsi.org 174 | opnsense.org 175 | untangle.com 176 | smoothwall.org 177 | snort.org 178 | ossec.net 179 | sugarcrm.com 180 | suitecrm.com 181 | zoho.com 182 | zendesk.com 183 | hubspot.com 184 | elastic.co 185 | splunk.com 186 | grafana.com 187 | prometheus.io 188 | influxdata.com 189 | wireshark.org 190 | gimp.org 191 | ansible.com 192 | puppet.com 193 | chef.io 194 | terraform.io 195 | fastly.com 196 | akamai.com 197 | squarespace.com 198 | weebly.com 199 | shopify.com 200 | magento.com 201 | oscommerce.com 202 | prestashop.com 203 | w3schools.com 204 | mozilla.org 205 | sqlite.org 206 | postgresql.org 207 | mysql.com 208 | mariadb.org 209 | redis.io 210 | mongodb.com 211 | hostgator.com 212 | greengeeks.com 213 | midphase.com 214 | westhost.com 215 | lunarpages.com 216 | arvixe.com 217 | liquidweb.com 218 | kinsta.com 219 | wpengine.com 220 | pantheon.io 221 | enom.com 222 | nic.ru 223 | 101domain.com 224 | namesilo.com 225 | eurodns.com 226 | whois.com 227 | wireguard.com 228 | openvpn.net 229 | strongswan.org 230 | freeswan.org 231 | softether.org 232 | endian.com 233 | ipfire.org 234 | clearos.com 235 | sophos.com 236 | shorewall.org 237 | odoo.com 238 | erpnext.com 239 | yetiforce.com 240 | espocrm.com 241 | crm.zoho.com 242 | flarepointcrm.com 243 | hubspot.com 244 | pipedrive.com 245 | fuxnoten.de 246 | horde.com 247 | petrosoftinc.com 248 | mikrotik.com 249 | vodia.com 250 | ccc.de 251 | yourlogo.com 252 | wallabag.org 253 | login.gov 254 | zulip.com 255 | keyhelp.de 256 | akaunting.com 257 | plausible.io 258 | mautic.com 259 | postfixadmin.com 260 | password.com 261 | ingresar.com 262 | example.com 263 | meshcentral.com 264 | glpi-project.org 265 | facturascripts.com 266 | keyweb.de 267 | blizzard.com 268 | meiguo.com 269 | ovh.com -------------------------------------------------------------------------------- /experiments/ablation_study/adapt_to_cryptocurrency_phishing.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | import openai 4 | from scripts.utils.web_utils.web_utils import * 5 | from scripts.utils.logger_utils import * 6 | import os 7 | from scripts.utils.PhishIntentionWrapper import PhishIntentionWrapper 8 | import yaml 9 | from scripts.pipeline.test_llm import TestVLM 10 | from tqdm import tqdm 11 | 12 | os.environ['OPENAI_API_KEY'] = open('./datasets/openai_key.txt').read().strip() 13 | os.environ['CURL_CA_BUNDLE'] = '' 14 | os.environ['CUDA_VISIBLE_DEVICES'] = '2' 15 | 16 | 17 | if __name__ == '__main__': 18 | 19 | # load hyperparameters 20 | with open('./param_dict.yaml') as file: 21 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 22 | 23 | # debug_list = './datasets/cryptocurrency_phishing/' 24 | # root_folder = './datasets/public_phishing_feeds' 25 | # result = './datasets/cryptocurrency_phishing.txt' 26 | 27 | debug_list = './datasets/LLM has insufficient information to determine its brand/' 28 | root_folder = './datasets/public_phishing_feeds' 29 | result = './datasets/insufficient_description_phishing_vlm.txt' 30 | 31 | phishintention_cls = PhishIntentionWrapper() 32 | llm_cls = TestVLM(phishintention_cls, 33 | param_dict=param_dict, 34 | proxies={"http": "http://127.0.0.1:7890", 35 | "https": "http://127.0.0.1:7890", 36 | } 37 | ) 38 | openai.api_key = os.getenv("OPENAI_API_KEY") 39 | openai.proxy = "http://127.0.0.1:7890" # proxy 40 | 41 | sleep_time = 3; timeout_time = 60 42 | driver = CustomWebDriver.boot(proxy_server="127.0.0.1:7890") # Using the proxy_url variable 43 | driver.set_script_timeout(timeout_time) 44 | driver.set_page_load_timeout(timeout_time) 45 | 46 | PhishLLMLogger.set_debug_on() 47 | PhishLLMLogger.set_verbose(True) 48 | # PhishLLMLogger.set_logfile("./datasets/cryptocurrency_phishing.log") 49 | 50 | sampled_crypto_phishing = os.listdir(debug_list) 51 | for filename in tqdm(sampled_crypto_phishing): 52 | 53 | pattern = r"(\d{4}-\d{2}-\d{2})_(.+)" 54 | match = re.match(pattern, filename) 55 | date = match.group(1) 56 | file = match.group(2).split('.png')[0] 57 | 58 | target_folder = os.path.join(root_folder, date, file) 59 | 60 | if os.path.exists(result) and target_folder in [x.strip().split('\t')[0] for x in open(result).readlines()]: 61 | continue 62 | 63 | shot_path = os.path.join(target_folder, 'shot.png') 64 | html_path = os.path.join(target_folder, 'index.html') 65 | info_path = os.path.join(target_folder, 'info.txt') 66 | if os.path.exists(info_path): 67 | URL = open(info_path, encoding='utf-8').read() 68 | else: 69 | URL = f'http://{file}' 70 | 71 | if os.path.exists(shot_path): 72 | logo_box, reference_logo = llm_cls.detect_logo(shot_path) 73 | PhishLLMLogger.spit(URL) 74 | pred, brand, brand_recog_time, crp_prediction_time, crp_transition_time, _ = llm_cls.test(URL, 75 | reference_logo, 76 | logo_box, 77 | shot_path, 78 | html_path, 79 | driver, 80 | ) 81 | with open(result, 'a+') as f: 82 | f.write(target_folder+'\t'+str(pred)+'\t'+str(brand)+'\t'+str(brand_recog_time)+'\t'+str(crp_prediction_time)+'\t'+str(crp_transition_time)+'\n') 83 | else: 84 | shutil.rmtree(target_folder) 85 | 86 | driver.delete_all_cookies() 87 | 88 | driver.quit() 89 | 90 | ct = 0 91 | total_ct = 0 92 | for line in open(result).readlines()[:50]: 93 | if "\tphish\t" in line: 94 | ct += 1 95 | total_ct += 1 96 | 97 | print(total_ct, ct, ct/total_ct) 98 | # Croptocurrency phishing VLM: 36 out of 50 99 | # Insufficient textual information: 11 out of 19, the left FNs switch to the 'non-standard credential' FN reason 100 | -------------------------------------------------------------------------------- /experiments/ablation_study/test_on_middle_ranked_benign.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import time 3 | import os 4 | import openai 5 | from scripts.brand_recognition.dataloader import * 6 | from scripts.utils.web_utils.web_utils import is_valid_domain 7 | from scripts.pipeline.test_llm import TestLLM 8 | import yaml 9 | import Levenshtein as lev 10 | os.environ['OPENAI_API_KEY'] = open('./datasets/openai_key.txt').read().strip() 11 | os.environ['http_proxy'] = "http://127.0.0.1:7890" 12 | os.environ['https_proxy'] = "http://127.0.0.1:7890" 13 | 14 | def test(result_file): 15 | 16 | success = 0 17 | strict_success = 0 18 | total = 0 19 | runtime = [] 20 | 21 | result_lines = open(result_file).readlines() 22 | pbar = tqdm(result_lines, leave=False) 23 | for line in pbar: 24 | data = line.split('\t') 25 | brand, URL, answer, failure_reason, brand_recog_time, brand_validation_time = data 26 | total += 1 27 | 28 | if is_valid_domain(answer): 29 | success += 1 30 | if len(failure_reason) == 0: 31 | strict_success += 1 32 | elif lev.ratio(answer, brand) >= 0.8: 33 | strict_success += 1 34 | 35 | pbar.set_description(f"Completeness (% brand recognized) = {success/total} ", refresh=True) 36 | 37 | print(f"Completeness (% brand recognized) = {success/total} \n " 38 | f"Completeness after brand validation (% brand recognized) = {strict_success/total}") 39 | 40 | if __name__ == '__main__': 41 | openai.api_key = os.getenv("OPENAI_API_KEY") 42 | openai.proxy = "http://127.0.0.1:7890" # proxy 43 | 44 | 45 | with open('./param_dict.yaml') as file: 46 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 47 | 48 | model = "gpt-3.5-turbo-16k" 49 | result_file = './datasets/alexa_middle_5k.txt' 50 | 51 | phishintention_cls = PhishIntentionWrapper() 52 | llm_cls = TestLLM(phishintention_cls, 53 | param_dict=param_dict, 54 | proxies={"http": "http://127.0.0.1:7890", 55 | "https": "http://127.0.0.1:7890", 56 | } 57 | ) 58 | 59 | for brand in tqdm(os.listdir('./datasets/alexa_middle_5k')): 60 | if brand.startswith('.'): 61 | continue 62 | 63 | folder = os.path.join('./datasets/alexa_middle_5k', brand) 64 | if os.path.exists(result_file) and brand in [x.strip().split('\t')[0] for x in open(result_file).readlines()]: 65 | continue 66 | 67 | # if subdomain != 'mymhs.nmfn.com': 68 | # continue 69 | 70 | shot_path = os.path.join(folder, 'shot.png') 71 | html_path = os.path.join(folder, 'index.html') 72 | info_path = os.path.join(folder, 'info.txt') 73 | if os.path.exists(info_path): 74 | URL = open(info_path, encoding='utf-8').read() 75 | else: 76 | URL = f'http://{brand}' 77 | DOMAIN = brand 78 | 79 | if not os.path.exists(shot_path): 80 | shutil.rmtree(folder) 81 | continue 82 | 83 | logo_cropping_time, logo_matching_time, google_search_time, brand_recog_time = 0, 0, 0, 0 84 | 85 | plotvis = Image.open(shot_path) 86 | logo_box, reference_logo = llm_cls.detect_logo(shot_path) 87 | answer = '' 88 | failure_reason = '' 89 | if reference_logo is None: 90 | failure_reason = 'cannot detect logo' 91 | else: 92 | image_width, image_height = plotvis.size 93 | (webpage_text, logo_caption, logo_ocr), (ocr_processing_time, image_caption_processing_time) = \ 94 | llm_cls.preprocessing(shot_path=shot_path, 95 | html_path=html_path, 96 | reference_logo=reference_logo, 97 | logo_box=logo_box, 98 | image_width=image_width, 99 | image_height=image_height, 100 | announcer=None) 101 | if len(logo_caption) == 0 and len(logo_ocr) == 0: 102 | failure_reason = 'no text in logo' 103 | else: 104 | ## Brand recognition model 105 | start_time = time.time() 106 | predicted_domain, *_ = llm_cls.brand_recognition_llm(reference_logo=reference_logo, 107 | webpage_text=webpage_text, 108 | logo_caption=logo_caption, 109 | logo_ocr=logo_ocr, 110 | announcer=None) 111 | brand_recog_time = time.time() - start_time 112 | 113 | ## Domain validation 114 | if predicted_domain and len(predicted_domain) > 0 and is_valid_domain(predicted_domain): 115 | answer = predicted_domain 116 | validation_success, logo_cropping_time, logo_matching_time = llm_cls.brand_validation( 117 | company_domain=predicted_domain, 118 | reference_logo=reference_logo) 119 | if not validation_success: 120 | failure_reason = 'failure in logo matching' 121 | 122 | else: 123 | failure_reason = 'no prediction' 124 | 125 | print(brand, answer, failure_reason) 126 | with open(result_file, 'a+', encoding='utf-8') as f: 127 | f.write(brand + '\t' + URL + '\t' + answer + '\t' + failure_reason + '\t' + str(brand_recog_time) + "\t" + str(logo_cropping_time+logo_matching_time) + '\n') 128 | 129 | test(result_file) 130 | # Completeness (% brand recognized) = 0.8674089068825911 131 | # Completeness after brand validation (% brand recognized) = 0.7064777327935222 -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/brand_recognition.py: -------------------------------------------------------------------------------- 1 | import os 2 | import openai 3 | from scripts.data.data_utils import ShotDataset_Caption 4 | from scripts.pipeline.test_llm import TestLLM 5 | import yaml 6 | from scripts.utils.PhishIntentionWrapper import PhishIntentionWrapper 7 | from tldextract import tldextract 8 | from tqdm import tqdm 9 | import numpy as np 10 | from scripts.utils.web_utils.web_utils import is_valid_domain 11 | 12 | 13 | def list_correct(result_file): 14 | correct = [] 15 | result_lines = open(result_file).readlines() 16 | for line in result_lines: 17 | data = line.strip().split('\t') 18 | url, gt, pred, time = data 19 | if is_valid_domain(pred): # has prediction 20 | correct.append(url) 21 | return correct 22 | 23 | 24 | def test(result_file): 25 | ct = 0 26 | total = 0 27 | runtime = [] 28 | 29 | result_lines = open(result_file).readlines() 30 | pbar = tqdm(result_lines, leave=False) 31 | for line in pbar: 32 | data = line.strip().split('\t') 33 | url, gt, pred, time = data 34 | total += 1 35 | runtime.append(float(time)) 36 | 37 | if is_valid_domain(pred): 38 | ct += 1 39 | else: 40 | print(url, gt, pred) 41 | 42 | print(f"Completeness (% brand recognized) = {ct / total} " 43 | f"Median runtime {np.median(runtime)}, Mean runtime {np.mean(runtime)}, " 44 | f"Min runtime {min(runtime)}, Max runtime {max(runtime)}, " 45 | f"Std runtime {np.std(runtime)}") 46 | 47 | 48 | def test_check_precision(result_file): 49 | ct_correct = 0 50 | ct = 0 51 | total = 0 52 | runtime = [] 53 | 54 | result_lines = open(result_file).readlines() 55 | pbar = tqdm(result_lines, leave=False) 56 | for line in pbar: 57 | data = line.strip().split('\t') 58 | url, gt, pred, time = data 59 | total += 1 60 | runtime.append(float(time)) 61 | 62 | if is_valid_domain(pred): 63 | ct += 1 64 | if tldextract.extract(pred).domain in gt: 65 | ct_correct += 1 66 | else: 67 | print(url, gt, pred) 68 | 69 | print(f"Completeness (% brand recognized) = {ct / total} " 70 | f"Precision = {ct_correct / ct} " 71 | f"Median runtime {np.median(runtime)}, Mean runtime {np.mean(runtime)}, " 72 | f"Min runtime {min(runtime)}, Max runtime {max(runtime)}, " 73 | f"Std runtime {np.std(runtime)}") 74 | 75 | 76 | if __name__ == '__main__': 77 | openai.api_key = os.getenv("OPENAI_API_KEY") 78 | proxy_url = "http://127.0.0.1:7890" 79 | openai.proxy = proxy_url # proxy 80 | 81 | with open('./param_dict.yaml') as file: 82 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 83 | 84 | phishintention_cls = PhishIntentionWrapper() 85 | llm_cls = TestLLM(phishintention_cls, 86 | param_dict=param_dict, 87 | proxies={"http": proxy_url, 88 | "https": proxy_url, 89 | } 90 | ) 91 | 92 | dataset = ShotDataset_Caption(annot_path='./datasets/alexa_screenshots_orig.txt') 93 | result_file = './datasets/alexa_brand_testllm_caption.txt' 94 | 95 | for it in tqdm(range(len(dataset))): 96 | 97 | if os.path.exists(result_file) and dataset.urls[it] in open(result_file).read(): 98 | continue 99 | 100 | url, _, logo_caption, logo_ocr, webpage_text, reference_logo = dataset.__getitem__(it) 101 | domain = tldextract.extract(url).domain + '.' + tldextract.extract(url).suffix 102 | print('Logo caption: ', logo_caption) 103 | print('Logo OCR: ', logo_ocr) 104 | total_time = 0 105 | 106 | if len(logo_caption) or len(logo_ocr): 107 | predicted_domain, _, brand_llm_pred_time = llm_cls.brand_recognition_llm(reference_logo=reference_logo, 108 | webpage_text=webpage_text, 109 | logo_caption=logo_caption, 110 | logo_ocr=logo_ocr, 111 | announcer=None) 112 | total_time += brand_llm_pred_time 113 | 114 | if predicted_domain and len(predicted_domain) > 0 and is_valid_domain(predicted_domain): 115 | validation_success, logo_cropping_time, logo_matching_time = llm_cls.brand_validation( 116 | company_domain=predicted_domain, 117 | reference_logo=reference_logo) 118 | 119 | total_time += logo_matching_time 120 | 121 | if not validation_success: 122 | answer = 'failure in logo matching' 123 | else: 124 | answer = 'no prediction' 125 | 126 | else: 127 | answer = 'no text in logo' 128 | 129 | with open(result_file, 'a+') as f: 130 | f.write(url + '\t' + domain + '\t' + answer + '\t' + str(total_time) + '\n') 131 | 132 | test(result_file) 133 | test_check_precision(result_file) 134 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/crp_prediction.py: -------------------------------------------------------------------------------- 1 | import os 2 | import openai 3 | from tqdm import tqdm 4 | from scripts.data.data_utils import ShotDataset 5 | import yaml 6 | from scripts.pipeline.test_llm import TestLLM 7 | from scripts.utils.PhishIntentionWrapper import PhishIntentionWrapper 8 | import numpy as np 9 | 10 | def test(result_file): 11 | correct = 0 12 | total = 0 13 | pred_pos = 0 14 | true_pos = 0 15 | pred_pos_and_true_pos = 0 16 | runtime = [] 17 | false_positives = [] 18 | false_negatives = [] 19 | 20 | result_lines = open(result_file).readlines() 21 | pbar = tqdm(result_lines, leave=False) 22 | for line in pbar: 23 | data = line.strip().split('\t') 24 | url, gt, pred, time = data 25 | 26 | pred_pos += float(' A.' in pred) 27 | true_pos += float(gt == 'A') 28 | pred_pos_and_true_pos += float(' A.' in pred) * float(gt == 'A') 29 | if (gt == 'A') and (' A.' in pred): 30 | correct += 1 31 | elif (gt == 'B') and (' A.' not in pred): # if the gt is non-crp, then no prediction is also correct prediction 32 | correct += 1 33 | elif gt == 'A': 34 | false_negatives.append(url) 35 | else: 36 | false_positives.append(url) 37 | 38 | total += 1 39 | runtime.append(float(time)) 40 | 41 | print(f"test classification acc: {correct / total}, " 42 | f"test precision: {pred_pos_and_true_pos / (pred_pos + 1e-8)} " 43 | f"test recall: {pred_pos_and_true_pos / (true_pos + 1e-8)} " 44 | f"Median runtime {np.median(runtime)}, Mean runtime {np.mean(runtime)} " 45 | f"Min runtime {min(runtime)}, Max runtime {max(runtime)}, " 46 | f"Std runtime {np.std(runtime)}" 47 | ) 48 | 49 | return false_positives, false_negatives 50 | 51 | 52 | if __name__ == '__main__': 53 | 54 | openai.api_key = os.getenv("OPENAI_API_KEY") 55 | proxy_url = "http://127.0.0.1:7890" 56 | openai.proxy = proxy_url # proxy 57 | 58 | with open('./param_dict.yaml') as file: 59 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 60 | 61 | phishintention_cls = PhishIntentionWrapper() 62 | llm_cls = TestLLM(phishintention_cls, 63 | param_dict=param_dict, 64 | proxies={"http": proxy_url, 65 | "https": proxy_url, 66 | } 67 | ) 68 | 69 | dataset = ShotDataset(annot_path='./datasets/alexa_screenshots.txt') 70 | print(len(dataset)) 71 | result_file = './datasets/alexa_shot_testllm2.txt' 72 | 73 | for it in tqdm(range(len(dataset))): 74 | 75 | if os.path.exists(result_file) and dataset.urls[it] in open(result_file).read(): 76 | continue 77 | url, gt, webpage_text = dataset.__getitem__(it, True) 78 | 79 | crp_cls, crp_prediction_time = llm_cls.crp_prediction_llm(html_text=webpage_text, 80 | announcer=None) 81 | 82 | with open(result_file, 'a+') as f: 83 | f.write(url+'\t'+gt+'\t'+str(crp_cls)+'\t'+str(crp_prediction_time)+'\n') 84 | 85 | false_positives, false_negatives = test(result_file) 86 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/crp_prediction_adversary.py: -------------------------------------------------------------------------------- 1 | import time 2 | import openai 3 | import pandas as pd 4 | import os 5 | from tqdm import tqdm 6 | import yaml 7 | from scripts.pipeline.test_llm import TestLLM 8 | from scripts.utils.PhishIntentionWrapper import PhishIntentionWrapper 9 | import numpy as np 10 | from PIL import Image 11 | 12 | def test(result_file): 13 | ct = 0 14 | total = 0 15 | result_lines = open(result_file).readlines() 16 | pbar = tqdm(result_lines, leave=False) 17 | for line in pbar: 18 | data = line.strip().split('\t') 19 | hash, gt, pred, time = data 20 | if 'A.' in pred: 21 | ct += 1 22 | total += 1 23 | 24 | pbar.set_description( f"After attack with defense recall: {ct / total} ", refresh=True) 25 | 26 | print(f"After attack with defense recall: {ct / total} ") 27 | 28 | 29 | if __name__ == '__main__': 30 | openai.api_key = os.getenv("OPENAI_API_KEY") 31 | proxy_url = "http://127.0.0.1:7890" 32 | openai.proxy = proxy_url # proxy 33 | 34 | with open('./param_dict.yaml') as file: 35 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 36 | 37 | phishintention_cls = PhishIntentionWrapper() 38 | llm_cls = TestLLM(phishintention_cls, 39 | param_dict=param_dict, 40 | proxies={"http": proxy_url, 41 | "https": proxy_url, 42 | } 43 | ) 44 | 45 | root_folder = './datasets/dynapd' 46 | all_folders = [x.strip().split('\t')[0] for x in open('./datasets/dynapd_wo_validation.txt').readlines()] 47 | df = pd.read_csv('./datasets/Brand_Labelled_130323.csv') 48 | crp_list = [x.strip() for x in open('./datasets/dynapd_crp_list.txt').readlines()] 49 | 50 | result = './datasets/dynapd_llm_adv_selection_defense.txt' 51 | 52 | for hash in tqdm(all_folders): 53 | target_folder = os.path.join(root_folder, hash) 54 | if os.path.exists(result) and hash in open(result).read(): 55 | continue 56 | if hash not in crp_list: 57 | continue 58 | 59 | shot_path = os.path.join(target_folder, 'shot.png') 60 | html_path = os.path.join(target_folder, 'index.html') 61 | if not os.path.exists(shot_path): 62 | continue 63 | 64 | pk_info = df.loc[df['HASH'] == hash] 65 | try: 66 | URL = list(pk_info['URL'])[0] 67 | except IndexError: 68 | URL = f'http://127.0.0.5/{hash}' 69 | 70 | logo_box, reference_logo = llm_cls.detect_logo(shot_path) 71 | plotvis = Image.open(shot_path) 72 | image_width, image_height = plotvis.size 73 | (webpage_text, logo_caption, logo_ocr), (ocr_processing_time, image_caption_processing_time) = \ 74 | llm_cls.preprocessing(shot_path=shot_path, 75 | html_path=html_path, 76 | reference_logo=reference_logo, 77 | logo_box=logo_box, 78 | image_width=image_width, 79 | image_height=image_height, 80 | announcer=None) 81 | 82 | crp_cls, crp_prediction_time = llm_cls.crp_prediction_llm(html_text=webpage_text, 83 | announcer=None) 84 | 85 | with open(result, 'a+') as f: 86 | f.write(hash+'\t'+'A'+'\t'+str(crp_cls)+'\t'+str(crp_prediction_time)+'\n') 87 | 88 | test(result) 89 | 90 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/crp_transition.py: -------------------------------------------------------------------------------- 1 | from scripts.data.data_utils import ButtonDataset 2 | import torch 3 | from tqdm import tqdm 4 | from torch.utils.data import DataLoader 5 | import clip 6 | import numpy as np 7 | from PIL import Image 8 | import pandas as pd 9 | 10 | @torch.no_grad() 11 | def tester(model, test_dataloader, device): 12 | 13 | model.eval() 14 | correct = 0 15 | total = 0 16 | pred_pos = 0 17 | true_pos = 0 18 | pred_pos_and_true_pos = 0 19 | 20 | pbar = tqdm(test_dataloader, leave=False) 21 | for batch in pbar: 22 | images, ground_truth, _, _, img_path, url = batch 23 | images = images.to(device) 24 | texts = clip.tokenize(["not a login button", "a login button"]).to(device) 25 | 26 | # align 27 | logits_per_image, logits_per_text = model(images, texts) 28 | pred = logits_per_image.argmax(dim=-1).cpu() 29 | 30 | correct += torch.eq(ground_truth, pred).sum().item() 31 | total += images.shape[0] 32 | 33 | pred_pos += pred.item() 34 | true_pos += ground_truth.item() 35 | pred_pos_and_true_pos += (pred.item()) * (ground_truth.item()) 36 | 37 | pbar.set_description(f"test classification acc: {correct/total}, " 38 | f"test precision: {pred_pos_and_true_pos/(pred_pos+1e-8)} " 39 | f"test recall: {pred_pos_and_true_pos/(true_pos+1e-8)} ", refresh=True) 40 | 41 | print(f"test classification acc: {correct/total}, " 42 | f"test precision: {pred_pos_and_true_pos/(pred_pos+1e-8)} " 43 | f"test recall: {pred_pos_and_true_pos/(true_pos+1e-8)} ") 44 | return correct, total 45 | 46 | 47 | @torch.no_grad() 48 | def tester_rank(model, test_dataset, preprocess, device): 49 | model.eval() 50 | correct = 0 51 | total = 0 52 | 53 | df = pd.DataFrame({'url': test_dataset.urls, 54 | 'path': test_dataset.img_paths, 55 | 'label': test_dataset.labels}) 56 | grp = df.groupby('url') 57 | grp = dict(list(grp), keys=lambda x: x[0]) # {url: List[dom_path, save_path]} 58 | 59 | for url, data in tqdm(grp.items()): 60 | try: 61 | img_paths = data.path 62 | labels = data.label 63 | except: 64 | continue 65 | labels = torch.tensor(np.asarray(labels)) 66 | images = [] 67 | for path in img_paths: 68 | img_process = preprocess(Image.open(path)) 69 | images.append(img_process) 70 | 71 | images = torch.stack(images).to(device) 72 | texts = clip.tokenize(["not a login button", "a login button"]).to(device) 73 | logits_per_image, logits_per_text = model(images, texts) 74 | probs = logits_per_image.softmax(dim=-1) # (N, C) 75 | conf = probs[torch.arange(probs.shape[0]), 1] # take the confidence (N, 1) 76 | _, ind = torch.topk(conf, min(10, len(conf))) # top1 index 77 | 78 | if (labels == 1).sum().item(): # has login button 79 | if (labels[ind] == 1).sum().item(): # has login button and it is reported 80 | correct += 1 81 | 82 | total += 1 83 | 84 | print(correct, total) 85 | 86 | 87 | 88 | if __name__ == '__main__': 89 | device = "cuda" if torch.cuda.is_available() else "cpu" 90 | model, preprocess = clip.load("ViT-B/32", device=device) 91 | if device == "cpu": 92 | model.float() # https://github.com/openai/CLIP/issues/57 93 | 94 | model, preprocess = clip.load("ViT-B/32", device=device, jit=False) 95 | 96 | train_dataset = ButtonDataset(annot_path='./datasets/alexa_login_train.txt', 97 | root='./datasets/alexa_login', 98 | preprocess=preprocess) 99 | 100 | test_dataset = ButtonDataset(annot_path='./datasets/alexa_login_test.txt', 101 | root='./datasets/alexa_login', 102 | preprocess=preprocess) 103 | 104 | test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False) 105 | train_dataloader = DataLoader(train_dataset, batch_size=1, shuffle=False) 106 | 107 | print(len(train_dataloader)) 108 | print(len(test_dataloader)) 109 | 110 | state_dict = torch.load("./checkpoints/epoch{}_model.pt".format(4)) 111 | model.load_state_dict(state_dict) 112 | # tester(model, test_dataloader, device) 113 | # overall image test classification acc: 0.9960878724044538, test precision: 0.9518987341531164 test recall: 0.8392857142669802 114 | 115 | tester_rank(model, test_dataset, preprocess, device) 116 | # top1 recall: 0.9128, top3 recall: 0.9283, top5 recall: 0.9470, top10 recall: 0.9720 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/crp_transition_adversary.py: -------------------------------------------------------------------------------- 1 | from scripts.data.data_utils import ButtonDataset 2 | from tqdm import tqdm 3 | from torch.utils.data import DataLoader 4 | import numpy as np 5 | from PIL import Image 6 | import pandas as pd 7 | from experiments.componentwise_evaluation.torchattacks.attacks.fgsm import FGSM 8 | from experiments.componentwise_evaluation.torchattacks.attacks.bim import BIM 9 | from experiments.componentwise_evaluation.torchattacks.attacks.deepfool import DeepFool 10 | from experiments.componentwise_evaluation.torchattacks.attacks.protect import * 11 | 12 | 13 | def tester_rank(model, test_dataset, device, protect_enabled, attack_method, adv_attack=True): 14 | total = 0 15 | perturb_correct = 0 16 | 17 | df = pd.DataFrame({'url': test_dataset.urls, 18 | 'path': test_dataset.img_paths, 19 | 'label': test_dataset.labels}) 20 | grp = df.groupby('url') 21 | grp = dict(list(grp), keys=lambda x: x[0]) # {url: List[dom_path, save_path]} 22 | 23 | for url, data in tqdm(grp.items()): 24 | torch.cuda.empty_cache() 25 | try: 26 | img_paths = data.path 27 | labels = data.label 28 | except: 29 | continue 30 | labels = torch.tensor(np.asarray(labels)) 31 | images = [] 32 | for path in img_paths: 33 | img_process = preprocess(Image.open(path)) 34 | images.append(img_process) 35 | 36 | if (labels == 1).sum().item(): # has login button 37 | total += 1 38 | else: 39 | continue 40 | 41 | images = torch.stack(images).to(device) 42 | if adv_attack: 43 | # adversary attack 44 | target_labels = torch.zeros_like(labels) 45 | target_labels = target_labels.long().to(device) 46 | 47 | if protect_enabled: 48 | protect_act(model.visual) 49 | protect_resnetblock(model.visual) 50 | model = model.to(device) 51 | if attack_method == 'DeepFool': 52 | attack_cls = DeepFool(model, device=device) 53 | elif attack_method == 'FGSM': 54 | attack_cls = FGSM(model, device=device) 55 | elif attack_method == 'BIM': 56 | attack_cls = BIM(model, device=device) 57 | adv_images = attack_cls(images, labels, target_labels) 58 | images.detach() 59 | adv_images.detach() 60 | del attack_cls 61 | 62 | # perturbed prediction 63 | del model 64 | model, _ = clip.load("ViT-B/32", device=device) 65 | model.load_state_dict(torch.load("./checkpoints/epoch4_model.pt")) 66 | model = model.to(device) 67 | freeze_params(model) 68 | 69 | texts = clip.tokenize(["not a login button", "a login button"]).to(device) 70 | with torch.no_grad(): 71 | if adv_attack: 72 | logits_per_image, logits_per_text = model(adv_images, texts) 73 | else: 74 | logits_per_image, logits_per_text = model(images, texts) 75 | 76 | probs = logits_per_image.softmax(dim=-1) # (N, C) 77 | conf = probs[torch.arange(probs.shape[0]), 1] # take the confidence (N, 1) 78 | _, ind = torch.topk(conf, min(1, len(conf))) # top1 index 79 | 80 | if (labels[ind] == 1).sum().item(): # has login button and it is reported 81 | perturb_correct += 1 82 | 83 | print(f"After attack correct count = {perturb_correct}, Total = {total}, Recall@K = {perturb_correct/total}") 84 | 85 | print(f"After attack correct count = {perturb_correct}, Total = {total}, Recall@K = {perturb_correct/total}") 86 | 87 | 88 | if __name__ == '__main__': 89 | protect_enabled = False # protect or not 90 | attack_method = 'FGSM' 91 | assert attack_method in ['DeepFool', 'FGSM', 'BIM'] 92 | 93 | device = "cuda" if torch.cuda.is_available() else "cpu" 94 | model, preprocess = clip.load("ViT-B/32", device=device) 95 | # https://github.com/openai/CLIP/issues/57 96 | if device == "cpu": 97 | model.float() 98 | 99 | test_dataset = ButtonDataset(annot_path='./datasets/alexa_login_test.txt', 100 | root='./datasets/alexa_login', 101 | preprocess=preprocess) 102 | test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False) 103 | 104 | state_dict = torch.load("./checkpoints/epoch{}_model.pt".format(4)) 105 | model.load_state_dict(state_dict) 106 | 107 | if protect_enabled: 108 | protect_act(model.visual) # 109 | protect_resnetblock(model.visual) 110 | model = model.to(device) 111 | freeze_params(model) 112 | 113 | tester_rank(model, test_dataset, device, protect_enabled=protect_enabled, attack_method=attack_method, 114 | image_save_dir='./ranking_model/test_case') -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## This part of code is cloned from https://github.com/Harry24k/adversarial-attacks-pytorch/ -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/__init__.py: -------------------------------------------------------------------------------- 1 | # None attacks 2 | 3 | # Linf attacks 4 | from .attacks.fgsm import FGSM 5 | from .attacks.bim import BIM 6 | from .attacks.rfgsm import RFGSM 7 | from .attacks.pgd import PGD 8 | from .attacks.ffgsm import FFGSM 9 | from .attacks.mifgsm import MIFGSM 10 | from .attacks.difgsm import DIFGSM 11 | from .attacks.jitter import Jitter 12 | from .attacks.nifgsm import NIFGSM 13 | from .attacks.pgdrs import PGDRS 14 | from .attacks.sinifgsm import SINIFGSM 15 | 16 | # L2 attacks 17 | from .attacks.pgdl2 import PGDL2 18 | from .attacks.pgdrsl2 import PGDRSL2 19 | from .attacks.deepfool import DeepFool 20 | 21 | # L1 attacks 22 | 23 | # L0 attacks 24 | from .attacks.sparsefool import SparseFool 25 | from .attacks.jsma import JSMA 26 | 27 | # Linf, L2 attacks 28 | from .attacks.fab import FAB 29 | 30 | # Wrapper Class 31 | from .wrappers.multiattack import MultiAttack 32 | from .wrappers.lgv import LGV 33 | 34 | __version__ = '3.4.1' 35 | __all__ = [ 36 | "FGSM", "BIM", "RFGSM", "PGD", "FFGSM", 37 | "MIFGSM", "DIFGSM", 38 | "Jitter", "NIFGSM", "PGDRS", "SINIFGSM", 39 | "JSMA", "PGDL2", "DeepFool", "PGDRSL2", 40 | "SparseFool", 41 | "FAB", 42 | "MultiAttack", "LGV", 43 | ] 44 | __wrapper__ = [ 45 | "LGV", "MultiAttack", 46 | ] 47 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/componentwise_evaluation/torchattacks/attacks/__init__.py -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/autoattack.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from ..attack import Attack 4 | from ..wrappers.multiattack import MultiAttack 5 | from .apgd import APGD 6 | from .apgdt import APGDT 7 | from .fab import FAB 8 | from .square import Square 9 | 10 | 11 | class AutoAttack(Attack): 12 | r""" 13 | AutoAttack in the paper 'Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks' 14 | [https://arxiv.org/abs/2003.01690] 15 | [https://github.com/fra31/auto-attack] 16 | 17 | Distance Measure : Linf, L2 18 | 19 | Arguments: 20 | model (nn.Module): model to attack. 21 | norm (str) : Lp-norm to minimize. ['Linf', 'L2'] (Default: 'Linf') 22 | eps (float): maximum perturbation. (Default: 0.3) 23 | version (bool): version. ['standard', 'plus', 'rand'] (Default: 'standard') 24 | n_classes (int): number of classes. (Default: 10) 25 | seed (int): random seed for the starting point. (Default: 0) 26 | verbose (bool): print progress. (Default: False) 27 | 28 | Shape: 29 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 30 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 31 | - output: :math:`(N, C, H, W)`. 32 | 33 | Examples:: 34 | >>> attack = torchattacks.AutoAttack(model, norm='Linf', eps=8/255, version='standard', n_classes=10, seed=None, verbose=False) 35 | >>> adv_images = attack(images, labels) 36 | 37 | """ 38 | 39 | def __init__(self, model, device=None, norm='Linf', eps=8/255, version='standard', n_classes=10, seed=None, verbose=False): 40 | super().__init__('AutoAttack', model, device) 41 | self.norm = norm 42 | self.eps = eps 43 | self.version = version 44 | self.n_classes = n_classes 45 | self.seed = seed 46 | self.verbose = verbose 47 | self.supported_mode = ['default'] 48 | 49 | if version == 'standard': # ['apgd-ce', 'apgd-t', 'fab-t', 'square'] 50 | self._autoattack = MultiAttack([ 51 | APGD(model, eps=eps, norm=norm, seed=self.get_seed(), 52 | verbose=verbose, loss='ce', n_restarts=1), 53 | APGDT(model, eps=eps, norm=norm, seed=self.get_seed(), 54 | verbose=verbose, n_classes=n_classes, n_restarts=1), 55 | FAB(model, eps=eps, norm=norm, seed=self.get_seed( 56 | ), verbose=verbose, multi_targeted=True, n_classes=n_classes, n_restarts=1), 57 | Square(model, eps=eps, norm=norm, seed=self.get_seed(), 58 | verbose=verbose, n_queries=5000, n_restarts=1), 59 | ]) 60 | 61 | # ['apgd-ce', 'apgd-dlr', 'fab', 'square', 'apgd-t', 'fab-t'] 62 | elif version == 'plus': 63 | self._autoattack = MultiAttack([ 64 | APGD(model, eps=eps, norm=norm, seed=self.get_seed(), 65 | verbose=verbose, loss='ce', n_restarts=5), 66 | APGD(model, eps=eps, norm=norm, seed=self.get_seed(), 67 | verbose=verbose, loss='dlr', n_restarts=5), 68 | FAB(model, eps=eps, norm=norm, seed=self.get_seed(), 69 | verbose=verbose, n_classes=n_classes, n_restarts=5), 70 | Square(model, eps=eps, norm=norm, seed=self.get_seed(), 71 | verbose=verbose, n_queries=5000, n_restarts=1), 72 | APGDT(model, eps=eps, norm=norm, seed=self.get_seed(), 73 | verbose=verbose, n_classes=n_classes, n_restarts=1), 74 | FAB(model, eps=eps, norm=norm, seed=self.get_seed( 75 | ), verbose=verbose, multi_targeted=True, n_classes=n_classes, n_restarts=1), 76 | ]) 77 | 78 | elif version == 'rand': # ['apgd-ce', 'apgd-dlr'] 79 | self._autoattack = MultiAttack([ 80 | APGD(model, eps=eps, norm=norm, seed=self.get_seed(), 81 | verbose=verbose, loss='ce', eot_iter=20, n_restarts=1), 82 | APGD(model, eps=eps, norm=norm, seed=self.get_seed(), 83 | verbose=verbose, loss='dlr', eot_iter=20, n_restarts=1), 84 | ]) 85 | 86 | else: 87 | raise ValueError("Not valid version. ['standard', 'plus', 'rand']") 88 | 89 | def forward(self, images, labels): 90 | r""" 91 | Overridden. 92 | """ 93 | 94 | images = images.clone().detach().to(self.device) 95 | labels = labels.clone().detach().to(self.device) 96 | adv_images = self._autoattack(images, labels) 97 | 98 | return adv_images 99 | 100 | def get_seed(self): 101 | return time.time() if self.seed is None else self.seed 102 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/bim.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | from .protect import * 6 | 7 | class BIM(Attack): 8 | r""" 9 | BIM or iterative-FGSM in the paper 'Adversarial Examples in the Physical World' 10 | [https://arxiv.org/abs/1607.02533] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 8/255) 17 | alpha (float): step size. (Default: 2/255) 18 | steps (int): number of steps. (Default: 10) 19 | 20 | .. note:: If steps set to 0, steps will be automatically decided following the paper. 21 | 22 | Shape: 23 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 24 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 25 | - output: :math:`(N, C, H, W)`. 26 | 27 | Examples:: 28 | >>> attack = torchattacks.BIM(model, eps=8/255, alpha=2/255, steps=10) 29 | >>> adv_images = attack(images, labels) 30 | """ 31 | 32 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10): 33 | super().__init__('BIM', model, device) 34 | self.eps = eps 35 | self.alpha = alpha 36 | if steps == 0: 37 | self.steps = int(min(eps*255 + 4, 1.25*eps*255)) 38 | else: 39 | self.steps = steps 40 | self.supported_mode = ['default', 'targeted'] 41 | 42 | def forward(self, images, labels, target_labels): 43 | r""" 44 | Overridden. 45 | """ 46 | 47 | images = images.clone().detach().to(self.device) 48 | 49 | loss = nn.CrossEntropyLoss() 50 | 51 | for _ in range(self.steps): 52 | images.requires_grad = True 53 | outputs = self.get_logits(images) 54 | 55 | # Calculate loss 56 | cost = -loss(outputs, target_labels) 57 | 58 | # Update adversarial images 59 | grad = torch.autograd.grad(cost, images, 60 | retain_graph=False, 61 | create_graph=False)[0] 62 | 63 | adv_images = images + self.alpha * grad.sign() 64 | # a = torch.clamp(ori_images - self.eps, min=0) 65 | # b = (adv_images >= a).float() * adv_images + (adv_images < a).float() * a # nopep8 66 | # c = (b > ori_images+self.eps).float() * (ori_images+self.eps) + (b <= ori_images + self.eps).float() * b # nopep8 67 | # images = torch.clamp(c, max=1).detach() 68 | images = adv_images.detach() 69 | 70 | return images 71 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/cw.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | import torch.nn as nn 4 | import torch.optim as optim 5 | 6 | from ..attack import Attack 7 | 8 | 9 | class CW(Attack): 10 | r""" 11 | CW in the paper 'Towards Evaluating the Robustness of Neural Networks' 12 | [https://arxiv.org/abs/1608.04644] 13 | 14 | Distance Measure : L2 15 | 16 | Arguments: 17 | model (nn.Module): model to attack. 18 | c (float): c in the paper. parameter for box-constraint. (Default: 1) 19 | :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` 20 | kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) 21 | :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` 22 | steps (int): number of steps. (Default: 50) 23 | lr (float): learning rate of the Adam optimizer. (Default: 0.01) 24 | 25 | .. warning:: With default c, you can't easily get adversarial images. Set higher c like 1. 26 | 27 | Shape: 28 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 29 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 30 | - output: :math:`(N, C, H, W)`. 31 | 32 | Examples:: 33 | >>> attack = torchattacks.CW(model, c=1, kappa=0, steps=50, lr=0.01) 34 | >>> adv_images = attack(images, labels) 35 | 36 | .. note:: Binary search for c is NOT IMPLEMENTED methods in the paper due to time consuming. 37 | 38 | """ 39 | 40 | def __init__(self, model, device=None, c=1, kappa=0, steps=50, lr=0.01): 41 | super().__init__('CW', model, device) 42 | self.c = c 43 | self.kappa = kappa 44 | self.steps = steps 45 | self.lr = lr 46 | self.supported_mode = ['default', 'targeted'] 47 | 48 | def forward(self, images, labels, target_labels): 49 | r""" 50 | Overridden. 51 | """ 52 | 53 | images = images.clone().detach().to(self.device) 54 | labels = labels.clone().detach().to(self.device) 55 | 56 | # w = torch.zeros_like(images).detach() # Requires 2x times 57 | w = self.inverse_tanh_space(images).detach() 58 | w.requires_grad = True 59 | 60 | best_adv_images = images.clone().detach() 61 | best_L2 = 1e10*torch.ones((len(images))).to(self.device) 62 | prev_cost = 1e10 63 | dim = len(images.shape) 64 | 65 | MSELoss = nn.MSELoss(reduction='none') 66 | Flatten = nn.Flatten() 67 | 68 | optimizer = optim.Adam([w], lr=self.lr) 69 | 70 | for step in range(self.steps): 71 | # Get adversarial images 72 | adv_images = self.tanh_space(w) 73 | 74 | # Calculate loss 75 | current_L2 = MSELoss(Flatten(adv_images), 76 | Flatten(images)).sum(dim=1) 77 | L2_loss = current_L2.sum() 78 | 79 | outputs = self.get_logits(adv_images) 80 | f_loss = self.f(outputs, target_labels).sum() 81 | 82 | cost = L2_loss + self.c*f_loss 83 | 84 | optimizer.zero_grad() 85 | cost.backward() 86 | optimizer.step() 87 | 88 | # Update adversarial images 89 | pre = torch.argmax(outputs.detach(), 1) 90 | if self.targeted: 91 | # We want to let pre == target_labels in a targeted attack 92 | condition = (pre == target_labels).float() 93 | else: 94 | # If the attack is not targeted we simply make these two values unequal 95 | condition = (pre != labels).float() 96 | 97 | # Filter out images that get either correct predictions or non-decreasing loss, 98 | # i.e., only images that are both misclassified and loss-decreasing are left 99 | mask = condition*(best_L2 > current_L2.detach()) 100 | best_L2 = mask*current_L2.detach() + (1-mask)*best_L2 101 | 102 | mask = mask.view([-1]+[1]*(dim-1)) 103 | best_adv_images = mask*adv_images.detach() + (1-mask)*best_adv_images 104 | 105 | # Early stop when loss does not converge. 106 | # max(.,1) To prevent MODULO BY ZERO error in the next step. 107 | if step % max(self.steps//10, 1) == 0: 108 | if cost.item() > prev_cost: 109 | return best_adv_images 110 | prev_cost = cost.item() 111 | 112 | return best_adv_images 113 | 114 | def tanh_space(self, x): 115 | return 1/2*(torch.tanh(x) + 1) 116 | 117 | def inverse_tanh_space(self, x): 118 | # torch.atanh is only for torch >= 1.7.0 119 | # atanh is defined in the range -1 to 1 120 | return self.atanh(torch.clamp(x*2-1, min=-1, max=1)) 121 | 122 | def atanh(self, x): 123 | return 0.5*torch.log((1+x)/(1-x)) 124 | 125 | # f-function in the paper 126 | def f(self, outputs, labels): 127 | one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] 128 | 129 | # find the max logit other than the target class 130 | other = torch.max((1-one_hot_labels)*outputs, dim=1)[0] 131 | # get the target class's logit 132 | real = torch.max(one_hot_labels*outputs, dim=1)[0] 133 | 134 | if self.targeted: 135 | return torch.clamp((other-real), min=-self.kappa) 136 | else: 137 | return torch.clamp((real-other), min=-self.kappa) 138 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/deepfool.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | from .protect import * 6 | 7 | class DeepFool(Attack): 8 | r""" 9 | 'DeepFool: A Simple and Accurate Method to Fool Deep Neural Networks' 10 | [https://arxiv.org/abs/1511.04599] 11 | Distance Measure : L2 12 | Arguments: 13 | model (nn.Module): model to attack. 14 | steps (int): number of steps. (Default: 50) 15 | overshoot (float): parameter for enhancing the noise. (Default: 0.02) 16 | Shape: 17 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 18 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 19 | - output: :math:`(N, C, H, W)`. 20 | Examples:: 21 | >>> attack = torchattacks.DeepFool(model, steps=50, overshoot=0.02) 22 | >>> adv_images = attack(images, labels) 23 | """ 24 | def __init__(self, model, device=None, steps=5, overshoot=0.02): 25 | super().__init__('DeepFool', model, device) 26 | self.steps = steps 27 | self.overshoot = overshoot 28 | self.supported_mode = ['default'] 29 | 30 | def forward(self, images, labels, target_labels): 31 | r""" 32 | Overridden. 33 | """ 34 | adv_images, _ = self.forward_return_target_labels(images, labels) 35 | return adv_images 36 | 37 | def forward_return_target_labels(self, images, labels): 38 | r""" 39 | Overridden. 40 | """ 41 | images = images.clone().detach().to(self.device) 42 | labels = labels.clone().detach().to(self.device) 43 | 44 | batch_size = len(images) 45 | correct = torch.tensor([True]*batch_size) 46 | target_labels = labels.clone().detach().to(self.device) 47 | curr_steps = 0 48 | 49 | adv_images = [] 50 | for idx in range(batch_size): 51 | image = images[idx:idx+1].clone().detach() 52 | adv_images.append(image) 53 | 54 | while (True in correct) and (curr_steps < self.steps): 55 | for idx in range(batch_size): 56 | if not correct[idx]: 57 | continue 58 | early_stop, pre, adv_image = self._forward_indiv(adv_images[idx], labels[idx]) 59 | adv_images[idx] = adv_image 60 | target_labels[idx] = pre 61 | if early_stop: 62 | correct[idx] = False 63 | curr_steps += 1 64 | 65 | adv_images = torch.cat(adv_images).detach() 66 | return adv_images, target_labels 67 | 68 | def _forward_indiv(self, image, label): 69 | image.requires_grad = True 70 | 71 | fs = self.get_logits(image)[0] # output 72 | _, pre = torch.max(fs, dim=0) 73 | if pre != label: 74 | return (True, pre, image) 75 | 76 | ws = self._construct_jacobian(fs, image) # output to image 77 | image = image.detach() 78 | 79 | f_0 = fs[label] 80 | w_0 = ws[label] 81 | 82 | wrong_classes = [i for i in range(len(fs)) if i != label] 83 | f_k = fs[wrong_classes] 84 | w_k = ws[wrong_classes] 85 | 86 | f_prime = f_k - f_0 87 | w_prime = w_k - w_0 88 | value = torch.abs(f_prime) \ 89 | / (torch.norm(nn.Flatten()(w_prime), p=2, dim=1) + 1e-8) 90 | _, hat_L = torch.min(value, 0) 91 | 92 | delta = (torch.abs(f_prime[hat_L])*w_prime[hat_L] \ 93 | / (torch.norm(w_prime[hat_L], p=2)**2 + 1e-8)) # project to the boundary 94 | 95 | target_label = hat_L if hat_L < label else hat_L+1 96 | 97 | adv_image = image + (1+self.overshoot) * delta 98 | adv_image = adv_image.detach() 99 | return (False, target_label, adv_image) 100 | 101 | # https://stackoverflow.com/questions/63096122/pytorch-is-it-possible-to-differentiate-a-matrix 102 | # torch.autograd.functional.jacobian is only for torch >= 1.5.1 103 | def _construct_jacobian(self, y, x): 104 | x_grads = [] 105 | for idx, y_element in enumerate(y): 106 | if x.grad is not None: 107 | x.grad.zero_() 108 | y_element.backward(retain_graph=(False or idx+1 < len(y))) 109 | x_grads.append(x.grad.clone().detach()) 110 | return torch.stack(x_grads).reshape(*y.shape, *x.shape) -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/difgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from ..attack import Attack 6 | 7 | 8 | class DIFGSM(Attack): 9 | r""" 10 | DI2-FGSM in the paper 'Improving Transferability of Adversarial Examples with Input Diversity' 11 | [https://arxiv.org/abs/1803.06978] 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): maximum perturbation. (Default: 8/255) 18 | alpha (float): step size. (Default: 2/255) 19 | decay (float): momentum factor. (Default: 0.0) 20 | steps (int): number of iterations. (Default: 10) 21 | resize_rate (float): resize factor used in input diversity. (Default: 0.9) 22 | diversity_prob (float) : the probability of applying input diversity. (Default: 0.5) 23 | random_start (bool): using random initialization of delta. (Default: False) 24 | 25 | Shape: 26 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 27 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 28 | - output: :math:`(N, C, H, W)`. 29 | 30 | Examples:: 31 | >>> attack = torchattacks.DIFGSM(model, eps=8/255, alpha=2/255, steps=10, decay=0.0, resize_rate=0.9, diversity_prob=0.5, random_start=False) 32 | >>> adv_images = attack(images, labels) 33 | 34 | """ 35 | 36 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, decay=0.0, 37 | resize_rate=0.9, diversity_prob=0.5, random_start=False): 38 | super().__init__('DIFGSM', model, device) 39 | self.eps = eps 40 | self.steps = steps 41 | self.decay = decay 42 | self.alpha = alpha 43 | self.resize_rate = resize_rate 44 | self.diversity_prob = diversity_prob 45 | self.random_start = random_start 46 | self.supported_mode = ['default', 'targeted'] 47 | 48 | def input_diversity(self, x): 49 | img_size = x.shape[-1] 50 | img_resize = int(img_size * self.resize_rate) 51 | 52 | if self.resize_rate < 1: 53 | img_size = img_resize 54 | img_resize = x.shape[-1] 55 | 56 | rnd = torch.randint(low=img_size, high=img_resize, size=(1,), dtype=torch.int32) 57 | rescaled = F.interpolate(x, size=[rnd, rnd], mode='bilinear', align_corners=False) 58 | h_rem = img_resize - rnd 59 | w_rem = img_resize - rnd 60 | pad_top = torch.randint(low=0, high=h_rem.item(), size=(1,), dtype=torch.int32) 61 | pad_bottom = h_rem - pad_top 62 | pad_left = torch.randint(low=0, high=w_rem.item(), size=(1,), dtype=torch.int32) 63 | pad_right = w_rem - pad_left 64 | 65 | padded = F.pad(rescaled, [pad_left.item(), pad_right.item(), pad_top.item(), pad_bottom.item()], value=0) 66 | 67 | return padded if torch.rand(1) < self.diversity_prob else x 68 | 69 | def forward(self, images, labels): 70 | r""" 71 | Overridden. 72 | """ 73 | 74 | images = images.clone().detach().to(self.device) 75 | labels = labels.clone().detach().to(self.device) 76 | 77 | if self.targeted: 78 | target_labels = self.get_target_label(images, labels) 79 | 80 | loss = nn.CrossEntropyLoss() 81 | momentum = torch.zeros_like(images).detach().to(self.device) 82 | 83 | adv_images = images.clone().detach() 84 | 85 | if self.random_start: 86 | # Starting at a uniformly random point 87 | adv_images = adv_images + torch.empty_like(adv_images).uniform_(-self.eps, self.eps) 88 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 89 | 90 | for _ in range(self.steps): 91 | adv_images.requires_grad = True 92 | outputs = self.get_logits(self.input_diversity(adv_images)) 93 | 94 | # Calculate loss 95 | if self.targeted: 96 | cost = -loss(outputs, target_labels) 97 | else: 98 | cost = loss(outputs, labels) 99 | 100 | # Update adversarial images 101 | grad = torch.autograd.grad(cost, adv_images, 102 | retain_graph=False, create_graph=False)[0] 103 | 104 | grad = grad / torch.mean(torch.abs(grad), dim=(1,2,3), keepdim=True) 105 | grad = grad + momentum*self.decay 106 | momentum = grad 107 | 108 | adv_images = adv_images.detach() + self.alpha*grad.sign() 109 | delta = torch.clamp(adv_images - images, min=-self.eps, max=self.eps) 110 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 111 | 112 | return adv_images 113 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/eotpgd.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class EOTPGD(Attack): 8 | r""" 9 | Comment on "Adv-BNN: Improved Adversarial Defense through Robust Bayesian Neural Network" 10 | [https://arxiv.org/abs/1907.00895] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 8/255) 17 | alpha (float): step size. (Default: 2/255) 18 | steps (int): number of steps. (Default: 10) 19 | eot_iter (int) : number of models to estimate the mean gradient. (Default: 2) 20 | 21 | Shape: 22 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 23 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 24 | - output: :math:`(N, C, H, W)`. 25 | 26 | Examples:: 27 | >>> attack = torchattacks.EOTPGD(model, eps=8/255, alpha=2/255, steps=10, eot_iter=2) 28 | >>> adv_images = attack(images, labels) 29 | 30 | """ 31 | 32 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, 33 | eot_iter=2, random_start=True): 34 | super().__init__('EOTPGD', model, device) 35 | self.eps = eps 36 | self.alpha = alpha 37 | self.steps = steps 38 | self.eot_iter = eot_iter 39 | self.random_start = random_start 40 | self.supported_mode = ['default', 'targeted'] 41 | 42 | def forward(self, images, labels): 43 | r""" 44 | Overridden. 45 | """ 46 | 47 | images = images.clone().detach().to(self.device) 48 | labels = labels.clone().detach().to(self.device) 49 | 50 | if self.targeted: 51 | target_labels = self.get_target_label(images, labels) 52 | 53 | loss = nn.CrossEntropyLoss() 54 | 55 | adv_images = images.clone().detach() 56 | 57 | if self.random_start: 58 | # Starting at a uniformly random point 59 | adv_images = adv_images + torch.empty_like(adv_images).uniform_(-self.eps, self.eps) # nopep8 60 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 61 | 62 | for _ in range(self.steps): 63 | grad = torch.zeros_like(adv_images) 64 | adv_images.requires_grad = True 65 | 66 | for j in range(self.eot_iter): 67 | outputs = self.get_logits(adv_images) 68 | 69 | # Calculate loss 70 | if self.targeted: 71 | cost = -loss(outputs, target_labels) 72 | else: 73 | cost = loss(outputs, labels) 74 | 75 | # Update adversarial images 76 | grad += torch.autograd.grad(cost, adv_images, 77 | retain_graph=False, 78 | create_graph=False)[0] 79 | 80 | # (grad/self.eot_iter).sign() == grad.sign() 81 | adv_images = adv_images.detach() + self.alpha*grad.sign() 82 | delta = torch.clamp(adv_images - images, 83 | min=-self.eps, max=self.eps) 84 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 85 | 86 | return adv_images 87 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/ffgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class FFGSM(Attack): 8 | r""" 9 | New FGSM proposed in 'Fast is better than free: Revisiting adversarial training' 10 | [https://arxiv.org/abs/2001.03994] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 8/255) 17 | alpha (float): step size. (Default: 10/255) 18 | 19 | Shape: 20 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 21 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 22 | - output: :math:`(N, C, H, W)`. 23 | 24 | Examples:: 25 | >>> attack = torchattacks.FFGSM(model, eps=8/255, alpha=10/255) 26 | >>> adv_images = attack(images, labels) 27 | """ 28 | 29 | def __init__(self, model, device=None, eps=8/255, alpha=10/255): 30 | super().__init__('FFGSM', model, device) 31 | self.eps = eps 32 | self.alpha = alpha 33 | self.supported_mode = ['default', 'targeted'] 34 | 35 | def forward(self, images, labels): 36 | r""" 37 | Overridden. 38 | """ 39 | 40 | images = images.clone().detach().to(self.device) 41 | labels = labels.clone().detach().to(self.device) 42 | 43 | if self.targeted: 44 | target_labels = self.get_target_label(images, labels) 45 | 46 | loss = nn.CrossEntropyLoss() 47 | 48 | adv_images = images + torch.randn_like(images).uniform_(-self.eps, self.eps) # nopep8 49 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 50 | adv_images.requires_grad = True 51 | 52 | outputs = self.get_logits(adv_images) 53 | 54 | # Calculate loss 55 | if self.targeted: 56 | cost = -loss(outputs, target_labels) 57 | else: 58 | cost = loss(outputs, labels) 59 | 60 | # Update adversarial images 61 | grad = torch.autograd.grad(cost, adv_images, 62 | retain_graph=False, create_graph=False)[0] 63 | 64 | adv_images = adv_images + self.alpha*grad.sign() 65 | delta = torch.clamp(adv_images - images, min=-self.eps, max=self.eps) 66 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 67 | 68 | return adv_images 69 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/fgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class FGSM(Attack): 8 | r""" 9 | FGSM in the paper 'Explaining and harnessing adversarial examples' 10 | [https://arxiv.org/abs/1412.6572] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 8/255) 17 | 18 | Shape: 19 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 20 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 21 | - output: :math:`(N, C, H, W)`. 22 | 23 | Examples:: 24 | >>> attack = torchattacks.FGSM(model, eps=8/255) 25 | >>> adv_images = attack(images, labels) 26 | 27 | """ 28 | def __init__(self, model, device=None, eps=8/255): 29 | super().__init__('FGSM', model, device) 30 | self.eps = eps 31 | self.supported_mode = ['default', 'targeted'] 32 | 33 | 34 | def forward(self, images, labels, target_labels): 35 | r""" 36 | Overridden. 37 | """ 38 | 39 | images = images.clone().detach().to(self.device) 40 | labels = labels.clone().detach().to(self.device) 41 | 42 | loss = nn.CrossEntropyLoss() 43 | 44 | images.requires_grad = True 45 | outputs = self.get_logits(images) 46 | 47 | # Calculate loss 48 | cost = -loss(outputs, target_labels) 49 | 50 | # Update adversarial images 51 | grad = torch.autograd.grad(cost, images, 52 | retain_graph=False, create_graph=False)[0] 53 | 54 | adv_images = images + self.eps*grad.sign() 55 | 56 | return adv_images 57 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/gn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..attack import Attack 4 | 5 | 6 | class GN(Attack): 7 | r""" 8 | Add Gaussian Noise. 9 | 10 | Arguments: 11 | model (nn.Module): model to attack. 12 | std (nn.Module): standard deviation (Default: 0.1). 13 | 14 | Shape: 15 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 16 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 17 | - output: :math:`(N, C, H, W)`. 18 | 19 | Examples:: 20 | >>> attack = torchattacks.GN(model) 21 | >>> adv_images = attack(images, labels) 22 | 23 | """ 24 | 25 | def __init__(self, model, device=None, std=0.1): 26 | super().__init__('GN', model, device) 27 | self.std = std 28 | self.supported_mode = ['default'] 29 | 30 | def forward(self, images, labels=None): 31 | r""" 32 | Overridden. 33 | """ 34 | 35 | images = images.clone().detach().to(self.device) 36 | adv_images = images + self.std*torch.randn_like(images) 37 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 38 | 39 | return adv_images 40 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/jitter.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from ..attack import Attack 6 | 7 | 8 | class Jitter(Attack): 9 | r""" 10 | Jitter in the paper 'Exploring Misclassifications of Robust Neural Networks to Enhance Adversarial Attacks' 11 | [https://arxiv.org/abs/2105.10304] 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): maximum perturbation. (Default: 8/255) 18 | alpha (float): step size. (Default: 2/255) 19 | steps (int): number of steps. (Default: 10) 20 | random_start (bool): using random initialization of delta. (Default: True) 21 | 22 | Shape: 23 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 24 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 25 | - output: :math:`(N, C, H, W)`. 26 | 27 | Examples:: 28 | >>> attack = torchattacks.Jitter(model, eps=8/255, alpha=2/255, steps=10, 29 | scale=10, std=0.1, random_start=True) 30 | >>> adv_images = attack(images, labels) 31 | 32 | """ 33 | 34 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, 35 | scale=10, std=0.1, random_start=True): 36 | super().__init__('Jitter', model, device) 37 | self.eps = eps 38 | self.alpha = alpha 39 | self.steps = steps 40 | self.random_start = random_start 41 | self.scale = scale 42 | self.std = std 43 | self.supported_mode = ['default', 'targeted'] 44 | 45 | def forward(self, images, labels): 46 | r""" 47 | Overridden. 48 | """ 49 | 50 | images = images.clone().detach().to(self.device) 51 | labels = labels.clone().detach().to(self.device) 52 | 53 | if self.targeted: 54 | target_labels = self.get_target_label(images, labels) 55 | 56 | loss = nn.MSELoss(reduction='none') 57 | 58 | adv_images = images.clone().detach() 59 | 60 | if self.random_start: 61 | # Starting at a uniformly random point 62 | adv_images = adv_images + \ 63 | torch.empty_like(adv_images).uniform_(-self.eps, self.eps) 64 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 65 | 66 | for _ in range(self.steps): 67 | adv_images.requires_grad = True 68 | logits = self.get_logits(adv_images) 69 | 70 | _, pre = torch.max(logits, dim=1) 71 | wrong = (pre != labels) 72 | 73 | norm_z = torch.norm(logits, p=float('inf'), dim=1, keepdim=True) 74 | hat_z = nn.Softmax(dim=1)(self.scale*logits/norm_z) 75 | 76 | if self.std != 0: 77 | hat_z = hat_z + self.std*torch.randn_like(hat_z) 78 | 79 | # Calculate loss 80 | if self.targeted: 81 | target_Y = F.one_hot( 82 | target_labels, num_classes=logits.shape[-1]).float() 83 | cost = -loss(hat_z, target_Y).mean(dim=1) 84 | else: 85 | Y = F.one_hot(labels, num_classes=logits.shape[-1]).float() 86 | cost = loss(hat_z, Y).mean(dim=1) 87 | 88 | norm_r = torch.norm((adv_images - images), p=float('inf'), dim=[1, 2, 3]) # nopep8 89 | nonzero_r = (norm_r != 0) 90 | cost[wrong*nonzero_r] /= norm_r[wrong*nonzero_r] 91 | 92 | cost = cost.mean() 93 | 94 | # Update adversarial images 95 | grad = torch.autograd.grad(cost, adv_images, 96 | retain_graph=False, create_graph=False)[0] 97 | 98 | adv_images = adv_images.detach() + self.alpha*grad.sign() 99 | delta = torch.clamp(adv_images-images, min=-self.eps, max=self.eps) # nopep8 100 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 101 | 102 | return adv_images 103 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/mifgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class MIFGSM(Attack): 8 | r""" 9 | MI-FGSM in the paper 'Boosting Adversarial Attacks with Momentum' 10 | [https://arxiv.org/abs/1710.06081] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 8/255) 17 | alpha (float): step size. (Default: 2/255) 18 | decay (float): momentum factor. (Default: 1.0) 19 | steps (int): number of iterations. (Default: 10) 20 | 21 | Shape: 22 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 23 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 24 | - output: :math:`(N, C, H, W)`. 25 | 26 | Examples:: 27 | >>> attack = torchattacks.MIFGSM(model, eps=8/255, steps=10, decay=1.0) 28 | >>> adv_images = attack(images, labels) 29 | 30 | """ 31 | 32 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, decay=1.0): 33 | super().__init__('MIFGSM', model, device) 34 | self.eps = eps 35 | self.steps = steps 36 | self.decay = decay 37 | self.alpha = alpha 38 | self.supported_mode = ['default', 'targeted'] 39 | 40 | def forward(self, images, labels): 41 | r""" 42 | Overridden. 43 | """ 44 | 45 | images = images.clone().detach().to(self.device) 46 | labels = labels.clone().detach().to(self.device) 47 | 48 | if self.targeted: 49 | target_labels = self.get_target_label(images, labels) 50 | 51 | momentum = torch.zeros_like(images).detach().to(self.device) 52 | 53 | loss = nn.CrossEntropyLoss() 54 | 55 | adv_images = images.clone().detach() 56 | 57 | for _ in range(self.steps): 58 | adv_images.requires_grad = True 59 | outputs = self.get_logits(adv_images) 60 | 61 | # Calculate loss 62 | if self.targeted: 63 | cost = -loss(outputs, target_labels) 64 | else: 65 | cost = loss(outputs, labels) 66 | 67 | # Update adversarial images 68 | grad = torch.autograd.grad(cost, adv_images, 69 | retain_graph=False, create_graph=False)[0] 70 | 71 | grad = grad / torch.mean(torch.abs(grad), 72 | dim=(1, 2, 3), keepdim=True) 73 | grad = grad + momentum*self.decay 74 | momentum = grad 75 | 76 | adv_images = adv_images.detach() + self.alpha*grad.sign() 77 | delta = torch.clamp(adv_images - images, 78 | min=-self.eps, max=self.eps) 79 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 80 | 81 | return adv_images 82 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/nifgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | class NIFGSM(Attack): 6 | r""" 7 | NI-FGSM in the paper 'NESTEROV ACCELERATED GRADIENT AND SCALEINVARIANCE FOR ADVERSARIAL ATTACKS' 8 | [https://arxiv.org/abs/1908.06281], Published as a conference paper at ICLR 2020 9 | 10 | Distance Measure : Linf 11 | 12 | Arguments: 13 | model (nn.Module): model to attack. 14 | eps (float): maximum perturbation. (Default: 8/255) 15 | alpha (float): step size. (Default: 2/255) 16 | decay (float): momentum factor. (Default: 1.0) 17 | steps (int): number of iterations. (Default: 10) 18 | 19 | Shape: 20 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 21 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 22 | - output: :math:`(N, C, H, W)`. 23 | 24 | Examples:: 25 | >>> attack = torchattacks.NIFGSM(model, eps=8/255, alpha=2/255, steps=10, decay=1.0) 26 | >>> adv_images = attack(images, labels) 27 | 28 | """ 29 | 30 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, decay=1.0): 31 | super().__init__('NIFGSM', model, device) 32 | self.eps = eps 33 | self.steps = steps 34 | self.decay = decay 35 | self.alpha = alpha 36 | self.supported_mode = ['default', 'targeted'] 37 | 38 | def forward(self, images, labels): 39 | r""" 40 | Overridden. 41 | """ 42 | 43 | images = images.clone().detach().to(self.device) 44 | labels = labels.clone().detach().to(self.device) 45 | 46 | if self.targeted: 47 | target_labels = self.get_target_label(images, labels) 48 | 49 | momentum = torch.zeros_like(images).detach().to(self.device) 50 | 51 | loss = nn.CrossEntropyLoss() 52 | 53 | adv_images = images.clone().detach() 54 | 55 | for _ in range(self.steps): 56 | adv_images.requires_grad = True 57 | nes_images = adv_images + self.decay*self.alpha*momentum 58 | outputs = self.get_logits(nes_images) 59 | # Calculate loss 60 | if self.targeted: 61 | cost = -loss(outputs, target_labels) 62 | else: 63 | cost = loss(outputs, labels) 64 | 65 | # Update adversarial images 66 | grad = torch.autograd.grad(cost, adv_images, 67 | retain_graph=False, create_graph=False)[0] 68 | grad = self.decay*momentum + grad / torch.mean(torch.abs(grad), dim=(1,2,3), keepdim=True) 69 | momentum = grad 70 | adv_images = adv_images.detach() + self.alpha*grad.sign() 71 | delta = torch.clamp(adv_images - images, min=-self.eps, max=self.eps) 72 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 73 | 74 | return adv_images 75 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/onepixel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | 6 | from ..attack import Attack 7 | from ._differential_evolution import differential_evolution 8 | 9 | 10 | class OnePixel(Attack): 11 | r""" 12 | Attack in the paper 'One pixel attack for fooling deep neural networks' 13 | [https://arxiv.org/abs/1710.08864] 14 | 15 | Modified from "https://github.com/DebangLi/one-pixel-attack-pytorch/" and 16 | "https://github.com/sarathknv/adversarial-examples-pytorch/blob/master/one_pixel_attack/" 17 | 18 | Distance Measure : L0 19 | 20 | Arguments: 21 | model (nn.Module): model to attack. 22 | pixels (int): number of pixels to change (Default: 1) 23 | steps (int): number of steps. (Default: 10) 24 | popsize (int): population size, i.e. the number of candidate agents or "parents" in differential evolution (Default: 10) 25 | inf_batch (int): maximum batch size during inference (Default: 128) 26 | 27 | Shape: 28 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 29 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 30 | - output: :math:`(N, C, H, W)`. 31 | 32 | Examples:: 33 | >>> attack = torchattacks.OnePixel(model, pixels=1, steps=10, popsize=10, inf_batch=128) 34 | >>> adv_images = attack(images, labels) 35 | 36 | """ 37 | 38 | def __init__(self, model, device=None, pixels=1, steps=10, popsize=10, inf_batch=128): 39 | super().__init__('OnePixel', model, device) 40 | self.pixels = pixels 41 | self.steps = steps 42 | self.popsize = popsize 43 | self.inf_batch = inf_batch 44 | self.supported_mode = ['default', 'targeted'] 45 | 46 | def forward(self, images, labels): 47 | r""" 48 | Overridden. 49 | """ 50 | 51 | images = images.clone().detach().to(self.device) 52 | labels = labels.clone().detach().to(self.device) 53 | 54 | if self.targeted: 55 | target_labels = self.get_target_label(images, labels) 56 | 57 | batch_size, channel, height, width = images.shape 58 | 59 | bounds = [(0, height), (0, width)]+[(0, 1)]*channel 60 | bounds = bounds*self.pixels 61 | 62 | popmul = max(1, int(self.popsize/len(bounds))) 63 | 64 | adv_images = [] 65 | for idx in range(batch_size): 66 | image, label = images[idx:idx+1], labels[idx:idx+1] 67 | 68 | if self.targeted: 69 | target_label = target_labels[idx:idx+1] 70 | 71 | def func(delta): 72 | return self._loss(image, target_label, delta) 73 | 74 | def callback(delta, convergence): 75 | return self._attack_success(image, target_label, delta) 76 | 77 | else: 78 | def func(delta): 79 | return self._loss(image, label, delta) 80 | 81 | def callback(delta, convergence): 82 | return self._attack_success(image, label, delta) 83 | 84 | delta = differential_evolution(func=func, 85 | bounds=bounds, 86 | callback=callback, 87 | maxiter=self.steps, popsize=popmul, 88 | init='random', 89 | recombination=1, atol=-1, 90 | polish=False).x 91 | delta = np.split(delta, len(delta)/len(bounds)) 92 | adv_image = self._perturb(image, delta) 93 | adv_images.append(adv_image) 94 | 95 | adv_images = torch.cat(adv_images) 96 | return adv_images 97 | 98 | def _loss(self, image, label, delta): 99 | adv_images = self._perturb(image, delta) # Mutiple delta 100 | prob = self._get_prob(adv_images)[:, label] 101 | if self.targeted: 102 | return 1-prob # If targeted, increase prob 103 | else: 104 | return prob # If non-targeted, decrease prob 105 | 106 | def _attack_success(self, image, label, delta): 107 | adv_image = self._perturb(image, delta) # Single delta 108 | prob = self._get_prob(adv_image) 109 | pre = np.argmax(prob) 110 | if self.targeted and (pre == label): 111 | return True 112 | elif (not self.targeted) and (pre != label): 113 | return True 114 | return False 115 | 116 | def _get_prob(self, images): 117 | with torch.no_grad(): 118 | batches = torch.split(images, self.inf_batch) 119 | outs = [] 120 | for batch in batches: 121 | out = self.get_logits(batch) 122 | outs.append(out) 123 | outs = torch.cat(outs) 124 | prob = F.softmax(outs, dim=1) 125 | return prob.detach().cpu().numpy() 126 | 127 | def _perturb(self, image, delta): 128 | delta = np.array(delta) 129 | if len(delta.shape) < 2: 130 | delta = np.array([delta]) 131 | num_delta = len(delta) 132 | adv_image = image.clone().detach().to(self.device) 133 | adv_images = torch.cat([adv_image]*num_delta, dim=0) 134 | for idx in range(num_delta): 135 | pixel_info = delta[idx].reshape(self.pixels, -1) 136 | for pixel in pixel_info: 137 | pos_x, pos_y = pixel[:2] 138 | channel_v = pixel[2:] 139 | for channel, v in enumerate(channel_v): 140 | adv_images[idx, channel, int(pos_x), int(pos_y)] = v 141 | return adv_images 142 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/pgd.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | from .protect import * 6 | 7 | class PGD(Attack): 8 | r""" 9 | PGD in the paper 'Towards Deep Learning Models Resistant to Adversarial Attacks' 10 | [https://arxiv.org/abs/1706.06083] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 8/255) 17 | alpha (float): step size. (Default: 2/255) 18 | steps (int): number of steps. (Default: 10) 19 | random_start (bool): using random initialization of delta. (Default: True) 20 | 21 | Shape: 22 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 23 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 24 | - output: :math:`(N, C, H, W)`. 25 | 26 | Examples:: 27 | >>> attack = torchattacks.PGD(model, eps=8/255, alpha=1/255, steps=10, random_start=True) 28 | >>> adv_images = attack(images, labels) 29 | 30 | """ 31 | 32 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, random_start=False): 33 | super().__init__('PGD', model, device) 34 | self.eps = eps 35 | self.alpha = alpha 36 | self.steps = steps 37 | self.random_start = random_start 38 | self.supported_mode = ['default', 'targeted'] 39 | 40 | def forward(self, images, labels, target_labels): 41 | r""" 42 | Overridden. 43 | """ 44 | 45 | images = images.clone().detach().to(self.device) 46 | labels = labels.clone().detach().to(self.device) 47 | 48 | loss = nn.CrossEntropyLoss() 49 | adv_images = images.clone().detach() 50 | 51 | if self.random_start: 52 | # Starting at a uniformly random point 53 | adv_images = adv_images + \ 54 | torch.empty_like(adv_images).uniform_(-self.eps, self.eps) 55 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 56 | 57 | for _ in range(self.steps): 58 | adv_images.requires_grad = True 59 | self.model = reset_model('./checkpoints/epoch4_model.pt', protect=True) 60 | outputs = self.get_logits(adv_images) 61 | 62 | # Calculate loss 63 | cost = -loss(outputs, target_labels) 64 | 65 | # Update adversarial images 66 | grad = torch.autograd.grad(cost, adv_images, 67 | retain_graph=False, create_graph=False)[0] 68 | 69 | adv_images = adv_images.detach() + self.alpha * grad.sign() 70 | 71 | return adv_images 72 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/pgdl2.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class PGDL2(Attack): 8 | r""" 9 | PGD in the paper 'Towards Deep Learning Models Resistant to Adversarial Attacks' 10 | [https://arxiv.org/abs/1706.06083] 11 | 12 | Distance Measure : L2 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): maximum perturbation. (Default: 1.0) 17 | alpha (float): step size. (Default: 0.2) 18 | steps (int): number of steps. (Default: 10) 19 | random_start (bool): using random initialization of delta. (Default: True) 20 | 21 | Shape: 22 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 23 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 24 | - output: :math:`(N, C, H, W)`. 25 | 26 | Examples:: 27 | >>> attack = torchattacks.PGDL2(model, eps=1.0, alpha=0.2, steps=10, random_start=True) 28 | >>> adv_images = attack(images, labels) 29 | 30 | """ 31 | 32 | def __init__(self, model, device=None, eps=1.0, alpha=0.2, steps=10, random_start=True, eps_for_division=1e-10): 33 | super().__init__('PGDL2', model, device) 34 | self.eps = eps 35 | self.alpha = alpha 36 | self.steps = steps 37 | self.random_start = random_start 38 | self.eps_for_division = eps_for_division 39 | self.supported_mode = ['default', 'targeted'] 40 | 41 | def forward(self, images, labels, target_labels): 42 | r""" 43 | Overridden. 44 | """ 45 | 46 | images = images.clone().detach().to(self.device) 47 | labels = labels.clone().detach().to(self.device) 48 | loss = nn.CrossEntropyLoss() 49 | 50 | adv_images = images.clone().detach() 51 | batch_size = len(images) 52 | 53 | if self.random_start: 54 | # Starting at a uniformly random point 55 | delta = torch.empty_like(adv_images).normal_() 56 | d_flat = delta.view(adv_images.size(0), -1) 57 | n = d_flat.norm(p=2, dim=1).view(adv_images.size(0), 1, 1, 1) 58 | r = torch.zeros_like(n).uniform_(0, 1) 59 | delta *= r/n*self.eps 60 | adv_images = torch.clamp(adv_images + delta, min=0, max=1).detach() 61 | 62 | for _ in range(self.steps): 63 | adv_images.requires_grad = True 64 | outputs = self.get_logits(adv_images) 65 | 66 | # Calculate loss 67 | cost = -loss(outputs, target_labels) 68 | 69 | # Update adversarial images 70 | grad = torch.autograd.grad(cost, adv_images, 71 | retain_graph=False, create_graph=False)[0] 72 | grad_norms = torch.norm(grad.view(batch_size, -1), p=2, dim=1) + self.eps_for_division # nopep8 73 | grad = grad / grad_norms.view(batch_size, 1, 1, 1) 74 | adv_images = adv_images.detach() + self.alpha * grad 75 | 76 | delta = adv_images - images 77 | delta_norms = torch.norm(delta.view(batch_size, -1), p=2, dim=1) 78 | factor = self.eps / delta_norms 79 | factor = torch.min(factor, torch.ones_like(delta_norms)) 80 | delta = delta * factor.view(-1, 1, 1, 1) 81 | 82 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 83 | 84 | return adv_images 85 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/pgdrs.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import copy 4 | 5 | from ..attack import Attack 6 | 7 | 8 | class Noise(): 9 | def __init__(self, noise_type, noise_sd): 10 | self.noise_type = noise_type 11 | self.noise_sd = noise_sd 12 | 13 | def __call__(self, img): 14 | if self.noise_type == "guassian": 15 | noise = torch.randn_like(img.float())*self.noise_sd 16 | elif self.noise_type == "uniform": 17 | noise = (torch.rand_like(img.float()) - 0.5)*2*self.noise_sd 18 | return noise 19 | 20 | 21 | class PGDRS(Attack): 22 | r""" 23 | PGD for randmized smoothing in the paper 'Provably Robust Deep Learning via Adversarially Trained Smoothed Classifiers' 24 | [https://arxiv.org/abs/1906.04584] 25 | Modification of the code from https://github.com/Hadisalman/smoothing-adversarial/blob/master/code/attacks.py 26 | 27 | Distance Measure : Linf 28 | 29 | Arguments: 30 | model (nn.Module): model to attack. 31 | eps (float): maximum perturbation. (Default: 8/255) 32 | alpha (float): step size. (Default: 2/255) 33 | steps (int): number of steps. (Default: 40) 34 | noise_type (str): guassian or uniform. (Default: guassian) 35 | noise_sd (float): standard deviation for normal distributio, or range for . (Default: 0.5) 36 | noise_batch_size (int): guassian or uniform. (Default: 5) 37 | batch_max (int): split data into small chunk if the total number of augmented data points, len(inputs)*noise_batch_size, are larger than batch_max, in case GPU memory is insufficient. (Default: 2048) 38 | 39 | Shape: 40 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 41 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 42 | - output: :math:`(N, C, H, W)`. 43 | 44 | Examples:: 45 | >>> attack = torchattacks.PGDRS(model, eps=8/255, alpha=2/255, steps=10, noise_type="guassian", noise_sd=0.5, noise_batch_size=5, batch_max=2048) 46 | >>> adv_images = attack(images, labels) 47 | 48 | """ 49 | 50 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, noise_type="guassian", noise_sd=0.5, noise_batch_size=5, batch_max=2048): 51 | super().__init__('PGDRS', model, device) 52 | self.eps = eps 53 | self.alpha = alpha 54 | self.steps = steps 55 | self.noise_func = Noise(noise_type, noise_sd) 56 | self.noise_batch_size = noise_batch_size 57 | self.supported_mode = ['default', 'targeted'] 58 | self.batch_max = batch_max 59 | 60 | def forward(self, inputs: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: 61 | if inputs.shape[0]*self.noise_batch_size > self.batch_max: 62 | split_num = int(self.batch_max/self.noise_batch_size) 63 | inputs_split = torch.split(inputs, 64 | split_size_or_sections=split_num) 65 | labels_split = torch.split(labels, 66 | split_size_or_sections=split_num) 67 | img_list = [] 68 | for img_sub, lab_sub in zip(inputs_split, labels_split): 69 | img_adv = self._forward(img_sub, lab_sub) 70 | img_list.append(img_adv) 71 | return torch.vstack(img_list) 72 | else: 73 | return self._forward(inputs, labels) 74 | 75 | def _forward(self, images, labels): 76 | r""" 77 | Overridden. 78 | """ 79 | 80 | images = images.clone().detach().to(self.device) 81 | labels = labels.clone().detach().to(self.device) 82 | # expend the inputs over noise_batch_size 83 | shape = torch.Size([images.shape[0], self.noise_batch_size]) + images.shape[1:] # nopep8 84 | inputs_exp = images.unsqueeze(1).expand(shape) 85 | inputs_exp = inputs_exp.reshape(torch.Size([-1]) + inputs_exp.shape[2:]) # nopep8 86 | 87 | delta = torch.zeros( 88 | (len(labels), *inputs_exp.shape[1:]), requires_grad=True, device=self.device) 89 | delta_last = torch.zeros( 90 | (len(labels), *inputs_exp.shape[1:]), requires_grad=False, device=self.device) 91 | 92 | if self.targeted: 93 | target_labels = self.get_target_label(images, labels) 94 | 95 | for _ in range(self.steps): 96 | delta.requires_grad = True 97 | # img_adv is the perturbed data for randmized smoothing 98 | # delta.repeat(1,self.noise_batch_size,1,1).view_as(inputs_exp) 99 | img_adv = inputs_exp + delta.unsqueeze(1).repeat((1, self.noise_batch_size, 1, 1, 1)).view_as(inputs_exp) # nopep8 100 | img_adv = torch.clamp(img_adv, min=0, max=1) 101 | 102 | noise_added = self.noise_func(img_adv.view(len(img_adv), -1)) 103 | noise_added = noise_added.view(img_adv.shape) 104 | 105 | adv_noise = img_adv + noise_added 106 | logits = self.get_logits(adv_noise) 107 | 108 | softmax = F.softmax(logits, dim=1) 109 | # average the probabilities across noise 110 | average_softmax = softmax.reshape( 111 | -1, self.noise_batch_size, logits.shape[-1]).mean(1, keepdim=True).squeeze(1) 112 | logsoftmax = torch.log(average_softmax.clamp(min=1e-20)) 113 | ce_loss = F.nll_loss( 114 | logsoftmax, labels) if not self.targeted else -F.nll_loss(logsoftmax, target_labels) 115 | 116 | grad = torch.autograd.grad( 117 | ce_loss, delta, retain_graph=False, create_graph=False)[0] 118 | delta = delta_last + self.alpha*torch.sign(grad) 119 | delta = torch.clamp(delta, min=-self.eps, max=self.eps) 120 | delta_last.data = copy.deepcopy(delta.data) 121 | 122 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 123 | return adv_images 124 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/pgdrsl2.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | from ..attack import Attack 5 | import copy 6 | 7 | 8 | class Noise(): 9 | def __init__(self, noise_type, noise_sd): 10 | self.noise_type = noise_type 11 | self.noise_sd = noise_sd 12 | 13 | def __call__(self, img): 14 | if self.noise_type == "guassian": 15 | noise = torch.randn_like(img.float())*self.noise_sd 16 | elif self.noise_type == "uniform": 17 | noise = (torch.rand_like(img.float()) - 0.5)*2*self.noise_sd 18 | return noise 19 | 20 | 21 | class PGDRSL2(Attack): 22 | r""" 23 | PGD for randmized smoothing in the paper 'Provably Robust Deep Learning via Adversarially Trained Smoothed Classifiers' 24 | [https://arxiv.org/abs/1906.04584] 25 | Modification of the code from https://github.com/Hadisalman/smoothing-adversarial/blob/master/code/attacks.py 26 | 27 | Distance Measure : L2 28 | 29 | Arguments: 30 | model (nn.Module): model to attack. 31 | eps (float): maximum perturbation. (Default: 1.0) 32 | alpha (float): step size. (Default: 0.2) 33 | steps (int): number of steps. (Default: 10) 34 | noise_type (str): guassian or uniform. (Default: guassian) 35 | noise_sd (float): standard deviation for normal distributio, or range for . (Default: 0.5) 36 | noise_batch_size (int): guassian or uniform. (Default: 5) 37 | batch_max (int): split data into small chunk if the total number of augmented data points, len(inputs)*noise_batch_size, are larger than batch_max, in case GPU memory is insufficient. (Default: 2048) 38 | random_start (bool): using random initialization of delta. (Default: True) 39 | 40 | Shape: 41 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 42 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 43 | - output: :math:`(N, C, H, W)`. 44 | 45 | Examples:: 46 | >>> attack = torchattacks.PGDRSL2(model, eps=1.0, alpha=0.2, steps=10, noise_type="guassian", noise_sd=0.5, noise_batch_size=5, batch_max=2048) 47 | >>> adv_images = attack(images, labels) 48 | 49 | """ 50 | 51 | def __init__(self, model, device=None, eps=1.0, alpha=0.2, steps=10, noise_type="guassian", noise_sd=0.5, noise_batch_size=5, batch_max=2048, eps_for_division=1e-10): 52 | super().__init__('PGDRSL2', model, device) 53 | self.eps = eps 54 | self.alpha = alpha 55 | self.steps = steps 56 | self.noise_func = Noise(noise_type, noise_sd) 57 | self.noise_batch_size = noise_batch_size 58 | self.eps_for_division = eps_for_division 59 | self.supported_mode = ['default', 'targeted'] 60 | self.batch_max = batch_max 61 | 62 | def forward(self, inputs: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: 63 | if inputs.shape[0]*self.noise_batch_size > self.batch_max: 64 | img_list = [] 65 | split_num = int(self.batch_max/self.noise_batch_size) 66 | inputs_split = torch.split(inputs, 67 | split_size_or_sections=split_num) 68 | labels_split = torch.split(labels, 69 | split_size_or_sections=split_num) 70 | for img_sub, lab_sub in zip(inputs_split, labels_split): 71 | img_adv = self._forward(img_sub, lab_sub) 72 | img_list.append(img_adv) 73 | return torch.vstack(img_list) 74 | else: 75 | return self._forward(inputs, labels) 76 | 77 | def _forward(self, images, labels): 78 | r""" 79 | Overridden. 80 | """ 81 | 82 | images = images.clone().detach().to(self.device) 83 | labels = labels.clone().detach().to(self.device) 84 | # expend the inputs over noise_batch_size 85 | shape = torch.Size([images.shape[0], self.noise_batch_size]) + images.shape[1:] # nopep8 86 | inputs_exp = images.unsqueeze(1).expand(shape) 87 | inputs_exp = inputs_exp.reshape(torch.Size([-1]) + inputs_exp.shape[2:]) # nopep8 88 | 89 | data_batch_size = labels.shape[0] 90 | delta = torch.zeros( 91 | (len(labels), *inputs_exp.shape[1:]), requires_grad=True, device=self.device) 92 | delta_last = torch.zeros( 93 | (len(labels), *inputs_exp.shape[1:]), requires_grad=False, device=self.device) 94 | 95 | if self.targeted: 96 | target_labels = self.get_target_label(images, labels) 97 | 98 | for _ in range(self.steps): 99 | delta.requires_grad = True 100 | # img_adv is the perturbed data for randmized smoothing 101 | # delta.repeat(1,self.noise_batch_size,1,1) 102 | img_adv = inputs_exp + delta.unsqueeze(1).repeat((1, self.noise_batch_size, 1, 1, 1)).view_as(inputs_exp) # nopep8 103 | img_adv = torch.clamp(img_adv, min=0, max=1) 104 | 105 | noise_added = self.noise_func(img_adv.view(len(img_adv), -1)) 106 | noise_added = noise_added.view(img_adv.shape) 107 | 108 | adv_noise = img_adv + noise_added 109 | logits = self.get_logits(adv_noise) 110 | 111 | softmax = F.softmax(logits, dim=1) 112 | # average the probabilities across noise 113 | average_softmax = softmax.reshape( 114 | -1, self.noise_batch_size, logits.shape[-1]).mean(1, keepdim=True).squeeze(1) 115 | logsoftmax = torch.log(average_softmax.clamp(min=1e-20)) 116 | ce_loss = F.nll_loss( 117 | logsoftmax, labels) if not self.targeted else -F.nll_loss(logsoftmax, target_labels) 118 | 119 | grad = torch.autograd.grad( 120 | ce_loss, delta, retain_graph=False, create_graph=False)[0] 121 | grad_norms = torch.norm( 122 | grad.view(data_batch_size, -1), p=2, dim=1) + self.eps_for_division 123 | grad = grad / grad_norms.view(data_batch_size, 1, 1, 1) 124 | 125 | delta = delta_last + self.alpha*grad 126 | delta_norms = torch.norm(delta.view( 127 | data_batch_size, -1), p=2, dim=1) 128 | factor = self.eps / delta_norms 129 | factor = torch.min(factor, torch.ones_like(delta_norms)) 130 | delta = delta * factor.view(-1, 1, 1, 1) 131 | delta_last.data = copy.deepcopy(delta.data) 132 | 133 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 134 | return adv_images 135 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/pifgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import numpy as np 4 | 5 | from ..attack import Attack 6 | 7 | 8 | class PIFGSM(Attack): 9 | r""" 10 | PIFGSM in the paper 'Patch-wise Attack for Fooling Deep Neural Network' 11 | [https://arxiv.org/abs/2007.06765] 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | max_epsilon (float): maximum size of adversarial perturbation. (Default: 16/255) 18 | num_iter_set (float): number of iterations. (Default: 10) 19 | momentum (float): momentum. (Default: 1.0) 20 | amplification (float): to amplifythe step size. (Default: 10.0) 21 | prob (float): probability of using diverse inputs. (Default: 0.7) 22 | 23 | Shape: 24 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 25 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 26 | - output: :math:`(N, C, H, W)`. 27 | 28 | Examples:: 29 | >>> attack = torchattacks.PIFGSM(model, eps=16/255, num_iter_set=10) 30 | >>> adv_images = attack(images, labels) 31 | 32 | """ 33 | 34 | def __init__(self, model, device=None, max_epsilon=16/255, num_iter_set=10, momentum=1.0, amplification=10.0, prob=0.7): 35 | super().__init__('PIFGSM', model, device) 36 | self.max_epsilon = max_epsilon 37 | self.num_iter_set = num_iter_set 38 | self.momentum = momentum 39 | self.amplification = amplification 40 | self.prob = prob 41 | self.supported_mode = ['default'] 42 | 43 | def forward(self, images, labels): 44 | r""" 45 | Overridden. 46 | """ 47 | 48 | images = images.clone().detach().to(self.device) 49 | labels = labels.clone().detach().to(self.device) 50 | 51 | images_min = self.clip_by_tensor( 52 | images-self.max_epsilon, t_min=0, t_max=1) 53 | images_max = self.clip_by_tensor( 54 | images+self.max_epsilon, t_min=0, t_max=1) 55 | adv_images = self.graph(images, labels, images_min, images_max) 56 | 57 | return adv_images 58 | 59 | def clip_by_tensor(self, t, t_min, t_max): 60 | result = (t >= t_min).float() * t + (t < t_min).float() * t_min 61 | result = (result <= t_max).float() * result + \ 62 | (result > t_max).float() * t_max 63 | return result 64 | 65 | def project_noise(self, images, P_kern, padding_size): 66 | images = F.conv2d(images, P_kern, padding=( 67 | padding_size, padding_size), groups=3) 68 | return images 69 | 70 | def project_kern(self, kern_size): 71 | kern = np.ones((kern_size, kern_size), 72 | dtype=np.float32) / (kern_size ** 2 - 1) 73 | kern[kern_size // 2, kern_size // 2] = 0.0 74 | kern = kern.astype(np.float32) 75 | stack_kern = np.stack([kern, kern, kern]) 76 | stack_kern = np.expand_dims(stack_kern, 1) 77 | stack_kern = torch.tensor(stack_kern).to(self.device) 78 | return stack_kern, kern_size // 2 79 | 80 | def graph(self, images, labels, images_min, images_max): 81 | eps = self.max_epsilon 82 | num_iter = self.num_iter_set 83 | alpha = eps / num_iter 84 | alpha_beta = alpha * self.amplification 85 | gamma = alpha_beta 86 | P_kern, padding_size = self.project_kern(3) 87 | 88 | images.requires_grad = True 89 | amplification = 0.0 90 | for _ in range(num_iter): 91 | # zero_gradients(images) 92 | if images.grad is not None: 93 | images.grad.detach_() 94 | images.grad.zero_() 95 | 96 | output_v3 = self.get_logits(images) 97 | loss = F.cross_entropy(output_v3, labels) 98 | loss.backward() 99 | noise = images.grad.data 100 | 101 | amplification += alpha_beta * torch.sign(noise) 102 | cut_noise = torch.clamp( 103 | abs(amplification) - eps, 0, 10000.0) * torch.sign(amplification) 104 | projection = gamma * \ 105 | torch.sign(self.project_noise( 106 | cut_noise, P_kern, padding_size)) 107 | amplification += projection 108 | 109 | images = images + alpha_beta * torch.sign(noise) + projection 110 | images = self.clip_by_tensor(images, images_min, images_max) 111 | images = images.detach().requires_grad_(True) 112 | 113 | return images.detach() 114 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/pifgsmplusplus.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import numpy as np 4 | 5 | from ..attack import Attack 6 | 7 | 8 | class PIFGSMPLUSPLUS(Attack): 9 | r""" 10 | Patch-wise++ Perturbation for Adversarial Targeted Attacks' 11 | [https://arxiv.org/abs/2012.15503] 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | max_epsilon (float): maximum size of adversarial perturbation. (Default: 16/255) 18 | num_iter_set (float): number of iterations. (Default: 10) 19 | momentum (float): momentum. (Default: 1.0) 20 | amplification (float): to amplifythe step size. (Default: 10.0) 21 | prob (float): probability of using diverse inputs. (Default: 0.7) 22 | project_factor (float): To control the weight of project term. (Default: 0.8) 23 | 24 | Shape: 25 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 26 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 27 | - output: :math:`(N, C, H, W)`. 28 | 29 | Examples:: 30 | >>> attack = torchattacks.PIFGSMPLUSPLUS(model, eps=16/255, num_iter_set=10) 31 | >>> adv_images = attack(images, labels) 32 | 33 | """ 34 | 35 | def __init__(self, model, device=None, max_epsilon=16/255, num_iter_set=10, momentum=1.0, amplification=10.0, prob=0.7, project_factor=0.8): 36 | super().__init__('PIFGSMPLUSPLUS', model, device) 37 | self.max_epsilon = max_epsilon 38 | self.num_iter_set = num_iter_set 39 | self.momentum = momentum 40 | self.amplification = amplification 41 | self.prob = prob 42 | self.project_factor = project_factor 43 | self.supported_mode = ['default', 'targeted'] 44 | 45 | def forward(self, images, labels): 46 | r""" 47 | Overridden. 48 | """ 49 | 50 | images = images.clone().detach().to(self.device) 51 | labels = labels.clone().detach().to(self.device) 52 | 53 | images_min = self.clip_by_tensor( 54 | images-self.max_epsilon, t_min=0, t_max=1) 55 | images_max = self.clip_by_tensor( 56 | images+self.max_epsilon, t_min=0, t_max=1) 57 | adv_images = self.graph(images, labels, images_min, images_max) 58 | 59 | return adv_images 60 | 61 | def clip_by_tensor(self, t, t_min, t_max): 62 | result = (t >= t_min).float() * t + (t < t_min).float() * t_min 63 | result = (result <= t_max).float() * result + \ 64 | (result > t_max).float() * t_max 65 | return result 66 | 67 | def project_noise(self, images, P_kern, padding_size): 68 | images = F.conv2d(images, P_kern, padding=( 69 | padding_size, padding_size), groups=3) 70 | return images 71 | 72 | def gaussian_kern(self, kernlen=21, nsig=3): 73 | """Returns a 2D Gaussian kernel array.""" 74 | import scipy.stats as st 75 | 76 | x = np.linspace(-nsig, nsig, kernlen) 77 | kern1d = st.norm.pdf(x) 78 | kernel_raw = np.outer(kern1d, kern1d) 79 | kernel = kernel_raw / kernel_raw.sum() 80 | kernel = kernel.astype(np.float32) 81 | stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0) 82 | stack_kernel = np.expand_dims(stack_kernel, 1) 83 | stack_kernel = torch.tensor(stack_kernel).to(self.device) 84 | return stack_kernel 85 | 86 | def project_kern(self, kern_size): 87 | kern = np.ones((kern_size, kern_size), 88 | dtype=np.float32) / (kern_size ** 2 - 1) 89 | kern[kern_size // 2, kern_size // 2] = 0.0 90 | kern = kern.astype(np.float32) 91 | stack_kern = np.stack([kern, kern, kern]) 92 | stack_kern = np.expand_dims(stack_kern, 1) 93 | stack_kern = torch.tensor(stack_kern).to(self.device) 94 | return stack_kern, kern_size // 2 95 | 96 | def graph(self, images, labels, images_min, images_max): 97 | eps = self.max_epsilon 98 | num_iter = self.num_iter_set 99 | alpha = eps / num_iter 100 | alpha_beta = alpha * self.amplification 101 | gamma = alpha_beta * self.project_factor 102 | P_kern, padding_size = self.project_kern(3) 103 | T_kern = self.gaussian_kern(3, 3) 104 | 105 | images.requires_grad = True 106 | amplification = 0.0 107 | for _ in range(num_iter): 108 | # zero_gradients(images) 109 | if images.grad is not None: 110 | images.grad.detach_() 111 | images.grad.zero_() 112 | 113 | output_v3 = self.get_logits(images) 114 | loss = F.cross_entropy(output_v3, labels) 115 | loss.backward() 116 | noise = images.grad.data 117 | noise = F.conv2d(noise, T_kern, padding=( 118 | padding_size, padding_size), groups=3) 119 | 120 | amplification += alpha_beta * torch.sign(noise) 121 | cut_noise = torch.clamp( 122 | abs(amplification) - eps, 0, 10000.0) * torch.sign(amplification) 123 | projection = gamma * \ 124 | torch.sign(self.project_noise( 125 | cut_noise, P_kern, padding_size)) 126 | 127 | if self.targeted: 128 | images = images - alpha_beta * torch.sign(noise) - projection 129 | else: 130 | images = images + alpha_beta * torch.sign(noise) + projection 131 | 132 | images = self.clip_by_tensor(images, images_min, images_max) 133 | images = images.detach().requires_grad_(True) 134 | 135 | return images.detach() 136 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/protect.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | from clip.model import LayerNorm, OrderedDict, ResidualAttentionBlock, QuickGELU 5 | import torchvision.transforms as transforms 6 | import clip 7 | 8 | class QuantizeRelu(nn.Module): 9 | # Assuming you've defined this somewhere 10 | def __init__(self, step_size=0.05): 11 | super(QuantizeRelu, self).__init__() 12 | self.step_size = step_size 13 | 14 | def forward(self, x): 15 | return torch.mul(torch.floor(x / self.step_size), self.step_size) 16 | 17 | 18 | class QuantizeGelu(nn.Module): 19 | def __init__(self, step_size=0.05): 20 | super(QuantizeGelu, self).__init__() 21 | self.step_size = step_size 22 | self.qrelu = QuantizeRelu(step_size) 23 | self.qrelu.register_backward_hook(self.zero_gradients_hook) 24 | 25 | def forward(self, x): 26 | out = self.qrelu(x.half()) 27 | out = out * torch.sigmoid(1.702 * out) 28 | return out 29 | 30 | def zero_gradients_hook(self, module, grad_input, grad_output): 31 | # Returns a new gradient tuple with all gradients set to zero 32 | return tuple(torch.zeros_like(g) if g is not None else None for g in grad_input) 33 | 34 | 35 | class ResidualAttentionBlockQuantize(nn.Module): 36 | def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): 37 | super().__init__() 38 | 39 | self.attn = nn.MultiheadAttention(d_model, n_head).to(dtype=torch.half) # Convert to 'half' data type 40 | self.ln_1 = LayerNorm(d_model) 41 | self.mlp = nn.Sequential(OrderedDict([ 42 | ("c_fc", nn.Linear(d_model, d_model * 4).to(dtype=torch.half)), # Convert to 'half' data type 43 | ("gelu", QuantizeGelu()), 44 | ("c_proj", nn.Linear(d_model * 4, d_model).to(dtype=torch.half)) # Convert to 'half' data type 45 | ])) 46 | self.ln_2 = LayerNorm(d_model) 47 | self.attn_mask = attn_mask 48 | self.qrelu = QuantizeRelu() 49 | 50 | # Registering the backward hook to zero out gradients 51 | self.register_backward_hook(self.zero_gradients_hook) 52 | 53 | def attention(self, x: torch.Tensor): 54 | self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None 55 | return self.attn(x.half(), x.half(), x.half(), need_weights=False, attn_mask=self.attn_mask)[ 56 | 0] # Convert to 'half' data type 57 | 58 | def forward(self, x: torch.Tensor): 59 | x = x + self.attention(self.ln_1(x)) 60 | x = self.qrelu(x.half()) + self.mlp(self.ln_2(x)) 61 | return x 62 | 63 | def zero_gradients_hook(self, module, grad_input, grad_output): 64 | # Returns a new gradient tuple with all gradients set to zero 65 | return tuple(torch.zeros_like(g) if g is not None else None for g in grad_input) 66 | 67 | 68 | def protect_act(model): 69 | for name, module in model.named_modules(): 70 | if module != model: 71 | if name == 'gelu': 72 | setattr(model, name, QuantizeGelu()) 73 | elif isinstance(module, nn.Module) or isinstance(module, nn.Sequential): 74 | protect_act(module) 75 | 76 | def protect_resnetblock(model): 77 | modules = [] 78 | block_names = [] 79 | for name, module in model.transformer.resblocks.named_modules(): 80 | if isinstance(module, ResidualAttentionBlock): # Replace with the actual ResNet block class name 81 | modules.append(module) 82 | block_names.append(name) 83 | 84 | if modules: 85 | module = modules[-1] 86 | name = block_names[-1] 87 | new_block = ResidualAttentionBlockQuantize(module.attn.embed_dim, 88 | module.attn.num_heads, None) 89 | new_block.load_state_dict(module.state_dict()) 90 | new_block.training = module.training 91 | setattr(model.transformer.resblocks, name, new_block) 92 | 93 | 94 | def freeze_params(model): 95 | model.eval() 96 | model.token_embedding.eval() 97 | model.transformer.eval() 98 | model.ln_final.eval() 99 | model.positional_embedding.requires_grad = False 100 | model.text_projection.requires_grad = False 101 | model.logit_scale.requires_grad = False 102 | 103 | 104 | 105 | def print_model(model): 106 | for name, module in model.named_modules(): 107 | print(module) 108 | 109 | 110 | def reset_model(path, protect=True): 111 | device = "cuda" if torch.cuda.is_available() else "cpu" 112 | model, preprocess = clip.load("ViT-B/32", device=device) 113 | model.load_state_dict(torch.load(path)) # load state_dict 114 | if device == "cpu": 115 | model.float() 116 | if protect: # protect activations functions 117 | protect_act(model.visual) # 118 | protect_resnetblock(model.visual) 119 | model = model.to(device) 120 | freeze_params(model) 121 | return model -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/rfgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class RFGSM(Attack): 8 | r""" 9 | R+FGSM in the paper 'Ensemble Adversarial Training : Attacks and Defences' 10 | [https://arxiv.org/abs/1705.07204] 11 | 12 | Distance Measure : Linf 13 | 14 | Arguments: 15 | model (nn.Module): model to attack. 16 | eps (float): strength of the attack or maximum perturbation. (Default: 8/255) 17 | alpha (float): step size. (Default: 2/255) 18 | steps (int): number of steps. (Default: 10) 19 | 20 | Shape: 21 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 22 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 23 | - output: :math:`(N, C, H, W)`. 24 | 25 | Examples:: 26 | >>> attack = torchattacks.RFGSM(model, eps=8/255, alpha=2/255, steps=10) 27 | >>> adv_images = attack(images, labels) 28 | """ 29 | 30 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10): 31 | super().__init__('RFGSM', model, device) 32 | self.eps = eps 33 | self.alpha = alpha 34 | self.steps = steps 35 | self.supported_mode = ['default', 'targeted'] 36 | 37 | def forward(self, images, labels): 38 | r""" 39 | Overridden. 40 | """ 41 | 42 | images = images.clone().detach().to(self.device) 43 | labels = labels.clone().detach().to(self.device) 44 | 45 | if self.targeted: 46 | target_labels = self.get_target_label(images, labels) 47 | 48 | loss = nn.CrossEntropyLoss() 49 | 50 | adv_images = images + self.alpha*torch.randn_like(images).sign() 51 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 52 | 53 | for _ in range(self.steps): 54 | adv_images.requires_grad = True 55 | outputs = self.get_logits(adv_images) 56 | 57 | # Calculate loss 58 | if self.targeted: 59 | cost = -loss(outputs, target_labels) 60 | else: 61 | cost = loss(outputs, labels) 62 | 63 | # Update adversarial images 64 | grad = torch.autograd.grad(cost, adv_images, 65 | retain_graph=False, create_graph=False)[0] 66 | adv_images = adv_images.detach() + self.alpha*grad.sign() 67 | delta = torch.clamp(adv_images - images, 68 | min=-self.eps, max=self.eps) 69 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 70 | 71 | return adv_images 72 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/sinifgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class SINIFGSM(Attack): 8 | r""" 9 | SI-NI-FGSM in the paper 'NESTEROV ACCELERATED GRADIENT AND SCALEINVARIANCE FOR ADVERSARIAL ATTACKS' 10 | [https://arxiv.org/abs/1908.06281], Published as a conference paper at ICLR 2020 11 | Modified from "https://githuba.com/JHL-HUST/SI-NI-FGSM" 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): maximum perturbation. (Default: 8/255) 18 | alpha (float): step size. (Default: 2/255) 19 | steps (int): number of iterations. (Default: 10) 20 | decay (float): momentum factor. (Default: 1.0) 21 | m (int): number of scale copies. (Default: 5) 22 | 23 | Shape: 24 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 25 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 26 | - output: :math:`(N, C, H, W)`. 27 | 28 | Examples:: 29 | >>> attack = torchattacks.SINIFGSM(model, eps=8/255, alpha=2/255, steps=10, decay=1.0, m=5) 30 | >>> adv_images = attack(images, labels) 31 | 32 | """ 33 | 34 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, decay=1.0, m=5): 35 | super().__init__('SINIFGSM', model, device) 36 | self.eps = eps 37 | self.steps = steps 38 | self.decay = decay 39 | self.alpha = alpha 40 | self.m = m 41 | self.supported_mode = ['default', 'targeted'] 42 | 43 | def forward(self, images, labels): 44 | r""" 45 | Overridden. 46 | """ 47 | 48 | images = images.clone().detach().to(self.device) 49 | labels = labels.clone().detach().to(self.device) 50 | 51 | if self.targeted: 52 | target_labels = self.get_target_label(images, labels) 53 | 54 | momentum = torch.zeros_like(images).detach().to(self.device) 55 | 56 | loss = nn.CrossEntropyLoss() 57 | 58 | adv_images = images.clone().detach() 59 | 60 | for _ in range(self.steps): 61 | adv_images.requires_grad = True 62 | nes_image = adv_images + self.decay*self.alpha*momentum 63 | # Calculate sum the gradients over the scale copies of the input image 64 | adv_grad = torch.zeros_like(images).detach().to(self.device) 65 | for i in torch.arange(self.m): 66 | nes_images = nes_image / torch.pow(2, i) 67 | outputs = self.get_logits(nes_images) 68 | # Calculate loss 69 | if self.targeted: 70 | cost = -loss(outputs, target_labels) 71 | else: 72 | cost = loss(outputs, labels) 73 | adv_grad += torch.autograd.grad(cost, adv_images, 74 | retain_graph=False, create_graph=False)[0] 75 | adv_grad = adv_grad / self.m 76 | 77 | # Update adversarial images 78 | grad = self.decay*momentum + adv_grad / \ 79 | torch.mean(torch.abs(adv_grad), dim=(1, 2, 3), keepdim=True) 80 | momentum = grad 81 | adv_images = adv_images.detach() + self.alpha*grad.sign() 82 | delta = torch.clamp(adv_images - images, 83 | min=-self.eps, max=self.eps) 84 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 85 | 86 | return adv_images 87 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/sparsefool.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | 5 | from ..attack import Attack 6 | from .deepfool import DeepFool 7 | 8 | 9 | class SparseFool(Attack): 10 | r""" 11 | Attack in the paper 'SparseFool: a few pixels make a big difference' 12 | [https://arxiv.org/abs/1811.02248] 13 | 14 | Modified from "https://github.com/LTS4/SparseFool/" 15 | 16 | Distance Measure : L0 17 | 18 | Arguments: 19 | model (nn.Module): model to attack. 20 | steps (int): number of steps. (Default: 10) 21 | lam (float): parameter for scaling DeepFool noise. (Default: 3) 22 | overshoot (float): parameter for enhancing the noise. (Default: 0.02) 23 | 24 | Shape: 25 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 26 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 27 | - output: :math:`(N, C, H, W)`. 28 | 29 | Examples:: 30 | >>> attack = torchattacks.SparseFool(model, steps=10, lam=3, overshoot=0.02) 31 | >>> adv_images = attack(images, labels) 32 | 33 | """ 34 | 35 | def __init__(self, model, device=None, steps=10, lam=3, overshoot=0.02): 36 | super().__init__('SparseFool', model, device) 37 | self.steps = steps 38 | self.lam = lam 39 | self.overshoot = overshoot 40 | self.deepfool = DeepFool(model) 41 | self.supported_mode = ['default'] 42 | 43 | def forward(self, images, labels): 44 | r""" 45 | Overridden. 46 | """ 47 | 48 | images = images.clone().detach().to(self.device) 49 | labels = labels.clone().detach().to(self.device) 50 | 51 | batch_size = len(images) 52 | correct = torch.tensor([True]*batch_size) 53 | curr_steps = 0 54 | 55 | adv_images = [] 56 | for idx in range(batch_size): 57 | image = images[idx:idx+1].clone().detach() 58 | adv_images.append(image) 59 | 60 | while (True in correct) and (curr_steps < self.steps): 61 | for idx in range(batch_size): 62 | image = images[idx:idx+1] 63 | label = labels[idx:idx+1] 64 | adv_image = adv_images[idx] 65 | 66 | fs = self.get_logits(adv_image)[0] 67 | _, pre = torch.max(fs, dim=0) 68 | if pre != label: 69 | correct[idx] = False 70 | continue 71 | 72 | adv_image, target_label = self.deepfool.forward_return_target_labels(adv_image, label) 73 | adv_image = image + self.lam*(adv_image - image) 74 | 75 | adv_image.requires_grad = True 76 | fs = self.get_logits(adv_image)[0] 77 | _, pre = torch.max(fs, dim=0) 78 | 79 | if pre == label: 80 | pre = target_label 81 | 82 | cost = fs[pre] - fs[label] 83 | grad = torch.autograd.grad(cost, adv_image, 84 | retain_graph=False, create_graph=False)[0] 85 | grad = grad / grad.norm() 86 | 87 | adv_image = self._linear_solver(image, grad, adv_image) 88 | adv_image = image + (1+self.overshoot)*(adv_image - image) 89 | adv_images[idx] = torch.clamp(adv_image, min=0, max=1).detach() 90 | 91 | curr_steps += 1 92 | 93 | adv_images = torch.cat(adv_images).detach() 94 | 95 | return adv_images 96 | 97 | def _linear_solver(self, x_0, coord_vec, boundary_point): 98 | input_shape = x_0.size() 99 | 100 | plane_normal = coord_vec.clone().detach().view(-1) 101 | plane_point = boundary_point.clone().detach().view(-1) 102 | 103 | x_i = x_0.clone().detach() 104 | 105 | f_k = torch.dot(plane_normal, x_0.view(-1) - plane_point) 106 | sign_true = f_k.sign().item() 107 | 108 | beta = 0.001 * sign_true 109 | current_sign = sign_true 110 | 111 | while current_sign == sign_true and coord_vec.nonzero().size()[0] > 0: 112 | 113 | f_k = torch.dot(plane_normal, x_i.view(-1) - plane_point) + beta 114 | 115 | pert = f_k.abs() / coord_vec.abs().max() 116 | 117 | mask = torch.zeros_like(coord_vec) 118 | mask[np.unravel_index(torch.argmax(coord_vec.abs()).cpu(), input_shape)] = 1. # nopep8 119 | 120 | r_i = torch.clamp(pert, min=1e-4) * mask * coord_vec.sign() 121 | 122 | x_i = x_i + r_i 123 | x_i = torch.clamp(x_i, min=0, max=1) 124 | 125 | f_k = torch.dot(plane_normal, x_i.view(-1) - plane_point) 126 | current_sign = f_k.sign().item() 127 | 128 | coord_vec[r_i != 0] = 0 129 | 130 | return x_i 131 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/spsa.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Code is from https://github.com/BorealisAI/advertorch/blob/master/advertorch/attacks/spsa.py 3 | ''' 4 | 5 | import torch 6 | from torch.nn.modules.loss import _Loss 7 | 8 | from ..attack import Attack 9 | 10 | 11 | class MarginalLoss(_Loss): 12 | def forward(self, logits, targets): 13 | assert logits.shape[-1] >= 2 14 | top_logits, top_classes = torch.topk(logits, 2, dim=-1) 15 | target_logits = logits[torch.arange(logits.shape[0]), targets] 16 | max_nontarget_logits = torch.where( 17 | top_classes[..., 0] == targets, 18 | top_logits[..., 1], 19 | top_logits[..., 0], 20 | ) 21 | 22 | loss = max_nontarget_logits - target_logits 23 | if self.reduction == "none": 24 | pass 25 | elif self.reduction == "sum": 26 | loss = loss.sum() 27 | elif self.reduction == "mean": 28 | loss = loss.mean() 29 | else: 30 | raise ValueError("unknown reduction: '%s'" % (self.recution,)) 31 | 32 | return loss 33 | 34 | 35 | class SPSA(Attack): 36 | r""" 37 | SPSA in the paper 'Adversarial Risk and the Dangers of Evaluating Against Weak Attacks' 38 | [https://arxiv.org/abs/1802.05666] 39 | 40 | Distance Measure : Linf 41 | 42 | Arguments: 43 | model (nn.Module): model to attack. 44 | eps (float): maximum perturbation. (Default: 8/255) 45 | delta (float): scaling parameter of SPSA. (Default: 0.01) 46 | lr (float): the learning rate of the `Adam` optimizer. (Default: 0.01) 47 | nb_iter (int): number of iterations of the attack. (Default: 1) 48 | nb_sample (int): number of samples for SPSA gradient approximation. (Default: 128) 49 | max_batch_size (int): maximum batch size to be evaluated at once. (Default: 64) 50 | 51 | Shape: 52 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 53 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 54 | - output: :math:`(N, C, H, W)`. 55 | 56 | Examples:: 57 | >>> attack = torchattacks.SPSA(model, eps=0.3) 58 | >>> adv_images = attack(images, labels) 59 | 60 | """ 61 | 62 | def __init__(self, model, device=None, eps=0.3, delta=0.01, lr=0.01, nb_iter=1, nb_sample=128, max_batch_size=64): 63 | super().__init__('SPSA', model, device) 64 | self.eps = eps 65 | self.delta = delta 66 | self.lr = lr 67 | self.nb_iter = nb_iter 68 | self.nb_sample = nb_sample 69 | self.max_batch_size = max_batch_size 70 | self.loss_fn = MarginalLoss(reduction="none") 71 | self.supported_mode = ['default', 'targeted'] 72 | 73 | def forward(self, images, labels): 74 | r""" 75 | Overridden. 76 | """ 77 | 78 | images = images.clone().detach().to(self.device) 79 | labels = labels.clone().detach().to(self.device) 80 | 81 | adv_images = self.spsa_perturb(images, labels) 82 | return adv_images 83 | 84 | def loss(self, *args): 85 | if self.targeted: 86 | return self.loss_fn(*args) 87 | else: 88 | return -self.loss_fn(*args) 89 | 90 | def linf_clamp_(self, dx, x, eps): 91 | """Clamps perturbation `dx` to fit L_inf norm and image bounds. 92 | 93 | Limit the L_inf norm of `dx` to be <= `eps`, and the bounds of `x + dx` 94 | to be in `[clip_min, clip_max]`. 95 | 96 | Return: the clamped perturbation `dx`. 97 | """ 98 | 99 | # dx_clamped = self.batch_clamp(eps, dx) 100 | dx_clamped = torch.clamp(dx, min=-eps, max=eps) 101 | # x_adv = self.clamp(x + dx_clamped, clip_min, clip_max) 102 | x_adv = torch.clamp(x + dx_clamped, min=0, max=1) 103 | # `dx` is changed *inplace* so the optimizer will keep 104 | # tracking it. the simplest mechanism for inplace was 105 | # adding the difference between the new value `x_adv - x` 106 | # and the old value `dx`. 107 | dx += x_adv - x - dx 108 | return dx 109 | 110 | def _get_batch_sizes(self, n, max_batch_size): 111 | batches = [max_batch_size for _ in range(n // max_batch_size)] 112 | if n % max_batch_size > 0: 113 | batches.append(n % max_batch_size) 114 | return batches 115 | 116 | @torch.no_grad() 117 | def spsa_grad(self, images, labels, delta, nb_sample, max_batch_size): 118 | """Uses SPSA method to apprixmate gradient w.r.t `x`. 119 | 120 | Use the SPSA method to approximate the gradient of `loss(predict(x), y)` 121 | with respect to `x`, based on the nonce `v`. 122 | 123 | Return the approximated gradient of `loss_fn(predict(x), y)` with respect to `x`. 124 | """ 125 | 126 | grad = torch.zeros_like(images) 127 | images = torch.unsqueeze(images, 0) 128 | labels = torch.unsqueeze(labels, 0) 129 | 130 | def f(xvar, yvar): 131 | return self.loss(self.get_logits(xvar), yvar) 132 | 133 | images = images.expand(max_batch_size, *images.shape[1:]).contiguous() 134 | labels = labels.expand(max_batch_size, *labels.shape[1:]).contiguous() 135 | 136 | v = torch.empty_like(images[:, :1, ...]) 137 | for batch_size in self._get_batch_sizes(nb_sample, max_batch_size): 138 | x_ = images[:batch_size] 139 | y_ = labels[:batch_size] 140 | vb = v[:batch_size] 141 | vb = vb.bernoulli_().mul_(2.0).sub_(1.0) 142 | v_ = vb.expand_as(x_).contiguous() 143 | x_shape = x_.shape 144 | x_ = x_.view(-1, *images.shape[2:]) 145 | y_ = y_.view(-1, *labels.shape[2:]) 146 | v_ = v_.view(-1, *v.shape[2:]) 147 | df = f(x_+delta*v_, y_) - f(x_-delta*v_, y_) 148 | df = df.view(-1, *[1 for _ in v_.shape[1:]]) 149 | grad_ = df / (2.*delta*v_) 150 | grad_ = grad_.view(x_shape) 151 | grad_ = grad_.sum(dim=0, keepdim=False) 152 | grad += grad_ 153 | 154 | grad /= nb_sample 155 | return grad 156 | 157 | def spsa_perturb(self, x, y): 158 | dx = torch.zeros_like(x) 159 | dx.grad = torch.zeros_like(dx) 160 | optimizer = torch.optim.Adam([dx], lr=self.lr) 161 | for _ in range(self.nb_iter): 162 | optimizer.zero_grad() 163 | dx.grad = self.spsa_grad( 164 | x + dx, y, self.delta, self.nb_sample, self.max_batch_size) 165 | optimizer.step() 166 | dx = self.linf_clamp_(dx, x, self.eps) 167 | 168 | x_adv = x + dx 169 | return x_adv 170 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/tpgd.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from ..attack import Attack 6 | 7 | 8 | class TPGD(Attack): 9 | r""" 10 | PGD based on KL-Divergence loss in the paper 'Theoretically Principled Trade-off between Robustness and Accuracy' 11 | [https://arxiv.org/abs/1901.08573] 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): strength of the attack or maximum perturbation. (Default: 8/255) 18 | alpha (float): step size. (Default: 2/255) 19 | steps (int): number of steps. (Default: 10) 20 | 21 | Shape: 22 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 23 | - output: :math:`(N, C, H, W)`. 24 | 25 | Examples:: 26 | >>> attack = torchattacks.TPGD(model, eps=8/255, alpha=2/255, steps=10) 27 | >>> adv_images = attack(images) 28 | 29 | """ 30 | 31 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10): 32 | super().__init__('TPGD', model, device) 33 | self.eps = eps 34 | self.alpha = alpha 35 | self.steps = steps 36 | self.supported_mode = ['default'] 37 | 38 | def forward(self, images, labels=None): 39 | r""" 40 | Overridden. 41 | """ 42 | 43 | images = images.clone().detach().to(self.device) 44 | logit_ori = self.get_logits(images).detach() 45 | 46 | adv_images = images + 0.001*torch.randn_like(images) 47 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 48 | 49 | loss = nn.KLDivLoss(reduction='sum') 50 | 51 | for _ in range(self.steps): 52 | adv_images.requires_grad = True 53 | logit_adv = self.get_logits(adv_images) 54 | 55 | # Calculate loss 56 | cost = loss(F.log_softmax(logit_adv, dim=1), 57 | F.softmax(logit_ori, dim=1)) 58 | 59 | # Update adversarial images 60 | grad = torch.autograd.grad(cost, adv_images, 61 | retain_graph=False, create_graph=False)[0] 62 | 63 | adv_images = adv_images.detach() + self.alpha*grad.sign() 64 | delta = torch.clamp(adv_images - images, 65 | min=-self.eps, max=self.eps) 66 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 67 | 68 | return adv_images 69 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/upgd.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | from ..attack import Attack 7 | 8 | 9 | class UPGD(Attack): 10 | r""" 11 | Ultimate PGD that supports various options of gradient-based adversarial attacks. 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): maximum perturbation. (Default: 8/255) 18 | alpha (float): step size. (Default: 2/255) 19 | steps (int): number of steps. (Default: 10) 20 | random_start (bool): using random initialization of delta. (Default: False) 21 | loss (str): loss function. ['ce', 'margin', 'dlr'] (Default: 'ce') 22 | decay (float): momentum factor. (Default: 1.0) 23 | eot_iter (int) : number of models to estimate the mean gradient. (Default: 1) 24 | 25 | Shape: 26 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 27 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 28 | - output: :math:`(N, C, H, W)`. 29 | 30 | Examples:: 31 | >>> attack = torchattacks.UPGD(model, eps=8/255, alpha=2/255, steps=10, random_start=False) 32 | >>> adv_images = attack(images, labels) 33 | 34 | """ 35 | 36 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, 37 | random_start=False, loss='ce', decay=1.0, eot_iter=1): 38 | super().__init__('UPGD', model, device) 39 | self.eps = eps 40 | self.alpha = alpha 41 | self.steps = steps 42 | self.random_start = random_start 43 | self.loss = loss 44 | self.decay = decay 45 | self.eot_iter = eot_iter 46 | self.supported_mode = ['default', 'targeted'] 47 | 48 | def forward(self, images, labels): 49 | r""" 50 | Overridden. 51 | """ 52 | 53 | images = images.clone().detach().to(self.device) 54 | labels = labels.clone().detach().to(self.device) 55 | 56 | if self.targeted: 57 | target_labels = self.get_target_label(images, labels) 58 | 59 | momentum = torch.zeros_like(images).detach().to(self.device) 60 | 61 | adv_images = images.clone().detach() 62 | 63 | if self.random_start: 64 | # Starting at a uniformly random point 65 | adv_images = adv_images + \ 66 | torch.empty_like(adv_images).uniform_(-self.eps, self.eps) 67 | adv_images = torch.clamp(adv_images, min=0, max=1).detach() 68 | 69 | for _ in range(self.steps): 70 | grad = torch.zeros_like(adv_images) 71 | adv_images.requires_grad = True 72 | 73 | for j in range(self.eot_iter): 74 | # Calculate loss 75 | if self.targeted: 76 | cost = self.get_loss(adv_images, labels, target_labels) 77 | else: 78 | cost = self.get_loss(adv_images, labels) 79 | 80 | grad += torch.autograd.grad(cost, adv_images, 81 | retain_graph=False, 82 | create_graph=False)[0] / self.eot_iter 83 | 84 | # Update adversarial images 85 | grad = grad / torch.mean(torch.abs(grad), 86 | dim=(1, 2, 3), keepdim=True) 87 | grad = grad + momentum*self.decay 88 | momentum = grad 89 | 90 | adv_images = adv_images.detach() + self.alpha*grad.sign() 91 | delta = torch.clamp(adv_images - images, 92 | min=-self.eps, max=self.eps) 93 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 94 | 95 | return adv_images 96 | 97 | def get_loss(self, images, labels, target_labels=None): 98 | if isinstance(self.loss, str): 99 | if self.loss == 'ce': 100 | return self.ce_loss(images, labels, target_labels) 101 | elif self.loss == 'dlr': 102 | return self.dlr_loss(images, labels, target_labels) 103 | elif self.loss == 'margin': 104 | return self.margin_loss(images, labels, target_labels) 105 | else: 106 | raise ValueError(self.loss + " is not valid.") 107 | else: 108 | return self.loss(images, labels, target_labels) 109 | 110 | def ce_loss(self, images, labels, target_labels): 111 | loss = nn.CrossEntropyLoss() 112 | outputs = self.get_logits(images) 113 | 114 | if self.targeted: 115 | cost = -loss(outputs, target_labels) 116 | else: 117 | cost = loss(outputs, labels) 118 | return cost 119 | 120 | def dlr_loss(self, images, labels, target_labels): 121 | outputs = self.get_logits(images) 122 | outputs_sorted, ind_sorted = outputs.sort(dim=1) 123 | if self.targeted: 124 | cost = -(outputs[np.arange(outputs.shape[0]), labels] - outputs[np.arange(outputs.shape[0]), target_labels]) \ 125 | / (outputs_sorted[:, -1] - .5 * outputs_sorted[:, -3] - .5 * outputs_sorted[:, -4] + 1e-12) 126 | else: 127 | ind = (ind_sorted[:, -1] == labels).float() 128 | cost = -(outputs[np.arange(outputs.shape[0]), labels] - outputs_sorted[:, -2] * ind - outputs_sorted[:, -1] * (1. - ind)) \ 129 | / (outputs_sorted[:, -1] - outputs_sorted[:, -3] + 1e-12) 130 | return cost.sum() 131 | 132 | # f-function in the paper 133 | def margin_loss(self, images, labels, target_labels): 134 | outputs = self.get_logits(images) 135 | if self.targeted: 136 | # one_hot_labels = torch.eye(len(outputs[0]))[target_labels].to(self.device) 137 | one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[target_labels] # nopep8 138 | i, _ = torch.max((1-one_hot_labels)*outputs, dim=1) 139 | j = torch.masked_select(outputs, one_hot_labels.bool()) 140 | cost = -torch.clamp((i-j), min=0) # -self.kappa=0 141 | else: 142 | one_hot_labels = torch.eye(len(outputs[0]))[labels].to(self.device) 143 | i, _ = torch.max((1-one_hot_labels)*outputs, dim=1) 144 | j = torch.masked_select(outputs, one_hot_labels.bool()) 145 | cost = -torch.clamp((j-i), min=0) # -self.kappa=0 146 | return cost.sum() 147 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/vanila.py: -------------------------------------------------------------------------------- 1 | from ..attack import Attack 2 | 3 | 4 | class VANILA(Attack): 5 | r""" 6 | Vanila version of Attack. 7 | It just returns the input images. 8 | 9 | Arguments: 10 | model (nn.Module): model to attack. 11 | 12 | Shape: 13 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 14 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 15 | - output: :math:`(N, C, H, W)`. 16 | 17 | Examples:: 18 | >>> attack = torchattacks.VANILA(model) 19 | >>> adv_images = attack(images, labels) 20 | 21 | """ 22 | 23 | def __init__(self, model, device): 24 | super().__init__('VANILA', model, device) 25 | self.supported_mode = ['default'] 26 | 27 | def forward(self, images, labels=None): 28 | r""" 29 | Overridden. 30 | """ 31 | 32 | adv_images = images.clone().detach().to(self.device) 33 | 34 | return adv_images 35 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/vmifgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class VMIFGSM(Attack): 8 | r""" 9 | VMI-FGSM in the paper 'Enhancing the Transferability of Adversarial Attacks through Variance Tuning 10 | [https://arxiv.org/abs/2103.15571], Published as a conference paper at CVPR 2021 11 | Modified from "https://github.com/JHL-HUST/VT" 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): maximum perturbation. (Default: 8/255) 18 | steps (int): number of iterations. (Default: 10) 19 | alpha (float): step size. (Default: 2/255) 20 | decay (float): momentum factor. (Default: 1.0) 21 | N (int): the number of sampled examples in the neighborhood. (Default: 5) 22 | beta (float): the upper bound of neighborhood. (Default: 3/2) 23 | 24 | Shape: 25 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 26 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 27 | - output: :math:`(N, C, H, W)`. 28 | 29 | Examples:: 30 | >>> attack = torchattacks.VMIFGSM(model, eps=8/255, alpha=2/255, steps=10, decay=1.0, N=5, beta=3/2) 31 | >>> adv_images = attack(images, labels) 32 | 33 | """ 34 | 35 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, decay=1.0, N=5, beta=3/2): 36 | super().__init__('VMIFGSM', model, device) 37 | self.eps = eps 38 | self.steps = steps 39 | self.decay = decay 40 | self.alpha = alpha 41 | self.N = N 42 | self.beta = beta 43 | self.supported_mode = ['default', 'targeted'] 44 | 45 | def forward(self, images, labels): 46 | r""" 47 | Overridden. 48 | """ 49 | 50 | images = images.clone().detach().to(self.device) 51 | labels = labels.clone().detach().to(self.device) 52 | 53 | if self.targeted: 54 | target_labels = self.get_target_label(images, labels) 55 | 56 | momentum = torch.zeros_like(images).detach().to(self.device) 57 | v = torch.zeros_like(images).detach().to(self.device) 58 | loss = nn.CrossEntropyLoss() 59 | adv_images = images.clone().detach() 60 | 61 | for _ in range(self.steps): 62 | adv_images.requires_grad = True 63 | outputs = self.get_logits(adv_images) 64 | 65 | # Calculate loss 66 | if self.targeted: 67 | cost = -loss(outputs, target_labels) 68 | else: 69 | cost = loss(outputs, labels) 70 | 71 | # Update adversarial images 72 | adv_grad = torch.autograd.grad(cost, adv_images, 73 | retain_graph=False, create_graph=False)[0] 74 | 75 | grad = (adv_grad + v) / torch.mean(torch.abs(adv_grad + v), 76 | dim=(1, 2, 3), keepdim=True) 77 | grad = grad + momentum * self.decay 78 | momentum = grad 79 | 80 | # Calculate Gradient Variance 81 | GV_grad = torch.zeros_like(images).detach().to(self.device) 82 | for _ in range(self.N): 83 | neighbor_images = adv_images.detach() + \ 84 | torch.randn_like(images).uniform_(-self.eps * 85 | self.beta, self.eps*self.beta) 86 | neighbor_images.requires_grad = True 87 | outputs = self.get_logits(neighbor_images) 88 | 89 | # Calculate loss 90 | if self.targeted: 91 | cost = -loss(outputs, target_labels) 92 | else: 93 | cost = loss(outputs, labels) 94 | GV_grad += torch.autograd.grad(cost, neighbor_images, 95 | retain_graph=False, create_graph=False)[0] 96 | # obtaining the gradient variance 97 | v = GV_grad / self.N - adv_grad 98 | 99 | adv_images = adv_images.detach() + self.alpha*grad.sign() 100 | delta = torch.clamp(adv_images - images, 101 | min=-self.eps, max=self.eps) 102 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 103 | 104 | return adv_images 105 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/attacks/vnifgsm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ..attack import Attack 5 | 6 | 7 | class VNIFGSM(Attack): 8 | r""" 9 | VNI-FGSM in the paper 'Enhancing the Transferability of Adversarial Attacks through Variance Tuning 10 | [https://arxiv.org/abs/2103.15571], Published as a conference paper at CVPR 2021 11 | Modified from "https://github.com/JHL-HUST/VT" 12 | 13 | Distance Measure : Linf 14 | 15 | Arguments: 16 | model (nn.Module): model to attack. 17 | eps (float): maximum perturbation. (Default: 8/255) 18 | alpha (float): step size. (Default: 2/255) 19 | steps (int): number of iterations. (Default: 10) 20 | decay (float): momentum factor. (Default: 1.0) 21 | N (int): the number of sampled examples in the neighborhood. (Default: 5) 22 | beta (float): the upper bound of neighborhood. (Default: 3/2) 23 | 24 | Shape: 25 | - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. 26 | - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. 27 | - output: :math:`(N, C, H, W)`. 28 | 29 | Examples:: 30 | >>> attack = torchattacks.VNIFGSM(model, eps=8/255, alpha=2/255, steps=10, decay=1.0, N=5, beta=3/2) 31 | >>> adv_images = attack(images, labels) 32 | 33 | """ 34 | 35 | def __init__(self, model, device=None, eps=8/255, alpha=2/255, steps=10, decay=1.0, N=5, beta=3/2): 36 | super().__init__('VNIFGSM', model, device) 37 | self.eps = eps 38 | self.steps = steps 39 | self.decay = decay 40 | self.alpha = alpha 41 | self.N = N 42 | self.beta = beta 43 | self.supported_mode = ['default', 'targeted'] 44 | 45 | def forward(self, images, labels): 46 | r""" 47 | Overridden. 48 | """ 49 | 50 | images = images.clone().detach().to(self.device) 51 | labels = labels.clone().detach().to(self.device) 52 | 53 | if self.targeted: 54 | target_labels = self.get_target_label(images, labels) 55 | 56 | momentum = torch.zeros_like(images).detach().to(self.device) 57 | v = torch.zeros_like(images).detach().to(self.device) 58 | loss = nn.CrossEntropyLoss() 59 | adv_images = images.clone().detach() 60 | 61 | for _ in range(self.steps): 62 | adv_images.requires_grad = True 63 | nes_images = adv_images + self.decay * self.alpha * momentum 64 | outputs = self.get_logits(nes_images) 65 | 66 | # Calculate loss 67 | if self.targeted: 68 | cost = -loss(outputs, target_labels) 69 | else: 70 | cost = loss(outputs, labels) 71 | 72 | # Update adversarial images 73 | adv_grad = torch.autograd.grad(cost, adv_images, 74 | retain_graph=False, create_graph=False)[0] 75 | 76 | grad = (adv_grad + v) / torch.mean(torch.abs(adv_grad + v), 77 | dim=(1, 2, 3), keepdim=True) 78 | grad = grad + momentum * self.decay 79 | momentum = grad 80 | 81 | # Calculate Gradient Variance 82 | GV_grad = torch.zeros_like(images).detach().to(self.device) 83 | for _ in range(self.N): 84 | neighbor_images = adv_images.detach() + \ 85 | torch.randn_like(images).uniform_(-self.eps * 86 | self.beta, self.eps*self.beta) 87 | neighbor_images.requires_grad = True 88 | outputs = self.get_logits(neighbor_images) 89 | 90 | # Calculate loss 91 | if self.targeted: 92 | cost = -loss(outputs, target_labels) 93 | else: 94 | cost = loss(outputs, labels) 95 | GV_grad += torch.autograd.grad(cost, neighbor_images, 96 | retain_graph=False, create_graph=False)[0] 97 | # obtaining the gradient variance 98 | v = GV_grad / self.N - adv_grad 99 | 100 | adv_images = adv_images.detach() + self.alpha*grad.sign() 101 | delta = torch.clamp(adv_images - images, 102 | min=-self.eps, max=self.eps) 103 | adv_images = torch.clamp(images + delta, min=0, max=1).detach() 104 | 105 | return adv_images 106 | -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/wrappers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/componentwise_evaluation/torchattacks/wrappers/__init__.py -------------------------------------------------------------------------------- /experiments/componentwise_evaluation/torchattacks/wrappers/multiattack.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..attack import Attack 4 | 5 | 6 | class MultiAttack(Attack): 7 | r""" 8 | MultiAttack is a class to attack a model with various attacks agains same images and labels. 9 | 10 | Arguments: 11 | model (nn.Module): model to attack. 12 | attacks (list): list of attacks. 13 | 14 | Examples:: 15 | >>> atk1 = torchattacks.PGD(model, eps=8/255, alpha=2/255, iters=40, random_start=True) 16 | >>> atk2 = torchattacks.PGD(model, eps=8/255, alpha=2/255, iters=40, random_start=True) 17 | >>> atk = torchattacks.MultiAttack([atk1, atk2]) 18 | >>> adv_images = attack(images, labels) 19 | 20 | """ 21 | def __init__(self, attacks, device=None, verbose=False): 22 | super().__init__("MultiAttack", attacks[0].model, device) 23 | self.attacks = attacks 24 | self.verbose = verbose 25 | self.supported_mode = ['default'] 26 | 27 | self.check_validity() 28 | 29 | self._accumulate_multi_atk_records = False 30 | self._multi_atk_records = [0.0] 31 | 32 | def check_validity(self): 33 | if len(self.attacks) < 2: 34 | raise ValueError("More than two attacks should be given.") 35 | 36 | ids = [id(attack.model) for attack in self.attacks] 37 | if len(set(ids)) != 1: 38 | raise ValueError("At least one of attacks is referencing a different model.") 39 | 40 | def forward(self, images, labels): 41 | r""" 42 | Overridden. 43 | """ 44 | batch_size = images.shape[0] 45 | fails = torch.arange(batch_size).to(self.device) 46 | final_images = images.clone().detach().to(self.device) 47 | labels = labels.clone().detach().to(self.device) 48 | 49 | multi_atk_records = [batch_size] 50 | 51 | for _, attack in enumerate(self.attacks): 52 | adv_images = attack(images[fails], labels[fails]) 53 | 54 | outputs = self.get_logits(adv_images) 55 | _, pre = torch.max(outputs.data, 1) 56 | 57 | corrects = (pre == labels[fails]) 58 | wrongs = ~corrects 59 | 60 | succeeds = torch.masked_select(fails, wrongs) 61 | succeeds_of_fails = torch.masked_select(torch.arange(fails.shape[0]).to(self.device), wrongs) 62 | 63 | final_images[succeeds] = adv_images[succeeds_of_fails] 64 | 65 | fails = torch.masked_select(fails, corrects) 66 | multi_atk_records.append(len(fails)) 67 | 68 | if len(fails) == 0: 69 | break 70 | 71 | if self.verbose: 72 | print(self._return_sr_record(multi_atk_records)) 73 | 74 | if self._accumulate_multi_atk_records: 75 | self._update_multi_atk_records(multi_atk_records) 76 | 77 | return final_images 78 | 79 | def _clear_multi_atk_records(self): 80 | self._multi_atk_records = [0.0] 81 | 82 | def _covert_to_success_rates(self, multi_atk_records): 83 | sr = [((1-multi_atk_records[i]/multi_atk_records[0])*100) for i in range(1, len(multi_atk_records))] 84 | return sr 85 | 86 | def _return_sr_record(self, multi_atk_records): 87 | sr = self._covert_to_success_rates(multi_atk_records) 88 | return "Attack success rate: "+" | ".join(["%2.2f %%"%item for item in sr]) 89 | 90 | def _update_multi_atk_records(self, multi_atk_records): 91 | for i, item in enumerate(multi_atk_records): 92 | self._multi_atk_records[i] += item 93 | 94 | def save(self, data_loader, save_path=None, verbose=True, return_verbose=False, 95 | save_predictions=False, save_clean_images=False): 96 | r""" 97 | Overridden. 98 | """ 99 | self._clear_multi_atk_records() 100 | prev_verbose = self.verbose 101 | self.verbose = False 102 | self._accumulate_multi_atk_records = True 103 | 104 | for i, attack in enumerate(self.attacks): 105 | self._multi_atk_records.append(0.0) 106 | 107 | if return_verbose: 108 | rob_acc, l2, elapsed_time = super().save(data_loader, save_path, 109 | verbose, return_verbose, 110 | save_predictions, 111 | save_clean_images) 112 | sr = self._covert_to_success_rates(self._multi_atk_records) 113 | elif verbose: 114 | super().save(data_loader, save_path, verbose, 115 | return_verbose, save_predictions, 116 | save_clean_images) 117 | sr = self._covert_to_success_rates(self._multi_atk_records) 118 | else: 119 | super().save(data_loader, save_path, False, 120 | False, save_predictions, 121 | save_clean_images) 122 | 123 | self._clear_multi_atk_records() 124 | self._accumulate_multi_atk_records = False 125 | self.verbose = prev_verbose 126 | 127 | if return_verbose: 128 | return rob_acc, sr, l2, elapsed_time 129 | 130 | def _save_print(self, progress, rob_acc, l2, elapsed_time, end): 131 | r""" 132 | Overridden. 133 | """ 134 | print("- Save progress: %2.2f %% / Robust accuracy: %2.2f %%"%(progress, rob_acc)+\ 135 | " / "+self._return_sr_record(self._multi_atk_records)+\ 136 | ' / L2: %1.5f (%2.3f it/s) \t'%(l2, elapsed_time), end=end) 137 | -------------------------------------------------------------------------------- /experiments/field_study/fonts/arabic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/arabic.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/arialbd.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/arialbd.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/chinese_cht.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/chinese_cht.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/cyrillic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/cyrillic.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/french.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/french.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/german.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/german.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/hindi.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/hindi.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/japan.ttc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/japan.ttc -------------------------------------------------------------------------------- /experiments/field_study/fonts/kannada.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/kannada.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/korean.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/korean.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/latin.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/latin.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/marathi.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/marathi.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/nepali.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/nepali.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/persian.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/persian.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/simfang.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/simfang.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/spanish.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/spanish.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/tamil.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/tamil.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/telugu.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/telugu.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/urdu.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/urdu.ttf -------------------------------------------------------------------------------- /experiments/field_study/fonts/uyghur.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/fonts/uyghur.ttf -------------------------------------------------------------------------------- /experiments/field_study/plots/brand_freq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/brand_freq.png -------------------------------------------------------------------------------- /experiments/field_study/plots/brand_sector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/brand_sector.png -------------------------------------------------------------------------------- /experiments/field_study/plots/campaign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/campaign.png -------------------------------------------------------------------------------- /experiments/field_study/plots/domain_age.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/domain_age.png -------------------------------------------------------------------------------- /experiments/field_study/plots/geo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/geo.png -------------------------------------------------------------------------------- /experiments/field_study/plots/num_phish.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/num_phish.png -------------------------------------------------------------------------------- /experiments/field_study/plots/phishllm_cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/experiments/field_study/plots/phishllm_cost.png -------------------------------------------------------------------------------- /experiments/field_study/tele/gsheets.py: -------------------------------------------------------------------------------- 1 | import gspread 2 | from oauth2client.service_account import ServiceAccountCredentials 3 | import pprint 4 | import pandas as pd 5 | from gspread import Cell 6 | 7 | class gwrapper(): 8 | def __init__(self): 9 | scope = [ 10 | 'https://www.googleapis.com/auth/drive', 11 | 'https://www.googleapis.com/auth/drive.file' 12 | ] 13 | file_name = '/home/ruofan/git_space/ScamDet/datasets/google_cloud.json' 14 | creds = ServiceAccountCredentials.from_json_keyfile_name(file_name, scope) 15 | client = gspread.authorize(creds) 16 | 17 | # Fetch the sheet 18 | self.sheet = client.open('phishllm label small-scale').sheet1 19 | 20 | def get_records(self): 21 | print('get records') 22 | return self.sheet.get_all_records() 23 | 24 | def update_file(self, filename, date): 25 | df = pd.read_csv(filename, delimiter='\t') 26 | print(df.values) 27 | df['input_score'] = 'ERROR' 28 | df['yes'] = 0 29 | df['no'] = 0 30 | df['unsure'] = 0 31 | df = df[df['phish'] == 1] 32 | df = df.drop_duplicates('url', ) 33 | df['date'] = date 34 | df = df[['date', 'url', 'foldername', 'prediction', 35 | 'input_score', 'vt_result', 36 | 'yes', 'no', 'unsure']] # columns here 37 | list2 = df.values.tolist()[1:] 38 | print(list2) 39 | results = self.sheet.append_rows(list2) 40 | print(results) 41 | 42 | def update_list(self, to_update): 43 | self.sheet.append_rows(to_update) 44 | 45 | def update_cell(self, question_id, yes, no, unsure): 46 | # update google sheet yes, no, unsure columns 47 | cells = [] 48 | id = int(question_id) 49 | cells.append(Cell(row=id, col=7, value=yes)) 50 | cells.append(Cell(row=id, col=8, value=no)) 51 | cells.append(Cell(row=id, col=9, value=unsure)) 52 | self.sheet.update_cells(cells) 53 | 54 | 55 | if __name__ == '__main__': 56 | g = gwrapper() 57 | -------------------------------------------------------------------------------- /experiments/field_study/test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | from scripts.pipeline.test_llm import * 5 | import argparse 6 | from tqdm import tqdm 7 | import yaml 8 | import openai 9 | from datetime import datetime, date, timedelta 10 | os.environ["CUDA_LAUNCH_BLOCKING"] = "1" 11 | os.environ['OPENAI_API_KEY'] = open('./datasets/openai_key.txt').read().strip() 12 | 13 | if __name__ == '__main__': 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--folder", default="./datasets/field_study/2023-09-02/") 17 | parser.add_argument("--config", default='./param_dict.yaml', help="Config .yaml path") 18 | args = parser.parse_args() 19 | 20 | PhishLLMLogger.set_debug_on() 21 | PhishLLMLogger.set_verbose(True) 22 | 23 | # load hyperparameters 24 | with open(args.config) as file: 25 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 26 | 27 | # PhishLLM 28 | proxy_url = os.environ.get('proxy_url', None) 29 | phishintention_cls = PhishIntentionWrapper() 30 | llm_cls = TestLLM(phishintention_cls, 31 | param_dict=param_dict, 32 | proxies={"http": proxy_url, 33 | "https": proxy_url, 34 | }) 35 | openai.api_key = os.getenv("OPENAI_API_KEY") 36 | openai.proxy = proxy_url # set openai proxy 37 | 38 | # boot driver 39 | driver = CustomWebDriver.boot(proxy_server=proxy_url) # Using the proxy_url variable 40 | driver.set_script_timeout(param_dict['rank']['script_timeout']) 41 | driver.set_page_load_timeout(param_dict['rank']['page_load_timeout']) 42 | 43 | day = date.today().strftime("%Y-%m-%d") 44 | result_txt = '{}_phishllm.txt'.format(day) 45 | 46 | if not os.path.exists(result_txt): 47 | with open(result_txt, "w+") as f: 48 | f.write("folder" + "\t") 49 | f.write("phish_prediction" + "\t") 50 | f.write("target_prediction" + "\t") # write top1 prediction only 51 | f.write("brand_recog_time" + "\t") 52 | f.write("crp_prediction_time" + "\t") 53 | f.write("crp_transition_time" + "\n") 54 | 55 | for ct, folder in tqdm(enumerate(os.listdir(args.folder))): 56 | if folder in [x.split('\t')[0] for x in open(result_txt, encoding='ISO-8859-1').readlines()]: 57 | continue 58 | 59 | info_path = os.path.join(args.folder, folder, 'info.txt') 60 | html_path = os.path.join(args.folder, folder, 'html.txt') 61 | shot_path = os.path.join(args.folder, folder, 'shot.png') 62 | predict_path = os.path.join(args.folder, folder, 'predict.png') 63 | if not os.path.exists(shot_path): 64 | continue 65 | 66 | try: 67 | if len(open(info_path, encoding='ISO-8859-1').read()) > 0: 68 | url = open(info_path, encoding='ISO-8859-1').read() 69 | else: 70 | url = 'https://' + folder 71 | except FileNotFoundError: 72 | url = 'https://' + folder 73 | 74 | logo_box, reference_logo = llm_cls.detect_logo(shot_path) 75 | while True: 76 | try: 77 | pred, brand, brand_recog_time, crp_prediction_time, crp_transition_time, plotvis = llm_cls.test(url=url, 78 | reference_logo=reference_logo, 79 | logo_box=logo_box, 80 | shot_path=shot_path, 81 | html_path=html_path, 82 | driver=driver, 83 | ) 84 | driver.delete_all_cookies() 85 | break 86 | 87 | except (WebDriverException) as e: 88 | print(f"Driver crashed or encountered an error: {e}. Restarting driver.") 89 | driver.quit() 90 | time.sleep(1) 91 | driver = CustomWebDriver.boot(proxy_server=proxy_url) # Using the proxy_url variable 92 | driver.set_script_timeout(param_dict['rank']['script_timeout']) 93 | driver.set_page_load_timeout(param_dict['rank']['page_load_timeout']) 94 | continue 95 | 96 | try: 97 | with open(result_txt, "a+", encoding='ISO-8859-1') as f: 98 | f.write(f"{folder}\t{pred}\t{brand}\t{brand_recog_time}\t{crp_prediction_time}\t{crp_transition_time}\n") 99 | if pred == 'phish': 100 | plotvis.save(predict_path) 101 | 102 | except UnicodeEncodeError: 103 | continue 104 | 105 | 106 | driver.quit() 107 | 108 | -------------------------------------------------------------------------------- /experiments/field_study/test_baseline.py: -------------------------------------------------------------------------------- 1 | from scripts.pipeline.test_baseline import * 2 | from scripts.utils.web_utils.web_utils import CustomWebDriver 3 | from scripts.utils.dynaphish.brand_knowledge_utils import BrandKnowledgeConstruction 4 | from scripts.pipeline.test_dynaphish import DynaPhish, SubmissionButtonLocator 5 | import argparse 6 | import cv2 7 | from mmocr.apis import MMOCRInferencer 8 | 9 | os.environ['HTTP_PROXY'] = 'http://127.0.0.1:7890' 10 | os.environ['HTTPS_PROXY'] = 'http://127.0.0.1:7890' 11 | 12 | if __name__ == '__main__': 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument("--folder", default="./datasets/field_study/2024-01-29/") 16 | parser.add_argument("--date", default="2024-01-29", help="%Y-%m-%d") 17 | parser.add_argument("--method", default='dynaphish', choices=['phishpedia', 'phishintention', 'dynaphish']) 18 | args = parser.parse_args() 19 | 20 | # PhishLLM 21 | phishintention_cls = PhishIntentionWrapper() 22 | base_cls = TestBaseline(phishintention_cls) 23 | 24 | # Xdriver 25 | if args.method != 'phishpedia': 26 | sleep_time = 3; timeout_time = 60 27 | driver = CustomWebDriver.boot(proxy_server="127.0.0.1:7890") # Using the proxy_url variable 28 | driver.set_script_timeout(timeout_time / 2) 29 | driver.set_page_load_timeout(timeout_time) 30 | 31 | if args.method == 'dynaphish': 32 | phishintention_config_path = '/home/ruofan/git_space/ScamDet/model_chain/dynaphish/configs.yaml' # todo: copy a new one 33 | phishintention_cls.reset_model(phishintention_config_path, False) 34 | 35 | API_KEY, SEARCH_ENGINE_ID = [x.strip() for x in open('./datasets/google_api_key.txt').readlines()] 36 | KnowledgeExpansionModule = BrandKnowledgeConstruction(API_KEY, SEARCH_ENGINE_ID, phishintention_cls, 37 | proxies={"http": "http://127.0.0.1:7890", 38 | "https": "http://127.0.0.1:7890", 39 | }) 40 | 41 | mmocr_model = MMOCRInferencer(det=None, 42 | rec='ABINet', 43 | device='cuda') 44 | button_locator_model = SubmissionButtonLocator( 45 | button_locator_config="/home/ruofan/git_space/MyXdriver_pub/xutils/forms/button_locator_models/config.yaml", 46 | button_locator_weights_path="/home/ruofan/git_space/MyXdriver_pub/xutils/forms/button_locator_models/model_final.pth") 47 | 48 | dynaphish_cls = DynaPhish(phishintention_cls, 49 | phishintention_config_path, 50 | None, 51 | KnowledgeExpansionModule) 52 | 53 | os.makedirs('./field_study/results/', exist_ok=True) 54 | result_txt = './field_study/results/{}_{}.txt'.format(args.date, args.method) 55 | 56 | if not os.path.exists(result_txt): 57 | with open(result_txt, "w+") as f: 58 | f.write("folder" + "\t") 59 | f.write("phish_prediction" + "\t") 60 | f.write("target_prediction" + "\t") # write top1 prediction only 61 | f.write("runtime" + "\n") 62 | 63 | for ct, folder in tqdm(enumerate(os.listdir(args.folder))): 64 | if folder in [x.split('\t')[0] for x in open(result_txt, encoding='ISO-8859-1').readlines()]: 65 | continue 66 | 67 | info_path = os.path.join(args.folder, folder, 'info.txt') 68 | html_path = os.path.join(args.folder, folder, 'html.txt') 69 | shot_path = os.path.join(args.folder, folder, 'shot.png') 70 | predict_path = os.path.join(args.folder, folder, 'predict_{}.png'.format(args.method)) 71 | if not os.path.exists(shot_path): 72 | continue 73 | 74 | try: 75 | if len(open(info_path, encoding='ISO-8859-1').read()) > 0: 76 | url = open(info_path, encoding='ISO-8859-1').read() 77 | else: 78 | url = 'https://' + folder 79 | except: 80 | url = 'https://' + folder 81 | 82 | try: 83 | if args.method == 'phishpedia': 84 | pred, brand, runtime, plotvis = base_cls.test_phishpedia(url, shot_path) 85 | elif args.method == 'phishintention': 86 | pred, brand, runtime, plotvis = base_cls.test_phishintention(url, shot_path, driver) 87 | elif args.method == 'dynaphish': 88 | pred, brand, plotvis, _, _, \ 89 | _, _, runtime_breakdown, _ = \ 90 | dynaphish_cls.test_dynaphish(URL=url, 91 | screenshot_path=shot_path, 92 | kb_driver=driver, 93 | base_model='phishintention', 94 | knowledge_expansion_branch='logo2brand', 95 | kb_enabled=True) 96 | parts = runtime_breakdown.split('|') 97 | float_parts = [float(part) for part in parts] 98 | runtime = sum(float_parts) 99 | 100 | except KeyError: 101 | continue 102 | 103 | try: 104 | with open(result_txt, "a+", encoding='ISO-8859-1') as f: 105 | f.write(folder + "\t") 106 | f.write(str(pred) + "\t") 107 | f.write(str(brand) + "\t") # write top1 prediction only 108 | f.write(str(runtime) + "\n") 109 | if pred == 1: 110 | cv2.imwrite(predict_path, plotvis) 111 | 112 | except UnicodeEncodeError: 113 | continue 114 | 115 | if (ct + 501) % 500 == 0: 116 | if args.method != 'phishpedia': 117 | driver.quit() 118 | driver = CustomWebDriver.boot(proxy_server="127.0.0.1:7890") # Using the proxy_url variable 119 | driver.set_script_timeout(timeout_time / 2) 120 | driver.set_page_load_timeout(timeout_time) 121 | 122 | if args.method != 'phishpedia': 123 | driver.quit() -------------------------------------------------------------------------------- /figures/phishllm.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/figures/phishllm.pdf -------------------------------------------------------------------------------- /figures/phishllm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/figures/phishllm.png -------------------------------------------------------------------------------- /param_dict.yaml: -------------------------------------------------------------------------------- 1 | ocr: # paddleOCR model 2 | sure_thre: 0.98 # OCR will iterate through all supported languages, if one of the language exceeds 0.98, use this language 3 | unsure_thre: 0.9 # OCR will iterate through all supported languages, if none of the language exceeds 0.98, but one of it exceeds 0.9 and is higher than the previous 2 languages, use this language 4 | local_best_window: 2 # higher than the previous 2 languages 5 | supported_langs: # all supported languages, you can add more from https://github.com/Mushroomcat9998/PaddleOCR/blob/main/doc/doc_en/multi_languages_en.md#5-support-languages-and-abbreviations 6 | - 'en' 7 | - 'ch' 8 | - 'ru' 9 | - 'japan' 10 | - 'fa' 11 | - 'korean' 12 | - 'fr' 13 | - 'german' 14 | 15 | logo_caption: # image captioning model 16 | model_name: "blip_caption" # model name 17 | model_type: "base_coco" # 18 | expand_ratio: 1.5 # look at the logo region and expand this region by a factor of 1.5 to look at its surrounding things as well 19 | 20 | LLM_model: "gpt-3.5-turbo-16k" # use gpt3.5 for LLM 21 | 22 | brand_recog: # brand recognition model 23 | temperature: 0 # deterministic response 24 | max_tokens: 10 # limit the maximum number of generated tokens 25 | sleep_time: 0.5 # 26 | prompt_path: "./prompts/brand_recog_prompt.json" # path to the prompt 27 | ask_industry: True # ask the industry sector as well to give more information to the LLM to infer the brand 28 | industry: 29 | temperature: 0 30 | max_tokens: 5 31 | 32 | brand_valid: 33 | activate: False # whether to activate the brand validation? 34 | k: 10 # look at the top-10 google image results to check whether the webpage logo is similar to any one of them 35 | siamese_thre: 0.7 # whether the webpage logo is similar to any one of them with similarity threshold as 0.7 36 | 37 | crp_pred: # CRP prediction model 38 | temperature: 0 # deterministic response 39 | max_tokens: 200 # limit the maximum number of generated tokens 40 | sleep_time: 0.5 41 | prompt_path: "./prompts/crp_pred_prompt.json" # path to the prompt 42 | 43 | rank: # CRP transition model 44 | model_name: "ViT-B/32" # use vision transformer as backbone 45 | checkpoint_path: "./checkpoints/epoch4_model.pt" # checkpoint path 46 | max_uis_process: 50 # only look at the first 50 UI elements because the login UI is likely to be located on the top of the screenshot 47 | batch_size: 64 48 | depth_limit: 1 49 | driver_sleep_time: 3 50 | script_timeout: 5 51 | page_load_timeout: 5 52 | -------------------------------------------------------------------------------- /prompts/brand_recog_prompt.json: -------------------------------------------------------------------------------- 1 | [{"role": "system", 2 | "content": "You are knowledgeable about brands and their associated logos. Given the description of a logo, logo's OCR text, and the industry sector, your task is to decide the brand of the logo. Logo OCR is more important than logo's description. Just give the brand's domain in English, do not output any explanation. If there are multiple possible brands, output the most likely domain."}, 3 | 4 | {"role": "user", "content": "Given the following description on the brand's logo: 'the logo for icy vein news and guides', the logo's OCR text: 'GO PREMIUM ICY VEINS M NoW', and the industry sector: 'Entertainment', Question: What is the brand's domain? Answer: "}, 5 | 6 | {"role": "assistant", "content": "icy-veins.com"} 7 | ] -------------------------------------------------------------------------------- /prompts/crp_pred_prompt.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"role": "system", 3 | "content": "Given the webpage HTML, your task is to decide the status of the webpage. A credential-requiring page is where the users are asked to fill-in their sensitive information, including usernames, passwords; contact details such as addresses, phone numbers, emails, and financial information such as credit card numbers, social security numbers etc. Please first delete the tokens that may confuse your decision or contraditionary to the rest of the HTML content."}, 4 | 5 | {"role": "user", 6 | "content": "Given the HTML webpage text: MAX BOUNTY Affiliate Login: Email address Password Sign in +Forgot your password? MaxBounty Inci[2] non-credential-requiring, \n Question: A. This is a credential-requiring page. B. This is not a credential-requiring page. \n Answer: "}, 7 | 8 | {"role": "assistant", 9 | "content": "The token 'non-credential-requiring' may confuse your decision. Therefore we delete the token 'non-credential-requiring'. Then we filter the keywords that are related to sensitive information: Email address, Password. After that we find the keywords that are related to login: Sign in, Login. Therefore the answer would be A."}, 10 | 11 | {"role": "user", 12 | "content": "Given the HTML webpage text: 360\u5bfc\u822a \u8d44\u8baf \u89c6\u9891 \u56fe\u7247 \u826f\u533b \u5730\u56fe \u767e\u79d1 \u6587\u5e93 \u95ee\u7b54 \u66f4\u591a \u8bbe\u7f6e \u9690\u85cf\u8d44\u8baf \u6362\u80a4 \u767b\u5f55 360\u641c\u7d20 \u641c\u7d22 \u610f\u89c1\u53cd\u9988\u8fdd\u6cd5\u4e3e\u62a5\u672a\u6210\u5e74\u4eba\u4e3e\u62a5\u4f7f\u7528\u534f\u8bae\u9690\u79c1\u7ba1\u7406\u9690\u79c1\u6761\u6b3e\u514d\u8d23\u58f0\u660e\u63a8\u5e7f\u5408\u4f5c360\u641c\u7d22\u8054\u76df, \n Question: A. This is a credential-requiring page. B. This is not a credential-requiring page. \n Answer: "}, 13 | 14 | {"role": "assistant", 15 | "content": "There is no confusing token. Then we find the keywords that are related to login: 登录. But there is no keyword related to sensitive information. Therefore the answer would be B."} 16 | 17 | ] -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client 2 | openai 3 | Pillow 4 | numpy 5 | lxml 6 | ftfy 7 | regex 8 | tqdm 9 | pandas 10 | git+https://github.com/openai/CLIP.git 11 | openai 12 | backoff 13 | idna 14 | geopandas 15 | seaborn 16 | python-Levenshtein 17 | tldextract 18 | scikit-learn 19 | matplotlib 20 | opencv-python 21 | opencv-contrib-python 22 | transformers==4.25 23 | tldextract 24 | -------------------------------------------------------------------------------- /scripts/infer/test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | from scripts.pipeline.test_llm import * 5 | import argparse 6 | from tqdm import tqdm 7 | import yaml 8 | import openai 9 | from datetime import datetime, date, timedelta 10 | os.environ["CUDA_LAUNCH_BLOCKING"] = "1" 11 | os.environ['OPENAI_API_KEY'] = open('./datasets/openai_key.txt').read().strip() 12 | 13 | if __name__ == '__main__': 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--folder", default="./datasets/field_study/2023-09-02/") 17 | parser.add_argument("--config", default='./param_dict.yaml', help="Config .yaml path") 18 | args = parser.parse_args() 19 | 20 | PhishLLMLogger.set_debug_on() 21 | PhishLLMLogger.set_verbose(True) 22 | 23 | # load hyperparameters 24 | with open(args.config) as file: 25 | param_dict = yaml.load(file, Loader=yaml.FullLoader) 26 | 27 | # PhishLLM 28 | proxy_url = os.environ.get('proxy_url', None) 29 | phishintention_cls = PhishIntentionWrapper() 30 | llm_cls = TestLLM(phishintention_cls, 31 | param_dict=param_dict, 32 | proxies={"http": proxy_url, 33 | "https": proxy_url, 34 | }) 35 | openai.api_key = os.getenv("OPENAI_API_KEY") 36 | openai.proxy = proxy_url # set openai proxy 37 | 38 | # boot driver 39 | driver = CustomWebDriver.boot(proxy_server=proxy_url) # Using the proxy_url variable 40 | driver.set_script_timeout(param_dict['rank']['script_timeout']) 41 | driver.set_page_load_timeout(param_dict['rank']['page_load_timeout']) 42 | 43 | day = date.today().strftime("%Y-%m-%d") 44 | result_txt = '{}_phishllm.txt'.format(day) 45 | 46 | if not os.path.exists(result_txt): 47 | with open(result_txt, "w+") as f: 48 | f.write("folder" + "\t") 49 | f.write("phish_prediction" + "\t") 50 | f.write("target_prediction" + "\t") # write top1 prediction only 51 | f.write("brand_recog_time" + "\t") 52 | f.write("crp_prediction_time" + "\t") 53 | f.write("crp_transition_time" + "\n") 54 | 55 | for ct, folder in tqdm(enumerate(os.listdir(args.folder))): 56 | if folder in [x.split('\t')[0] for x in open(result_txt, encoding='ISO-8859-1').readlines()]: 57 | continue 58 | 59 | info_path = os.path.join(args.folder, folder, 'info.txt') 60 | html_path = os.path.join(args.folder, folder, 'html.txt') 61 | shot_path = os.path.join(args.folder, folder, 'shot.png') 62 | predict_path = os.path.join(args.folder, folder, 'predict.png') 63 | if not os.path.exists(shot_path): 64 | continue 65 | 66 | try: 67 | if len(open(info_path, encoding='ISO-8859-1').read()) > 0: 68 | url = open(info_path, encoding='ISO-8859-1').read() 69 | else: 70 | url = 'https://' + folder 71 | except FileNotFoundError: 72 | url = 'https://' + folder 73 | 74 | logo_box, reference_logo = llm_cls.detect_logo(shot_path) 75 | while True: 76 | try: 77 | pred, brand, brand_recog_time, crp_prediction_time, crp_transition_time, plotvis = llm_cls.test(url=url, 78 | reference_logo=reference_logo, 79 | logo_box=logo_box, 80 | shot_path=shot_path, 81 | html_path=html_path, 82 | driver=driver, 83 | ) 84 | driver.delete_all_cookies() 85 | break 86 | 87 | except (WebDriverException) as e: 88 | print(f"Driver crashed or encountered an error: {e}. Restarting driver.") 89 | driver.quit() 90 | time.sleep(1) 91 | driver = CustomWebDriver.boot(proxy_server=proxy_url) # Using the proxy_url variable 92 | driver.set_script_timeout(param_dict['rank']['script_timeout']) 93 | driver.set_page_load_timeout(param_dict['rank']['page_load_timeout']) 94 | continue 95 | 96 | try: 97 | with open(result_txt, "a+", encoding='ISO-8859-1') as f: 98 | f.write(f"{folder}\t{pred}\t{brand}\t{brand_recog_time}\t{crp_prediction_time}\t{crp_transition_time}\n") 99 | if pred == 'phish': 100 | plotvis.save(predict_path) 101 | 102 | except UnicodeEncodeError: 103 | continue 104 | 105 | 106 | driver.quit() 107 | 108 | -------------------------------------------------------------------------------- /scripts/train/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from torch import nn, optim 4 | from PIL import Image 5 | from torch.utils.data import DataLoader 6 | import torch 7 | import clip 8 | try: 9 | from torchvision.transforms import InterpolationMode 10 | BICUBIC = InterpolationMode.BICUBIC 11 | except ImportError: 12 | BICUBIC = Image.BICUBIC 13 | from tqdm import tqdm 14 | from scripts.data.data_utils import ButtonDataset, BalancedBatchSampler 15 | import argparse 16 | 17 | def trainer(epoch, model, train_dataloader, device, lr): 18 | loss_img = nn.CrossEntropyLoss() 19 | optimizer = optim.Adam(model.visual.parameters(), lr=lr) # only change the image encoder part 20 | scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, len(train_dataloader) * epoch) 21 | model.train() 22 | model.token_embedding.eval() 23 | model.transformer.eval() 24 | model.ln_final.eval() 25 | 26 | model.positional_embedding.requires_grad = False 27 | model.text_projection.requires_grad = False 28 | model.logit_scale.requires_grad = False 29 | 30 | for ep in range(epoch): 31 | print(f"Running epoch {ep}") 32 | step = 0 33 | tr_loss = 0 34 | 35 | pbar = tqdm(train_dataloader, leave=False) 36 | for batch in pbar: 37 | step += 1 38 | 39 | images, ground_truth, *_ = batch 40 | images = images.to(device) 41 | texts = clip.tokenize(["not a login button", "a login button"]).to(device) 42 | logits_per_image, logits_per_text = model(images, texts) 43 | ground_truth = ground_truth.to(device) 44 | total_loss = loss_img(logits_per_image, ground_truth) # only cross entropy for images 45 | 46 | optimizer.zero_grad() 47 | total_loss.backward() 48 | tr_loss += total_loss.item() 49 | if device == "cpu": 50 | optimizer.step() 51 | scheduler.step() 52 | else: 53 | convert_models_to_fp32(model) 54 | optimizer.step() 55 | scheduler.step() 56 | clip.model.convert_weights(model) 57 | pbar.set_description(f"Train batch CE: {total_loss.item()}", refresh=True) 58 | tr_loss /= step 59 | 60 | os.makedirs("./checkpoints", exist_ok=True) 61 | torch.save(model.state_dict(), f"./checkpoints/epoch{ep}_model.pt") 62 | print(f"Epoch {ep}, tr_loss {tr_loss}") 63 | def convert_models_to_fp32(model): 64 | for p in model.visual.parameters(): 65 | p.data = p.data.float() 66 | if p.grad is not None: 67 | p.grad.data = p.grad.data.float() 68 | 69 | def main(args): 70 | device = "cuda" if torch.cuda.is_available() else "cpu" 71 | model, preprocess = clip.load("ViT-B/32", device=device, jit=False) 72 | 73 | # Adjust model for CPU if necessary 74 | if device == "cpu": 75 | model.float() 76 | 77 | train_dataset = ButtonDataset( 78 | annot_path=args.annot_path, 79 | root=args.dataset_root, 80 | preprocess=preprocess 81 | ) 82 | 83 | train_sampler = BalancedBatchSampler(train_dataset.labels, args.batch_size) 84 | train_dataloader = DataLoader(train_dataset, batch_sampler=train_sampler) 85 | print(f"Number of batches: {len(train_dataloader)}") 86 | 87 | trainer(args.epoch, model, train_dataloader, device, args.lr) 88 | 89 | if __name__ == '__main__': 90 | parser = argparse.ArgumentParser(description="Train a CLIP model on button images") 91 | parser.add_argument('--epoch', type=int, default=5, help='Number of epochs to train') 92 | parser.add_argument('--batch_size', type=int, default=128, help='Batch size for training') 93 | parser.add_argument('--lr', type=float, default=1e-5, help='Learning rate') 94 | parser.add_argument('--annot_path', type=str, required=True, help='Path to the annotation file') 95 | parser.add_argument('--dataset_root', type=str, required=True, help='Root directory of the dataset') 96 | 97 | args = parser.parse_args() 98 | main(args) -------------------------------------------------------------------------------- /scripts/utils/dynaphish/configs.yaml: -------------------------------------------------------------------------------- 1 | # please set the environment variable first 2 | # echo 'export ANACONDA_ENV_PATH="/home/xxx/anaconda3/envs/myenv"' >> ~/.bashrc 3 | # source ~/.bashrc 4 | 5 | AWL_MODEL: 6 | CFG_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/AWL_detector_utils/configs/faster_rcnn_web.yaml" 7 | WEIGHTS_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/AWL_detector_utils/output/website_lr0.001/model_final.pth" 8 | 9 | CRP_CLASSIFIER: 10 | WEIGHTS_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/crp_classifier_utils/output/Increase_resolution_lr0.005/BiT-M-R50x1V2_0.005.pth.tar" 11 | MODEL_TYPE: 'mixed' 12 | 13 | CRP_LOCATOR: 14 | CFG_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/crp_locator_utils/login_finder/configs/faster_rcnn_login_lr0.001_finetune.yaml" 15 | WEIGHTS_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/crp_locator_utils/login_finder/output/lr0.001_finetune/model_final.pth" 16 | 17 | SIAMESE_MODEL: 18 | NUM_CLASSES: 277 19 | WEIGHTS_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/OCR_siamese_utils/output/targetlist_lr0.01/bit.pth.tar" 20 | OCR_WEIGHTS_PATH: "${ANACONDA_ENV_PATH}/lib/python3.8/site-packages/phishintention/src/OCR_siamese_utils/demo_downgrade.pth.tar" 21 | TARGETLIST_PATH: "expand_targetlist.zip" # Relative path 22 | MATCH_THRE: 0.87 23 | DOMAIN_MAP_PATH: "domain_map.pkl" # Relative path 24 | -------------------------------------------------------------------------------- /scripts/utils/dynaphish/google_safebrowsing.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import requests 3 | import json 4 | 5 | class SafeBrowsingInvalidApiKey(Exception): 6 | def __init__(self): 7 | Exception.__init__(self, "Invalid API key for Google Safe Browsing") 8 | 9 | class SafeBrowsingPermissionDenied(Exception): 10 | def __init__(self, detail): 11 | Exception.__init__(self, detail) 12 | 13 | class SafeBrowsingWeirdError(Exception): 14 | def __init__(self, code, status, message): 15 | self.message = "%s(%i): %s" % ( 16 | status, 17 | code, 18 | message 19 | ) 20 | Exception.__init__(self, message) 21 | 22 | 23 | def chunks(lst, n): 24 | """Yield successive n-sized chunks from lst.""" 25 | for i in range(0, len(lst), n): 26 | yield lst[i:i + n] 27 | 28 | 29 | class SafeBrowsing(object): 30 | def __init__(self, key, api_url='https://safebrowsing.googleapis.com/v4/threatMatches:find'): 31 | self.api_key = key 32 | self.api_url = api_url 33 | 34 | def lookup_urls(self, urls, platforms=["ANY_PLATFORM"]): 35 | results = {} 36 | for urll in chunks(urls, 50): 37 | data = { 38 | "client": { 39 | "clientId": "pysafebrowsing", 40 | "clientVersion": "1.5.2" 41 | }, 42 | "threatInfo": { 43 | "threatTypes": 44 | [ 45 | "MALWARE", 46 | "SOCIAL_ENGINEERING", 47 | "THREAT_TYPE_UNSPECIFIED", 48 | "UNWANTED_SOFTWARE", 49 | "POTENTIALLY_HARMFUL_APPLICATION" 50 | ], 51 | "platformTypes": platforms, 52 | "threatEntryTypes": ["URL"], 53 | "threatEntries": [{'url': u} for u in urll], 54 | } 55 | } # include all threattypes 56 | headers = {'Content-type': 'application/json'} 57 | 58 | try: 59 | r = requests.post( 60 | self.api_url, 61 | data=json.dumps(data), 62 | params={'key': self.api_key}, 63 | headers=headers 64 | ) 65 | except requests.exceptions.ConnectionError: 66 | results.update(dict([(u, {"malicious": False}) for u in urls])) 67 | continue 68 | 69 | if r.status_code == 200: 70 | # Return clean results 71 | if r.json() == {}: 72 | results.update(dict([(u, {"malicious": False}) for u in urls])) 73 | else: 74 | for url in urll: 75 | # Get matches 76 | matches = [match for match in r.json()['matches'] if match['threat']['url'] == url] 77 | if len(matches) > 0: 78 | results[url] = { 79 | 'malicious': True, 80 | 'platforms': list(set([b['platformType'] for b in matches])), 81 | 'threats': list(set([b['threatType'] for b in matches])), 82 | 'cache': min([b["cacheDuration"] for b in matches]) 83 | } 84 | else: 85 | results[url] = {"malicious": False} 86 | else: 87 | results.update(dict([(u, {"malicious": False}) for u in urls])) 88 | continue 89 | 90 | return results 91 | 92 | def lookup_url(self, url, platforms=["ANY_PLATFORM"]): 93 | """ 94 | Online lookup of a single url 95 | """ 96 | r = self.lookup_urls([url], platforms=platforms) 97 | return r[url] 98 | 99 | -------------------------------------------------------------------------------- /scripts/utils/logger_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | 4 | class TxtColors: 5 | OK = '\033[92m' 6 | DEBUG = '\033[94m' 7 | WARNING = "\033[93m" 8 | FATAL = '\033[91m' 9 | EXCEPTION = '\033[100m' 10 | ENDC = '\033[0m' 11 | 12 | '''Logging Utils''' 13 | class PhishLLMLogger(): 14 | _caller_prefix = "PhishLLMLogger" 15 | _verbose = True 16 | _logfile = None 17 | _debug = False # Off by default 18 | _warning = True 19 | 20 | @classmethod 21 | def set_verbose(cls, verbose): 22 | cls._verbose = verbose 23 | 24 | @classmethod 25 | def set_logfile(cls, logfile): 26 | # if os.path.isfile(logfile): 27 | # os.remove(logfile) # Remove the existing log file 28 | PhishLLMLogger._logfile = logfile 29 | 30 | @classmethod 31 | def unset_logfile(cls): 32 | PhishLLMLogger.set_logfile(None) 33 | 34 | @classmethod 35 | def set_debug_on(cls): 36 | PhishLLMLogger._debug = True 37 | 38 | @classmethod 39 | def set_debug_off(cls): # Call if need to turn debug messages off 40 | PhishLLMLogger._debug = False 41 | 42 | @classmethod 43 | def set_warning_on(cls): 44 | PhishLLMLogger._warning = True 45 | 46 | @classmethod 47 | def set_warning_off(cls): # Call if need to turn warnings off 48 | PhishLLMLogger._warning = False 49 | 50 | @classmethod 51 | def spit(cls, msg, warning=False, debug=False, error=False, exception=False, caller_prefix=""): 52 | logging.basicConfig(level=logging.DEBUG if PhishLLMLogger._debug else logging.WARNING) 53 | caller_prefix = f"[{caller_prefix}]" if caller_prefix else "" 54 | prefix = "[FATAL]" if error else "[DEBUG]" if debug else "[WARNING]" if warning else "[EXCEPTION]" if exception else "" 55 | logger = logging.getLogger("custom_logger") # Choose an appropriate logger name 56 | if PhishLLMLogger._logfile: 57 | log_msg = re.sub(r"\033\[\d+m", "", msg) 58 | log_handler = logging.FileHandler(PhishLLMLogger._logfile, mode='a') 59 | log_formatter = logging.Formatter('%(message)s') 60 | log_handler.setFormatter(log_formatter) 61 | logger.addHandler(log_handler) 62 | logger.propagate = False 63 | logger.setLevel(logging.DEBUG if PhishLLMLogger._debug else logging.WARNING) 64 | logger.debug("%s%s %s" % (caller_prefix, prefix, log_msg)) 65 | logger.removeHandler(log_handler) 66 | else: 67 | if PhishLLMLogger._verbose: 68 | txtcolor = TxtColors.FATAL if error else TxtColors.DEBUG if debug else TxtColors.WARNING if warning else "[EXCEPTION]" if exception else TxtColors.OK 69 | # if not debug or Logger._debug: 70 | if (not debug and not warning) or (debug and PhishLLMLogger._debug) or (warning and PhishLLMLogger._warning): 71 | print("%s%s%s %s" % (txtcolor, caller_prefix, prefix, msg)) -------------------------------------------------------------------------------- /scripts/utils/web_utils/collect_html_screenshots_benign.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import shutil 3 | import time 4 | import pandas as pd 5 | from scripts.utils.web_utils.web_utils import is_uniform_color, cleanup_drivers, apply_custom_styles, init_driver_pool 6 | from multiprocessing import Pool 7 | from itertools import cycle 8 | 9 | # Function to process domains 10 | def process_domains(data): 11 | domain_list, root_dir = data 12 | drivers = init_driver_pool(size=2) # Adjust size based on your resource capacity 13 | driver_cycle = cycle(drivers) # Use itertools.cycle for round-robin usage of drivers 14 | 15 | for domain in domain_list: 16 | if len(os.listdir(root_dir)) >= 5000: 17 | cleanup_drivers(drivers) 18 | exit() 19 | driver = next(driver_cycle) 20 | target = f'https://{domain}' 21 | domain_path = os.path.join(root_dir, domain) 22 | if os.path.exists(os.path.join(domain_path, 'shot.png')): 23 | continue 24 | os.makedirs(domain_path, exist_ok=True) 25 | print(f'Target URL = {target}') 26 | 27 | try: 28 | driver.get(target) 29 | time.sleep(3) 30 | driver.scroll_to_top() 31 | apply_custom_styles(driver) 32 | 33 | # Save the page source and screenshot 34 | with open(os.path.join(domain_path, 'index.html'), 'w', encoding='utf-8') as f: 35 | f.write(driver.page_source()) 36 | with open(os.path.join(domain_path, 'info.txt'), 'w', encoding='utf-8') as f: 37 | f.write(target) 38 | driver.save_screenshot(os.path.join(domain_path, 'shot.png')) 39 | 40 | # Remove domain directory if screenshot shows mostly uniform color 41 | if is_uniform_color(os.path.join(domain_path, 'shot.png')): 42 | shutil.rmtree(domain_path) 43 | print(f'{domain} - removed, mostly uniform color.') 44 | else: 45 | print(f'{domain} - valid screenshot saved.') 46 | # Assume further actions here 47 | except Exception as e: 48 | print(f'Error with {domain}: {e}') 49 | shutil.rmtree(domain_path) 50 | 51 | cleanup_drivers(drivers) 52 | 53 | 54 | def main(): 55 | root_dir = './datasets/alexa_middle_5k' 56 | os.makedirs(root_dir, exist_ok=True) 57 | popular_1m = pd.read_csv('./datasets/tranco-top-1m.csv', header=None) 58 | middle_domains = popular_1m.iloc[:, 1].tolist()[100000:105000] 59 | 60 | # Split the domain list into chunks for multiprocessing 61 | chunk_size = len(middle_domains) // os.cpu_count() 62 | domain_chunks = [middle_domains[i:i + chunk_size] for i in range(0, len(middle_domains), chunk_size)] 63 | 64 | # Start multiprocessing 65 | try: 66 | with Pool(os.cpu_count()) as pool: 67 | pool.map(process_domains, [(chunk, root_dir) for chunk in domain_chunks]) 68 | except KeyboardInterrupt: 69 | print("Interrupted by user, cleaning up...") 70 | finally: 71 | print("Final cleanup if any") 72 | 73 | 74 | if __name__ == "__main__": 75 | 76 | main() 77 | 78 | -------------------------------------------------------------------------------- /scripts/utils/web_utils/download_github_phishing_feed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd datasets/ 4 | rm phishing-links-ACTIVE-TODAY.txt 5 | rm phishing-links-ACTIVE-NOW.txt 6 | rm phishing-links-ACTIVE.txt 7 | rm phishing-links-NEW-last-hour.txt 8 | rm phishing-links-NEW-today.txt 9 | 10 | rm phishing-domains-NEW-today.txt 11 | rm phishing-domains-NEW-last-hour.txt 12 | rm phishing-domains-ACTIVE.txt 13 | 14 | # Setting up retry variables 15 | max_retries=5 16 | retry_delay=10 # Time delay in seconds between retries 17 | 18 | # Retry loop for wget 19 | for (( i=1; i<=max_retries; i++ )); do 20 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-TODAY.txt 21 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt 22 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE.txt 23 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-last-hour.txt 24 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt 25 | 26 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-NEW-today.txt 27 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-NEW-last-hour.txt 28 | https_proxy=127.0.0.1:7890 wget https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt 29 | 30 | if [ $? -eq 0 ]; then 31 | echo "Download successful." 32 | break 33 | else 34 | echo "Download failed. Retrying in $retry_delay seconds... (Attempt $i of $max_retries)" 35 | sleep $retry_delay 36 | fi 37 | done 38 | cd ../ -------------------------------------------------------------------------------- /server/.gitignore: -------------------------------------------------------------------------------- 1 | logs -------------------------------------------------------------------------------- /server/README.md: -------------------------------------------------------------------------------- 1 | # PhishLLM Interface 2 | 3 | Interface architecture 4 | - client: HTML, CSS, JS with picoCSS for styling 5 | - server: Flask as a bridge between PhishLLM and client 6 | 7 | Overview 8 | - `/static`: JS files for client interaction 9 | - `/templates`: static client UI 10 | - `announcer.py`: handles responses, act as messaging bridge between PhishLLM and client 11 | - `server.py`: handles requests from client 12 | 13 | ## Announcer 14 | `.spit()` method converts PhishLLM logs into SSE messages and send it to a user session. Must specify event type: 15 | - `AnnouncerEvent.PROMPT`: this message is a prompt for PhishLLM 16 | - `AnnouncerEvent.RESPONSE`: this message is a response from PhishLLM 17 | - `AnnouncerEvent.SUCCESS`: PhishLLM has finished analysing, terminates user session 18 | - `AnnouncerEvent.FAIL`: something went wrong, terminates user session 19 | 20 | ## Requirements 21 | ```txt 22 | flask_cors 23 | flask_session 24 | gevent 25 | apscheduler 26 | ``` 27 | 28 | ## Usage 29 | Start the server 30 | ``` 31 | python -m server.server 32 | ``` -------------------------------------------------------------------------------- /server/__init__.py: -------------------------------------------------------------------------------- 1 | from server import * -------------------------------------------------------------------------------- /server/announcer.py: -------------------------------------------------------------------------------- 1 | import queue 2 | 3 | class AnnouncerPrompt: 4 | @staticmethod 5 | def question_template_prediction(html_text): 6 | return f"Given the HTML webpage text, Question: \ 7 | A. This is a credential-requiring page. B.\ 8 | This is not a credential-requiring page.
Answer: " 9 | 10 | @staticmethod 11 | def question_template_brand(logo_caption, logo_ocr): 12 | return f"Given the description on the brand's logo, and the logo's OCR text \ 13 | Question: What is the brand's domain?
Answer: " 14 | 15 | @staticmethod 16 | def question_template_brand_industry(logo_caption, logo_ocr, industry): 17 | return f"Given the description on the brand's logo, the logo's OCR text, and the industry sector.\ 18 | Question: What is the brand's domain?
Answer: " 19 | 20 | @staticmethod 21 | def question_template_industry(html_text): 22 | return f"Your task is to predict the industry sector given webpage content. \ 23 | Only give the industry sector, do not output any explanation.\ 24 | Given the webpage text, Question: What is the webpage's industry sector?
Answer: " 25 | 26 | class AnnouncerEvent: 27 | PROMPT = "prompt" 28 | RESPONSE = "response" 29 | FAIL = "fail" 30 | SUCCESS = "success" 31 | 32 | class Announcer: 33 | def __init__(self): 34 | """ 35 | Each Announcer instance keeps track of a single user session. 36 | Creates a queue to store server-side events (SSE) messages. 37 | Sends output logs from PhishLLM to the user via SSE. 38 | """ 39 | self.spit = self._spit 40 | self.message_queue = queue.Queue(maxsize=1) 41 | 42 | def format(self, msg: str, event: str) -> str: 43 | """ 44 | Format a string as a SSE message. 45 | """ 46 | msg = msg.replace(" \n ", "
") 47 | return f'event: {event}\ndata: {msg}\n\n' 48 | 49 | @staticmethod 50 | def spit(msg: str, event: str): 51 | """ 52 | Ignore PhishLLM output log if Announcer is not an instance (no defined user session) 53 | """ 54 | return 55 | 56 | def _spit(self, msg: str, event: str): 57 | """ 58 | Convert PhishLLM output log into SSE message, send to user 59 | """ 60 | try: 61 | msg = self.format(msg, event) 62 | self.message_queue.put_nowait(msg) 63 | except: 64 | pass 65 | 66 | announcer = Announcer() -------------------------------------------------------------------------------- /server/static/facebookchatone.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/PhishLLM/9c3a337b25d6ee8a7b48c85d04a50721841f01c7/server/static/facebookchatone.mp3 -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | install_dir="/tmp/" 2 | 3 | function success { 4 | echo "[+] $1" 5 | } 6 | 7 | function fail { 8 | echo "[-] $1" 9 | } 10 | 11 | function warn { 12 | echo "[!] $1" 13 | } 14 | 15 | function prompt { 16 | ret="" 17 | while true; do 18 | read -p "$1 [y/n]: " yn 19 | case $yn in 20 | [Yy]* ) ret=1; break;; 21 | [Nn]* ) ret=0; break;; 22 | * ) echo "Please answer yes or no.";; 23 | esac 24 | done 25 | return $ret 26 | } 27 | 28 | function install_chrome { 29 | debfile="google-chrome-stable_current_amd64.deb" 30 | wget "https://dl.google.com/linux/direct/$debfile" -P "$install_dir" 31 | if [ $? -ne 0 ]; 32 | then 33 | fail "Could not download Chrome" 34 | return 1 35 | fi 36 | sudo dpkg -i "$install_dir$debfile" 37 | if [ $? -ne 0 ]; 38 | then 39 | fail "Could not install Chrome package" 40 | return 2 41 | fi 42 | success "Successfully installed Chrome" 43 | return 0 44 | } 45 | 46 | 47 | declare -A browsers 48 | browsers=(["google-chrome-stable"]=install_chrome) 49 | 50 | function check_browsers { 51 | for browser in ${!browsers[@]}; 52 | do 53 | installed=false 54 | sudo dpkg -l "$browser" > /dev/null 2>&1 55 | if [ $? -eq 0 ]; 56 | then 57 | success "$browser is installed. (version: $($browser --version))" 58 | installed=true 59 | else 60 | warn "$browser does not seem to be installed" 61 | prompt "Do you want to install its latest stable version?" 62 | if [ $? -eq 1 ]; 63 | then 64 | success "Installing $browser" 65 | ${browsers[$browser]} 66 | if [ $? -eq 0 ]; 67 | then 68 | installed=true 69 | fi 70 | else 71 | fail "Skipping $browser installation" 72 | fi 73 | fi 74 | echo -e "" 75 | done 76 | return 0 77 | } 78 | 79 | # Install chrome binary 80 | #check_browsers 81 | 82 | # Create a new conda environment with Python 3.8 83 | # Check if the environment already exists 84 | conda info --envs | grep -w "$ENV_NAME" > /dev/null 85 | if [ $? -eq 0 ]; then 86 | echo "Activating Conda environment $ENV_NAME" 87 | else 88 | echo "Creating and activating new Conda environment $ENV_NAME with Python 3.8" 89 | conda create -n "$ENV_NAME" python=3.8 90 | fi 91 | 92 | PACKAGE_NAME="phishintention" 93 | if conda list -n "$ENV_NAME" | grep -q "$PACKAGE_NAME"; then 94 | echo "$PACKAGE_NAME is already installed, skip installation" 95 | elif [ -d "PhishIntention" ]; then 96 | echo "Directory PhishIntention already exists, skip cloning" 97 | cd PhishIntention 98 | chmod +x ./setup.sh 99 | export ENV_NAME="$ENV_NAME" && ./setup.sh 100 | cd ../ 101 | rm -rf PhishIntention 102 | else 103 | git clone -b development --single-branch https://github.com/lindsey98/PhishIntention.git 104 | cd PhishIntention 105 | chmod +x ./setup.sh 106 | export ENV_NAME="$ENV_NAME" && ./setup.sh 107 | cd ../ 108 | rm -rf PhishIntention 109 | fi 110 | 111 | # Install PaddleOCR 112 | if command -v nvcc &> /dev/null; then 113 | # cuda is available 114 | conda run -n "$ENV_NAME" pip install paddlepaddle-gpu -i https://pypi.tuna.tsinghua.edu.cn/simple 115 | else # cpu-only 116 | conda run -n "$ENV_NAME" pip install paddlepaddle -i https://pypi.tuna.tsinghua.edu.cn/simple 117 | fi 118 | conda run -n "$ENV_NAME" pip install "paddleocr>=2.0.1" 119 | 120 | # Install Image Captioning model 121 | PACKAGE_NAME="lavis" 122 | if conda list -n "$ENV_NAME" | grep -q "$PACKAGE_NAME"; then 123 | echo "$PACKAGE_NAME is already installed, skip installation" 124 | else 125 | git clone https://github.com/lindsey98/LAVIS.git 126 | cd LAVIS 127 | conda run -n "$ENV_NAME" pip install -e . 128 | cd ../ 129 | rm -rf LAVIS 130 | fi 131 | 132 | ## Install other requirements 133 | if [ -f requirements.txt ]; then 134 | conda run -n "$ENV_NAME" pip install -r requirements.txt 135 | else 136 | echo "requirements.txt not found. Skipping additional package installations." 137 | fi 138 | 139 | # Download the ranking model 140 | mkdir -p checkpoints 141 | cd checkpoints 142 | conda run -n "$ENV_NAME" pip install gdown 143 | conda run -n "$ENV_NAME" gdown --id 1bpy-SRDOkL96j9r3ErBd7L5mDUdLAWaU 144 | --------------------------------------------------------------------------------