├── README.md ├── awvs_script.py ├── config.yaml ├── log ├── add_log.txt └── error_url.txt ├── pic └── pic01.png ├── proxy_pool ├── README.md ├── config.yml ├── goProxyPool-linux-amd64 └── goProxyPool-windows-amd64.exe └── url.txt /README.md: -------------------------------------------------------------------------------- 1 | # awvs-scan-script 2 | 3 | ## 脚本介绍 4 | 5 | 调用Acunetix AWVS的API实现批量扫描，并且使用代理池（项目地址[goProxyPool](https://github.com/pingc0y/go_proxy_pool)），实现批量扫描时的每个扫描目标都使用不同的代理IP。提供常驻后台监控功能，控制最大扫描任务数量以及最大扫描时间。 6 | 7 | ## 脚本功能 8 | 9 | * 仅批量添加目标（可设置添加数量），并设置扫描使用的代理，不进行扫描（会同时删除存放URL的文件中已经成功添加到扫描器的URL，并记录到log文件夹下） 10 | * 批量添加目标（可设置添加数量），并使用代理开始扫描（会同时删除存放URL的文件中已经成功添加到扫描器的URL，并记录到log文件夹下） 11 | * 对扫描器内已有目标进行扫描 12 | * 中止所有扫描任务 13 | * 获取扫描失败的目标，删除扫描器中扫描失败的目标，同时将扫描失败的URL保存在`log/error_url.txt`中 14 | * 删除扫描器中的所有扫描任务和目标 15 | * 对扫描失败的目标重新扫描，即将保存在`log/error_url.txt`中的URL重新添加到扫描器并开始扫描。执行该操作前请先执行【获取扫描失败的目标】。 16 | * **新增常驻后台监控功能，控制最大扫描任务数量以及最大扫描时间：脚本定时检查扫描器中正在进行的扫描任务，若未达到最大扫描任务数量，会自动添加扫描任务；同时会定时检查是否有扫描任务的扫描时间超过最大扫描时间，脚本会自动中止超过最大扫描时间的任务。** 17 | 18 | ``` 19 | 正在扫描: 0 ，等待扫描: 0 ，漏洞数量: {'high': None, 'low': None, 'med': None} 20 | [*] 请选择要进行的操作： 21 | 1、批量添加目标，不进行扫描 22 | 2、批量添加目标并开始扫描 23 | 3、对扫描器内已有目标进行扫描 24 | 4、中止所有扫描任务 25 | 5、获取扫描失败的目标 26 | 6、对扫描失败的目标重新扫描 27 | 7、删除所有目标和扫描任务 28 | 29 | 请输入数字：3 30 | 未扫描的目标数量为: 0 31 | 请输入要开始扫描的目标个数(留空则开始扫描所有未扫描目标): 32 | ------------------------------------------------ 33 | 请输入数字：5 34 | [*] 该操作会删除扫描器中扫描失败的目标，并将扫描失败的URL保存至log文件夹下 35 | [*] 是否要删除扫描器中扫描失败的目标(y/n)： 36 | ------------------------------------------------ 37 | 请输入数字：6 38 | [*] 是否已经执行【获取扫描失败的目标】(y/n)：y 39 | [*] 正在尝试对扫描失败的目标进行重新扫描 40 | [*] 请输入要添加的目标数量(留空则添加./log/error_url.txt中全部url)： 41 | ------------------------------------------------ 42 | 请输入数字：7 43 | [*] 已删除所有目标，当前目标列表为空 44 | ``` 45 | 46 | ![](https://raw.githubusercontent.com/colind0pe/awvs-scan-script/master/pic/pic01.png) 47 | 48 | ## 使用方法 49 | 50 | 1. 启动proxy_pool目录下的代理池程序，程序会开始爬取免费代理并验证代理，大约需要三分钟。代理池程序的详细使用请访问原项目仓库：[goProxyPool](https://github.com/pingc0y/go_proxy_pool)。感谢作者[pingc0y](https://github.com/pingc0y)的贡献。 51 | 52 | 2. 在config.yaml配置文件中替换proxy_pool、awvs_url、api_key为你的代理池、awvs访问地址和API 密钥，同时可以选择修改标签和扫描速度以及扫描的类型。**新增最大扫描数量和最大扫描时间配置项。** 53 | 54 | 3. * 基本模式：执行`python3 awvs_script.py`使用基本模式 55 | * 监控模式：执行`python3 awvs_script.py monitor`使用监控模式（Linux可执行`nohup python3 -u awvs_script.py monitor &`让脚本后台运行） -------------------------------------------------------------------------------- /awvs_script.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import threading 5 | import time 6 | from datetime import datetime 7 | 8 | import requests 9 | import urllib3 10 | import yaml 11 | 12 | urllib3.disable_warnings() 13 | 14 | 15 | class Config: 16 | # 从配置文件中读取配置 17 | try: 18 | with open('config.yaml', 'r', encoding='utf-8') as f: 19 | cfg = yaml.load(f, Loader=yaml.FullLoader) 20 | proxy_pool = cfg['proxy_pool'] 21 | awvs_url = cfg['awvs_url'] 22 | api_key = cfg['api_key'] 23 | headers = {'Content-Type': 'application/json', "X-Auth": api_key} 24 | scan_label = cfg['scan_label'] 25 | scan_speed = cfg['scan_speed'] 26 | proxy_enabled = cfg['proxy_enabled'] 27 | profile_id = cfg['profile_id'][cfg['scan_mode']] 28 | max_scan_count = cfg['max_scan_count'] 29 | max_scan_time = cfg['max_scan_time'] 30 | except Exception as e: 31 | print("读取配置文件出错，请检查配置是否规范") 32 | 33 | # 验证AWVS地址和API Token 34 | def check_api(self): 35 | try: 36 | resp = requests.get(self.awvs_url + '/api/v1/me/stats', 37 | headers=self.headers, 38 | timeout=10, 39 | verify=False) 40 | if resp.status_code == 401: 41 | print("awvs认证失败，请检查您设置的api_key") 42 | sys.exit() 43 | result = json.loads(resp.content.decode()) 44 | scans_running_count, scans_waiting_count = \ 45 | result['scans_running_count'], result['scans_waiting_count'] 46 | vuln_count = result['vuln_count'] 47 | # 输出扫描基本信息 48 | print("正在扫描:", scans_running_count, "，等待扫描:", scans_waiting_count, 49 | "，漏洞数量:", vuln_count) 50 | except Exception as e: 51 | print('初始化失败，请检查您设置的awvs_url是否正确\n', e) 52 | sys.exit() 53 | return scans_running_count 54 | 55 | 56 | class Proxy: 57 | 58 | def __init__(self): 59 | self.cfg = Config() 60 | 61 | # 获取代理（如果proxy_enabled选项为False不进行抓取） 62 | def get_proxy(self): 63 | if not self.cfg.proxy_enabled: 64 | return "127.0.0.1", 8111 65 | while 1: 66 | try: 67 | proxy_str = requests.get( 68 | self.cfg.proxy_pool + 69 | "/get?type=HTTP&count=1&anonymity=all").content.decode() 70 | proxy_list = json.loads(proxy_str) 71 | proxy_ip = proxy_list['Ip'] 72 | proxy_port = proxy_list['Port'] 73 | anonymity = proxy_list['Anonymity'] 74 | # 去掉透明代理 75 | if anonymity != "透明": 76 | break 77 | except Exception as e: 78 | print("获取代理失败\n", e) 79 | sys.exit() 80 | return proxy_ip, proxy_port 81 | 82 | 83 | class GetUrl: 84 | # 从url文件中读取url（url.txt或者./log/error_url.txt） 85 | @staticmethod 86 | def get_url_from_txt(count, txt, add_log): 87 | try: 88 | url_list = [] 89 | if not os.path.getsize(txt): 90 | print("[*] {}中没有url，请将待检测的url写入txt文件".format(txt)) 91 | sys.exit() 92 | 93 | # 把添加成功的url写入add_log.txt或者readd_log.txt 94 | with open(txt, 'r') as f: 95 | for url in f.readlines()[0:count]: 96 | with open(add_log, 'a') as log: 97 | log.write(url) 98 | 99 | # url预处理 100 | if 'http' not in url[0:7]: 101 | url = "http://" + url 102 | url = url.strip().rstrip('/') 103 | url_list.append(url) 104 | except Exception as e: 105 | print("[*] 请确认{}的位置是否正确\n".format(txt), e) 106 | sys.exit() 107 | return url_list 108 | 109 | 110 | class Target: 111 | 112 | def __init__(self): 113 | self.cfg = Config() 114 | 115 | # 根据target_id获取单个目标的信息 116 | def get_target(self, target_id): 117 | try: 118 | r = requests.get(self.cfg.awvs_url + 119 | "/api/v1/targets/{}".format(target_id), 120 | headers=self.cfg.headers, 121 | timeout=10, 122 | verify=False) 123 | target = json.loads(r.content.decode()) 124 | return target 125 | except Exception as e: 126 | print("获取目标失败\n", e) 127 | 128 | # 获取扫描器中的所有目标的信息 129 | def get_targets(self): 130 | try: 131 | resp = requests.get(self.cfg.awvs_url + "/api/v1/targets", 132 | headers=self.cfg.headers, 133 | timeout=10, 134 | verify=False) 135 | result = json.loads(resp.content.decode()) 136 | return result 137 | except Exception as e: 138 | print("获取目标失败\n", e) 139 | 140 | # 批量添加目标到扫描器，并会从url文件中删除已经添加成功的url 141 | def add_targets(self, txt, add_log): 142 | count = input("[*] 请输入要添加的目标数量(留空则添加{}中全部url)：".format(txt)) or None 143 | if count is not None: 144 | count = int(count) 145 | url_list = GetUrl().get_url_from_txt(count, txt, add_log) 146 | target_id_list = [] 147 | flag = None 148 | for url in url_list: 149 | try: 150 | data = { 151 | "address": url, 152 | "description": self.cfg.scan_label, 153 | "criticality": "10" 154 | } 155 | response = requests.post(self.cfg.awvs_url + "/api/v1/targets", 156 | data=json.dumps(data), 157 | headers=self.cfg.headers, 158 | timeout=30, 159 | verify=False) 160 | result = json.loads(response.content) 161 | target_id = result['target_id'] 162 | target_address = result['address'] 163 | target_id_list.append(target_id) 164 | if response.status_code == 201: 165 | print(target_address, " 目标添加成功") 166 | flag = True 167 | except Exception as e: 168 | flag = False 169 | print("[*] 添加目标失败\n", e) 170 | 171 | # 从存放url的文件中删除已经添加成功的url 172 | if flag: 173 | try: 174 | with open(txt, 'r') as f_r: 175 | lines = f_r.readlines()[count:] 176 | with open(txt, 'w') as f_w: 177 | if count is None: 178 | lines = {} 179 | f_w.writelines(lines) 180 | except Exception as e: 181 | print("[*] 去除txt中已添加到扫描器的url时出错\n", e) 182 | self.configuration(target_id_list, self.cfg.profile_id) 183 | return target_id_list 184 | 185 | # 配置已添加到扫描器的目标的扫描参数，添加目标时会调用该方法 186 | def configuration(self, target_id_list, default_scanning_profile_id): 187 | for target_id in target_id_list: 188 | configuration_url = self.cfg.awvs_url + \ 189 | "/api/v1/targets/{}/configuration".format(target_id) 190 | 191 | proxy_ip, proxy_port = Proxy().get_proxy() 192 | try: 193 | data = { 194 | "scan_speed": self.cfg.scan_speed, 195 | "default_scanning_profile_id": default_scanning_profile_id, 196 | "proxy": { 197 | "enabled": self.cfg.proxy_enabled, 198 | "protocol": "http", 199 | "address": proxy_ip, 200 | "port": proxy_port 201 | } 202 | } 203 | requests.patch(url=configuration_url, 204 | data=json.dumps(data), 205 | headers=self.cfg.headers, 206 | timeout=30, 207 | verify=False) 208 | except Exception as e: 209 | print(e) 210 | 211 | # 根据传入的target_id批量删除目标 212 | def del_target(self, target_id, target_address): 213 | try: 214 | r = requests.delete(self.cfg.awvs_url + "/api/v1/targets/" + 215 | target_id, 216 | headers=self.cfg.headers, 217 | timeout=30, 218 | verify=False) 219 | return r.status_code 220 | except Exception as e: 221 | print("删除目标{}时发送错误\n".format(target_address), e) 222 | 223 | # 删除扫描器中的所有目标，同时会删除扫描任务 224 | def del_targets(self): 225 | targets_info = self.get_targets() 226 | targets_count = targets_info['pagination']['count'] 227 | targets = targets_info['targets'] 228 | if targets_count == 0: 229 | print("[*] 已删除所有目标，当前目标列表为空") 230 | print("[*] 当前目标数量为{}".format(targets_count)) 231 | for target in targets: 232 | target_id = target['target_id'] 233 | target_address = target['address'] 234 | status_code = self.del_target(target_id, target_address) 235 | if status_code == 204: 236 | print(target_address, " 删除目标成功") 237 | 238 | 239 | class Scan: 240 | 241 | def __init__(self): 242 | self.cfg = Config() 243 | 244 | # 根据传入的target_id批量添加扫描任务 245 | def scan_targets(self, target_id_list): 246 | for target_id in target_id_list: 247 | data = { 248 | "target_id": target_id, 249 | "profile_id": self.cfg.profile_id, 250 | "incremental": False, 251 | "schedule": { 252 | "disable": False, 253 | "start_date": None, 254 | "time_sensitive": False 255 | } 256 | } 257 | try: 258 | r = requests.post(self.cfg.awvs_url + "/api/v1/scans", 259 | data=json.dumps(data), 260 | headers=self.cfg.headers, 261 | timeout=30, 262 | verify=False) 263 | if r.status_code == 201: 264 | target = Target().get_target(target_id) 265 | target_address = target['address'] 266 | print(target_address, " 添加扫描任务成功") 267 | except Exception as e: 268 | print("[*] 添加扫描任务失败\n", e) 269 | 270 | # 对扫描器内已有的目标进行扫描，同样可以控制扫描的数量 271 | def scan_exist_targets(self): 272 | not_scan_target_list = [] 273 | targets_info = Target().get_targets() 274 | targets = targets_info['targets'] 275 | for target in targets: 276 | last_scan_id = target['last_scan_id'] 277 | target_id = target['target_id'] 278 | if last_scan_id is None: 279 | not_scan_target_list.append(target_id) 280 | print("未扫描的目标数量为:", len(not_scan_target_list)) 281 | count = input("请输入要开始扫描的目标个数(留空则开始扫描所有未扫描目标):") or None 282 | if count is not None: 283 | count = int(count) 284 | target_id_list = not_scan_target_list[0:count] 285 | self.scan_targets(target_id_list) 286 | 287 | # 获取扫描器内所有扫描任务的信息 288 | def get_scans(self): 289 | try: 290 | resp = requests.get(self.cfg.awvs_url + "/api/v1/scans", 291 | headers=self.cfg.headers, 292 | timeout=30, 293 | verify=False) 294 | scan_list = json.loads(resp.content.decode()) 295 | return scan_list 296 | except Exception as e: 297 | print(e) 298 | 299 | # 中止扫描器内所有正在扫描的任务 300 | def abort_scans(self): 301 | scan_list = self.get_scans()['scans'] 302 | for scan in scan_list: 303 | scan_id = scan['scan_id'] 304 | scan_status = scan['current_session']['status'] 305 | target_address = scan['target']['address'] 306 | if scan_status == "processing": 307 | try: 308 | resp = requests.post( 309 | self.cfg.awvs_url + 310 | "/api/v1/scans/{}/abort".format(scan_id), 311 | headers=self.cfg.headers, 312 | timeout=30, 313 | verify=False) 314 | if resp.status_code == 204: 315 | print(target_address, " 扫描任务已中止") 316 | except Exception as e: 317 | print("[*] 中止扫描任务时出现错误\n", e) 318 | 319 | # 获取扫描器内扫描失败的url，保存到'./log/error_url.txt中'，同时会删除扫描器内扫描失败的url 320 | def get_error_scans(self): 321 | print("[*] 该操作会删除扫描器中扫描失败的目标，并将扫描失败的URL保存至log文件夹下") 322 | confirm = input("[*] 是否要删除扫描器中扫描失败的目标(y/n)：") 323 | if confirm != "y": 324 | print("[*] 您已取消该操作") 325 | sys.exit() 326 | scan_list = self.get_scans()['scans'] 327 | try: 328 | with open("./log/error_url.txt", 'a') as f: 329 | for scan in scan_list: 330 | if scan['current_session']['event_level'] == 2: 331 | error_target_id = scan['target_id'] 332 | error_url = scan['target']['address'] 333 | f.write(error_url + '\n') 334 | # 删除扫描器中扫描失败的url 335 | status_code = Target().del_target( 336 | error_target_id, error_url) 337 | if status_code == 204: 338 | print(error_url, " 扫描失败，已从扫描器中删除目标") 339 | print("[*] 扫描失败的URL已保存至log文件夹下的error_url.txt中") 340 | except Exception as e: 341 | print("[*] 获取扫描失败的URL时出现错误\n", e) 342 | 343 | # 将‘./log/error_url.txt’中的url重新添加到扫描器并开始扫描，同样可以控制扫描的数量 344 | def rescan_error_scans(self): 345 | confirm = input("[*] 是否已经执行【获取扫描失败的目标】(y/n)：") 346 | if confirm != "y": 347 | print("[*] 已取消操作") 348 | sys.exit() 349 | print("[*] 正在尝试对扫描失败的目标进行重新扫描") 350 | self.scan_targets(Target().add_targets(txt="./log/error_url.txt", 351 | add_log="./log/readd_log.txt")) 352 | 353 | 354 | class Monitor: 355 | 356 | def __init__(self): 357 | self.cfg = Config() 358 | 359 | # 根据传入的scan_id中止扫描任务 360 | def abort_scans(self, scan_dict): 361 | for key in scan_dict: 362 | try: 363 | resp = requests.post(self.cfg.awvs_url + 364 | "/api/v1/scans/{}/abort".format(key), 365 | headers=self.cfg.headers, 366 | timeout=30, 367 | verify=False) 368 | if resp.status_code == 204: 369 | print("\n", scan_dict[key], " 扫描任务超过两小时，已中止扫描任务\n") 370 | except Exception as e: 371 | print("[*] 中止扫描任务时出现错误\n", e) 372 | 373 | # 将扫描器中已有的目标添加到扫描任务 374 | def add_scans(self, count): 375 | not_scan_target_list = [] 376 | targets_info = Target().get_targets() 377 | targets = targets_info['targets'] 378 | for target in targets: 379 | last_scan_id = target['last_scan_id'] 380 | target_id = target['target_id'] 381 | if last_scan_id is None: 382 | not_scan_target_list.append(target_id) 383 | print("未扫描的目标数量为:", len(not_scan_target_list)) 384 | target_id_list = not_scan_target_list[0:count] 385 | if len(not_scan_target_list) > 0: 386 | Scan().scan_targets(target_id_list) 387 | return target_id_list 388 | 389 | # 对比所有正在扫描的任务的开始时间和当前时间，返回超过最大扫描时间的scan_id 390 | def time_diff(self): 391 | scans = Scan().get_scans()['scans'] 392 | scan_dict = {} 393 | for scan in scans: 394 | try: 395 | status = scan['current_session']['status'] 396 | scan_id = scan['scan_id'] 397 | target_address = scan['target']['address'] 398 | if status == 'processing': 399 | start_time = scan['current_session']['start_date'][0:-6] 400 | UTC_FORMAT = '%Y-%m-%dT%H:%M:%S.%f' 401 | start_time = datetime.strptime(start_time, UTC_FORMAT) 402 | now_time = datetime.now() 403 | is_time_out = int( 404 | str(now_time - start_time).split(":") 405 | [0]) > self.cfg.max_scan_time - 1 406 | if is_time_out is True: 407 | scan_dict[scan_id] = target_address 408 | except Exception as e: 409 | print("获取时间差失败", e) 410 | return scan_dict 411 | 412 | # 检查当前正在进行的扫描任务的数量，未达到最大扫描任务数量时，向扫描器中添加扫描任务 413 | def add_to_limit(self): 414 | count = Config().check_api() 415 | if count < self.cfg.max_scan_count: 416 | diff_count = self.cfg.max_scan_count - count 417 | target_id_list = Monitor().add_scans(diff_count) 418 | print("[*] 已向扫描器添加{}个扫描任务\n".format(len(target_id_list))) 419 | 420 | 421 | def main(): 422 | Config().check_api() 423 | 424 | usage = """[*] 请选择要进行的操作： 425 | 1、批量添加目标，不进行扫描 426 | 2、批量添加目标并开始扫描 427 | 3、对扫描器内已有目标进行扫描 428 | 4、中止所有扫描任务 429 | 5、获取扫描失败的目标 430 | 6、对扫描失败的目标重新扫描 431 | 7、删除所有目标和扫描任务 432 | """ 433 | 434 | print(usage) 435 | selection = str(input("请输入数字：")) 436 | 437 | if selection == "1": 438 | Target().add_targets(txt="url.txt", add_log="./log/add_log.txt") 439 | 440 | elif selection == "2": 441 | Scan().scan_targets(Target().add_targets(txt="url.txt", 442 | add_log="./log/add_log.txt")) 443 | 444 | elif selection == "3": 445 | Scan().scan_exist_targets() 446 | 447 | elif selection == "4": 448 | Scan().abort_scans() 449 | 450 | elif selection == "5": 451 | Scan().get_error_scans() 452 | 453 | elif selection == "6": 454 | Scan().rescan_error_scans() 455 | 456 | elif selection == "7": 457 | Target().del_targets() 458 | 459 | else: 460 | print("输入的内容有误") 461 | 462 | 463 | # 监控是否有超过最大扫描时间的任务 464 | def monitor_time(): 465 | try: 466 | while 1: 467 | time.sleep(600) 468 | scan_dict = Monitor().time_diff() 469 | if len(scan_dict) != 0: 470 | Monitor().abort_scans(scan_dict) 471 | except KeyboardInterrupt: 472 | print("\nApplication exit!") 473 | 474 | 475 | # 监控是否达到最大扫描任务数量，未达到则向扫描器添加扫描任务 476 | def monitor_count(): 477 | try: 478 | while 1: 479 | Monitor().add_to_limit() 480 | time.sleep(180) 481 | except KeyboardInterrupt: 482 | print("\nApplication exit!") 483 | 484 | 485 | if __name__ == '__main__': 486 | try: 487 | if len(sys.argv) == 1: 488 | main() 489 | elif len(sys.argv) == 2 and sys.argv[1] == 'monitor': 490 | threads = [] 491 | t1 = threading.Thread(target=monitor_time) 492 | t2 = threading.Thread(target=monitor_count) 493 | threads.append(t1) 494 | threads.append(t2) 495 | t1.start() 496 | t2.start() 497 | t1.join() 498 | t2.join() 499 | else: 500 | print("""1、执行python3 awvs_script.py使用基本模式 501 | 2、执行python3 awvs_script.py monitor使用监控模式（Linux可执行"nohup python3 -u awvs_script.py monitor &"让脚本后台运行）""" 502 | ) 503 | except Exception as e: 504 | print(e) 505 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | # 代理池地址 2 | proxy_pool: http://127.0.0.1:8080 3 | # awvs访问地址 4 | awvs_url: https://127.0.0.1:3443 5 | # awvs api的key 6 | api_key: 1986ad8c0a5b3df4d7028d5f3c06e936c784f1aa4c2d349ba9d463623900c1226 7 | # 设置标签 8 | scan_label: 脚本批量添加 9 | # 由慢到快:sequential slow moderate fast 10 | scan_speed: fast 11 | # 是否启用代理 12 | proxy_enabled: True 13 | # 设置最大扫描数量 14 | max_scan_count: 10 # 仅监控模式下生效 15 | # 设置最大扫描时间(小时) 16 | max_scan_time: 2 # 仅监控模式下生效 17 | # 设置扫描类型（从profile_id中选择，将scan_mode后的数字替换即可） 18 | scan_mode: 2 19 | # 扫描类型 20 | profile_id: 21 | 1: 11111111-1111-1111-1111-111111111111 # 完全扫描 22 | 2: 11111111-1111-1111-1111-111111111112 # 高风险漏洞 23 | 3: 11111111-1111-1111-1111-111111111113 # SQL 注入漏洞 24 | 4: 11111111-1111-1111-1111-111111111115 # 弱口令检测 25 | 5: 11111111-1111-1111-1111-111111111116 # XSS 漏洞 26 | 6: 11111111-1111-1111-1111-111111111117 # Crawl Only 27 | 7: 11111111-1111-1111-1111-111111111120 # 恶意软件扫描 28 | -------------------------------------------------------------------------------- /log/add_log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colind0pe/awvs-scan-script/4ae77024b9d9a6fd9a2df2c4b7db82fbf37fa9c5/log/add_log.txt -------------------------------------------------------------------------------- /log/error_url.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colind0pe/awvs-scan-script/4ae77024b9d9a6fd9a2df2c4b7db82fbf37fa9c5/log/error_url.txt -------------------------------------------------------------------------------- /pic/pic01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colind0pe/awvs-scan-script/4ae77024b9d9a6fd9a2df2c4b7db82fbf37fa9c5/pic/pic01.png -------------------------------------------------------------------------------- /proxy_pool/README.md: -------------------------------------------------------------------------------- 1 | # goProxyPool 2 | 3 | 一款无环境依赖开箱即用的免费代理IP池 4 | 5 | 内置14个免费代理源，均使用内置的简单正则获取 6 | 7 | 支持调用插件扩展代理源，返回的数据符合格式即可，无开发语言限制 8 | 9 | 支持webApi获取、删除、更新等代理池内的IP 10 | 11 | 支持 http，socket5 隧道代理模式，无需手动更换IP 12 | 13 | 遇到bug或有好的建议，欢迎提issue 14 | 15 | ## 隧道代理 16 | 隧道代理是代理IP存在的一种方式。 17 | 相对于传统固定代理IP，它的特点是自动地在代理服务器上改变IP，这样每个请求都使用一个不同的IP。 18 | 19 | ## 代理IP特征 20 | 这里提供一些代理IP的特征，师傅们可通过特征自己写代理源，api获取的话内置的正则方式就能写 21 | 360网络空间测绘_socket5： 22 | ```text 23 | protocol:"socks5" AND "Accepted Auth Method: 0x0" AND "connection: close" AND country: "China" 24 | ``` 25 | fofa_http: 26 | ```text 27 | "HTTP/1.1 403 Forbidden Server: nginx/1.12.1" && port="9091" 28 | 29 | port="3128" && title="ERROR: The requested URL could not be retrieved" 30 | 31 | "X-Cache: 'MISS from VideoCacheBox/CE8265A63696DECD7F0D17858B1BDADC37771805'" && "X-Squid-Error: ERR_ACCESS_DENIED 0" 32 | ``` 33 | hunter_http： 34 | ```text 35 | header.server="nginx/2.2.200603d"&&web.title="502 Bad Gateway" && ip.port="8085" 36 | ``` 37 | 38 | # 截图 39 | [![zuz6TU.png](https://s1.ax1x.com/2022/11/19/zuz6TU.png)](https://s1.ax1x.com/2022/11/19/zuz6TU.png) 40 | # 使用说明 41 | 下载 42 | ``` 43 | git clone git@github.com:pingc0y/goProxyPool.git 44 | ``` 45 | 编译（直接使用成品，就无需编译） 46 | 以下是在windows环境下，编译出各平台可执行文件的命令 47 | ``` 48 | SET CGO_ENABLED=0 49 | SET GOOS=windows 50 | SET GOARCH=amd64 51 | go build -ldflags "-s -w" -o ../goProxyPool-windows-amd64.exe 52 | 53 | SET CGO_ENABLED=0 54 | SET GOOS=windows 55 | SET GOARCH=386 56 | go build -ldflags "-s -w" -o ../goProxyPool-windows-386.exe 57 | 58 | SET CGO_ENABLED=0 59 | SET GOOS=linux 60 | SET GOARCH=amd64 61 | go build -ldflags "-s -w" -o ../goProxyPool-linux-amd64 62 | 63 | SET CGO_ENABLED=0 64 | SET GOOS=linux 65 | SET GOARCH=arm64 66 | go build -ldflags "-s -w" -o ../goProxyPool-linux-arm64 67 | 68 | SET CGO_ENABLED=0 69 | SET GOOS=linux 70 | SET GOARCH=386 71 | go build -ldflags "-s -w" -o ../goProxyPool-linux-386 72 | 73 | SET CGO_ENABLED=0 74 | SET GOOS=darwin 75 | SET GOARCH=amd64 76 | go build -ldflags "-s -w" -o ../goProxyPool-macos-amd64 77 | 78 | SET CGO_ENABLED=0 79 | SET GOOS=darwin 80 | SET GOARCH=arm64 81 | go build -ldflags "-s -w" -o ../goProxyPool-macos-arm64 82 | 83 | ``` 84 | 运行 85 | 需要与config.yml在同一目录 86 | 注意：抓取代理会进行类型地区等验证会比较缓慢，存活验证会快很多 87 | ``` 88 | .\goProxyPool.exe 89 | ``` 90 | 91 | 代理源中有部分需要翻墙才能访问，有条件就设置下config.yml的代理配置 92 | ```yml 93 | proxy: 94 | host: 127.0.0.1 95 | port: 10809 96 | ``` 97 | ## webAPi说明 98 | 查看代理池情况 99 | ``` 100 | http://127.0.0.1:8080/ 101 | ``` 102 | 103 | 获取代理 104 | ``` 105 | http://127.0.0.1:8080/get?type=HTTP&count=10&anonymity=all 106 | 可选参数： 107 | type 代理类型 108 | anonymity 匿名度 109 | region 地区 110 | source 代理源 111 | count 代理数量 112 | 获取所有：all 113 | ``` 114 | 115 | 删除代理 116 | ``` 117 | http://127.0.0.1:8080/delete?ip=127.0.0.1&port=8888 118 | 必须传参： 119 | ip 代理ip 120 | port 代理端口 121 | ``` 122 | 123 | 验证代理 124 | ``` 125 | http://127.0.0.1:8080/verify 126 | ``` 127 | 128 | 更换隧道代理IP 129 | ``` 130 | http://127.0.0.1:8080/tunnelUpdate 131 | ``` 132 | 133 | 抓取代理 134 | ``` 135 | http://127.0.0.1:8080/upload 136 | ``` 137 | ## 代理字段解读 138 | ```go 139 | type ProxyIp struct { 140 | Ip string //IP地址 141 | Port string //代理端口 142 | Info1 string //代理地区信息1 143 | Info2 string //代理地区信息2 144 | Info3 string //代理地区信息3 145 | Isp string //IP提供商 146 | Type string //代理类型 147 | Anonymity string //代理匿名度, 透明：显示真实IP, 普匿：显示假的IP, 高匿：无代理IP特征 148 | Time string //最近一次代理验证时间 149 | Speed string //代理响应速度 150 | SuccessNum int //验证请求成功的次数 151 | RequestNum int //验证请求的次数 152 | Source string //代理源 153 | } 154 | ``` 155 | ## 配置文件 156 | ```yaml 157 | #使用代理去获取代理IP 158 | proxy: 159 | host: 127.0.0.1 160 | port: 10809 161 | 162 | # 配置信息 163 | config: 164 | #监听IP 165 | ip: 0.0.0.0 166 | #web监听端口 167 | port: 8080 168 | #http隧道代理端口 169 | httpTunnelPort: 8111 170 | #socket隧道代理端口 171 | socketTunnelPort: 8112 172 | #隧道代理更换时间秒 173 | tunnelTime: 60 174 | #可用IP数量小于‘proxyNum’时就去抓取 175 | proxyNum: 50 176 | #代理IP验证间隔秒 177 | verifyTime: 1800 178 | #抓取/检测状态线程数 179 | threadNum: 200 180 | 181 | #ip源 182 | spider: 183 | #代理获取源1 184 | - name: '齐云代理' 185 | #请求方式 186 | method: 'GET' 187 | #POST传参用的请求体 188 | body: '' 189 | #urls请求间隔/秒，防止频率过快被限制 190 | interval: 0 191 | #使用的请求头 192 | Headers: 193 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 194 | #获取的地址 195 | urls: 'https://proxy.ip3366.net/free/?action=china&page=1,https://proxy.ip3366.net/free/?action=china&page=2,https://proxy.ip3366.net/free/?action=china&page=3' 196 | #获取IP的正则表达式， 197 | ip: '\"IP\">(\d+?\.\d+?.\d+?\.\d+?)' 198 | #获取端口的正则表达式 199 | port: '\"PORT\">(\d+?)' 200 | #是否使用代理去请求 201 | proxy: false 202 | 203 | #通过插件，扩展ip源 204 | spiderPlugin: 205 | #插件名 206 | - name: test 207 | #运行命令，返回的结果要符合格式 208 | run: '.\test1.exe' 209 | 210 | #通过文件，导入IP 211 | spiderFile: 212 | #文件名 213 | - name: test1 214 | #文件路径 215 | path: 'ip.txt' 216 | ``` 217 | ### 扩展返回格式 218 | 通过,分割 219 | ```text 220 | 110.179.64.89:1080,111.2.155.180:1090,111.172.3.212:1090,111.196.186.95:6669 221 | ``` 222 | ### 文件导入格式 223 | 通过换行分割 224 | ```text 225 | 110.179.64.89:1080 226 | 111.2.155.180:1090 227 | 111.172.3.212:1090 228 | 111.196.186.95:6669 229 | 111.201.103.29:1080 230 | 113.12.200.66:1080 231 | 113.67.96.67:1090 232 | 113.104.217.45:1080 233 | 113.110.246.76:1080 234 | 113.116.9.18:1080 235 | 113.119.193.183:1090 236 | 113.119.193.187:1090 237 | 113.249.93.219:1080 238 | 114.95.200.164:1080 239 | 115.193.161.177:1080 240 | ``` 241 | 242 | 243 | ## 更新说明 244 | 2022/11/19 245 | 新增 socket5代理 246 | 新增 文件导入代理 247 | 新增 显示验证进度 248 | 新增 验证webApi 249 | 修改 扩展导入格式 250 | 优化 代理验证方式 251 | 优化 匿名度改为自动识别 252 | 修复 若干bug 253 | 254 | 255 | -------------------------------------------------------------------------------- /proxy_pool/config.yml: -------------------------------------------------------------------------------- 1 | #使用代理去获取代理IP 2 | proxy: 3 | host: 127.0.0.1 4 | port: 7890 5 | 6 | #配置信息 7 | config: 8 | #监听IP 9 | ip: 0.0.0.0 10 | #webApi监听端口 11 | port: 8080 12 | #http隧道代理端口 13 | httpTunnelPort: 8111 14 | #socket隧道代理端口 15 | socketTunnelPort: 8112 16 | #隧道代理更换时间秒 17 | tunnelTime: 60 18 | #可用IP数量小于‘proxyNum’时就去抓取 19 | proxyNum: 10 20 | #代理IP验证间隔秒 21 | verifyTime: 1800 22 | #抓取/检测状态线程数 23 | threadNum: 200 24 | 25 | #ip源 26 | spider: 27 | - name: '齐云代理' 28 | #请求方式 29 | method: 'GET' 30 | #post传参用的请求体 31 | #body: '' 32 | #urls请求间隔/秒，防止频率过快被限制 33 | interval: 0 34 | #使用的请求头 35 | Headers: 36 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 37 | #获取的地址 38 | urls: 'https://proxy.ip3366.net/free/?action=china&page=1,https://proxy.ip3366.net/free/?action=china&page=2,https://proxy.ip3366.net/free/?action=china&page=3' 39 | #获取IP的正则表达式， 40 | ip: '\"IP\">(\d+?\.\d+?.\d+?\.\d+?)' 41 | #获取端口的正则表达式 42 | port: '\"PORT\">(\d+?)' 43 | #是否使用代理去请求 44 | proxy: false 45 | 46 | - name: "89代理" 47 | method: 'GET' 48 | Headers: 49 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 50 | urls: "https://www.89ip.cn/index_1.html,https://www.89ip.cn/index_2.html,https://www.89ip.cn/index_3.html" 51 | ip: '[\s]*?(\d+?\.\d+?.\d+?\.\d+?)[\s]*?' 52 | port: '[\s]*?\d+?\.\d+?.\d+?\.\d+?[\s]*?[\s]*?[\s]*?(\d+?)[\s]*?' 53 | proxy: false 54 | 55 | #每天需要跟换地址 56 | - name: "开心代理" 57 | method: 'GET' 58 | Headers: 59 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 60 | urls: "http://www.kxdaili.com/daili/ip/7713.html" 61 | ip: '\[.+?\](\d+?\.\d+?.\d+?\.\d+?):\d+?@.+?#' 62 | port: '\[.+?\]\d+?\.\d+?.\d+?\.\d+?:(\d+?)@.+?#' 63 | proxy: false 64 | 65 | - name: "快代理" 66 | method: 'GET' 67 | Headers: 68 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 69 | urls: "http://www.ip3366.net/?stype=1&page=1,http://www.ip3366.net/?stype=1&page=2,http://www.ip3366.net/?stype=3&page=1,http://www.ip3366.net/?stype=3&page=2" 70 | ip: '[\s]*?(\d+?\.\d+?.\d+?\.\d+?)[\s]*?' 71 | port: '[\s]*?\d+?\.\d+?.\d+?\.\d+?[\s]*?[\s]*?[\s]*?(\d+?)[\s]*?' 72 | proxy: false 73 | 74 | - name: "高可用代理" 75 | method: 'GET' 76 | Headers: 77 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 78 | urls: "https://ip.jiangxianli.com/?page=1" 79 | ip: '[\s]*?(\d+?\.\d+?.\d+?\.\d+?)[\s]*?' 80 | port: '[\s]*?\d+?\.\d+?.\d+?\.\d+?[\s]*?[\s]*?[\s]*?(\d+?)[\s]*?' 81 | proxy: false 82 | 83 | 84 | - name: "小舒代理" 85 | method: 'GET' 86 | Headers: 87 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 88 | urls: "https://xsdaili.cn/dayProxy/ip/1796.html" 89 | ip: '(\d+?\.\d+?.\d+?\.\d+?):\d+?@.+?#\[.+?\]' 90 | port: '\d+?\.\d+?.\d+?\.\d+?:(\d+?)@.+?#\[.+?\]' 91 | proxy: false 92 | 93 | - name: "命运零代理" 94 | method: 'GET' 95 | Headers: 96 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 97 | urls: "http://proxylist.fatezero.org/proxy.list" 98 | ip: '\"host\": \"(.+?)\"' 99 | port: '\"port\": (\d+)' 100 | proxy: false 101 | 102 | 103 | - name: '自由代理' 104 | method: 'GET' 105 | Headers: 106 | #需要使用cookie,不然会有验证 107 | User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36' 108 | urls: "https://www.freeproxylists.net/zh/?c=CN&u=50&page=1" 109 | ip: '%3e(.{0,60}?)%3c%2f' 110 | port: 'center\">(\d+?)' 119 | port: 'href=\"/\d+?\.\d+?.\d+?\.\d+?/(\d+?)#http.{0,1}\">' 120 | proxy: false 121 | 122 | 123 | - name: "hidemy代理" 124 | method: 'GET' 125 | Headers: 126 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 127 | urls: "https://hidemy.name/cn/proxy-list/?maxtime=1000&type=h#list,https://hidemy.name/cn/proxy-list/?maxtime=1000&type=h&start=64#list,https://hidemy.name/cn/proxy-list/?maxtime=1000&type=h&start=128#list,https://hidemy.name/cn/proxy-list/?maxtime=5000&type=5#list,https://hidemy.name/cn/proxy-list/?maxtime=5000&type=s#list" 128 | ip: '(\d+?\.\d+?.\d+?\.\d+?)\d+' 129 | port: '\d+?\.\d+?.\d+?\.\d+?(\d+)' 130 | proxy: true 131 | 132 | - name: "scrape代理" 133 | method: 'GET' 134 | Headers: 135 | User-Agent: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 136 | urls: "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=CN&ssl=all&anonymity=all" 137 | ip: '(\d+?\.\d+?.\d+?\.\d+?):\d+' 138 | port: '\d+?\.\d+?.\d+?\.\d+?:(\d+)' 139 | anonymity: '透明' 140 | proxy: true 141 | 142 | - name: "my代理" 143 | method: 'GET' 144 | Headers: 145 | User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' 146 | urls: "https://www.my-proxy.com/free-socks-5-proxy.html,https://www.my-proxy.com/free-elite-proxy.html,https://www.my-proxy.com/free-anonymous-proxy.html" 147 | ip: '>(\d+?\.\d+?.\d+?\.\d+?):\d+#' 148 | port: '>\d+?\.\d+?.\d+?\.\d+?:(\d+)#' 149 | proxy: true 150 | 151 | - name: "proxy代理" 152 | method: 'GET' 153 | Headers: 154 | User-Agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' 155 | urls: "https://free-proxy-list.net/,https://www.us-proxy.org/,https://www.socks-proxy.net/" 156 | ip: '(\d+?\.\d+?.\d+?\.\d+?):\d+' 157 | port: '\d+?\.\d+?.\d+?\.\d+?:(\d+)' 158 | proxy: true 159 | 160 | 161 | #通过插件，扩展ip源 162 | spiderPlugin: 163 | #插件名 164 | - name: test 165 | #运行命令，返回的结果要符合格式 166 | run: 'text.exe' 167 | 168 | #通过文件导入ip 169 | spiderFile: 170 | #插件名 171 | - name: test1 172 | #运行命令，返回的结果要符合格式 173 | path: 'ip.txt' 174 | 175 | 176 | 177 | -------------------------------------------------------------------------------- /proxy_pool/goProxyPool-linux-amd64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colind0pe/awvs-scan-script/4ae77024b9d9a6fd9a2df2c4b7db82fbf37fa9c5/proxy_pool/goProxyPool-linux-amd64 -------------------------------------------------------------------------------- /proxy_pool/goProxyPool-windows-amd64.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colind0pe/awvs-scan-script/4ae77024b9d9a6fd9a2df2c4b7db82fbf37fa9c5/proxy_pool/goProxyPool-windows-amd64.exe -------------------------------------------------------------------------------- /url.txt: -------------------------------------------------------------------------------- 1 | http://testhtml5.vulnweb.com/ 2 | http://testphp.vulnweb.com/ 3 | http://testasp.vulnweb.com/ 4 | http://testaspnet.vulnweb.com/ 5 | http://rest.vulnweb.com/ --------------------------------------------------------------------------------