├── .gitignore ├── AD_Scanner_Base.py ├── BruteXSS.py ├── Burp_force_directory.py ├── Burp_force_directory └── dictionary │ ├── ASP.txt │ ├── ASPX.txt │ ├── DIR.txt │ ├── JSP.txt │ ├── MDB.txt │ └── PHP.txt ├── PortScanner.py ├── README.md ├── Sqliscan ├── __init__.py ├── serverinfo.py ├── sqlerrors.py ├── std.py ├── useragents.py └── web.py ├── XSS_payload └── wordlist.txt ├── burp_user.py ├── data.txt ├── data1.txt ├── dict ├── password.txt ├── pinyin2.txt └── user.txt ├── index.py ├── reids_demo.py ├── scanner.py ├── static ├── css │ ├── bootstrap-theme.css │ ├── bootstrap-theme.css.map │ ├── bootstrap-theme.min.css │ ├── bootstrap-theme.min.css.map │ ├── bootstrap.css │ ├── bootstrap.css.map │ ├── bootstrap.min.css │ ├── bootstrap.min.css.map │ ├── style.css │ └── style2.css ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 ├── images │ └── banner.jpg └── js │ ├── bootstrap.js │ ├── bootstrap.min.js │ ├── jquery.min.js │ └── npm.js ├── tHar_lib ├── __init__.py ├── engine_search.py ├── graphs.py ├── hostchecker.py ├── htmlExport.py ├── markup.py ├── myparser.py └── port_scanner.py ├── templates ├── AD.html └── content.html ├── test.py ├── the_harvest.py └── url_spider.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | *.iml 103 | *.xml 104 | scanner.py 105 | scanner.py 106 | -------------------------------------------------------------------------------- /AD_Scanner_Base.py: -------------------------------------------------------------------------------- 1 | #Author:Chernobyl 2018/5/28 2 | from url_spider import * 3 | from Burp_force_directory import * 4 | from scanner import * 5 | from the_harvest import * 6 | from burp_user import * 7 | from BruteXSS import * 8 | import re 9 | import os 10 | import sys 11 | import getopt 12 | import argparse 13 | import redis 14 | import _thread 15 | import time 16 | import json 17 | 18 | 19 | def terminal_input(): 20 | ''' 21 | 命令行输入处理函数 22 | 传入参数:无 23 | 返回值:包含参数和对应值的dict 24 | 25 | 命令行参数: 26 | -u/--url= : 传入的URL 27 | -h :帮助 28 | --spider-threads : 爬虫线程 29 | --burp-threads : 目录爆破线程 30 | -S : 爬虫参数 31 | -I : SQLMAP参数 32 | -B : 目录爆破参数 33 | --cookie : 手动输入cookie 34 | --file : 输出文件名 35 | 36 | ''' 37 | ter_opt={} 38 | if len(sys.argv) == 1: 39 | sys.argv.append('-h') 40 | parser = argparse.ArgumentParser(description='AnD Web Scanner',add_help=True) 41 | parser.add_argument('-u','--url',help='目标url') 42 | parser.add_argument('--login_url',default=None,help='账户爆破URL') 43 | parser.add_argument('--cookie',default=None,help='扫描器cookie') 44 | parser.add_argument('-F','--file',default=None,help='输出目标文件') 45 | parser.add_argument('-S','--spider_args',default=None,help='全站爬虫模块方法(craw)') 46 | parser.add_argument('--spider_threads',default=10,help='全站爬虫模块线程数',type=int) 47 | parser.add_argument('-I','--sqli_args',default=None,help='SQL注入漏洞扫描模块方法(run)') 48 | parser.add_argument('-B','--burp_args',default=None,help='路径爆破模块方法(run)') 49 | parser.add_argument('--burp_threads',default=10,help='路径爆破模块线程数',type=int) 50 | parser.add_argument('-R','--harvest_args',default=None,help='子域名收集模块参数(search)') 51 | parser.add_argument('-U','--burp_user_args',default=None,help='用户爆破模块参数(burp)') 52 | parser.add_argument('-X','--burp_XSS_args',default=None,help='XSS模块参数(run)') 53 | 54 | parser.add_argument 55 | parser.add_argument('--debug',default=None,help='开启Debug模式') 56 | args = parser.parse_args() 57 | for x,y in args._get_kwargs(): 58 | ter_opt[x]=y 59 | return ter_opt 60 | 61 | 62 | class base: 63 | ''' 64 | 参数:扫描网址 65 | ''' 66 | def url_check(self,url): 67 | ''' 68 | URL检测函数 69 | 传入参数:待检测的URL 70 | 返回值:无 71 | 72 | URL类型: 73 | Case0:http://www.test.com/...||https://www.test.com/... 74 | Case1:http://test.com/...||https://test.com/... 75 | Case1:www||aaa.test.com/..... 76 | Case2:test.com/... 77 | other:error 78 | ''' 79 | if re.match('(http|https)://(.*?)\.(.*?)\.(.*)',url) != None: #Case0: 80 | self.url = url 81 | self.base_redis.hset('base','url',url) 82 | self.base_redis.hset('base','url_type',0) 83 | 84 | if re.match('(http|https)://(.*?)\.(.*)',url) != None: #Case1: 85 | self.url = url 86 | self.base_redis.hset('base','url',url) 87 | self.base_redis.hset('base','url_type',1) 88 | 89 | elif re.match('(.*?)\.(.*?)\.(.*)',url) != None:#case 2 90 | self.url = url 91 | self.base_redis.hset('base','url',url) 92 | self.base_redis.hset('base','url_type',2) 93 | elif re.match('(.*?)\.(.*)',url) != None:#case 3: 94 | self.url = url 95 | self.base_redis.hset('base','url',url) 96 | self.base_redis.hset('base','url_type',3) 97 | else: 98 | print('URL Type Error!') 99 | sys.exit(1)#URL_ERROR 100 | 101 | def opt_handler(self): 102 | '''命令行参数处理 103 | 针对各模块特有设置项,设置相应键值对,用于初始化和存储信息 104 | ''' 105 | #所有参数传入redis 106 | for x in self.info.keys(): 107 | if self.info[x] != None: 108 | self.base_redis.hset('base',x,self.info[x]) 109 | print('optiopns:\n') 110 | for x in self.base_redis.hkeys('base'): 111 | print(x+':'+self.base_redis.hget('base',x)) 112 | self.output_dict={} 113 | if self.info['spider_args'] == 'craw': 114 | self.output_dict['Url_Spider'] = 'Spider_full_urls' 115 | if self.info['burp_args'] == 'run': 116 | self.output_dict['Burp_force_directory'] = 'Burp_force_directory_url' 117 | if self.info['sqli_args'] == 'run': 118 | self.output_dict['Sqli_scanner'] = 'Vulnerable_urls' 119 | if self.info['harvest_args'] == 'search': 120 | self.output_dict['the_harvest'] = ['Harvest_subdomain','Harvest_emails'] 121 | if self.info['burp_user_args'] == 'burp' and self.info['login_url'] != None: 122 | self.output_dict['burp_user'] = True 123 | if self.info['burp_XSS_args'] == 'run': 124 | self.output_dict['XSS'] = 'XSS_hole' 125 | 126 | print('go') 127 | time.sleep(1) 128 | 129 | '''处理输出文件''' 130 | if self.info['file'] != None: 131 | self.file_status = True 132 | 133 | def print_data(self): 134 | '''格式化输出模块返回的数据 135 | 格式:模块名 136 | -------- 137 | 数据 138 | ''' 139 | 140 | #如果传入了输出文件的参数则打开相应的文件 141 | if self.file_status : 142 | self.data_file = self.info['file'] 143 | self.data_file = open(self.data_file,'w') 144 | 145 | print('URL:'+self.url+'\n') 146 | if self.file_status: 147 | print('URL:'+self.url+'\n',file=self.data_file) 148 | 149 | for x in self.output_dict.keys(): 150 | num = 0 151 | print('\n\n'+x+':\n--------------------------------------') 152 | if self.file_status: 153 | print('\n\n'+x+':\n--------------------------------------',file=self.data_file) 154 | if x == 'the_harvest': 155 | for x,y in zip(self.base_redis.smembers(self.output_dict[x][0]),self.base_redis.smembers(self.output_dict[x][1])): 156 | print(str(num+1)+': domain:'+str(x)+' mail:'+str(y)) 157 | if self.file_status: 158 | print(str(num+1)+': domain:'+str(x)+' mail:'+str(y),file=self.data_file) 159 | continue 160 | 161 | if x == 'burp_user': 162 | print('account: '+self.base_redis.hget('burp_user', 'user')+ 'password: '+self.base_redis.hget('burp_user', 'password')) 163 | if self.file_status: 164 | print('account: '+self.base_redis.hget('burp_user', 'user')+ 'password: '+self.base_redis.hget('burp_user', 'password'),file=self.data_file) 165 | continue 166 | 167 | for y in self.base_redis.smembers(self.output_dict[x]): 168 | if x != 'Sqli_scanner' : 169 | print(str(num+1)+':'+y) 170 | if self.file_status: 171 | print(str(num+1)+':'+y,file=self.data_file) 172 | num+=1 173 | 174 | else: 175 | tmp = json.loads(y) 176 | print('url:'+str(tmp['url'])+' database:'+str(tmp['db'])+' server:'+str(tmp['server'])) 177 | if self.file_status: 178 | print('url:'+str(tmp['url'])+' database:'+str(tmp['db'])+' server:'+str(tmp['server']),file=self.data_file) 179 | 180 | if self.file_status : 181 | self.data_file.close() 182 | 183 | def __init__(self): 184 | self.info = terminal_input() 185 | self.url = self.info['url'] 186 | self.lg_url = self.info['login_url'] 187 | self.file_status = False 188 | self.save_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True)#开启本地radis 189 | self.base_redis = redis.Redis(connection_pool=self.save_pool) 190 | self.base_redis.flushdb() 191 | self.url_check(self.url) 192 | self.url_type = self.base_redis.hget('base','url_type') 193 | self.opt_handler() 194 | '''各模块初始化''' 195 | #对传入的URL进行处理,增加http://前缀 196 | if self.url_type == '2' or self.url_type == '3': 197 | self.url = 'http://'+self.url 198 | self.spider = SpiderMain(self.url,self.save_pool) 199 | self.burp_force_diectory = Scanner(self.url,self.save_pool) 200 | self.sqli = SqliMain(self.save_pool) 201 | self.harvest = TheHarvester(self.url,self.save_pool) 202 | self.burp_user = BurpUser(self.lg_url,self.save_pool) 203 | self.burp_XSS = BruteXSS(self.save_pool) 204 | 205 | def start_modules(self): 206 | '''多线程执行模块的运行方法''' 207 | _thread.start_new_thread(self.spider.run,()) 208 | _thread.start_new_thread(self.burp_force_diectory.more_threads,()) 209 | _thread.start_new_thread(self.sqli.run,()) 210 | _thread.start_new_thread(self.harvest.start_search,()) 211 | _thread.start_new_thread(self.burp_user.run,()) 212 | _thread.start_new_thread(self.burp_XSS.run,()) 213 | 214 | 215 | def module_check(self): 216 | '''查询模块的线程是否执行完成''' 217 | return_list=[] 218 | if self.info['spider_args'] == 'craw': 219 | return_list.append(self.spider.is_finished()) 220 | if self.info['burp_args'] == 'run': 221 | return_list.append(self.burp_force_diectory.is_finished()) 222 | if self.info['sqli_args'] == 'run': 223 | return_list.append(self.sqli.is_finished()) 224 | if self.info['harvest_args'] == 'search': 225 | return_list.append(self.harvest.is_finished()) 226 | if self.info['burp_user_args'] == 'burp': 227 | return_list.append(self.burp_user.is_finished()) 228 | if self.info['burp_XSS_args'] == 'run': 229 | return_list.append(self.burp_XSS.is_finished()) 230 | return return_list 231 | 232 | 233 | #if '__name__' == '__main__': 234 | ma = base() 235 | ma.start_modules() 236 | timer=0 237 | while False in ma.module_check() : 238 | print('time=%d stat:'%(timer),end=' ') 239 | for x in ma.output_dict.keys(): 240 | print(' '+x+':',end='') 241 | if x == 'Url_Spider': 242 | print(ma.spider.is_finished(),end='') 243 | elif x == 'Burp_force_directory': 244 | print(ma.burp_force_diectory.is_finished(),end='') 245 | elif x == 'Sqli_scanner': 246 | print(ma.sqli.is_finished(),end='') 247 | elif x == 'the_harvest': 248 | print(ma.harvest.is_finished(),end='') 249 | elif x == 'burp_user': 250 | print(ma.burp_user.is_finished(),end='') 251 | elif x == 'XSS': 252 | print(ma.burp_XSS.is_finished(),end='') 253 | print(' ',end='\r') 254 | time.sleep(5) 255 | timer+=5 256 | continue 257 | os.system('cls') 258 | print('finished') 259 | ma.print_data() 260 | input() 261 | -------------------------------------------------------------------------------- /BruteXSS.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-coding:utf8-*- 3 | #!BruteXSS 4 | #!author: Dlangman 5 | ''' 6 | v0.1easy 7 | GET方法对url进行解析,将url分解出value。然后对value进行XSS, 8 | 检测机制是检查response中是否有插入信息,所以payload很重要,不能乱改payload文件。 9 | ''' 10 | 11 | import http.client 12 | import urllib.request, urllib.parse, urllib.error 13 | import urllib.parse 14 | import threading 15 | import redis 16 | import sys 17 | 18 | class BruteXSS(object): 19 | 20 | def __init__(self, savepool): 21 | self.redis_out = '' #输出给共享池的 22 | self.isfinish = False #是否跑完 23 | self.save_pool = savepool # 开启本地radis 24 | self.pool = redis.Redis(connection_pool=self.save_pool) # 创建一个连接实例 25 | 26 | self.thread_num = 50 27 | self.thread_max = threading.BoundedSemaphore(self.thread_num) 28 | 29 | def run(self): 30 | ''' 31 | 获取base 参数 burp_XSS_args 32 | :return: 33 | ''' 34 | action = self.pool.hget('base', 'burp_XSS_args') 35 | if action == 'run': 36 | self.brute() 37 | 38 | def brute(self): 39 | old_url =[] 40 | while self.pool.get('spider_redis') == 'False': 41 | urls = self.pool.smembers("Spider_full_urls") 42 | for url in urls : 43 | if url not in old_url: 44 | self.thread_max.acquire() 45 | url_real = threading.Thread(target=self.GET,args=(url,)) 46 | url_real.start() 47 | url_real.join() 48 | old_url = urls 49 | 50 | self.isfinish = True 51 | 52 | def is_finished(self): 53 | return self.isfinish 54 | 55 | def Redis_Outputer(self): 56 | ''' 57 | 键设置为 :XSS_hole 58 | :return: 59 | ''' 60 | self.pool.sadd('XSS_hole', self.redis_out) 61 | 62 | 63 | def wordlistimport(self, file,lst): 64 | try: 65 | with open(file,'r') as f: 66 | ''' 67 | 访问payload文件,并将结果存入lst[]中 68 | ''' 69 | for line in f: 70 | final = str(line.replace("\n","")) 71 | lst.append(final) 72 | except IOError: 73 | print("[!] Wordlist not found!") 74 | 75 | def GET(self, url): 76 | try: 77 | try: 78 | #print(threading.current_thread(),url) 79 | site = url 80 | if 'https://' in site or 'http://' in site: 81 | pass 82 | else: 83 | site = "http://" +site 84 | 85 | finalurl = urllib.parse.urlparse(site) #分割url为几部分 86 | domain0 = '{uri.scheme}://{uri.netloc}'.format(uri=finalurl) 87 | domain = domain0.replace("https://","").replace("http://", "").replace("www.", "").replace("/","") 88 | 89 | #print("[+] Checking if " + domain + " is available") 90 | connection = http.client.HTTPConnection(domain) 91 | connection.connect() 92 | #print("[+] "+ domain +" is available!") 93 | 94 | url = site 95 | paraname =[] 96 | wordlist = 'XSS_payload/wordlist.txt' 97 | 98 | payloads = [] 99 | self.wordlistimport(wordlist,payloads) #把payload放进[]里 100 | 101 | parameters = urllib.parse.parse_qs(finalurl.query,keep_blank_values=True) 102 | path = finalurl.scheme+"://"+finalurl.netloc+finalurl.path #网址路径 103 | for para in parameters: 104 | paraname.append(para) 105 | 106 | for pn in paraname: 107 | # print("[+] Testing "+pn+ " parameter...") 108 | for x in payloads: 109 | enc = urllib.parse.quote_plus(x) 110 | data = path +"?"+pn +"=" +enc #在网址路径上补上参数 111 | page = urllib.request.urlopen(data) 112 | sourececode = page.read().decode() 113 | if x in sourececode: #如果输入的内容完整在网页里,则认为是存在XSS 114 | print(("\n[!] XSS Vulnerability Found!\n"+"value: "+pn +" \npayload: " +x)) 115 | self.redis_out = url +" "+ pn +" "+ x 116 | self.Redis_Outputer() 117 | 118 | break 119 | 120 | except(http.client.HTTPResponse) as Exit: 121 | print(("[!] Site"+domain +" is offline")) 122 | except(KeyboardInterrupt) as Exit: 123 | print("\nExit...") 124 | self.thread_max.release() 125 | 126 | if __name__ == '__main__': 127 | url = 'http://localhost/crack_demo.php?value=aaaa&pwd=#' 128 | savepool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True) 129 | brute = BruteXSS(savepool) 130 | print(brute.is_finished()) 131 | brute.run() 132 | print(brute.is_finished()) -------------------------------------------------------------------------------- /Burp_force_directory.py: -------------------------------------------------------------------------------- 1 | #Burp_force_directory by xuxu 2 | import requests 3 | import threading 4 | import redis 5 | import os 6 | import time 7 | from urllib.parse import urlparse 8 | 9 | class Scanner(): 10 | def __init__(self, url, save_pool): 11 | self.burp_redis = redis.Redis(connection_pool=save_pool) 12 | self.url = self.Urlparse(url) 13 | self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 14 | 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36'} 15 | self.dic_list = [] #字典名目录 16 | self.get_url = [] #获取到的所存在的url 17 | self.get_url_len = 0 #获取到的有效url数量(可重复) 18 | self.len = 0 #获取到的有效url数量(不重复) 19 | self.threads_max = self.get_threads() # 最大线程数 20 | self.check = False #线程运行状态 21 | 22 | def Urlparse(self, url): 23 | ''' 24 | 把传入的url进行截取,只要scheme + netloc部分 25 | :return: 26 | ''' 27 | k = urlparse(url) 28 | l = (k[0] + '://' + k[1]) 29 | return l.rstrip() 30 | 31 | def get_threads(self): 32 | ''' 33 | 从redis中取线程数,如果返回为None,则默认50 34 | ''' 35 | return int(self.burp_redis.hget('base','burp_threads')) 36 | 37 | def run(self): 38 | ''' 39 | 获取base模块的参数,决定是否运行 40 | :return: 41 | ''' 42 | key = self.burp_redis.hget('base','burp_arg') 43 | if key == 'run': 44 | self.more_threads() 45 | 46 | 47 | def get_dic(self): 48 | ''' 49 | 获取字典目录下的文件名到self.dic_list 50 | 增加把相对路径换成绝对路径的功能 51 | :return: 52 | ''' 53 | for root, files, self.dic_list in os.walk('./Burp_force_directory/dictionary'): 54 | pass 55 | 56 | def more_threads(self): 57 | self.get_dic() 58 | threads = [] 59 | self.check = False 60 | for k in range(0,len(self.dic_list)): 61 | print(self.dic_list[k]) 62 | #t = threading.Thread(target=self.combine_url,args=(self.dic_list[k],)) 63 | #threads.append(t) 64 | self.combine_url(self.dic_list[k]) 65 | 66 | for k in threads: 67 | k.start() 68 | 69 | #for k in threads: 70 | #k.join() 71 | 72 | self.check = True 73 | 74 | def combine_url(self,doc_name): 75 | ''' 76 | 从字典中逐行取出子目录,并将其与传入的网址组合 77 | ''' 78 | #print(doc_name) 79 | with open(os.getcwd() + r'\Burp_force_directory\dictionary\\'+doc_name,'r') as file_obj: 80 | for line in file_obj: 81 | test_url = self.url + line 82 | # print(test_url) 83 | if threading.activeCount() >= self.threads_max: 84 | time.sleep(0.7) 85 | else: 86 | t = threading.Thread(target=self.judge, args=(test_url.rstrip(),)) 87 | t.start() 88 | # t.join() 89 | # print(threading.activeCount()) 90 | # self.judge(test_url.rstrip()) 91 | 92 | def judge(self, test_url): 93 | ''' 94 | 判断所传入的连接是否存在 95 | ''' 96 | try: 97 | #print(test_url) 98 | k = self.request(test_url) 99 | #print(k.status_code) 100 | if k.status_code == 200: 101 | print(test_url) 102 | ''' 103 | self.get_url.append(test_url) 104 | self.len = len(set(self.get_url)) 105 | print(self.len,self.get_url_len) 106 | if self.len > self.get_url_len: 107 | self.get_url_len = self.len 108 | ''' 109 | if test_url in self.get_url: 110 | pass 111 | else: 112 | self.get_url.append(test_url) 113 | ''' 114 | try: 115 | self.burp_redis.hset('Burp_force_directory_scanned_url','scanned_url',self.get_url) 116 | print(self.burp_redis.hget('Burp_force_directory','scanned_url')) 117 | except Exception as p: 118 | pass 119 | #测试模式下开启报错 120 | #print(p) 121 | ''' 122 | 123 | 124 | try: 125 | print(test_url) 126 | self.burp_redis.sadd('Burp_force_directory_url',test_url) 127 | except Exception as e: 128 | print(e) 129 | 130 | except requests.exceptions.Timeout: 131 | pass 132 | except Exception as e: 133 | pass 134 | #测试模式下开启报错输出 135 | #print(e) 136 | 137 | 138 | def request(self, test_url): 139 | ''' 140 | 用get方法会请求整个【头部+正文】,浪费资源 141 | 利用head方法,只请求【资源头部】 142 | ''' 143 | r = requests.head(test_url, headers=self.headers, timeout=1) 144 | return r 145 | 146 | def print_get_url(self): 147 | self.print_get_url = set(self.print_get_url) 148 | print(self.get_url) 149 | 150 | def is_finished(self): 151 | return self.check 152 | 153 | 154 | if __name__ == '__main__': 155 | save_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True)#开启本地radis 156 | url = 'http://www.sdlongli.com' 157 | Web_scanner = Scanner(url,save_pool) 158 | Web_scanner.more_threads() 159 | print(Web_scanner.burp_redis.hget('Burp_force_directory','scanned_url')) 160 | #print(Web_scanner.module_redis.hget('Burp_force_directory','scanned_url')) 161 | -------------------------------------------------------------------------------- /Burp_force_directory/dictionary/ASP.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AttackandDefenceSecurityLab/AD_WebScanner/59124f421202d20a2def1291799443c5635a4b69/Burp_force_directory/dictionary/ASP.txt -------------------------------------------------------------------------------- /Burp_force_directory/dictionary/ASPX.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AttackandDefenceSecurityLab/AD_WebScanner/59124f421202d20a2def1291799443c5635a4b69/Burp_force_directory/dictionary/ASPX.txt -------------------------------------------------------------------------------- /Burp_force_directory/dictionary/DIR.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AttackandDefenceSecurityLab/AD_WebScanner/59124f421202d20a2def1291799443c5635a4b69/Burp_force_directory/dictionary/DIR.txt -------------------------------------------------------------------------------- /Burp_force_directory/dictionary/JSP.txt: -------------------------------------------------------------------------------- 1 | /FCKeditor/editor/filemanager/browser/default/browser.html?Type=Image&Connector=connectors/jsp/connector.jsp 2 | /fccmsres/admin/default.jsp 3 | /login.jsp 4 | /luntanLogin.jsp 5 | /domain_manage.jsp 6 | /s8main.jsp 7 | /login/index.jsp 8 | /admin/index.jsp 9 | /message/admin_login.jsp 10 | /admin.jsp 11 | /cms/admin.jsp 12 | /admin/admin.jsp 13 | /manage/admin.jsp 14 | /plc/admin.jsp 15 | /EducationManager/admin.jsp 16 | /bbs-admin.jsp 17 | /login/admin.jsp 18 | /book/admin.jsp 19 | /servicesystem/login-admin.jsp 20 | /login-admin.jsp 21 | /admins/admin.jsp 22 | /newsadmin/admin.jsp 23 | /user/admin.jsp 24 | /orderadmin/admin.jsp 25 | /penit-admin.jsp 26 | /clearadmin/admin.jsp 27 | /WebAdmin/admin.jsp 28 | /relogin-admin.jsp 29 | /manage/index.jsp 30 | /oa/login.jsp 31 | /oa/admin_login.jsp 32 | /coon.jsp 33 | /adminis/login.jsp 34 | /02nfdiy.jsp 35 | /0x5emyup.jsp 36 | /1.jsp 37 | /1/1/gif.jsp 38 | /10f4digshell0.jsp 39 | /11.jsp 40 | /111.jsp 41 | /11111/index.jsp 42 | /115cn.jsp 43 | /123.jsp 44 | /1234.jsp 45 | /12345.jsp 46 | /123456.jsp 47 | /12912.jsp 48 | /1dppdiy.jsp 49 | /1hmmdigshell2.jsp 50 | /1iyydiy.jsp 51 | /1ndex.jsp 52 | /1tufmyup.jsp 53 | /1uuqmyup.jsp 54 | /2005kycj/2005kycj/login.jsp 55 | /2006.jsp 56 | /21ex/jihe.jsp 57 | /22.jsp 58 | /222.jsp 59 | /2ir9myup.jsp 60 | /2m8ydigshell0.jsp 61 | /2r8idiy.jsp 62 | /3.jsp 63 | /30wfdigshell0.jsp 64 | /33.jsp 65 | /333.jsp 66 | /3800cc.jsp 67 | /3upxmyup.jsp 68 | /41x6digshell0.jsp 69 | /47rfmyup.jsp 70 | /4fpndigshell0.jsp 71 | /4p5xdiy.jsp 72 | /5u3qdigshell0.jsp 73 | /5xc4diy.jsp 74 | /6crwdiy.jsp 75 | /6k.jsp 76 | /6qv4myup.jsp 77 | /6yaqmyup.jsp 78 | /79hlmyup.jsp 79 | /7am5xiao.jsp 80 | /7hsfdigshell0.jsp 81 | /8000/welcome.jsp 82 | /8080/anything.jsp 83 | /8080/servlet/org.apache.catalina.servlets.DefaultServlet/index.jsp 84 | /80f9digshell0.jsp 85 | /87d6diy.jsp 86 | /88888/index.jsp 87 | /89wjdiy.jsp 88 | /8vt2digshell0.jsp 89 | /8wr8myup.jsp 90 | /92vrmyup.jsp 91 | /9g42shell.jsp 92 | /ASPAdmin.jsp 93 | /ASPAdmin_A.jsp 94 | /ASPXspy2.jsp 95 | /A_Login.jsp 96 | /AddNews.jsp 97 | /Admin.jsp 98 | /Admin/Admin_Index.jsp 99 | /Admin/Database/%23tourdata.jsp 100 | /Admin/Database/%23tourdatabak.jsp 101 | /AdminCenter/AdminLogin.jsp 102 | /AdminFile/Admin_Login.jsp 103 | /AdminLogin1.jsp 104 | /AdminMain.jsp 105 | /AdminMenu.jsp 106 | /AdminUserModule/AdminUserLogin.jsp 107 | /Admin_BatchLink.jsp 108 | /Admin_Cy/DataBackup/DataBack.jsp 109 | /Admin_Cy/DataCy/%23%23cyweb_cn.jsp 110 | /Admin_Cy/Zzm.jsp 111 | /Admin_DataBackup.jsp 112 | /Admin_Database.jsp 113 | /Admin_Field.jsp 114 | /Admin_Help_User.jsp 115 | /Admin_Label.jsp 116 | /Admin_Login588.jsp 117 | /Admin_Login8.jsp 118 | /Admin_Login888.jsp 119 | /Admin_Maillist.jsp 120 | /Admin_Message.jsp 121 | /Admin_Photo.jsp 122 | /Admin_SoftCateMenu.jsp 123 | /Admin_SoftInfo.jsp 124 | /Admin_SoftLink.jsp 125 | /Admin_SoftList.jsp 126 | /Admin_SubCate.jsp 127 | /Admin_UpdateSoftNum.jsp 128 | /Admin_UploadFile.jsp 129 | /Admin_UploadFile_Style.jsp 130 | /Admin_UserSetting.jsp 131 | /Admin_ZgTea_Art/Login.jsp 132 | /Admin_jsCreate.jsp 133 | /Administration/Default.jsp 134 | /Article/admin/login.jsp 135 | /ArticleShow.jsp 136 | /Articlelogin.jsp 137 | /CEO.jsp 138 | /Char.jsp 139 | /CmsEditor/Upload.jsp 140 | /Comment.jsp 141 | /Connections/Connections.jsp 142 | /Connections/baseinc.jsp 143 | /Connections/cnn.jsp 144 | /Connections/conn.jsp 145 | /ConsoleHelp/login.jsp 146 | /Create_Commend.jsp 147 | /Create_Default.jsp 148 | /Create_New.jsp 149 | /Create_Other.jsp 150 | /Create_SoftCate.jsp 151 | /Create_SoftList_All.jsp 152 | /Create_SoftList_Cate.jsp 153 | /Create_jsNews.jsp 154 | /Create_jsSearch.jsp 155 | /DATA/%23echuang%23.jsp 156 | /Data.project/%23zxData.project%23.jsp 157 | /Data/%23vvskybbs.jsp 158 | /Data/MeCMS_data.jsp 159 | /Data/YxBBs.jsp 160 | /Data/db.jsp 161 | /Data/wrtxcnshop2.jsp 162 | /DataBackup/1.jsp 163 | /DataBackup/111.jsp 164 | /DataBackup/123.jsp 165 | /DataBackup/222.jsp 166 | /DataBackup/ASPAdmin.jsp 167 | /DataBackup/ASPAdmin_A.jsp 168 | /DataBackup/a.jsp 169 | /DataBackup/aa.jsp 170 | /DataBackup/ad.jsp 171 | /DataBackup/asdf.jsp 172 | /DataBackup/c99.jsp 173 | /DataBackup/cao.jsp 174 | /DataBackup/caonima.jsp 175 | /DataBackup/cmd.jsp 176 | /DataBackup/command.jsp 177 | /DataBackup/cshell.jsp 178 | /DataBackup/css.jsp 179 | /DataBackup/d99.jsp 180 | /DataBackup/default1.jsp 181 | /DataBackup/digshell0.jsp 182 | /DataBackup/digshell2.jsp 183 | /DataBackup/diy.jsp 184 | /DataBackup/diy3.jsp 185 | /DataBackup/dm.jsp 186 | /DataBackup/do.jsp 187 | /DataBackup/error.jsp 188 | /DataBackup/fuck.jsp 189 | /DataBackup/fuckyou.jsp 190 | /DataBackup/hack.jsp 191 | /DataBackup/hacker.jsp 192 | /DataBackup/hate.jsp 193 | /DataBackup/hello.jsp 194 | /DataBackup/index1.jsp 195 | /DataBackup/log.jsp 196 | /DataBackup/love.jsp 197 | /DataBackup/luck.jsp 198 | /DataBackup/m.jsp 199 | /DataBackup/main1.jsp 200 | /DataBackup/mm.jsp 201 | /DataBackup/mmm.jsp 202 | /DataBackup/my.jsp 203 | /DataBackup/myup.jsp 204 | /DataBackup/new.jsp 205 | /DataBackup/news.jsp 206 | /DataBackup/ok.jsp 207 | /DataBackup/phpinfo.jsp 208 | /DataBackup/phpspy.jsp 209 | /DataBackup/root.jsp 210 | /DataBackup/servu.jsp 211 | /DataBackup/shell.jsp 212 | /DataBackup/spy.jsp 213 | /DataBackup/su.jsp 214 | /DataBackup/temp.jsp 215 | /DataBackup/webshell.jsp 216 | /DataBackup/wish.jsp 217 | /DataBackup/woaini.jsp 218 | /DataBackup/ws.jsp 219 | /DataBackup/x.jsp 220 | /DataBackup/xiao.jsp 221 | /DataBackup/xiaolu.jsp 222 | /DataBackup/xm.jsp 223 | /DataBackup/xx.jsp 224 | /DataBackup/xxx.jsp 225 | /DataBackup/yes.jsp 226 | /DataBackup/z.jsp 227 | /DataBackup/zz.jsp 228 | /DataBackup/zzz.jsp 229 | /DataBase/%23GBooK.jsp 230 | /DataBase/DB.jsp 231 | /DataBase/TCBBS7.jsp 232 | /DataBases/%23%23%23fdkjgzschool.V2009%23.jsp 233 | /DataShop).jsp 234 | /Data_Backup.jsp 235 | /Data_Return.jsp 236 | /Database/%23database%23.jsp 237 | /Database/%23tyqiye.jsp 238 | /Database/%23tyqiyechina.jsp 239 | /Database/%23wygkcnalibaba.jsp 240 | /Database/Data.jsp 241 | /Database/DataShop).jsp 242 | /Database/DataShop.jsp 243 | /Databases/%23wrtxcn2007.jsp 244 | /Databases/%23wygkcnqywz4.jsp 245 | /Databases/wrtxcnqywz4.jsp 246 | /Databases/wygkcnqyhtml.jsp 247 | /Databases/wygkcnqywz.jsp 248 | /Databases/wygkcnqywz3.jsp 249 | /DbConnect.jsp 250 | /Default_index.jsp 251 | /EC_Admin/EC_AdminLogin.jsp 252 | /EduAdmin/Admin_Login.jsp 253 | /FCKeditor/editor/filemanager/browser/default/browser.jsp 254 | /FCKeditor/editor/filemanager/browser/default/browser.jsp?Type=all&Connector=connectors/asp/connector.jsp 255 | /Fl_Web.jsp 256 | /Foosun/Admin/login.jsp 257 | /Function/UploadProductPic.jsp 258 | /Fuzhuang_Fushi/index.jsp 259 | /Fy_SqlX.jsp 260 | /GOOGLE1bb9e40669bc959a.jsp 261 | /Gas_login.jsp 262 | /Gehang_Geye/index.jsp 263 | /GetPassword.jsp 264 | /Gongye_Zhipin/index.jsp 265 | /Guowai_Wangzhan/index.jsp 266 | /HX_LOGIN.jsp 267 | /Heike_Anquan/index.jsp 268 | /HomeManagement/Login.jsp 269 | /Hradmin/admin.jsp 270 | /Huagong_Nengyuan/index.jsp 271 | /Hz@host!.jsp 272 | /ImageMap.jsp 273 | /Images/config_inc.jsp 274 | /Inc/conndb.jsp 275 | /Include/setting.jsp 276 | /Index.jsp 277 | /InsertEmotion.jsp 278 | /Jianzhan_Sheji/index.jsp 279 | /Keji_IT/index.jsp 280 | /Kes/Admin/Admin_Login.jsp 281 | /KesAdmin_Login.jsp 282 | /Library/DbConnect.jsp 283 | /Link/upload/upload.jsp 284 | /Log.jsp 285 | /LoginAdministrator.jsp 286 | /Login_ok.jsp 287 | /LookupPass.jsp 288 | /MSOffice/cltreq.jsp 289 | /Manag_onlinedb.jsp 290 | /Manage/Default.jsp 291 | /ManageAdmin/ManageLogin.jsp 292 | /ManageLogin.jsp 293 | /Manage_backup.jsp 294 | /Manager/default.jsp 295 | /MeCMS_data.jsp 296 | /Member/FileUpLoad.jsp 297 | /Mianfei_Ziyuan/index.jsp 298 | /My-login.jsp 299 | /MySql.jsp 300 | /NBA_lanqiu/index.jsp 301 | /NBArticle.jsp 302 | /Neeao.jsp 303 | /Neeao_SqlIn.jsp 304 | /Neeao_sql_admin.jsp 305 | /NewFucker.jsp 306 | /NewsInfr.jsp 307 | /NewsUpLoad.jsp 308 | /Nonglin_Muyu/index.jsp 309 | /OaLogin.jsp 310 | /PBlog1.jsp 311 | /PBlog2.jsp 312 | /PBlog3.jsp 313 | /PoolMan.jsp 314 | /Preview.jsp 315 | /Product/manage/login.jsp 316 | /Qiche_Qipei/index.jsp 317 | /Reg/z9v8User_Reg.jsp 318 | /Reg/z9v8User_Reg1.jsp 319 | /Register/UserReg_Step1.jsp 320 | /Register/UserReg_Step2.jsp 321 | /SEM_User/admin_php/login.jsp 322 | /SK_login.jsp 323 | /SaveUpFile.jsp 324 | /Saveannounce_upload.jsp 325 | /ScanShell.jsp 326 | /Select_feedback.jsp 327 | /Server.jsp 328 | /ServerInfo.jsp 329 | /Shangwu_Maoyi/index.jsp 330 | /Shop_Login.jsp 331 | /ShowHost.jsp 332 | /ShowNews.jsp 333 | /Skyj.jsp 334 | /Sousuo_Yinqing/index.jsp 335 | /Southidceditor/upload.jsp 336 | /SqlIn/sqlIn_admin.jsp 337 | /Stats.jsp 338 | /Subsitemanage/login.jsp 339 | /Super/Index.jsp 340 | /SysAdmin/AdminLogin.jsp 341 | /SysAdmin/login.jsp 342 | /SysConfig.jsp 343 | /SysUser.jsp 344 | /Sys_admin.jsp 345 | /System/Function/UploadProductPic.jsp 346 | /SystemAdmin/AdminLogin.jsp 347 | /TUNGSTENDATA.jsp 348 | /UP/UpFilea.jsp 349 | /USERok.jsp 350 | /Up_BookPicPro.jsp 351 | /Upfile_AdPia.jsp 352 | /Upfile_AdPic.jsp 353 | /Upfile_Articla.jsp 354 | /Upfile_Article.jsp 355 | /Upfile_Image.jsp 356 | /Upfile_OrderPic.jsp 357 | /Upfile_Product.jsp 358 | /Upfile_ProductPic.jsp 359 | /Upfile_Soft.jsp 360 | /Upfile_SoftPic.jsp 361 | /Upfile_pic.jsp 362 | /Upfile_pics.jsp 363 | /Upfiledd.jsp 364 | /Upfilem.jsp 365 | /Upfilep.jsp 366 | /UploadAttachment.jsp 367 | /UploadFace.jsp 368 | /UploadImage3_upload.jsp 369 | /UploadProductPic.jsp 370 | /UploadSoft/diy.jsp 371 | /Upload_Dialog.jsp 372 | /Upload_Photo.jsp 373 | /Upload_Product.jsp 374 | /Upload_ProductPic.jsp 375 | /Upload_SoftPic.jsp 376 | /Upload_user.jsp 377 | /Uploaddd.jsp 378 | /User/Reg_service.jsp 379 | /User/UserReg.jsp 380 | /User/User_Article.jsp 381 | /User/User_Space.jsp 382 | /UserJoin.jsp 383 | /UserList.jsp 384 | /UserLogin.jsp 385 | /UserManage.jsp 386 | /UserModify.jsp 387 | /UserReg.jsp 388 | /User_GetPassword.jsp 389 | /Users/Login.jsp 390 | /Wangba_Lianmeng/index.jsp 391 | /WebAdmin/eWebEditor/Admin_Login.jsp 392 | /WebAdmin/login.jsp 393 | /WebEdit/admin/upload.jsp 394 | /WebEdit/admin_login.jsp 395 | /WebEdit/db/dbwebedit%23cc495898.jsp 396 | /WebEditor/admin_login.jsp 397 | /Yingjian_Zixun/index.jsp 398 | /Yinshua_Chuban/index.jsp 399 | /Zuqiu_Tianxia/1025.jsp 400 | /Zuqiu_Tianxia/index.jsp 401 | /Zzm.jsp 402 | /__vti_inf.jsp 403 | /_admin.jsp 404 | /_vt_bin/contents.jsp 405 | /_vt_bin/fpadmin.jsp 406 | /_vti_bin/shtml.dll/nosuch.jsp 407 | /_vti_log/_vti_cnf/default.jsp 408 | /_vti_log/default.jsp 409 | /a.jsp 410 | /a0p7digshell2.jsp 411 | /a_admin.jsp 412 | /a_main.jsp 413 | /aa.jsp 414 | /aaa.jsp 415 | /about.jsp 416 | /acblog.jsp 417 | /account.jsp 418 | /acct/login.jsp 419 | /ad.jsp 420 | /ad/ad_edit.jsp 421 | /ad/upload.jsp 422 | /ad/uploadsave.jsp 423 | /ad_admin/admin_login.jsp 424 | /ad_admin_login.jsp 425 | /ad_edit.jsp 426 | /ad_index.jsp 427 | /ad_login.jsp 428 | /ad_manage.jsp 429 | /add.jsp 430 | /addFile.jsp 431 | /addPicture.jsp 432 | /add_admin.jsp 433 | /add_user.jsp 434 | /addlb.jsp 435 | /addmember.jsp 436 | /adduser.jsp 437 | /adlogin.jsp 438 | /adm.jsp 439 | /adm_login.jsp 440 | /adm_menu.jsp 441 | /adm_user.jsp 442 | /admcheck.jsp 443 | /admcheckform.jsp 444 | /admin-login.jsp 445 | /admin-login/login.jsp 446 | /admin/%23m_x%23data.jsp 447 | /admin/AdminLogin1.jsp 448 | /admin/AdminMenu.jsp 449 | /admin/Admin_Database.jsp 450 | /admin/BathUpdate.jsp 451 | /admin/FCKeditor/editor/filemanager/browser/default/browser.jsp?Type=all&Connector=connectors/asp/connector.jsp 452 | /admin/FCKeditor/editor/filemanager/upload/test.jsp 453 | /admin/LoginAdministrator.jsp 454 | /admin/Select_feedback.jsp 455 | /admin/SiteConfig.jsp 456 | /admin/SouthidcEditor/PopUp.jsp 457 | /admin/SouthidcEditor/admin_login.jsp 458 | /admin/Southidceditor/upload.jsp 459 | /admin/SysConfig.jsp 460 | /admin/Sys_db.jsp 461 | /admin/Upfile_Image.jsp 462 | /admin/Upfile_Soft.jsp 463 | /admin/Upfile_SoftPic.jsp 464 | /admin/UploadImage3_upload.jsp 465 | /admin/Upload_Image.jsp 466 | /admin/Upload_Soft.jsp 467 | /admin/Upload_SoftPic.jsp 468 | /admin/WEB-INF/classes/ContextAdmin.java/x00.jsp 469 | /admin/WebEdit/admin_login.jsp 470 | /admin/WebEditor/admin_login.jsp 471 | /admin/account.jsp 472 | /admin/ad_edit.jsp 473 | /admin/ad_login.jsp 474 | /admin/adm_menu.jsp 475 | /admin/admin_6list.jsp 476 | /admin/admin_NUpLoad.jsp 477 | /admin/admin_admin.jsp 478 | /admin/admin_ads.jsp 479 | /admin/admin_copy.jsp 480 | /admin/admin_fileup.jsp 481 | /admin/admin_h.jsp 482 | /admin/admin_index.jsp 483 | /admin/admin_login.jsp 484 | /admin/admin_main.jsp 485 | /admin/admin_mb.jsp 486 | /admin/admin_menu.jsp 487 | /admin/admin_setup.jsp 488 | /admin/admin_styles.jsp 489 | /admin/admin_template.jsp 490 | /admin/admin_upfile.jsp 491 | /admin/admin_upload.jsp 492 | /admin/admin_uploadfile.jsp 493 | /admin/admin_user.jsp 494 | /admin/adminlogin.jsp 495 | /admin/adminn.jsp 496 | /admin/admlogin.jsp 497 | /admin/asp.jsp 498 | /admin/aspcheck.jsp 499 | /admin/aspinfo.jsp 500 | /admin/b2b_sysdata.jsp 501 | /admin/backdata.jsp 502 | /admin/backdate.jsp 503 | /admin/backlogin.jsp 504 | /admin/backup.jsp 505 | /admin/code.jsp 506 | /admin/config.jsp 507 | /admin/conn.jsp 508 | /admin/controlpanel.jsp 509 | /admin/cp.jsp 510 | /admin/cz_login.jsp 511 | /admin/dama.jsp 512 | /admin/data/%23down19827.jsp 513 | /admin/data/data.jsp 514 | /admin/data/user.jsp 515 | /admin/database.jsp 516 | /admin/db.jsp 517 | /admin/dbb.jsp 518 | /admin/default.jsp 519 | /admin/default/admin.jsp 520 | /admin/default/login.jsp 521 | /admin/diy.jsp 522 | /admin/downfile.jsp 523 | /admin/eWeb/admin_login.jsp 524 | /admin/eWebEditor/admin_login.jsp 525 | /admin/eWebEditor_v280_Free/admin_login.jsp 526 | /admin/edit/admin_login.jsp 527 | /admin/edit/upload.jsp 528 | /admin/editor.jsp 529 | /admin/editor/admin_login.jsp 530 | /admin/editor/admin_style.jsp 531 | /admin/editor/editor/filemanager/upload/test.jsp 532 | /admin/editor/upload.jsp 533 | /admin/enda.jsp 534 | /admin/ew/upload.jsp 535 | /admin/ewebedit/admin_login.jsp 536 | /admin/ewebeditor/upload.jsp 537 | /admin/fckeditor/editor/filemanager/browser/default/browser.jsp?Type=Image&Connector=connectors/asp/connector.jsp 538 | /admin/get_your_passport.jsp 539 | /admin/go.jsp 540 | /admin/helps.jsp 541 | /admin/home.jsp 542 | /admin/htmedit/admin_login.jsp 543 | /admin/htmedit/db/ewebeditor.jsp 544 | /admin/htmledit/admin_login.jsp 545 | /admin/htmleditor/admin_login.jsp 546 | /admin/htmleditor/upload.jsp 547 | /admin/inc_config.jsp 548 | /admin/index_login.jsp 549 | /admin/info.jsp 550 | /admin/left.jsp 551 | /admin/login.jsp 552 | /admin/login1.jsp 553 | /admin/logina.jsp 554 | /admin/logo.jsp 555 | /admin/logout.jsp 556 | /admin/lygofa.jsp 557 | /admin/m_bian/db/%23ewebeditor.jsp 558 | /admin/main.jsp 559 | /admin/manage.jsp 560 | /admin/manage/admin.jsp 561 | /admin/manage/login.jsp 562 | /admin/md5.jsp 563 | /admin/member/login.jsp 564 | /admin/menu.jsp 565 | /admin/myup.jsp 566 | /admin/news.jsp 567 | /admin/newsinput.jsp 568 | /admin/nsclass.jsp 569 | /admin/open.jsp 570 | /admin/ows_login.jsp 571 | /admin/picup.jsp 572 | /admin/print/data_1.jsp 573 | /admin/save_upfile.jsp 574 | /admin/saveup.jsp 575 | /admin/test.jsp/info.jsp 576 | /admin/unloadimg.jsp 577 | /admin/up.jsp 578 | /admin/up_images.jsp 579 | /admin/upfile-flash.jsp 580 | /admin/upfile.jsp 581 | /admin/upfile1.jsp 582 | /admin/upfile2.jsp 583 | /admin/upfile_flash.jsp 584 | /admin/upload.jsp 585 | /admin/upload1.jsp 586 | /admin/upload2.jsp 587 | /admin/uploadPic.jsp 588 | /admin/uploadPic.jsp?actionType=mod&picName=miao.jsp 589 | /admin/upload_.jsp 590 | /admin/upload_1.jsp 591 | /admin/upload_2.jsp 592 | /admin/upload_3.jsp 593 | /admin/uploadfaceok.jsp 594 | /admin/uploadfileBanner.jsp 595 | /admin/uploadfileCases.jsp 596 | /admin/uploadfileCasesType.jsp 597 | /admin/uploadfileDown.jsp 598 | /admin/uploadfileLink.jsp 599 | /admin/uploadfileNews.jsp 600 | /admin/uploadfileNewsPic.jsp 601 | /admin/uploadfilePartners.jsp 602 | /admin/uploadfileServices.jsp 603 | /admin/uploadfileServicesType.jsp 604 | /admin/uploadfiletemp_pic.jsp 605 | /admin/uploadsave.jsp 606 | /admin/uppic.jsp 607 | /admin/user/User_Admin.jsp 608 | /admin/user/login.jsp 609 | /admin/user_login.jsp 610 | /admin/web.jsp 611 | /admin/web_login.jsp 612 | /admin/webeditor/admin_login.jsp 613 | /admin/wolf.jsp 614 | /admin/xh_login.jsp 615 | /admin/ydxzdate.jsp 616 | /admin/yns_login.jsp 617 | /admin/z9v8config.jsp 618 | /admin/z9v8conn.jsp 619 | /admin/z9v8login.jsp 620 | /admin/z9v8md5.jsp 621 | /admin/z9v8myup.jsp 622 | /admin/z9v8upfile_flash.jsp 623 | /admin/z9v8uploadPic.jsp 624 | /admin1.jsp 625 | /admin1/Admin_Login.jsp 626 | /admin123.jsp 627 | /admin2.jsp 628 | /admin3.jsp 629 | /admin4.jsp 630 | /admin666.jsp 631 | /admin888.jsp 632 | /admin999.jsp 633 | /Main.jsp 634 | /testno404page.jsp 635 | /test.jsp 636 | -------------------------------------------------------------------------------- /Burp_force_directory/dictionary/MDB.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AttackandDefenceSecurityLab/AD_WebScanner/59124f421202d20a2def1291799443c5635a4b69/Burp_force_directory/dictionary/MDB.txt -------------------------------------------------------------------------------- /Burp_force_directory/dictionary/PHP.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AttackandDefenceSecurityLab/AD_WebScanner/59124f421202d20a2def1291799443c5635a4b69/Burp_force_directory/dictionary/PHP.txt -------------------------------------------------------------------------------- /PortScanner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import nmap 3 | import socket 4 | from urllib import parse 5 | 6 | ports = '21, 22, 23, 25, 53, 80, 161, 162, 443, 445, 1080, 1433, 3306, 3389, 8080' 7 | ''' 8 | 21 ftp 9 | 22 ssh 10 | 23 telnet 11 | 25 smtp 12 | 53 domain 13 | 80 http 14 | 139 netbios-ssn 15 | 161/162 snmp 16 | 443 https 17 | 445 microsoft-ds 18 | 1080 socks 19 | 1433 mssql 20 | 1521 oracle 21 | 3306 mysql 22 | 3389 ms-wbt-server 23 | 8080 http-proxy 24 | ''' 25 | 26 | class PortScanner: 27 | global ports 28 | 29 | def __init__(self, url): 30 | ''' 31 | 从url中获取主机名,并将其解析为对应的ip地址 32 | ''' 33 | name = parse.urlparse(url).hostname 34 | self.host = socket.gethostbyname(name) 35 | 36 | def ports_scan(self): 37 | ''' 38 | 使用nmap对指定的端口进行扫描,并将每个端口的扫描结果逐一输出 39 | ''' 40 | host = self.host 41 | try: 42 | nm = nmap.PortScanner() 43 | nm.scan(host, ports) 44 | 45 | print('----------------------------------------------------') 46 | print('Host: %s (%s)' % (host, nm[host].hostname())) 47 | print('State: %s' % nm[host].state()) 48 | 49 | for proto in nm[host].all_protocols(): 50 | print('-------------') 51 | print('Protocol: %s' % proto) 52 | list_ports = nm[host][proto].keys() 53 | for port in list_ports: 54 | print('port: %-6s\tname: %-12s\tstate: %-8s\tproduct: %-16s\textrainfo: %-12s\tversion: %-6s' 55 | % (port, nm[host][proto][port]['name'], nm[host][proto][port]['state'], nm[host][proto][port]['product'], nm[host][proto][port]['extrainfo'], nm[host][proto][port]['version'])) 56 | except Exception as e: 57 | raise e 58 | 59 | if __name__ == '__main__': 60 | url = 'https://www.baidu.com' 61 | portscanner = PortScanner(url) 62 | portscanner.ports_scan() 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AD_WebScanner 2 | 3 | AD工作室精心研发漏洞安全扫描器 4 | > 整合各大开源模块,自行加以整合 5 | > 6 | > python版本 :3以上 7 | 8 | # 开发约束 9 | ## 模块构造器 10 | - 构造器的参数为URL+redis+模块特有参数, 11 | 12 | ```python 13 | class demo(url,save_pool,....): 14 |  self.module_redis = redis.Redis(connection_pool=save_pool) 15 | ... 16 | 17 | ``` 18 | 19 | - `savepool`由基础模块初始化子模块时提供,子模块可直接使用`redis.Redis(connection_pool=save_pool)`连接共用存储池 20 | - 特有参数需指定默认参数,即只传入URL模块亦可单独执行 21 | - 在构造函数的方法声明内注释说明特有参数的类型 22 | - 构造器内需要包含模块的执行方法 23 | - 从基础模块的相应键名获取各模块的特殊设置,如`hget('base','spider-threads)#获取爬虫模块的线程设置值` 24 | 25 | ## redis 26 | - 连接redis实例的名称为模块名_redis,如`spider_redis` 27 | - 模块中应省略创建存储池的过程,直接连接基础模块所创建的存储池 28 | - 建议使用redis的hash存储,类型为`'模块名':'键':'值'`,如`hset('base','url',url)` 29 | - 存储聚合数据类型时(如list/set),使用redis的list/set存储,键名为模块名-键名,如`redis.sadd('base-input_opt','100)` 30 | - 如果使用string\list\set存储类型,即视为共用的存储对象,各模块均有读写权限 31 | - 进行redis的写入/读取操作的方法后注释说明传入/读取值的名称和类型 32 | 33 | ## 通用约束 34 | - 模块内每个方法声明后注释传入参数的类型/说明,返回值的类型/说明 35 | - 模块的关键处理步骤需进行注释 36 | - 个人负责各自的模块,需修改他人部分的请与相应模块的负责人交流 37 | - 模块开头用注释标明作者/修改日期 38 | - 模块包含`is_finished()`方法,返回值为True或False,当模块的执行方法完成返回True,否则返回False 39 | - 模块执行返回的信息应存入redis中 40 | 41 | 42 | # 主要功能 43 | 44 | - [x] 爬虫 (leslie) 45 | - [x] 目录爆破 (xuxu) 46 | - [x] 模块化设计,框架设计 (Chernobyl) 47 | - [x] 子域名爆破 (leslie) 48 | - [ ] 命令执行类(leslie) 49 | - [x] 数据库漏洞扫描(threeworld)     50 | - [x] 弱密码爆破(leslie) 51 | - [ ] XSS类 52 | - [ ] 敏感个人信息泄露 53 | - [ ] 内网渗透 54 | - [ ] 中间件扫描或者指纹扫描 55 | - [ ] 无线网络扫描 56 | - [ ] 端口扫描(xuxu) 57 | - [x] 图形化界面 58 | - to be continued 59 | 60 | # 依赖 61 | 62 | - requests 63 | - redis 64 | - bs4 65 | - urllib 66 | 67 | # 可视化页面 68 | - 使用flask+html+css编写 69 | - 确保templates+static这两个文件夹在index.py的同一文件夹下 70 | - 需手动把AD_Scanner_Base.py文件中最后一行的input()给注释掉方可正常使用该模块 71 | - 用flask编译器启动index.py,在浏览器输入127.0.0.1:5000访问 72 | 73 | # 参考 74 | https://github.com/We5ter/Scanners-Box 75 | -------------------------------------------------------------------------------- /Sqliscan/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AttackandDefenceSecurityLab/AD_WebScanner/59124f421202d20a2def1291799443c5635a4b69/Sqliscan/__init__.py -------------------------------------------------------------------------------- /Sqliscan/serverinfo.py: -------------------------------------------------------------------------------- 1 | import time 2 | import signal 3 | import multiprocessing 4 | import bs4 5 | from urllib.parse import urlparse 6 | 7 | from Sqliscan import std 8 | from Sqliscan import web 9 | 10 | 11 | def init(): 12 | signal.signal(signal.SIGINT, signal.SIG_IGN) 13 | 14 | def check(urls): 15 | """ 16 | 17 | """ 18 | 19 | domains_info = [] # 20 | results = {} # 21 | 22 | childs = [] # 23 | max_processes = multiprocessing.cpu_count() * 2 24 | pool = multiprocessing.Pool(max_processes, init) 25 | 26 | for url in urls: 27 | def callback(result, url=url): 28 | results[url] = result 29 | childs.append(pool.apply_async(__getserverinfo, (url, ), callback=callback)) 30 | 31 | # try: 32 | while True: 33 | time.sleep(0.5) 34 | if all([child.ready() for child in childs]): 35 | break 36 | # except Exception: 37 | # pool.terminate() 38 | # pool.join() 39 | # else: 40 | pool.close() 41 | pool.join() 42 | 43 | # 44 | # 45 | for url in urls: 46 | if url in results.keys(): 47 | data = results.get(url) 48 | domains_info.append([url, data[0], data[1]]) 49 | continue 50 | 51 | domains_info.append([url, '', '',]) 52 | 53 | return domains_info 54 | 55 | def __getserverinfo(url): 56 | """get server name and version of given domain""" 57 | 58 | url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0] 59 | 60 | info = [] # to store server info 61 | url = "https://aruljohn.com/webserver/" + url 62 | 63 | try: 64 | result = web.gethtml(url) 65 | except Exception: 66 | raise 67 | 68 | try: 69 | soup = bs4.BeautifulSoup(result, "lxml") 70 | except: 71 | return ['', ''] 72 | 73 | if soup.findAll('p', {"class" : "err"}): 74 | return ['', ''] 75 | 76 | for row in soup.findAll('tr'): 77 | if row.findAll('td', {"class": "title"}): 78 | info.append(row.findAll('td')[1].text.rstrip('\r')) 79 | 80 | return info 81 | -------------------------------------------------------------------------------- /Sqliscan/sqlerrors.py: -------------------------------------------------------------------------------- 1 | #coding = 'utf-8' 2 | 3 | import re 4 | 5 | #页面错误返回的特征 6 | sql_errors = { 7 | "MySQL": (r"SQL syntax.*MySQL", r"Warning.*mysql_.*", r"MySQL Query fail.*", r"SQL syntax.*MariaDB server"), 8 | "PostgreSQL": (r"PostgreSQL.*ERROR", r"Warning.*\Wpg_.*", r"Warning.*PostgreSQL"), 9 | "Microsoft SQL Server": (r"OLE DB.* SQL Server", r"(\W|\A)SQL Server.*Driver", r"Warning.*odbc_.*", r"Warning.*mssql_", r"Msg \d+, Level \d+, State \d+", r"Unclosed quotation mark after the character string", r"Microsoft OLE DB Provider for ODBC Drivers"), 10 | "Microsoft Access": (r"Microsoft Access Driver", r"Access Database Engine", r"Microsoft JET Database Engine", r".*Syntax error.*query expression"), 11 | "Oracle": (r"\bORA-[0-9][0-9][0-9][0-9]", r"Oracle error", r"Warning.*oci_.*", "Microsoft OLE DB Provider for Oracle"), 12 | "IBM DB2": (r"CLI Driver.*DB2", r"DB2 SQL error"), 13 | "SQLite": (r"SQLite/JDBCDriver", r"System.Data.SQLite.SQLiteException"), 14 | "Informix": (r"Warning.*ibase_.*", r"com.informix.jdbc"), 15 | "Sybase": (r"Warning.*sybase.*", r"Sybase message") 16 | } 17 | 18 | def check(html): 19 | """ 20 | 检查html页面是否含有SQLerror 21 | :param html: 22 | :return: (True, db)或(False, None) 23 | """ 24 | for db, errors in sql_errors.items(): 25 | for error in errors: 26 | if re.compile(error): 27 | return True,db 28 | return False,None -------------------------------------------------------------------------------- /Sqliscan/std.py: -------------------------------------------------------------------------------- 1 | #coding = 'utf-8' 2 | import time 3 | import json 4 | from termcolor import colored 5 | from terminaltables import SingleTable 6 | 7 | def stderr(message, end="\n"): 8 | """ 9 | 输出一个错误给用户 10 | :param message: 11 | :param end: 12 | :return: 13 | """ 14 | symbol = colored("[ERR]","red") 15 | currenttime = colored("[{}]".format(time.strftime("%H:%M:%S")),"green") 16 | print("{}{}{}".format(symbol, currenttime,message),end=end) 17 | 18 | def stdout(message, end="\n"): 19 | """ 20 | 输出一个信息给用户 21 | :param message: 22 | :param end: 23 | :return: 24 | """ 25 | symbol = colored("[MSG]", "yellow") 26 | currentime = colored("[{}]".format(time.strftime("%H:%M:%S")), "green") 27 | print("{} {} {}".format(symbol, currentime, message), end=end) 28 | 29 | def stdin(message, params, upper=False, lower=False): 30 | """ 31 | 询问用户输入信息 32 | :param message: 33 | :param params: 34 | :param upper: 35 | :param lower: 36 | :return: 用户输入的信息 37 | """ 38 | symbol = colored("[OPT]","magenta") 39 | currentime = colored("[{}]".format(time.strftime("%H:%M:%S")), "green") 40 | option = input("{} {} {}: ".format(symbol, currentime, message)) 41 | 42 | if upper: 43 | option =option.upper() 44 | elif lower: 45 | option = option.lower() 46 | 47 | while option not in params: 48 | option = input("{} {} {}: ".format(symbol, currentime, message)) 49 | 50 | if upper: 51 | option = option.upper() 52 | elif lower: 53 | option = option.lower() 54 | 55 | return option 56 | 57 | def showsign(message): 58 | """ 59 | 输出一个漏洞信息 60 | :param message: 61 | :return: 62 | """ 63 | print(colored(message, "magenta")) 64 | 65 | 66 | def fullprint(data): 67 | """ 68 | 输出漏洞网址的服务器信息 69 | :param data: 70 | :return: 71 | """ 72 | 73 | # [ 74 | # ["index", "url", "db", server", "lang"], 75 | # ["1", "sql.com", "mysql", apache", "php/5.5xxx"] 76 | # ] 77 | 78 | title = " VULNERABLE URLS " 79 | table_data = [["index", "url", "db", "server", "lang"]] 80 | # add into table_data by one by one 81 | for index, each in enumerate(data): 82 | table_data.append([index+1, each[0], each[1], each[2][0:30], each[3][0:30]]) 83 | 84 | table = SingleTable(table_data, title) 85 | print(table.table) 86 | 87 | 88 | def dumpjson(array): 89 | """ 90 | 以json格式存储 91 | :param array: 92 | :return: 93 | """ 94 | jsondata = {} 95 | 96 | for index, result in enumerate(array): 97 | jsondata[index] = { 98 | 'url': result[0].encode('utf-8'), 99 | 'db': result[1].encode('utf-8'), 100 | 'server': result[2].encode('utf-8') 101 | } 102 | jsonresult = json.dumps(jsondata,cls=MyEncoder) 103 | return jsonresult 104 | 105 | class MyEncoder(json.JSONEncoder): 106 | def default(self, obj): 107 | if isinstance(obj, bytes): 108 | return str(obj, encoding='utf-8'); 109 | return json.JSONEncoder.default(self, obj) 110 | 111 | if __name__ == '__main__': 112 | stderr('error') 113 | stdout("OK") 114 | showsign("vulnerable") 115 | stdin("do you want to continue scanning? [Y/N]", ["Y", "N"], upper=True) -------------------------------------------------------------------------------- /Sqliscan/useragents.py: -------------------------------------------------------------------------------- 1 | #coding='utf-8 2 | import random 3 | 4 | def get(): 5 | """ 6 | 随机返回user-agents 7 | :return: user-agents 8 | """ 9 | 10 | return random.choice(useragents) 11 | 12 | 13 | useragents = [ 14 | { 15 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' 16 | }, 17 | { 18 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36' 19 | }, 20 | { 21 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36' 22 | }, 23 | { 24 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36' 25 | }, 26 | { 27 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246' 28 | }, 29 | { 30 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1' 31 | }, 32 | { 33 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0' 34 | }, 35 | { 36 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0' 37 | }, 38 | { 39 | 'User-Agent': 'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0' 40 | }, 41 | { 42 | 'User-Agent': 'Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0' 43 | }, 44 | { 45 | 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Maxthon/3.0.8.2 Safari/533.1' 46 | }, 47 | { 48 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A' 49 | }, 50 | { 51 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2' 52 | } 53 | ] 54 | 55 | if __name__ == '__main__': 56 | print(get()) -------------------------------------------------------------------------------- /Sqliscan/web.py: -------------------------------------------------------------------------------- 1 | #coding ='utf-8' 2 | 3 | import urllib.request,urllib.error,urllib.parse 4 | from Sqliscan import useragents 5 | 6 | def gethtml(url, lastURL=False): 7 | """ 8 | 给定url返回html 9 | :param url: 10 | :param lastURL: 11 | :return: HTML或False 12 | """ 13 | #对url的预处理 14 | if not (url.startswith("http://") or url.startswith("https://")): 15 | url = "http://"+url 16 | header = useragents.get() 17 | request = urllib.request.Request(url, None, header) 18 | html = None 19 | 20 | try: 21 | response = urllib.request.urlopen(request,timeout=10) 22 | except urllib.HTPPError as e: 23 | #返回500 24 | if e.getcode() == 500: 25 | html = e.read() 26 | pass 27 | except urllib.URLError as e: 28 | pass 29 | 30 | except Exception as e0: 31 | raise e0 32 | 33 | except: 34 | pass 35 | 36 | else: 37 | html = response.read() 38 | if html: 39 | if lastURL == True: 40 | return (html, response.url) 41 | else: 42 | return html 43 | return False 44 | 45 | 46 | if __name__ == '__main__': 47 | html = gethtml("http://testphp.vulnweb.com:80/listproducts.php?cat=1") 48 | print(html) -------------------------------------------------------------------------------- /XSS_payload/wordlist.txt: -------------------------------------------------------------------------------- 1 | 2 | "> 3 | "> 4 | "> 5 | "> 6 | ">

Clickme

8 | ">Clickme 9 | ">Clickme 10 | ">click 11 | "> 34 | 35 | 36 | 56 | 57 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | save_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True)#开启本地radis 4 | test1 = redis.Redis(connection_pool=save_pool)#创建一个连接实例 5 | 6 | data = 'aa' 7 | test1.sadd('data',data) 8 | data = 'bb' 9 | test1.sadd('data',data) 10 | 11 | data = ['dd','ff'] 12 | test1.sadd('data',data) 13 | for x in test1.smembers('data'): 14 | print(type(x),x) 15 | -------------------------------------------------------------------------------- /the_harvest.py: -------------------------------------------------------------------------------- 1 | from tHar_lib.engine_search import Search 2 | from tHar_lib import hostchecker 3 | import redis 4 | import tldextract 5 | 6 | class TheHarvester(): 7 | def __init__(self, url, savepool, limit=200, engine='baidu'): 8 | val = tldextract.extract(url) 9 | self.word = "{0}.{1}".format(val.domain, val.suffix) 10 | self.limit = limit 11 | self.engine = engine 12 | self.savepool = savepool 13 | self.finished = False 14 | self.redis_connect() 15 | 16 | 17 | def is_finished(self): 18 | return self.finished 19 | 20 | def redis_connect(self): 21 | self.harvest_redis = redis.Redis(connection_pool=self.savepool) 22 | 23 | 24 | def start_search(self): 25 | search = Search(self.word, self.limit, self.engine) 26 | search.process() 27 | self.all_emails = search.get_emails() 28 | self.all_hosts = search.get_hostnames() 29 | self.host_check() 30 | for i in self.all_hosts: 31 | self.harvest_redis.sadd('Harvest_subdomain', i) 32 | for i in self.all_emails: 33 | self.harvest_redis.sadd('Harvest_emails', i) 34 | self.finished = True 35 | 36 | 37 | # print(self.all_hosts, self.all_emails) 38 | 39 | def host_check(self): 40 | self.total_length = len(self.all_hosts) 41 | self.all_hosts = sorted(set(self.all_hosts)) 42 | self.hosts = hostchecker.Checker(self.all_hosts).check() 43 | 44 | def run(self): 45 | self.action = self.harvest_redis.hget('base','harvest_args') 46 | if self.action == 'search': 47 | self.start_search() 48 | 49 | 50 | if __name__ == '__main__': 51 | save_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True) 52 | Harvester = TheHarvester('baidu.com',savepool=save_pool) 53 | 54 | Harvester.start_search() 55 | -------------------------------------------------------------------------------- /url_spider.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import threading 3 | from bs4 import BeautifulSoup 4 | from urllib.parse import urljoin, urlparse 5 | import time 6 | import redis 7 | 8 | 9 | def turn_num(url, length): 10 | ''' 11 | 转化为特征向量 12 | :param url: url 13 | :param length: 域名和协议的长度,我们只需要后面的部分,前面的部分每个url都一样,节省计算 14 | :return: url的特征向量 15 | ''' 16 | url = url[length:] 17 | dim = 75 18 | 19 | char_index = [i for i in range(len(url)) if url[i] == '/'] 20 | char_index.insert(0, 0) 21 | 22 | char_weight = [] 23 | for i in range(len(char_index)): 24 | try: 25 | char_weight.append(url[char_index[i]:char_index[i + 1]]) 26 | except: 27 | char_weight.append(url[char_index[i]:]) 28 | 29 | num = len(char_weight) 30 | 31 | url_weight = [ord(j)*(num-i)*(num-i) for i in range(len(char_weight)) for j in char_weight[i]] 32 | 33 | for i in range(len(url_weight), dim): 34 | url_weight.append(0) 35 | return url_weight 36 | 37 | 38 | def cos(vector1,vector2): 39 | ''' 40 | 余弦相似度计算 41 | :param vector1: url1 42 | :param vector2: url2 43 | :return: 相似度大小 44 | ''' 45 | dot_product = 0.0 46 | normA = 0.0 47 | normB = 0.0 48 | for a,b in zip(vector1,vector2): 49 | dot_product += a*b 50 | normA += a**2 51 | normB += b**2 52 | if normA == 0.0 or normB == 0.0: 53 | return None 54 | else: 55 | return dot_product / ((normA*normB)**0.5) 56 | 57 | 58 | def similarities(data, url, length): 59 | ''' 60 | 将url与一组url比较相似度 61 | :param data: 数据集 62 | :param url: 目标url 63 | :param length: 长度 64 | :return: 判定结果 65 | ''' 66 | url_list = [turn_num(i, length) for i in data] 67 | target_url = turn_num(url, length) 68 | for i in url_list: 69 | try: 70 | if cos(target_url, i) > 0.9995: 71 | return 1 72 | except: 73 | return 0 74 | return 0 75 | 76 | 77 | class Downloader: # 发起请求,获取内容 78 | def get(self, url, content): 79 | try: 80 | r = requests.get(url, timeout=100) 81 | if r.status_code != 200: 82 | return None 83 | content.append(r.text) 84 | return content 85 | except: 86 | pass 87 | 88 | 89 | class UrlManager: 90 | def __init__(self): 91 | self.new_urls = set() 92 | self.old_urls = set() 93 | 94 | def add_new_url(self, url, length): 95 | if url is None or similarities(self.old_urls, url, length): # 判断爬过的url中有没有相似的url,如果有就跳过 96 | return 97 | if url not in self.new_urls and url not in self.old_urls: 98 | self.new_urls.add(url) 99 | 100 | def add_new_urls(self,urls, length): 101 | if urls is None or len(urls) == 0: 102 | return 103 | for url in urls: 104 | self.add_new_url(url, length) 105 | 106 | def has_new_url(self): 107 | return len(self.new_urls) != 0 108 | 109 | def get_new_url(self, redis_add): 110 | new_url = self.new_urls.pop() 111 | redis_add(new_url) 112 | self.old_urls.add(new_url) 113 | return new_url 114 | 115 | 116 | class SpiderMain: 117 | def __init__(self, root, savepool): 118 | self.urls = UrlManager() 119 | self.down = Downloader() 120 | self.root = root 121 | self.domain = urlparse(root).hostname 122 | self.rootlength = len(self.root) 123 | self.savepool = savepool 124 | self.redis_connect() 125 | self.finished = False 126 | self.spider_redis.set('spider_redis', 'False') 127 | 128 | def run(self): 129 | self.redis_get() 130 | if self.action == 'craw': 131 | self.craw() 132 | self.finished = True 133 | self.spider_redis.set('spider_redis', 'True') 134 | 135 | def is_finished(self): 136 | return self.finished 137 | 138 | def redis_get(self): 139 | self.action = self.spider_redis.hget('base', 'spider_args') 140 | self.threadnum = self.spider_redis.hget('base', 'spider_threads') 141 | 142 | def redis_set(self, url): 143 | #print('spider add!') 144 | try: 145 | self.spider_redis.sadd('Spider_full_urls', url) 146 | except Exception as e: 147 | print(e) 148 | 149 | def redis_connect(self): 150 | #save_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True) 151 | self.spider_redis = redis.Redis(connection_pool=self.savepool) 152 | 153 | def judge(self, domain, url): # 判断链接的域名 154 | if(url.find(domain) != -1): 155 | return True 156 | else: 157 | return False 158 | 159 | def parse(self, page_url, content): # 解析页面 160 | if content is None: 161 | return 162 | soup = BeautifulSoup(content, 'lxml') 163 | news = self.get_new_urls(page_url, soup) 164 | return news 165 | 166 | def get_new_urls(self, page_url, soup): # 从页面里面获得a标签列表,并组成新地址 167 | new_urls = set() 168 | links = soup.find_all('a') 169 | for link in links: 170 | new_url = link.get('href') 171 | new_full_url = urljoin(page_url, new_url) 172 | if self.judge(self.domain, new_full_url): 173 | new_urls.add(new_full_url) 174 | return new_urls 175 | 176 | def craw(self): # 控制流程,利用多线程发起请求 177 | self.urls.add_new_url(self.root, self.rootlength) 178 | while self.urls.has_new_url(): 179 | content = [] 180 | th = [] 181 | for _ in list(range(int(self.threadnum))): 182 | if self.urls.has_new_url() is False: 183 | break 184 | new_url = self.urls.get_new_url(self.redis_set) 185 | 186 | # print("craw: " + new_url) 187 | t = threading.Thread(target=self.down.get, args=(new_url, content)) 188 | t.start() 189 | th.append(t) 190 | for t in th: 191 | t.join() 192 | 193 | for _str in content: 194 | if _str is None: 195 | continue 196 | 197 | new_urls = self.parse(new_url, _str) 198 | self.urls.add_new_urls(new_urls, self.rootlength) 199 | 200 | def all(self): 201 | print('[+] ALL ' + str(len(self.urls.old_urls))) 202 | 203 | def check(self): 204 | reponse = requests.get(self.root) 205 | if reponse == 200: 206 | if len(self.urls.old_urls) > 1: 207 | #print('the status of spider [success]') 208 | return 1 209 | else: 210 | #print('the status of spider [fail]') 211 | return 0 212 | else: 213 | #print('target url error') 214 | return 0 215 | 216 | 217 | 218 | if __name__ == '__main__': 219 | url = 'http://www.leslie2018.com' 220 | save_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True) 221 | spider = SpiderMain(url, save_pool) 222 | spider.run() 223 | print('[+] All ' + str(len(spider.urls.old_urls))) 224 | --------------------------------------------------------------------------------