├── requirements.txt ├── .DS_Store ├── bufferflic.png ├── README.md └── bufferflic.py /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | lxml 3 | argparse 4 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dr0op/bufferfly/HEAD/.DS_Store -------------------------------------------------------------------------------- /bufferflic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dr0op/bufferfly/HEAD/bufferflic.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bufferfly 2 | 攻防资产处理小工具，对攻防前的信息搜集到的大批量资产/域名进行存活检测、获取标题头、语料提取、常见web端口检测、简单中间识别，去重等，便于筛选有价值资产。 3 | 4 | ```python 5 | __ ________ ______ 6 | / /_ __ __/ __/ __/__ _____/ __/ /_ __ 7 | / __ \/ / / / /_/ /_/ _ \/ ___/ /_/ / / / / 8 | / /_/ / /_/ / __/ __/ __/ / / __/ / /_/ / 9 | /_.___/\__,_/_/ /_/ \___/_/ /_/ /_/\__, / 10 | /____/ v1.2.1 11 | 1.高速资产存活检测，获取标题 12 | 2.常见Web端口访问测试/获取标题 lxml方式速度较快 13 | 3.资产去重 14 | 4.随机UA 15 | 5.C段web端口探测/获取标题 16 | 6.C段识别 17 | 7.shiro识别 18 | 8.简单中间件识别 19 | 20 | 适用用于外网资产梳理 21 | 22 | TODO: 23 | 1.在不发送更多请求的情况下模糊识别weblogic/jboss/jenkins/zabbix/activeMQ/solr/gitlab/spring等 24 | 2.常见端口扫描(22/445/3389/3306/6379/1521等常见端口 与masscan、nmap结合) 25 | 26 | 27 | ``` 28 | 29 | ![bufferfly](bufferflic.png) 30 | 31 | # 使用 32 | ``` 33 | usage: bufferflic.py [-h] [-t] [-f] [--mvdups] [-c] 34 | 35 | 攻防资产处理工具，用于简单处理筛选攻防前的有价值资产 36 | 37 | optional arguments: 38 | -h, --help show this help message and exit 39 | -t , --thread 线程参数 40 | -f , --file 从文件读取 41 | --mvdups 文本去重 42 | -c , --c C段探测 43 | 44 | ``` 45 | 46 | # DEFF 47 | 48 | 1. 添加C段探测 49 | 2. 添加C段识别 50 | 3. 添加shiro中间件检测 51 | 4. 修复一些显示BUG 52 | 53 | # requirements 54 | 55 | 1. requests 56 | 2. lxml 57 | 3. argparse 58 | 59 | ```shell 60 | pip3 install -r requirements.txt 61 | ``` 62 | 63 | # TODO 64 | 65 | 1. 在不发送更多请求的情况下模糊识别weblogic/jboss/jenkins/zabbix/activeMQ/solr/gitlab/spring等 66 | 2. 常见端口扫描(22/445/3389/3306/6379/1521等常见端口 与masscan、nmap结合) 67 | 68 | # help 69 | 70 | 主要是方便前期收集到的大量资产梳理的小工具，代码比较简陋，但还算实用。不要太相信Fofa。 -------------------------------------------------------------------------------- /bufferflic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/evn python3 2 | #_*_ coding:utf-8 _*- 3 | #攻防演习信息搜集资产处理框架v1.2 4 | #author Ra1ndr0op 5 | 6 | import requests 7 | import re 8 | import sys 9 | import threading 10 | import argparse 11 | from lxml import etree 12 | import random 13 | from collections import Counter 14 | import socket 15 | 16 | if sys.version > '3': 17 | import queue as Queue 18 | from urllib.parse import urlparse 19 | else: 20 | import Queue 21 | import urlparse 22 | 23 | 24 | 25 | URIList = [] 26 | IPCsubList = [] 27 | threadList = [] 28 | CsubThreadList = [] 29 | active_url_list = [] 30 | 31 | urlQueue = Queue.Queue(1000*100) 32 | lineQueue = Queue.Queue(1000*100) 33 | port = list(range(80,90))+list(range(8080,8091))+list(range(8000,8010))+[7001,8032,8023,9200,2375,5904,6066,7077,8161] 34 | URIList = [] 35 | IPCsubList = [] 36 | lineQueue = Queue(1000*100) 37 | 38 | 39 | banner = ''' 40 | __ ________ ______ 41 | / /_ __ __/ __/ __/__ _____/ __/ /_ __ 42 | / __ \/ / / / /_/ /_/ _ \/ ___/ /_/ / / / / 43 | / /_/ / /_/ / __/ __/ __/ / / __/ / /_/ / 44 | /_.___/\__,_/_/ /_/ \___/_/ /_/ /_/\__, / 45 | /____/ v1.2.1 46 | 1.高速资产存活检测，获取标题 47 | 2.常见Web端口访问测试/获取标题 lxml方式速度较快 48 | 3.资产去重：单文件去重，双文件去重 49 | 4.随机UA 50 | 5.C段端口访问测试/获取标题 51 | 6.识别C段IP 52 | 7.shiro识别 53 | 54 | 适用用于外网资产梳理 55 | 56 | TODO: 57 | 1.在不发送更多请求的情况下模糊识别weblogic/jboss/jenkins/zabbix/activeMQ/solr/gitlab/spring等 58 | 2.常见端口扫描(22/445/3389/3306/6379/1521等常见端口 与masscan、nmap结合) 59 | 60 | ''' 61 | 62 | 63 | # 获取url请求title，返回title值 利用正则方式获取title信息 64 | def getTitle(url): 65 | 66 | headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0'} 67 | 68 | if "http://" or "https://" not in url: 69 | url = "http://"+url 70 | try: 71 | sys.stdout.write("\r Going to access {0} ...".format(url)) 72 | res = requests.get(url,headers=headers,timeout=2) 73 | except: 74 | return 75 | 76 | split = " ------ " 77 | code = res.status_code 78 | enc = res.encoding 79 | 80 | if code in [200,301,302,404,403,500]: 81 | try: 82 | text=res.text.encode(enc).decode('utf-8') 83 | except: 84 | try: 85 | text=res.text.encode(enc).decode('gbk') 86 | except: 87 | pass 88 | try: 89 | title =re.search(r'(.*?)',text,re.I).group(1) 90 | except: 91 | title="Null" 92 | 93 | print(url+split+str(code)+split+title) 94 | return str(url)+split+str(code)+split+title 95 | else: 96 | return 97 | 98 | 99 | # 获取url请求title，返回title值 利用lxml方式获取title信息 100 | 101 | def getTitle2(url): 102 | headers={'User-Agent':get_user_agent(),'Cookie':'rememberMe=b69375edcb2b3c5084c02bd9690b6625',} 103 | 104 | if "http://" not in url: 105 | url = "http://"+url 106 | try: 107 | sys.stdout.write("\r[*]Recording to access {0} ...".format(url)) 108 | res = requests.get(url,headers=headers,timeout=2) 109 | except: 110 | return 111 | 112 | split = " ------ " 113 | code = res.status_code 114 | enc = res.encoding 115 | server = get_url_servers(res) 116 | ctext = '' 117 | if code in [200,301,302,404,403,500]: 118 | try: 119 | text=res.text.encode(enc).decode('utf-8') 120 | except: 121 | try: 122 | text=res.text.encode(enc).decode('gbk') 123 | except: 124 | pass 125 | try: 126 | html = etree.HTML(text) 127 | Title = html.findtext('.//title') 128 | title = Title if Title !=None else 'Null' 129 | if None == server: 130 | server = 'Null' 131 | ctext = get_context(text) 132 | if None == ctext: 133 | ctext ='Null' 134 | except: 135 | title="Null" 136 | result = '{}[+]{}{}{}{}{}{}'.format(Color.OKGREEN,url.ljust(36),str(code).ljust(10),server.ljust(20),title[:20].ljust(30),''.join(ctext.split()).ljust(20),Color.ENDC) 137 | print('\r{0}'.format(result)) 138 | #print(url+split+str(code)+split+server+split+title+split+ctext) 139 | #return str(url)+split+str(code)+split+server+split+title 140 | return result 141 | else: 142 | return 143 | 144 | #文本去重, 145 | def MovDups(file): 146 | # 大文件，使用上下文管理,利用set的键值去重 147 | with open(file,'r') as f: 148 | with open(file.split(".")[0]+'-rmdups.txt','w') as ff: 149 | while True: 150 | ulist = f.readlines(1024*10) 151 | if not ulist: 152 | break 153 | rustr = "".join(list(set(ulist))) 154 | ff.write(rustr) 155 | 156 | #双文本去重 157 | def MovDups2(file1,file2): 158 | #去除file1中含有file2内容的部分，然后将两个文件合并 159 | cache = [] 160 | with open(file1,'r') as f: 161 | for ln in f.readlines(): 162 | cache.append(ln) 163 | print(ln) 164 | 165 | with open(file2,'r') as ff: 166 | for ln in ff.readlines(): 167 | if ln in cache: 168 | cache.remove(ln) 169 | continue 170 | cache.append(ln) 171 | 172 | with open("mv2dups.txt",'w') as fff: 173 | for ln in cache: 174 | print(ln) 175 | fff.write(ln) 176 | # 随机UA 177 | def get_user_agent(): 178 | user_agents = [ 179 | "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", 180 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", 181 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)", 182 | "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", 183 | "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", 184 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", 185 | "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", 186 | "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)", 187 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)", 188 | "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6", 189 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1", 190 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0", 191 | "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5", 192 | "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6", 193 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", 194 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20", 195 | "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",] 196 | return random.choice(user_agents) 197 | 198 | # 获取服务器容器 199 | def get_url_server(resp): 200 | try: 201 | for k in resp.headers.keys(): 202 | if k.upper() == 'SERVER': 203 | header_server = resp.headers[k].upper() 204 | return header_server 205 | except: 206 | return 'Unkonw' 207 | 208 | 209 | 210 | # 获取服务器容器 格式化 211 | def get_url_servers(resp): 212 | try: 213 | for k in resp.headers.keys(): 214 | if k.upper() == 'SERVER': 215 | header_server = resp.headers[k].upper() 216 | if re.search('iis/6.0'.upper(), header_server): 217 | short_server = 'IIS/6.0' 218 | elif re.search('iis/7.0'.upper(), header_server): 219 | short_server = 'IIS/7.0' 220 | elif re.search('iis/7.5'.upper(), header_server): 221 | short_server = 'IIS/7.5' 222 | elif re.search('iis/8.0'.upper(), header_server): 223 | short_server = 'IIS/8.0' 224 | elif re.search('iis/8.5'.upper(), header_server): 225 | short_server = 'IIS/8.5' 226 | elif re.search('iis'.upper(), header_server): 227 | short_server = 'IIS' 228 | elif re.search('apache'.upper(), header_server): 229 | short_server = 'Apache' 230 | elif re.search('nginx'.upper(), header_server): 231 | short_server = 'Nginx' 232 | elif re.search('vWebServer'.upper(), header_server): 233 | short_server = 'vWebServer' 234 | elif re.search('openresty'.upper(), header_server): 235 | short_server = 'OpebResty' 236 | elif re.search('tengine'.upper(), header_server): 237 | short_server = 'Tengine' 238 | elif re.search('apusic'.upper(), header_server): 239 | short_server = 'APUSIC' 240 | elif re.search('marco'.upper(), header_server): 241 | short_server = 'Marco' 242 | elif re.search('twebap'.upper(), header_server): 243 | short_server = 'TWebAP' 244 | elif re.search('360'.upper(), header_server): 245 | short_server = '360wzws' 246 | elif re.search('cdn'.upper(), header_server): 247 | short_server = 'CDN' 248 | else: 249 | short_server = get_url_server(resp) 250 | if k.upper() == 'SET-COOKIE': 251 | header_cookie = resp.headers[k] 252 | if 'rememberMe=deleteMe' in header_cookie: 253 | short_server= 'shiro' 254 | return short_server 255 | except: 256 | return "Unkonw" 257 | 258 | 259 | # 获取中间部分内容信息 260 | # TODO: jenkins/weblogic/zabbix/shiro/elasticsearch/gitlab/shiro 261 | def get_context(html): 262 | context = etree.HTML(html) 263 | for bad in context.xpath(".//script"): 264 | bad.getparent().remove(bad) 265 | for bad in context.xpath(".//style"): 266 | bad.getparent().remove(bad) 267 | content = context.xpath('string(.)').replace(" ","").replace("\n","") 268 | n = int(len(content)/2) 269 | ct = content[n-20:n+20] 270 | return ct.strip()[:25] 271 | 272 | # 提取url中的域名或IP 273 | def url2Domain(line): 274 | if ("http://" in line) or ("https://" in line): 275 | return urlparse(line)[1] 276 | else: 277 | return line 278 | 279 | 280 | # 域名转为ip 281 | def Domain2Ip(host): 282 | try: 283 | ip=socket.getaddrinfo(host, None)[0][4][0] 284 | return ip 285 | except Exception as e: 286 | pass 287 | 288 | # # C段识别 289 | # def FindCsub(urls): 290 | # ipc = [] 291 | # for url in urls: 292 | # ip=Domain2Ip(url2Domain(url)) 293 | # if None != ip: 294 | # x =ip.rfind('.') 295 | # ip=ip[:x+1]+"0/24" 296 | # ipc.append(ip) 297 | # for value,count in Counter(ipc).most_common(): 298 | # print(value, "-----------------",count) 299 | # return Counter(ipc).most_common() 300 | 301 | 302 | # 将url转为C段ip地址表示 303 | def Domain2Csub2(url): 304 | sys.stdout.write("\r[*]Recording to desolve {0} ...".format(url)) 305 | ip=Domain2Ip(url2Domain(url)) 306 | if '' != ip and None !=ip: 307 | x =ip.rfind('.') 308 | ip=ip[:x+1]+"0/24" 309 | return ip 310 | 311 | 312 | 313 | class Color: 314 | HEADER = '\033[95m' 315 | OKBLUE = '\033[90m' 316 | OKGREEN = '\033[92m' 317 | OKYELLOW = '\33[93m' 318 | WARNING = '\033[91m' 319 | FAIL = '\033[91m' 320 | ENDC = '\033[0m' 321 | 322 | 323 | #多线程 324 | class MyThread(threading.Thread): 325 | def __init__(self,q): 326 | threading.Thread.__init__(self) 327 | self.q = q 328 | 329 | def run(self): 330 | while not self.q.empty(): 331 | getTitle2(self.q.get()) 332 | 333 | class FindCsubThread(threading.Thread): 334 | def __init__(self,q): 335 | threading.Thread.__init__(self) 336 | self.q = q 337 | def run(self): 338 | while not self.q.empty(): 339 | IPCsubList.append(Domain2Csub2(self.q.get())) 340 | 341 | 342 | def main(): 343 | print(Color.OKYELLOW+banner+Color.ENDC) 344 | parser = argparse.ArgumentParser(description='攻防资产处理工具，用于简单处理筛选攻防前的有价值资产') 345 | parser.add_argument('-t','--thread',metavar='',type=int,default='10',help='线程参数') 346 | parser.add_argument('-f','--file',metavar='',default='',help='从文件读取') 347 | parser.add_argument('--mvdups',metavar='',default='',help='文本去重') 348 | # parser.add_argument('--mvdups2',metavar='',default='',help='去除file1中含有file2内容的部分，然后将两个文件合并') 349 | parser.add_argument('-c','--c',metavar='',default='',help='C段探测') 350 | args = parser.parse_args() 351 | 352 | target = args.file 353 | thread_nums = args.thread 354 | movdup = args.mvdups 355 | # mvdups2 = args.mvdups2 356 | ip = args.c 357 | print(target) 358 | 359 | if '' != target: 360 | with open(target,'r') as f: 361 | for line in f.readlines(): 362 | lineQueue.put(line.strip()) 363 | for p in port: 364 | #print(line.strip()+":"+str(p)) 365 | 366 | urlQueue.put(line.strip()+":"+str(p)) 367 | print("Queue OK! !") 368 | print("thread nums:",thread_nums,"!") 369 | print("识别C段IP中...") 370 | 371 | for j in range(thread_nums): 372 | CsubThreadList.append(FindCsubThread(lineQueue)) 373 | for k in CsubThreadList: 374 | k.start() 375 | for m in CsubThreadList: 376 | m.join() 377 | if lineQueue.empty(): 378 | for value,count in Counter(IPCsubList).most_common(): 379 | if count >= 5 and None != value: 380 | result = '{}[+]{}{}'.format(Color.OKYELLOW,value.ljust(50),count,Color.ENDC) 381 | print('\r{0}'.format(result)) 382 | #print(value," ",count) 383 | 384 | 385 | for i in range(thread_nums): 386 | threadList.append(MyThread(urlQueue)) 387 | for t in threadList: 388 | t.start() 389 | for l in threadList: 390 | l.join() 391 | 392 | if '' != movdup: 393 | MovDups(movdup) 394 | if '' !=ip: 395 | #ipl = ip.split('.') 396 | for i in range(1,254): 397 | x =ip.rfind('.') 398 | ip=ip[:x+1]+str(i) 399 | for p in port: 400 | urlQueue.put(ip.strip()+":"+str(p)) 401 | print("Queue OK! !") 402 | print("thread nums:",thread_nums,"!") 403 | for i in range(thread_nums): 404 | threadList.append(MyThread(urlQueue)) 405 | for t in threadList: 406 | t.start() 407 | for l in threadList: 408 | l.join() 409 | if __name__ == '__main__': 410 | main() 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | --------------------------------------------------------------------------------