├── README.md ├── lib ├── __init__.py ├── config.py ├── filter.py └── sniffer.py ├── pic ├── 111.png ├── 222.png └── 333.png ├── requirements.txt └── run.py /README.md: -------------------------------------------------------------------------------- 1 | # PassiveDataSorting 0.1 2 | 3 | 自动化被动扫描系统分为数据源、数据处理、漏洞验证等三个子系统，本系统属于数据处理部分，抓取流量镜像的数据，进行分析过滤去重等操作，发送至消息队列中，等待PassiveSecCheck消费 4 | 5 | ## Author ## 6 | 咚咚呛 7 | 8 | 如有其他建议，可联系微信280495355 9 | 10 | ## Support ## 11 | 12 | 满足如下安全需求 13 | 14 | 1、支持大流量镜像(比如核心交换机上对负载均衡nginx或waf流量镜像)数据处理，当流量过大会主动丢弃 15 | 2、支持流量对http解析 16 | 2、支持按照规则进行去重、去静态、去相似等方法，已集成部分规则，另一部分规则需要按照场景和日志进行统计设定 17 | 3、支持按照五元素json存储结果到redis(方法、协议、域名、接口、参数) 18 | 19 | 20 | 技术细节如下： 21 | 22 | 1、基于scapy进行网卡流量数据处理，流量过大会主动丢弃部分数据 23 | 2、基于scapy_http进行流量的http解析 24 | 3、支持http协议的GET/POST方法，针对POST防止，暂只支持application/x-www-form-urlencoded(如body为user=test&password=123456)类型 25 | 4、自定义url过滤规则保存在redis的中，可动态修改，支持如下：匹配任意五元素，进行放行、拦截、替换等操作，如去静态资源、去相似url、去敏感域名等。 26 | 5、去重过滤，计算出MD5(方法+接口)值，利用redis唯一key进行去重 27 | 6、结果redis存储Key是去重md5值，VALUE是json存储的五元素 28 | 7、结果为反复覆盖式存储，可保持请求最新 29 | 30 | 31 | ## Test Environment ## 32 | 33 | >centos 7 34 | > 35 | >python 2.7.5 36 | 37 | ## Tree ## 38 | 39 | PassiveDataSorting 40 | ----lib #模块库文件 41 | ----run.py #程序主程序 42 | 43 | ## Deploy ## 44 | 45 | 1、任意机子安装redis 46 | $ yum install redis 47 | $ vim /etc/redis.conf 48 | # 更改bind 127.0.0.1 改成了 bind 0.0.0.0 49 | # 添加一行requirepass xxxxx密码 50 | # 修改daemonize yes 51 | $ redis-server /etc/redis.conf 52 | 53 | 2、流量镜像数据处理server 54 | 1） $ yum install scapy 55 | 2） $ pip install -r requirements.txt 56 | 3） 配置./lib/config.py 文件，填入Redis和相关规则，后期可以直接在redis进行规则更改添加 57 | 4） $ nohup python run.py & 58 | 59 | 60 | 61 | ## Config ## 62 | 63 | 配置目录：./lib/config.py 64 | 65 | # redis信息 66 | REDIS_HOST = '127.0.0.1' 67 | REDIS_PORT = 6379 68 | REDIS_PASSWORD = 'xxxxxxx' 69 | REDIS_DB = 0 70 | 71 | # field 代表5元素中字段名称，method/protocol/domain/ng_request_url_short/arg 72 | # rule 代表需要匹配的正则 73 | # remarks 代表备注信息 74 | # action代表行为，open匹配上则放过没匹配则拦截、lock匹配上拦截，replace匹配上替换 75 | # replace代表，当action为replace为时，匹配上的替换为replace中的字串 76 | conf_sniffer_rule = [ 77 | {'field': 'method', 'rule': '^(GE|POS)T$', 'remarks': '方法过滤只允许GET/POST', 'action': 'open'}, 78 | {'field': 'protocol', 'rule': '^http://$', 'remarks': '协议过滤只允许http://', 'action': 'open'}, 79 | {'field': 'domain', 'rule': 'www\.test\.com', 'remarks': '禁止www.test.com', 'action': 'lock'}, 80 | {'field': 'ng_request_url_short', 'rule': '(.+)\.(ico|mp3|js|jpg|jped|gif|xml|zip|css|png|txt|ttf|rar|gz)$', 81 | 'remarks': '排除静态文件', 'action': 'lock'}, 82 | {'field': 'ng_request_url_short', 'rule': '(\d+){5,}', 'replace': 'xxxxxx', 'remarks': '接口出现5位以上数字则把相关数字替换为xxxxx', 83 | 'action': 'replace'}, 84 | {'field': 'ng_request_url_short', 'rule': '/$', 'replace': '', 'remarks': '假如接口最后一位为/进行删除', 85 | 'action': 'replace'} 86 | ] 87 | 88 | 89 | ## Screenshot ## 90 | 91 | ![Screenshot](pic/111.png) 92 | 93 | ![Screenshot](pic/222.png) 94 | 95 | ![Screenshot](pic/333.png) 96 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __author__ = 'grayddq' 4 | __version__ = '0.1' 5 | -------------------------------------------------------------------------------- /lib/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # redis信息 3 | REDIS_HOST = '127.0.0.1' 4 | REDIS_PORT = 6379 5 | REDIS_PASSWORD = '11111' 6 | REDIS_DB = 5 7 | 8 | # field 代表5元素中字段名称，method/protocol/domain/ng_request_url_short/arg 9 | # rule 代表需要匹配的正则 10 | # remarks 代表备注信息 11 | # action代表行为，open匹配放过不匹配拦截、lock匹配拦截，replace匹配替换 12 | # replace代表，当action为replace为时，匹配替换 13 | conf_sniffer_rule = [ 14 | {'field': 'method', 'rule': '^(GE|POS)T$', 'remarks': '方法过滤只允许GET/POST', 'action': 'open'}, 15 | {'field': 'protocol', 'rule': '^http://$', 'remarks': '协议过滤只允许http://', 'action': 'open'}, 16 | {'field': 'domain', 'rule': 'www\.test\.com', 'remarks': '禁止www.test.com', 'action': 'lock'}, 17 | {'field': 'ng_request_url_short', 'rule': '(.+)\.(ico|mp3|js|jpg|jped|gif|xml|zip|css|png|txt|ttf|rar|gz)$', 18 | 'remarks': '排除静态文件', 'action': 'lock'}, 19 | {'field': 'ng_request_url_short', 'rule': '(\d+){5,}', 'replace': 'xxxxxx', 'remarks': '接口出现5位以上数字进行替换', 20 | 'action': 'replace'}, 21 | {'field': 'ng_request_url_short', 'rule': '/$', 'replace': '', 'remarks': '假如接口最后一位为/进行删除', 22 | 'action': 'replace'} 23 | ] 24 | -------------------------------------------------------------------------------- /lib/filter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*-\ 2 | import re 3 | 4 | NAME, VERSION, AUTHOR, LICENSE = "PublicDataSorting", "V0.1", "咚咚呛", "Public (FREE)" 5 | 6 | 7 | class Filter(): 8 | def __init__(self, target, rules): 9 | self.target = target 10 | self.protocol = target['protocol'] 11 | self.ng_request_url_short = target['ng_request_url_short'] 12 | self.domain = target['domain'] 13 | self.method = target['method'] 14 | self.arg = target['arg'] 15 | self.rules = rules 16 | 17 | def filter(self): 18 | for rule in self.rules: 19 | pattern = re.compile(rule['rule']) 20 | short_list = [] 21 | if rule['field'] == 'method': 22 | str = self.method 23 | elif rule['field'] == 'protocol': 24 | str = self.protocol 25 | elif rule['field'] == 'domain': 26 | str = self.domain 27 | elif rule['field'] == 'arg': 28 | str = self.arg 29 | else: 30 | short_list = self.ng_request_url_short.split('?') 31 | str = short_list[0] 32 | 33 | if rule['action'] == 'open': 34 | if pattern.search(str): 35 | continue 36 | else: 37 | return True 38 | elif rule['action'] == 'replace': 39 | if pattern.search(str): 40 | result, number = pattern.subn(rule['replace'], str) 41 | if rule['field'] == 'method': 42 | self.method = result 43 | self.target['method'] = result 44 | elif rule['field'] == 'protocol': 45 | self.protocol = result 46 | self.target['protocol'] = result 47 | elif rule['field'] == 'domain': 48 | self.domain = result 49 | self.target['domain'] = result 50 | elif rule['field'] == 'arg': 51 | self.arg = result 52 | self.target['arg'] = result 53 | else: 54 | short_list[0] = result 55 | self.ng_request_url_short = '?'.join(short_list) 56 | self.target['ng_request_url_short'] = '?'.join(short_list) 57 | else: 58 | if pattern.search(str): 59 | return True 60 | return False 61 | -------------------------------------------------------------------------------- /lib/sniffer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import scapy_http.http as HTTP 3 | from scapy.all import * 4 | from filter import * 5 | from config import * 6 | import time, hashlib, redis 7 | 8 | NAME, VERSION, AUTHOR, LICENSE = "PublicDataSorting", "V0.1", "咚咚呛", "Public (FREE)" 9 | 10 | 11 | class Capute(): 12 | def __init__(self): 13 | self.port = 80 14 | self.redis_r = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB) 15 | self.rules = eval(self.redis_r.hget('passive_config', 'conf_sniffer_rule')) \ 16 | if 'conf_sniffer_rule' in self.redis_r.hkeys('passive_config') else [] 17 | 18 | def md5(self, str): 19 | m = hashlib.md5() 20 | m.update(str) 21 | return m.hexdigest() 22 | 23 | def pktTCP(self, pkt): 24 | if HTTP.HTTPRequest in pkt: 25 | request_py = pkt[TCP].payload 26 | request_json = {} 27 | if request_py.Method == "POST": 28 | if 'application/x-www-form-urlencoded' in pkt[HTTP.HTTPRequest].fields['Content-Type'].strip().lower() \ 29 | and int(pkt[HTTP.HTTPRequest].fields['Content-Length'].strip().lower()) > 0: 30 | headers, body = str(request_py).split("\r\n\r\n", 1) 31 | # 不允许参数为空 32 | if len(body) > 0: 33 | host, ng_request_url_short = request_py.Host, request_py.Path 34 | request_json = {'method': 'POST', 35 | 'protocol': 'http://', 36 | 'domain': host, 37 | 'ng_request_url_short': ng_request_url_short, 38 | 'arg': body} 39 | time.sleep(0.01) 40 | elif (request_py.Method == "GET"): 41 | if request_py.Path.find('?') > 0: 42 | query, ng_request_url_short = \ 43 | request_py.Path[request_py.Path.find('?') + 1:], request_py.Path[0:request_py.Path.find('?')] 44 | # 不允许参数为空 45 | if query and ng_request_url_short: 46 | host = request_py.Host 47 | request_json = {'method': 'POST', 48 | 'protocol': 'http://', 49 | 'domain': host, 50 | 'ng_request_url_short': ng_request_url_short, 51 | 'arg': query} 52 | time.sleep(0.01) 53 | else: 54 | pass 55 | if request_json: 56 | if Filter(request_json, self.rules).filter(): 57 | return 58 | MD5 = self.md5(request_json['method'] + request_json['ng_request_url_short'].split('?')[0]) 59 | self.redis_r.set('DataSort_' + MD5, request_json) 60 | 61 | def run(self): 62 | while True: 63 | try: 64 | sniff(filter='tcp and port %d' % self.port, prn=self.pktTCP, store=0) 65 | except BaseException, e: 66 | print e 67 | 68 | 69 | if __name__ == "__main__": 70 | Capute().run() 71 | -------------------------------------------------------------------------------- /pic/111.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grayddq/PassiveDataSorting/0c04fb9755553542f285df6f18f5aec7a83a1b0f/pic/111.png -------------------------------------------------------------------------------- /pic/222.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grayddq/PassiveDataSorting/0c04fb9755553542f285df6f18f5aec7a83a1b0f/pic/222.png -------------------------------------------------------------------------------- /pic/333.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grayddq/PassiveDataSorting/0c04fb9755553542f285df6f18f5aec7a83a1b0f/pic/333.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | redis==2.10.5 2 | scapy==2.3.3 3 | scapy_http==1.8 4 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from lib.sniffer import * 3 | from lib.config import * 4 | 5 | NAME, VERSION, AUTHOR, LICENSE = "PublicLogSorting", "V0.1", "咚咚呛", "Public (FREE)" 6 | 7 | if __name__ == "__main__": 8 | redis_r = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB) 9 | if not 'conf_sniffer_rule' in redis_r.hkeys('passive_config'): 10 | redis_r.hset('passive_config', 'conf_sniffer_rule', conf_sniffer_rule) 11 | Capute().run() 12 | --------------------------------------------------------------------------------