├── .gitignore
├── requirements.txt
├── config
├── test_sub.txt
├── config.py
├── html_template.py
├── remove_repeat.py
└── next_subdomains.txt
├── scripts
├── base.py
├── virustotal.py
├── threatcrowd.py
└── ca.py
├── LICENSE
├── lib
├── parser.py
├── title.py
└── core.py
├── README.md
└── subdomain.py
/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__/
2 | *.pyc
3 | .vscode/
4 | output
5 | # Environments
6 | .env
7 | .venv
8 | env/
9 | venv/
10 | ENV/
11 | env.bak/
12 | venv.bak/
13 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | aiodns==3.0.0
2 | certifi==2019.6.16
3 | cffi==1.12.3
4 | chardet==3.0.4
5 | dnspython==2.1.0
6 | idna==2.8
7 | pycares==4.0.0
8 | pycparser==2.19
9 | requests==2.22.0
10 | urllib3==1.25.8
11 |
--------------------------------------------------------------------------------
/config/test_sub.txt:
--------------------------------------------------------------------------------
1 | bbs
2 | a
3 | b
4 | c
5 | d
6 | test
7 | wjsso
8 | oa
9 | my
10 | eprint
11 | o
12 | j
13 | w
14 | www
15 | sms
16 | test
17 | app
18 | admin
19 | bdp
20 | oil
21 | 60s
22 | gsp
23 | ufs
24 | iotx
25 |
--------------------------------------------------------------------------------
/scripts/base.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-30 18:05:15
6 | @LastEditTime: 2019-05-31 16:20:40
7 | '''
8 | class Base(object):
9 | def __init__(self, scan_domain):
10 | self.scan_domain = scan_domain
11 | self.sub = set()
12 | self.enable = True
13 |
14 | def run(self):
15 | print("base")
16 |
17 |
18 |
--------------------------------------------------------------------------------
/config/config.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-30 16:08:13
6 | @LastEditTime: 2019-07-19 17:40:48
7 | '''
8 |
9 | VERSION = "0.1"
10 |
11 | BANNER = """
12 |
13 | __ __ _____ _ ______ _
14 | \ \ / / / ___| | | | _ \ (_)
15 | \ V /_____\ `--. _ _| |__ | | | |___ _ __ ___ __ _ _ _ __
16 | \ /______|`--. \ | | | '_ \| | | / _ \| '_ ` _ \ / _` | | '_ \
17 | | | /\__/ / |_| | |_) | |/ / (_) | | | | | | (_| | | | | |
18 | \_/ \____/ \__,_|_.__/|___/ \___/|_| |_| |_|\__,_|_|_| |_|
19 | https://github.com/Ciyfly/Y-SubDomain Author: @Recar
20 | """
21 |
22 |
23 | VIRUSTOTAL_APIKEY = ""
24 | WEIBU_APIKEY = ""
25 | # 是对重复的ip达到多少进行删除数据 jd的话差不多超过50个域名指向一个ip就可以删除了
26 | DNS_THRESHOLD = 50
--------------------------------------------------------------------------------
/config/html_template.py:
--------------------------------------------------------------------------------
1 | html_head = """
2 |
3 |
4 |
5 |
6 | """
7 |
8 | html_title ="""{0} 子域名扫描结果"""
9 |
10 | html_body_head = """"""
11 |
12 | html_body_title = """{0} 子域名扫描结果
"""
13 |
14 | html_body_a = """{0}
"""
15 |
16 | html_body_end = """"""
17 |
18 | html_style = """
36 | """
37 |
38 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Recar
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/scripts/virustotal.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-30 18:05:15
6 | @LastEditTime: 2019-06-27 17:10:18
7 | '''
8 | from base import Base
9 | import sys
10 | sys.path.append("..")
11 | from y_subdomain.config.config import VIRUSTOTAL_APIKEY
12 | import requests
13 | import json
14 |
15 | class Scan(Base):
16 | """一分钟只能使用四次"""
17 | def __init__(self, scan_domain):
18 | super().__init__(scan_domain)
19 | self.name = "virustotal"
20 | self.base_url = 'https://www.virustotal.com/vtapi/v2/domain/report'
21 |
22 | def run(self):
23 | try:
24 | if not VIRUSTOTAL_APIKEY:# 如果没有配置 api key 直接返回空
25 | return set()
26 | params = {'apikey': VIRUSTOTAL_APIKEY,'domain': self.scan_domain}
27 | response = requests.get(self.base_url, params=params)
28 | if response.status_code == 200:
29 | data = json.loads(response.content).get("subdomains")
30 | for domain in data:
31 | self.sub.add(domain)
32 | return self.sub
33 | else:
34 | return set()
35 | except Exception as e:
36 | print("ERROR: "+self.name+" : "+str(e))
37 | return set()
38 |
39 |
--------------------------------------------------------------------------------
/config/remove_repeat.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-06-11 18:04:51
6 | @LastEditTime: 2019-06-11 18:18:57
7 | '''
8 | import os
9 | import sys
10 |
11 | def main(add_subdomains_path):
12 | print("start ")
13 | subs = set()
14 | base_path = os.path.dirname(os.path.abspath(__file__))
15 | big_subdomains_path = os.path.join(base_path, "big_subdomains.txt")
16 | tmp_file_path = os.path.join(base_path, "tmp.txt")
17 |
18 | with open(big_subdomains_path, "r") as f:
19 | line = f.readline()
20 | while line:
21 | subs.add(line)
22 | line = f.readline()
23 | old_dic_len = len(subs)
24 | with open(add_subdomains_path, "r") as f:
25 | line = f.readline()
26 | while line:
27 | subs.add(line)
28 | line = f.readline()
29 | print(f"add dict: {len(subs) - old_dic_len}")
30 | print("remove repeat after: {0} ".format(len(subs)))
31 | with open(tmp_file_path, "w") as f:
32 | for sub in subs:
33 | f.write(str(sub))
34 | os.remove(big_subdomains_path)
35 | print(big_subdomains_path)
36 | os.rename(tmp_file_path, big_subdomains_path)
37 | print("end ")
38 |
39 | if __name__ == "__main__":
40 | add_subdomains_path = sys.argv[1]
41 | print(f"merge {add_subdomains_path}")
42 | main(add_subdomains_path)
--------------------------------------------------------------------------------
/scripts/threatcrowd.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-30 18:05:15
6 | @LastEditTime: 2019-06-27 17:09:51
7 | '''
8 | from base import Base
9 | import requests
10 | import json
11 |
12 | class Scan(Base):
13 | def __init__(self, scan_domain):
14 | super().__init__(scan_domain)
15 | self.name = "threatcrowd"
16 | self.base_url = "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={0}"
17 | self.headers = {
18 | 'authority': 'www.threatcrowd.org',
19 | 'accept': 'application/json',
20 | 'content-type': 'application/json',
21 | 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
22 | }
23 | self.enable = False
24 |
25 | def run(self):
26 | try:
27 | get_url = self.base_url.format(self.scan_domain)
28 | response = requests.get(url=get_url, headers = self.headers )
29 | if response.status_code == 200:
30 | data = json.loads(response.content).get('subdomains')
31 | # data = str(data,encoding='utf-8')
32 | for domain in data:
33 | self.sub.add(domain)
34 | return self.sub
35 | else:
36 | return set()
37 | except Exception as e:
38 | print("ERROR: "+self.name+" : "+str(e))
39 | return set()
40 |
--------------------------------------------------------------------------------
/config/next_subdomains.txt:
--------------------------------------------------------------------------------
1 | test
2 | test2
3 | test3
4 | test4
5 | t
6 | dev
7 | 1
8 | 2
9 | 3
10 | s1
11 | s2
12 | s3
13 | admin
14 | adm
15 | a
16 | b
17 | c
18 | m
19 | ht
20 | adminht
21 | webht
22 | web
23 | gm
24 | sys
25 | system
26 | manage
27 | manager
28 | mgr
29 | passport
30 | bata
31 | wei
32 | weixin
33 | wechat
34 | wx
35 | wiki
36 | upload
37 | ftp
38 | pic
39 | jira
40 | zabbix
41 | nagios
42 | bug
43 | bugzilla
44 | sql
45 | mysql
46 | db
47 | stmp
48 | pop
49 | imap
50 | mail
51 | zimbra
52 | exchange
53 | forum
54 | bbs
55 | list
56 | count
57 | counter
58 | img
59 | img01
60 | img02
61 | img03
62 | img04
63 | api
64 | cache
65 | js
66 | css
67 | app
68 | apps
69 | wap
70 | sms
71 | zip
72 | monitor
73 | proxy
74 | update
75 | upgrade
76 | stat
77 | stats
78 | data
79 | portal
80 | blog
81 | autodiscover
82 | en
83 | search
84 | so
85 | oa
86 | database
87 | home
88 | sso
89 | help
90 | vip
91 | s
92 | w
93 | down
94 | download
95 | downloads
96 | dl
97 | svn
98 | git
99 | log
100 | staff
101 | vpn
102 | sslvpn
103 | ssh
104 | scanner
105 | sandbox
106 | ldap
107 | lab
108 | go
109 | demo
110 | console
111 | cms
112 | auth
113 | crm
114 | erp
115 | res
116 | static
117 | old
118 | new
119 | beta
120 | image
121 | service
122 | login
123 | 3g
124 | docs
125 | it
126 | e
127 | live
128 | library
129 | files
130 | i
131 | d
132 | cp
133 | connect
134 | gateway
135 | lib
136 | preview
137 | backup
138 | share
139 | status
140 | assets
141 | user
142 | vote
143 | bugs
144 | cas
145 | feedback
146 | id
147 | edm
148 | survey
149 | union
150 | ceshi
151 | dev1
152 | updates
153 | phpmyadmin
154 | pma
155 | edit
156 | master
157 | xml
158 | control
159 | profile
160 | zhidao
161 | tool
162 | toolbox
163 | boss
164 | activity
165 | www
166 |
--------------------------------------------------------------------------------
/lib/parser.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-30 16:07:49
6 | LastEditTime: 2021-05-20 14:56:30
7 | '''
8 | from optparse import OptionParser
9 | from config.config import VERSION, BANNER
10 | USAGE = "python domain -d xxx.com"
11 | import sys
12 | def get_options():
13 | print(BANNER)
14 | parser = OptionParser(usage=USAGE,version=VERSION)
15 |
16 | parser.add_option('-d', type=str, dest="domain", help="指定要测试的域名")
17 |
18 | parser.add_option('-e', type=str, dest="engine", help="指定使用的引擎 逗号间隔")
19 |
20 | parser.add_option('-c', type=str, dest="sub_dict", help="指定使用的字典 不指定默认使用默认的")
21 |
22 | parser.add_option('-f', type=str, dest="domain_file", help="指定域名列表文件 默认使用小字典2w")
23 |
24 | parser.add_option('-t', type=int, dest="thread_count", help="指定线程数 默认500")
25 |
26 | parser.add_option('--private',action='store_true', dest="is_private", default=False, help="是否对内网ip进行清除")
27 |
28 | parser.add_option('--title',action='store_true', dest="get_title", default=False, help="是否直接获取域名的title信息")
29 |
30 | parser.add_option('--gen',action='store_true', dest="gen_rule", default=False, help="是否加上动态规则生成的字典")
31 |
32 | parser.add_option('--exh',action='store_true', dest="exhaustion", default=False, help="是否进行暴力穷举")
33 |
34 | parser.add_option('--exo',action='store_true', dest="exhaustion_only", default=False, help="只进行暴力穷举")
35 |
36 | parser.add_option('--next',action='store_true', dest="next_sub", default=False, help="默认穷举到三级域名 开启则穷举到四级域名")
37 |
38 | parser.add_option('--big_dict',action='store_true', dest="big_dict", default=False, help="默认使用小字典2w 开启则使用大字典200w")
39 |
40 | parser.add_option('--json',action='store_true', dest="is_json", default=False, help="是否生成json报告")
41 |
42 | parser.add_option('--html',action='store_true', dest="is_html", default=False, help="是否生成html报告")
43 | (options,args) = parser.parse_args()
44 | if options.domain==None and options.domain_file==None:
45 | parser.print_help()
46 | sys.exit(0)
47 | if options.domain:
48 | if "www" in options.domain or "http://" in options.domain :
49 | options.domain =options.domain.replace("www.","").replace("http://", "")
50 | if options.engine:
51 | options.engine = options.engine.split(",")
52 |
53 | return options,args
54 |
--------------------------------------------------------------------------------
/scripts/ca.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import os
3 | import asyncio
4 | import aiodns
5 | import socket
6 | import ssl
7 | import traceback
8 | import sys
9 | sys.path.append("../")
10 | from base import Base
11 |
12 | class Scan(Base):
13 | def __init__(self, scan_domain):
14 | super().__init__(scan_domain)
15 | self.name = "ca"
16 |
17 | @staticmethod
18 | def query_a(domain):
19 | """
20 | 查询根域名A记录
21 | :param domain: 被查询的域名(某些情况下根域名可能没有A记录，可尝试www的A记录)
22 | :return:
23 | """
24 | loop = asyncio.new_event_loop()
25 | asyncio.set_event_loop(loop)
26 | resolver = aiodns.DNSResolver(loop=loop)
27 | f = resolver.query(domain, 'A')
28 | result = loop.run_until_complete(f)
29 | # 可以多个IP都尝试下，但正常情况下没有区别
30 | return result[0].host
31 |
32 | @staticmethod
33 | def get_cert_domains(ip):
34 | """
35 | 向IP的443端口查询支持的域名情况
36 | :param ip:
37 | :return:
38 | """
39 | s = socket.socket()
40 | s.settimeout(2)
41 | cert_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'cacert.pem')
42 | connect = ssl.wrap_socket(s, cert_reqs=ssl.CERT_REQUIRED, ca_certs=cert_path)
43 | connect.settimeout(2)
44 | connect.connect((ip, 443))
45 | cert_data = connect.getpeercert().get('subjectAltName')
46 | return cert_data
47 |
48 | def run(self, only_subdomains=False):
49 | """
50 | 根据HTTPS证书获取支持的域名
51 | :param only_subdomains: 是否只需要子域名，比如搜索baidu.com只需要*.baidu.com，不需要baidu.cn
52 | :return:
53 | """
54 | domains = []
55 | try:
56 | ip = self.query_a(self.scan_domain)
57 | cert_domains = self.get_cert_domains(ip)
58 | for cert_domain in cert_domains:
59 | domain = cert_domain[1]
60 | if not domain.startswith('*'):
61 | if only_subdomains:
62 | if domain.endswith(self.scan_domain):
63 | domains.append(domain)
64 | else:
65 | domains.append(domain)
66 | return set(domains)
67 | except Exception as e:
68 | return set(domains)
69 |
70 |
71 | # if __name__ == '__main__':
72 | # try:
73 | # ret_domains = Scan('baidu.com').run(only_subdomains=False)
74 | # print(ret_domains)
75 | # except Exception as e:
76 | # traceback.print_exc()
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # y_subdomain
2 |
3 | 子域名获取工具
4 |
5 |
6 | ## 使用
7 |
8 | 使用加速下载
9 | `git clone https://github.com.cnpmjs.org/Ciyfly/y_subdomain.git`
10 |
11 | **如果是下载的zip 或者tar包需要修改文件夹名为 y_subdomain**
12 |
13 | **基于python3 linux环境下**
14 |
15 | 适用于 扫描器前置信息收集 简单子域名获取 学习等
16 | 基本使用内置原生库
17 |
18 | [](https://asciinema.org/a/m7mqlsHux1TinM2oWB6D6LZoD)
19 |
20 |
21 | ### 泛解析的解决办法
22 | 先测试一个不存在的域名然后是否成功解析
23 |
24 |
25 | ### 为什么要做
26 | 1. 重复造轮子 深入理解轮子 做出更好用的轮子
27 | 2. 是大扫描器的信息收集的一部分功能的实现
28 | 3. 工作中擅于使用轮子 学习中擅于造轮子
29 |
30 |
31 | ## 实现
32 | ### 接口的实现
33 | 这里参考 poc的形式 动态的从script文件夹下的脚本
34 | 动态载入实例化并执
35 | 继承Base类默认有个 `self.enable=True` 可以控制是否开启脚本
36 | 如果使用 -e 指定接口 enable是False也会强制执行
37 | 对于接口返回的域名会使用线程池再去dns解析验证 (默认线程池大小 50)
38 |
39 | **接口引擎脚本完成：**
40 | 1. 百度云检测
41 | 2. hackertarget
42 | 3. virustotal (**需要在config/config.py 中添加api key才能使用不然默认不会执行**)
43 | 4. 通过证书获取
44 |
45 | ### 暴力穷举的实现
46 | 一口气加载字典到内存中 占内存不大
47 | 使用共享队列+多线程的形式进行解析dns 默认线程开启100个
48 | 这个地方有进度条功能 可以在实例化穷举类的时候进行选择是否开启
49 |
50 |
51 | ## 版本
52 |
53 | **v1.2**
54 | 2021 0520
55 | 去掉大字典 影响下载速度
56 | 保存的结果拆分为 ip ipc段 域名 域名+ip
57 | 增加获取title功能 `--title`
58 | 增加动态生成字典加入测试字典功能 `--gen`
59 |
60 | **v1.1.0**
61 | 修改一些bug
62 | 增加三级四级域名穷举
63 | 修改为默认小字典2w (使用的onforall的字典)
64 | 指定`big_dict` 可以使用大字典200w
65 | 增加可以指定字典参数
66 | 穷举增加超时时间 默认为2h
67 | dns增加超时时间10s
68 |
69 | **v1.0.0**
70 | 修改bug
71 | 进度条增加find输出
72 | 增加对内网ip的剔除参数
73 | 增加指定字典的参数
74 |
75 | **V0.2**
76 | 修改 穷举采用共享队列+多线程的形式
77 | 去除多余代码
78 | 修改了代码结构 便于作为api使用
79 | 穷举增加了进度条
80 | 扩大了字典 目前字典 2350706 (235w) (跑一次大概 40分钟左右)
81 | 增加了HUP小调试 发起HUP信号会输出 此时队列大小 (在 api形式使用不输出进度条的时候 如果长时间未结束可调试使用)
82 | `kill -HUP 进程id`
83 |
84 | **V0.1**
85 | 采用多接口的形式获取域名 动态插件形式添加接口脚本
86 | 接口验证和穷举都采用线程池的形式进行dns解析
87 |
88 |
89 |
90 | # api形式使用
91 | 使用接口解析
92 | ```python
93 | from y_subdomain.lib.core import EngineScan
94 |
95 | engine_scan = EngineScan(scan_domain, engine)
96 | # scan_domain 为扫描的域名 engine 是指定的接口 空的话就全部都跑 默认也是全部都跑
97 | engine_domain_ips_dict = engine_scan.run()
98 | # 调用 run方法会返回 解析的结果 是 字典形式 子域名对应ip列表
99 |
100 | 接口类如下:
101 | class EngineScan(object):
102 | """接口解析类
103 | :param scan_domain 测试域名
104 | :param engine 指定引擎列表 默认全部
105 | :param thread_count 线程 默认100
106 | :param get_black_ip 是否返回泛解析的ip 默认不返回
107 | :param is_private 是否保留内网ip 默认不保留
108 | :return domain_ips_dict, (black_ip) 选择返回泛解析ip则有第二个 否则只返回域名对于ip
109 |
110 | """
111 | ```
112 | 使用穷举解析
113 | ```python
114 | from y_subdomain.lib.core import ExhaustionScan
115 | scan_domain = "测试域名" # 穷举只支持单个域名
116 | exhaustion_scan = ExhaustionScan(scan_domain, thread_count=100, is_output=True)
117 | # is_output 是否输出进度条 默认是False的
118 | exh_domain_ips_dict = exhaustion_scan.run()
119 |
120 | 穷举类如下:
121 | class ExhaustionScan(object):
122 | """暴力穷举
123 | :param scan_domain 要测试的域名
124 | :param thread_count 线程数 默认100
125 | :param is_output 是否输出进度条 默认不输出
126 | :param black_ip 泛解析的ip 默认为空
127 | :param is_private 是否保留内网ip 默认不保留
128 | :param sub_dict 指定的字典 默认读取配置文件下字典
129 | :param next_sub 是否是三级或者四级的域名扫描
130 | :param timeout 超时时间 默认穷举超时时间为 2h
131 | :return domain_ips_dict 域名对应解析的ip结果
132 |
133 | """
134 | ```
135 |
136 | ## update
137 | 2019 0813
138 | 修改一些bug
139 | 增加三级四级域名穷举
140 | 修改为默认小字典2w (使用的onforall的字典)
141 | 指定`big_dict` 可以使用大字典200w
142 | 增加可以指定字典参数
143 | 穷举增加超时时间 默认为2h
144 | dns增加超时时间10s
145 |
146 | ---
147 |
148 | 2019 0719
149 | 修改bug
150 | 进度条增加find输出
151 | 增加对内网ip的剔除参数
152 | 增加指定字典的参数
153 |
154 | ---
155 | 2019 0711
156 | 增加对内网ip的忽略 默认是清除掉的 可以通过命令行参数进行控制
157 | 增加指定字典的功能参数
158 | 增加只进行穷举的功能参数
159 | 没有把去除内网ip的方法写到savedata中是为了当接口api使用的时候 对内网ip的处理类内部就进行处理
160 |
161 | ---
162 | 2019 0709
163 | 经测试 修改对于泛解析的解决
164 | 如果随机字符串跑出来泛解析还继续进行穷举但是过滤掉这个泛解析的ip
165 | 对于接口里的也进行泛解析剔除
166 | 对于暴力穷举的会默认使用配置文件中的 50 阈值 作为判断泛解析的条件 当超过50个域名指向一个ip进行剔除
167 |
168 |
169 | # TODO
170 | 多增加几个模块
171 | 优化下代码结构
172 |
173 | ## 参考
174 |
175 | [ESD](https://github.com/FeeiCN/ESD)
176 |
177 | [wydomain](https://github.com/ring04h/wydomain)
178 |
--------------------------------------------------------------------------------
/lib/title.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | Author: Recar
5 | Date: 2020-09-02 10:31:45
6 | LastEditTime: 2021-03-25 18:25:54
7 | '''
8 | from optparse import OptionParser
9 | import traceback
10 | import threading
11 | import urllib3
12 | import logging
13 | import signal
14 | try:
15 | import queue
16 | except:
17 | import Queue as queue
18 | import time
19 | import sys
20 | import re
21 | import os
22 |
23 |
24 | logging.getLogger("urllib3").setLevel(logging.ERROR)
25 | urllib3.disable_warnings()
26 | header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36'}
27 |
28 | result = []
29 |
30 | def ctrl_c(signum, frame):
31 | print()
32 | print("ctrl c")
33 | sys.exit()
34 | # ctrl+c
35 | signal.signal(signal.SIGINT, ctrl_c)
36 |
37 | class BaseWork(object):
38 | def __init__(self, consumer_count=50):
39 | self.consumer_count = consumer_count
40 | self.work_queue = queue.Queue()
41 |
42 | def put(self, item):
43 | '''
44 | @params item 数据
45 | '''
46 | try:
47 | if type(item) == list:
48 | for d in item:
49 | self.work_queue.put(d)
50 | else:
51 | self.work_queue.put(item)
52 | except Exception as e:
53 | pass
54 |
55 | def producer(self, func):
56 | pass
57 |
58 | def consumer(self, func):
59 | '''
60 | 消费者
61 | @params func 消费者函数
62 | '''
63 | while not self.work_queue.empty():
64 | item = self.work_queue.get(timeout=3)
65 | if item is None:
66 | break
67 | func(item)
68 |
69 | def run(self, consumer_func):
70 | '''
71 | 运行方法
72 | @params consumer_func 消费者函数
73 | '''
74 | start_time = time.time()
75 | threads = []
76 | for i in range(self.consumer_count):
77 | t = threading.Thread(target=self.consumer,args=(consumer_func,))
78 | t.setDaemon(True)
79 | t.start()
80 | threads.append(t)
81 | while not self.work_queue.empty():
82 | # logging.debug("queue size: {0}".format(self.work_queue.qsize()))
83 | time.sleep(1)
84 | alive = True
85 | while alive:
86 | alive = False
87 | for thread in threads:
88 | if thread.isAlive():
89 | alive = True
90 | time.sleep(0.1)
91 | use_time = time.time() - start_time
92 |
93 | class Worker(BaseWork):
94 | '''普通消费队列'''
95 | def __init__(self, consumer_count=50):
96 | super(Worker, self).__init__(consumer_count)
97 |
98 | class WorkerPrior(BaseWork):
99 | '''优先消费队列'''
100 | def __init__(self, consumer_count=50):
101 | super(WorkerPrior, self).__init__(consumer_count)
102 | from queue import PriorityQueue
103 | self.work_queue = PriorityQueue()
104 |
105 | def put(self, item, priority=1):
106 | '''
107 | @params item 数据
108 | @params priority 优先级 默认是1
109 | '''
110 | try:
111 | if type(item) == list:
112 | for d in item:
113 | self.work_queue.put((priority, d))
114 | else:
115 | self.work_queue.put((priority, item))
116 | except Exception as e:
117 | pass
118 |
119 | def consumer(self, func):
120 | '''
121 | 消费者
122 | @params func 消费者函数
123 | '''
124 | while not self.work_queue.empty():
125 | item = self.work_queue.get(timeout=3)
126 | priority, data = item
127 | if data is None:
128 | break
129 | func(data)
130 |
131 |
132 | class GetTitle(object):
133 | def __init__(self, urls, logger=logging):
134 | self.urls = urls
135 | self.logger = logger
136 | self.result = list()
137 |
138 | def get_title(self, url):
139 | try:
140 | http = urllib3.PoolManager()
141 | response = http.request('GET',url,headers=header,timeout=1)
142 | re_title = re.findall("(.+)", response.data.decode())
143 | title = re_title[0] if re_title else ""
144 | info = {
145 | "status_code": response.status,
146 | "title": title,
147 | }
148 | return info
149 | except Exception as e:
150 | return None
151 |
152 | def create_worker(self):
153 | worker = Worker(consumer_count=100)
154 | for url in self.urls:
155 | worker.put({
156 | "url": url
157 | })
158 | return worker
159 |
160 | def consumer(self, data):
161 | url = data["url"]
162 | info = self.get_title(url)
163 | if not info:
164 | return
165 | output = "{:30}\t{:5}\t{:30}\n".format(url, info["status_code"], info["title"])
166 | self.logger.info("[+] {0}".format(output))
167 | self.result.append(output)
168 |
169 | def run(self):
170 | worker = self.create_worker()
171 | self.logger.info("[*] task create success {0}".format(worker.work_queue.qsize()))
172 | output = "{:30}\t{:5}\t{:30}".format("url","status", "title")
173 | self.logger.info("[*] {0}".format(output))
174 | worker.run(self.consumer)
175 | self.logger.info("[+] find title: {0}".format(len(self.result)))
176 | return self.result
177 |
178 |
--------------------------------------------------------------------------------
/subdomain.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-15 18:40:51
6 | LastEditTime: 2021-05-20 17:38:54
7 | '''
8 |
9 | from lib.parser import get_options
10 | from lib.core import (
11 | EngineScan, ExhaustionScan, SaveDate,
12 | print_log, print_info, print_progress,
13 | GenSubdomain
14 | )
15 | from lib.title import GetTitle
16 | import time
17 | import sys
18 | import os
19 |
20 | def main():
21 | # 获取命令行参数
22 | options,args = get_options()
23 | scan_domain = options.domain
24 | domain_file = options.domain_file
25 | is_html = options.is_html
26 | is_json = options.is_json
27 | is_private = options.is_private
28 | engine = options.engine
29 | thread_count = options.thread_count
30 | sub_dict = options.sub_dict
31 | big_dict = options.big_dict
32 | exhaustion = options.exhaustion
33 | exhaustion_only = options.exhaustion_only
34 | get_title = options.get_title
35 | gen_rule = options.gen_rule
36 | domains = list()
37 | if not thread_count:
38 | thread_count = 500
39 | if domain_file:
40 | with open(domain_file, "r") as f:
41 | for domain in f:
42 | domains.append(domain.strip())
43 | else:
44 | domains.append(scan_domain)
45 | for scan_domain in domains:
46 | print_info("scan {0}\n".format(scan_domain))
47 |
48 | engine_domain_ips_dict =None # 初始化
49 | if not exhaustion_only: # 是否只进行穷举解析
50 | # # 接口解析
51 | start = time.perf_counter()
52 | engine_scan = EngineScan(scan_domain, engine, is_private=is_private)
53 | engine_domain_ips_dict = engine_scan.run()
54 | print_info(len(engine_domain_ips_dict))
55 | engine_end = (time.perf_counter() - start)
56 | print_info(f"引擎接口消耗时间:{engine_end}s")
57 |
58 | exh_domain_ips_dict = None
59 | all_exh_domain_ips_dict = dict()
60 | if exhaustion or exhaustion_only:
61 | # 穷举解析
62 | start = time.perf_counter()
63 | exhaustion_scan = ExhaustionScan(
64 | scan_domain, thread_count=thread_count,
65 | is_output=True, is_private=is_private,
66 | sub_dict = sub_dict, big_dict=big_dict,
67 | gen_rule=gen_rule
68 | )
69 | exh_domain_ips_dict = exhaustion_scan.run()
70 | if exh_domain_ips_dict:
71 | print_info(f"穷举发现 : {len(exh_domain_ips_dict)}")
72 | all_exh_domain_ips_dict.update(exh_domain_ips_dict)
73 | exh_end = (time.perf_counter() - start)
74 | print_info(f"穷举消耗时间:{exh_end}s")
75 | print_info(f"开始三级域名穷举")
76 | engine_exh_domain = set()
77 | # 先去重
78 | if engine_domain_ips_dict:
79 | for domain in engine_domain_ips_dict.keys():
80 | engine_exh_domain.add(domain)
81 | for domain in list(exh_domain_ips_dict.keys()):
82 | engine_exh_domain.add(domain)
83 | # 三级域名穷举
84 | three_exh_domain_ips_dict = dict()
85 | # all size 用于输出进度条
86 | all_size = len(engine_exh_domain)
87 | start = time.perf_counter()
88 | for i, domain in enumerate(engine_exh_domain):
89 | print_info(f"开始测试 {domain}")
90 | next_exhaustion_scan = ExhaustionScan(
91 | domain, thread_count=100,
92 | is_output=False, is_private=is_private,
93 | next_sub=True
94 | )
95 | # run & update result echo progress
96 | next_exh_domain_ips_dict = next_exhaustion_scan.run()
97 | three_exh_domain_ips_dict.update(next_exh_domain_ips_dict)
98 | all_exh_domain_ips_dict.update(next_exh_domain_ips_dict)
99 | # 输出进度条
100 | print_progress(all_size-i, all_size, start, len(three_exh_domain_ips_dict))
101 | print()
102 | print_info(f"三级域名穷举发现 : {len(three_exh_domain_ips_dict)}")
103 | # 如果穷举的三级域名有结果则进行四级域名穷举
104 | four_exh_domain_ips_dict = dict()
105 | all_size = len(three_exh_domain_ips_dict)
106 | start = time.perf_counter()
107 | if three_exh_domain_ips_dict:
108 | print_info("开始四级域名穷举")
109 | for i, domain in enumerate(three_exh_domain_ips_dict.keys()):
110 | next_exhaustion_scan = ExhaustionScan(
111 | domain, thread_count=100,
112 | is_output=False, is_private=is_private,
113 | next_sub=True
114 | )
115 | next_exh_domain_ips_dict = next_exhaustion_scan.run()
116 | four_exh_domain_ips_dict.update(next_exh_domain_ips_dict)
117 | all_exh_domain_ips_dict.update(next_exh_domain_ips_dict)
118 | # 输出进度条
119 | print_progress(all_size-i, all_size, start, len(four_exh_domain_ips_dict))
120 | print()
121 | print_info(f"四级域名穷举发现: {len(four_exh_domain_ips_dict)}")
122 | print_info(f"所有穷举发现: {len(all_exh_domain_ips_dict)}")
123 | print_info("start save result")
124 | # 保存结果
125 | save_data = SaveDate(
126 | scan_domain,
127 | engine_domain_ips_dict= engine_domain_ips_dict,
128 | exh_domain_ips_dict=all_exh_domain_ips_dict,
129 | is_text=True,
130 | is_json=is_json,
131 | is_html=is_html
132 | )
133 | domain_ips_dict = save_data.save_doamin_ips()
134 | # 增加title
135 | if not get_title:
136 | return
137 | urls = domain_ips_dict.keys()
138 | print_info("get title")
139 | title_result = GetTitle(urls).run()
140 | print_info(f"title get count: {len(title_result)}")
141 | title_path = os.path.join(save_data.out_out_dir, "title.txt")
142 | with open(title_path, "w") as f:
143 | for title in title_result:
144 | f.write(title)
145 | print_info(f"title save: {title_path}")
146 |
147 |
148 | if __name__ == "__main__":
149 | main()
150 |
--------------------------------------------------------------------------------
/lib/core.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/import pdb;pdb.set_trace()
2 | # coding=UTF-8
3 | '''
4 | @Author: recar
5 | @Date: 2019-05-30 17:49:08
6 | LastEditTime: 2021-05-20 17:19:42
7 | '''
8 |
9 | import sys
10 | sys.path.append("..")
11 | from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
12 | from collections import defaultdict
13 | from y_subdomain.config.html_template import (
14 | html_head, html_title, html_body_head, html_body_title,
15 | html_body_a, html_body_end, html_style
16 | )
17 |
18 | from y_subdomain.config.config import DNS_THRESHOLD
19 | import dns
20 | from dns import resolver
21 | import os
22 | import importlib
23 | import ipaddress
24 | import sys
25 | import re
26 | import json
27 | import random
28 | import string
29 | import queue
30 | import threading
31 | import time
32 | import signal
33 |
34 |
35 | def ctrl_c(signum,frame):
36 | print()
37 | print("[-] input ctrl c")
38 | sys.exit()
39 |
40 | # def print_queue_size(signum, frame):
41 | # print()
42 | # print(f"[*] queue size: {frame.f_locals['self'].sub_dict_queue.qsize()}")
43 | # print(f"[*] find subdomain: {len(frame.f_locals['self'].domain_ips_dict)}")
44 |
45 | # ctrl+c
46 | signal.signal(signal.SIGINT, ctrl_c)
47 | # HUP
48 | # signal.signal(signal.SIGHUP, print_queue_size)
49 |
50 | def print_log(message):
51 | # ljust(50) 实现长度不够存在显示残留 左对齐以空格达到指定长度
52 | print ("\r[*] {0}".format(message).ljust(50), end="")
53 |
54 | def print_flush():
55 | print ("\r\r", end="")
56 |
57 | def print_info(message):
58 | print(("[+] {0}".format(message)))
59 |
60 | def print_debug(message):
61 | print("[*] {0}".format(message))
62 |
63 | def print_error(message):
64 | print(("\n[-] {0}".format(message)))
65 |
66 | def print_progress(currne_size, all_size, start, find_size):
67 | """输出进度条
68 | :param currne_size 当前队列大小
69 | :param all_size 总体队列大小
70 | :param start 开启时间
71 | :param find_size 找到多少条
72 | """
73 | out_u = int(currne_size/all_size*50) # ##
74 | out_l = 50 - out_u
75 | percentage = 100-(currne_size/all_size*100)
76 | use_time = time.perf_counter() - start
77 | print(
78 | '\r'+'[' + '>' * out_l + '-' * out_u +']'
79 | + f'{percentage:.2f}%'
80 | + f'|size: {currne_size}'
81 | + f'|use time: {use_time:.2f}s'
82 | + f'|find: {find_size} ', end="")
83 |
84 | class SaveDate(object):
85 | """用于保存域名结果"""
86 | def __init__(
87 | self, scan_domain, engine_domain_ips_dict=None, exh_domain_ips_dict=None,
88 | is_text=False, is_json=False, is_html=False
89 | ):
90 | self.engine_domain_ips_dict = engine_domain_ips_dict
91 | self.exh_domain_ips_dict = exh_domain_ips_dict
92 | self.domain_ips_dict = dict()
93 | self.clean_data()
94 | self.scan_domain = scan_domain
95 | self.is_text = is_text
96 | self.is_json = is_json
97 | self.is_html = is_html
98 |
99 | self.get_output()
100 |
101 | def clean_data(self):
102 | if self.engine_domain_ips_dict and not self.exh_domain_ips_dict:
103 | # 只有 engine_domain_ips_dict
104 | self.domain_ips_dict = self.engine_domain_ips_dict
105 | elif self.exh_domain_ips_dict and not self.engine_domain_ips_dict:
106 | # 只有 exh_domain_ips_dict
107 | self.domain_ips_dict = self.exh_domain_ips_dict
108 | elif self.engine_domain_ips_dict and self.exh_domain_ips_dict:
109 | # 都有
110 | for domain, ips in self.engine_domain_ips_dict.items():
111 | if domain in self.exh_domain_ips_dict.keys():
112 | self.exh_domain_ips_dict[domain] = self.exh_domain_ips_dict[domain] + ips
113 | else:
114 | self.exh_domain_ips_dict[domain] = ips
115 | self.domain_ips_dict = self.exh_domain_ips_dict
116 |
117 | def get_output(self):
118 | base_path = os.path.dirname(os.path.abspath(__file__))
119 | output_dir = os.path.join(base_path, "../", "output", self.scan_domain)
120 | if not os.path.isdir(output_dir):
121 | os.makedirs(output_dir)
122 | self.out_out_dir = output_dir
123 | self.output_txt = os.path.join(output_dir, "subdomain_ips.txt")
124 | self.output_html = os.path.join(output_dir, "subdomain_ips.html")
125 | self.output_json = os.path.join(output_dir, "subdomain_ips.json")
126 |
127 | def save_text(self):
128 | with open(self.output_txt, "w") as f:
129 | for domain, ips in self.domain_ips_dict.items():
130 | f.write(f"{domain} {ips}\n")
131 | print_info("save txt success\n")
132 |
133 | def save_json(self):
134 | with open(self.output_json, "w") as f:
135 | json.dump(self.domain_ips_dict, f, indent=3)
136 | print_info("save json success\n")
137 |
138 | def save_html(self):
139 | html = html_head
140 | html += html_title.format(self.scan_domain)
141 | html += html_body_head
142 | html += html_body_title.format(self.scan_domain)
143 | for domain in self.domain_ips_dict.keys():
144 | html += html_body_a.format(domain)
145 | html += html_body_end
146 | html += html_style
147 | with open(self.output_html, "w") as f:
148 | f.write(html)
149 | print_info("save html success\n")
150 |
151 | def save_other(self):
152 | subdomain_txt = os.path.join(self.out_out_dir, "subdomain.txt")
153 | ip_txt = os.path.join(self.out_out_dir, "ip.txt")
154 | ipc_txt = os.path.join(self.out_out_dir, "ipc.txt")
155 | domains = self.domain_ips_dict.keys()
156 | ips_list = self.domain_ips_dict.values()
157 | ipcs = list()
158 | for ips in ips_list:
159 | for ip in ips:
160 | ip_net = ".".join([str(i) for i in ip.split(".")[0:-1]+[0]])
161 | ipcs.append(ip_net+"/24")
162 | ipcs = list(set(ipcs))
163 | with open(subdomain_txt, "w") as f:
164 | for domain in domains:
165 | f.write(domain+"\n")
166 | with open(ip_txt, "w") as f:
167 | for ips in ips_list:
168 | for ip in ips:
169 | f.write(ip+"\n")
170 | with open(ipc_txt, "w") as f:
171 | for ipc in ipcs:
172 | f.write(ipc+"\n")
173 |
174 | def save_doamin_ips(self):
175 | # 应朋友需求这里将结果都分开
176 | if not self.domain_ips_dict: # 空的话就不保存文件
177 | return
178 | print_info(f"output_dir {self.out_out_dir}")
179 | if self.is_text:
180 | self.save_text()
181 | if self.is_json:
182 | self.save_json()
183 | if self.is_html:
184 | self.save_html()
185 | self.save_other()
186 | return self.domain_ips_dict
187 |
188 | class EngineScan(object):
189 | """接口解析类
190 | :param scan_domain 测试域名
191 | :param engine 指定引擎列表 默认全部
192 | :param thread_count 线程 默认100
193 | :param get_black_ip 是否返回泛解析的ip 默认不返回
194 | :param is_private 是否保留内网ip 默认不保留
195 | :return domain_ips_dict, (black_ip) 选择返回泛解析ip则有第二个 否则只返回域名对于ip
196 |
197 | """
198 | def __init__(self, scan_domain, engine=None, thread_count=100, get_black_ip=False, is_private=False):
199 | self.scan_domain = scan_domain
200 | self.engine = engine
201 | self.thread_count = thread_count
202 | # dns
203 | #self.resolver = resolver
204 | self.resolver = dns.resolver.Resolver()
205 | # 设置dns超时时间
206 | self.resolver.timeout = 2
207 | self.resolver.lifetime = 2
208 | #self.resolver.nameservers=['8.8.8.8', '114.114.114.114']
209 | # 存储变量
210 | self.domains_set = set()
211 | self.domain_ips_dict = defaultdict(list)
212 | self.get_black_ip = get_black_ip
213 | # 去除泛解析
214 | self.black_ip = list()
215 | # {ip:{ domains: [域名], count: 计数} }
216 | self.ip_domain_count_dict = dict()
217 | # 去除内网ip
218 | self.is_private = is_private
219 | def add_domain(self, domains):
220 | # 去除掉其他后缀不是 .scan_domain的域名
221 | for domain in domains:
222 | if domain.endswith(f".{self.scan_domain}"):
223 | self.domains_set.add(domain)
224 |
225 | def run_scripts(self):
226 | base_path = os.path.dirname(os.path.abspath(__file__))
227 | scripts_path = os.path.join(base_path, "../","scripts")
228 | # 添加到搜索路径
229 | sys.path.append(scripts_path)
230 | scrips_list = list()
231 | scripts_class = list()
232 | if not self.engine: # 没有指定引擎 遍历scrips文件夹
233 | for root, dirs, files in os.walk(scripts_path):
234 | for filename in files:
235 | name = os.path.splitext(filename)[0]
236 | suffix = os.path.splitext(filename)[1]
237 | if suffix == '.py' and name!="base":
238 | metaclass=importlib.import_module(os.path.splitext(filename)[0])
239 | # 通过脚本的 enable属性判断脚本是否执行
240 | if metaclass.Scan(self.scan_domain).enable:
241 | print_info("run script: "+metaclass.Scan(self.scan_domain).name)
242 | result = metaclass.Scan(self.scan_domain).run()
243 | self.add_domain(result)
244 | print_info(f"add: {len(result)} all count: {len(self.domains_set)}")
245 | else: # 指定了引擎
246 | for name in self.engine: # 这里不判断是否开启引擎 直接使用
247 | metaclass=importlib.import_module(name)
248 | print_info("run script: "+metaclass.Scan(self.scan_domain).name)
249 | result = metaclass.Scan(self.scan_domain).run()
250 | self.domains_set = self.domains_set | result
251 | print_info(f"add {len(result)} all count: {len(self.domains_set)}")
252 |
253 | def threadpool_dns(self):
254 | pool = ThreadPoolExecutor(self.thread_count) # 定义线程池
255 | all_task = list()
256 | for domain in self.domains_set:
257 | all_task.append(pool.submit(self.analysis_dns, domain))
258 | for task in all_task:
259 | task.result()
260 |
261 | def analysis_dns(self, domain):
262 | try:
263 | ans = self.resolver.resolve(domain, "A")
264 | if ans:
265 | ips = list()
266 | for i in ans.response.answer:
267 | for j in i.items:
268 | if hasattr(j, "address"):
269 | self.domain_ips_dict[domain].append(j.address)
270 | except dns.resolver.NoAnswer:
271 | pass
272 | except dns.exception.Timeout:
273 | pass
274 | except Exception as e:
275 | pass
276 |
277 | def remove_black_ip(self):
278 | # 对于接口返回域名的泛解析结果的去除
279 | for domain, ips in self.domain_ips_dict.items():
280 | for ip in ips:
281 | if ip in self.ip_domain_count_dict.keys():
282 | self.ip_domain_count_dict[ip]["count"] +=1
283 | self.ip_domain_count_dict[ip]["domains"].append(domain)
284 | else:
285 | self.ip_domain_count_dict[ip] = {"domains": [domain], "count": 1}
286 | # remove
287 | for ip, domains_count in self.ip_domain_count_dict.items():
288 | if domains_count["count"] > DNS_THRESHOLD: # 有50个域名指向了同一个ip jd的有大量指向一个ip
289 | # 将泛解析的ip存下来 返回回去
290 | self.black_ip.append(ip)
291 | for domain in domains_count["domains"]:
292 | if domain in self.domain_ips_dict.keys():
293 | # print(domain, ip, domains_count["count"])
294 | self.domain_ips_dict.pop(domain)
295 |
296 | def remove_private(self):
297 | # 移除内网ip
298 | print_info("del private ip domain")
299 | for domain in list(self.domain_ips_dict.keys()):
300 | ips = self.domain_ips_dict[domain]
301 | if ipaddress.ip_address(ips[0]).is_private: # if private ip del
302 | self.domain_ips_dict.pop(domain)
303 |
304 | def run(self):
305 | # 先用script下的接口获取子域名
306 | self.run_scripts()
307 | # 对这些接口进行dns解析 获取对应的ip列表
308 | print_info("start dns query")
309 | self.threadpool_dns()
310 | # 是否保留内网ip结果
311 | if not self.is_private:
312 | self.remove_private()
313 | # 对接口返回含有泛解析的域名去除
314 | self.remove_black_ip()
315 | if self.get_black_ip: # 是否返回泛解析的ip
316 | return self.domain_ips_dict, self.black_ip
317 | else:
318 | return self.domain_ips_dict
319 |
320 |
321 | # 穷举类
322 | class ExhaustionScan(object):
323 | """暴力穷举
324 | :param scan_domain 要测试的域名
325 | :param thread_count 线程数 默认100
326 | :param is_output 是否输出进度条 默认不输出
327 | :param black_ip 泛解析的ip 默认为空
328 | :param is_private 是否保留内网ip 默认不保留
329 | :param sub_dict 指定的字典 默认读取配置文件下字典
330 | :param next_sub 是否是三级或者四级的域名扫描
331 | :param timeout 超时时间 默认穷举超时时间为 2h
332 | :return domain_ips_dict 域名对应解析的ip结果
333 |
334 | """
335 | def __init__ (
336 | self, scan_domain, thread_count=100,
337 | is_output=False, black_ip=list(),
338 | is_private=False, sub_dict=None, next_sub=False,
339 | big_dict=False, timeout=7200, gen_rule=False
340 | ):
341 | self.base_path = os.path.dirname(os.path.abspath(__file__))
342 | # dns
343 | self.resolver = resolver
344 | #self.resolver.nameservers=['8.8.8.8', '114.114.114.114']
345 | self.scan_domain = scan_domain
346 | # 默认线程100个
347 | self.thread_count = thread_count
348 | self.is_output = is_output
349 | self.timeout = timeout
350 | self.sub_dict = sub_dict
351 | self.big_dict = big_dict
352 | self.next_sub = next_sub
353 | self.domain_ips_dict = defaultdict(list)
354 | self.sub_dict_queue = queue.Queue()
355 | # 是否添加动态字典
356 | self.gen_rule = gen_rule
357 | self.load_subdomain_dict()
358 | self.all_size = self.sub_dict_queue.qsize()
359 | # 泛解析的ip 默认是空 可以将接口返回的 black_ip 传入这里
360 | self.black_ip = black_ip
361 | # {ip:{ domains: [域名], count: 计数} }
362 | self.ip_domain_count_dict = dict()
363 | # 内网ip
364 | self.is_private = is_private
365 |
366 |
367 | def load_subdomain_dict(self):
368 | if not self.next_sub:
369 | print_info("load sub dict")
370 | if self.sub_dict: # 使用指定的字典
371 | if os.path.exists( self.sub_dict):
372 | dict_path = self.sub_dict
373 | else:
374 | print_error("字典不存在")
375 | else:
376 | dict_path = os.path.join(self.base_path, "../","config", "subdomains.txt")
377 | # 如果使用大字典
378 | if self.big_dict:
379 | dict_path = os.path.join(self.base_path, "../","config", "big_subdomains.txt")
380 | # 是否是三级或者四级扫描使用 小字典
381 | if self.next_sub:
382 | dict_path = os.path.join(self.base_path, "../","config", "next_subdomains.txt")
383 | with open(dict_path, "r") as f:
384 | for sub in f:
385 | self.sub_dict_queue.put(f"{sub.strip()}.{self.scan_domain}")
386 | if self.gen_rule:
387 | genrule_subdomain_list = GenSubdomain(self.scan_domain).gen()
388 | for sub in genrule_subdomain_list:
389 | self.sub_dict_queue.put(f"{sub.strip()}.{self.scan_domain}")
390 | #if not self.next_sub:
391 | print_info(f"quque all size: {self.sub_dict_queue.qsize()}")
392 |
393 | def is_analysis(self):
394 | """
395 | 泛解析判断
396 | 通过不存在的域名进行判断
397 | """
398 | try:
399 | ans = self.resolver.resolve(
400 | ''.join(random.sample(string.ascii_lowercase,5))+"."+self.scan_domain , "A")
401 | if ans:
402 | ips = list()
403 | for i in ans.response.answer:
404 | for j in i.items:
405 | ip = j.to_text()
406 | if ip:
407 | return True
408 | except dns.resolver.NoAnswer:
409 | return False
410 | except dns.exception.Timeout:
411 | return False
412 | except dns.resolver.NXDOMAIN:
413 | return False
414 | except Exception as e:
415 | print(e)
416 | return False
417 |
418 | def analysis_dns(self, domain):
419 | try:
420 | # print(domain)
421 | ans = resolver.query(domain, "A")
422 | if ans:
423 | ips = list()
424 | for i in ans.response.answer:
425 | for j in i.items:
426 | if hasattr(j, "address"):
427 | # 增加对内网ip的判断 if not save private
428 | if not self.is_private:
429 | if not ipaddress.ip_address(j.address).is_private:
430 | # 增加对泛解析的操作
431 | if self.remove_black_ip(domain, j.address):
432 | self.domain_ips_dict[domain].append(j.address)
433 | else: # 如果对内网结果进行保存
434 | if self.remove_black_ip(domain, j.address):
435 | self.domain_ips_dict[domain].append(j.address)
436 | except dns.resolver.NoAnswer:
437 | pass
438 | except dns.exception.Timeout:
439 | pass
440 | except Exception as e:
441 | pass
442 |
443 | def remove_black_ip(self, domain, ip):
444 | """ 对解析后的域名ip判断是否是泛解析 超过阈值进行忽略和删除"""
445 | if ip in self.black_ip:
446 | return False
447 | if ip in self.ip_domain_count_dict.keys():
448 | self.ip_domain_count_dict[ip]["count"] +=1
449 | self.ip_domain_count_dict[ip]["domains"].append(domain)
450 | else:
451 | self.ip_domain_count_dict[ip] = {"domains": [domain], "count": 1}
452 | # 判断是否达到泛解析ip阈值 达到则删除记录下来的并增加黑ip
453 | if self.ip_domain_count_dict[ip]["count"] > DNS_THRESHOLD:
454 | for domain in self.ip_domain_count_dict[ip]["domains"]:
455 | if domain in self.domain_ips_dict.keys():
456 | self.domain_ips_dict.pop(domain)
457 | self.black_ip.append(ip)
458 | return False
459 | else:# 没有达到阈值
460 | return True
461 |
462 | def worker(self):
463 | while not self.sub_dict_queue.empty():
464 | domain = self.sub_dict_queue.get()
465 | if domain is None:
466 | break
467 | self.analysis_dns(domain)
468 | # self.sub_dict_queue.task_done()
469 |
470 | def all_done(self, threads):
471 | done_count = 0
472 | for t in threads:
473 | if not t.is_alive():
474 | done_count +=1
475 | if done_count == len(threads):
476 | return True
477 | return False
478 |
479 | def run(self):
480 | # 先进行泛解析判断
481 | if self.is_analysis():
482 | if not self.next_sub:
483 | print_debug("存在泛解析")
484 | threads = []
485 | for i in range(self.thread_count):
486 | t = threading.Thread(target=self.worker)
487 | t.setDaemon(True)
488 | t.start()
489 | threads.append(t)
490 | # 阻塞 等待队列消耗完
491 | if not self.next_sub:
492 | print_info("start thread ")
493 | start = time.perf_counter()
494 | #while not self.sub_dict_queue.empty() and self.all_done(threads):
495 | while not self.sub_dict_queue.empty():
496 | time.sleep(1)
497 | if self.is_output:
498 | # 输出进度条
499 | print_progress(
500 | self.sub_dict_queue.qsize(), self.all_size,
501 | start, len(self.domain_ips_dict)
502 | )
503 | if not self.next_sub: # 不然下一级扫描的进度条会有问题
504 | print() # 最后输出的问题加一个print来实现换行
505 | # self.sub_dict_queue.join() # 这里暂时不需要使用队列来阻塞
506 | return self.domain_ips_dict
507 |
508 | class GenSubdomain(object):
509 | def __init__(self, domain):
510 | # 我们这里拿三级子域名字典与domain结合生成字典
511 | self.scan_domain = domain
512 | self.base_dir = os.path.dirname(os.path.abspath(__file__))
513 |
514 | def gen(self):
515 | subdomain_dict_path = os.path.join(self.base_dir, "../","config", "next_subdomains.txt")
516 | gen_subdomain_list = list()
517 | rule_list = [
518 | "-",
519 | ".",
520 | "_",
521 | "@"
522 | ]
523 | with open(subdomain_dict_path, "r") as f:
524 | for sub in f:
525 | for rule in rule_list:
526 | gen_subdomain_list.append(f'{self.scan_domain}{sub}')
527 | gen_subdomain_list.append(f'{sub}{self.scan_domain}')
528 | gen_subdomain_list.append(f'{sub}{rule}{self.scan_domain}')
529 | gen_subdomain_list.append(f'{self.scan_domain}{rule}{sub}')
530 | print_info(f"gen subdomain count: {len(gen_subdomain_list)}")
531 | return gen_subdomain_list
532 |
--------------------------------------------------------------------------------