├── .codeclimate.yml ├── .coveragerc ├── .gitignore ├── .history ├── .travis.yml ├── CHNAGELOG.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── VERSION ├── build.sh ├── encomtech-net-1557193837 ├── encomtech-net.grep ├── encomtech-net.json └── encomtech-net.txt ├── publish.sh ├── release.sh ├── setup.py └── simplydomain ├── __init__.py ├── bin ├── simply_domain.py └── simply_domain_checker.py ├── config.py ├── docs ├── SimplyDomain-logo.png ├── Simplydomain.png ├── _config.yml ├── index.md └── sd-run.gif ├── setup └── requirements.txt ├── src ├── __init__.py ├── core_logger.py ├── core_output.py ├── core_printer.py ├── core_processes.py ├── core_progress.py ├── core_runtime.py ├── core_scrub.py ├── core_serialization.py ├── dynamic_modules │ ├── __init__.py │ ├── bing_search.py │ ├── crtsh_search.py │ ├── dnsdumpster_search.py │ ├── module_template.py │ ├── virustotal_search.py │ └── yahoo_search.py.test ├── module_checker.py ├── module_helpers.py ├── module_loader.py ├── module_multiprocessing.py ├── module_provider.py ├── module_recursion.py ├── module_resolvers.py └── static_modules │ ├── __init__.py │ ├── subdomain_bruteforce.py │ └── subdomain_raw_bruteforce.py └── tests ├── __init__.py ├── test_corePrinters.py ├── test_dnsServers.py └── test_requestsHelpers.py /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | plugins: 2 | pylint: 3 | enabled: true 4 | markdownlint: 5 | enabled: true 6 | pep8: 7 | enabled: true 8 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # build file coveralls and nosetests 2 | 3 | [report] 4 | exclude_lines = 5 | pragma: no cover 6 | def __repr__ 7 | raise AssertionError 8 | raise NotImplementedError 9 | if __name__ == .__main__.: 10 | omit = 11 | *setup.py 12 | */docs/* 13 | */python?.?/* 14 | */site-packages/nose/* 15 | *__init__* 16 | */thrid_party/* 17 | */tests/* -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | *.DS_Store 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | *.idea 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | #.env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | -------------------------------------------------------------------------------- /.history: -------------------------------------------------------------------------------- 1 | hf mf rdbl 0 a ffffffffffff 2 | hf mf rdsc 0 a ffffffffffff 3 | hf mf mifare 4 | hf mf mifare 5 | hf mf mifare 6 | quit 7 | help 8 | hf 9 | hf 10 | hw tune 11 | hw tune 12 | hw tune 13 | hw tune 14 | hw tune 15 | hw tune 16 | hw tune 17 | hf 14a 18 | hw version 19 | quit 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # sudo false implies containerized builds 2 | sudo: false 3 | language: python 4 | python: 5 | - 3.6 6 | install: 7 | - pip install nose 8 | - pip install coverage 9 | - pip install coveralls 10 | - pip install . 11 | script: 12 | - simply_domain.py -h 13 | 14 | 15 | -------------------------------------------------------------------------------- /CHNAGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | ## [1.0.8] - 2017-12-04 5 | ### Added 6 | - add code coverage with coveralls.io 7 | - add code .coveragerc file for build 8 | 9 | ## [1.0.7] - 2017-12-04 10 | ### Added 11 | - add OSX & Linux CI to expand coverage 12 | 13 | 14 | ## [1.0.6] - 2017-12-04 15 | ### Added 16 | - update readme to add TL;DR docker 17 | 18 | ## [1.0.-5] - 2017-12-04 19 | ### Added 20 | - update build.sh to auto release and publish 21 | 22 | ## [1.0.3] - 2017-11-26 23 | ### Added 24 | - add docker image 25 | - add build.sh 26 | - add release.sh 27 | 28 | 29 | ## [1.0.2] - 2017-11-26 30 | ### Added 31 | - progress bar on main search 32 | - progress bar write 33 | - now prints subdomain to std.out 34 | - new msg queue for output 35 | - _pbar_thread() for output 36 | 37 | ### Changed 38 | - broke up static vs dynamic modules 39 | - core_runtime now executes dynamic than static 40 | - runtime / process improvements 41 | - print style to metasploit look / feel 42 | 43 | ### Removed 44 | - N/A 45 | 46 | ## [1.0.1] - 2017-11-23 47 | ### Added 48 | - dnspop third_party lists 49 | 50 | ### Changed 51 | - N/A 52 | 53 | ### Removed 54 | - N/A 55 | 56 | ## [1.0.0] - 2017-11-23 57 | ### Added 58 | - Project created 59 | - Public release 60 | 61 | ### Changed 62 | - N/A 63 | 64 | ### Removed 65 | - N/A 66 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # NOTE: Only use this when you want to build image locally 2 | # else use `docker pull simplysecurity\simplydomain:{VERSION}` 3 | # all image versions can be found at: 4 | 5 | # -----BUILD ENTRY----- 6 | 7 | # image base 8 | FROM python:3 9 | 10 | # author 11 | MAINTAINER Killswitch-GUI 12 | ADD VERSION . 13 | LABEL description="Dockerfile base for SimplyDomain." 14 | 15 | RUN python3 -m pip install simplydomain 16 | 17 | ENTRYPOINT ["simply_domain.py"] 18 | 19 | # -----END OF BUILD----- 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, ⭕Alexander Rymdeko-Harvey 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | incluce simplydomain/.config.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/SimplySecurity/simplydomain-pkg.svg?branch=master)](https://travis-ci.org/SimplySecurity/simplydomain-pkg) 2 | [![codebeat badge](https://codebeat.co/badges/981ba393-f661-47d1-95dc-6aa5a3c87e2c)](https://codebeat.co/projects/github-com-simplysecurity-simplydomain-master) 3 | 4 | ![Alt text](simplydomain/docs/SimplyDomain-logo.png?raw=true "SimplyDomain") 5 | #### Table of Contents 6 | --- 7 | - [simplydomain-pkg](#simplydomain-pkg) 8 | * [Three Core Design Principals:](#three-core-design-principals-) 9 | - [Install simplydomain](#install-simplydomain) 10 | * [Using PIP Package Managment](#using-pip-package-managment) 11 | * [Using `setup.py` to build from source](#using--setuppy--to-build-from-source) 12 | * [simplydomain CLI tools (Quickstart)](#simplydomain-cli-tools--quickstart-) 13 | * [Install via Docker](#install-via-docker) 14 | - [simplydomain programming API](#simplydomain-programming-api) 15 | * [Importing simplydomain into your project](#importing-simplydomain-into-your-project) 16 | * [Executing a search](#executing-a-search) 17 | + [simplydomain.execute_search(domain)](#simplydomainexecute-search-domain-) 18 | + [simplydomain.execute_raw_bruteforce(domain)](#simplydomainexecute-raw-bruteforce-domain-) 19 | + [simplydomain.execute_wordlist_bruteforce(domain)](#simplydomainexecute-wordlist-bruteforce-domain-) 20 | 21 | # simplydomain-pkg 22 | Subdomain brute force focused on speed and data serialization. 23 | SimplyDomain uses a framework approach to build and deploy modules within. This allows 24 | for fast, easy and concise output to feed into larger OSINT feeds. 25 | 26 | ## Three Core Design Principals: 27 | * Easy install - support as many *NIX* based platforms. 28 | * Pure Python - no other arbitrary setup processes and Python-3 support 29 | * Expose public API - allows for simplydomain to integrate into other toolsets. 30 | 31 | # Install simplydomain 32 | You have a few fundamental choices when installing simplydomain; you can use your host systems python install, you can use `virtualenv` to ensure maximum capability, or Docker to have a clean environment. 33 | 34 | ## Using PIP Package Managment 35 | ```python 36 | pip3 install simplydomain 37 | ``` 38 | or 39 | ```python 40 | python3 -m pip install simplydomain 41 | ``` 42 | ## Using `setup.py` to build from source 43 | ```bash 44 | git clone git@github.com:SimplySecurity/simplydomain-pkg.git | cd simplydomain-pkg 45 | python3 -m pip install 46 | ``` 47 | ## simplydomain CLI tools (Quickstart) 48 | simplydomain supports a `bin` directory which is installed during the Python Setup PKG install. This now allows users to use their terminal of choice to use simplydomain. 49 | 50 | To display Help: 51 | ```bash 52 | simply_domain.py -h 53 | ``` 54 | To run a basic passive sub-domain search: 55 | ```bash 56 | simply_domain.py -all uber.com 57 | ``` 58 | 59 | ## Install via Docker 60 | The developed `Dockerfile` provides you with an easy way to spin up an instance and gain results in a short period without breaking certain dependencies. I highly suggest you use docker Volumes to ensure data persistence: 61 | 62 | ```bash 63 | docker run -ti simplysecurity/simplydomain -h 64 | ``` 65 | 66 | # simplydomain programming API 67 | The simplydomain Python package allows you to expose a few critical areas of simplydomain to enable you easily extend or implement simplydomain in existing projects. 68 | 69 | *For reference the exposed API lives at https://github.com/SimplySecurity/simplydomain-pkg/simplydomain/__init__.py* 70 | 71 | ## Importing simplydomain into your project 72 | Since simplydomain really at the core is a suite of high-level functions, there are only a few **High Level** API calls that can be made. For this reason, the exposed api is purely functioning vs. Class structures. 73 | ```python 74 | import simplydomain 75 | 76 | simplydomain.() 77 | ``` 78 | ## Executing a search 79 | simplydomain consists of many `Dynamic` modules and `Static` modules too allow a programmer to search a large subset of sources for subdomains easily. Within the simplydomain API, this concept is broken down into executing a large scale search function, and specific `Static` modules. 80 | 81 | ---- 82 | ### simplydomain.execute_search(domain) 83 | Executes the main search function(s) of simplydomain. 84 | 85 | **Required Parameters**: 86 | * domain (str) - sets the domain to search sub-domains for 87 | 88 | **Optional Parameters**: 89 | * config (dict) - sets the JSON config settings 90 | * dnsservers (list) - sets a list of DNS servers for resolving Questions 91 | * debug (bool) - sets the log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) 92 | * verbose (bool) - set to enable verbose console messaging 93 | * wordlist_bruteforce (bool) - sets to enable wordlist bruteforcing 94 | * wordlist_count (bool) - top sub-domains count to bruteforce (1-1000000) 95 | * raw_bruteforce (bool) - set to enable to brute force keyspace 96 | * raw_depth (int) - depth to brute force keyspace (1-5) 97 | * return_type (str) - (dict || json) 98 | 99 | **Implemented Definition** 100 | ```python 101 | simplydomain.execute_search( 102 | domain, 103 | config={}, 104 | dnsservers=[], 105 | debug='CRITICAL', 106 | verbose=False, 107 | wordlist_bruteforce=True, 108 | wordlist_count=100, 109 | raw_bruteforce=True, 110 | raw_depth=3, 111 | return_type='json', 112 | ): 113 | ``` 114 | 115 | **Example(s)** 116 | ```python 117 | >>> import simplydomain 118 | >>> simplydomain.execute_search() 119 | ``` 120 | 121 | ---- 122 | ### simplydomain.execute_raw_bruteforce(domain) 123 | Executes the static raw brute-force module of simplydomain. This allows simplydomain to generate all applicable RFC character sets off a subdomain keyspace. This can range from 1 char() to 5 char() which can feasibly be brute forced. 124 | 125 | **Required Parameters**: 126 | * domain (str) - sets the domain to search sub-domains for 127 | 128 | **Optional Parameters**: 129 | * config (dict) - sets the JSON config settings 130 | * dnsservers (list) - sets a list of DNS servers for resolving Questions 131 | * debug (bool) - sets the log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) 132 | * verbose (bool) - set to enable verbose console messaging 133 | * wordlist_bruteforce (bool) - sets to enable wordlist bruteforcing 134 | * wordlist_count (bool) - top sub-domains count to bruteforce (1-1000000) 135 | * raw_bruteforce (bool) - set to enable to brute force keyspace 136 | * raw_depth (int) - depth to brute force keyspace (1-5) 137 | * return_type (str) - (dict || json) 138 | 139 | **Implemented Definition** 140 | ```python 141 | simplydomain.execute_raw_bruteforce( 142 | domain, 143 | config={}, 144 | dnsservers=[], 145 | debug='CRITICAL', 146 | verbose=False, 147 | wordlist_count=0, 148 | return_type='json', 149 | wordlist_bruteforce=False, 150 | raw_bruteforce=True, 151 | raw_depth=2 152 | ): 153 | ``` 154 | 155 | **Example(s)** 156 | ```python 157 | >>> import simplydomain 158 | >>> simplydomain.execute_raw_bruteforce('uber.com', raw_depth=3) 159 | 160 | '{"args": {"debug": true, "domain": "uber.com",..}, "data":...."}' 161 | ``` 162 | 163 | ---- 164 | ### simplydomain.execute_wordlist_bruteforce(domain) 165 | Executes the static wordlist brute-force module of simplydomain. This allows simplydomain to get a range() of X subdomains for to be brute-forced. This can range from 1-1 Million words. 166 | 167 | **Required Parameters**: 168 | * domain (str) - sets the domain to search sub-domains for 169 | 170 | **Optional Parameters**: 171 | * config (dict) - sets the JSON config settings 172 | * dnsservers (list) - sets a list of DNS servers for resolving Questions 173 | * debug (bool) - sets the log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) 174 | * verbose (bool) - set to enable verbose console messaging 175 | * wordlist_bruteforce (bool) - sets to enable wordlist bruteforcing 176 | * wordlist_count (bool) - top sub-domains count to bruteforce (1-1000000) 177 | * raw_bruteforce (bool) - set to enable to brute force keyspace 178 | * raw_depth (int) - depth to brute force keyspace (1-5) 179 | * return_type (str) - (dict || json) 180 | 181 | **Implemented Definition** 182 | ```python 183 | simplydomain.execute_raw_bruteforce( 184 | domain, 185 | config={}, 186 | dnsservers=[], 187 | debug='CRITICAL', 188 | verbose=False, 189 | wordlist_count=100, 190 | return_type='json', 191 | wordlist_bruteforce=True, 192 | raw_bruteforce=False, 193 | raw_depth=0 194 | ): 195 | ``` 196 | 197 | **Example(s)** 198 | ```python 199 | >>> import simplydomain 200 | >>> simplydomain.execute_raw_bruteforce('uber.com', wordlist_count=100) 201 | 202 | '{"args": {"debug": true, "domain": "uber.com",..}, "data":...."}' 203 | ``` 204 | 205 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 2.0.9 2 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | # SET THE FOLLOWING VARIABLES 4 | # docker hub username 5 | USERNAME=simplysecurity 6 | # image name 7 | IMAGE=simplydomain 8 | docker build -t $USERNAME/$IMAGE:latest . -------------------------------------------------------------------------------- /encomtech-net-1557193837/encomtech-net.grep: -------------------------------------------------------------------------------- 1 | name:Virus Total Subdomain Search module_name:virus_total.py module_version:https://www.virustotal.com/ui/domains/%s/subdomains source:1.0 time:1557184319.1424341 toolname:SimplyDomain subdomain:mail.encomtech.net vaild:True 2 | name:Virus Total Subdomain Search module_name:virus_total.py module_version:https://www.virustotal.com/ui/domains/%s/subdomains source:1.0 time:1557184319.1426492 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 3 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=1 time:1557184319.253505 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 4 | name:Python API for Dnsdumpster module_name:dnsdumpster_search.py module_version:https://dnsdumpster.com source:1.0 time:1557184319.8449721 toolname:SimplyDomain subdomain:encomtech.net vaild:True 5 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=11 time:1557184320.038201 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 6 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=21 time:1557184320.762511 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 7 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=31 time:1557184321.638444 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 8 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=41 time:1557184322.395355 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 9 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=51 time:1557184323.280381 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 10 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=61 time:1557184324.031577 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 11 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=71 time:1557184324.778116 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 12 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=81 time:1557184325.5136461 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 13 | name:Bing Subdomain Search module_name:bing_search.py module_version:http://www.bing.com/search?q=site%3A%s&first=%scount source:http://www.bing.com/search?q=site%3Aencomtech.net&first=91 time:1557184326.2586591 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 14 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557184335.842459 toolname:SimplyDomain subdomain:www.encomtech.net vaild:True 15 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557184336.056353 toolname:SimplyDomain subdomain:mail.encomtech.net vaild:True 16 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557184336.21666 toolname:SimplyDomain subdomain:lyncdiscover.encomtech.net vaild:True 17 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557184339.607589 toolname:SimplyDomain subdomain:sip.encomtech.net vaild:True 18 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557184340.3216908 toolname:SimplyDomain subdomain:autodiscover.encomtech.net vaild:True 19 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557184343.9657989 toolname:SimplyDomain subdomain:citrix.encomtech.net vaild:True 20 | name:Recursive Subdomain Bruteforce Using Wordlist module_name:subdomain_bruteforce.py module_version: source:1.0 time:1557186663.567428 toolname:SimplyDomain subdomain:msoid.encomtech.net vaild:True 21 | -------------------------------------------------------------------------------- /encomtech-net-1557193837/encomtech-net.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": { 3 | "debug": false, 4 | "domain": "encomtech.net", 5 | "verbose": false 6 | }, 7 | "data": [ 8 | { 9 | "module_name": "virus_total.py", 10 | "module_version": "https://www.virustotal.com/ui/domains/%s/subdomains", 11 | "name": "Virus Total Subdomain Search", 12 | "source": "1.0", 13 | "subdomain": "mail.encomtech.net", 14 | "time": 1557184319.1424341, 15 | "toolname": "SimplyDomain", 16 | "valid": true 17 | }, 18 | { 19 | "module_name": "virus_total.py", 20 | "module_version": "https://www.virustotal.com/ui/domains/%s/subdomains", 21 | "name": "Virus Total Subdomain Search", 22 | "source": "1.0", 23 | "subdomain": "www.encomtech.net", 24 | "time": 1557184319.1426492, 25 | "toolname": "SimplyDomain", 26 | "valid": true 27 | }, 28 | { 29 | "module_name": "bing_search.py", 30 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 31 | "name": "Bing Subdomain Search", 32 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=1", 33 | "subdomain": "www.encomtech.net", 34 | "time": 1557184319.253505, 35 | "toolname": "SimplyDomain", 36 | "valid": true 37 | }, 38 | { 39 | "module_name": "dnsdumpster_search.py", 40 | "module_version": "https://dnsdumpster.com", 41 | "name": "Python API for Dnsdumpster", 42 | "source": "1.0", 43 | "subdomain": "encomtech.net", 44 | "time": 1557184319.8449721, 45 | "toolname": "SimplyDomain", 46 | "valid": true 47 | }, 48 | { 49 | "module_name": "bing_search.py", 50 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 51 | "name": "Bing Subdomain Search", 52 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=11", 53 | "subdomain": "www.encomtech.net", 54 | "time": 1557184320.038201, 55 | "toolname": "SimplyDomain", 56 | "valid": true 57 | }, 58 | { 59 | "module_name": "bing_search.py", 60 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 61 | "name": "Bing Subdomain Search", 62 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=21", 63 | "subdomain": "www.encomtech.net", 64 | "time": 1557184320.762511, 65 | "toolname": "SimplyDomain", 66 | "valid": true 67 | }, 68 | { 69 | "module_name": "bing_search.py", 70 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 71 | "name": "Bing Subdomain Search", 72 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=31", 73 | "subdomain": "www.encomtech.net", 74 | "time": 1557184321.638444, 75 | "toolname": "SimplyDomain", 76 | "valid": true 77 | }, 78 | { 79 | "module_name": "bing_search.py", 80 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 81 | "name": "Bing Subdomain Search", 82 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=41", 83 | "subdomain": "www.encomtech.net", 84 | "time": 1557184322.395355, 85 | "toolname": "SimplyDomain", 86 | "valid": true 87 | }, 88 | { 89 | "module_name": "bing_search.py", 90 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 91 | "name": "Bing Subdomain Search", 92 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=51", 93 | "subdomain": "www.encomtech.net", 94 | "time": 1557184323.280381, 95 | "toolname": "SimplyDomain", 96 | "valid": true 97 | }, 98 | { 99 | "module_name": "bing_search.py", 100 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 101 | "name": "Bing Subdomain Search", 102 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=61", 103 | "subdomain": "www.encomtech.net", 104 | "time": 1557184324.031577, 105 | "toolname": "SimplyDomain", 106 | "valid": true 107 | }, 108 | { 109 | "module_name": "bing_search.py", 110 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 111 | "name": "Bing Subdomain Search", 112 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=71", 113 | "subdomain": "www.encomtech.net", 114 | "time": 1557184324.778116, 115 | "toolname": "SimplyDomain", 116 | "valid": true 117 | }, 118 | { 119 | "module_name": "bing_search.py", 120 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 121 | "name": "Bing Subdomain Search", 122 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=81", 123 | "subdomain": "www.encomtech.net", 124 | "time": 1557184325.5136461, 125 | "toolname": "SimplyDomain", 126 | "valid": true 127 | }, 128 | { 129 | "module_name": "bing_search.py", 130 | "module_version": "http://www.bing.com/search?q=site%3A%s&first=%scount", 131 | "name": "Bing Subdomain Search", 132 | "source": "http://www.bing.com/search?q=site%3Aencomtech.net&first=91", 133 | "subdomain": "www.encomtech.net", 134 | "time": 1557184326.2586591, 135 | "toolname": "SimplyDomain", 136 | "valid": true 137 | }, 138 | { 139 | "module_name": "subdomain_bruteforce.py", 140 | "module_version": "", 141 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 142 | "source": "1.0", 143 | "subdomain": "www.encomtech.net", 144 | "time": 1557184335.842459, 145 | "toolname": "SimplyDomain", 146 | "valid": true 147 | }, 148 | { 149 | "module_name": "subdomain_bruteforce.py", 150 | "module_version": "", 151 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 152 | "source": "1.0", 153 | "subdomain": "mail.encomtech.net", 154 | "time": 1557184336.056353, 155 | "toolname": "SimplyDomain", 156 | "valid": true 157 | }, 158 | { 159 | "module_name": "subdomain_bruteforce.py", 160 | "module_version": "", 161 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 162 | "source": "1.0", 163 | "subdomain": "lyncdiscover.encomtech.net", 164 | "time": 1557184336.21666, 165 | "toolname": "SimplyDomain", 166 | "valid": true 167 | }, 168 | { 169 | "module_name": "subdomain_bruteforce.py", 170 | "module_version": "", 171 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 172 | "source": "1.0", 173 | "subdomain": "sip.encomtech.net", 174 | "time": 1557184339.607589, 175 | "toolname": "SimplyDomain", 176 | "valid": true 177 | }, 178 | { 179 | "module_name": "subdomain_bruteforce.py", 180 | "module_version": "", 181 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 182 | "source": "1.0", 183 | "subdomain": "autodiscover.encomtech.net", 184 | "time": 1557184340.3216908, 185 | "toolname": "SimplyDomain", 186 | "valid": true 187 | }, 188 | { 189 | "module_name": "subdomain_bruteforce.py", 190 | "module_version": "", 191 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 192 | "source": "1.0", 193 | "subdomain": "citrix.encomtech.net", 194 | "time": 1557184343.9657989, 195 | "toolname": "SimplyDomain", 196 | "valid": true 197 | }, 198 | { 199 | "module_name": "subdomain_bruteforce.py", 200 | "module_version": "", 201 | "name": "Recursive Subdomain Bruteforce Using Wordlist", 202 | "source": "1.0", 203 | "subdomain": "msoid.encomtech.net", 204 | "time": 1557186663.567428, 205 | "toolname": "SimplyDomain", 206 | "valid": true 207 | } 208 | ], 209 | "meta_data": { 210 | "author": "Alexander Rymdeko-Harvey", 211 | "github_repo": "https://github.com/killswitch-GUI/SimplyDomain", 212 | "twitter": "Killswitch-GUI", 213 | "version": 0.1 214 | } 215 | } -------------------------------------------------------------------------------- /encomtech-net-1557193837/encomtech-net.txt: -------------------------------------------------------------------------------- 1 | autodiscover.encomtech.net 2 | citrix.encomtech.net 3 | encomtech.net 4 | lyncdiscover.encomtech.net 5 | mail.encomtech.net 6 | msoid.encomtech.net 7 | sip.encomtech.net 8 | www.encomtech.net 9 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | rm -rf dist/ 2 | python3 -m pip install --user --upgrade setuptools wheel 3 | python3 setup.py sdist bdist_wheel 4 | python3 -m pip install --user --upgrade twine 5 | python -m pip install --user --upgrade twine 6 | twine upload dist/* 7 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # SET THE FOLLOWING VARIABLES 5 | USERNAME=simplysecurity 6 | IMAGE=simplydomain 7 | 8 | # UPDATE THE SOURCE CODE 9 | git pull 10 | 11 | # bump version 12 | docker run --rm -v "$PWD":/app treeder/bump patch 13 | VERSION=`cat VERSION` 14 | echo "version: $VERSION" 15 | 16 | # ALERT VERSION 17 | echo "Building Version: $VERSION" 18 | 19 | # START BUILD 20 | ./build.sh 21 | 22 | # TAG IT 23 | git checkout -b "Version-$VERSION" 24 | git add --all 25 | git commit -m "version $VERSION" 26 | git tag -a "$VERSION" -m "version $VERSION" 27 | git push origin "Version-$VERSION" 28 | git push origin "Version-$VERSION" --tags 29 | git checkout master 30 | git merge "Version-$VERSION" 31 | git push 32 | hub release create Version-$VERSION -m "Version compiled by 'build.sh': $VERSION" 33 | 34 | # DOCKER TAG/VERSIONING 35 | docker tag $USERNAME/$IMAGE:latest $USERNAME/$IMAGE:$VERSION 36 | 37 | # PUSH TO DOCKER HUB 38 | docker push $USERNAME/$IMAGE:latest 39 | echo "Docker image pushed: $USERNAME/$IMAGE:latest" 40 | docker push $USERNAME/$IMAGE:$VERSION 41 | echo "Docker image pushed: $USERNAME/$IMAGE:$VERSION" 42 | 43 | ./publish.sh 44 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | VERSIONFILE = open("VERSION").read() 5 | 6 | setup(name='simplydomain', 7 | version=VERSIONFILE, 8 | description='simplydomain is a very basic framework to automate domain brute forcing.', 9 | url='http://github.com/SimplySecurity/simplydomain-pkg', 10 | author='Alexander Rymdeko-Harvey', 11 | author_email='a.rymdekoharvey@obscuritylabs.com', 12 | license='BSD 3.0', 13 | packages=[ 14 | 'simplydomain', 15 | 'simplydomain.src', 16 | 'simplydomain.src.dynamic_modules', 17 | 'simplydomain.src.static_modules', 18 | 'simplydomain.tests' 19 | ], 20 | classifiers=[ 21 | # How mature is this project? Common values are 22 | # 3 - Alpha 23 | # 4 - Beta 24 | # 5 - Production/Stable 25 | 'Development Status :: 4 - Beta', 26 | # Specify the Python versions you support here. In particular, ensure 27 | # that you indicate whether you support Python 2, Python 3 or both. 28 | 'Programming Language :: Python :: 3.6', 29 | 'Programming Language :: Python :: 3.7' 30 | ], 31 | install_requires=[ 32 | 'aiodns', 33 | 'aiohttp', 34 | 'beautifulsoup4', 35 | 'crtsh', 36 | 'dnsdumpster', 37 | 'fake_useragent', 38 | 'json2xml', 39 | 'requests', 40 | 'setuptools', 41 | 'termcolor', 42 | 'tqdm', 43 | 'uvloop', 44 | 'validators', 45 | 'click' 46 | ], 47 | scripts=[ 48 | 'simplydomain/bin/simply_domain.py' 49 | ], 50 | include_package_data=True, 51 | zip_safe=False) 52 | -------------------------------------------------------------------------------- /simplydomain/__init__.py: -------------------------------------------------------------------------------- 1 | from simplydomain.src import core_printer 2 | from simplydomain.src import core_runtime 3 | from simplydomain.src import module_resolvers 4 | from simplydomain.src import core_logger 5 | 6 | # import json config.. 7 | from simplydomain import config 8 | 9 | import os 10 | import sys 11 | import json 12 | import logging 13 | import argparse 14 | 15 | ##### SIMPLYDOMAIN API FUNCTIONS ##### 16 | 17 | # STATICS 18 | __core_printer = core_printer.CorePrinters() 19 | __core_logger = core_logger.CoreLogging() 20 | __core_dns_servers = module_resolvers.DnsServers() 21 | 22 | """ 23 | Current JSON config struc: 24 | wordlist_bruteforce: BOOL 25 | wordlist_count: INT 26 | raw_bruteforce: BOOL 27 | raw_depth: INT 28 | verbose: BOOL 29 | debug: BOOL 30 | """ 31 | 32 | 33 | def __raw_depth_check(value): 34 | """Check if value is positive int. 35 | 36 | Arguments: 37 | value {number} -- value to check 38 | 39 | Returns: 40 | number -- value that was checked 41 | 42 | Raises: 43 | argparse.ArgumentTypeError -- is an invalid positive int value 44 | argparse.ArgumentTypeError -- is too large of a keyspace for raw depth 45 | """ 46 | ivalue = int(value) 47 | if ivalue <= 0: 48 | raise argparse.ArgumentTypeError( 49 | "%s is an invalid positive int value" % value) 50 | if ivalue >= 6: 51 | raise argparse.ArgumentTypeError( 52 | "%s is too large of a keyspace for raw depth" % value) 53 | return ivalue 54 | 55 | 56 | def __load_dns(config): 57 | __core_dns_servers.populate_servers() 58 | __core_dns_servers.populate_config(config) 59 | 60 | 61 | def __load_config(): 62 | return config.__json_config 63 | 64 | 65 | def __set_logging(value='INFO'): 66 | if value == 'CRITICAL': 67 | __core_logger.start(logging.CRITICAL) 68 | if value == 'ERROR': 69 | __core_logger.start(logging.ERROR) 70 | if value == 'WARNING': 71 | __core_logger.start(logging.WARNING) 72 | if value == 'INFO': 73 | __core_logger.start(logging.INFO) 74 | if value == 'DEBUG': 75 | __core_logger.start(logging.DEBUG) 76 | 77 | 78 | def __parse_values(domain, debug, verbose, wordlist_bruteforce, wordlist_count, raw_bruteforce, raw_depth): 79 | parser = argparse.ArgumentParser() 80 | parser.add_argument("DOMAIN", help="domain to query") 81 | # opts 82 | parser.add_argument("-wb", "--wordlist-bruteforce", help="enable word list bruteforce module", 83 | action="store_true") 84 | parser.add_argument("-wc", "--wordlist-count", help="set the count of the top words to use DEFAULT: 100", 85 | action="store", default=100, type=int) 86 | parser.add_argument("-rb", "--raw-bruteforce", help="enable raw bruteforce module", 87 | action="store_true") 88 | parser.add_argument("-rd", "--raw-depth", help="set the count of depth to raw bruteforce DEFAULT: 3", 89 | action="store", default=3, type=__raw_depth_check) 90 | parser.add_argument("-m", "--module", help="module to hit", 91 | action="store") 92 | parser.add_argument( 93 | "-o", "--output", help="output directory location (Ex. /users/test/)") 94 | parser.add_argument("-on", "--output-name", 95 | help="output directory name (Ex. test-2017)",) 96 | parser.add_argument("-v", "--verbose", help="increase output verbosity", 97 | action="store_true") 98 | parser.add_argument("-d", "--debug", help="enable debug logging to .SimplyDns.log file, default WARNING only", 99 | action="store_true") 100 | args = [] 101 | if debug: 102 | args.append('-d') 103 | if verbose: 104 | args.append('-v') 105 | if wordlist_bruteforce: 106 | args.append('-wb') 107 | if wordlist_count: 108 | args.append('-wc') 109 | args.append(str(wordlist_count)) 110 | if raw_bruteforce: 111 | args.append('-rb') 112 | if raw_depth: 113 | args.append('-rd') 114 | args.append(str(raw_depth)) 115 | if domain: 116 | args.append(str(domain)) 117 | return parser.parse_args(args) 118 | 119 | 120 | def execute_raw_bruteforce(domain, config={}, dnsservers=[], debug='CRITICAL', verbose=False, wordlist_count=0, return_type='json', wordlist_bruteforce=False, raw_bruteforce=True, raw_depth=2): 121 | """Executes only the raw bruteforce search function of simplydomain. 122 | 123 | Executes the static raw brute-force module of simplydomain. 124 | This allows simplydomain to generate all applicable RFC character 125 | sets off a subdomain keyspace. This can range from 1 char() to 5 char() 126 | which can feasibly be brute forced. 127 | 128 | Arguments: 129 | domain {str} -- domain to search 130 | 131 | Keyword Arguments: 132 | config {dict} -- sets the JSON config settings for the opperation (default: {{}}) 133 | dnsservers {list} -- sets a list of top DNS servers to resolve (default: {[]}) 134 | debug {str} -- sets the python logging setting: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: {'CRITICAL'}) 135 | verbose {bool} -- sets the verbosity count (default: {False}) 136 | wordlist_count {number} -- count to bruteforce from the word list 0-1,000,000 (default: {0}) 137 | return_type {str} -- sets the return output type to (default: {'json'}) 138 | wordlist_bruteforce {bool} -- enable wordlist bruteforce (default: {False}) 139 | raw_bruteforce {bool} -- enable raw bruteforce (default: {True}) 140 | raw_depth {number} -- depth of keyspace (default: {2}) 141 | 142 | Returns: 143 | str -- (dict | json) object 144 | """ 145 | __set_logging(debug) 146 | if not config: 147 | config = __load_config() 148 | if not dnsservers: 149 | dnsservers = __load_dns(config) 150 | config['args'] = __parse_values( 151 | domain, 152 | debug, 153 | verbose, 154 | wordlist_bruteforce, 155 | wordlist_count, 156 | raw_bruteforce, 157 | raw_depth 158 | ) 159 | cr = core_runtime.CoreRuntime(__core_logger, config) 160 | return cr.execute_raw_bruteforce(return_type=return_type) 161 | 162 | 163 | def execute_wordlist_bruteforce(domain, config={}, dnsservers=[], debug='CRITICAL', verbose=False, wordlist_count=100, return_type='json', wordlist_bruteforce=True, raw_bruteforce=False, raw_depth=0): 164 | """Executes only the wordlist bruteforce search function of simplydomain. 165 | 166 | Executes the static wordlist brute-force module of simplydomain. 167 | This allows simplydomain to get a range() of X subdomains for to be brute-forced. 168 | This can range from 1-1 Million words. 169 | 170 | Arguments: 171 | domain {str} -- domain to search 172 | 173 | Keyword Arguments: 174 | config {dict} -- sets the JSON config settings for the opperation (default: {{}}) 175 | dnsservers {list} -- sets a list of top DNS servers to resolve (default: {[]}) 176 | debug {str} -- sets the python logging setting: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: {'CRITICAL'}) 177 | verbose {bool} -- sets the verbosity count (default: {False}) 178 | wordlist_count {number} -- count to bruteforce from the word list 0-1,000,000 (default: {100}) 179 | return_type {str} -- sets the return output type to (default: {'json'}) 180 | wordlist_bruteforce {bool} -- enable wordlist bruteforce (default: {True}) 181 | raw_bruteforce {bool} -- enable raw bruteforce (default: {False}) 182 | raw_depth {number} -- depth of keyspace (default: {0}) 183 | 184 | Returns: 185 | str -- (dict | json) object 186 | """ 187 | __set_logging(debug) 188 | if not config: 189 | config = __load_config() 190 | if not dnsservers: 191 | dnsservers = __load_dns(config) 192 | config['args'] = __parse_values( 193 | domain, 194 | debug, 195 | verbose, 196 | wordlist_bruteforce, 197 | wordlist_count, 198 | raw_bruteforce, 199 | raw_depth 200 | ) 201 | cr = core_runtime.CoreRuntime(__core_logger, config) 202 | return cr.execute_bruteforce(return_type=return_type) 203 | 204 | 205 | def execute_search(domain, config={}, dnsservers=[], debug='CRITICAL', verbose=False, wordlist_bruteforce=True, wordlist_count=100, raw_bruteforce=True, raw_depth=3, return_type='json'): 206 | """Executes the main search function(s) of simplydomain. 207 | 208 | simplydomain consists of many Dynamic modules and Static modules too 209 | allow a programmer to search a large subset of sources for subdomains 210 | easily. Within the simplydomain API, this concept is broken down into 211 | executing a large scale search function, and specific Static modules. 212 | 213 | Arguments: 214 | domain {str} -- domain to search 215 | 216 | Keyword Arguments: 217 | config {dict} -- sets the JSON config settings for the opperation (default: {{}}) 218 | dnsservers {list} -- sets a list of top DNS servers to resolve (default: {[]}) 219 | debug {str} -- sets the python logging setting: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: {'CRITICAL'}) 220 | verbose {bool} -- sets the verbosity count (default: {False}) 221 | wordlist_bruteforce {bool} -- enable wordlist bruteforce (default: {True}) 222 | wordlist_count {number} -- count to bruteforce from the word list 0-1,000,000 (default: {100}) 223 | raw_bruteforce {bool} -- enable raw bruteforce (default: {True}) 224 | raw_depth {number} -- enable raw bruteforce (default: {3}) 225 | return_type {str} -- sets the return output type to (default: {'json'}) 226 | 227 | Returns: 228 | str -- (dict | json) object 229 | """ 230 | __set_logging(debug) 231 | if not config: 232 | config = __load_config() 233 | if not dnsservers: 234 | dnsservers = __load_dns(config) 235 | # now setup the config file 236 | # in JSON format for the core 237 | # to use within simplydomain 238 | config['args'] = __parse_values( 239 | domain, 240 | debug, 241 | verbose, 242 | wordlist_bruteforce, 243 | wordlist_count, 244 | raw_bruteforce, 245 | raw_depth 246 | ) 247 | cr = core_runtime.CoreRuntime(__core_logger, config) 248 | return cr.execute_amp(return_type=return_type) 249 | -------------------------------------------------------------------------------- /simplydomain/bin/simply_domain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.6 2 | import argparse 3 | import json 4 | import logging 5 | import os 6 | import sys 7 | 8 | from simplydomain.src import core_printer 9 | from simplydomain.src import core_runtime 10 | 11 | from simplydomain.src import module_resolvers 12 | from simplydomain.src import core_logger 13 | 14 | # import json config.. 15 | from simplydomain import config 16 | 17 | 18 | def _raw_depth_check(value): 19 | ivalue = int(value) 20 | if ivalue <= 0: 21 | raise argparse.ArgumentTypeError( 22 | "%s is an invalid positive int value" % value) 23 | if ivalue >= 6: 24 | raise argparse.ArgumentTypeError( 25 | "%s is too large of a keyspace for raw depth" % value) 26 | return ivalue 27 | 28 | 29 | def cli_parse(): 30 | """ 31 | Parse the CLI args passed to the script. 32 | :return: args 33 | """ 34 | # required 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument("DOMAIN", help="domain to query") 37 | # opts 38 | parser.add_argument("-wb", "--wordlist-bruteforce", help="enable word list bruteforce module", 39 | action="store_true") 40 | parser.add_argument("-wc", "--wordlist-count", help="set the count of the top words to use DEFAULT: 100", 41 | action="store", default=100, type=int) 42 | parser.add_argument("-rb", "--raw-bruteforce", help="enable raw bruteforce module", 43 | action="store_true") 44 | parser.add_argument("-rd", "--raw-depth", help="set the count of depth to raw bruteforce DEFAULT: 3", 45 | action="store", default=3, type=_raw_depth_check) 46 | parser.add_argument("-m", "--module", help="module to hit", 47 | action="store") 48 | parser.add_argument( 49 | "-o", "--output", help="output directory location (Ex. /users/test/)") 50 | parser.add_argument("-on", "--output-name", 51 | help="output directory name (Ex. test-2017)",) 52 | parser.add_argument("-l", "--list", help="list loaded modules", 53 | action="store_true") 54 | parser.add_argument("-ll", "--long-list", help="list loaded modules and info about each module", 55 | action="store_true") 56 | parser.add_argument("-v", "--verbose", help="increase output verbosity", 57 | action="store_true") 58 | parser.add_argument("-d", "--debug", help="enable debug logging to .SimplyDns.log file, default WARNING only", 59 | action="store_true") 60 | args = parser.parse_args() 61 | if args.verbose: 62 | print("[!] verbosity turned on") 63 | return args, parser 64 | 65 | 66 | def load_config(pr): 67 | """ 68 | Loads .config.json file for use 69 | :return: dict obj 70 | """ 71 | json_file = config.__json_config 72 | ds = module_resolvers.DnsServers() 73 | ds.populate_servers() 74 | json_file = ds.populate_config(json_file) 75 | print(pr.blue_text('Public DNS resolvers populated: (SERVER COUNT: %s)' % 76 | (str(ds.count_resolvers())))) 77 | return json_file 78 | 79 | 80 | def main(): 81 | """ 82 | Print entry screen and pass execution to CLI, 83 | and task core. 84 | :return: 85 | """ 86 | pr = core_printer.CorePrinters() 87 | pr.print_entry() 88 | args, parser = cli_parse() 89 | logger = core_logger.CoreLogging() 90 | pr.print_config_start() 91 | config = load_config(pr) 92 | config['args'] = args 93 | if args.debug: 94 | pr.print_green_on_bold('[!] DEBUGGING ENABLED!') 95 | logger.start(logging.DEBUG) 96 | else: 97 | logger.start(logging.INFO) 98 | logger.infomsg('main', 'startup') 99 | if args.module: 100 | if not args.DOMAIN: 101 | parser.print_help() 102 | config['silent'] = False 103 | c = core_runtime.CoreRuntime(logger, config) 104 | c.execute_mp() 105 | elif args.list: 106 | c = core_runtime.CoreRuntime(logger, config) 107 | c.list_modules() 108 | elif args.long_list: 109 | c = core_runtime.CoreRuntime(logger, config) 110 | c.list_modules_long() 111 | else: 112 | if not args.DOMAIN: 113 | parser.print_help() 114 | config['silent'] = False 115 | c = core_runtime.CoreRuntime(logger, config) 116 | c.execute_mp() 117 | 118 | 119 | if __name__ == "__main__": 120 | try: 121 | main() 122 | except KeyboardInterrupt: 123 | print('Interrupted') 124 | try: 125 | sys.exit(0) 126 | except SystemExit: 127 | os._exit(0) 128 | -------------------------------------------------------------------------------- /simplydomain/bin/simply_domain_checker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.6 2 | 3 | 4 | """ 5 | SimplyDomainChecker.python 6 | Author: Brad Crawford 7 | 8 | Takes JSON input from SimplyDomain and resolves subdomain CNAMES 9 | looking for possible subdomain takeover opportunities. 10 | 11 | """ 12 | 13 | import sys 14 | import json 15 | import argparse 16 | 17 | from provider import PROVIDER_LIST 18 | from simplydomain.src import module_checker 19 | 20 | 21 | def lookup_cname(filename): 22 | 23 | 24 | print("\nLoaded {} providers from provider.py\n".format(len(PROVIDER_LIST))) 25 | 26 | 27 | #Open JSON file and build a list of domains marked as valid 28 | with open(filename) as json_file: 29 | json_data = json.load(json_file) 30 | json_list = json_data['data'] 31 | valid_domain_list = {obj['subdomain'] for obj in json_list if obj['valid']} 32 | print("\nLoaded {} unique domains\n".format(len(valid_domain_list))) 33 | 34 | myDns = module_checker.DnsChecker(valid_domain_list) 35 | pair_dict = myDns.run() 36 | for key,value in pair_dict.items(): 37 | print("Subdomain: {} ||| CNAME: {}".format(key,value)) 38 | #TODO: Add native HTTPS support 39 | 40 | myHttp = module_checker.HttpChecker(myDns.cname_results) 41 | output = myHttp.run() 42 | print(output) 43 | 44 | def cli_parser(): 45 | """ 46 | Parse the CLI args passed to the script. 47 | :return: args 48 | """ 49 | 50 | parser = argparse.ArgumentParser() 51 | # mandatory 52 | parser.add_argument("FILENAME", help="full path and filename") 53 | 54 | # optional 55 | 56 | args = parser.parse_args() 57 | return args 58 | 59 | def main(): 60 | args = cli_parser() 61 | lookup_cname(args.FILENAME) 62 | 63 | 64 | if __name__ == "__main__": 65 | try: 66 | main() 67 | except KeyboardInterrupt: 68 | print('Interrupted') 69 | try: 70 | sys.exit(0) 71 | except SystemExit: 72 | os._exit(0) -------------------------------------------------------------------------------- /simplydomain/config.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | __json_config = { 5 | 6 | "meta_data": { 7 | "version": 0.1, 8 | "author": "Alexander Rymdeko-Harvey", 9 | "twitter": "Killswitch-GUI", 10 | "github_repo": "https://github.com/killswitch-GUI/SimplyDomain" 11 | }, 12 | "subdomain_bruteforce": { 13 | "top_1000": [ 14 | "third_party", 15 | "dnspop", 16 | "results", 17 | "bitquark_20160227_subdomains_popular_1000" 18 | ], 19 | "top_10000": [ 20 | "third_party", 21 | "dnspop", 22 | "results", 23 | "bitquark_20160227_subdomains_popular_10000" 24 | ], 25 | "top_100000": [ 26 | "third_party", 27 | "dnspop", 28 | "results", 29 | "bitquark_20160227_subdomains_popular_100000" 30 | ], 31 | "top_1000000": [ 32 | "third_party", 33 | "dnspop", 34 | "results", 35 | "bitquark_20160227_subdomains_popular_1000000" 36 | ] 37 | }, 38 | "yahoo_search": { 39 | "start_count": 0, 40 | "end_count": 1000, 41 | "quantity": 100, 42 | "sleep_time": 3, 43 | "sleep_jitter": 0.30 44 | }, 45 | "bing_search": { 46 | "start_count": 1, 47 | "end_count": 100, 48 | "sleep_time": 3, 49 | "sleep_jitter": 0.30 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /simplydomain/docs/SimplyDomain-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/docs/SimplyDomain-logo.png -------------------------------------------------------------------------------- /simplydomain/docs/Simplydomain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/docs/Simplydomain.png -------------------------------------------------------------------------------- /simplydomain/docs/_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-architect -------------------------------------------------------------------------------- /simplydomain/docs/index.md: -------------------------------------------------------------------------------- 1 | ![Alt text](SimplyDomain-logo.png?raw=true "SimplyDomain") 2 | Subdomain brute force focused on speed and data serialization. 3 | SimplyDomain uses a framework approach to build and deploy modules within. This allows 4 | for fast, easy and concise output to feed into larger OSINT feeds. 5 | 6 | ## Demo 7 | ![GitHub Logo](https://github.com/SimplySecurity/SimplyDomain/blob/master/docs/sd-run.gif?raw=true) 8 | 9 | ## Installation 10 | SimplyDomain is pure Python, and *should* support any platform. 11 | **Docker Install** 12 | ```bash 13 | docker pull simplysecurity/simplydomain 14 | ``` 15 | 16 | 17 | **One-line** install with bash: 18 | ```bash 19 | root@kali:~# curl -s https://raw.githubusercontent.com/SimplySecurity/SimplyDomain/master/setup/oneline-setup.sh | bash 20 | root@kali:~# cd SimplyDomain 21 | (SD) root@kali:~/SimplyDomain# ./SimplyDomain.py 22 | ``` 23 | 24 | **Git clone** install with `setup.sh` which builds a Python virtual env: 25 | ```bash 26 | root@kali:~# git clone https://github.com/SimplySecurity/SimplyDomain.git 27 | root@kali:~# ./SimplyDomain/setup.sh 28 | root@kali:~# cd SimplyDomain 29 | (SD) root@kali:~/SimplyDomain# ./SimplyDomain.py 30 | ``` 31 | 32 | **Source** install with **no** Python virtual end: 33 | ```bash 34 | root@kali:~# git clone https://github.com/SimplySecurity/SimplyDomain.git 35 | root@kali:~# cd SimplyDomain/setup/ 36 | root@kali:~# pip install -r requirements.txt 37 | root@kali:~# cd .. 38 | root@kali:~# python3.6 SimplyDomain.py -h 39 | ``` 40 | 41 | ### Install FAQ 42 | * Python: Python 3.6 required 43 | * OS: SimplyDomain is pure Python if it runs Python it should work 44 | * Priv: Requires `sudo` to install 45 | 46 | ## Module Support 47 | 48 | Module | Name | Description | Version 49 | --- | --- | --- | --- 50 | crtsh_search.py | Comodo Certificate Fingerprint | Uses https://crt.sh search with unofficial search engine support. | 1.0 51 | bing_search.py | Bing Subdomain Search | Uses Bing search engine with unofficial search engine API support. | 1.0 52 | dnsdumpster_search.py | Python API for Dnsdumpster | (Unofficial) Python API for https://dnsdumpster.com/ using @paulsec lib | 1.0 53 | virus_total.py | Virus Total Subdomain Search | Uses https://virustotal.com search with unofficial search engine API support. | 1.0 54 | 55 | ## Running SimplyDomain 56 | 57 | ``` 58 | root@kali:~# python3.6 SimplyDomain.py -h 59 | 60 | ------------------------------------------------------------ 61 | ______ _______ __ 62 | / \/ \ / | 63 | /$$$$$$ $$$$$$$ | ______ _____ ____ ______ $$/ _______ 64 | $$ \__$$/$$ | $$ |/ \/ \/ \ / \/ / \ 65 | $$ \$$ | $$ /$$$$$$ $$$$$$ $$$$ |$$$$$$ $$ $$$$$$$ | 66 | $$$$$$ $$ | $$ $$ | $$ $$ | $$ | $$ |/ $$ $$ $$ | $$ | 67 | / \__$$ $$ |__$$ $$ \__$$ $$ | $$ | $$ /$$$$$$$ $$ $$ | $$ | 68 | $$ $$/$$ $$/$$ $$/$$ | $$ | $$ $$ $$ $$ $$ | $$ | 69 | $$$$$$/ $$$$$$$/ $$$$$$/ $$/ $$/ $$/ $$$$$$$/$$/$$/ $$/ 70 | ------------------------------------------------------------ 71 | 72 | usage: SimplyDomain.py [-h] [-m MODULE] [-o OUTPUT] [-on OUTPUT_NAME] [-l] 73 | [-ll] [-v] [-d] 74 | DOMAIN 75 | 76 | positional arguments: 77 | DOMAIN domain to query 78 | 79 | optional arguments: 80 | -h, --help show this help message and exit 81 | -m MODULE, --module MODULE 82 | module to hit 83 | -o OUTPUT, --output OUTPUT 84 | output directory location (Ex. /users/test/) 85 | -on OUTPUT_NAME, --output-name OUTPUT_NAME 86 | output directory name (Ex. test-2017) 87 | -l, --list list loaded modules 88 | -ll, --long-list list loaded modules and info about each module 89 | -v, --verbose increase output verbosity 90 | -d, --debug enable debug logging to .SimplyDns.log file, default 91 | WARNING only 92 | ``` 93 | 94 | ### Passive Collection 95 | Using just the dynamic modules, SimplyDomain will not attempt to brute force any `A` records. You can easily run SimplyDomain via the virtual-env by using the `cd` command into the `SimplyDomain` dir. This will activate the python virtual environment and allow you to use python3.6 within. 96 | 97 | ``` 98 | python3.6 SimplyDomain.py test.com 99 | ``` 100 | 101 | ### Active Collection 102 | Enabling the wordlist brute force will allow you to discover new domains and attempt to brute force unknown or undiscovered domain names. Passing the `-wc` also allows you to specify up to the top 1 Million seen subdomains. 103 | 104 | ``` 105 | python3.6 SimplyDomain.py test.com -wb -wc 10000 106 | ``` 107 | 108 | ### Output / Data Injector 109 | All runs by default create a folder structure based on the domain name and time, this also includes: 110 | * Grepable Text 111 | * JSON Object 112 | * Sorted Uniq Text File 113 | * XML (In dev) 114 | 115 | To change the location and output name use the following: 116 | 117 | ``` 118 | python3.6 SimplyDomain.py test.com -wb -wc 10000 -o /user/test/ -on foldername-to-pickup 119 | ``` 120 | 121 | ## Contributing 122 | This project is built with PyCharms and should be imported via the `.idea` Folder. Please make sure the following take place before submitting a pull request: 123 | 124 | 1. Passes all code `Inspections` - Python & General 125 | 2. If possible unit tests of new code 126 | 127 | 128 | -------------------------------------------------------------------------------- /simplydomain/docs/sd-run.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/docs/sd-run.gif -------------------------------------------------------------------------------- /simplydomain/setup/requirements.txt: -------------------------------------------------------------------------------- 1 | # Requirements automatically generated by pigar. 2 | # https://github.com/Damnever/pigar 3 | 4 | # src/static_modules/subdomain_bruteforce.py: 5 5 | aiodns == 1.1.1 6 | 7 | # src/dynamic_modules/bing_search.py: 4 8 | beautifulsoup4 == 4.6.0 9 | 10 | # src/static_modules/subdomain_bruteforce.py: 9 11 | click == 6.7 12 | 13 | 14 | # src/dynamic_modules/crtsh_search.py: 3 15 | crtsh == 0.1.0 16 | 17 | # src/dynamic_modules/dnsdumpster_search.py: 3 18 | dnsdumpster == 0.3 19 | 20 | # src/module_helpers.py: 3 21 | # src/module_resolvers.py: 1 22 | fake_useragent == 0.1.10 23 | 24 | # SimplyDomain.py: 8,9,11,12 25 | # src/dynamic_modules/bing_search.py: 5,6,8 26 | # src/dynamic_modules/crtsh_search.py: 4,6 27 | # src/dynamic_modules/dnsdumpster_search.py: 4,5,7 28 | # src/dynamic_modules/virustotal_search.py: 4,5,7 29 | # src/static_modules/subdomain_bruteforce.py: 13,14,17 30 | json2xml == 2.2.0 31 | 32 | # src/module_helpers.py: 1 33 | requests == 2.20.0 34 | 35 | # setup/setup.py: 1 36 | setuptools == 32.2.0 37 | 38 | # src/core_printer.py: 1 39 | termcolor == 1.1.0 40 | 41 | # src/core_progress.py: 1 42 | # src/static_modules/subdomain_bruteforce.py: 11 43 | tqdm == 4.19.4 44 | 45 | # src/static_modules/subdomain_bruteforce.py: 7 46 | uvloop == 0.8.1 47 | 48 | # src/core_scrub.py: 1 49 | # src/module_helpers.py: 2 50 | validators == 0.12.0 51 | -------------------------------------------------------------------------------- /simplydomain/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/src/__init__.py -------------------------------------------------------------------------------- /simplydomain/src/core_logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class CoreLogging(object): 5 | """simple logging testing and dev""" 6 | 7 | def __init__(self): 8 | self.name = ".simplydomain.log" 9 | 10 | def start(self, level=logging.INFO): 11 | logger = logging.getLogger("simplydomain") 12 | logger.setLevel(level) 13 | fh = logging.FileHandler(self.name) 14 | formatter = logging.Formatter( 15 | '%(asctime)s-[%(name)s]-[%(levelname)s]- %(message)s') 16 | fh.setFormatter(formatter) 17 | logger.addHandler(fh) 18 | logger.info("Program started") 19 | logging.captureWarnings(True) 20 | logger.info("Set Logging Warning Capture: True") 21 | 22 | def debugmsg(self, message, modulename): 23 | try: 24 | msg = 'simplydomain.' + str(modulename) 25 | logger = logging.getLogger(msg) 26 | logger.debug(str(message)) 27 | except Exception as e: 28 | print(e) 29 | 30 | def infomsg(self, message, modulename): 31 | try: 32 | msg = 'simplydomain.' + str(modulename) 33 | logger = logging.getLogger(msg) 34 | logger.info(str(message)) 35 | except Exception as e: 36 | print(e) 37 | 38 | def warningmsg(self, message, modulename): 39 | try: 40 | msg = 'simplydomain.' + str(modulename) 41 | logger = logging.getLogger(msg) 42 | logger.warning(str(message)) 43 | except Exception as e: 44 | print(e) 45 | -------------------------------------------------------------------------------- /simplydomain/src/core_output.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | import time 5 | 6 | from . import core_printer 7 | 8 | 9 | class CoreOutput(core_printer.CorePrinters): 10 | """ 11 | Core class to handle final output. 12 | """ 13 | 14 | def __init__(self): 15 | """ 16 | Init class. 17 | """ 18 | core_printer.CorePrinters.__init__(self) 19 | self.json_data = {} 20 | 21 | def output_json_obj(self, json_data): 22 | """ 23 | Output json data file. 24 | :param json_data: json obj 25 | :return: NONE 26 | """ 27 | return json.dumps(json_data.subdomains, sort_keys=True) 28 | 29 | def output_json(self, json_data): 30 | """ 31 | Output json data file. 32 | :param json_data: json obj 33 | :return: NONE 34 | """ 35 | args = self.config['args'] 36 | s = str(args.DOMAIN) 37 | s = s.replace('.', '-') 38 | loc = "" 39 | dir_name = s + '-' + str(int(time.time())) 40 | def_name = s + '.json' 41 | if args.output: 42 | loc += str(args.output) 43 | if args.output_name: 44 | dir_name = str(args.output_name) 45 | dir_to_write = os.path.join(loc, dir_name) 46 | pathlib.Path(dir_to_write).mkdir(parents=True, exist_ok=True) 47 | with open(os.path.join(dir_to_write, def_name), 'a') as outfile: 48 | json.dump(json_data.subdomains, outfile, sort_keys=True, indent=4) 49 | self.logger.infomsg('output_json() JSON output file created at: : ' 50 | + str((os.path.join(dir_to_write, def_name))), 'CoreOutput') 51 | print(self.blue_text("JSON text file created: %s" % 52 | (os.path.join(dir_to_write, def_name)))) 53 | 54 | def print_text(self, json_data): 55 | """ 56 | Output to text file 57 | :param json_data: json obj 58 | :return: NONE 59 | """ 60 | for item in json_data.subdomains['data']: 61 | print("name:%s module_name:%s module_version:%s source:%s time:%s toolname:%s subdomain:%s vaild:%s" % 62 | (item['name'], item['module_name'], item['module_version'], item['source'], item['time'], 63 | item['toolname'], item['subdomain'], item['valid'])) 64 | 65 | def output_text(self, json_data): 66 | """ 67 | Output to text file 68 | :param json_data: json obj 69 | :return: NONE 70 | """ 71 | args = self.config['args'] 72 | s = str(args.DOMAIN) 73 | s = s.replace('.', '-') 74 | loc = "" 75 | dir_name = s + '-' + str(int(time.time())) 76 | def_name = s + '.grep' 77 | if args.output: 78 | loc += str(args.output) 79 | if args.output_name: 80 | dir_name = str(args.output_name) 81 | dir_to_write = os.path.join(loc, dir_name) 82 | pathlib.Path(dir_to_write).mkdir(parents=True, exist_ok=True) 83 | with open(os.path.join(dir_to_write, def_name), 'a') as outfile: 84 | for item in json_data.subdomains['data']: 85 | x = ("name:%s module_name:%s module_version:%s source:%s time:%s toolname:%s subdomain:%s vaild:%s\n" % 86 | (item['name'], item['module_name'], item['module_version'], item['source'], item['time'], 87 | item['toolname'], item['subdomain'], item['valid'])) 88 | outfile.write(x) 89 | self.logger.infomsg('output_text() TXT grep output file created at: : ' 90 | + str(os.path.join(dir_to_write, def_name)), 'CoreOutput') 91 | print(self.blue_text("Grepable text file created: %s" % 92 | (os.path.join(dir_to_write, def_name)))) 93 | 94 | def output_text_std(self, json_data): 95 | """ 96 | Output to text file 97 | :param json_data: json obj 98 | :return: NONE 99 | """ 100 | args = self.config['args'] 101 | s = str(args.DOMAIN) 102 | s = s.replace('.', '-') 103 | loc = "" 104 | dir_name = s + '-' + str(int(time.time())) 105 | def_name = s + '.txt' 106 | if args.output: 107 | loc += str(args.output) 108 | if args.output_name: 109 | dir_name = str(args.output_name) 110 | dir_to_write = os.path.join(loc, dir_name) 111 | pathlib.Path(dir_to_write).mkdir(parents=True, exist_ok=True) 112 | flist = [] 113 | with open(os.path.join(dir_to_write, def_name), 'a') as outfile: 114 | for item in json_data.subdomains['data']: 115 | flist.append(item['subdomain']) 116 | for item in sorted(set(flist)): 117 | x = ("%s\n" % (item)) 118 | outfile.write(x) 119 | self.logger.infomsg('output_text_std() TXT output file created at: : ' 120 | + str((os.path.join(dir_to_write, def_name))), 'CoreOutput') 121 | print(self.blue_text("Standard text file created: %s" % 122 | (os.path.join(dir_to_write, def_name)))) 123 | -------------------------------------------------------------------------------- /simplydomain/src/core_printer.py: -------------------------------------------------------------------------------- 1 | from termcolor import colored, cprint 2 | import json 3 | 4 | 5 | class CorePrinters(object): 6 | """ 7 | Core class: handles all data output within the project. 8 | """ 9 | 10 | __title_screen = """ 11 | ------------------------------------------------------------ 12 | ______ _______ __ 13 | / \/ \ / | 14 | /$$$$$$ $$$$$$$ | ______ _____ ____ ______ $$/ _______ 15 | $$ \__$$/$$ | $$ |/ \/ \/ \ / \/ / \ 16 | $$ \$$ | $$ /$$$$$$ $$$$$$ $$$$ |$$$$$$ $$ $$$$$$$ | 17 | $$$$$$ $$ | $$ $$ | $$ $$ | $$ | $$ |/ $$ $$ $$ | $$ | 18 | / \__$$ $$ |__$$ $$ \__$$ $$ | $$ | $$ /$$$$$$$ $$ $$ | $$ | 19 | $$ $$/$$ $$/$$ $$/$$ | $$ | $$ $$ $$ $$ $$ | $$ | 20 | $$$$$$/ $$$$$$$/ $$$$$$/ $$/ $$/ $$/ $$$$$$$/$$/$$/ $$/ 21 | ------------------------------------------------------------ 22 | """ 23 | __config_startup = """ 24 | *----------------------------------* 25 | | CONFIGURATION INITIALIZATION | 26 | *----------------------------------* 27 | """ 28 | 29 | __d_module_change = """ 30 | *----------------------------------* 31 | | DYNAMIC MODULES INITIALIZATION | 32 | *----------------------------------* 33 | """ 34 | __s_module_change = """ 35 | *----------------------------------* 36 | | STATIC MODULES INITIALIZATION | 37 | *----------------------------------* 38 | """ 39 | 40 | def __init__(self): 41 | """ 42 | INIT class object and define 43 | statics. 44 | """ 45 | self.print_green = lambda x: cprint(x, 'green') 46 | self.print_green_on_bold = lambda x: cprint(x, 'green', attrs=['bold']) 47 | self.print_yellow = lambda x: cprint(x, 'yellow') 48 | self.print_yellow_on_bold = lambda x: cprint( 49 | x, 'yellow', attrs=['bold']) 50 | self.print_red = lambda x: cprint(x, 'red') 51 | self.print_red_on_bold = lambda x: cprint(x, 'red', attrs=['bold']) 52 | self.print_white = lambda x: cprint(x, 'white') 53 | 54 | def blue_text(self, msg): 55 | """ 56 | Return green text obj. 57 | :param msg: TEXT 58 | :return: OBJ 59 | """ 60 | s = colored(' [*] ', color='blue') 61 | msg = s + msg 62 | return msg 63 | 64 | def green_text(self, msg): 65 | """ 66 | Return green text obj. 67 | :param msg: TEXT 68 | :return: OBJ 69 | """ 70 | s = colored(' [+] ', color='green') 71 | msg = s + msg 72 | return msg 73 | 74 | def print_entry(self): 75 | """ 76 | Print entry screen to the project. 77 | :return: NONE 78 | """ 79 | self.print_green_on_bold(self.__title_screen) 80 | 81 | def print_d_module_start(self): 82 | """ 83 | Print entry to dynamic modules 84 | :return: 85 | """ 86 | self.print_yellow(self.__d_module_change) 87 | 88 | def print_s_module_start(self): 89 | """ 90 | Print entry to dynamic modules 91 | :return: 92 | """ 93 | self.print_yellow(self.__s_module_change) 94 | 95 | def print_config_start(self): 96 | """ 97 | Print entry to dynamic modules 98 | :return: 99 | """ 100 | self.print_yellow(self.__config_startup) 101 | 102 | def print_modules(self, module_list): 103 | """ 104 | Print all modules within the framework to be run 105 | :param module_list: mod list of loaded functions 106 | :return: NONE 107 | """ 108 | self.print_red_on_bold(" [*] Available modules are:") 109 | x = 1 110 | ordlist = [] 111 | finalList = [] 112 | for name in module_list: 113 | parts = name.split("/") 114 | ordlist.append(parts[-1]) 115 | ordlist = sorted(ordlist) 116 | for name in ordlist: 117 | name = 'modules/' + name 118 | finalList.append(name) 119 | for name in finalList: 120 | print("\t%s)\t%s" % (x, '{0: <24}'.format(name))) 121 | x += 1 122 | 123 | def print_modules_long(self, module_list): 124 | """ 125 | Print all modules within the framework to be run 126 | :param module_list: mod list of loaded functions 127 | :return: NONE 128 | """ 129 | self.print_red_on_bold(" [*] Available modules are:") 130 | print("-" * 60) 131 | for mod in module_list: 132 | dynamic_module = module_list[mod] 133 | dm = dynamic_module.DynamicModule() 134 | parts = mod.split("/") 135 | name = 'modules/' + parts[-1] 136 | self.print_yellow_on_bold( 137 | " %s" % ('{0: <24}'.format(name).ljust(40))) 138 | print(json.dumps(dm.info, indent=4)) 139 | print("-" * 60) 140 | -------------------------------------------------------------------------------- /simplydomain/src/core_processes.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | from itertools import product 3 | import string 4 | import threading 5 | import time 6 | 7 | from . import core_printer 8 | from . import core_progress 9 | from . import core_serialization 10 | from . import module_recursion 11 | 12 | 13 | class CoreProcess(core_printer.CorePrinters, core_progress.CoreProgress, module_recursion.ModuleRecursion): 14 | """ 15 | Core class to handle threading and process 16 | creation. 17 | """ 18 | 19 | def __init__(self): 20 | """class init 21 | 22 | setups for a full MP stack. 23 | """ 24 | core_printer.CorePrinters.__init__(self) 25 | core_progress.CoreProgress.__init__(self) 26 | module_recursion.ModuleRecursion.__init__(self) 27 | self.procs = [] 28 | self.threads = [] 29 | self.processors = mp.cpu_count() 30 | self.module_count = 0 31 | self.mp = mp 32 | self.mpq = self.mp.Queue() 33 | self.task_queue = mp.Queue() 34 | self.task_output_queue = mp.Queue() 35 | self.task_msg_queue = mp.Queue() 36 | self.progress_bar_pickup = mp.Queue() 37 | 38 | # output handlers 39 | self.serialize_json_output = core_serialization.SerializeJSON( 40 | self.config) 41 | 42 | def _configure_mp(self): 43 | """ 44 | Sets the configuration props for 45 | mp to handle taskings. 46 | :return: NONE 47 | """ 48 | # use SPAWN since FORK is not supported on windows 49 | # SPAWN is slower since it creates a entire python 50 | # interpter 51 | 52 | self.logger.infomsg( 53 | 'setting MP to SPAWN for cross platform support', 'CoreRuntime') 54 | self.mp.set_start_method('spawn') 55 | 56 | def _configure_processes(self, mod_count): 57 | """ 58 | Check to make sure we dont start 59 | more procs than needed. Also fill taskQ with 60 | None values (PILLS) 61 | :return: NONE 62 | """ 63 | self.logger.infomsg('current # of modules loaded: ' + 64 | str(mod_count), 'CoreProcess') 65 | self.logger.infomsg('current # of processors of SYSTEM: ' + 66 | str(self.processors), 'CoreProcess') 67 | if self.processors > mod_count: 68 | self.logger.infomsg('setting MP count of: ' + 69 | str(mod_count), 'CoreProcess') 70 | self.processors = mod_count 71 | # populate the PILL == None 72 | for p in range(self.processors + 1): 73 | self.task_queue.put(None) 74 | 75 | def _task_output_queue_consumer(self): 76 | """ 77 | Consumes task_output_queue data at set interval, 78 | to be run as a Thread at a interval of 1 sec. If this res 79 | is to low queue mem could lock. 80 | :return: NONE 81 | """ 82 | while True: 83 | item = self.task_output_queue.get() 84 | if item: 85 | self.logger.debugmsg('_task_output_queue_consumer recv a subdomain: ' 86 | + str(item.subdomain), 'CoreProcess') 87 | msg = self.green_text("Subdomain: %s Vaild: (%s)" % 88 | ('{0: <30}'.format('('+str(item.subdomain)+')'), str(item.valid))) 89 | if not self.config['silent']: 90 | self.progress_print(msg) 91 | self.serialize_json_output.add_subdomain(item) 92 | self.add_subdomain(item) 93 | if item == None: 94 | self.logger.infomsg( 95 | '_task_output_queue_consumer is (NONE) exiting thread', 'CoreProcess') 96 | break 97 | if self.task_output_queue.empty(): 98 | pass 99 | # time.sleep(0.1) 100 | 101 | def populate_task_queue(self, modules): 102 | """ 103 | Populats the queue of module objects to be executed. 104 | :param modules: passed list of dynamic loaded modules 105 | :param module_name: if name passed it will only place that module into the q 106 | :return: NONE 107 | """ 108 | if self.config['args'].module: 109 | # populate only one module in the q 110 | for mod in modules: 111 | # populate the q with module data if name hits 112 | if self.config['args'].module in mod: 113 | self.task_queue.put(mod) 114 | else: 115 | for mod in modules: 116 | # populate the q with module data 117 | self.logger.infomsg('adding module to task queue: ' + 118 | str(mod), 'CoreProcess') 119 | self.task_queue.put(mod) 120 | self.module_count = len(modules) 121 | self._configure_processes(len(modules)) 122 | 123 | def clear_task_queue(self): 124 | """ 125 | If tasked to shutdown, clear the queue for cleanup 126 | :return: NONE 127 | """ 128 | # empty the pipe q 129 | self.logger.infomsg('tasked to clear_task_queue()', 'CoreProcess') 130 | while not self.task_queue.empty(): 131 | obj = self.task_queue.get() 132 | del obj 133 | # allow GC to pick up and flush pipe 134 | self.logger.infomsg( 135 | 'clear_task_queue() completed empty', 'CoreProcess') 136 | self.task_queue.close() 137 | 138 | def _pbar_thread(self): 139 | """ 140 | Built be called from a thread and do simple 141 | math to watch progress of Dynamic modules. 142 | :return: NONE 143 | """ 144 | if not self.config['silent']: 145 | start_count = len(self.procs) 146 | self.start_progress_bar(self.module_count) 147 | while self.check_active(): 148 | try: 149 | dm = self.progress_bar_pickup.get() 150 | except Exception as e: 151 | print(e) 152 | if dm == None: 153 | self.close_progress_bar() 154 | break 155 | if dm: 156 | if dm[0] == 'complete': 157 | self.progress_print(self.blue_text(dm[1])) 158 | self.inc_progress_bar(1) 159 | if dm[0] == 'execute': 160 | self.progress_print(self.blue_text(dm[1])) 161 | if self.progress_bar_pickup.empty(): 162 | time.sleep(0.1) 163 | 164 | def _start_thread_function(self, pointer): 165 | """ 166 | starts a late or early thread. 167 | :param pointer: Function def 168 | :return: NONE 169 | """ 170 | # TODO: FIX this hack and use real pointer? 171 | self.threads.insert(0, threading.Thread( 172 | target=self._pbar_thread())) 173 | 174 | t = self.threads[0] 175 | t.start() 176 | 177 | def _start_threads(self): 178 | """ 179 | Function to handle threads to be spawned. 180 | :return: NONE 181 | """ 182 | self.threads.append(threading.Thread( 183 | target=self._task_output_queue_consumer)) 184 | 185 | for t in self.threads: 186 | # TODO: Fix issue where threads die before job is parsed 187 | # TODO: Make some threads daemon? 188 | # t.daemon = True 189 | t.start() 190 | 191 | def stop_threads(self): 192 | """ 193 | Attempt to clean up threads before bail. 194 | :return: NONE 195 | """ 196 | self.logger.infomsg( 197 | 'tasked to stop_threads() putting (NONE)', 'CoreProcess') 198 | self.task_output_queue.put(None) 199 | for t in self.threads: 200 | self.logger.infomsg( 201 | 'Attempting to shutting down thread in stop_threads()', 'CoreProcess') 202 | t.join 203 | self.print_red("[!] All consumer threads have been joined") 204 | self.logger.infomsg( 205 | 'All consumer threads joined in stop_threads()', 'CoreProcess') 206 | 207 | def join_threads(self): 208 | """ 209 | Attempt to clean up threads before bail. 210 | :return: NONE 211 | """ 212 | self.logger.infomsg( 213 | 'tasked to join_threads() putting (NONE)', 'CoreProcess') 214 | self.task_output_queue.put(None) 215 | while True: 216 | if self.task_output_queue.empty() == True: 217 | self.logger.infomsg( 218 | 'self.task_output_queue is empty! breaking loop', 'CoreProcess') 219 | break 220 | else: 221 | self.logger.infomsg( 222 | 'self.task_output_queue not empty sleeping for 1 second', 'CoreProcess') 223 | time.sleep(1) 224 | for t in self.threads: 225 | t.join 226 | 227 | def start_processes(self): 228 | """ 229 | Executes all procs with a given module and 230 | passes it the proper objects to communicate with 231 | the core run time. 232 | 233 | :param module_obj: Module Ooject 234 | :param queues: A list of queue objects 235 | :return: BOOL 236 | """ 237 | self._start_threads() 238 | for _ in range(self.processors): 239 | self.logger.infomsg( 240 | 'start_processes() is kicking off empty procs with queue objects', 'CoreProcess') 241 | self.start_process(self.config, self.task_queue, 242 | self.task_output_queue, self.progress_bar_pickup) 243 | for p in self.procs: 244 | p.daemon = True 245 | p.start() 246 | self.logger.infomsg( 247 | 'start_process() started proc with daemon mode', 'CoreProcess') 248 | 249 | def start_process(self, config, task_queue, task_output_queue, progress_bar_pickup): 250 | """ 251 | Executes a proc with a given module and 252 | passes it the proper objects to communicate with 253 | the core run time. 254 | 255 | :param config: Module Ooject 256 | :param task_queue: A list of queue objects 257 | :param task_output_queue: 258 | :return: BOOL 259 | """ 260 | # add all process to a list so we itt over them 261 | queue_dict = { 262 | 'task_queue': task_queue, 263 | 'task_output_queue': task_output_queue, 264 | 'progress_bar_pickup': progress_bar_pickup 265 | } 266 | self.logger.infomsg('start_process() built queue_dict and appending to self.mp.procs, current proc count: ' 267 | + str(len(self.procs)), 'CoreProcess') 268 | self.procs.append( 269 | self.mp.Process(target=self.execute_processes, 270 | args=(config, queue_dict, self.modules))) 271 | 272 | def execute_processes(self, config, queue_dict, modules): 273 | """ 274 | Executes the module required and passed. 275 | :param module_obj: module settings 276 | :param queues: passed list obj 277 | :return: 278 | """ 279 | while True: 280 | # loop to execute taskings from taskQ 281 | q = queue_dict['task_queue'].get() 282 | pbq = queue_dict['progress_bar_pickup'] 283 | if q == None: 284 | self.logger.infomsg( 285 | 'execute_processes() dynamic module task_queue empty, EXITING process', 'CoreProcess') 286 | break 287 | dynamic_module = modules[q] 288 | try: 289 | dm = dynamic_module.DynamicModule(config) 290 | self.logger.infomsg('execute_processes() starting module: ' 291 | + str(dm.info['Name']), 'CoreProcess') 292 | msg = "Executing module: %s %s" % ('{0: <22}'.format( 293 | "("+dm.info['Module']+")"), "("+dm.info['Name']+")") 294 | if not self.config['silent']: 295 | pbq.put(['execute', msg]) 296 | # blocking 297 | dm.dynamic_main(queue_dict) 298 | self.logger.infomsg('execute_processes() completed module: ' 299 | + str(dm.info['Name']), 'CoreProcess') 300 | msg = "Module completed: %s %s" % ( 301 | '{0: <22}'.format("(" + dm.info['Module'] + ")"), "(" + dm.info['Name'] + ")") 302 | if not self.config['silent']: 303 | pbq.put(['complete', msg]) 304 | except Exception as e: 305 | self.logger.warningmsg('execute_processes hit fatal error: ' 306 | + str(e), 'CoreProcess') 307 | self.print_red(" [!] Module process failed: %s %s" % ( 308 | '{0: <22}'.format("(" + dm.info['Module'] + ")"), "(" + e + ")")) 309 | 310 | def execute_process(self, mod_name, config, queue_dict): 311 | """ 312 | Executes the module required and passed. 313 | :param module_obj: module settings 314 | :param queues: passed list obj 315 | :return: 316 | """ 317 | static_module = self.static_modules[mod_name] 318 | try: 319 | sm = static_module.DynamicModule(config) 320 | if not self.config['silent']: 321 | self.print_green(" [*] Executing module: %s %s" % ( 322 | '{0: <22}'.format("("+sm.info['Module']+")"), "("+sm.info['Name']+")")) 323 | sm.dynamic_main(queue_dict) 324 | if not self.config['silent']: 325 | self.print_green(" [*] Module completed: %s %s" % ( 326 | '{0: <22}'.format("(" + sm.info['Module'] + ")"), "(" + sm.info['Name'] + ")")) 327 | except Exception as e: 328 | if not self.config['silent']: 329 | self.print_red(" [!] Module process failed: %s %s" % ( 330 | '{0: <22}'.format("(" + sm.info['Module'] + ")"), "(" + str(e) + ")")) 331 | 332 | def check_active_len(self): 333 | """ 334 | Checks for active pids and returns count. 335 | :return: int 336 | """ 337 | _value = len(self.mp.active_children()) 338 | self.logger.infomsg('check_active_len() checking current active pids: ' 339 | + str(_value), 'CoreProcess') 340 | return _value 341 | 342 | def check_active(self): 343 | """ 344 | Check if mp is has active pids 345 | :return: BOOL 346 | """ 347 | if len(self.mp.active_children()): 348 | self.logger.infomsg( 349 | 'check_active() for MP pids: True', 'CoreProcess') 350 | return True 351 | else: 352 | self.logger.infomsg( 353 | 'check_active() for MP pids: False', 'CoreProcess') 354 | return False 355 | 356 | def join_processes(self): 357 | """ 358 | Attempt to join all the process to clean 359 | them up. 360 | :return: NONE 361 | """ 362 | for p in self.procs: 363 | self.logger.infomsg('join_processes() starting to join self.procs, current active: ' 364 | + str(len(self.procs)), 'CoreProcess') 365 | p.join() 366 | 367 | def list_processes(self): 368 | """ 369 | List all procs and pids. 370 | :return: NONE 371 | """ 372 | for p in self.procs: 373 | pid = p.pid 374 | p_name = p.name 375 | self.print_yellow( 376 | "[!] Process info: (PID: %s) (NAME: %s)" % (str(pid), str(p_name))) 377 | 378 | def list_processes_exitcode(self): 379 | """ 380 | List all procs and exitcode 381 | :return: 382 | """ 383 | for p in self.procs: 384 | pid = p.pid 385 | ec = p.exitcode 386 | self.print_yellow( 387 | "[!] Process info: (PID: %s) (EXITCODE: %s)" % (str(pid), str(ec))) 388 | 389 | def kill_processes(self): 390 | """ 391 | Attempt to kill all child pids 392 | and clean up from CTRL+C. 393 | :return: NONE 394 | """ 395 | for p in self.procs: 396 | pid = p.pid 397 | p_name = p.name 398 | while p.is_alive(): 399 | p.terminate() 400 | self.print_red("[!] Process has been terminated: (PID: %s) (NAME: %s)" % ( 401 | str(pid), str(p_name))) 402 | -------------------------------------------------------------------------------- /simplydomain/src/core_progress.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | 3 | 4 | class CoreProgress(object): 5 | 6 | """ 7 | Core class for dynamic modules and progress bars. 8 | Also allows to use core printers with tqdm.write() 9 | """ 10 | 11 | def __init__(self): 12 | """ 13 | INIT class object and define 14 | statics. 15 | """ 16 | # self.progress_bar 17 | pass 18 | 19 | def update_progress_bar(self): 20 | """ 21 | Adds a completion record to the progress bar. 22 | :return: 23 | """ 24 | self.progress_bar.update() 25 | 26 | def close_progress_bar(self): 27 | """ 28 | Progress bar cleanup 29 | :return: NONE 30 | """ 31 | self.progress_bar.close() 32 | 33 | def progress_print(self, msg_obj): 34 | """ 35 | Prints a message obj to te tqdm loop. 36 | :param msg_obj: A obj to be be printed 37 | :return: NONE 38 | """ 39 | self.progress_bar.write(msg_obj) 40 | 41 | def start_progress_bar(self, count, maxinterval=1, mininterval=0): 42 | """ 43 | Start up pbar and show on screen. 44 | :param count: count of modules 45 | :return: NONE 46 | """ 47 | self.progress_bar = tqdm( 48 | total=count, unit="module", maxinterval=maxinterval, mininterval=mininterval) 49 | 50 | def inc_progress_bar(self, size=0): 51 | """ 52 | Incs the progrss by (1) 53 | :param size: If size pass size, otherwise do min inc. 54 | :return: NONE 55 | """ 56 | if size: 57 | self.progress_bar.update(size) 58 | else: 59 | self.progress_bar.update() 60 | -------------------------------------------------------------------------------- /simplydomain/src/core_runtime.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | 4 | from . import core_output 5 | from . import core_processes 6 | from . import module_loader 7 | 8 | 9 | class CoreRuntime(module_loader.LoadModules, 10 | core_processes.CoreProcess, 11 | core_output.CoreOutput): 12 | """ 13 | Core Runtime Class. 14 | """ 15 | 16 | def __init__(self, logger, config): 17 | """ 18 | Init class and passed objects. 19 | """ 20 | self.config = config 21 | core_output.CoreOutput.__init__(self) 22 | module_loader.LoadModules.__init__(self) 23 | # ore_printer.CorePrinters.__init__(self) 24 | core_processes.CoreProcess.__init__(self) 25 | self.logger = logger 26 | 27 | def list_modules(self): 28 | """ 29 | List the modules loaded. 30 | :return: 31 | """ 32 | self.logger.infomsg('tasked to list modules', 'CoreRuntime') 33 | self.print_modules(self.modules) 34 | 35 | def list_modules_long(self): 36 | """ 37 | List the modules loaded. 38 | :return: 39 | """ 40 | self.logger.infomsg('tasked to list modules', 'CoreRuntime') 41 | self.print_modules_long(self.modules) 42 | 43 | def execute_output(self): 44 | """ 45 | Execute the output of formatted data stucs. 46 | :return: NONE 47 | """ 48 | self.logger.infomsg('starting execute_output()', 'CoreRuntime') 49 | self.output_text(self.serialize_json_output) 50 | self.output_json(self.serialize_json_output) 51 | self.output_text_std(self.serialize_json_output) 52 | 53 | def execute_output_object(self, return_type): 54 | """ 55 | Execute the output of formatted data stucs. 56 | :return: NONE 57 | """ 58 | self.logger.infomsg('starting execute_output_object()', 'CoreRuntime') 59 | if return_type.lower() == 'dict': 60 | return self.serialize_json_output 61 | else: 62 | return self.output_json_obj(self.serialize_json_output) 63 | 64 | def execute_startup(self): 65 | """ 66 | setup Q. 67 | :return: NONE 68 | """ 69 | self.logger.infomsg('execute_startup() setup began', 'CoreRuntime') 70 | self.print_d_module_start() 71 | self.logger.infomsg( 72 | 'execute_startup() start to populate task queue', 'CoreRuntime') 73 | self.populate_task_queue(self.modules) 74 | self.logger.infomsg( 75 | 'execute_startup() start to create hollow processes for future work', 'CoreRuntime') 76 | self.start_processes() 77 | self.config['silent'] = False 78 | 79 | def execute_dynamic(self): 80 | """ 81 | Execute only the dynamic modules. 82 | :return: NONE 83 | """ 84 | self.logger.infomsg( 85 | 'execute_dynamic() start ONLY dynamic modules', 'CoreRuntime') 86 | self._start_thread_function(self._pbar_thread) 87 | while self.check_active(): 88 | try: 89 | self.logger.infomsg( 90 | 'execute_dynamic() checking for active PIDs', 'CoreRuntime') 91 | time.sleep(2) 92 | except KeyboardInterrupt: 93 | self.logger.warningmsg( 94 | 'execute_dynamic() CRITICAL: CTRL+C Captured', 'CoreRuntime') 95 | self.print_red_on_bold("\n[!] CRITICAL: CTRL+C Captured - Trying to clean up!\n" 96 | "[!] WARNING: Press CTRL+C AGAIN to bypass and MANUALLY cleanup") 97 | try: 98 | time.sleep(0.1) 99 | self.stop_threads() 100 | self.kill_processes() 101 | sys.exit(0) 102 | except KeyboardInterrupt: 103 | self.list_processes() 104 | sys.exit(0) 105 | # cleanup dynamic mod pbar 106 | self.logger.infomsg( 107 | 'execute_dynamic() dynamic modules completed', 'CoreRuntime') 108 | self.progress_bar_pickup.put(None) 109 | self.close_progress_bar() 110 | 111 | def execute_static(self): 112 | """ 113 | Execute static modules in sorted order by EXEC order. 114 | :return: NONE 115 | """ 116 | self.logger.infomsg( 117 | 'execute_static() start ONLY static modules', 'CoreRuntime') 118 | self.print_s_module_start() 119 | queue_dict = { 120 | 'task_queue': self.task_queue, 121 | 'task_output_queue': self.task_output_queue, 122 | 'subdomain_list': self.get_subdomain_list(), 123 | } 124 | if self.config['args'].wordlist_bruteforce: 125 | self.execute_process( 126 | 'simplydomain/src/static_modules/subdomain_bruteforce.py', self.config, queue_dict) 127 | if self.config['args'].raw_bruteforce: 128 | self.execute_process( 129 | 'simplydomain/src/static_modules/subdomain_raw_bruteforce.py', self.config, queue_dict) 130 | self.logger.infomsg( 131 | 'execute_static() static modules completed', 'CoreRuntime') 132 | 133 | def execute_bruteforce(self, return_type='json'): 134 | """ 135 | Executes only the domain Bruteforce module and retunr data. 136 | """ 137 | try: 138 | self.config['silent'] = True 139 | self.populate_task_queue(self.modules) 140 | self._start_threads() 141 | self._start_thread_function(self._pbar_thread) 142 | queue_dict = { 143 | 'task_queue': self.task_queue, 144 | 'task_output_queue': self.task_output_queue, 145 | 'subdomain_list': self.get_subdomain_list(), 146 | } 147 | if self.config['args'].wordlist_bruteforce: 148 | self.execute_process( 149 | 'simplydomain/src/static_modules/subdomain_bruteforce.py', self.config, queue_dict) 150 | finally: 151 | self.join_threads() 152 | # return JSON object 153 | return self.execute_output_object(return_type) 154 | 155 | def execute_raw_bruteforce(self, return_type='json'): 156 | """ 157 | Executes only the domain Bruteforce module and retunr data. 158 | """ 159 | try: 160 | self.config['silent'] = True 161 | self.populate_task_queue(self.modules) 162 | self._start_threads() 163 | self._start_thread_function(self._pbar_thread) 164 | queue_dict = { 165 | 'task_queue': self.task_queue, 166 | 'task_output_queue': self.task_output_queue, 167 | } 168 | if self.config['args'].raw_bruteforce: 169 | self.execute_process( 170 | 'simplydomain/src/static_modules/subdomain_raw_bruteforce.py', self.config, queue_dict) 171 | finally: 172 | self.join_threads() 173 | # return JSON object 174 | return self.execute_output_object(return_type) 175 | 176 | def execute_amp(self, return_type='json'): 177 | """ 178 | Executes all the Dynamic Modules to be 179 | sent to processes. Returns raw object. 180 | """ 181 | try: 182 | # startup 183 | self.execute_startup() 184 | self.execute_dynamic() 185 | self.execute_static() 186 | finally: 187 | # cleanup 188 | self.join_processes() 189 | self.join_threads() 190 | # return JSON object 191 | return self.execute_output_object(return_type) 192 | 193 | def execute_mp(self): 194 | """ 195 | Executes all the Dynamic Modules to be 196 | sent to processes 197 | :return: 198 | """ 199 | try: 200 | # startup 201 | self.execute_startup() 202 | self.execute_dynamic() 203 | self.execute_static() 204 | finally: 205 | # cleanup 206 | self.join_processes() 207 | self.join_threads() 208 | self.execute_output() 209 | -------------------------------------------------------------------------------- /simplydomain/src/core_scrub.py: -------------------------------------------------------------------------------- 1 | from validators import domain 2 | 3 | 4 | class Scrub(object): 5 | 6 | """ 7 | Core data handler to clean, and post results in proper 8 | DataSerialization format for SimplyDomain. 9 | 10 | Attributes: 11 | subdomain: subdomain to parse 12 | """ 13 | 14 | def __init__(self, subdomain=""): 15 | """ 16 | Init class struc. Used as a object to parse and populate 17 | results. 18 | """ 19 | self.subdomain = subdomain 20 | 21 | def validate_domain(self): 22 | """ 23 | Use domain validator to confirm domain name. 24 | :return: BOOL 25 | """ 26 | try: 27 | val = domain(str(self.subdomain)) 28 | if val: 29 | # domain is valid 30 | return True 31 | else: 32 | # domain validation failed 33 | return False 34 | except Exception as e: 35 | # TODO: add in logger class for errors 36 | return False 37 | -------------------------------------------------------------------------------- /simplydomain/src/core_serialization.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class SerializeJSON(object): 5 | 6 | """ 7 | Core data handler for json output. Stores all final objects 8 | and allows a standard way to ingest data from SimplyDomain. 9 | 10 | Sample JSON Data Struc: 11 | 12 | JSON_OUTPUT = { 13 | "meta_data": { 14 | "version": 0.1, 15 | "author": "Alexander Rymdeko-Harvey", 16 | "twitter": "Killswitch-GUI", 17 | "github_repo": "https://github.com/killswitch-GUI/SimplyDomain" 18 | }, 19 | 20 | 21 | "args": { 22 | "domain": "google.com", 23 | "verbose": false, 24 | "debug": false 25 | }, 26 | 27 | "data": [ 28 | { 29 | "name": "cert search", 30 | "module_name": "crtsh_search.py", 31 | "source": "github.com", 32 | "time": 1222121212.11, 33 | "subdomain": "test.test.com" 34 | }, 35 | { 36 | "name": "cert search", 37 | "module_name": "crtsh_search.py", 38 | "source": "github.com", 39 | "time": 1222121212.11, 40 | "subdomain": "test.test.com" 41 | } 42 | ] 43 | } 44 | """ 45 | 46 | def __init__(self, config): 47 | """ 48 | Init class struc. Used as a object to parse and populate 49 | results. 50 | """ 51 | self.subdomains = {} 52 | self.subdomains['args'] = {} 53 | self.subdomains['data'] = [] 54 | self.subdomains['meta_data'] = config['meta_data'] 55 | self.subdomains['args']['domain'] = config['args'].DOMAIN 56 | self.subdomains['args']['debug'] = config['args'].debug 57 | self.subdomains['args']['verbose'] = config['args'].verbose 58 | 59 | def add_subdomain(self, input_obj): 60 | """ 61 | Add subdomain to class kbject for storage untill output is needed. 62 | :param input_obj: class object of subdomain 63 | :param config: core .config.json and params loaded 64 | :return: 65 | """ 66 | subdomain = {} 67 | subdomain['name'] = input_obj.name 68 | subdomain['module_name'] = input_obj.module_name 69 | subdomain['module_version'] = input_obj.module_version 70 | subdomain['source'] = input_obj.source 71 | subdomain['time'] = input_obj.time 72 | subdomain['toolname'] = input_obj.toolname 73 | subdomain['subdomain'] = input_obj.subdomain 74 | subdomain['valid'] = input_obj.valid 75 | self.subdomains['data'].append(subdomain) 76 | 77 | def print_json_subdomains(self): 78 | """ 79 | Simple test print. 80 | :return: 81 | """ 82 | json_str = json.dumps(self.subdomains, sort_keys=True, indent=4) 83 | print(json_str) 84 | 85 | 86 | class SubDomain(object): 87 | 88 | """ 89 | Core data handler to clean, and post results in proper 90 | DataSerialization format for SimplyDomain. 91 | 92 | Attributes: 93 | name: long name of method 94 | module_name: name of the module that performed collection 95 | source: source of the subdomain or resource of collection 96 | module_version: version from meta 97 | source: source of the collection 98 | time: time the result obj was built 99 | toolname: tool used to collect data 100 | subdomain: subdomain to use 101 | valid: is domain valid 102 | """ 103 | 104 | def __init__(self, name, module_name, module_version, source, time, subdomain, valid): 105 | """ 106 | Init class struc. Used as a object to parse and populate 107 | results. 108 | """ 109 | self.name = name 110 | self.module_name = module_name 111 | self.module_version = module_version 112 | self.source = source 113 | self.time = time 114 | self.toolname = "SimplyDomain" 115 | self.subdomain = subdomain 116 | self.valid = valid 117 | -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/src/dynamic_modules/__init__.py -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/bing_search.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.parse import urlparse 3 | 4 | from bs4 import BeautifulSoup 5 | from simplydomain.src import core_serialization 6 | from simplydomain.src import module_helpers 7 | 8 | from simplydomain.src import core_scrub 9 | 10 | 11 | # use RequestsHelpers() class to make requests to target URL 12 | class DynamicModule(module_helpers.RequestsHelpers): 13 | """ 14 | Dynamic module class that will be loaded and called 15 | at runtime. This will allow modules to easily be independent of the 16 | core runtime. 17 | """ 18 | 19 | def __init__(self, json_entry): 20 | """ 21 | Init class structure. Each module takes a JSON entry object which 22 | can pass different values to the module with out changing up the API. 23 | adapted form Empire Project: 24 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 25 | 26 | :param json_entry: JSON data object passed to the module. 27 | """ 28 | module_helpers.RequestsHelpers.__init__(self) 29 | self.json_entry = json_entry 30 | self.info = { 31 | # mod name 32 | 'Module': 'bing_search.py', 33 | 34 | # long name of the module to be used 35 | 'Name': 'Bing Subdomain Search', 36 | 37 | # version of the module to be used 38 | 'Version': '1.0', 39 | 40 | # description 41 | 'Description': ['Uses Bing search engine', 42 | 'with unofficial search engine API support.'], 43 | 44 | # authors or sources to be quoted 45 | 'Authors': ['@Killswitch-GUI', '@ecjx'], 46 | 47 | # list of resources or comments 48 | 'comments': [ 49 | 'https://github.com/ejcx/subdomainer/blob/master/subdomainer.py' 50 | ] 51 | } 52 | 53 | self.options = { 54 | 55 | 'url': 'http://www.bing.com/search?q=site%3A%s&first=%s' 56 | 'count' 57 | } 58 | 59 | def dynamic_main(self, queue_dict): 60 | """ 61 | Main entry point for process to call. 62 | 63 | core_serialization.SubDomain Attributes: 64 | name: long name of method 65 | module_name: name of the module that performed collection 66 | source: source of the subdomain or resource of collection 67 | module_version: version from meta 68 | source: source of the collection 69 | time: time the result obj was built 70 | subdomain: subdomain to use 71 | valid: is domain valid 72 | 73 | :return: NONE 74 | """ 75 | foundsubdomains = [] 76 | core_args = self.json_entry['args'] 77 | task_output_queue = queue_dict['task_output_queue'] 78 | cs = core_scrub.Scrub() 79 | start_count = int(self.json_entry['bing_search']['start_count']) 80 | end_count = int(self.json_entry['bing_search']['end_count']) 81 | while start_count <= end_count: 82 | domain = "http://www.bing.com/search?q=site%3A" + \ 83 | str(core_args.DOMAIN) + "&first=" + str(start_count) 84 | data, status = self.request_content(domain) 85 | soup = BeautifulSoup(data, 'html.parser') 86 | for i in soup.find_all('a', href=True): 87 | possiblesubdomain = i['href'] 88 | if "." + str(core_args.DOMAIN) in possiblesubdomain: 89 | parsed = urlparse(possiblesubdomain) 90 | if parsed.netloc not in foundsubdomains: 91 | foundsubdomains.append(str(parsed.netloc)) 92 | if parsed.hostname not in foundsubdomains: 93 | foundsubdomains.append(str(parsed.hostname)) 94 | for sub in foundsubdomains: 95 | cs.subdomain = sub 96 | # check if domain name is valid 97 | valid = cs.validate_domain() 98 | # build the SubDomain Object to pass 99 | sub_obj = core_serialization.SubDomain( 100 | self.info["Name"], 101 | self.info["Module"], 102 | self.options['url'], 103 | domain, 104 | time.time(), 105 | sub, 106 | valid 107 | ) 108 | task_output_queue.put(sub_obj) 109 | # results inc at rate of 10 per page 110 | start_count += 10 111 | time.sleep(0.5) 112 | -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/crtsh_search.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from crtsh import crtshAPI 4 | from simplydomain.src import core_serialization 5 | from simplydomain.src import module_helpers 6 | 7 | from simplydomain.src import core_scrub 8 | 9 | 10 | class DynamicModule(object): 11 | """ 12 | Dynamic module class that will be loaded and called 13 | at runtime. This will allow modules to easily be independent of the 14 | core runtime. 15 | """ 16 | 17 | def __init__(self, json_entry): 18 | """ 19 | Init class structure. Each module takes a JSON entry object which 20 | can pass different values to the module with out changing up the API. 21 | adapted form Empire Project: 22 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 23 | 24 | :param json_entry: JSON data object passed to the module. 25 | """ 26 | self.json_entry = json_entry 27 | self.info = { 28 | # mod name 29 | 'Module': 'crtsh_search.py', 30 | 31 | # long name of the module to be used 32 | 'Name': 'Comodo Certificate Fingerprint', 33 | 34 | # version of the module to be used 35 | 'Version': '1.0', 36 | 37 | # description 38 | 'Description': ['Uses https://crt.sh search', 39 | 'with unofficial search engine support.'], 40 | 41 | # authors or sources to be quoted 42 | 'Authors': ['@Killswitch-GUI', '@PaulSec'], 43 | 44 | # list of resources or comments 45 | 'comments': [ 46 | 'SHA-1 or SHA-256 lookup.' 47 | ] 48 | } 49 | 50 | self.options = { 51 | # threads for the module to use 52 | 'Threads': 1 53 | } 54 | 55 | def dynamic_main(self, queue_dict): 56 | """ 57 | Main entry point for process to call. 58 | 59 | core_serialization.SubDomain Attributes: 60 | name: long name of method 61 | module_name: name of the module that performed collection 62 | source: source of the subdomain or resource of collection 63 | module_version: version from meta 64 | source: source of the collection 65 | time: time the result obj was built 66 | subdomain: subdomain to use 67 | valid: is domain valid 68 | 69 | :return: 70 | """ 71 | core_args = self.json_entry['args'] 72 | task_output_queue = queue_dict['task_output_queue'] 73 | cs = core_scrub.Scrub() 74 | rd = [] 75 | data = crtshAPI().search(str(core_args.DOMAIN)) 76 | for d in data: 77 | cs.subdomain = d['issuer'] 78 | # check if domain name is valid 79 | valid = cs.validate_domain() 80 | # build the SubDomain Object to pass 81 | sub_obj = core_serialization.SubDomain( 82 | self.info["Name"], 83 | self.info["Module"], 84 | "https://crt.sh", 85 | self.info["Version"], 86 | time.time(), 87 | d['issuer'], 88 | valid 89 | ) 90 | # populate queue with return data objects 91 | task_output_queue.put(sub_obj) 92 | -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/dnsdumpster_search.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from dnsdumpster.DNSDumpsterAPI import DNSDumpsterAPI 4 | from simplydomain.src import core_serialization 5 | from simplydomain.src import module_helpers 6 | 7 | from simplydomain.src import core_scrub 8 | 9 | 10 | # use RequestsHelpers() class to make requests to target URL 11 | class DynamicModule(module_helpers.RequestsHelpers): 12 | """ 13 | Dynamic module class that will be loaded and called 14 | at runtime. This will allow modules to easily be independent of the 15 | core runtime. 16 | """ 17 | 18 | def __init__(self, json_entry): 19 | """ 20 | Init class structure. Each module takes a JSON entry object which 21 | can pass different values to the module with out changing up the API. 22 | adapted form Empire Project: 23 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 24 | 25 | :param json_entry: JSON data object passed to the module. 26 | """ 27 | module_helpers.RequestsHelpers.__init__(self) 28 | self.json_entry = json_entry 29 | self.info = { 30 | # mod name 31 | 'Module': 'dnsdumpster_search.py', 32 | 33 | # long name of the module to be used 34 | 'Name': 'Python API for Dnsdumpster', 35 | 36 | # version of the module to be used 37 | 'Version': '1.0', 38 | 39 | # description 40 | 'Description': ['(Unofficial) Python API for', 41 | 'https://dnsdumpster.com/ using @paulsec lib.'], 42 | 43 | # authors or sources to be quoted 44 | 'Authors': ['@Killswitch-GUI', '@PaulSec'], 45 | 46 | # list of resources or comments 47 | 'comments': [ 48 | 'Searches for prior seen domains, as well as data that goes with those.' 49 | ] 50 | } 51 | 52 | self.options = { 53 | 54 | 'threads': '', 55 | 'url': 'https://dnsdumpster.com', 56 | } 57 | 58 | def dynamic_main(self, queue_dict): 59 | """ 60 | Main entry point for process to call. 61 | 62 | core_serialization.SubDomain Attributes: 63 | name: long name of method 64 | module_name: name of the module that performed collection 65 | source: source of the subdomain or resource of collection 66 | module_version: version from meta 67 | source: source of the collection 68 | time: time the result obj was built 69 | subdomain: subdomain to use 70 | valid: is domain valid 71 | 72 | :return: NONE 73 | """ 74 | core_args = self.json_entry['args'] 75 | task_output_queue = queue_dict['task_output_queue'] 76 | cs = core_scrub.Scrub() 77 | 78 | data = DNSDumpsterAPI().search(str(core_args.DOMAIN)) 79 | for d in data['dns_records']['host']: 80 | cs.subdomain = d['domain'] 81 | # check if domain name is valid 82 | valid = cs.validate_domain() 83 | # build the SubDomain Object to pass 84 | sub_obj = core_serialization.SubDomain( 85 | self.info["Name"], 86 | self.info["Module"], 87 | self.options['url'], 88 | self.info["Version"], 89 | time.time(), 90 | d['domain'], 91 | valid 92 | ) 93 | # populate queue with return data object 94 | task_output_queue.put(sub_obj) 95 | -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/module_template.py: -------------------------------------------------------------------------------- 1 | class DynamicModule(object): 2 | """ 3 | Dynamic module class that will be loaded and called 4 | at runtime. This will allow modules to easily be independent of the 5 | core runtime. 6 | """ 7 | 8 | def __init__(self, json_entry): 9 | """ 10 | Init class structure. Each module takes a JSON entry object which 11 | can pass different values to the module with out changing up the API. 12 | adapted form Empire Project: 13 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 14 | 15 | :param json_entry: JSON data object passed to the module. 16 | """ 17 | self.json_entry = json_entry 18 | self.info = { 19 | # name of the module to be used 20 | 'Name': 'Template enumerator module', 21 | 22 | # version of the module to be used 23 | 'Version': '0.0', 24 | 25 | # description 26 | 'Description': ('Template module to model after', 27 | 'while using 2 lines'), 28 | 29 | # authors or sources to be quoted 30 | 'Authors': ['@Killswitch-GUI', '@Killswitch-GUI'], 31 | 32 | # list of resources or comments 33 | 'comments': [ 34 | 'Please make sure all of the required fields are filled in.', 35 | 'Http://www.google.com' 36 | ] 37 | } 38 | 39 | self.options = { 40 | # options for the module to use by default these will be dynamic in nature 41 | 42 | # threads for the module to use 43 | 'Threads': 10 44 | 45 | } 46 | 47 | def dynamic_main(self, queue_dict): 48 | """ 49 | Main entry point for process to call. 50 | :return: 51 | """ 52 | -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/virustotal_search.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | from simplydomain.src import core_serialization 5 | from simplydomain.src import module_helpers 6 | 7 | from simplydomain.src import core_scrub 8 | 9 | 10 | # use RequestsHelpers() class to make requests to target URL 11 | class DynamicModule(module_helpers.RequestsHelpers): 12 | """ 13 | Dynamic module class that will be loaded and called 14 | at runtime. This will allow modules to easily be independent of the 15 | core runtime. 16 | """ 17 | 18 | def __init__(self, json_entry): 19 | """ 20 | Init class structure. Each module takes a JSON entry object which 21 | can pass different values to the module with out changing up the API. 22 | adapted form Empire Project: 23 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 24 | 25 | :param json_entry: JSON data object passed to the module. 26 | """ 27 | module_helpers.RequestsHelpers.__init__(self) 28 | self.json_entry = json_entry 29 | self.info = { 30 | # mod name 31 | 'Module': 'virus_total.py', 32 | 33 | # long name of the module to be used 34 | 'Name': 'Virus Total Subdomain Search', 35 | 36 | # version of the module to be used 37 | 'Version': '1.0', 38 | 39 | # description 40 | 'Description': ['Uses https://virustotal.com search', 41 | 'with unofficial search engine API support.'], 42 | 43 | # authors or sources to be quoted 44 | 'Authors': ['@Killswitch-GUI'], 45 | 46 | # list of resources or comments 47 | 'comments': [ 48 | 'Searches for seen subdomains at one point and time.' 49 | ] 50 | } 51 | 52 | self.options = { 53 | 54 | 'url': 'https://www.virustotal.com/ui/domains/%s/subdomains' 55 | } 56 | 57 | def dynamic_main(self, queue_dict): 58 | """ 59 | Main entry point for process to call. 60 | 61 | core_serialization.SubDomain Attributes: 62 | name: long name of method 63 | module_name: name of the module that performed collection 64 | source: source of the subdomain or resource of collection 65 | module_version: version from meta 66 | source: source of the collection 67 | time: time the result obj was built 68 | subdomain: subdomain to use 69 | valid: is domain valid 70 | 71 | :return: NONE 72 | """ 73 | core_args = self.json_entry['args'] 74 | task_output_queue = queue_dict['task_output_queue'] 75 | cs = core_scrub.Scrub() 76 | domain = self.options['url'] % (str(core_args.DOMAIN)) 77 | data, status = self.request_json(domain) 78 | data = json.loads(data) 79 | if status: 80 | for d in data['data']: 81 | cs.subdomain = d['id'] 82 | # check if domain name is valid 83 | valid = cs.validate_domain() 84 | # build the SubDomain Object to pass 85 | sub_obj = core_serialization.SubDomain( 86 | self.info["Name"], 87 | self.info["Module"], 88 | self.options['url'], 89 | self.info["Version"], 90 | time.time(), 91 | d['id'], 92 | valid 93 | ) 94 | # populate queue with return data object 95 | task_output_queue.put(sub_obj) 96 | -------------------------------------------------------------------------------- /simplydomain/src/dynamic_modules/yahoo_search.py.test: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.parse import urlparse 3 | 4 | from bs4 import BeautifulSoup 5 | from src import core_serialization 6 | from src import module_helpers 7 | 8 | from src import core_scrub 9 | 10 | 11 | # use RequestsHelpers() class to make requests to target URL 12 | class DynamicModule(module_helpers.RequestsHelpers): 13 | """ 14 | Dynamic module class that will be loaded and called 15 | at runtime. This will allow modules to easily be independent of the 16 | core runtime. 17 | """ 18 | 19 | def __init__(self, json_entry): 20 | """ 21 | Init class structure. Each module takes a JSON entry object which 22 | can pass different values to the module with out changing up the API. 23 | adapted form Empire Project: 24 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 25 | 26 | :param json_entry: JSON data object passed to the module. 27 | """ 28 | module_helpers.RequestsHelpers.__init__(self) 29 | self.json_entry = json_entry 30 | self.info = { 31 | # mod name 32 | 'Module': 'yahoo_search.py', 33 | 34 | # long name of the module to be used 35 | 'Name': 'Yahoo Subdomain Search', 36 | 37 | # version of the module to be used 38 | 'Version': '1.0', 39 | 40 | # description 41 | 'Description': ['Uses yahoo search engine', 42 | 'with unofficial search engine API support.'], 43 | 44 | # authors or sources to be quoted 45 | 'Authors': ['@Killswitch-GUI'], 46 | 47 | # list of resources or comments 48 | 'comments': [ 49 | 'https://search.yahoo.com/search?p=' 50 | ] 51 | } 52 | 53 | self.options = { 54 | 55 | 'url': 'http://www.bing.com/search?q=site%3A%s&first=%s' 56 | } 57 | 58 | def dynamic_main(self, queue_dict): 59 | """ 60 | Main entry point for process to call. 61 | 62 | core_serialization.SubDomain Attributes: 63 | name: long name of method 64 | module_name: name of the module that performed collection 65 | source: source of the subdomain or resource of collection 66 | module_version: version from meta 67 | source: source of the collection 68 | time: time the result obj was built 69 | subdomain: subdomain to use 70 | valid: is domain valid 71 | 72 | :return: NONE 73 | """ 74 | foundsubdomains = [] 75 | core_args = self.json_entry['args'] 76 | task_output_queue = queue_dict['task_output_queue'] 77 | cs = core_scrub.Scrub() 78 | start_count = int(self.json_entry['yahoo_search']['start_count']) 79 | end_count = int(self.json_entry['yahoo_search']['end_count']) 80 | quantity = int(self.json_entry['yahoo_search']['quantity']) 81 | while start_count <= end_count: 82 | domain = "https://search.yahoo.com/search?p=" + str(core_args.DOMAIN) + "&b=" + \ 83 | str(start_count) + "&pz=" + str(quantity) 84 | data, status = self.request_content(domain) 85 | soup = BeautifulSoup(data, 'html.parser') 86 | for i in soup.find_all('a', href=True): 87 | possiblesubdomain = i['href'] 88 | if "." + str(core_args.DOMAIN) in possiblesubdomain: 89 | d, s = self.request_raw(possiblesubdomain) 90 | if s and "." + str(core_args.DOMAIN) in d.url: 91 | parsed = urlparse(d.url) 92 | if parsed.netloc not in foundsubdomains: 93 | foundsubdomains.append(str(parsed.netloc)) 94 | if parsed.hostname not in foundsubdomains: 95 | foundsubdomains.append(str(parsed.hostname)) 96 | for sub in foundsubdomains: 97 | cs.subdomain = sub 98 | # check if domain name is valid 99 | valid = cs.validate_domain() 100 | # build the SubDomain Object to pass 101 | sub_obj = core_serialization.SubDomain( 102 | self.info["Name"], 103 | self.info["Module"], 104 | self.options['url'], 105 | domain, 106 | time.time(), 107 | sub, 108 | valid 109 | ) 110 | task_output_queue.put(sub_obj) 111 | # results inc at rate of 10 per page 112 | start_count += 10 113 | time.sleep(1) -------------------------------------------------------------------------------- /simplydomain/src/module_checker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.6 2 | 3 | 4 | """ 5 | SimplyDomainChecker.python 6 | Author: Brad Crawford 7 | 8 | Takes JSON input from SimplyDomain and resolves subdomain CNAMES 9 | looking for possible subdomain takeover opportunities. 10 | 11 | """ 12 | 13 | import sys 14 | import json 15 | import requests 16 | import argparse 17 | import asyncio 18 | import aiodns 19 | from aiohttp import ClientSession 20 | from aiohttp.client_exceptions import ClientConnectorError 21 | from simplydomain.src.module_provider import PROVIDER_LIST 22 | 23 | class HttpChecker(object): 24 | 25 | def __init__(self, url_obj): 26 | """ 27 | url_obj=DICT: url:cname 28 | """ 29 | self.url_obj = url_obj 30 | 31 | async def fetch(self, sem, url, cname, session): 32 | async with sem: 33 | response_obj = {} 34 | response_obj['subdomain'] = "http://" + url 35 | response_obj['cname'] = cname 36 | response_obj['takeover'] = False 37 | for obj in PROVIDER_LIST: 38 | if all(key in cname for key in obj['cname']): 39 | try: 40 | async with session.get(response_obj['subdomain']) as response: 41 | print("Testing: {}".format(url)) 42 | print("URL: {} Status: {}".format(response.url, response.status)) 43 | data = await response.read() 44 | for r in obj['response']: 45 | if r in str(data): 46 | response_obj['takeover'] = True 47 | response_obj['type'] = {} 48 | response_obj['type']['confidence'] = "HIGH" 49 | response_obj['type']['provider'] = obj['name'] 50 | response_obj['type']['response'] = r 51 | response_obj['type']['response_status'] = response.status 52 | print("Got one: {}".format(response_obj)) 53 | return response_obj 54 | return response_obj 55 | except ClientConnectorError as e: 56 | print("Connection Error: {} CNAME: {}".format(e,cname)) 57 | response_obj['takeover'] = True 58 | response_obj['type'] = {} 59 | response_obj['type']['confidence'] = "MEDIUM" 60 | response_obj['type']['provider'] = obj['name'] 61 | response_obj['type']['response'] = e 62 | response_obj['type']['response_status'] = None 63 | return response_obj 64 | except Exception as e: 65 | print("Doh!: {} ErrorType: {} CNAME: {}".format(e, type(e),cname)) 66 | return None 67 | 68 | async def tasker(self): 69 | tasks = [] 70 | sem = asyncio.Semaphore(512) 71 | async with ClientSession(conn_timeout=3) as session: 72 | for url,cname in self.url_obj.items(): 73 | task = asyncio.ensure_future(self.fetch(sem, url, cname, session)) 74 | tasks.append(task) 75 | responses = asyncio.gather(*tasks) 76 | await responses 77 | return responses.result() 78 | 79 | def run(self): 80 | loop = asyncio.get_event_loop() 81 | future = asyncio.ensure_future(self.tasker()) 82 | loop.run_until_complete(future) 83 | return [x for x in future.result() if x is not None] 84 | 85 | 86 | class DnsChecker(object): 87 | 88 | def __init__(self, url_list, dns_server_count = 20): 89 | """ 90 | url_list=LIST: urls to check for CNAME 91 | :return: cname_results DICT: url:cname 92 | """ 93 | self.url_list = url_list 94 | self.sem = asyncio.Semaphore(512) 95 | self.loop = asyncio.get_event_loop() 96 | self.dns_servers = get_dns_servers(dns_server_count) 97 | self.resolver = aiodns.DNSResolver(loop=self.loop, servers=self.dns_servers, timeout=2, tries=2, rotate=True) 98 | self.cname_results = {} 99 | 100 | async def fetch(self, url): 101 | async with self.sem: 102 | try: 103 | response = await self.resolver.query(url,'CNAME') 104 | self.cname_results.update({url:response.cname}) 105 | return { "url": url, "cname": response.cname } if response.cname is not None else None 106 | except Exception as e: 107 | print("Error: {}".format(e)) 108 | return None 109 | 110 | async def tasker(self): 111 | tasks = [] 112 | for url in self.url_list: 113 | task = asyncio.ensure_future(self.fetch(url)) 114 | tasks.append(task) 115 | responses = asyncio.gather(*tasks, return_exceptions=True, loop=self.loop) 116 | await responses 117 | 118 | def run(self): 119 | print("DNS servers: {}".format(self.dns_servers)) 120 | future = asyncio.ensure_future(self.tasker()) 121 | self.loop.run_until_complete(future) 122 | return self.cname_results 123 | 124 | def get_dns_servers(count): 125 | try: 126 | r = requests.get('https://public-dns.info/nameserver/us.json') 127 | data = json.loads(r.text) 128 | servers = [i['ip'] for i in data if (i['reliability'] == 1)] 129 | servers_prune = servers[:count] 130 | return(servers_prune) 131 | except Exception as e: 132 | print("get_dns_servers error: {}".format(e)) 133 | #TODO return None or something -------------------------------------------------------------------------------- /simplydomain/src/module_helpers.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import validators 3 | from fake_useragent import UserAgent 4 | 5 | from . import core_output 6 | 7 | 8 | class RequestsHelpers(core_output.CoreOutput): 9 | 10 | """ 11 | A set of functions to make requests standard and handle fail cases 12 | to prevent error msgs. 13 | """ 14 | 15 | def __init__(self): 16 | """ 17 | Init class structure. 18 | """ 19 | # TODO: Init Logging class 20 | core_output.CoreOutput.__init__(self) 21 | self.ua = UserAgent() 22 | 23 | def get_dns_wildcard(self, domain): 24 | try: 25 | x = "*.{}".format(domain) 26 | url = 'https://dns.google.com/resolve?name=%s&type=A' % (str(x)) 27 | headers = {"Accept": "application/json"} 28 | response = requests.get(url,headers=headers,verify=True) 29 | return response.json() 30 | except Exception as e: 31 | print(response.text) 32 | return {} 33 | 34 | # split up json from raw for future support 35 | def request_json(self, url, return_code=200): 36 | """ 37 | Request JSON content and expected return code. 38 | :param url: URL to request 39 | :param return_code: expected return code or raise error 40 | :return: JSON, SuccessState 41 | """ 42 | try: 43 | header = { 44 | 'User-Agent': str(self.ua.google) 45 | } 46 | if not validators.url(url): 47 | self.print_yellow( 48 | " [!] Invalid URL Requested: %s" % (str(url))) 49 | return {}, False 50 | r = requests.get(url, headers=header) 51 | if r.status_code != return_code: 52 | self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" % 53 | (str(r.status_code), str(return_code))) 54 | return {}, False 55 | return r.content, True 56 | except requests.ConnectTimeout as e: 57 | self.print_red( 58 | " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e))) 59 | return {}, False 60 | except requests.TooManyRedirects as e: 61 | self.print_red( 62 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 63 | return {}, False 64 | except requests.HTTPError as e: 65 | self.print_red( 66 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 67 | return {}, False 68 | except ConnectionError as e: 69 | self.print_red( 70 | " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e))) 71 | return {}, False 72 | except Exception as e: 73 | self.print_red( 74 | " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e))) 75 | return {}, False 76 | 77 | def request_content(self, url, return_code=200): 78 | """ 79 | Request content and expected return code. 80 | :param url: URL to request 81 | :param return_code: expected return code or raise error 82 | :return: JSON, SuccessState 83 | """ 84 | try: 85 | header = { 86 | 'User-Agent': str(self.ua.google) 87 | } 88 | if not validators.url(url): 89 | self.print_yellow( 90 | " [!] Invalid URL Requested: %s" % (str(url))) 91 | return {}, False 92 | r = requests.get(url, headers=header) 93 | if r.status_code != return_code: 94 | self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" % 95 | (str(r.status_code), str(return_code))) 96 | return {}, False 97 | return r.content, True 98 | except requests.ConnectTimeout as e: 99 | self.print_red( 100 | " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e))) 101 | return {}, False 102 | except requests.TooManyRedirects as e: 103 | self.print_red( 104 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 105 | return {}, False 106 | except requests.HTTPError as e: 107 | self.print_red( 108 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 109 | return {}, False 110 | except ConnectionError as e: 111 | self.print_red( 112 | " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e))) 113 | return {}, False 114 | except Exception as e: 115 | self.print_red( 116 | " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e))) 117 | return {}, False 118 | 119 | def request_text(self, url, return_code=200): 120 | """ 121 | Request content and expected return code. 122 | :param url: URL to request 123 | :param return_code: expected return code or raise error 124 | :return: JSON, SuccessState 125 | """ 126 | try: 127 | header = { 128 | 'User-Agent': str(self.ua.google) 129 | } 130 | if not validators.url(url): 131 | self.print_yellow( 132 | " [!] Invalid URL Requested: %s" % (str(url))) 133 | return {}, False 134 | r = requests.get(url, headers=header) 135 | if r.status_code != return_code: 136 | self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" % 137 | (str(r.status_code), str(return_code))) 138 | return {}, False 139 | return r.text, True 140 | except requests.ConnectTimeout as e: 141 | self.print_red( 142 | " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e))) 143 | return {}, False 144 | except requests.TooManyRedirects as e: 145 | self.print_red( 146 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 147 | return {}, False 148 | except requests.HTTPError as e: 149 | self.print_red( 150 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 151 | return {}, False 152 | except ConnectionError as e: 153 | self.print_red( 154 | " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e))) 155 | return {}, False 156 | except Exception as e: 157 | self.print_red( 158 | " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e))) 159 | return {}, False 160 | 161 | def request_raw(self, url, return_code=200): 162 | """ 163 | Request content and expected return code. 164 | :param url: URL to request 165 | :param return_code: expected return code or raise error 166 | :return: JSON, SuccessState 167 | """ 168 | try: 169 | header = { 170 | 'User-Agent': str(self.ua.google) 171 | } 172 | if not validators.url(url): 173 | self.print_yellow( 174 | " [!] Invalid URL Requested: %s" % (str(url))) 175 | return {}, False 176 | r = requests.get(url, headers=header) 177 | if r.status_code != return_code: 178 | self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" % 179 | (str(r.status_code), str(return_code))) 180 | return {}, False 181 | return r, True 182 | except requests.ConnectTimeout as e: 183 | self.print_red( 184 | " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e))) 185 | return {}, False 186 | except requests.TooManyRedirects as e: 187 | self.print_red( 188 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 189 | return {}, False 190 | except requests.HTTPError as e: 191 | self.print_red( 192 | " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e))) 193 | return {}, False 194 | except ConnectionError as e: 195 | self.print_red( 196 | " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e))) 197 | return {}, False 198 | except Exception as e: 199 | self.print_red( 200 | " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e))) 201 | return {}, False 202 | -------------------------------------------------------------------------------- /simplydomain/src/module_loader.py: -------------------------------------------------------------------------------- 1 | # Loads modules into the system 2 | import os 3 | import glob 4 | import warnings 5 | import importlib 6 | 7 | 8 | class LoadModules(object): 9 | """ 10 | Loads modules from the "modules" directory allowing 11 | operators to drop modules into the folder. 12 | """ 13 | 14 | def __init__(self): 15 | """ 16 | Create objects used in class. 17 | """ 18 | self.modules = {} 19 | self.dmodules = {} 20 | self.static_modules = {} 21 | self.static_dmodules = {} 22 | self.load_dynamic_modules() 23 | self.load_static_modules() 24 | 25 | def load_dynamic_modules(self): 26 | """ 27 | Loads modules into static class variables, 28 | these can than be referenced easily. 29 | :return: NONE 30 | """ 31 | # loop and assign key and name 32 | current_script_patch = os.path.dirname(os.path.abspath(__file__)) 33 | warnings.filterwarnings('ignore', '.*Parent module*', ) 34 | x = 1 35 | path = os.path.join('dynamic_modules', '*.py') 36 | path = current_script_patch + "/" + path 37 | for name in glob.glob(path): 38 | if name.endswith(".py") and ("__init__" not in name) \ 39 | and ("module_template" not in name): 40 | quick_path = os.path.join( 41 | 'simplydomain', 'src', 'dynamic_modules', name.split('/')[-1]) 42 | module_name = quick_path.replace("/", ".").rstrip('.py') 43 | loaded_modules = self.dynamic_import(module_name) 44 | self.modules[quick_path] = loaded_modules 45 | self.dmodules[x] = loaded_modules 46 | x += 1 47 | 48 | def load_static_modules(self): 49 | """ 50 | Loads modules into static class variables, 51 | these can than be referenced easily. 52 | :return: NONE 53 | """ 54 | # loop and assign key and name 55 | current_script_patch = os.path.dirname(os.path.abspath(__file__)) 56 | warnings.filterwarnings('ignore', '.*Parent module*', ) 57 | x = 1 58 | path = os.path.join('static_modules', '*.py') 59 | path = current_script_patch + "/" + path 60 | for name in glob.glob(path): 61 | if name.endswith(".py") and ("__init__" not in name) \ 62 | and ("module_template" not in name): 63 | quick_path = os.path.join( 64 | 'simplydomain', 'src', 'static_modules', name.split('/')[-1]) 65 | module_name = quick_path.replace("/", ".").rstrip('.py') 66 | loaded_modules = self.dynamic_import(module_name) 67 | self.static_modules[quick_path] = loaded_modules 68 | self.static_dmodules[x] = loaded_modules 69 | x += 1 70 | 71 | def dynamic_import(self, module): 72 | """ 73 | Loads a module at runtime. 74 | :param module: module str (name) 75 | :return: module obj 76 | """ 77 | return importlib.import_module(module) 78 | -------------------------------------------------------------------------------- /simplydomain/src/module_multiprocessing.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | 3 | 4 | class ModuleMultiProcessing(object): 5 | 6 | """ 7 | A multiprocessing class to handle execution for all modules 8 | requiring multiprocessing. Use module* before all items to we dont 9 | crush other mp classes. 10 | """ 11 | 12 | def __init__(self): 13 | """ 14 | Create objects used in class. 15 | """ 16 | self.module_procs = [] 17 | self.module_threads = [] 18 | self.module_processors = mp.cpu_count() 19 | self.module_mp = mp 20 | 21 | def _module_configure_mp(self): 22 | """ 23 | Sets the configuration props for 24 | mp to handle taskings. 25 | :return: NONE 26 | """ 27 | # use SPAWN since FORK is not supported on windows 28 | # SPAWN is slower since it creates a entire python 29 | # interpter 30 | self.module_mp.set_start_method('spawn') 31 | 32 | def module_join_processes(self): 33 | """ 34 | Attempt to join all the process to clean 35 | them up. 36 | :return: NONE 37 | """ 38 | for p in self.module_procs: 39 | p.join() 40 | 41 | def modue_list_processes(self): 42 | """ 43 | List all procs and pids. 44 | :return: NONE 45 | """ 46 | for p in self.module_procs: 47 | pid = p.pid 48 | p_name = p.name 49 | print("[!] Process info: (PID: %s) (NAME: %s)" % 50 | (str(pid), str(p_name))) 51 | 52 | def module_check_active(self): 53 | """ 54 | Check if mp is has active pids 55 | :return: BOOL 56 | """ 57 | if len(self.module_mp.active_children()): 58 | return True 59 | else: 60 | return 61 | 62 | def module_start_processes(self): 63 | """ 64 | Executes all procs with a given module and 65 | passes it the proper objects to communicate with 66 | the core run time. 67 | 68 | :param module_obj: Module Ooject 69 | :param queues: A list of queue objects 70 | :return: BOOL 71 | """ 72 | for _ in range(self.module_processors): 73 | self.module_start_process() 74 | for p in self.module_procs: 75 | p.daemon = True 76 | p.start() 77 | 78 | def module_start_process(self, function_pointer, queue_list): 79 | """ 80 | Executes a proc with a given module and 81 | passes it the proper objects to communicate with 82 | the core run time. 83 | 84 | :param function_pointer: Module Object 85 | :return: BOOL 86 | """ 87 | self.module_procs.append( 88 | self.module_mp.Process(target=function_pointer, args=(queue_list,))) 89 | -------------------------------------------------------------------------------- /simplydomain/src/module_provider.py: -------------------------------------------------------------------------------- 1 | PROVIDER_LIST = [ 2 | { 3 | "name":"amazonaws s3", 4 | "cname":["amazonaws.com", "s3"], 5 | "response":["NoSuchBucket", "The specified bucket does not exist"] 6 | }, 7 | { 8 | "name":"heroku", 9 | "cname":["herokudns.com"], 10 | "response":["There's nothing here, yet.", "herokucdn.com/error-pages/no-such-app.html", "No such app"] 11 | }, 12 | { 13 | "name":"amazonaws elastic beanstalk", 14 | "cname":["elasticbeanstalk.com"], 15 | "response":[] 16 | } 17 | ] -------------------------------------------------------------------------------- /simplydomain/src/module_recursion.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | 3 | 4 | class ModuleRecursion(object): 5 | """Class to handle recursion. 6 | 7 | Simple class to handle tracking and storing prior 8 | sub-domains discovred. 9 | """ 10 | 11 | def __init__(self): 12 | """class init. 13 | """ 14 | self.recursion_queue = mp.Queue() 15 | 16 | def add_subdomain(self, domain): 17 | """add subdomain to Q. 18 | 19 | uses a non-blocking call to add to the Q 20 | to prevent any errors with size. 21 | 22 | Arguments: 23 | domain {str} -- subdomain to add to Q 24 | """ 25 | self.recursion_queue.put(domain) 26 | 27 | def get_subdomain_list(self, valid_only=True): 28 | """build subdomain list. 29 | 30 | Using the JSON from the event consumer, we 31 | can easily build a unique list of 32 | subdomains for module use. 33 | 34 | Keyword Arguments: 35 | valid_only {bool} -- filter only valid subdomains (default: {True}) 36 | 37 | Returns: 38 | list -- list of raw subdomains 39 | """ 40 | data = [] 41 | refill = [] 42 | while True: 43 | try: 44 | x = self.recursion_queue.get_nowait() 45 | if valid_only and x.valid: 46 | data.append(x.subdomain) 47 | if not valid_only: 48 | data.append(x.subdomain) 49 | except Exception as e: 50 | print(e) 51 | break 52 | return set(data) -------------------------------------------------------------------------------- /simplydomain/src/module_resolvers.py: -------------------------------------------------------------------------------- 1 | from fake_useragent import UserAgent 2 | import json 3 | 4 | from . import module_helpers 5 | 6 | 7 | class DnsServers(module_helpers.RequestsHelpers): 8 | 9 | """ 10 | A set of functions to find resolvers to use 11 | of high quality. 12 | """ 13 | 14 | def __init__(self): 15 | """ 16 | Init class structure. 17 | """ 18 | module_helpers.RequestsHelpers.__init__(self) 19 | self.ua = UserAgent() 20 | self.nameservers = [] 21 | self.nameserver_ips = [] 22 | 23 | def populate_servers(self): 24 | """ 25 | Populate server list. 26 | :return: NONE 27 | """ 28 | data, status = self.request_json( 29 | 'https://public-dns.info/nameserver/us.json') 30 | if status: 31 | data = json.loads(data) 32 | for d in data: 33 | self.nameservers.append(d) 34 | self.clean_servers() 35 | 36 | def populate_config(self, json_config): 37 | """ 38 | Populate the json config file at runtime with 39 | public dns servers. 40 | :param json_config: start JSON config 41 | :return: json_config: final JSON config 42 | """ 43 | json_config['resolvers'] = self.nameserver_ips[0:10] 44 | return json_config 45 | 46 | def clean_servers(self): 47 | """ 48 | Sort name servers. 49 | :return: NONE 50 | """ 51 | for i in self.nameservers: 52 | # check for 100% reliability 53 | if i['reliability'] == 1: 54 | self.nameserver_ips.append(i['ip']) 55 | 56 | def count_resolvers(self): 57 | """ 58 | Count resolvers. 59 | :return: INT nameserver list count 60 | """ 61 | return len(self.nameserver_ips) 62 | -------------------------------------------------------------------------------- /simplydomain/src/static_modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/src/static_modules/__init__.py -------------------------------------------------------------------------------- /simplydomain/src/static_modules/subdomain_bruteforce.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import queue 4 | import asyncio 5 | import aiodns 6 | import functools 7 | # disable uvloop until supports windows 8 | # import uvloop 9 | import socket 10 | import click 11 | import time 12 | import dns.resolver 13 | from tqdm import tqdm 14 | 15 | from simplydomain.src import core_serialization 16 | from simplydomain.src import core_scrub 17 | from simplydomain.src import module_helpers 18 | from multiprocessing import Process 19 | 20 | 21 | # use RequestsHelpers() class to make requests to target URL 22 | 23 | 24 | class DynamicModule(module_helpers.RequestsHelpers): 25 | """ 26 | Dynamic module class that will be loaded and called 27 | at runtime. This will allow modules to easily be independent of the 28 | core runtime. 29 | """ 30 | 31 | def __init__(self, json_entry): 32 | """ 33 | Init class structure. Each module takes a JSON entry object which 34 | can pass different values to the module with out changing up the API. 35 | adapted form Empire Project: 36 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 37 | 38 | :param json_entry: JSON data object passed to the module. 39 | """ 40 | module_helpers.RequestsHelpers.__init__(self) 41 | self.json_entry = json_entry 42 | self.info = { 43 | # mod name 44 | 'Module': 'subdomain_bruteforce.py', 45 | 46 | # long name of the module to be used 47 | 'Name': 'Recursive Subdomain Bruteforce Using Wordlist', 48 | 49 | # version of the module to be used 50 | 'Version': '1.0', 51 | 52 | # description 53 | 'Description': ['Uses lists from dnspop', 54 | 'with high quality dns resolvers.'], 55 | 56 | # authors or sources to be quoted 57 | 'Authors': ['@Killswitch-GUI', '@blark'], 58 | 59 | # list of resources or comments 60 | 'comments': [ 61 | 'Searches and performs recursive dns-lookup.', 62 | ' adapted from https://github.com/blark/aiodnsbrute/blob/master/aiodnsbrute/cli.py' 63 | ], 64 | # priority of module (0) being first to execute 65 | 'priority': 0 66 | } 67 | 68 | self.options = { 69 | } 70 | # ~ queue object 71 | self.word_count = int(self.json_entry['args'].wordlist_count) 72 | self.word_list_queue = queue.Queue(maxsize=0) 73 | self.tasks = [] 74 | self.domain = '' 75 | self.errors = [] 76 | self.fqdn = [] 77 | self.runtime_queue = [] 78 | # disable uvloop until supports windows 79 | # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 80 | self.loop = asyncio.get_event_loop() 81 | self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True) 82 | # TODO: make max tasks defined in config.json 83 | self.max_tasks = 500 84 | # TODO: make total set from wordcount in config.json 85 | self.sem = asyncio.BoundedSemaphore(self.max_tasks) 86 | self.cs = core_scrub.Scrub() 87 | self.core_args = self.json_entry['args'] 88 | self.silent = self.json_entry['silent'] 89 | 90 | def dynamic_main(self, queue_dict): 91 | """ 92 | Main entry point for process to call. 93 | 94 | core_serialization.SubDomain Attributes: 95 | name: long name of method 96 | module_name: name of the module that performed collection 97 | source: source of the subdomain or resource of collection 98 | module_version: version from meta 99 | source: source of the collection 100 | time: time the result obj was built 101 | subdomain: subdomain to use 102 | valid: is domain valid 103 | 104 | :return: NONE 105 | """ 106 | self.task_output_queue = queue_dict['task_output_queue'] 107 | self.subdomain_list = queue_dict['subdomain_list'] 108 | self.domain = str(self.core_args.DOMAIN) 109 | self._execute_resolve(self.domain) 110 | 111 | async def _process_dns_wordlist(self, domain): 112 | """ 113 | Populates word list queue with words to brute 114 | force a domain with. 115 | :return: NONE 116 | """ 117 | h = module_helpers.RequestsHelpers() 118 | content, ret = h.request_text( 119 | 'https://github.com/bitquark/dnspop/raw/master/results/bitquark_20160227_subdomains_popular_1000000') 120 | if not ret: 121 | raise Exception( 122 | 'Failed to request bitquark top_1000000 sub domains.') 123 | # file_path = os.path.join(*self.json_entry['subdomain_bruteforce']['top_1000000']) 124 | # fancy iter so we can pull out only (N) lines 125 | sub_doamins = content.split() 126 | #sub_doamins = [next(content).strip() for x in range(self.word_count)] 127 | for word in sub_doamins[0:self.word_count]: 128 | # Wait on the semaphore before adding more tasks 129 | await self.sem.acquire() 130 | host = '{}.{}'.format(word.strip(), domain) 131 | task = asyncio.ensure_future(self._dns_lookup(host)) 132 | task.add_done_callback(functools.partial( 133 | self._dns_result_callback, host)) 134 | self.tasks.append(task) 135 | await asyncio.gather(*self.tasks, return_exceptions=True) 136 | 137 | def _select_random_resolver(self): 138 | """ 139 | Select a random resolver from the JSON config, allows 140 | for procs to easily obtain a IP. 141 | :return: STR: ip 142 | """ 143 | ip = random.choice(self.json_entry['resolvers']) 144 | return ip 145 | 146 | def _runtime_loop(self): 147 | """runtime time that allows recursive search 148 | """ 149 | try: 150 | for x in self.runtime_queue: 151 | # the inital loop branch to control 152 | # the runtime of a search 153 | self._execute_resolve(x, exit=True) 154 | self.runtime_queue.remove(x) 155 | if self.runtime_queue: 156 | # only hit this branch if more values have 157 | # been added to the queue 158 | self._runtime_loop() 159 | else: 160 | # runtime completed 161 | return 162 | except KeyboardInterrupt: 163 | print('Interrupted') 164 | try: 165 | self.logger("Caught keyboard interrupt, cleaning up...") 166 | asyncio.gather(*asyncio.Task.all_tasks()).cancel() 167 | self.loop.stop() 168 | except SystemExit: 169 | os._exit(0) 170 | 171 | def _execute_resolve(self, domain, recursive=True, exit=False): 172 | """ 173 | Execs a single thread based / adapted from: 174 | https://github.com/blark/aiodnsbrute/blob/master/aiodnsbrute/cli.py 175 | :return: NONE 176 | """ 177 | try: 178 | _valid = True 179 | self.logger("Brute forcing {} with a maximum of {} concurrent tasks...".format( 180 | domain, self.max_tasks)) 181 | self.logger( 182 | "Wordlist loaded, brute forcing {} DNS records".format(self.word_count)) 183 | if self._check_wildcard_domain(domain): 184 | self.logger("Core domain: {} is a wildcard DNS A record domain! Check yourself..".format(domain), msg_type='err') 185 | self.logger("Moving striaght to past subdomains", msg_type='err') 186 | # TODO: enable verbose 187 | if not self.silent: 188 | self.pbar = tqdm(total=self.word_count, 189 | unit="records", maxinterval=0.1, mininterval=0) 190 | if recursive: 191 | self.resolver.nameservers = self.json_entry['resolvers'] 192 | self.logger("Using recursive DNS with the following servers: {}".format( 193 | len(self.resolver.nameservers))) 194 | else: 195 | domain_ns = self.loop.run_until_complete( 196 | self._dns_lookup(domain, 'NS')) 197 | self.logger( 198 | "Setting nameservers to {} domain NS servers: {}".format(domain, [host.host for host in domain_ns])) 199 | self.resolver.nameservers = [ 200 | socket.gethostbyname(host.host) for host in domain_ns] 201 | #self.resolver.nameservers = self.core_resolvers 202 | self.loop.run_until_complete(self._process_dns_wordlist(domain)) 203 | except KeyboardInterrupt: 204 | self.logger("Caught keyboard interrupt, cleaning up...") 205 | asyncio.gather(*asyncio.Task.all_tasks()).cancel() 206 | self.loop.stop() 207 | finally: 208 | # 209 | # TODO: enable verbose 210 | if not self.silent: 211 | # 212 | self.pbar.close() 213 | self.logger( 214 | "completed, {} subdomains found.".format(len(self.fqdn))) 215 | if exit: 216 | # to prevent runtime loops from spawning 217 | return 218 | for x in self.subdomain_list: 219 | if not self._check_wildcard_domain(x): 220 | self.runtime_queue.append(x) 221 | if self.runtime_queue: 222 | self._runtime_loop() 223 | self.loop.close() 224 | self.pbar.close() 225 | return self.fqdn 226 | 227 | def logger(self, msg, msg_type='info'): 228 | """A quick and dirty msfconsole style stdout logger. 229 | 230 | Uses style decorator to print a console message that is tqdm safe and 231 | prevents the console bars from being altered. Only prints WHEN 232 | no silent is enabled. 233 | 234 | Arguments: 235 | msg {str} -- message you want to print 236 | 237 | Keyword Arguments: 238 | msg_type {str} -- sets color / style setting (default: {'info'}) 239 | """ 240 | if not self.silent: 241 | style = {'info': ('[*]', 'blue'), 'pos': ('[+]', 'green'), 'err': ('[-]', 'red'), 242 | 'warn': ('[!]', 'yellow'), 'dbg': ('[D]', 'cyan')} 243 | if msg_type is not 0: 244 | decorator = click.style('{}'.format( 245 | style[msg_type][0]), fg=style[msg_type][1], bold=True) 246 | else: 247 | decorator = '' 248 | m = " {} {}".format(decorator, msg) 249 | tqdm.write(m) 250 | 251 | async def _dns_lookup(self, name, _type='A'): 252 | """Performs a DNS request using aiodns, returns an asyncio future.""" 253 | response = await self.resolver.query(name, _type) 254 | return response 255 | 256 | def _check_wildcard_domain(sef, name, _type='A'): 257 | """check if domain A record is *. 258 | 259 | [description] 260 | 261 | Arguments: 262 | sef {[type]} -- [description] 263 | name {[type]} -- [description] 264 | 265 | Keyword Arguments: 266 | _type {str} -- [description] (default: {'A'}) 267 | 268 | Returns: 269 | bool -- is domain wildcard domain A record. 270 | """ 271 | x = "*.{}".format(name) 272 | try: 273 | answers = dns.resolver.query(x, _type) 274 | if answers: 275 | return True 276 | except Exception as e: 277 | pass 278 | return False 279 | 280 | def _dns_result_callback(self, name, future): 281 | """Handles the result passed by the _dns_lookup function.""" 282 | # Record processed we can now release the lock 283 | self.sem.release() 284 | # Handle known exceptions, barf on other ones 285 | if future.exception() is not None: 286 | try: 287 | err_num = future.exception().args[0] 288 | err_text = future.exception().args[1] 289 | except IndexError: 290 | self.logger("Couldn't parse exception: {}".format( 291 | future.exception()), 'err') 292 | if (err_num == 4): # This is domain name not found, ignore it. 293 | pass 294 | elif (err_num == 12): # Timeout from DNS server 295 | self.logger("Timeout for {}".format(name), 'warn', 2) 296 | elif (err_num == 1): # Server answered with no data 297 | pass 298 | else: 299 | self.logger('{} generated an unexpected exception: {}'.format( 300 | name, future.exception()), 'err') 301 | self.errors.append({'hostname': name, 'error': err_text}) 302 | # Output result 303 | else: 304 | self.cs.subdomain = name 305 | # check if domain name is valid 306 | valid = self.cs.validate_domain() 307 | # build the SubDomain Object to pass 308 | sub_obj = core_serialization.SubDomain( 309 | self.info["Name"], 310 | self.info["Module"], 311 | "", 312 | self.info["Version"], 313 | time.time(), 314 | name, 315 | valid 316 | ) 317 | if valid: 318 | if not self._check_wildcard_domain(name): 319 | self.runtime_queue.append(name) 320 | self.task_output_queue.put(sub_obj) 321 | ip = ', '.join([ip.host for ip in future.result()]) 322 | self.fqdn.append((name, ip)) 323 | # self.logger("{:<30}\t{}".format(name, ip), 'pos') 324 | # self.logger(future.result(), 'dbg', 3) 325 | self.tasks.remove(future) 326 | # TODO: enable verbose 327 | if not self.silent: 328 | self.pbar.update() 329 | -------------------------------------------------------------------------------- /simplydomain/src/static_modules/subdomain_raw_bruteforce.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import queue 4 | import asyncio 5 | import aiodns 6 | import functools 7 | # disable uvloop until supports windows 8 | # import uvloop 9 | import socket 10 | import click 11 | import time 12 | import string 13 | from tqdm import tqdm 14 | from itertools import product 15 | 16 | from simplydomain.src import core_serialization 17 | from simplydomain.src import module_helpers 18 | from simplydomain.src import core_scrub 19 | 20 | 21 | # use RequestsHelpers() class to make requests to target URL 22 | class DynamicModule(module_helpers.RequestsHelpers): 23 | """ 24 | Dynamic module class that will be loaded and called 25 | at runtime. This will allow modules to easily be independent of the 26 | core runtime. 27 | """ 28 | 29 | def __init__(self, json_entry): 30 | """ 31 | Init class structure. Each module takes a JSON entry object which 32 | can pass different values to the module with out changing up the API. 33 | adapted form Empire Project: 34 | https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py 35 | 36 | :param json_entry: JSON data object passed to the module. 37 | """ 38 | module_helpers.RequestsHelpers.__init__(self) 39 | self.json_entry = json_entry 40 | self.info = { 41 | # mod name 42 | 'Module': 'subdomain_bruteforce.py', 43 | 44 | # long name of the module to be used 45 | 'Name': 'Recursive Subdomain Bruteforce Using Wordlist', 46 | 47 | # version of the module to be used 48 | 'Version': '1.0', 49 | 50 | # description 51 | 'Description': ['Uses lists from dnspop', 52 | 'with high quality dns resolvers.'], 53 | 54 | # authors or sources to be quoted 55 | 'Authors': ['@Killswitch-GUI', '@blark'], 56 | 57 | # list of resources or comments 58 | 'comments': [ 59 | 'Searches and performs recursive dns-lookup.', 60 | ' adapted from https://github.com/blark/aiodnsbrute/blob/master/aiodnsbrute/cli.py' 61 | ], 62 | # priority of module (0) being first to execute 63 | 'priority': 0 64 | } 65 | 66 | self.options = { 67 | } 68 | # ~ queue object 69 | self.word_count = int(self.json_entry['args'].raw_depth) 70 | self.word_list_queue = queue.Queue(maxsize=0) 71 | self.tasks = [] 72 | self.domain = '' 73 | self.errors = [] 74 | self.fqdn = [] 75 | self.sub_gen_count = 0 76 | # disable uvloop until supports windows 77 | # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 78 | self.loop = asyncio.get_event_loop() 79 | self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True) 80 | # TODO: make max tasks defined in config.json 81 | self.max_tasks = 1024 82 | # TODO: make total set from wordcount in config.json 83 | self.sem = asyncio.BoundedSemaphore(self.max_tasks) 84 | self.cs = core_scrub.Scrub() 85 | self.core_args = self.json_entry['args'] 86 | self.core_resolvers = self.json_entry['resolvers'] 87 | self.silent = self.json_entry['silent'] 88 | 89 | def dynamic_main(self, queue_dict): 90 | """ 91 | Main entry point for process to call. 92 | 93 | core_serialization.SubDomain Attributes: 94 | name: long name of method 95 | module_name: name of the module that performed collection 96 | source: source of the subdomain or resource of collection 97 | module_version: version from meta 98 | source: source of the collection 99 | time: time the result obj was built 100 | subdomain: subdomain to use 101 | valid: is domain valid 102 | 103 | :return: NONE 104 | """ 105 | self.task_output_queue = queue_dict['task_output_queue'] 106 | self.domain = str(self.core_args.DOMAIN) 107 | self._execute_resolve() 108 | 109 | async def _process_dns_wordlist(self): 110 | """ 111 | Populates word list queue with words to brute 112 | force a domain with. 113 | :return: NONE 114 | """ 115 | # fancy iter so we can pull out only (N) lines 116 | sub_doamins = [] 117 | for x in range(1, self.word_count+1, 1): 118 | sub_doamins += [''.join(i) for i in product( 119 | string.ascii_lowercase + string.digits, repeat=x)] 120 | self.sub_gen_count = len(sub_doamins) 121 | # move pbar here to ensure we have proper count at start 122 | if not self.silent: 123 | self.pbar = tqdm(total=self.sub_gen_count, 124 | unit="records", maxinterval=0.1, mininterval=0) 125 | for word in sub_doamins: 126 | # Wait on the semaphore before adding more tasks 127 | await self.sem.acquire() 128 | host = '{}.{}'.format(word.strip(), self.domain) 129 | task = asyncio.ensure_future(self._dns_lookup(host)) 130 | task.add_done_callback(functools.partial( 131 | self._dns_result_callback, host)) 132 | self.tasks.append(task) 133 | await asyncio.gather(*self.tasks, return_exceptions=True) 134 | 135 | def _select_random_resolver(self): 136 | """ 137 | Select a random resolver from the JSON config, allows 138 | for procs to easily obtain a IP. 139 | :return: STR: ip 140 | """ 141 | ip = random.choice(self.json_entry['resolvers']) 142 | return ip 143 | 144 | def _execute_resolve(self, recursive=True): 145 | """ 146 | Execs a single thread based / adapted from: 147 | https://github.com/blark/aiodnsbrute/blob/master/aiodnsbrute/cli.py 148 | :return: NONE 149 | """ 150 | try: 151 | self.logger("Brute forcing {} with a maximum of {} concurrent tasks...".format( 152 | self.domain, self.max_tasks)) 153 | self.logger("Wordlist loaded, brute forcing {} DNS records".format( 154 | self.sub_gen_count)) 155 | # TODO: enable verbose 156 | if recursive: 157 | self.logger("Using recursive DNS with the following servers: {}".format( 158 | self.resolver.nameservers)) 159 | else: 160 | domain_ns = self.loop.run_until_complete( 161 | self._dns_lookup(self.domain, 'NS')) 162 | self.logger( 163 | "Setting nameservers to {} domain NS servers: {}".format(self.domain, [host.host for host in domain_ns])) 164 | self.resolver.nameservers = [ 165 | socket.gethostbyname(host.host) for host in domain_ns] 166 | #self.resolver.nameservers = self.core_resolvers 167 | self.loop.run_until_complete(self._process_dns_wordlist()) 168 | except KeyboardInterrupt: 169 | self.logger("Caught keyboard interrupt, cleaning up...") 170 | asyncio.gather(*asyncio.Task.all_tasks()).cancel() 171 | self.loop.stop() 172 | finally: 173 | self.loop.close() 174 | # TODO: enable verbose 175 | if not self.silent: 176 | self.pbar.close() 177 | self.logger( 178 | "completed, {} subdomains found.".format(len(self.fqdn))) 179 | return self.fqdn 180 | 181 | def logger(self, msg, msg_type='info', level=0): 182 | """A quick and dirty msfconsole style stdout logger.""" 183 | # TODO: enable verbose 184 | if not self.silent: 185 | style = {'info': ('[*]', 'blue'), 'pos': ('[+]', 'green'), 'err': ('[-]', 'red'), 186 | 'warn': ('[!]', 'yellow'), 'dbg': ('[D]', 'cyan')} 187 | if msg_type is not 0: 188 | decorator = click.style('{}'.format( 189 | style[msg_type][0]), fg=style[msg_type][1], bold=True) 190 | else: 191 | decorator = '' 192 | m = " {} {}".format(decorator, msg) 193 | tqdm.write(m) 194 | 195 | async def _dns_lookup(self, name, _type='A'): 196 | """Performs a DNS request using aiodns, returns an asyncio future.""" 197 | response = await self.resolver.query(name, _type) 198 | return response 199 | 200 | def _dns_result_callback(self, name, future): 201 | """Handles the result passed by the _dns_lookup function.""" 202 | # Record processed we can now release the lock 203 | self.sem.release() 204 | # Handle known exceptions, barf on other ones 205 | if future.exception() is not None: 206 | try: 207 | err_num = future.exception().args[0] 208 | err_text = future.exception().args[1] 209 | except IndexError: 210 | self.logger("Couldn't parse exception: {}".format( 211 | future.exception()), 'err') 212 | if (err_num == 4): # This is domain name not found, ignore it. 213 | pass 214 | elif (err_num == 12): # Timeout from DNS server 215 | self.logger("Timeout for {}".format(name), 'warn', 2) 216 | elif (err_num == 1): # Server answered with no data 217 | pass 218 | else: 219 | self.logger('{} generated an unexpected exception: {}'.format( 220 | name, future.exception()), 'err') 221 | self.errors.append({'hostname': name, 'error': err_text}) 222 | # Output result 223 | else: 224 | self.cs.subdomain = name 225 | # check if domain name is valid 226 | valid = self.cs.validate_domain() 227 | # build the SubDomain Object to pass 228 | sub_obj = core_serialization.SubDomain( 229 | self.info["Name"], 230 | self.info["Module"], 231 | "", 232 | self.info["Version"], 233 | time.time(), 234 | name, 235 | valid 236 | ) 237 | self.task_output_queue.put(sub_obj) 238 | ip = ', '.join([ip.host for ip in future.result()]) 239 | self.fqdn.append((name, ip)) 240 | # self.logger("{:<30}\t{}".format(name, ip), 'pos') 241 | # self.logger(future.result(), 'dbg', 3) 242 | self.tasks.remove(future) 243 | # TODO: enable verbose 244 | if not self.silent: 245 | self.pbar.update() 246 | -------------------------------------------------------------------------------- /simplydomain/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimplySecurity/simplydomain/101dd55b213009b449a96a1fa8b143d85dcdba88/simplydomain/tests/__init__.py -------------------------------------------------------------------------------- /simplydomain/tests/test_corePrinters.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from src import core_printer 3 | 4 | 5 | class TestCorePrinters(TestCase): 6 | p = core_printer.CorePrinters() 7 | 8 | def test_blue_text(self): 9 | msg1 = self.p.blue_text("test") 10 | msg2 = "\x1b[34m [*] \x1b[0mtest" 11 | self.assertEqual(msg1, msg2) 12 | 13 | def test_green_text(self): 14 | msg1 = self.p.green_text("test") 15 | msg2 = "\x1b[32m [+] \x1b[0mtest" 16 | self.assertEqual(msg1, msg2) 17 | 18 | def test_print_entry(self): 19 | self.p.print_entry() 20 | 21 | def test_print_d_module_start(self): 22 | self.p.print_d_module_start() 23 | 24 | def test_print_s_module_start(self): 25 | self.p.print_s_module_start() 26 | 27 | def test_print_config_start(self): 28 | self.p.print_config_start() 29 | 30 | def test_print_modules(self): 31 | self.p.print_modules(['modules/bing_search.py']) 32 | -------------------------------------------------------------------------------- /simplydomain/tests/test_dnsServers.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from src import module_resolvers 3 | 4 | 5 | class TestDnsServers(TestCase): 6 | 7 | m = module_resolvers.DnsServers() 8 | m.populate_servers() 9 | 10 | def test_populate_servers(self): 11 | a = self.m.nameserver_ips 12 | if '8.8.8.8' not in a: 13 | self.fail("missing google resolver in list!") 14 | 15 | def test_populate_config(self): 16 | j = {} 17 | j = self.m.populate_config(j) 18 | if '8.8.8.8' not in j['resolvers']: 19 | self.fail('resolver json missing google') 20 | 21 | def test_clean_servers(self): 22 | self.m.clean_servers() 23 | if not self.m.nameserver_ips: 24 | self.fail() 25 | 26 | def test_count_resolvers(self): 27 | if self.m.count_resolvers() < 1: 28 | self.fail('no resolvers populated') 29 | -------------------------------------------------------------------------------- /simplydomain/tests/test_requestsHelpers.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from src import module_helpers 3 | import json 4 | 5 | 6 | class TestRequestsHelpers(TestCase): 7 | 8 | h = module_helpers.RequestsHelpers() 9 | 10 | def test_request_json(self): 11 | msg, val = self.h.request_json( 12 | 'http://echo.jsontest.com/key1/key-val/key2/key-val2', return_code=200) 13 | if not val: 14 | self.fail() 15 | j = json.loads(msg) 16 | self.assertDictEqual({'key2': 'key-val2', 'key1': 'key-val'}, j) 17 | 18 | def test_request_content(self): 19 | msg, val = self.h.request_content('https://httpbin.org/ip') 20 | if not val: 21 | self.fail() 22 | j = json.loads(msg) 23 | 24 | def test_request_raw(self): 25 | r, val = self.h.request_raw('https://httpbin.org/ip') 26 | if not val: 27 | self.fail() 28 | j = json.loads(r.content) 29 | --------------------------------------------------------------------------------