├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── application.py ├── docker-compose.yaml ├── env.example ├── init-mongo.js ├── models.py ├── requirements.txt ├── routes ├── domain.py ├── index.py ├── subdomains.py └── utils.py ├── run.py └── templates └── index.html /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | Data/ 6 | .DS_Store 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # pdm 106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 107 | #pdm.lock 108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 109 | # in version control. 110 | # https://pdm.fming.dev/#use-with-ide 111 | .pdm.toml 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | .env 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | #.idea/ 162 | 163 | 164 | # Docker volumes 165 | data/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile 2 | FROM python:3.10-slim 3 | 4 | WORKDIR /app 5 | RUN mkdir -p /app/files 6 | 7 | COPY requirements.txt requirements.txt 8 | RUN pip install -r requirements.txt 9 | 10 | COPY . . 11 | 12 | CMD ["python", "./run.py"] 13 | 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Patrik 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vilicus 2 | 3 | ![Dashboard](https://github.com/PatrikFehrenbach/vilicus/assets/9072595/74e8e738-8f9b-40c5-8cf9-641ede1c279f) 4 | 5 | Vilicus (from Latin, meaning overseer or supervisor) is a Bug Bounty API Dashboard. This platform is designed to simplify the process of bug bounty hunting by aggregating data from various sources and presenting it in an easy-to-understand dashboard. 6 | 7 | ## Requirements: 8 | 9 | To get Vilicus up and running, you'll need the following: 10 | 11 | - Python3 12 | - Docker 13 | - Docker-compose 14 | 15 | ## Installation Steps: 16 | 17 | Follow these steps to install and run Vilicus: 18 | 19 | 1. Clone the Vilicus repository to your local machine. 20 | 21 | ``` 22 | git clone https://github.com/PatrikFehrenbach/vilicus.git 23 | cd vilicus 24 | ``` 25 | 26 | 2. Start the Docker services. 27 | 28 | ``` 29 | docker-compose up 30 | ``` 31 | 32 | Wait for Docker to pull the necessary images and start the services. This may take a while. 33 | 34 | 35 | This will start the server and the application will be accessible at `localhost:5000` (or whatever port you've configured). 36 | 37 | 3. Visit the dashboard in your web browser. 38 | 39 | ### Optional SecurityTrails integration 40 | 41 | The tool has the ability to automatically query the (https://securitytrails.com/) Securitytrails API once a domain has been added. If youwant too enable this feature, you have to rename the `env.example` to `.env` and insert your own API Key. It is also recommended to rebuild the container like so `docker-compose build --no-cache` 42 | 43 | Screenshot 2023-07-09 at 19 38 06 44 | 45 | 46 | ## Contributing: 47 | 48 | Contributions are always welcome. If you find a bug or want to add a new feature, feel free to create a new issue or open a pull request. 49 | 50 | ## License: 51 | 52 | This project is open-source and available under the [MIT License](https://github.com/PatrikFehrenbach/vilicus/blob/main/LICENSE). 53 | 54 | 55 | # Subdomain and Domain API 56 | 57 | ## Routes 58 | 59 | ### POST /add_domain 60 | 61 | Create a new domain. The request body should contain a JSON object with a "name" field. 62 | 63 | Request Body: 64 | 65 | ```{ "name": "domain name" }``` 66 | 67 | Responses: 68 | - 201: 'Domain added successfully!' 69 | - 400: 'No domain name provided' 70 | 71 | --- 72 | 73 | ### POST /update_domain/ 74 | 75 | Update the name of an existing domain. The request body should contain a JSON object with a "name" field. 76 | 77 | Request Body: 78 | 79 | ```{ "name": "new domain name" }``` 80 | 81 | Responses: 82 | - 200: 'Domain updated successfully!' 83 | - 400: 'No new domain name provided' 84 | - 404: 'Domain not found' 85 | 86 | --- 87 | 88 | ### POST /delete_domain/ 89 | 90 | Delete a specific domain by its name. 91 | 92 | Responses: 93 | - 200: 'Domain deleted successfully!' 94 | - 404: 'Domain not found' 95 | 96 | --- 97 | 98 | ### GET /domains/export 99 | 100 | Export all domains. 101 | 102 | Responses: 103 | 104 | - 200: List of all domains 105 | 106 | --- 107 | 108 | ### GET /domains/search?q=test 109 | 110 | Search domains by query. The query should be passed as a URL parameter. 111 | 112 | Responses: 113 | 114 | - 200: Search results 115 | 116 | --- 117 | 118 | ### POST /add_subdomain/ 119 | 120 | Create a new subdomain for a specific domain. The request body should contain a JSON object with a "subdomain_name" field. 121 | 122 | Request Body: 123 | 124 | ```{ "subdomain_name": "subdomain name" }``` 125 | 126 | Responses: 127 | - 201: 'Subdomain added successfully!' 128 | - 400: 'No subdomain name provided' 129 | - 404: 'Main domain not found' 130 | - 409: 'Conflict' 131 | 132 | --- 133 | 134 | ### POST /update_subdomain// 135 | 136 | Update the name of an existing subdomain for a specific domain. The request body should contain a JSON object with a "name" field. 137 | 138 | Request Body: 139 | 140 | ```{ "name": "new subdomain name" }``` 141 | 142 | Responses: 143 | - 200: 'Subdomain updated successfully!' 144 | - 400: 'No new subdomain name provided' 145 | - 404: 'Main domain not found' 146 | - 404: 'Subdomain not found' 147 | 148 | --- 149 | 150 | ### POST /delete_subdomain// 151 | 152 | Delete a specific subdomain for a specific domain. 153 | 154 | Responses: 155 | - 200: 'Subdomain deleted successfully!' 156 | - 404: 'Main domain not found' 157 | - 404: 'Subdomain not found' 158 | 159 | --- 160 | 161 | ### GET /subdomains/export 162 | 163 | Export all subdomains. 164 | 165 | Responses: 166 | 167 | - 200: List of all subdomains 168 | 169 | --- 170 | 171 | ### GET //subdomains/export 172 | 173 | Export all subdomains of a specific domain. 174 | 175 | Responses: 176 | - 200: List of all subdomains of the specified domain 177 | - 404: 'Domain not found' 178 | 179 | --- 180 | 181 | ### GET /subdomains/search?q=test 182 | 183 | Search subdomains by query. The query should be passed as a URL parameter. 184 | 185 | Responses: 186 | 187 | - 200: Search results 188 | 189 | --- 190 | 191 | ### GET /lastupdated 192 | 193 | Fetch all subdomains added in the last hour. 194 | 195 | Responses: 196 | 197 | - 200: List of all subdomains added in the last hour 198 | 199 | --- 200 | 201 | ### GET /sort_subdomains 202 | 203 | Fetch all domains sorted by the count of their subdomains in descending order. 204 | 205 | Responses: 206 | 207 | - 200: List of all domains sorted by subdomains count 208 | -------------------------------------------------------------------------------- /application.py: -------------------------------------------------------------------------------- 1 | # application.py 2 | 3 | from flask import Flask 4 | import os 5 | from flask_executor import Executor 6 | 7 | UPLOAD_FOLDER = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'files') 8 | ALLOWED_EXTENSIONS = {'txt'} 9 | executor = Executor() # make executor a global variable 10 | 11 | def allowed_file(filename): 12 | return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS 13 | 14 | def create_app(): 15 | app = Flask(__name__) 16 | executor.init_app(app) # initialize the global executor with your app 17 | app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER 18 | 19 | return app 20 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | mongodb: 4 | image: mongo:latest 5 | ports: 6 | - 27017:27017 7 | volumes: 8 | - ./data:/data/db 9 | - ./init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro 10 | environment: 11 | - MONGO_INITDB_ROOT_USERNAME=admin 12 | - MONGO_INITDB_ROOT_PASSWORD=admin123 13 | 14 | web: 15 | build: . 16 | ports: 17 | - 127.0.0.1:5000:5000 # Bind to localhost only 18 | depends_on: 19 | - mongodb 20 | environment: 21 | - FLASK_RUN_HOST=0.0.0.0 # Listen on all interfaces 22 | -------------------------------------------------------------------------------- /env.example: -------------------------------------------------------------------------------- 1 | SECURITYTRAILS="" -------------------------------------------------------------------------------- /init-mongo.js: -------------------------------------------------------------------------------- 1 | db = db.getSiblingDB('recon'); 2 | 3 | db.createCollection('domains'); 4 | db.createCollection('subdomains'); 5 | 6 | 7 | db.createUser({ 8 | user: 'admin', 9 | pwd: 'admin123', 10 | roles: [ 11 | { role: 'readWrite', db: 'recon' } 12 | ] 13 | }); 14 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | # models.py 2 | from pymongo import MongoClient,DESCENDING 3 | 4 | client = MongoClient('mongodb://admin:admin123@mongodb:27017/') 5 | db = client['recon'] 6 | domains_collection = db['domains'] 7 | subdomains_collection = db['subdomains'] 8 | from datetime import datetime 9 | 10 | # Create index 11 | subdomains_collection.create_index([("added_at", DESCENDING)]) 12 | subdomains_collection.create_index('domain_id') 13 | subdomains_collection.create_index([('domain_id', 1), ('added_at', -1)]) 14 | 15 | 16 | 17 | # models.py 18 | from bson.objectid import ObjectId 19 | 20 | class Domain: 21 | def __init__(self, name): 22 | self.name = name 23 | self.subdomains_count = 0 24 | self.id = None # The ID assigned by MongoDB. 25 | self.subdomains = [] # Define subdomains here 26 | self.last_updated = None # Define last_updated here 27 | 28 | def subdomain_count(self): 29 | return self.subdomains_count 30 | 31 | def get_all_domains(): 32 | domains = domains_collection.find({}) 33 | return domains 34 | 35 | def last_updated(self): 36 | # Get the most recent subdomain of this domain 37 | subdomain_data = subdomains_collection.find({'domain_id': self.id}).sort('added_at', -1).limit(1) 38 | 39 | subdomain_data = list(subdomain_data) 40 | 41 | # If there is a subdomain, return the added_at of it (the most recently added), 42 | # otherwise return None 43 | return subdomain_data[0]['added_at'] if subdomain_data else None 44 | 45 | 46 | def save(self): 47 | existing_domain = domains_collection.find_one({'name': self.name}) 48 | if existing_domain: 49 | return "Domain already exists" 50 | else: 51 | domain_data = {'name': self.name, 'subdomains': []} 52 | domain_id = domains_collection.insert_one(domain_data).inserted_id 53 | self.id = domain_id 54 | 55 | def update(self): 56 | existing_domain = domains_collection.find_one({'_id': ObjectId(self.id)}) 57 | if existing_domain: 58 | domains_collection.update_one({'_id': ObjectId(self.id)}, {'$set': {'name': self.name}}) 59 | return "Domain updated successfully" 60 | else: 61 | return "Domain not found" 62 | 63 | def delete(self): 64 | domains_collection.delete_one({'_id': ObjectId(self.id)}) 65 | subdomains_collection.delete_many({'domain_id': self.id}) 66 | 67 | def add_subdomain(self, subdomain): 68 | existing_subdomain = subdomains_collection.find_one({'name': subdomain, 'domain_id': self.id}) 69 | if not existing_subdomain: 70 | subdomain_obj = Subdomain(self.id, subdomain) 71 | subdomain_obj.save() 72 | self.subdomains_count += 1 # Increase the count. 73 | self.last_updated = datetime.utcnow() # Update last_updated with the current time 74 | domains_collection.update_one({'_id': ObjectId(self.id)}, {'$inc': {'subdomains_count': 1}, '$set': {'last_updated': self.last_updated}}) # Update the count and last_updated in the database. 75 | return "Subdomain added successfully" 76 | else: 77 | return "Subdomain already exists" 78 | 79 | 80 | def remove_subdomain(self, subdomain): 81 | subdomains_collection.delete_one({'name': subdomain, 'domain_id': self.id}) 82 | self.subdomains_count -= 1 # Decrease the count. 83 | domains_collection.update_one({'_id': ObjectId(self.id)}, {'$inc': {'subdomains_count': -1}}) # Update the count in the database. 84 | 85 | 86 | def get_subdomains(self): 87 | subdomains_data = subdomains_collection.find({'domain_id': self.id}) 88 | self.subdomains = [Subdomain(subdomain['domain_id'], subdomain['name']) for subdomain in subdomains_data] 89 | return self.subdomains 90 | 91 | @staticmethod 92 | def get_summary(): 93 | domain_count = domains_collection.count_documents({}) 94 | subdomain_count = subdomains_collection.count_documents({}) 95 | return {'domain_count': domain_count, 'subdomain_count': subdomain_count} 96 | 97 | @staticmethod 98 | def get_by_name(name): 99 | domain_data = domains_collection.find_one({'name': name}) 100 | if domain_data: 101 | domain = Domain(name) 102 | domain.id = domain_data['_id'] 103 | domain.subdomains_count = domain_data.get('subdomains_count', 0) 104 | return domain 105 | else: 106 | return None 107 | 108 | class Subdomain: 109 | def __init__(self, domain_id, name): 110 | self.domain_id = domain_id 111 | self.name = name 112 | self.added_at = datetime.utcnow() # the timestamp for when the subdomain is created 113 | self.id = None # Store the subdomain ID 114 | 115 | def save(self): 116 | subdomains_collection.insert_one({ 117 | 'domain_id': self.domain_id, 118 | 'name': self.name, 119 | 'added_at': self.added_at 120 | }) 121 | def update(self): 122 | subdomains_collection.update_one({'_id': self.id}, {'$set': {'name': self.name}}) 123 | 124 | def get_total_subdomains(self): 125 | return subdomains_collection.count_documents({}) 126 | 127 | 128 | def delete(self): 129 | subdomains_collection.delete_one({'_id': self.id}) 130 | 131 | def update_domain_subdomains(domain_id, subdomains): 132 | domain = Domain.get_by_id(domain_id) 133 | if domain: 134 | for subdomain_name in subdomains: 135 | subdomain = Subdomain(domain_id, subdomain_name) 136 | existing_subdomain = subdomains_collection.find_one({'name': subdomain_name, 'domain_id': domain_id}) 137 | if not existing_subdomain: # Check if subdomain already exists 138 | subdomain.save() 139 | else: 140 | print("Domain not found") 141 | 142 | def get_subdomains(self): 143 | subdomains = subdomains_collection.find({'domain': self.name}) 144 | return subdomains 145 | 146 | @staticmethod 147 | def get_by_domain_id(domain_id): 148 | return subdomains_collection.find({'domain_id': domain_id}) 149 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | flask_executor 3 | pymongo 4 | tldextract 5 | apscheduler 6 | python-dotenv -------------------------------------------------------------------------------- /routes/domain.py: -------------------------------------------------------------------------------- 1 | from flask import request, Blueprint, jsonify, redirect, url_for 2 | from models import domains_collection, Domain, subdomains_collection, Subdomain 3 | from bson.objectid import ObjectId 4 | 5 | 6 | main = Blueprint('domain_main', __name__) # Create a Blueprint instance 7 | 8 | @main.route('/add_domain', methods=['POST']) 9 | def add_domain(): 10 | name = request.json.get('name') 11 | if not name: 12 | return 'No domain name provided', 400 13 | domain = Domain(name) 14 | domain.save() 15 | return 'Domain added successfully!', 201 16 | 17 | 18 | @main.route('/update_domain/', methods=['POST']) 19 | def update_domain(domain_name): 20 | new_name = request.json.get('name') 21 | if not new_name: 22 | return 'No new domain name provided', 400 23 | domain = domains_collection.find_one({'name': domain_name}) 24 | if not domain: 25 | return 'Domain not found', 404 26 | domain_obj = Domain(name=new_name) 27 | domain_obj.id = domain['_id'] 28 | domain_obj.update() 29 | return 'Domain updated successfully!' 30 | 31 | @main.route('/delete_domain/', methods=['POST']) 32 | def delete_domain(domain_name): 33 | domain = domains_collection.find_one({'name': domain_name}) 34 | if not domain: 35 | return 'Domain not found', 404 36 | domain_obj = Domain('') 37 | domain_obj.id = domain['_id'] 38 | domain_obj.delete() 39 | return 'Domain deleted successfully!' 40 | 41 | from bson.json_util import dumps 42 | 43 | @main.route('/domains/export', methods=['GET']) 44 | def export_domains(): 45 | domains = list(domains_collection.find()) 46 | return dumps(domains), 200 47 | 48 | @main.route('/domains/search', methods=['GET']) 49 | def search_domain(): 50 | query = request.args.get('q') 51 | search_results = domains_collection.find({'name': {'$regex': query}}) 52 | only_name = [domain['name'] for domain in search_results] 53 | return jsonify(dumps(search_results)) 54 | -------------------------------------------------------------------------------- /routes/index.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, render_template, request, jsonify 2 | from models import domains_collection, subdomains_collection, Domain,Subdomain 3 | from bson.json_util import dumps 4 | import tldextract 5 | from werkzeug.utils import secure_filename 6 | import os 7 | from application import create_app 8 | import requests 9 | import logging 10 | import threading 11 | from .subdomains import get_subdomains 12 | import threading 13 | from .index import get_subdomains 14 | 15 | app = create_app() 16 | main = Blueprint('main', __name__) 17 | 18 | ALLOWED_EXTENSIONS = {'txt'} 19 | 20 | def allowed_file(filename): 21 | return '.' in filename and \ 22 | filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS 23 | 24 | @main.route('/', methods=['GET']) 25 | def index(): 26 | # Get all domains from the database 27 | domains_data = Domain.get_all_domains() 28 | 29 | # Convert each domain in domains_data to a Domain object and assign the timestamp of the last subdomain added to the last_updated attribute 30 | domains = [] 31 | for domain_data in domains_data: 32 | domain = Domain(domain_data['name']) 33 | domain.id = domain_data['_id'] 34 | domain.subdomains_count = domain_data.get('subdomains_count', 0) 35 | 36 | # Get the last updated time of the domain 37 | last_subdomain = subdomains_collection.find_one( 38 | {'domain_id': domain.id}, 39 | projection={'added_at': 1}, 40 | sort=[('added_at', -1)] 41 | ) 42 | domain.last_updated = last_subdomain['added_at'] if last_subdomain else None 43 | domains.append(domain) 44 | 45 | return render_template('index.html', domains=domains) 46 | 47 | 48 | @main.route('/add_domain', methods=['POST']) 49 | def add_domain(): 50 | name = request.json.get('name') 51 | if not name: 52 | return jsonify({'message': 'No domain name provided'}), 400 53 | domain = Domain(name) 54 | domain.save() 55 | try: 56 | # Call the get_subdomains function 57 | subdomains = get_subdomains(name) 58 | logging.info(f'Subdomains added for {name}: {subdomains}') # Log the added subdomains 59 | return jsonify({'message': 'Domain and its subdomains added successfully!', 'subdomains': subdomains}), 201 60 | except Exception as e: 61 | logging.error(f'Error while getting subdomains for {name}: {e}') 62 | return jsonify({'message': 'Domain added, but failed to get its subdomains'}), 201 63 | 64 | 65 | @main.route('/add_domains', methods=['POST']) 66 | def add_domains(): 67 | names = request.json.get('names') 68 | if not names: 69 | return jsonify({'message': 'No domain names provided'}), 400 70 | 71 | response = {} 72 | for name in names: 73 | domain = Domain(name) 74 | domain.save() 75 | try: 76 | # Call the get_subdomains function 77 | subdomains = get_subdomains(name) 78 | logging.info(f'Subdomains added for {name}: {subdomains}') # Log the added subdomains 79 | response[name] = {'message': 'Domain and its subdomains added successfully!', 'subdomains': subdomains} 80 | except Exception as e: 81 | logging.error(f'Error while getting subdomains for {name}: {e}') 82 | response[name] = {'message': 'Domain added, but failed to get its subdomains'} 83 | return jsonify(response), 201 84 | 85 | 86 | @main.route('/reset', methods=['GET']) 87 | def reset_database(): 88 | domains_collection.delete_many({}) 89 | subdomains_collection.delete_many({}) 90 | return jsonify({"message": "Database has been reset"}) 91 | 92 | @main.route('/search', methods=['GET']) 93 | def search(): 94 | query = request.args.get('q') 95 | domain_results = domains_collection.find({'name': {'$regex': query}}) 96 | subdomain_results = subdomains_collection.find({'name': {'$regex': query}}) 97 | # Combine results and convert to JSON 98 | all_results = list(domain_results) + list(subdomain_results) 99 | all_results_json = dumps(all_results) 100 | return all_results_json 101 | 102 | @main.route('/upload_subdomains', methods=['POST']) 103 | def upload_subdomains(): 104 | # check if the post request has the file part 105 | if 'file' not in request.files: 106 | return 'No file part', 400 107 | file = request.files['file'] 108 | # if user does not select file, browser also 109 | # submit an empty part without filename 110 | if file.filename == '': 111 | return 'No selected file', 400 112 | if file and allowed_file(file.filename): 113 | filename = secure_filename(file.filename) 114 | file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) 115 | 116 | # Open the file and read the subdomains 117 | with open(os.path.join(app.config['UPLOAD_FOLDER'], filename)) as f: 118 | subdomain_names = f.read().splitlines() 119 | 120 | # Add each subdomain to the database 121 | domain_obj = None # Define domain_obj with a default value 122 | for name in subdomain_names: 123 | # Extract the main domain name and subdomain from each domain in the list using tldextract 124 | extracted = tldextract.extract(name) 125 | print(extracted) 126 | main_domain = "{}.{}".format(extracted.domain, extracted.suffix) 127 | print(main_domain) 128 | subdomain_name = extracted.subdomain 129 | subdomain_name_with_main_domain = "{}.{}".format(subdomain_name, main_domain) 130 | print(subdomain_name_with_main_domain) 131 | print(subdomain_name) 132 | 133 | # Check if the main domain exists in the database 134 | domain = domains_collection.find_one({'name': main_domain}) 135 | if domain is None: 136 | # If not, add it to the database 137 | domain = Domain(main_domain) 138 | domain.save() 139 | print("Domain %s added successfully!" % main_domain) 140 | # Create the Domain object 141 | domain_obj = Domain(name=domain.name) 142 | domain_obj.id = domain.id 143 | else: 144 | # If yes, create a Domain object for it 145 | domain_obj = Domain(name=domain['name']) 146 | domain_obj.id = domain['_id'] 147 | 148 | # Add the subdomain 149 | domain_obj.add_subdomain(subdomain_name_with_main_domain) 150 | 151 | return 'Subdomains uploaded successfully!', 200 152 | else: 153 | return 'File not allowed', 400 154 | 155 | def update_subdomains_task(): 156 | # Read the domains from the file 157 | with open('domains_to_monitor.txt', 'r') as file: 158 | domains = file.read().splitlines() 159 | 160 | for domain_name in domains: 161 | # Get the domain object from the database 162 | domain_object = Domain.get_by_name(domain_name) 163 | 164 | # If the domain does not exist in the database, create a new one and save it 165 | if not domain_object: 166 | domain_object = Domain(domain_name) 167 | domain_object.save() 168 | 169 | # Get the subdomains from the API 170 | print("Getting subdomains for %s" % domain_object.name) 171 | subdomains = get_subdomains(domain_object.name) 172 | 173 | -------------------------------------------------------------------------------- /routes/subdomains.py: -------------------------------------------------------------------------------- 1 | # routes/subdomain.py 2 | from flask import request, Blueprint, jsonify, render_template 3 | from models import subdomains_collection, Subdomain, domains_collection, Domain 4 | from pymongo import MongoClient 5 | from bson.objectid import ObjectId 6 | from bson.json_util import dumps 7 | from datetime import datetime, timedelta 8 | import logging 9 | import requests 10 | from models import Domain 11 | from .utils import get_subdomains 12 | 13 | main = Blueprint('subdomain_main', __name__) # Create a Blueprint instance 14 | 15 | @main.route('/add_subdomain/', methods=['POST']) 16 | def add_subdomain(main_domain): 17 | # Retrieve the domain document from the database using the main domain name 18 | domain = domains_collection.find_one({'name': main_domain}) 19 | if not domain: 20 | return 'Main domain not found', 404 21 | 22 | # Create the Domain object 23 | domain_obj = Domain(name=domain['name']) 24 | domain_obj.id = domain['_id'] # set the id of the domain object 25 | 26 | # Get the subdomain name from the request body 27 | subdomain_name = request.json.get('subdomain_name') 28 | 29 | if not subdomain_name: 30 | return 'No subdomain name provided', 400 31 | 32 | # Add the subdomain 33 | response_message = domain_obj.add_subdomain(subdomain_name) 34 | if "successfully" in response_message: 35 | return response_message, 201 36 | else: 37 | return response_message, 409 # conflict 38 | 39 | 40 | @main.route('/update_subdomain//', methods=['POST']) 41 | def update_subdomain(main_domain, subdomain_name): 42 | new_name = request.json.get('name') 43 | if not new_name: 44 | return 'No new subdomain name provided', 400 45 | domain = domains_collection.find_one({'name': main_domain}) 46 | if not domain: 47 | return 'Main domain not found', 404 48 | subdomain = subdomains_collection.find_one({'name': subdomain_name, 'domain_id': domain['_id']}) 49 | if not subdomain: 50 | return 'Subdomain not found', 404 51 | subdomain_obj = Subdomain(domain_id=domain['_id'], name=new_name) 52 | subdomain_obj.id = subdomain['_id'] 53 | subdomain_obj.update() 54 | return 'Subdomain updated successfully!' 55 | 56 | @main.route('/delete_subdomain//', methods=['POST']) 57 | def delete_subdomain(main_domain, subdomain_name): 58 | domain = domains_collection.find_one({'name': main_domain}) 59 | if not domain: 60 | return 'Main domain not found', 404 61 | subdomain = subdomains_collection.find_one({'name': subdomain_name, 'domain_id': domain['_id']}) 62 | if not subdomain: 63 | return 'Subdomain not found', 404 64 | subdomain_obj = Subdomain(domain_id=domain['_id'], name='') 65 | subdomain_obj.id = subdomain['_id'] 66 | subdomain_obj.delete() 67 | return 'Subdomain deleted successfully!' 68 | 69 | @main.route('/subdomains/export', methods=['GET']) 70 | def export_subdomains(): 71 | subdomains = list(subdomains_collection.find()) 72 | returm_only_name = [subdomain['name'] for subdomain in subdomains] 73 | return dumps(returm_only_name), 200 74 | 75 | 76 | @main.route('//subdomains/export', methods=['GET']) 77 | def export_subdomains_by_domain(domain_name): 78 | domain = domains_collection.find_one({'name': domain_name}) 79 | if not domain: 80 | return 'Domain not found', 404 81 | subdomains = subdomains_collection.find({'domain_id': domain['_id']}) 82 | names = [subdomain['name'] for subdomain in subdomains] 83 | return jsonify(names) 84 | 85 | @main.route('/subdomains/search', methods=['GET']) 86 | def search_subdomain(): 87 | query = request.args.get('q') 88 | search_results = subdomains_collection.find({'name': {'$regex': query}}) 89 | only_name = [subdomain['name'] for subdomain in search_results] 90 | return jsonify(dumps(only_name)) 91 | #return jsonify(dumps(search_results)) 92 | 93 | @main.route('/lastupdated', methods=['GET']) 94 | def get_last_updated(): 95 | # Calculate the time 30 minutes ago 96 | half_hour_ago = datetime.utcnow() - timedelta(minutes=1) 97 | 98 | # Query the database for any subdomains added after half_hour_ago 99 | new_subdomains = list(subdomains_collection.find({"added_at": {"$gte": half_hour_ago}})) 100 | 101 | # Convert the query result to JSON and return it 102 | return dumps(new_subdomains) 103 | 104 | @main.route('/sort_subdomains') 105 | def sort_subdomains(): 106 | domains = Domain.get_all_domains() 107 | domains = sorted(domains, key=lambda domain: domain.subdomains_count, reverse=True) 108 | summary = Domain.get_summary() 109 | return render_template('index.html', domains=domains, summary=summary) 110 | 111 | -------------------------------------------------------------------------------- /routes/utils.py: -------------------------------------------------------------------------------- 1 | # utils.py 2 | import requests 3 | import logging 4 | from models import Domain 5 | import os 6 | 7 | API_KEY = os.getenv("SECURITYTRAILS") 8 | BASE_URL = "https://api.securitytrails.com/v1" 9 | 10 | def get_subdomains(domain): 11 | subdomains = [] 12 | 13 | url = f"{BASE_URL}/domain/{domain}/subdomains?children_only=false&include_inactive=false" 14 | headers = {"accept": "application/json", "APIKEY": API_KEY} 15 | response = requests.get(url, headers=headers) 16 | logging.info(response) 17 | data = response.json() 18 | limit_reached = data.get("meta", {}).get("limit_reached") 19 | logging.info("Limit reached: %s % limit_reached") 20 | logging.info(data) 21 | domain_object = Domain.get_by_name(domain) 22 | 23 | # Check if limit has been reached after first request 24 | if data.get("meta", {}).get("limit_reached") == True: 25 | logging.info("Limit reached, using scroll API for Domain %s" % domain) 26 | url = f"{BASE_URL}/domains/list?include_ips=false&scroll=true" 27 | payload = {"query": f"apex_domain = \"{domain}\""} 28 | response = requests.post(url, json=payload, headers=headers) 29 | data = response.json() 30 | scroll_id = data.get("meta", {}).get("scroll_id") 31 | total_pages = data.get("meta", {}).get("total_pages") 32 | 33 | if scroll_id and total_pages: 34 | current_page = 1 35 | while current_page < total_pages: 36 | url = f"{BASE_URL}/scroll/{scroll_id}" 37 | response = requests.get(url, headers=headers) 38 | data = response.json() 39 | 40 | for record in data.get("records", []): 41 | subdomain = record.get("hostname") 42 | if subdomain: 43 | subdomains.append(f"{subdomain}.{domain}") 44 | domain_object.add_subdomain(f"{subdomain}.{domain}") 45 | 46 | current_page += 1 47 | else: 48 | # If limit not reached, add subdomains from initial response 49 | subdomains.extend(f"{subdomain}.{domain}" for subdomain in data.get("subdomains", [])) 50 | for subdomain_name in subdomains: 51 | print("Adding subdomain %s" % subdomain_name) 52 | domain_object.add_subdomain(subdomain_name) 53 | logging.info("Subdomain %s added successfully!" % subdomain_name) 54 | return subdomains 55 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | from application import create_app 2 | from routes.index import main as index_blueprint 3 | from routes.index import update_subdomains_task 4 | from routes.domain import main as domain_blueprint 5 | from routes.subdomains import main as subdomain_blueprint 6 | from apscheduler.schedulers.background import BackgroundScheduler 7 | 8 | app = create_app() 9 | 10 | # register blueprints 11 | app.register_blueprint(index_blueprint) 12 | app.register_blueprint(domain_blueprint) 13 | app.register_blueprint(subdomain_blueprint) 14 | 15 | scheduler = BackgroundScheduler() 16 | # scheduler.add_job(update_subdomains_task, 'interval', seconds=24) 17 | 18 | scheduler.start() 19 | print("Scheduler started!") 20 | 21 | if __name__ == "__main__": 22 | app.run(debug=True, host="0.0.0.0", port=5000) 23 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Domain Management 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 |

vīlicus

13 | 14 |
15 |

Domains

16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | {% for domain in domains %} 26 | 27 | 28 | 29 | 30 | 31 | {% endfor %} 32 | 33 |
DomainSubdomains countLast Updated
{{ domain.name }}{{ domain.subdomains_count }}{{ domain.last_updated.strftime('%Y-%m-%d %H:%M:%S') if domain.last_updated else 'N/A' }}
34 |
35 | 36 |
37 | 38 |
39 |
40 |

Search for a domain or subdomain

41 |
42 |
43 | 44 | 45 |
46 | 47 |
48 |
49 |
50 | 51 | 52 |
53 |

Upload subdomains

54 |
55 |
56 | 57 | 58 |
59 | 60 |
61 |
62 | 63 |
64 |
65 | 66 | 67 | 87 | 88 | 89 | 90 | --------------------------------------------------------------------------------