├── README.md └── db /README.md: -------------------------------------------------------------------------------- 1 | # db 2 | Mini utility to store the list of the enumerated subdomains into an sqlite3 db. [one liner Style ] 3 | 4 | ![Build Status](https://pbs.twimg.com/media/EiYn1w3XYAAAlN9?format=jpg&name=large) 5 | 6 | Status: **Development** 7 | ## Features 8 | 9 | * Remove duplicates 10 | * Query database 11 | * Clean results 12 | * Pipe results to store data 13 | * Export the saved records 14 | 15 | ## Chaining tools 16 | You can pipe the tools results into db 17 | > Example : 18 | 19 | > subfinder + db 20 | > ![Build Status](https://pbs.twimg.com/media/EiY1Sa1WsAIPSSp?format=jpg&name=large) 21 | 22 | 23 | # Prod 24 | #### Install Requirements 25 | `sudo pip3 install Flask flask_sqlalchemy loguru uuid fire csv` 26 | 27 | > Add it as a bash command 28 | ```bash 29 | cp db /usr/bin 30 | chmod +x /usr/bin/db 31 | ``` 32 | 33 | # Usage 34 | 35 | | Query | Command | 36 | |--------------------------------------|----------------------------| 37 | | Add list of subdomains | cat subdomains.txt | db or echo subdomains.txt | db | 38 | | Add **ONE** subdmain to database | `db save tesla.com` | 39 | | Search for domain | `db search tesla.com` | 40 | | Get the new added subdomains | `db new` | 41 | | Export list of subdomains | `db export tesla.com` or if csv `db export tesla.com csv` | 42 | | Remove all the database records | `db wipe` | 43 | | Remove records for a specific domain | `db removeall tesla.com` | 44 | | Remove **ONE** record | `db remove tesla.com` | 45 | | Get all the saved records | `db all` | 46 | 47 | > PS : 48 | > - Export function uses TXT format by default (if csv is not specified) 49 | > - new arguments shows the latest saved records for that day only 50 | -------------------------------------------------------------------------------- /db: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # @ih3bski 3 | from flask import Flask 4 | from flask_sqlalchemy import SQLAlchemy 5 | from sqlalchemy import select, func 6 | import uuid 7 | from loguru import logger 8 | import sys 9 | import argparse 10 | import fire 11 | from datetime import datetime, date 12 | import csv 13 | from sqlalchemy import cast, Date 14 | 15 | app = Flask(__name__) 16 | 17 | # Change the default db path 18 | dbname = "bb_record" 19 | dbpath = "/tmp" 20 | result_save_dir = "/tmp" 21 | 22 | # SQLAlchemy config/Disable warnings for a clean output $_$ 23 | app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{dbpath}/{dbname}.sqlite' 24 | app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False 25 | db = SQLAlchemy(app) 26 | stats = logger.level("STATS", no=38, color="", icon="📈") 27 | Counter = 0 28 | results = 0 29 | 30 | # Model -> T.B changed 31 | class Target(db.Model): 32 | __tablename__ = 'Target' 33 | id = db.Column(db.Text, primary_key=True) 34 | subdomain = db.Column(db.Text) 35 | created_date = db.Column(db.DateTime, default=datetime.today().replace(microsecond=0)) 36 | 37 | def __init__(self,subdomain,id): 38 | self.id = id 39 | self.subdomain = subdomain 40 | 41 | db.create_all() 42 | 43 | def save(subdomain): 44 | """ 45 | Save subdomains to databse 46 | """ 47 | global Counter 48 | if db.session.query(Target.id).filter_by(subdomain=subdomain).scalar() is None : 49 | db.session.add(Target(subdomain,str(uuid.uuid4()))) 50 | db.session.commit() 51 | Counter += 1 52 | logger.log('INFO',f'[+] {subdomain} added to database') 53 | else: 54 | logger.log('ERROR',f'[-] {subdomain} already exists') 55 | 56 | def searchByDomain(subdomain): 57 | return db.session.query(Target).filter(Target.subdomain.like(f"%{subdomain}%")).all() 58 | 59 | def search(subdomain): 60 | """ 61 | Search a list of subdomains 62 | """ 63 | sub = searchByDomain(subdomain) 64 | if sub : 65 | results = sub 66 | for row in sub: 67 | logger.log('INFO',row.subdomain) 68 | logger.log("STATS", f'We have found {len(sub)} subdomains ! Happy Hacking $_$') 69 | else: 70 | logger.log('WARNING',f'[-] {subdomain} NOT FOUND.') 71 | 72 | def remove(subdomain): 73 | """ 74 | Remove One subdomain from the database 75 | """ 76 | if Target.query.filter(Target.subdomain == subdomain).delete(): 77 | db.session.commit() 78 | logger.log("INFO",f'[+] {subdomain} was deleted successfully.') 79 | else: 80 | logger.log("ERROR",f'[+] Failed while trying to remove {subdomain} check your request!') 81 | 82 | def removeall(subdomain): 83 | """ 84 | Remove all domains related like subdomains 85 | """ 86 | Target.query.filter(Target.subdomain.like(f"%{subdomain}%")).delete(synchronize_session='fetch') 87 | db.session.commit() 88 | print("deleted",sub) 89 | 90 | def new(): 91 | """ 92 | Show the new added subdomains, filter by today date. 93 | """ 94 | sub = db.session.query(Target).filter(func.date(Target.created_date) == date.today()).all() 95 | if sub.count : 96 | for row in sub: 97 | logger.log("INFO",f"{row.subdomain}") 98 | else: 99 | logger.log('WARNING',f'[-] No record added for today') 100 | 101 | def all(): 102 | """ 103 | Get all database results 104 | """ 105 | sub = db.session.query(Target).all() 106 | if sub : 107 | for row in sub: 108 | logger.log('INFO',f"{row.created_date} {row.subdomain}") 109 | logger.log("STATS", f'{len(sub)} bulks! $_$') 110 | else: 111 | logger.log('WARNING',f'[-] Database is empty') 112 | 113 | def wipe(): 114 | """ 115 | Delete all record in the database 116 | """ 117 | db.session.query(Target).delete() 118 | db.session.commit() 119 | 120 | def exportToCSV(subdomain,sub,file_name): 121 | """ 122 | Export subdomains as csv format 123 | """ 124 | try: 125 | with open(file_name,"w") as file: 126 | fnames = ['subdomain', 'saved_date'] 127 | writer = csv.DictWriter(file, fieldnames=fnames) 128 | writer.writeheader() 129 | for _ in sub: 130 | writer.writerow({'subdomain' : _.subdomain, 'saved_date': _.created_date}) 131 | print(f'[+] {len(sub)} results of {subdomain} exported successfully.\n[+] Path {result_save_dir}/{subdomain}_{datetime.now().date()}.csv') 132 | except IOError as error: 133 | logger.log("ERROR","[+] Fails to open or create file! Please check your permissions") 134 | 135 | def exportToTXT(subdomain,sub,file_name): 136 | """ 137 | Export subdomains as txt format 138 | """ 139 | try: 140 | f = open(file_name, 'w') 141 | for _ in sub: 142 | f.write(f'{_.subdomain}\n') 143 | print(f'[+] {len(sub)} results of {subdomain} exported successfully.\n[+] Path {result_save_dir}/{subdomain}_{datetime.now().date()}.txt') 144 | except IOError as error: 145 | logger.log("ERROR","[+] Fails to open or create file! Please check your permissions") 146 | 147 | def export(subdomain,format="txt"): 148 | """ 149 | Export results as txt,csv format 150 | default_dir/domain.com_date.{txt,csv} 151 | """ 152 | rows = searchByDomain(subdomain) 153 | if rows: 154 | if format == "csv": 155 | exportToCSV(subdomain,rows,f'{result_save_dir}/{subdomain}_{datetime.now().date()}.csv') 156 | elif format == "txt" : 157 | exportToTXT(subdomain,rows,f'{result_save_dir}/{subdomain}_{datetime.now().date()}.txt') 158 | else: 159 | logger.log("ERROR","This format is not supported yet !") 160 | else: 161 | logger.log("ERROR","[+] Cannot export the file, please check your query :)") 162 | 163 | if __name__ == '__main__': 164 | # use stdin if it's full 165 | if not sys.stdin.isatty(): 166 | for subdomains in sys.stdin: 167 | save(subdomains.strip()) 168 | logger.log("STATS", f'{Counter} Bulk added ^_^ !') 169 | # read subdomain as argument 170 | else: 171 | fire.Fire() 172 | --------------------------------------------------------------------------------