├── .gitignore ├── README.md ├── censys.ini.default ├── censys_certif_crawl.py └── database.sql /.gitignore: -------------------------------------------------------------------------------- 1 | censys.ini 2 | censys.db 3 | censys_data.py 4 | data 5 | datafiles/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # censys_certif_crawl.py 2 | 3 | A script to retrieve the certificate transparancy information from Censys. 4 | 5 | This information can be used for OSINT recon. 6 | 7 | # Configuration 8 | 9 | All the configuration is done in the ini file censys.ini. You need to get an API key from Censys and add the SECRET and UID. 10 | 11 | # Usage 12 | 13 | Use the script from the command line and give the search query as an argument. 14 | 15 | For example to search for certificates related to ".be" (Belgium) you can use 16 | 17 | ``` 18 | censys_certif_crawl.py ".be" 19 | ``` 20 | 21 | # Database 22 | 23 | All the data is stored in a sqlite database with three tables 24 | - subject_dn 25 | - dns_names 26 | - issuer_dn 27 | 28 | You can use the database to analyze the information that was retrieved from Censys. 29 | -------------------------------------------------------------------------------- /censys.ini.default: -------------------------------------------------------------------------------- 1 | [censys] 2 | url = https://www.censys.io/api/v1 3 | index = /search/certificates 4 | uid = UID 5 | secret = SECRET 6 | 7 | 8 | [db] 9 | db = censys.db 10 | sql-create = database.sql -------------------------------------------------------------------------------- /censys_certif_crawl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Use certificate transparancy for OSINT 4 | # information from censys 5 | # 6 | # Koen Van Impe 7 | # 20160816 8 | # 9 | # Usage : censys.py myquery 10 | # 11 | # Configuration : see the censys.ini file 12 | # 13 | 14 | import sqlite3 15 | import os 16 | import os.path 17 | import sys 18 | import json 19 | import requests 20 | import ConfigParser 21 | import time 22 | from time import gmtime, strftime 23 | 24 | # Setup the variables 25 | Config = ConfigParser.ConfigParser() 26 | Config.read("censys.ini") 27 | SQLDB=Config.get("db", "db") 28 | SQLFILE=Config.get("db", "sql-create") 29 | API_URL = Config.get("censys", "url") 30 | API_INDEX = Config.get("censys", "index") 31 | UID = Config.get("censys", "uid") 32 | SECRET = Config.get("censys", "secret") 33 | 34 | # Censys Query 35 | if len(sys.argv) <= 1 : 36 | print "error occurred: No censys filter given" 37 | print "Run %s censys_filter" % sys.argv[0] 38 | sys.exit(1) 39 | censys_query = sys.argv[1] 40 | 41 | print "0. Starting %s" % strftime("%Y-%m-%d %H:%M:%S", gmtime()) 42 | 43 | # Remove any old databases 44 | if os.path.isfile(SQLDB): 45 | os.remove(SQLDB) 46 | 47 | # Setup the database 48 | query = open(SQLFILE, 'r').read() 49 | sqlite3.complete_statement(query) 50 | conn = sqlite3.connect(SQLDB) 51 | with conn: 52 | cur = conn.cursor() 53 | try: 54 | # Create the database 55 | cur.executescript(query) 56 | print "1. Database %s created." % SQLDB 57 | res = None 58 | 59 | # Contact the API 60 | current_page = 1 61 | fields = [ "parsed.fingerprint_sha256", "parsed.extensions.subject_alt_name.dns_names", "parsed.issuer_dn", "parsed.subject_dn"] 62 | data = { 'query': censys_query, 'page': current_page, 'fields': fields} 63 | data = json.dumps(data) 64 | res = requests.post(API_URL + API_INDEX, data=data, auth=(UID,SECRET)) 65 | 66 | # Check if we get a good reply 67 | if res.status_code != 200: 68 | print "error occurred: %s" % res.json()["error"] 69 | sys.exit(1) 70 | 71 | print "2. Received results for query %s" % censys_query 72 | metadata_pages = res.json()["metadata"]["pages"] 73 | metadata_count = res.json()["metadata"]["count"] 74 | 75 | print "3. Got %s results in %s pages." % (metadata_count, metadata_pages) 76 | 77 | while current_page <= metadata_pages: 78 | if res is None: 79 | data = { 'query': censys_query, 'page': current_page, 'fields': fields} 80 | data = json.dumps(data) 81 | res = requests.post(API_URL + API_INDEX, data=data, auth=(UID,SECRET)) 82 | 83 | print "4. Page %s / %s " % (current_page, metadata_pages) 84 | 85 | if "results" in res.json(): 86 | results = res.json()["results"] 87 | 88 | for cert in results: 89 | if "parsed.extensions.subject_alt_name.dns_names" in cert: 90 | dns_names = cert["parsed.extensions.subject_alt_name.dns_names"] 91 | dns_names_count = len(dns_names) 92 | else: 93 | dns_names = None 94 | dns_names_count = 0 95 | 96 | subject_dn = cert["parsed.subject_dn"][0] 97 | issuer_dn = cert["parsed.issuer_dn"][0] 98 | fingerprint_sha256 = cert["parsed.fingerprint_sha256"][0] 99 | 100 | issuer_dn_split = issuer_dn.split(",") 101 | issuer_c = "" 102 | issuer_o = "" 103 | issuer_cn = "" 104 | issuer_ou = "" 105 | for el in issuer_dn_split: 106 | el = el.strip() 107 | if el[0:2] == "C=": 108 | issuer_c = el[2:] 109 | elif el[0:2] == "O=": 110 | issuer_o = el[2:] 111 | elif el[0:3] == "CN=": 112 | issuer_cn = el[3:] 113 | elif el[0:3] == "OU=": 114 | issuer_ou = el[3:] 115 | 116 | subject_dn_split = subject_dn.split(",") 117 | subject_c = "" 118 | subject_o = "" 119 | subject_cn = "" 120 | subject_ou = "" 121 | for el in subject_dn_split: 122 | el = el.strip() 123 | if el[0:2] == "C=": 124 | subject_c = el[2:] 125 | elif el[0:2] == "O=": 126 | subject_o = el[2:] 127 | elif el[0:3] == "CN=": 128 | subject_cn = el[3:] 129 | elif el[0:3] == "OU=": 130 | subject_ou = el[3:] 131 | 132 | subject_dn_table = [ fingerprint_sha256, subject_dn, dns_names_count, subject_c, subject_ou, subject_o, subject_cn] 133 | cur.execute("INSERT INTO subject_dn VALUES(?, ?, ?, ?, ?, ?, ?)", subject_dn_table) 134 | 135 | issuer_dn_table = [ fingerprint_sha256, issuer_dn, issuer_c, issuer_ou, issuer_o, issuer_cn] 136 | cur.execute("INSERT INTO issuer_dn VALUES(?, ?, ?, ?, ?, ?)", issuer_dn_table) 137 | 138 | if dns_names is not None: 139 | for name in dns_names: 140 | dns_names_table = [ fingerprint_sha256, name ] 141 | cur.execute("INSERT INTO dns_names VALUES(?, ?)", dns_names_table) 142 | 143 | conn.commit() 144 | current_page += 1 145 | res = None 146 | time.sleep(2) 147 | 148 | cur.close() 149 | 150 | except Exception as e: 151 | cur.close() 152 | raise 153 | -------------------------------------------------------------------------------- /database.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS subject_dn ; 2 | 3 | CREATE TABLE subject_dn (sha256 TEXT, content TEXT, dns_names_count INT, subject_c TEXT, subject_ou TEXT, subject_o TEXT, subject_cn TEXT) ; 4 | CREATE TABLE dns_names (sha256 TEXT, content TEXT) ; 5 | CREATE TABLE issuer_dn (sha256 TEXT, content TEXT, issuer_c TEXT, issuer_ou TEXT, issuer_o TEXT, issuer_cn TEXT) ; --------------------------------------------------------------------------------