├── .gitignore ├── CertStreamMonitor.py ├── LICENCE ├── README.md ├── check_rules.py ├── conf └── example.conf ├── docker ├── Dockerfile ├── docker-compose.yml └── supervisord.conf ├── gethost.py ├── requirements.txt ├── safebrowsing.py ├── scanhost.py ├── useragent_list.txt └── utils ├── confparser.py ├── sqlite.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | alerts 3 | db 4 | log 5 | -------------------------------------------------------------------------------- /CertStreamMonitor.py: -------------------------------------------------------------------------------- 1 | #!/opt/venv/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2018-2021 Caisse nationale d'Assurance Maladie 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, version 3 of the License. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | 15 | import os 16 | import re 17 | import sys 18 | import getopt 19 | import logging 20 | import datetime 21 | import certstream 22 | from logging.handlers import RotatingFileHandler 23 | from utils.confparser import ConfParser 24 | from utils.utils import TimestampNow, VerifyPath 25 | from utils.sqlite import SqliteCmd 26 | 27 | VERSION = "0.7.2" 28 | 29 | def usage(): 30 | """ 31 | CLI usage printing 32 | """ 33 | usage = """ 34 | -h --help Print this help 35 | -c --config Configuration file to use 36 | """ 37 | print(usage) 38 | sys.exit(0) 39 | 40 | 41 | def ConfAnalysis(ConfFile): 42 | """ 43 | configuration file analysis. Load global variables with parameters found 44 | in configuration file. 45 | 46 | :param ConfFile: the configuration file 47 | """ 48 | global CONF 49 | global DBFile 50 | global TABLEname 51 | global LogFile 52 | global LogLevel 53 | global LogType 54 | global SearchKeywords 55 | global BlacklistKeywords 56 | global DetectionThreshold 57 | global ACTServer 58 | global Proxy_Host 59 | global Proxy_Port 60 | global Proxy_Username 61 | global Proxy_Password 62 | 63 | try: 64 | CONF = ConfParser(ConfFile) 65 | 66 | DBFile = CONF.DBFile 67 | TABLEname = CONF.TABLEname 68 | LogFile = CONF.LogFile 69 | LogLevel = CONF.LogLevel 70 | LogType = CONF.LogType 71 | SearchKeywords = CONF.SearchKeywords 72 | BlacklistKeywords = CONF.BlacklistKeywords 73 | DetectionThreshold = CONF.DetectionThreshold 74 | ACTServer = CONF.ACTServer 75 | Proxy_Host = CONF.Proxy_Host 76 | Proxy_Port = CONF.Proxy_Port 77 | Proxy_Username = CONF.Proxy_Username 78 | Proxy_Password = CONF.Proxy_Password 79 | 80 | except: 81 | err = sys.exc_info() 82 | logging.error(" ConfParser Error: " + str(err)) 83 | 84 | 85 | def args_parse(): 86 | """ 87 | Tool options 88 | """ 89 | global ConfFile 90 | if not len(sys.argv[1:]): 91 | usage() 92 | try: 93 | opts, args = getopt.getopt(sys.argv[1:], "hc:", ["help", "conf="]) 94 | except getopt.GetoptError as err: 95 | logging.error(" Option Error. Exiting..." + str(err)) 96 | usage() 97 | sys.exit(2) 98 | 99 | for o, a in opts: 100 | if o in ("-h", "--help"): 101 | usage() 102 | elif o in ("-c", "--config"): 103 | if os.path.isfile(a): 104 | ConfFile = a 105 | else: 106 | logging.error(" Can't find configuration file. Exiting...") 107 | sys.exit(1) 108 | else: 109 | assert False, "Unhandled Option" 110 | return 111 | 112 | # CertStream 113 | 114 | 115 | def print_callback(message, context): 116 | """ 117 | callback that is going to be called at each CertStream message reception 118 | """ 119 | if message['message_type'] == "heartbeat": 120 | return 121 | 122 | if message['message_type'] == "certificate_update": 123 | all_domains = message['data']['leaf_cert']['all_domains'] 124 | 125 | # look for pattern on *each* hostname 126 | for host in all_domains: 127 | is_blacklisted = False 128 | if BlacklistKeywords != str(): 129 | is_blacklisted = re.findall(BlacklistKeywords, host) 130 | results = re.findall(SearchKeywords, host) 131 | FindNb = len(set(results)) 132 | 133 | # Matching host whith blacklisted keywords are ignored 134 | if is_blacklisted and FindNb >= DetectionThreshold: 135 | continue 136 | 137 | # If search keywords occurence in the hostname is greater or equal to DetectionThreshold 138 | # we store the hostname into DB 139 | if FindNb >= DetectionThreshold: 140 | # Data extraction to populate DB 141 | Domain = host 142 | SAN = "" 143 | Issuer = message['data']['leaf_cert']['subject']['aggregated'] 144 | Fingerprint = message['data']['leaf_cert']['fingerprint'] 145 | Startime = datetime.datetime.utcfromtimestamp( 146 | message['data']['leaf_cert']['not_before']).isoformat() 147 | FirstSeen = format(datetime.datetime.utcnow( 148 | ).replace(microsecond=0).isoformat()) 149 | # Test if entry still exist in DB 150 | if SQL.SQLiteVerifyEntry(TABLEname, Domain) == 0: 151 | SQL.SQLiteInsert(TABLEname, Domain, SAN, Issuer, Fingerprint, Startime, FirstSeen) 152 | sys.stdout.write(u"[{}] {} (SAN: {}) (Issuer: {}) (Fingerprint: {}) (StartTime: {})\n".format(datetime.datetime.now().replace(microsecond=0).isoformat(), host, "", message['data']['leaf_cert']['issuer']['aggregated'], message['data']['leaf_cert']['fingerprint'], datetime.datetime.utcfromtimestamp(message['data']['leaf_cert']['not_before']).isoformat())) 153 | sys.stdout.flush() 154 | 155 | # If just one keyword occurence, put data into debug log file 156 | elif FindNb > 0 and FindNb < DetectionThreshold: 157 | logging.debug("DETECTION THRESHOLD VALUE NOT REACHED - {} (SAN: {}) (Issuer: {}) (Fingerprint: {}) (StartTime: {})".format(host, "", message['data']['leaf_cert']['issuer']['aggregated'], message['data']['leaf_cert']['fingerprint'], datetime.datetime.utcfromtimestamp(message['data']['leaf_cert']['not_before']).isoformat())) 158 | 159 | 160 | # Main 161 | def main(): 162 | global SQL 163 | try: 164 | # Config 165 | ConfAnalysis(ConfFile) 166 | P = VerifyPath() 167 | # Create files 168 | P.VerifyOrCreate(DBFile) 169 | P.VerifyOrCreate(LogFile) 170 | # Database 171 | SQL = SqliteCmd(DBFile) 172 | SQL.SQLiteCreateTable(TABLEname) 173 | 174 | # logging 175 | logger = logging.getLogger() 176 | logger.setLevel(LogLevel) 177 | 178 | # file handler (10MB, 10 rotations) 179 | if LogType == 'file': 180 | format = logging.Formatter( 181 | '[%(levelname)s:%(name)s] %(asctime)s - %(message)s') 182 | file_handler = RotatingFileHandler(LogFile, 'a', 10000000, 10) 183 | file_handler.setFormatter(format) 184 | logger.addHandler(file_handler) 185 | # syslog handler 186 | elif LogType == 'syslog': 187 | stream_handler = logging.StreamHandler() 188 | logger.addHandler(stream_handler) 189 | else: 190 | logging.error("Unsupported log type " + LogType + ". Exiting...") 191 | sys.exit(1) 192 | 193 | # Work, connection to the CT logs aggregator (ACTServer), through a HTTP proxy if configured into configuration file 194 | logging.info("Looking for these strings: " + SearchKeywords + 195 | ", detection threshold: " + str(DetectionThreshold)) 196 | certstream.listen_for_events(print_callback, ACTServer, http_proxy_host=Proxy_Host, 197 | http_proxy_port=Proxy_Port, http_proxy_auth=(Proxy_Username, Proxy_Password)) 198 | print_callback() 199 | 200 | SQL.SQLiteClose() 201 | 202 | except: 203 | err = sys.exc_info() 204 | logging.error(" Main error " + str(err)) 205 | 206 | 207 | # Start 208 | if __name__ == '__main__': 209 | args_parse() 210 | main() 211 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CertStreamMonitor 2 | 3 | Monitor certificates generated for specific domain strings and associated, store data into sqlite3 database, alert you when sites come online. 4 | 5 | CertStreamMonitor architecture relies on 3 scripts : 6 | 7 | - `certstreammonitor.py` 8 | - this script runs as a daemon. 9 | - reading the certstream feed from [CertStream service](https://certstream.calidog.io/) provided by [Calidog Security](https://calidog.io/) (thanks so much for this!), it selects hostnames covered by certificates that match your criteria (SearchKeyWords parameter in conf). 10 | - it writes these hostnames along with its certificate relevant informations to the database. 11 | - `scanhost.py` 12 | - this script can be executed as often as you like. 13 | - it checks if site corresponding to the hostanme stored in DB is UP ot not. 14 | - it collects informations about the sites that are up to DB and to a JSON file. 15 | - `gethost.py` 16 | - Due to [@nbeguier](https://github.com/nbeguier) contribution, the project has also a `gethost.py` script that provides a way for security operators to request the last hostnames which have been detected by `certstreammointor.py` since X seconds. 17 | - `check_rules.py` 18 | - Due to [@nbeguier](https://github.com/nbeguier) contribution, the project has also a `check_rules.py` script that provides a way for security operators to check their own rules, especially SearchKeywords, DetectionThreshold and BlacklistKeywords. 19 | 20 | ## Features 21 | - **Monitoring:** 22 | - monitor `wss://certstream.calidog.io` CT logs aggregator server with certstream-python (see [certstream-python](https://github.com/CaliDog/certstream-python)), but you can choose, and operate, your own server (see [certstream-server](https://github.com/CaliDog/certstream-server/)). 23 | - choose strings you want to monitor in `Subject Alt Names` field of certificates 24 | - **Storing:** 25 | - store hostnames found along with its certificate relevant data into a sqlite3 database 26 | - **Alerting:** 27 | - for each hostname not already flagged as up : check if corresponding site is up or not 28 | - if it's up : 29 | - collects informations (IP address, AS informations, HTTP code, web page title, abuse email, (optional) google safe browsing status) 30 | - write them to a JSON file in the `/alerts` directory (default value) to push forward investigation. 31 | - (optional) push them to a destination (through apprise package) such as an email address, a Slack channel or even Twitter account 32 | - flags the hostname in the DB as up 33 | - if it's not : 34 | - the hostname will be checked until the Alert_Monitor_timelapse configuration variable (in days) will be reached 35 | - when the limit is reached a string like `Stop checking on 2019-09-27` is included into the StillInvestig column 36 | 37 | ## Requirements 38 | - Python 3 39 | - certstream 40 | - sqlite3 41 | - ipwhois 42 | - PySocks 43 | - hues 44 | - websocket-client 45 | - apprise 46 | 47 | ## Install 48 | Install the requirements 49 | ~~~ 50 | $ pip3 install -r requirements.txt 51 | ~~~ 52 | 53 | ## Configuration file 54 | You can find a configuration file example in 'conf' directory. 55 | Configurable parameters are: 56 | - `SearchKeywords`: Keywords to look for (with '|' (or) as separator) 57 | - `DetectionThreshold`: set the minimum number of detected SearchKeywords in a hostname before writing it to DB. Under this value but above zero, detected hostnames are only written to logfile. Default value: 2. 58 | - `DBFile`: SQLite3 database file (the path and file will be created if don't exist) 59 | - `TABLEname`: The name of the database table 60 | - `LogFile`: The logging file (the path and file will be created if don't exist) 61 | - `UAfile`: you can provide a User-Agent file to masquerade this value of requests (random change for each request) 62 | - `Alerts_dir`: you can specify where JSON alert files are written 63 | You can use the following strings to add time/date hashed based subdirectories: 64 | %%m -> month, %%d -> day, %%Y -> year, %%H -> hour, %%M -> minute. 65 | Example: Alerts_dir = ./alerts/%%Y/%%m/%%d 66 | 67 | Optional: 68 | - `Proxy`: allows to give a SOCKS or HTTP proxy to process your scanhost.py's requests (as Tor) 69 | - `Proxy_*` parameters : allow you to specify HTTP proxy informations (server, port[, user, password]) for CertStreamMonitor.py script to connect to the CT logs aggregator server. 70 | - `ACTServer`: you can specify the CT logs aggregator server of your choice. By default, it is the server run by Calidog Security. 71 | - `Safe_Browsing_API_Key`: indicate (if you want) your Google Safe Browsing API key in order to check hostnames that are UP against Google Safe Browsing Lookup API ([How-To get an API key](https://developers.google.com/safe-browsing/v4/get-started) for the Safe Browsing Lookup API). 72 | - `Notification_Destination`: specify a notification destination as attended by apprise package. Documentation about the format of this parameter is available on the [apprise Github page](https://github.com/caronc/apprise). 73 | - `BlacklistKeywords`: Keywords to ignore matched hosts (with '|' (or) as separator) 74 | 75 | ## Usage 76 | 77 | ### CertStreamMonitor.py 78 | 79 | ~~~ 80 | $ python3 ./CertStreamMonitor.py -c conf/example.conf 81 | Looking for these strings: paypal|apple|account|secure|login, detection threshold: 2 82 | Connection established to CertStream! Listening for events... 83 | [2018-03-12T11:40:15] cpanel.my-appleid-apple.net (SAN: mail.my-appleid-apple.net,my-appleid-apple.net,webdisk.my-appleid-apple.net,webmail.my-appleid-apple.net,www.my-appleid-apple.net) (Issuer: /C=US/CN=Let's Encrypt Authority X3/O=Let's Encrypt) (Fingerprint: 45:11:51:2D:24:D3:04:6E:DF:49:46:6D:64:56:67:4B:0A:48:8D:93) (StartTime: 2018-03-12T10:39:40) 84 | [2018-03-12T11:41:19] cpanel.verification-account-apple-now.com (SAN: mail.verification-account-apple-now.com,verification-account-apple-now.com,webdisk.verification-account-apple-now.com,webmail.verification-account-apple-now.com,www.verification-account-apple-now.com) (Issuer: /C=US/CN=Let's Encrypt Authority X3/O=Let's Encrypt) (Fingerprint: 2D:90:F9:F7:83:F6:48:26:EF:C9:72:50:4B:06:FA:36:53:94:3C:8B) (StartTime: 2018-03-12T10:40:49) 85 | [2018-03-12T11:41:36] login-apple.sytes.net (SAN: ) (Issuer: /C=US/CN=Let's Encrypt Authority X3/O=Let's Encrypt) (Fingerprint: C7:78:2F:08:1E:CC:83:6C:06:EF:77:14:D2:1A:4E:06:A8:B3:F9:77) (StartTime: 2018-03-12T10:41:08) 86 | [2018-03-12T11:42:26] cpanel.restore-account-apple.com (SAN: mail.restore-account-apple.com,restore-account-apple.com,webdisk.restore-account-apple.com,webmail.restore-account-apple.com,www.restore-account-apple.com) (Issuer: /C=US/CN=Let's Encrypt Authority X3/O=Let's Encrypt) (Fingerprint: F3:CA:B1:C6:DE:4F:05:16:FD:06:F3:FF:29:8A:D3:1F:10:9D:50:1A) (StartTime: 2018-03-12T10:41:59) 87 | [2018-03-12T11:49:37] securelogin.here.att.thysseankrupp.com (SAN: ) (Issuer: /C=US/CN=Let's Encrypt Authority X3/O=Let's Encrypt) (Fingerprint: 8F:9B:98:8D:5D:9B:03:0B:4F:62:56:40:C1:DE:9A:A4:FB:2D:A3:3E) (StartTime: 2018-03-12T09:22:41) 88 | ... 89 | ~~~ 90 | 91 | ### scanhost.py 92 | 93 | ~~~ 94 | $ python3 scanhost.py --help 95 | 96 | -h --help Print this help 97 | -c --config Configuration file to use 98 | -f --fqdn-dirs Store JSON files in sub-directories based on the hostname 99 | ~~~ 100 | 101 | ~~~ 102 | $ python3 ./scanhost.py -c conf/example.conf 103 | Test all domains in DB for Internet Presence: 104 | ********************************************* 105 | 14:30:12 - ERROR - https://socialparadiseweb.cf.socialparadise.cf - Connection error 106 | 14:32:18 - ERROR - https://rapportannuel-assurancemaladie.paris - Connection error 107 | 14:32:23 - SUCCESS - HTTP 200 - socialmediaforsocialaction.com 108 | Creating ./alerts/socialmediaforsocialaction.com.json : {'hostname': 'socialmediaforsocialaction.com', 'http_code': 200, 'cert_serial_number': '89:6C:03:F6:82:57:03:2A:A8:D0:E1:2F:E8:56:0E:32:83:E5:EC:29', 'webpage_title': 'Social Media for Social Action', 'ip_addr': '198.49.23.145', 'asn': '53831', 'asn_cidr': '198.49.23.0/24', 'asn_country_code': 'US', 'asn_description': 'SQUARESPACE - Squarespace, Inc., US', 'asn_abuse_email': 'abuse-network@squarespace.com'} 109 | 14:32:25 - ERROR - https://social.socialbride.co.za - Connection error 110 | 14:32:34 - SUCCESS - HTTP 503 - assurances-sociales.com 111 | Creating ./alerts/assurances-sociales.com.json : {'hostname': 'assurances-sociales.com', 'http_code': 503, 'cert_serial_number': '1A:0D:45:D9:05:15:DC:17:6C:9F:9E:47:A5:62:03:D9:25:02:F9:3C', 'webpage_title': 'Accueil', 'ip_addr': '164.132.235.17', 'asn': '16276', 'asn_cidr': '164.132.0.0/16', 'asn_country_code': 'FR', 'asn_description': 'OVH, FR', 'asn_abuse_email': 'lir@ovh.net'} 112 | ~~~ 113 | 114 | ### gethost.py 115 | 116 | ~~~ 117 | $ python3 gethost.py --help 118 | 119 | -h --help Print this help 120 | -c --config Configuration file to use 121 | --since Since when it displays findings (seconds) 122 | ~~~ 123 | 124 | ~~~ 125 | $ python3 ./gethost.py -c conf/example.conf --since 36000 # 10 hours 126 | Display all domains in DB for Internet Presence: 127 | ************************************************ 128 | socialparadiseweb.cf.socialparadise.cf None 129 | rapportannuel-assurancemaladie.paris None 130 | socialmediaforsocialaction.com 2019-06-12T15:54:31 131 | social.socialbride.co.za None 132 | ~~~ 133 | 134 | ### check_rules.py 135 | 136 | ~~~ 137 | $ python3 check_rules.py --help 138 | 139 | -h --help Print this help 140 | -c --config Configuration file to use 141 | -d --domain Domain name to check 142 | ~~~ 143 | 144 | ~~~ 145 | # No match - Keywords not found. 146 | $ ./check_rules.py -c conf/example.conf -d www.google.com 147 | Looking for these strings: paypal|apple|account|secure|login, detection threshold: 2 148 | No match - Keywords not found. 149 | 150 | # No match - Detection threashold not reached. 151 | $ ./check_rules.py -c conf/example.conf -d paypal.com 152 | Looking for these strings: paypal|apple|account|secure|login, detection threshold: 2 153 | No match - Detection threashold not reached. 154 | 155 | # No match - Blacklisted keywords. 156 | $ ./check_rules.py -c conf/example.conf -d login-paypal.gouv 157 | Looking for these strings: paypal|apple|account|secure|login, detection threshold: 2 158 | No match - Blacklisted keywords. 159 | 160 | # This is a match, detection threashold reached. 161 | $ ./check_rules.py -c conf/example.conf -d login-paypal.com 162 | Looking for these strings: paypal|apple|account|secure|login, detection threshold: 2 163 | This is a match, detection threashold reached. 164 | ~~~ 165 | 166 | ## Authors 167 | - Thomas Damonneville ([thomas.damonneville@assurance-maladie.fr](mailto:thomas.damonneville@assurance-maladie.fr)) 168 | - Christophe Brocas ([christophe.brocas@assurance-maladie.fr](mailto:christophe.brocas@assurance-maladie.fr)) 169 | - And of course, a big shout out to [Calidog Security](https://calidog.io/) and its fantastic [CertStream service](https://certstream.calidog.io/) without which nothing would have been possible! 170 | 171 | ## Contributors 172 | - [@ant1](https://github.com/ant1): [#3](https://github.com/AssuranceMaladieSec/CertStreamMonitor/pull/3) 173 | - [@xme](https://github.com/xme): [#8](https://github.com/AssuranceMaladieSec/CertStreamMonitor/pull/8), [#9](https://github.com/AssuranceMaladieSec/CertStreamMonitor/pull/9) 174 | - [@nbeguier](https://github.com/nbeguier): [#20](https://github.com/AssuranceMaladieSec/CertStreamMonitor/pull/20), [#21](https://github.com/AssuranceMaladieSec/CertStreamMonitor/pull/21), [#28](https://github.com/AssuranceMaladieSec/CertStreamMonitor/pull/28) 175 | 176 | ## Presentations 177 | - [SSTIC 2018](https://www.sstic.org/2018/) | June 2018 - C.Brocas, T. Damonneville: *"Certificate Transparency ou comment un nouveau standard peut aider votre veille sur certaines menaces"*. [Slides (fr)](https://www.sstic.org/2018/presentation/certificate_transparency_ou_comment_un_nouveau_standard_peut_aider_votre_analyse_des_menaces/), [full article (fr)](https://www.sstic.org/media/SSTIC2018/SSTIC-actes/certificate_transparency_ou_comment_un_nouveau_sta/SSTIC2018-Article-certificate_transparency_ou_comment_un_nouveau_standard_peut_aider_votre_analyse_des_menaces-broc_AR1OQsw.pdf), [video (fr)](https://static.sstic.org/videos2018/SSTIC_2018-06-13_P04.mp4). 178 | - [Hack-it-n 2018bis](http://www.hack-it-n.com/event2018bis/) | December 2018 - C. Brocas, T. Damonneville (given by C. Brocas): *"CertStreamMonitor, use Certificate Transparency to improve your threats detection"*. [Slides (en)](https://speakerdeck.com/cbrocas/2018bis-hack-it-n-certstreammonitor-use-certificate-transparency-to-improve-your-threats-detection). 179 | - [Toulouse hacking Convention 2019](https://19.thcon.party/) | March 2019 - C. Brocas, T. Damonneville (given by C. Brocas): *"Certificate Transparency & threats detection, 24 months later"*. [Slides (en)](https://speakerdeck.com/cbrocas/thc19-certificate-transparency-and-threats-detection-24-months-later), [video (fr)](https://www.youtube.com/watch?v=rUOQE-2NG3Y&feature=youtu.be&t=11485). 180 | 181 | ## License 182 | GNU GENERAL PUBLIC LICENSE (GPL) Version 3 183 | -------------------------------------------------------------------------------- /check_rules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (c) 2018 Caisse nationale d'Assurance Maladie 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, version 3 of the License. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | 14 | import os 15 | import re 16 | import sys 17 | import getopt 18 | import logging 19 | from utils.confparser import ConfParser 20 | from utils.utils import VerifyPath 21 | 22 | VERSION = "0.0.1" 23 | 24 | def usage(): 25 | """ 26 | CLI usage printing 27 | """ 28 | usage_output = """ 29 | -h --help Print this help 30 | -c --config Configuration file to use 31 | -d --domain Domain name to check 32 | """ 33 | print(usage_output) 34 | sys.exit(0) 35 | 36 | 37 | def ConfAnalysis(configuration_file): 38 | """ 39 | configuration file analysis. Load global variables with parameters found 40 | in configuration file. 41 | 42 | :param configuration_file: the configuration file 43 | """ 44 | global CONF 45 | global SearchKeywords 46 | global BlacklistKeywords 47 | global DetectionThreshold 48 | 49 | try: 50 | CONF = ConfParser(configuration_file) 51 | 52 | SearchKeywords = CONF.SearchKeywords 53 | BlacklistKeywords = CONF.BlacklistKeywords 54 | DetectionThreshold = CONF.DetectionThreshold 55 | except: 56 | err = sys.exc_info() 57 | logging.error(" ConfParser Error: %s", err) 58 | 59 | 60 | def args_parse(): 61 | """ 62 | Tool options 63 | """ 64 | global ConfFile 65 | global DOMAIN 66 | if not len(sys.argv[1:]): 67 | usage() 68 | try: 69 | opts, _ = getopt.getopt(sys.argv[1:], "hc:d:", ["help", "conf="]) 70 | except getopt.GetoptError as err: 71 | logging.error(" Option Error. Exiting... %s", err) 72 | usage() 73 | sys.exit(2) 74 | 75 | DOMAIN = None 76 | for o, a in opts: 77 | if o in ("-h", "--help"): 78 | usage() 79 | elif o in ("-c", "--config"): 80 | if os.path.isfile(a): 81 | ConfFile = a 82 | else: 83 | logging.error(" Can't find configuration file. Exiting...") 84 | sys.exit(1) 85 | elif o in ("-d", "--domain"): 86 | DOMAIN = a 87 | else: 88 | assert False, "Unhandled Option" 89 | if not DOMAIN: 90 | usage() 91 | sys.exit(2) 92 | 93 | 94 | def print_callback(): 95 | """ 96 | Truncate CertStreamMonitor/print_callback function, SQL/Logging support removed 97 | """ 98 | is_blacklisted = False 99 | if BlacklistKeywords != str(): 100 | is_blacklisted = re.findall(BlacklistKeywords, DOMAIN) 101 | results = re.findall(SearchKeywords, DOMAIN) 102 | FindNb = len(set(results)) 103 | 104 | # Matching host whith blacklisted keywords are ignored 105 | if is_blacklisted and FindNb >= DetectionThreshold: 106 | logging.info("No match - Blacklisted keywords.") 107 | return 108 | 109 | # If search keywords occurence in the hostname is greater or equal to DetectionThreshold 110 | if FindNb >= DetectionThreshold: 111 | logging.info("This is a match, detection threashold reached.") 112 | elif FindNb > 0 and FindNb < DetectionThreshold: 113 | logging.info("No match - Detection threashold not reached.") 114 | else: 115 | logging.info("No match - Keywords not found.") 116 | return 117 | 118 | 119 | # Main 120 | def main(): 121 | # Config 122 | ConfAnalysis(ConfFile) 123 | VerifyPath() 124 | 125 | # logging 126 | logger = logging.getLogger() 127 | logger.setLevel(logging.DEBUG) 128 | 129 | # term handler 130 | stream_handler = logging.StreamHandler() 131 | stream_handler.setLevel(logging.INFO) 132 | logger.addHandler(stream_handler) 133 | logging.info( 134 | "Looking for these strings: %s, detection threshold: %s", 135 | SearchKeywords, 136 | DetectionThreshold) 137 | print_callback() 138 | 139 | 140 | # Start 141 | if __name__ == '__main__': 142 | args_parse() 143 | main() 144 | -------------------------------------------------------------------------------- /conf/example.conf: -------------------------------------------------------------------------------- 1 | 2 | ## 3 | ## Configuration file for CertStreamMonitor 4 | ## 5 | 6 | [SEARCH] 7 | # Keywords to look for (with '|' (or) as separator) 8 | SearchKeywords = paypal|apple|account|secure|login 9 | 10 | # Minimum number of SearchKeywords detected in a hostname before writing it to DB. If you choose to set it to 2, 11 | # the tool needs to detect 2 SearchKeywords matchs to write the match hostname to DB. 12 | # Under 2 matching SearchKeywords but above 0, the matching hostname will only be logged to the log file. 13 | DetectionThreshold = 2 14 | 15 | # Keywords to ignore matched hosts (with '|' (or) as separator). 16 | # Leave it empty or comment it to disable this feature. 17 | BlacklistKeywords = mail\.|\.gouv|bitcoin 18 | 19 | [DATABASE] 20 | # SQLite3 database path (will be created if not exist) 21 | DBFile = ./db/CertStreamMonitor.sqlite3 22 | TABLEname = CertMon 23 | 24 | [CONNECT] 25 | # (optional) Declare a HTTP or SOCKS proxy to use for HTTPS Get informations with scanhost.py 26 | # ex: Proxy = http://127.0.0.1:8080 for a HTTP proxy server 27 | # ex: Proxy = socks5://127.0.0.1:9050 for a SOCKS5 proxy server 28 | #Proxy = socks5://127.0.0.1:9050 29 | 30 | # Default user-agent (don't remove): 31 | http_UA = Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36 32 | # Use a HTTPS user-agents file for scanhost.py 33 | UAfile = ./useragent_list.txt 34 | 35 | [LOGGING] 36 | # Logging file (will be created if not exist) 37 | LogFile = ./log/certstreammonitor.log 38 | LogLevel = 10 39 | LogType = file 40 | 41 | [REPORTING] 42 | # Alerts reporting directory for scanhost.py 43 | # %%m -> month 44 | # %%d -> day 45 | # %%Y -> year 46 | # %%H -> hour 47 | # %%M -> minute 48 | Alerts_dir = ./alerts/%%Y/%%m/%%d 49 | 50 | # Number of days scanhost.py will try to get information about an entry, 51 | # after that the entry will not be checked anymore (default: 180 days). 52 | Alert_Monitor_timelapse = 180 53 | 54 | # (optional) Notifications: you can specify a notification destination. It will push to it each hostname detected as UP (+ all informations present in the JSON file). 55 | # Notification Destination syntax and supported services are described on the apprise package Github page: https://github.com/caronc/apprise 56 | # Example of Notification_Destination for sending email over SMTPS : 57 | #Notification_Destination = mailtos://domain.xyz?user=smtp-user&pass=user-password&smtp=smtp-server-hostname&to=recipient-address&from=send-address&format=text&name=CertStreamMonitor-notification 58 | 59 | [SERVER] 60 | # Connection to Certficate Transparency aggregator server 61 | ACTServer = wss://certstream.calidog.io 62 | 63 | # (optional) Declare a HTTP proxy to use to connect to Certificate Transparency aggregator server 64 | #Proxy_Host = 127.0.0.1 65 | #Proxy_Port = 8080 66 | #Proxy_Username = "Username" 67 | #Proxy_Password = "My_Proxy-Password" 68 | 69 | [SAFEBROWSING] 70 | # (optional) Set the Google Safe Browsing API key value with your own if you want to test each hostname that is up against the GSB Lookup API 71 | #Safe_Browsing_API_Key = value 72 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.19 2 | 3 | LABEL maintainer="Damonneville Thomas " 4 | 5 | ENV INITSYSTEM=on 6 | 7 | # install packages 8 | RUN apk --no-cache add --update \ 9 | git \ 10 | python3 \ 11 | sqlite \ 12 | openrc \ 13 | supervisor \ 14 | gcc \ 15 | python3-dev \ 16 | musl-dev \ 17 | libffi-dev \ 18 | openssl-dev \ 19 | py3-pip \ 20 | cargo \ 21 | tor 22 | 23 | # Clone the project files into the docker container and install it 24 | COPY / /opt/CertStreamMonitor 25 | 26 | # Activate python virtual environment 27 | RUN python3 -m venv /opt/venv 28 | 29 | WORKDIR /opt/CertStreamMonitor/ 30 | RUN /opt/venv/bin/python3 -m pip install --upgrade pip 31 | RUN /opt/venv/bin/pip install -r requirements.txt 32 | 33 | # Add custom supervisor config 34 | COPY docker/supervisord.conf /etc/supervisor/conf.d/ 35 | CMD ["/usr/bin/supervisord"; "-c"; "/etc/supervisor/conf.d/supervisord.conf"] 36 | 37 | # Make some clean 38 | RUN rm -rf /var/cache/apk/* 39 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | # launch: docker-compose up --build -d 4 | 5 | services: 6 | CertStreamMonitor: 7 | image: certstreammonitor 8 | container_name: certstreammonitor 9 | hostname: certstreammonitor 10 | build: 11 | context: ../ 12 | dockerfile: docker/Dockerfile 13 | #volumes: 14 | 15 | command: /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf 16 | -------------------------------------------------------------------------------- /docker/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:tor] 5 | command=/usr/bin/tor 6 | 7 | [program:CertStreamMonitor] 8 | command=/opt/venv/bin/python /opt/CertStreamMonitor/CertStreamMonitor.py -c /opt/CertStreamMonitor/conf/example.conf -------------------------------------------------------------------------------- /gethost.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | GetHost display the last findings 4 | """ 5 | 6 | # Copyright (c) 2018-2019 Caisse nationale d'Assurance Maladie 7 | # 8 | # This program is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, version 3 of the License. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # Standard library imports 18 | from __future__ import absolute_import 19 | from datetime import datetime 20 | import getopt 21 | import os 22 | from sqlite3 import connect, Error 23 | import sys 24 | 25 | # Own library imports 26 | from utils.confparser import ConfParser 27 | 28 | # Debug 29 | # from pdb import set_trace as st 30 | 31 | def create_connection(db_file): 32 | """ create a database connection to the SQLite database 33 | specified by the db_file 34 | 35 | :param db_file: database file 36 | :return: Connection object or None 37 | """ 38 | 39 | try: 40 | conn = connect(db_file, isolation_level=None) 41 | # debug SQL 42 | # conn.set_trace_callback(print) 43 | return conn 44 | except Error as err: 45 | print(err) 46 | return False 47 | 48 | 49 | def args_parse(): 50 | """ 51 | Tools options 52 | """ 53 | global CONFFILE 54 | global SINCE 55 | SINCE = 3600 # One hour 56 | 57 | if not len(sys.argv[1:]): 58 | usage() 59 | try: 60 | opts, args = getopt.getopt(sys.argv[1:], "hc:", ["help", "conf=", "since="]) 61 | except getopt.GetoptError as err: 62 | print(" Option Error. Exiting..."+str(err)) 63 | usage() 64 | sys.exit(2) 65 | 66 | for o, a in opts: 67 | if o in ("-h", "--help"): 68 | usage() 69 | elif o in ("-c", "--config"): 70 | if os.path.isfile(a): 71 | CONFFILE = a 72 | else: 73 | print(" Can't find configuration file. Exiting...") 74 | sys.exit(1) 75 | elif o in ("--since"): 76 | SINCE = int(a) 77 | else: 78 | assert False, "Unhandled Option" 79 | return 80 | 81 | 82 | def usage(): 83 | """ 84 | CLI usage printing 85 | """ 86 | usage = """ 87 | -h --help Print this help 88 | -c --config Configuration file to use 89 | --since Since when it displays findings (seconds) 90 | """ 91 | print(usage) 92 | sys.exit(0) 93 | 94 | 95 | def ConfAnalysis(ConfFile): 96 | """ 97 | configuration file analysis. Load global variables with parameters found 98 | in configuration file. 99 | 100 | :param confFile: the configuration file 101 | """ 102 | global CONF 103 | global DBFile 104 | global TABLEname 105 | 106 | try: 107 | CONF = ConfParser(ConfFile) 108 | DBFile = CONF.DBFile 109 | TABLEname = CONF.TABLEname 110 | except Exception as err: 111 | err = sys.exc_info() 112 | print("ConfParser Error: "+str(err)) 113 | 114 | 115 | def parse_and_display_all_hostnames(TABLEname, conn, print_output=False): 116 | """ 117 | Parse and display all hostnames present in DB "" 118 | 119 | :param TABLEname: the table name storing certificate informations in database 120 | :param conn: db connection 121 | 122 | :return: True if everything went fine, False if something went wrong 123 | """ 124 | try: 125 | # Query rows that have not StillInvestig column already set 126 | # get Domain and Fingerprint column 127 | cur = conn.cursor() 128 | cur.execute("SELECT Domain,Issuer,Fingerprint,FirstSeen,StillInvestig FROM "+TABLEname) 129 | rows = cur.fetchall() 130 | result = dict() 131 | 132 | # run scan on each hostname 133 | for row in rows: 134 | domain = row[0] 135 | issuer = row[1] 136 | fingerprint = row[2] 137 | first_seen = row[3] 138 | still_investing = row[4] 139 | first_seen_date = datetime.strptime(first_seen, '%Y-%m-%dT%H:%M:%S') 140 | since = (datetime.utcnow() - first_seen_date).total_seconds() 141 | if since < SINCE: 142 | result.update({domain: {"issuer": issuer, "fingerprint": fingerprint, "still_investing": still_investing}}) 143 | if print_output: 144 | print("{domain} {issuer} {fingerprint} {still_investing}".format( 145 | domain=domain, 146 | issuer=issuer, 147 | fingerprint=fingerprint, 148 | still_investing=still_investing)) 149 | return result 150 | 151 | except KeyboardInterrupt: 152 | if print_output: 153 | print("Interrupt received, stopping ...") 154 | print("start - committing, closing DB") 155 | conn.commit 156 | conn.close 157 | if print_output: 158 | print("ending - committing, closing DB") 159 | return result 160 | 161 | except Exception as err: 162 | if print_output: 163 | print(err) 164 | return result 165 | 166 | 167 | def main(): 168 | """ 169 | Main function 170 | """ 171 | ConfAnalysis(CONFFILE) 172 | 173 | # create a database connection 174 | conn = create_connection(DBFile) 175 | 176 | with conn: 177 | print("Display all domains in DB for Internet Presence:") 178 | print("************************************************") 179 | parse_and_display_all_hostnames(TABLEname, conn, print_output=True) 180 | 181 | 182 | if __name__ == '__main__': 183 | args_parse() 184 | main() 185 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.4 2 | apprise==1.7.1 3 | CacheControl==0.13.1 4 | certifi==2023.11.17 5 | certstream==1.12 6 | cffi==1.16.0 7 | chardet==5.2.0 8 | charset-normalizer==3.3.2 9 | click==8.1.7 10 | colorama==0.4.6 11 | contextlib2==21.6.0 12 | cryptography==41.0.7 13 | distlib==0.3.8 14 | distro==1.9.0 15 | dnspython==2.0.0 16 | html5lib==1.1 17 | hues==0.2.2 18 | idna==3.6 19 | ipwhois==1.2.0 20 | lockfile==0.12.2 21 | Markdown==3.5.2 22 | meld3==2.0.1 23 | msgpack==1.0.7 24 | oauthlib==3.2.2 25 | ordered-set==4.1.0 26 | packaging==23.2 27 | pep517==0.13.1 28 | progress==1.6 29 | pycparser==2.21 30 | pyparsing==3.1.1 31 | PySocks==1.7.1 32 | pytoml==0.1.21 33 | PyYAML==6.0.1 34 | requests==2.31.0 35 | requests-oauthlib==1.3.1 36 | retrying==1.3.4 37 | six==1.16.0 38 | supervisor==4.2.5 39 | termcolor==2.4.0 40 | toml==0.10.2 41 | urllib3==2.1.0 42 | webencodings==0.5.1 43 | websocket-client==1.7.0 -------------------------------------------------------------------------------- /safebrowsing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (c) 2016 Jun C. Valdez 4 | # Code is distrubuted under the terms of an MIT style license 5 | # http://www.opensource.org/licenses/mit-license 6 | # 7 | 8 | import requests 9 | import json 10 | 11 | SB_CLIENT_ID = "Python SafeBrowsing Client" 12 | SB_CLIENT_VER = "0.0.1" 13 | 14 | 15 | class LookupAPI(object): 16 | 17 | 18 | def __init__(self, apikey): 19 | 20 | self.apiurl = 'https://safebrowsing.googleapis.com/v4/threatMatches:find?key=%s' % (apikey) 21 | self.platform_types = ['ANY_PLATFORM'] 22 | self.threat_types = ['THREAT_TYPE_UNSPECIFIED', 23 | 'MALWARE', 24 | 'SOCIAL_ENGINEERING', 25 | 'UNWANTED_SOFTWARE', 26 | 'POTENTIALLY_HARMFUL_APPLICATION'] 27 | self.threat_entry_types = ['URL'] 28 | 29 | def set_threat_types(self, threats): 30 | 31 | self.threat_types = threats 32 | 33 | def set_platform_types(self, platforms): 34 | 35 | self.platform_types = platforms 36 | 37 | def threat_matches_find(self, *urls): 38 | 39 | threat_entries = [] 40 | results = {} 41 | 42 | for url_ in urls: 43 | url = {'url': url_} 44 | threat_entries.append(url) 45 | 46 | reqbody = { 47 | 'client': { 48 | 'clientId': SB_CLIENT_ID, 49 | 'clientVersion': SB_CLIENT_VER 50 | }, 51 | 'threatInfo': { 52 | 'threatTypes': self.threat_types, 53 | 'platformTypes': self.platform_types, 54 | 'threatEntryTypes': self.threat_entry_types, 55 | 'threatEntries': threat_entries 56 | } 57 | } 58 | 59 | headers = {'Content-Type': 'application/json'} 60 | r = requests.post(self.apiurl, 61 | data=json.dumps(reqbody), 62 | headers=headers) 63 | # 64 | # need to include exceptions here 65 | # 66 | 67 | return r.json() 68 | 69 | 70 | 71 | class UpdateAPI(object): 72 | 73 | 74 | def __init__(self, apikey): 75 | pass 76 | 77 | 78 | -------------------------------------------------------------------------------- /scanhost.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (c) 2018-2019 Caisse nationale d'Assurance Maladie 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, version 3 of the License. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | 14 | import os 15 | import re 16 | import sys 17 | import getopt 18 | import logging 19 | import datetime 20 | import random 21 | import socks 22 | import signal 23 | import json 24 | from logging.handlers import RotatingFileHandler 25 | from utils.confparser import ConfParser 26 | from utils.utils import TimestampNow, VerifyPath 27 | import sqlite3 28 | import hues 29 | import requests 30 | import socket 31 | from ipwhois import IPWhois 32 | import warnings 33 | import time 34 | import safebrowsing 35 | import apprise 36 | 37 | def create_connection(db_file): 38 | """ create a database connection to the SQLite database 39 | specified by the db_file 40 | 41 | :param db_file: database file 42 | :return: Connection object or None 43 | """ 44 | 45 | try: 46 | conn = sqlite3.connect(db_file, isolation_level=None) 47 | # debug SQL 48 | # conn.set_trace_callback(print) 49 | return conn 50 | except sqlite3.Error as e: 51 | print(e) 52 | return False 53 | 54 | 55 | def args_parse(): 56 | """ 57 | Tools options 58 | """ 59 | global ConfFile 60 | global fqdn_dirs 61 | fqdn_dirs = False 62 | 63 | if not len(sys.argv[1:]): 64 | usage() 65 | try: 66 | opts, args = getopt.getopt(sys.argv[1:], "hfc:", ["help", "fqdn-dirs", "conf="]) 67 | except getopt.GetoptError as err: 68 | logging.error(" Option Error. Exiting..." + str(err)) 69 | usage() 70 | sys.exit(2) 71 | 72 | for o, a in opts: 73 | if o in ("-h", "--help"): 74 | usage() 75 | elif o in ("-c", "--config"): 76 | if os.path.isfile(a): 77 | ConfFile = a 78 | else: 79 | logging.error(" Can't find configuration file. Exiting...") 80 | sys.exit(1) 81 | elif o in ("-f", "--fqdn-dirs"): 82 | fqdn_dirs = True 83 | else: 84 | assert False, "Unhandled Option" 85 | return 86 | 87 | 88 | def usage(): 89 | """ 90 | CLI usage printing 91 | """ 92 | usage = """ 93 | -h --help Print this help 94 | -c --config Configuration file to use 95 | -f --fqdn-dirs Store JSON files in sub-directories based on the hostname 96 | """ 97 | print (usage) 98 | sys.exit(0) 99 | 100 | 101 | def generate_alert_dir(path): 102 | """ 103 | Generate the hashed directory path based on current date 104 | """ 105 | # %m -> month 106 | # %d -> day 107 | # %Y -> year 108 | # %H -> hour 109 | # %M -> minute 110 | t_hour = time.strftime("%H") 111 | t_minute = time.strftime("%M") 112 | t_day = time.strftime("%d") 113 | t_month = time.strftime("%m") 114 | t_year = time.strftime("%Y") 115 | path = path.replace('%H', t_hour) 116 | path = path.replace('%M', t_minute) 117 | path = path.replace('%d', t_day) 118 | path = path.replace('%m', t_month) 119 | path = path.replace('%Y', t_year) 120 | return path 121 | 122 | 123 | def ConfAnalysis(ConfFile): 124 | """ 125 | configuration file analysis. Load global variables with parameters found 126 | in configuration file. 127 | 128 | :param confFile: the configuration file 129 | """ 130 | global CONF 131 | global DBFile 132 | global TABLEname 133 | global LogFile 134 | global Proxy 135 | global UA 136 | global UAFILE 137 | global Alerts_dir 138 | global Alert_Monitor_timelapse 139 | global Notification_Destination 140 | global Safe_Browsing_API_Key 141 | 142 | try: 143 | CONF = ConfParser(ConfFile) 144 | DBFile = CONF.DBFile 145 | TABLEname = CONF.TABLEname 146 | LogFile = CONF.LogFile 147 | Proxy = CONF.Proxy 148 | UA = CONF.http_UA 149 | Alerts_dir = generate_alert_dir(CONF.Alerts_dir) 150 | Alert_Monitor_timelapse = CONF.Alert_Monitor_timelapse 151 | Notification_Destination = CONF.Notification_Destination 152 | UAFILE = CONF.UAfile 153 | Safe_Browsing_API_Key = CONF.Safe_Browsing_API_Key 154 | 155 | except Exception as err: 156 | err = sys.exc_info() 157 | logging.error(" ConfParser Error: " + str(err)) 158 | 159 | 160 | def get_random_UserAgent_header(lines): 161 | """ 162 | build a string containing a user-agent header, randomly 163 | choosen inside a given list 164 | 165 | :param lines: the file containing the user-agent possible values. 166 | One value per line. 167 | :return: the header with user-agent value set. 168 | """ 169 | ua = random.choice(lines) 170 | headers = {'user-agent': ua} 171 | return headers 172 | 173 | 174 | def get_requests(hostname, lines, conn, Proxy): 175 | """ 176 | build a requests object for a hostname 177 | 178 | :param hostname: 179 | :param lines: content of the file containing user-agents strings 180 | :param conn: connection to the database 181 | :param Proxy: connection through proxy 182 | 183 | :return: the answer to the request content or None 184 | """ 185 | 186 | # if the certificate is a wildcard, display it but no testing. 187 | # and return. 188 | if '*' in hostname: 189 | hues.warn('wildcard certificate: no request for ' + hostname) 190 | return None 191 | 192 | url = 'https://' + hostname 193 | headers = get_random_UserAgent_header(lines) 194 | 195 | # set proxy 196 | if Proxy: 197 | proxy = {"https": Proxy} 198 | else: 199 | proxy = "" 200 | 201 | try: 202 | r = requests.get(url, headers=headers, proxies=proxy, timeout=5) 203 | return r 204 | except requests.exceptions.SSLError as errs: 205 | # SSL error 206 | hues.error(" {} - SSL error".format(url)) 207 | return None 208 | except requests.exceptions.ConnectionError as errc: 209 | # other connection error 210 | hues.error(" {} - Connection error".format(url)) 211 | return None 212 | except requests.exceptions.RequestException as e: 213 | # A serious problem happened 214 | hues.error(" {} Error: {}".format(url, e)) 215 | return None 216 | except KeyboardInterrupt: 217 | print("get_requests() - Interrupt received, stopping ...") 218 | print("start - committing, closing DB") 219 | conn.commit 220 | conn.close 221 | print("ending - committing, closing DB") 222 | sys.exit(0) 223 | except Exception as ex: 224 | hues.error("get_requests() - any other kind of error: {}".format(ex)) 225 | return None 226 | 227 | 228 | def get_webpage_title(request): 229 | """ 230 | Get the website page title 231 | 232 | :param resquest: request object 233 | 234 | :return: webpage title or "" 235 | """ 236 | try: 237 | page = request.text.strip() 238 | tit = re.search('(.*?)', page, re.IGNORECASE) 239 | if tit is not None: 240 | title = tit.group(1) 241 | else: 242 | title = "" 243 | return title 244 | except Exception as e: 245 | print("error in get_webpage_title(): " + str(e)) 246 | return "" 247 | 248 | 249 | def get_ASN_Infos(ipaddr): 250 | """ 251 | Get Autonomous System Number informations linked to an ip address 252 | 253 | :param ipaddr: ip address of the website linked to the certificate common name 254 | 255 | :return: list of ASN infos: asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email or the same with empty values 256 | """ 257 | try: 258 | warnings.filterwarnings("ignore") 259 | obj = IPWhois(ipaddr) 260 | results = obj.lookup_rdap(depth=1) 261 | 262 | asn = results['asn'] 263 | asn_cidr = results['asn_cidr'] 264 | asn_country_code = results['asn_country_code'] 265 | asn_description = results['asn_description'] 266 | 267 | # parsing of all the entities members of the ASN record. 268 | # -> when finding an entity with 'abuse' role, print the email present 269 | # in the contact object. 270 | try: 271 | for entity in results['objects'].values(): 272 | if 'abuse' in entity['roles']: 273 | asn_abuse_email = entity['contact']['email'][0]['value'] 274 | break 275 | except Exception as e: 276 | asn_abuse_email = "" 277 | 278 | return asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email 279 | 280 | except Exception as e: 281 | asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email = "", "", "", "", "" 282 | return asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email 283 | 284 | 285 | def scan_hostname(hostname, SerialNumber, lines, Proxy, conn, site_infos): 286 | """ 287 | try scan a hostname and get informations back 288 | (HTTP code, page title, IP address, ASN, abuse email etc). 289 | 290 | :param hostname: the hostname present in the certificate 291 | :param SerialNumber: the serial number of the certificate 292 | :param lines: list of user-agents strings 293 | :param Proxy: proxy settings 294 | :param conn: database connection 295 | :param site_infos: informations extracted on the net for the given hostname 296 | 297 | :return: True if everything went fine, False if any problem has been encountered 298 | """ 299 | 300 | title = "" 301 | try: 302 | r = get_requests(hostname, lines, conn, Proxy) 303 | if r is not None: 304 | hues.success('HTTP ' + str(r.status_code) + ' - ' + hostname) 305 | 306 | # retrieve the title of the homepage 307 | title = get_webpage_title(r) 308 | 309 | # retrieve ASN informations 310 | ipaddr = socket.gethostbyname(hostname) 311 | asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email = get_ASN_Infos( 312 | ipaddr) 313 | 314 | # retrieve Google Safe Browsing Lookup API status for this hostname 315 | if Safe_Browsing_API_Key is not '': 316 | sb = safebrowsing.LookupAPI(Safe_Browsing_API_Key) 317 | safe_browsing_status = sb.threat_matches_find(hostname) 318 | else: 319 | safe_browsing_status = "No API key in config file" 320 | 321 | # build the content of the alert file using certificate / webpage / ASN informations 322 | site_infos = { 323 | 'hostname': hostname, 324 | 'http_code': r.status_code, 325 | 'cert_serial_number': SerialNumber, 326 | 'webpage_title': title, 327 | 'ip_addr': ipaddr, 328 | 'asn': asn, 329 | 'asn_cidr': asn_cidr, 330 | 'asn_country_code': asn_country_code, 331 | 'asn_description': asn_description, 332 | 'asn_abuse_email': asn_abuse_email, 333 | 'safe_browsing_status': safe_browsing_status 334 | } 335 | return site_infos 336 | else: 337 | return {} 338 | 339 | except KeyboardInterrupt: 340 | print("scan_hostname() - Interrupt received, stopping ...") 341 | print("start - committing, closing DB") 342 | conn.commit 343 | conn.close 344 | print("ending - committing, closing DB") 345 | sys.exit(0) 346 | 347 | except Exception as ex: 348 | hues.error("scan_hostname() - any other kind of error: {}".format(ex)) 349 | return {} 350 | 351 | 352 | def parse_and_scan_all_hostnames(TABLEname, Proxy, conn): 353 | """ 354 | Parse and scan all hostnames present in DB and having StillInvestig set to null or "" 355 | 356 | :param TABLEname: the table name storing certificate informations in database 357 | :param Proxy: proxy value 358 | :param conn: db connection 359 | 360 | :return: True if everything went fine, False if something went wrong 361 | """ 362 | try: 363 | # Query rows that have not StillInvestig column already set 364 | # get Domain, Fingerprint and FirstSeen columns 365 | cur = conn.cursor() 366 | cur.execute("SELECT Domain,Fingerprint,FirstSeen FROM " + TABLEname + 367 | " WHERE StillInvestig IS NULL or StillInvestig = ''") 368 | rows = cur.fetchall() 369 | 370 | # creating Alerts_dir if don't exist 371 | try: 372 | os.makedirs(Alerts_dir, mode=0o777, exist_ok=True) 373 | except FileExistsError: 374 | pass 375 | except: 376 | err = sys.exc_info() 377 | logging.error(" Can't create Alerts_dir: " + str(err)) 378 | 379 | # read User Agent file 380 | try: 381 | lines = open(UAFILE).read().splitlines() 382 | except: 383 | lines = UA 384 | 385 | # load apprise instance with config file parameters if notifications are activated in the config file 386 | if Notification_Destination is not '': 387 | apobj = apprise.Apprise() 388 | apobj.add(Notification_Destination) 389 | 390 | # run scan on each hostname 391 | for row in rows: 392 | hostname = row[0] 393 | SerialNumber = row[1] 394 | 395 | site_infos = {} 396 | 397 | site_infos = scan_hostname(hostname, SerialNumber, lines, Proxy, conn, site_infos) 398 | 399 | # Is time check limit reached? 400 | now = datetime.datetime.strptime(str(datetime.datetime.utcnow().replace(microsecond=0).isoformat()), "%Y-%m-%dT%H:%M:%S") 401 | FirstSeen = datetime.datetime.strptime(row[2], "%Y-%m-%dT%H:%M:%S") 402 | time_delta = (now - FirstSeen).days 403 | if time_delta < int(Alert_Monitor_timelapse): 404 | if not site_infos: 405 | continue 406 | else: 407 | # if the site is UP, we log the timestamp in the database in order to not reprocess it 408 | cur.execute("UPDATE " + TABLEname + " SET StillInvestig= ? WHERE Domain = ? AND Fingerprint = ? ;", 409 | (format(datetime.datetime.utcnow().replace(microsecond=0).isoformat()), hostname, SerialNumber)) 410 | conn.commit 411 | if fqdn_dirs: 412 | # Split hostname into a reverse list (TLD first) 413 | words = hostname.split(".")[::-1] 414 | FQDN_dir = Alerts_dir 415 | for w in words: 416 | FQDN_dir = FQDN_dir + "/" + w 417 | try: 418 | os.makedirs(FQDN_dir, mode=0o777, exist_ok=True) 419 | except FileExistsError: 420 | pass 421 | except: 422 | err = sys.exc_info() 423 | logging.error(" Can't create Alerts_dir: " + str(err)) 424 | print("Creating " + FQDN_dir + "/" + hostname + ".json : " + str(site_infos)) 425 | f = open(FQDN_dir + "/" + hostname + ".json", "w") 426 | else: 427 | print("Creating " + Alerts_dir + "/" + hostname + ".json : " + str(site_infos)) 428 | # log the hostname under the form of a file under the /alerts subdirectory 429 | # + fill the file with informations like ASN/abuse email/IP/web page title etc 430 | # next task: the SOC/Cert has to investigate this host. 431 | f = open(Alerts_dir + "/" + hostname + ".json", "w") 432 | json.dump(site_infos, f, indent=4) 433 | f.close() 434 | if Notification_Destination is not '': 435 | body_site_infos = str(site_infos).replace(', \'', chr(10) + '\'') 436 | body_site_infos = body_site_infos.replace('{', '') 437 | body_site_infos = body_site_infos.replace('}', '') 438 | apobj.notify(title="[CertStreamMonitor] Alert: " + hostname, body=body_site_infos,) 439 | 440 | # Time's up, stop checking entry 441 | else: 442 | hues.warn(" {} - end of monitoring reached ({} days)".format(hostname, Alert_Monitor_timelapse)) 443 | cur.execute("UPDATE " + TABLEname + " SET StillInvestig= '{}{}' WHERE Domain = '{}' AND Fingerprint = '{}' ;".format("Stop checking on ", now.date(), hostname, SerialNumber)) 444 | conn.commit 445 | 446 | return True 447 | 448 | except KeyboardInterrupt: 449 | print("Interrupt received, stopping ...") 450 | print("start - committing, closing DB") 451 | conn.commit 452 | conn.close 453 | print("ending - committing, closing DB") 454 | return False 455 | 456 | except Exception as e: 457 | hues.error("parse_and_scan_all_hostnames function error: {}".format(e)) 458 | return False 459 | 460 | finally: 461 | conn.commit 462 | conn.close 463 | 464 | 465 | def main(): 466 | ConfAnalysis(ConfFile) 467 | 468 | # create a database connection 469 | conn = create_connection(DBFile) 470 | 471 | with conn: 472 | print("Test all domains in DB for Internet Presence:") 473 | print("*********************************************") 474 | parse_and_scan_all_hostnames(TABLEname, Proxy, conn) 475 | 476 | 477 | if __name__ == '__main__': 478 | args_parse() 479 | main() 480 | -------------------------------------------------------------------------------- /useragent_list.txt: -------------------------------------------------------------------------------- 1 | Mozilla/5.0 (Linux; Android 5.1.1; SM-J320FN Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36 2 | Mozilla/5.0 (Linux; Android 6.0; ALE-L21 Build/HuaweiALE-L21) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36 3 | Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 4 | Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 5 | Mozilla/5.0 (Windows NT 6.2; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0 6 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) 7 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G361F-ORANGE Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36 8 | Mozilla/5.0 (Linux; Android 6.0; M50 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36 9 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-J530F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 10 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-T580 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.2 Chrome/51.0.2704.106 Safari/537.36 11 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36 12 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0 13 | Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_2 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A421 Safari/604.1 14 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G531F Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Mobile Safari/537.36 15 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G955F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 16 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7 17 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; NP06; rv:11.0) like Gecko 18 | Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; Lumia 550) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15063 19 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_1_1 like Mac OS X) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0 Mobile/14B100 Safari/602.1 20 | Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-J510FN Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 21 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-J330FN Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 22 | Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 23 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; MATM; rv:11.0) like Gecko 24 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 OPR/47.0.2631.71 25 | Mozilla/5.0 (Windows NT 6.3; Win64; x64; Trident/7.0; rv:11.0) like Gecko 26 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-J320F Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 27 | Mozilla/5.0 (Linux; Android 7.0; SM-G935F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36 28 | Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0 29 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-J320FN Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 30 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-A520F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 31 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 32 | Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0 33 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 34 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 35 | Mozilla/4.0 (compatible; MSIE 4.01; Mac_PowerPC) 36 | Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-A520F Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 37 | Mozilla/5.0 (Linux; Android 7.0; SM-G930F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36 38 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8 39 | Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko 40 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-A310F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 41 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8 42 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36 43 | Mozilla/5.0 (Linux; Android 6.0.1; SM-A310F Build/MMB29K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.98 Mobile Safari/537.36 44 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) 45 | Mozilla/5.0 (Windows NT 6.1; rv:54.0) Gecko/20100101 Firefox/54.0 46 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36 47 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-G531F Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 48 | Mozilla/5.0 (Linux; U; Android 4.4.2; fr-fr; GT-P5210 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30 49 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_2 like Mac OS X) AppleWebKit/602.3.12 (KHTML, like Gecko) Version/10.0 Mobile/14C92 Safari/602.1 50 | Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-T550 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/4.0 Chrome/44.0.2403.133 Safari/537.36 51 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-J320FN-ORANGE Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.5 Chrome/38.0.2125.102 Mobile Safari/537.36 52 | Mozilla/5.0 (Linux; Android 5.1.1; SM-G531F Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36 53 | Mozilla/5.0 (Windows NT 10.0; rv:55.0) Gecko/20100101 Firefox/55.0 54 | Mozilla/5.0 (Linux; Android 4.1.2; GT-S7390G Build/JZO54K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.81 Mobile Safari/537.36 55 | Mozilla/5.0 (Linux; Android 6.0.1; SM-J510FN Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36 56 | Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:24.0) Gecko/20100101 Firefox/24.0 57 | Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 58 | Mozilla/5.0 (Linux; Android 6.0.1; ASUS_X007D Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36 59 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 60 | Mozilla/5.0 (Linux; Android 6.0.1; SM-A500FU Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36 61 | Mozilla/5.0 (Linux; Android 6.0; F3311 Build/37.0.A.2.156) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36 62 | Mozilla/5.0 (Linux; Android 6.0; L-ITE 552 HD Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36 63 | Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 64 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36 OPR/47.0.2631.55 65 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_2 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A456 Safari/602.1 66 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G925F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 67 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) 68 | Mozilla/5.0 (Windows NT 6.3; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0 69 | Mozilla/5.0 (Linux; Android 5.1.1; SM-J320FN Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 70 | Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-A320FL Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 71 | Mozilla/5.0 (Linux; Android 5.1.1; SAMSUNG SM-J320FN Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.5 Chrome/38.0.2125.102 Mobile Safari/537.36 72 | Mozilla/5.0 (Linux; Android 5.1.1; SM-G531F Build/LMY48B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36 73 | Mozilla/5.0 (Linux; Android 5.1.1; SM-J320FN Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36 74 | Mozilla/5.0 (Linux; U; Android 4.3; fr-fr; GT-I9300 Build/JSS15J) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 75 | Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0 76 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 77 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1 78 | Mozilla/5.0 (Linux; Android 6.0.1; SAMSUNG SM-A310F Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 79 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G950F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 80 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G920F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 81 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36 82 | Mozilla/5.0 (iPad; CPU OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.0 Mobile/14F89 Safari/602.1 83 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G935F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 84 | Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53 85 | Mozilla/5.0 (Windows NT 6.1; rv:55.0) Gecko/20100101 Firefox/55.0 86 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 87 | Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko 88 | Mozilla/5.0 (Windows NT 6.3; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0 89 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8 90 | Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 91 | Mozilla/5.0 (Windows NT 6.0; rv:52.0) Gecko/20100101 Firefox/52.0 92 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_2_1 like Mac OS X) AppleWebKit/602.4.6 (KHTML, like Gecko) Version/10.0 Mobile/14D27 Safari/602.1 93 | Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G930F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36 94 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 95 | Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_5 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13G36 Safari/601.1 96 | Mozilla/5.0 (Windows NT 5.1; rv:52.0) Gecko/20100101 Firefox/52.0 97 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 98 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393 99 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0 100 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 101 | Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko 102 | Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0 103 | Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_1 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A402 Safari/604.1 104 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 105 | Mozilla/5.0 (iPad; CPU OS 9_3_5 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13G36 Safari/601.1 106 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0 107 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0 108 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.0 Mobile/14F89 Safari/602.1 109 | Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko 110 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 111 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063 112 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko 113 | Mozilla/5.0 (iPad; CPU OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.0 Mobile/14G60 Safari/602.1 114 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 115 | Mozilla/5.0 (Windows NT 10.0; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0 116 | Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.0 Mobile/14G60 Safari/602.1 117 | -------------------------------------------------------------------------------- /utils/confparser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # This file is a part of CertStreamMonitor 5 | 6 | import os 7 | import sys 8 | import logging 9 | import configparser 10 | 11 | 12 | class ConfParser: 13 | '''Configuration file parser''' 14 | 15 | def __init__(self, Confile=None): 16 | try: 17 | self.config = configparser.ConfigParser() 18 | 19 | with open(Confile, 'r', encoding='utf-8') as f: 20 | self.config.read_file(f) 21 | 22 | # search strings 23 | self.SearchKeywords = self.config['SEARCH']['SearchKeywords'] 24 | try: 25 | self.BlacklistKeywords = self.config['SEARCH']['BlacklistKeywords'] 26 | except KeyError: 27 | self.BlacklistKeywords = str() 28 | 29 | # Databases 30 | self.DBFile = self.config['DATABASE']['DBFile'] 31 | self.TABLEname = self.config['DATABASE']['TABLEname'] 32 | 33 | # Logging 34 | self.LogFile = self.config['LOGGING']['LogFile'] 35 | try: 36 | self.LogLevel = self.config['LOGGING']['LogLevel'] 37 | except: 38 | self.LogLevel = logging.DEBUG 39 | try: 40 | self.LogType = self.config['LOGGING']['LogType'] 41 | except: 42 | self.LogType = 'file' 43 | 44 | # Proxy 45 | try: 46 | self.Proxy = self.config['CONNECT']['Proxy'] 47 | except: 48 | self.Proxy = None 49 | 50 | # Reporting 51 | self.Alerts_dir = self.config['REPORTING']['Alerts_dir'] 52 | 53 | # Reporting Monitor Time laspe 54 | try: 55 | self.Alert_Monitor_timelapse = self.config['REPORTING']['Alert_Monitor_timelapse'] 56 | except: 57 | self.Alert_Monitor_timelapse = "180" 58 | 59 | # Notifications destination 60 | try: 61 | self.Notification_Destination = self.config['REPORTING']['Notification_Destination'] 62 | except: 63 | self.Notification_Destination = None 64 | 65 | # User Agent 66 | self.http_UA = self.config['CONNECT']['http_UA'] 67 | 68 | # User Agent list file 69 | self.UAfile = self.config['CONNECT']['UAfile'] 70 | 71 | # Detection Threshold 72 | try: 73 | self.DetectionThreshold = int( 74 | self.config['SEARCH']['DetectionThreshold']) 75 | except: 76 | self.DetectionThreshold = 1 77 | 78 | # CT logs aggregator server connection 79 | try: 80 | self.ACTServer = self.config['SERVER']['ACTServer'] 81 | except: 82 | logging.error( 83 | " Configuration file Error: You need a CT logs Aggregator server to connect on...") 84 | sys.exit(1) 85 | 86 | try: 87 | self.Proxy_Host = self.config['SERVER']['Proxy_Host'] 88 | except: 89 | self.Proxy_Host = None 90 | 91 | try: 92 | self.Proxy_Port = self.config['SERVER']['Proxy_Port'] 93 | except: 94 | self.Proxy_Port = None 95 | 96 | try: 97 | self.Proxy_Username = self.config[ 98 | 'SERVER']['Proxy_Username'] 99 | except: 100 | self.Proxy_Username = None 101 | 102 | try: 103 | self.Proxy_Password = self.config[ 104 | 'SERVER']['Proxy_Password'] 105 | except: 106 | self.Proxy_Password = None 107 | 108 | # Safe Browsing Status check 109 | try: 110 | self.Safe_Browsing_API_Key = self.config['SAFEBROWSING']['Safe_Browsing_API_Key'] 111 | except: 112 | self.Safe_Browsing_API_Key = '' 113 | 114 | except IOError: 115 | #print("[!!!] Configuration file Error: "+Confile) 116 | logging.error(" Configuration file Error: " + Confile) 117 | 118 | except: 119 | err = sys.exc_info() 120 | #print("[!!!] ConfParser Error: "+str(err)) 121 | logging.error(" ConfParser Error: " + str(err)) 122 | -------------------------------------------------------------------------------- /utils/sqlite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # This file is a part of CertStreamMonitor 5 | 6 | import sqlite3 7 | import sys 8 | 9 | class SqliteCmd(object): 10 | '''Sqlite3 DB commands''' 11 | def __init__(self, DBfile): 12 | self.conn = sqlite3.connect(DBfile) 13 | self.cur = self.conn.cursor() 14 | 15 | ## Main DB operations 16 | def SQLiteCreateTable(self, TABLEname): 17 | '''Creating main Table if not exist''' 18 | self.cur.execute('CREATE TABLE IF NOT EXISTS '+TABLEname+' (Domain TEXT NOT NULL PRIMARY KEY, SAN TEXT, Issuer TEXT, Fingerprint TEXT, Startime TEXT, FirstSeen TEXT, StillInvestig TEXT)') 19 | 20 | def SQLiteInsert(self, TABLEname, Domain, SAN, Issuer, Fingerprint, Startime, FirstSeen): 21 | '''Insert new entry infos''' 22 | self.cur.execute('INSERT OR IGNORE INTO '+TABLEname+' (Domain, SAN, Issuer, Fingerprint, Startime, FirstSeen) VALUES (?,?,?,?,?,?);', (Domain, SAN, Issuer, Fingerprint, Startime, FirstSeen)) 23 | self.conn.commit() 24 | 25 | def SQLiteVerifyEntry(self, TABLEname, Domain): 26 | '''Verify if entry still exist''' 27 | res = self.cur.execute('SELECT EXISTS (SELECT 1 FROM '+TABLEname+' WHERE Domain='+"\""+Domain+"\""+' LIMIT 1);') 28 | fres = res.fetchone()[0] 29 | # 0ô 30 | if fres != 0: 31 | return 1 32 | else: 33 | return 0 34 | 35 | def __del__(self): 36 | try: 37 | self.cur.close() 38 | self.conn.close() 39 | except: 40 | pass 41 | 42 | def SQLiteClose(self): 43 | self.__del__() 44 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # This file is a part of CertStreamMonitor 5 | 6 | import os 7 | import sys 8 | import datetime 9 | 10 | 11 | class TimestampNow: 12 | '''Generate Timestamp''' 13 | def Timestamp(self): 14 | now = datetime.datetime.now().strftime("%c") 15 | return now 16 | 17 | class VerifyPath: 18 | '''Verify or create file if not exist''' 19 | def VerifyOrCreate(self, file): 20 | try: 21 | os.makedirs(os.path.dirname(file), mode=0o777, exist_ok=True) 22 | except FileExistsError: 23 | pass 24 | except: 25 | err = sys.exc_info() 26 | #print("[!!!] VerifyPath class Error: "+str(err)) 27 | logging.error(" VerifyPath class Error: "+str(err)) 28 | --------------------------------------------------------------------------------