├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── certstream ├── __init__.py ├── cli.py └── core.py ├── examples ├── __init__.py ├── echo.py ├── gui.py └── stat_windows.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # dotenv 80 | .env 81 | 82 | # virtualenv 83 | .venv/ 84 | venv/ 85 | ENV/ 86 | 87 | # Spyder project settings 88 | .spyderproject 89 | 90 | # Rope project settings 91 | .ropeproject 92 | 93 | .idea/ 94 | 95 | *.sqlite3 96 | phishfinder/secrets.py 97 | 98 | celerybeat-schedule 99 | .DS_Store 100 | 101 | *.retry -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Cali Dog Security 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include LICENSE 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | dist: 2 | python setup.py sdist upload -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

CertStream-Python

4 |

See SSL certs as they're issued live.

5 |

6 | 7 | **Certstream-python** is a library for interacting with the [certstream network](https://certstream.calidog.io/) to monitor an aggregated feed from a collection of [Certificate Transparency Lists](https://www.certificate-transparency.org/known-logs). 8 | 9 | It leverages the excellent 2/3 compatible [websocket-client](https://github.com/websocket-client/websocket-client) library and supports reconnecting automatically. 10 | 11 | # Installing 12 | 13 | ``` 14 | pip install certstream 15 | ``` 16 | 17 | # Usage 18 | 19 | Usage is about as simple as it gets, simply import the `certstream` module and register a callback with `certstream.listen_for_events`. Once you register a callback it will be called with 2 arguments - `message`, and `context`. 20 | 21 | ```python 22 | import logging 23 | import sys 24 | import datetime 25 | import certstream 26 | 27 | def print_callback(message, context): 28 | logging.debug("Message -> {}".format(message)) 29 | 30 | if message['message_type'] == "heartbeat": 31 | return 32 | 33 | if message['message_type'] == "certificate_update": 34 | all_domains = message['data']['leaf_cert']['all_domains'] 35 | 36 | if len(all_domains) == 0: 37 | domain = "NULL" 38 | else: 39 | domain = all_domains[0] 40 | 41 | sys.stdout.write(u"[{}] {} (SAN: {})\n".format(datetime.datetime.now().strftime('%m/%d/%y %H:%M:%S'), domain, ", ".join(message['data']['leaf_cert']['all_domains'][1:]))) 42 | sys.stdout.flush() 43 | 44 | logging.basicConfig(format='[%(levelname)s:%(name)s] %(asctime)s - %(message)s', level=logging.INFO) 45 | 46 | certstream.listen_for_events(print_callback, url='wss://certstream.calidog.io/') 47 | ``` 48 | 49 | You can also register an `on_open` and `on_error` handler as well, which do exactly what you'd expect: 50 | 51 | ```python 52 | 53 | import certstream 54 | 55 | def print_callback(message, context): 56 | print("Received messaged -> {}".format(message)) 57 | 58 | def on_open(): 59 | print("Connection successfully established!") 60 | 61 | def on_error(instance, exception): 62 | # Instance is the CertStreamClient instance that barfed 63 | print("Exception in CertStreamClient! -> {}".format(exception)) 64 | 65 | certstream.listen_for_events(print_callback, on_open=on_open, on_error=on_error, url='wss://certstream.calidog.io/') 66 | 67 | ``` 68 | 69 | We also support connection via http proxy: 70 | 71 | ```python 72 | import certstream 73 | 74 | def print_callback(message, context): 75 | print("Received messaged -> {}".format(message)) 76 | 77 | certstream.listen_for_events(print_callback, url='wss://certstream.calidog.io/', http_proxy_host="proxy_host", http_proxy_port=8080, http_proxy_auth=("user", "password")) 78 | ``` 79 | 80 | Need more connection options? Take a look at `**kwargs` in `certstream.listen_for_events`. We pass it to `run_forever` method of [websocket-client](https://github.com/websocket-client/websocket-client/blob/87861f951d1a65ed5d9080f7aaaf44310f376c56/websocket/_app.py#L169-L192). 81 | 82 | e.g. to skip SSL/TLS verification 83 | ```python 84 | import certstream 85 | import ssl 86 | 87 | certstream.listen_for_events(print_callback, url='wss://certstream.calidog.io/', sslopt={"cert_reqs":ssl.CERT_NONE}) 88 | ``` 89 | 90 | # Example data structure 91 | 92 | The data structure coming from CertStream looks like this: 93 | 94 | ``` 95 | { 96 | "message_type": "certificate_update", 97 | "data": { 98 | "update_type": "X509LogEntry", 99 | "leaf_cert": { 100 | "subject": { 101 | "aggregated": "/CN=app.theaffairsite.com", 102 | "C": null, 103 | "ST": null, 104 | "L": null, 105 | "O": null, 106 | "OU": null, 107 | "CN": "app.theaffairsite.com" 108 | }, 109 | "extensions": { 110 | "keyUsage": "Digital Signature, Key Encipherment", 111 | "extendedKeyUsage": "TLS Web Server Authentication, TLS Web Client Authentication", 112 | "basicConstraints": "CA:FALSE", 113 | "subjectKeyIdentifier": "01:BE:17:27:B8:D8:26:EF:E1:5C:7A:F6:14:A7:EA:B5:D0:D8:B5:9B", 114 | "authorityKeyIdentifier": "keyid:A8:4A:6A:63:04:7D:DD:BA:E6:D1:39:B7:A6:45:65:EF:F3:A8:EC:A1\n", 115 | "authorityInfoAccess": "OCSP - URI:http://ocsp.int-x3.letsencrypt.org\nCA Issuers - URI:http://cert.int-x3.letsencrypt.org/\n", 116 | "subjectAltName": "DNS:app.theaffairsite.com", 117 | "certificatePolicies": "Policy: 2.23.140.1.2.1\nPolicy: 1.3.6.1.4.1.44947.1.1.1\n CPS: http://cps.letsencrypt.org\n User Notice:\n Explicit Text: This Certificate may only be relied upon by Relying Parties and only in accordance with the Certificate Policy found at https://letsencrypt.org/repository/\n" 118 | }, 119 | "not_before": 1509908649.0, 120 | "not_after": 1517684649.0, 121 | "serial_number": "33980d1bef9b6a76cfc708e3139f55f33c5", 122 | "fingerprint": "95:CA:86:6B:B4:98:59:D2:EC:C7:CA:E8:42:70:80:0B:18:03:C7:75", 123 | "as_der": "MIIFDTCCA/WgAwIBAgISAzmA0b75tqds/HCOMTn1XzPFMA0GCSqGSIb3DQEBCwUAMEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQDExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMzAeFw0xNzExMDUxOTA0MDlaFw0xODAyMDMxOTA0MDlaMCAxHjAcBgNVBAMTFWFwcC50aGVhZmZhaXJzaXRlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALtVFBtTDAMq5yt/fRujvt3XbxjAb58NG6ThmXiFN/rDyysKt4tsqYcOQRZc5D/z4Pm8hI3lgLgmiZdxJF6zUnJ7GoYGdpPwItmYHmp1rWo735NNw16zFMKw9KPi1l+aiKQqZQA9hcgXpbWoyoIZBwHS5K5Id6/uXfLk//9nRxaKqDQzB1ZokIzlv0u+hJxKA4Q+JyOiZvfQKDBcC9lEXsNJ74MTkCwu75qjvHYHB4jSrb3aiCxn7q934bI+CFFjCK1adyGJVnckXOcumZrPo4c8GL0Fc1uwZ/PdLvU9/4d/PpbSHdaN94B3bVxCjio/KnSJ8QNJo60QoEOZ60aCFN0CAwEAAaOCAhUwggIRMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUAb4XJ7jYJu/hXHr2FKfqtdDYtZswHwYDVR0jBBgwFoAUqEpqYwR93brm0Tm3pkVl7/Oo7KEwbwYIKwYBBQUHAQEEYzBhMC4GCCsGAQUFBzABhiJodHRwOi8vb2NzcC5pbnQteDMubGV0c2VuY3J5cHQub3JnMC8GCCsGAQUFBzAChiNodHRwOi8vY2VydC5pbnQteDMubGV0c2VuY3J5cHQub3JnLzAgBgNVHREEGTAXghVhcHAudGhlYWZmYWlyc2l0ZS5jb20wgf4GA1UdIASB9jCB8zAIBgZngQwBAgEwgeYGCysGAQQBgt8TAQEBMIHWMCYGCCsGAQUFBwIBFhpodHRwOi8vY3BzLmxldHNlbmNyeXB0Lm9yZzCBqwYIKwYBBQUHAgIwgZ4MgZtUaGlzIENlcnRpZmljYXRlIG1heSBvbmx5IGJlIHJlbGllZCB1cG9uIGJ5IFJlbHlpbmcgUGFydGllcyBhbmQgb25seSBpbiBhY2NvcmRhbmNlIHdpdGggdGhlIENlcnRpZmljYXRlIFBvbGljeSBmb3VuZCBhdCBodHRwczovL2xldHNlbmNyeXB0Lm9yZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEASpYg0ISnbyXpqYYzgpLdc8o6GZwKrMDrTARm63aT+2L88s2Ff6JlMz4XRH3v4iihLpLVUDoiXbNUyggyVqbkQLFtHtgj8ScLvWku8n7l7lp6DpV7j3h6byM2K6a+jasJKplL+Zbqzng0RaJlFFnnBXYE9a5BW3JlOzNbOMUOSKTZSB0+6pmeohU1DhNiPQNqT2katRu0LLGbwtcEpsWyScVc3VkJVu1l0QNq8gC+F3C2MpBtiSjjz6umP1F1z+sXhUx9dFVzJ2nSk7XxZaH+DW4OAb6zjwqqYjjf2S0VQM398URhfYzLQX6xEyDuZG4W58g5SMtOWDnslPhlIax3LA==", 124 | "all_domains": [ 125 | "app.theaffairsite.com" 126 | ] 127 | }, 128 | "cert_index": 27910635, 129 | "seen": 1509912803.959279, 130 | "source": { 131 | "url": "sabre.ct.comodo.com", 132 | "name": "Comodo 'Sabre' CT log" 133 | } 134 | } 135 | } 136 | ``` 137 | -------------------------------------------------------------------------------- /certstream/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import listen_for_events -------------------------------------------------------------------------------- /certstream/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import datetime 3 | import json 4 | import logging 5 | import sys 6 | import termcolor 7 | 8 | from signal import signal, SIGPIPE, SIG_DFL 9 | 10 | import certstream 11 | 12 | parser = argparse.ArgumentParser(description='Connect to the CertStream and process CTL list updates.') 13 | 14 | parser.add_argument('--json', action='store_true', help='Output raw JSON to the console.') 15 | parser.add_argument('--full', action='store_true', help='Output all SAN addresses as well') 16 | parser.add_argument('--disable-colors', action='store_true', help='Disable colors when writing a human readable ') 17 | parser.add_argument('--verbose', action='store_true', default=False, dest='verbose', help='Display debug logging.') 18 | parser.add_argument('--url', default="wss://certstream.calidog.io", dest='url', help='Connect to a certstream server.') 19 | 20 | def main(): 21 | args = parser.parse_args() 22 | 23 | # Ignore broken pipes 24 | signal(SIGPIPE, SIG_DFL) 25 | 26 | log_level = logging.INFO 27 | if args.verbose: 28 | log_level = logging.DEBUG 29 | 30 | logging.basicConfig(format='[%(levelname)s:%(name)s] %(asctime)s - %(message)s', level=log_level) 31 | 32 | def _handle_messages(message, context): 33 | if args.json: 34 | sys.stdout.flush() 35 | sys.stdout.write(json.dumps(message) + "\n") 36 | sys.stdout.flush() 37 | else: 38 | if args.disable_colors: 39 | logging.debug("Starting normal output.") 40 | payload = "{} {} - {} {}\n".format( 41 | "[{}]".format(datetime.datetime.fromtimestamp(message['data']['seen']).isoformat()), 42 | message['data']['source']['url'], 43 | message['data']['leaf_cert']['subject']['CN'], 44 | "[{}]".format(", ".join(message['data']['leaf_cert']['all_domains'])) if args.full else "" 45 | ) 46 | 47 | sys.stdout.write(payload) 48 | else: 49 | logging.debug("Starting colored output.") 50 | payload = "{} {} - {} {}\n".format( 51 | termcolor.colored("[{}]".format(datetime.datetime.fromtimestamp(message['data']['seen']).isoformat()), 'cyan', attrs=["bold", ]), 52 | termcolor.colored(message['data']['source']['url'], 'blue', attrs=["bold",]), 53 | termcolor.colored(message['data']['leaf_cert']['subject']['CN'], 'green', attrs=["bold",]), 54 | termcolor.colored("[", 'blue') + "{}".format( 55 | termcolor.colored(", ", 'blue').join( 56 | [termcolor.colored(x, 'white', attrs=["bold",]) for x in message['data']['leaf_cert']['all_domains']] 57 | ) 58 | ) + termcolor.colored("]", 'blue') if args.full else "", 59 | ) 60 | sys.stdout.write(payload) 61 | 62 | sys.stdout.flush() 63 | 64 | certstream.listen_for_events(_handle_messages, args.url, skip_heartbeats=True) 65 | 66 | if __name__ == "__main__": 67 | main() 68 | -------------------------------------------------------------------------------- /certstream/core.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import json 4 | import logging 5 | 6 | import time 7 | from websocket import WebSocketApp 8 | 9 | class Context(dict): 10 | """dot.notation access to dictionary attributes""" 11 | __getattr__ = dict.get 12 | __setattr__ = dict.__setitem__ 13 | __delattr__ = dict.__delitem__ 14 | 15 | class CertStreamClient(WebSocketApp): 16 | _context = Context() 17 | 18 | def __init__(self, message_callback, url, skip_heartbeats=True, on_open=None, on_error=None): 19 | self.message_callback = message_callback 20 | self.skip_heartbeats = skip_heartbeats 21 | self.on_open_handler = on_open 22 | self.on_error_handler = on_error 23 | super(CertStreamClient, self).__init__( 24 | url=url, 25 | on_open=self._on_open, 26 | on_message=self._on_message, 27 | on_error=self._on_error, 28 | ) 29 | 30 | def _on_open(self, _): 31 | certstream_logger.info("Connection established to CertStream! Listening for events...") 32 | if self.on_open_handler: 33 | self.on_open_handler() 34 | 35 | def _on_message(self, _, message): 36 | frame = json.loads(message) 37 | 38 | if frame.get('message_type', None) == "heartbeat" and self.skip_heartbeats: 39 | return 40 | 41 | self.message_callback(frame, self._context) 42 | 43 | def _on_error(self, _, ex): 44 | if type(ex) == KeyboardInterrupt: 45 | raise 46 | if self.on_error_handler: 47 | self.on_error_handler(ex) 48 | certstream_logger.error("Error connecting to CertStream - {} - Sleeping for a few seconds and trying again...".format(ex)) 49 | 50 | def listen_for_events(message_callback, url, skip_heartbeats=True, setup_logger=True, on_open=None, on_error=None, **kwargs): 51 | try: 52 | while True: 53 | c = CertStreamClient(message_callback, url, skip_heartbeats=skip_heartbeats, on_open=on_open, on_error=on_error) 54 | c.run_forever(ping_interval=15, **kwargs) 55 | time.sleep(5) 56 | except KeyboardInterrupt: 57 | certstream_logger.info("Kill command received, exiting!!") 58 | 59 | certstream_logger = logging.getLogger('certstream') 60 | certstream_logger.setLevel(logging.INFO) 61 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CaliDog/certstream-python/97eb5b37e2c2077495e20d6df3459088a5262b48/examples/__init__.py -------------------------------------------------------------------------------- /examples/echo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import datetime 4 | import certstream 5 | 6 | def print_callback(message, context): 7 | logging.debug("Message -> {}".format(message)) 8 | 9 | if message['message_type'] == "heartbeat": 10 | return 11 | 12 | if message['message_type'] == "certificate_update": 13 | all_domains = message['data']['leaf_cert']['all_domains'] 14 | 15 | if len(all_domains) == 0: 16 | domain = "NULL" 17 | else: 18 | domain = all_domains[0] 19 | 20 | sys.stdout.write(u"[{}] {} (SAN: {})\n".format(datetime.datetime.now().strftime('%m/%d/%y %H:%M:%S'), domain, ", ".join(message['data']['leaf_cert']['all_domains'][1:]))) 21 | sys.stdout.flush() 22 | 23 | logging.basicConfig(format='[%(levelname)s:%(name)s] %(asctime)s - %(message)s', level=logging.INFO) 24 | 25 | certstream.listen_for_events(print_callback, url='wss://certstream.calidog.io/') -------------------------------------------------------------------------------- /examples/gui.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import Queue 3 | import json 4 | 5 | import os 6 | 7 | import itertools 8 | import urwid 9 | import urwid.curses_display 10 | import threading 11 | import certstream 12 | 13 | def show_or_exit(key): 14 | if key in ('q', 'Q'): 15 | raise urwid.ExitMainLoop() 16 | 17 | """ 18 | This is the very hacky result of a weekend fighting with Urwid to create a more gui-driven experience for CertStream (and an 19 | attempt to replicate the feed watcher on certstream.calidog.io). I spent more time than I care to admit to messing with this 20 | and hit my breaking point :-/ 21 | 22 | If anyone else feels like taking this up let me know, and I'll be happy to help! Otherwise this will inevitably be garbage 23 | collected at some point in the future. 24 | """ 25 | 26 | urwid.set_encoding("UTF-8") 27 | 28 | PALETTE = [ 29 | ('headings', 'white,underline', 'black', 'bold,underline'), # bold text in monochrome mode 30 | ('body_text', 'light green', 'black'), 31 | ('heartbeat_active', 'light red', 'black'), 32 | ('heartbeat_inactive', 'light green', 'black'), 33 | ('buttons', 'yellow', 'dark green', 'standout'), 34 | ('section_text', 'body_text'), # alias to body_text 35 | ] 36 | 37 | class CertStreamGui(object): 38 | INTRO_MESSAGE = u""" 39 | _____ _ _____ _ 40 | / ____| | | / ____| | 41 | | | ___ _ __| |_| (___ | |_ _ __ ___ __ _ _ __ ___ 42 | | | / _ \ '__| __|\___ \| __| '__/ _ \/ _` | '_ ` _ \ 43 | | |___| __/ | | |_ ____) | |_| | | __/ (_| | | | | | | 44 | \_____\___|_| \__|_____/ \__|_| \___|\__,_|_| |_| |_| 45 | 46 | Welcome to CertStream CLI 1.0! 47 | 48 | We're waiting on certificates to come in, so hold tight! 49 | 50 | Protip: Looking for the old CertStream CLI behavior? Use the --grep flag! 51 | 52 | {} 53 | """ 54 | 55 | COUNTER_FORMAT = u"┤ {}/{} ├" 56 | 57 | FOOTER_START = "Certstream 1.0 | " 58 | 59 | HEARTBEAT_ICON = u'\u2764' 60 | 61 | def __init__(self): 62 | self.message_queue = Queue.Queue() 63 | self.message_list = [] 64 | self.counter_text = urwid.Text(self.COUNTER_FORMAT.format('0', '0')) 65 | self.seen_message = False 66 | self._heartbeat_is_animating = False 67 | self.setup_widgets() 68 | self.setup_certstream_listener() 69 | self._animate_waiter() 70 | 71 | 72 | def setup_widgets(self): 73 | self.intro_frame = urwid.LineBox( 74 | urwid.Filler( 75 | urwid.Text(('body_text', self.INTRO_MESSAGE.format("")), align=urwid.CENTER), 76 | valign=urwid.MIDDLE, 77 | ) 78 | ) 79 | 80 | self.frame = urwid.Frame( 81 | body=self.intro_frame, 82 | footer=urwid.Text( 83 | [self.FOOTER_START, ('heartbeat_inactive', self.HEARTBEAT_ICON)], 84 | align=urwid.CENTER 85 | ) 86 | ) 87 | 88 | self.loop = urwid.MainLoop( 89 | urwid.AttrMap(self.frame, 'body_text'), 90 | unhandled_input=show_or_exit, 91 | palette=PALETTE, 92 | ) 93 | 94 | self.list_walker = urwid.SimpleListWalker(self.message_list) 95 | self.list_box = urwid.ListBox(self.list_walker) 96 | urwid.connect_signal(self.list_walker, "modified", self.item_focused) 97 | 98 | def setup_certstream_listener(self): 99 | self.draw_trigger = self.loop.watch_pipe(self.process_trigger) 100 | 101 | self.certstream_thread = threading.Thread( 102 | target=certstream.listen_for_events, 103 | kwargs={ 104 | "message_callback": self.cert_processor, 105 | "skip_heartbeats": False, 106 | # "on_connect": on_connect 107 | }, 108 | ) 109 | self.certstream_thread.setDaemon(True) 110 | self.certstream_thread.start() 111 | 112 | def cert_processor(self, message, context): 113 | self.message_queue.put(message) 114 | os.write(self.draw_trigger, "TRIGGER") 115 | 116 | def process_trigger(self, _): 117 | while True: 118 | try: 119 | message = self.message_queue.get_nowait() 120 | self.process_message(message) 121 | except Queue.Empty: 122 | break 123 | 124 | self.loop.draw_screen() 125 | 126 | def _animate_waiter(self): 127 | if self.seen_message: 128 | return 129 | 130 | skel = u"|{}==={}|" 131 | 132 | WIDTH = 28 133 | 134 | cycle = itertools.cycle(range(1, WIDTH) + list(reversed(range(0, WIDTH-1)))) 135 | 136 | def _anim(loop, args): 137 | INTRO_MESSAGE, gui = args 138 | if gui.seen_message: 139 | return 140 | 141 | step = next(cycle) 142 | 143 | text = INTRO_MESSAGE.format( 144 | skel.format( 145 | " " * step, 146 | " " * ((WIDTH - step) - 1) 147 | ) 148 | ) 149 | 150 | gui.intro_frame.original_widget.original_widget.set_text(text) 151 | 152 | loop.set_alarm_in(0.1, _anim, (INTRO_MESSAGE, gui)) 153 | 154 | self.loop.set_alarm_in(0.1, _anim, (self.INTRO_MESSAGE, self)) 155 | 156 | def _animate_heartbeat(self, _=None, stage=0): 157 | if stage == 0: 158 | self.frame.set_footer( 159 | urwid.Text( 160 | [self.FOOTER_START, ('heartbeat_active', self.HEARTBEAT_ICON)], 161 | align=urwid.CENTER 162 | ) 163 | ) 164 | self.loop.set_alarm_in(0.5, self._animate_heartbeat, 1) 165 | elif stage == 1: 166 | self.frame.set_footer( 167 | urwid.Text( 168 | [self.FOOTER_START, ('heartbeat_inactive', self.HEARTBEAT_ICON)], 169 | align=urwid.CENTER 170 | ) 171 | ) 172 | self._heartbeat_is_animating = False 173 | 174 | def focus_right_panel(self, button, user_data): 175 | pass 176 | 177 | def item_focused(self): 178 | total = len(self.list_box.body) 179 | 180 | logging.info("item_focused called...") 181 | 182 | logging.info("Len {} | {}".format( 183 | len(self.list_walker), 184 | self.list_box.get_focus()[1] 185 | ) 186 | ) 187 | 188 | self.counter_text.set_text( 189 | self.COUNTER_FORMAT.format( 190 | total - self.list_box.get_focus()[1], 191 | total 192 | ) 193 | ) 194 | 195 | self.right_text.set_text( 196 | json.dumps( 197 | self.list_walker[self.list_box.get_focus()[1]].original_widget.user_data['data']['leaf_cert'], 198 | indent=4 199 | ) 200 | ) 201 | 202 | def process_message(self, message): 203 | if message['message_type'] == 'heartbeat' and not self._heartbeat_is_animating: 204 | self._heartbeat_is_animating = True 205 | self._animate_heartbeat() 206 | 207 | if not self.seen_message and message['message_type'] == 'certificate_update': 208 | self.right_text = urwid.Text('') 209 | self.frame.set_body( 210 | urwid.Columns( 211 | widget_list=[ 212 | urwid.Pile( 213 | [ 214 | SidelessLineBox( 215 | self.list_box, 216 | title="CertStream Messages", 217 | bline="", 218 | ), 219 | ( 220 | 'flow', 221 | urwid.Columns([ 222 | ('fixed', 6, urwid.Text(u'└─────')), 223 | ('flow', self.counter_text), 224 | Divider('─'), 225 | ('fixed', 1, urwid.Text(u'┘')), 226 | ]) 227 | ) 228 | ] 229 | ), 230 | SidelessLineBox( 231 | urwid.Filler( 232 | self.right_text, 233 | valign=urwid.TOP 234 | ), 235 | title="Parsed JSON" 236 | ) 237 | ], 238 | ) 239 | ) 240 | self.seen_message = True 241 | 242 | if message['message_type'] == 'certificate_update': 243 | _, original_offset = self.list_box.get_focus() 244 | self.list_walker.insert(0, 245 | urwid.AttrMap( 246 | FauxButton( 247 | "[{}] {} - {}".format( 248 | message['data']['cert_index'], 249 | message['data']['source']['url'], 250 | message['data']['leaf_cert']['subject']['CN'], 251 | ), 252 | user_data=message, 253 | on_press=self.focus_right_panel 254 | ), 255 | '', 256 | focus_map='buttons' 257 | ) 258 | ) 259 | 260 | self.counter_text.set_text( 261 | self.COUNTER_FORMAT.format( 262 | original_offset, 263 | len(self.list_box.body) - 1 264 | ) 265 | ) 266 | 267 | offset = (len(self.list_box.body) - 1) 268 | 269 | logging.info("Disconnecting") 270 | urwid.disconnect_signal(self.list_walker, "modified", self.item_focused) 271 | logging.info("Setting focus") 272 | self.list_walker.set_focus(offset) 273 | logging.info("Reconnecting") 274 | urwid.connect_signal(self.list_walker, "modified", self.item_focused) 275 | logging.info("Done") 276 | 277 | self.right_text.set_text( 278 | json.dumps( 279 | self.list_walker[offset - self.list_box.get_focus()[1] - 1].original_widget.user_data['data']['leaf_cert'], 280 | indent=4 281 | ) 282 | ) 283 | 284 | self.loop.draw_screen() 285 | 286 | def run(self): 287 | self.loop.run() 288 | 289 | class Selector(urwid.SelectableIcon): 290 | def __init__(self, text, cursor_position): 291 | super(Selector, self).__init__(text, cursor_position) 292 | 293 | import logging 294 | logging.basicConfig(filename='out.log', level=logging.DEBUG) 295 | 296 | urwid.escape.SHOW_CURSOR = '' 297 | 298 | gui = CertStreamGui() 299 | 300 | from urwid.widget import WidgetWrap, Divider, SolidFill, Text 301 | from urwid.container import Pile, Columns 302 | from urwid.decoration import WidgetDecoration 303 | 304 | class FauxButton(urwid.Button): 305 | button_left = Text("") 306 | button_right = Text("") 307 | 308 | def __init__(self, label, on_press=None, user_data=None): 309 | self.user_data = user_data 310 | super(FauxButton, self).__init__(label, on_press, user_data) 311 | 312 | 313 | class SidelessLineBox(WidgetDecoration, WidgetWrap): 314 | 315 | def __init__(self, original_widget, title="", title_align="center", 316 | tlcorner='┌', tline='─', lline='│', 317 | trcorner='┐', blcorner='└', rline='│', 318 | bline='─', brcorner='┘'): 319 | """ 320 | Draw a line around original_widget. 321 | Use 'title' to set an initial title text with will be centered 322 | on top of the box. 323 | Use `title_align` to align the title to the 'left', 'right', or 'center'. 324 | The default is 'center'. 325 | You can also override the widgets used for the lines/corners: 326 | tline: top line 327 | bline: bottom line 328 | lline: left line 329 | rline: right line 330 | tlcorner: top left corner 331 | trcorner: top right corner 332 | blcorner: bottom left corner 333 | brcorner: bottom right corner 334 | .. note:: This differs from the vanilla urwid LineBox by discarding 335 | the a line if the middle of the line is set to either None or the 336 | empty string. 337 | """ 338 | 339 | if tline: 340 | tline = Divider(tline) 341 | if bline: 342 | bline = Divider(bline) 343 | if lline: 344 | lline = SolidFill(lline) 345 | if rline: 346 | rline = SolidFill(rline) 347 | tlcorner, trcorner = Text(tlcorner), Text(trcorner) 348 | blcorner, brcorner = Text(blcorner), Text(brcorner) 349 | 350 | if not tline and title: 351 | raise ValueError('Cannot have a title when tline is unset') 352 | 353 | self.title_widget = Text(self.format_title(title)) 354 | 355 | if tline: 356 | if title_align not in ('left', 'center', 'right'): 357 | raise ValueError('title_align must be one of "left", "right", or "center"') 358 | if title_align == 'left': 359 | tline_widgets = [('flow', self.title_widget), tline] 360 | else: 361 | tline_widgets = [tline, ('flow', self.title_widget)] 362 | if title_align == 'center': 363 | tline_widgets.append(tline) 364 | self.tline_widget = Columns(tline_widgets) 365 | top = Columns([ 366 | ('fixed', 1, tlcorner), 367 | self.tline_widget, 368 | ('fixed', 1, trcorner) 369 | ]) 370 | 371 | else: 372 | self.tline_widget = None 373 | top = None 374 | 375 | middle_widgets = [] 376 | if lline: 377 | middle_widgets.append(('fixed', 1, lline)) 378 | middle_widgets.append(original_widget) 379 | focus_col = len(middle_widgets) - 1 380 | if rline: 381 | middle_widgets.append(('fixed', 1, rline)) 382 | 383 | middle = Columns(middle_widgets, 384 | box_columns=[0, 2], focus_column=focus_col) 385 | 386 | if bline: 387 | bottom = Columns([ 388 | ('fixed', 1, blcorner), bline, ('fixed', 1, brcorner) 389 | ]) 390 | else: 391 | bottom = None 392 | 393 | pile_widgets = [] 394 | if top: 395 | pile_widgets.append(('flow', top)) 396 | pile_widgets.append(middle) 397 | focus_pos = len(pile_widgets) - 1 398 | if bottom: 399 | pile_widgets.append(('flow', bottom)) 400 | pile = Pile(pile_widgets, focus_item=focus_pos) 401 | 402 | WidgetDecoration.__init__(self, original_widget) 403 | WidgetWrap.__init__(self, pile) 404 | 405 | def format_title(self, text): 406 | if len(text) > 0: 407 | return "┤ {} ├".format(text) 408 | else: 409 | return "" 410 | 411 | def set_title(self, text): 412 | if not self.title_widget: 413 | raise ValueError('Cannot set title when tline is unset') 414 | self.title_widget.set_text(self.format_title(text)) 415 | self.tline_widget._invalidate() 416 | 417 | 418 | gui.run() 419 | 420 | 421 | -------------------------------------------------------------------------------- /examples/stat_windows.py: -------------------------------------------------------------------------------- 1 | import time 2 | from collections import deque 3 | 4 | import certstream 5 | 6 | import logging 7 | 8 | logger = logging.getLogger('stat_counter') 9 | 10 | NUM_MINUTES = 1 11 | INTERVAL = NUM_MINUTES * 60 12 | 13 | def callback(message, context): 14 | if not context.get('edge'): 15 | context.edge = time.time() + INTERVAL 16 | context.counter = deque() 17 | context.heartbeats = deque() 18 | 19 | if message['message_type'] != "heartbeat": 20 | context.counter.append(message) 21 | else: 22 | context.heartbeats.append(message) 23 | 24 | if time.time() > context['edge']: 25 | logger.info( 26 | "Edge has been broken, writing out. {} results for the last {} minute/s ({} heartbeats)".format( 27 | len(context.counter), 28 | NUM_MINUTES, 29 | len(context.heartbeats) 30 | ) 31 | ) 32 | 33 | with open('/tmp/out.csv', 'a') as f: 34 | f.write("{},{}\n".format(time.time(), len(context['counter']))) 35 | 36 | context.counter.clear() 37 | context.heartbeats.clear() 38 | context.edge = time.time() + INTERVAL 39 | 40 | certstream.listen_for_events(callback, url='wss://certstream.calidog.io/', skip_heartbeats=False) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | websocket-client>=0.58.0 2 | termcolor 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import os 3 | 4 | here = os.path.abspath(os.path.dirname(__file__)) 5 | 6 | with open('requirements.txt') as f: 7 | dependencies = f.read().splitlines() 8 | 9 | long_description = """ 10 | Certstream is a library to connect to the certstream network (certstream.calidog.io). 11 | 12 | It supports automatic reconnection when networks issues occur, and should be stable for long-running jobs. 13 | """ 14 | 15 | setup( 16 | name='certstream', 17 | version="1.12", 18 | url='https://github.com/CaliDog/certstream-python/', 19 | author='Ryan Sears', 20 | install_requires=dependencies, 21 | setup_requires=dependencies, 22 | author_email='ryan@calidog.io', 23 | description='CertStream is a library for receiving certificate transparency list updates in real time.', 24 | long_description=long_description, 25 | packages=['certstream',], 26 | include_package_data=True, 27 | entry_points={ 28 | 'console_scripts': [ 29 | 'certstream = certstream.cli:main', 30 | ], 31 | }, 32 | license = "MIT", 33 | classifiers = [ 34 | "Development Status :: 4 - Beta", 35 | "License :: OSI Approved :: MIT License", 36 | "Topic :: Internet :: WWW/HTTP", 37 | "Topic :: Security :: Cryptography", 38 | "Environment :: Console", 39 | "Operating System :: MacOS :: MacOS X", 40 | "Operating System :: POSIX", 41 | "Programming Language :: Python :: 3", 42 | "Programming Language :: Python :: 2", 43 | "Framework :: AsyncIO" 44 | ], 45 | ) 46 | --------------------------------------------------------------------------------