├── .gitignore ├── nzbverify ├── __init__.py ├── conf.py ├── thread.py ├── nntp.py └── cmdline.py ├── bin └── nzbverify ├── setup.py ├── PKG-INFO └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /nzbverify/__init__.py: -------------------------------------------------------------------------------- 1 | import nntp 2 | import thread 3 | 4 | __author__ = "Weston Nielson " 5 | __version__ = "0.2.1" 6 | -------------------------------------------------------------------------------- /bin/nzbverify: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import logging 3 | 4 | if __name__ == "__main__": 5 | from nzbverify import cmdline 6 | 7 | #logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) 8 | cmdline.run() 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | import nzbverify 4 | 5 | setup( 6 | name='nzbverify', 7 | version=nzbverify.__version__, 8 | author=nzbverify.__author__.rsplit(' ', 1)[0], 9 | author_email=nzbverify.__author__.split(' ', 2)[-1], 10 | packages=['nzbverify'], 11 | url='http://pypi.python.org/pypi/nzbverify/', 12 | license='LICENSE', 13 | description='Utility for verifying the completeness of an NZB.', 14 | long_description=open('README.md').read(), 15 | scripts=['bin/nzbverify'] 16 | ) 17 | -------------------------------------------------------------------------------- /nzbverify/conf.py: -------------------------------------------------------------------------------- 1 | import netrc 2 | import os 3 | 4 | DEFAULT_CONFIG_PATHS = ['~/.nzbverify', '~/.netrc'] 5 | 6 | def get_config(config=None): 7 | config_paths = [] 8 | if config is not None: 9 | config_paths.append(config) 10 | config_paths.extend(DEFAULT_CONFIG_PATHS) 11 | 12 | conf = None 13 | for path in config_paths: 14 | if path.startswith('~'): 15 | path = os.path.expanduser(path) 16 | try: 17 | conf = netrc.netrc(path) 18 | break 19 | except: 20 | pass 21 | 22 | return conf -------------------------------------------------------------------------------- /PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.0 2 | Name: nzbverify 3 | Version: 0.2.1 4 | Summary: Utility for verifying the completeness of an NZB. 5 | Home-page: http://pypi.python.org/pypi/nzbverify/ 6 | Author: Weston Nielson 7 | Author-email: 8 | License: LICENSE 9 | Description: nzbverify 10 | ========= 11 | 12 | ``nzbverify`` is a command-line tool and library for verifying the integrity of 13 | an NZB file. It is capable of supporting both standard and SSL-encrypted NNTP 14 | connections and employs threads for increased verification speed. 15 | Platform: UNKNOWN 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nzbverify 2 | 3 | `nzbverify` is a command-line tool and library for verifying the integrity of 4 | an NZB file. It is capable of supporting both standard and SSL-encrypted NNTP 5 | connections and employs threads for increased verification speed. 6 | 7 | ## Usage 8 | 9 | ``` 10 | nzbverify -s news.server.com -u myusername -p mypassword -n40 test.nzb 11 | nzbverify version 0.2.1, Copyright (C) 2012 Weston Nielson 12 | Created 40 threads 13 | Parsing NZB: test.nzb 14 | Found 207 files and 27866 segments totalling 10.31 GB 15 | Available: 26632 [100.00%], Missing: 0 [0.00%], Total: 26632 [100.00%] 16 | Result: all 27866 segments available 17 | Verification took 12.1951680183 seconds 18 | ``` 19 | -------------------------------------------------------------------------------- /nzbverify/thread.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import nntp 3 | import nntplib 4 | 5 | def stop_threads(threads): 6 | """ 7 | Stops all threads and disconnects each NNTP connection. 8 | """ 9 | for thread in threads: 10 | thread.stop = True 11 | thread.join() 12 | 13 | class SegmentCheckerThread(threading.Thread): 14 | """ 15 | Threaded NZB Segment Checker. 16 | """ 17 | def __init__(self, num, segments, missing, credentials): 18 | threading.Thread.__init__(self) 19 | self.num = num 20 | self.segments = segments # Queue.Queue 21 | self.missing = missing # Queue.Queue 22 | self.credentials = credentials 23 | self.stop = False # Set to True to stop thread 24 | 25 | def run(self): 26 | self.server = nntp.NNTP(**self.credentials) 27 | try: 28 | while True: 29 | if self.stop: 30 | self.server.quit() 31 | return 32 | 33 | # Try to grab a segment from queue 34 | f, segment, bytes = self.segments.get(False) 35 | 36 | # Check for the article on the server 37 | try: 38 | self.server.stat(segment) 39 | #print "Found: %s" % segment 40 | except nntplib.NNTPTemporaryError, e: 41 | # Error code 430 is "No such article" 42 | error = nntp.get_error_code(e) 43 | if error == '430': 44 | # Found missing segment 45 | self.missing.put((f, segment, bytes)) 46 | self.missing.task_done() 47 | #print "Missing: %s" % segment 48 | else: 49 | # Some other error, put the segment back in the 50 | # queue to be checked again 51 | print "Error: %s" % e 52 | self.segments.put(segment) 53 | except Exception, e: 54 | print "Unknown error: %s" % e 55 | return 56 | 57 | # Signals to queue that this task is done 58 | self.segments.task_done() 59 | except Exception, e: 60 | try: 61 | self.server.quit() 62 | except: 63 | pass -------------------------------------------------------------------------------- /nzbverify/nntp.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import nntplib 3 | import socket 4 | 5 | try: 6 | import ssl 7 | except ImportError: 8 | _have_ssl = False 9 | else: 10 | _have_ssl = True 11 | 12 | log = logging.getLogger('NNTP') 13 | 14 | # Add the 'CAPABILITIES' response 15 | nntplib.LONGRESP.append('101') 16 | 17 | SSL_PORTS = [443, 563] 18 | 19 | def get_error_code(error): 20 | """ 21 | Attempts to extract the NNTP error code number from an NNTPError, which 22 | are of the form: 23 | 24 | 430 No such article 25 | """ 26 | error = str(error) 27 | return error.split()[0] 28 | 29 | class NNTP(nntplib.NNTP): 30 | """ 31 | An NNTP client that supports SSL/TLS. Most of this code is back-ported from 32 | Python 3.2 (see source below). 33 | 34 | NOTE: SSL support has been tested but TLS support has not. 35 | 36 | Source: 37 | http://svn.python.org/view/python/branches/release32-maint/Lib/nntplib.py 38 | """ 39 | def __init__(self, host, port, user=None, password=None, use_ssl=None, 40 | timeout=10): 41 | self.host = host 42 | self.port = port 43 | self.sock = socket.create_connection((host, port), timeout) 44 | self.sock = self.wrap_socket(self.sock, use_ssl) 45 | self.file = self.sock.makefile('rrb') 46 | self.debugging = 0 47 | self.welcome = self.getresp() 48 | self._caps = None 49 | self.authenticated = False 50 | 51 | # RFC 4642 2.2.2: Both the client and the server MUST know if there is 52 | # a TLS session active. A client MUST NOT attempt to start a TLS 53 | # session if a TLS session is already active. 54 | self.tls_on = False 55 | 56 | # If TLS is supported start a TLS session. Note that we have to do this 57 | # before we try to authenticate. 58 | if 'STARTTLS' in self.getcapabilities(): 59 | self.starttls() 60 | 61 | # Perform authentication if needed. 62 | if user: 63 | self.login(user, password) 64 | 65 | def login(self, user, password): 66 | if self.authenticated: 67 | raise ValueError("Already logged in.") 68 | 69 | if user: 70 | resp = self.shortcmd('authinfo user ' + user) 71 | if resp[:3] == '381': 72 | if not password: 73 | raise nntplib.NNTPReplyError(resp) 74 | else: 75 | resp = self.shortcmd('authinfo pass ' + password) 76 | if resp[:3] != '281': 77 | raise nntplib.NNTPPermanentError(resp) 78 | 79 | self.authenticated = True 80 | 81 | def starttls(self, context=None): 82 | """ 83 | Process a STARTTLS command. Arguments: 84 | - context: SSL context to use for the encrypted connection 85 | """ 86 | # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if 87 | # a TLS session already exists. 88 | if _have_ssl: 89 | if self.tls_on: 90 | raise ValueError("TLS is already enabled.") 91 | if self.authenticated: 92 | raise ValueError("TLS cannot be started after authentication.") 93 | resp = self._shortcmd('STARTTLS') 94 | if resp.startswith('382'): 95 | self.file.close() 96 | self.sock = self.wrap_socket(self.sock) 97 | self.file = self.sock.makefile("rwb") 98 | self.tls_on = True 99 | # Capabilities may change after TLS starts up, so ask for them 100 | # again. 101 | self._caps = None 102 | self.getcapabilities() 103 | else: 104 | raise nntplib.NNTPError("TLS failed to start.") 105 | 106 | def getcapabilities(self): 107 | """ 108 | If the CAPABILITIES command is not supported, an empty dict is 109 | returned. 110 | """ 111 | if self._caps is None: 112 | self.nntp_version = 1 113 | self.nntp_implementation = None 114 | try: 115 | resp, self._caps = self.capabilities() 116 | except nntplib.NNTPPermanentError: 117 | # Server doesn't support capabilities 118 | self._caps = {} 119 | else: 120 | if 'VERSION' in self._caps: 121 | # The server can advertise several supported versions, 122 | # choose the highest. 123 | self.nntp_version = max(map(int, self._caps['VERSION'])) 124 | if 'IMPLEMENTATION' in self._caps: 125 | self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) 126 | return self._caps 127 | 128 | def capabilities(self): 129 | """ 130 | Process a CAPABILITIES command. Not supported by all servers. 131 | """ 132 | caps = {} 133 | resp, lines = self.longcmd("CAPABILITIES") 134 | for line in lines: 135 | bits = line.split() 136 | name, tokens = bits[0], bits[1:] 137 | caps[name] = tokens 138 | return resp, caps 139 | 140 | def wrap_socket(self, sock, use_ssl): 141 | """ 142 | Wrap a socket in SSL/TLS. Arguments: 143 | - sock: Socket to wrap 144 | 145 | Returns: 146 | - sock: New, encrypted socket. 147 | """ 148 | # If the user hasn't explicitly said no to SSL, we'll use SSL if the port 149 | # is a known-SSL port. 150 | if use_ssl is None and self.port in SSL_PORTS: 151 | use_ssl = True 152 | 153 | if _have_ssl and use_ssl: 154 | log.info("Using SSL") 155 | return ssl.wrap_socket(sock, ssl_version=ssl.PROTOCOL_TLSv1) 156 | return sock 157 | -------------------------------------------------------------------------------- /nzbverify/cmdline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | nzbverify - Weston Nielson 4 | 5 | TODO: 6 | * Add missing threshold, after which we quit (default 5%?) 7 | * Add date range in NZB info printout 8 | * Add debug mode 9 | * Add timing info 10 | * Add SSL support 11 | * Better handling of credentials 12 | * Use getopt 13 | * Check for existance of NZB file before starting threads 14 | * Enable default config files (~/.nzbverify, ~/.netrc) 15 | """ 16 | from __future__ import division 17 | 18 | import Queue 19 | import getopt 20 | import getpass 21 | import logging 22 | import netrc 23 | import nntplib 24 | import os 25 | import signal 26 | import sys 27 | import time 28 | 29 | sys.path.append('../') 30 | 31 | from nzbverify import __author__, __version__, conf, thread 32 | 33 | try: 34 | from xml.etree.cElementTree import iterparse 35 | except: 36 | from xml.etree.ElementTree import iterparse 37 | 38 | __prog__ = "nzbverify" 39 | 40 | __usage__ = """ 41 | Usage: 42 | %s [options] 43 | 44 | Options: 45 | -s : NNTP server 46 | -u : NNTP username 47 | -p : NNTP password, will be prompted for 48 | -P : NNTP port 49 | -c : Config file to use (defaults: ~/.nzbverify, ~/.netrc) 50 | -n : Number of NNTP connections to use 51 | -e : Use SSL/TLS encryption 52 | -h : Show help text and exit 53 | """ 54 | 55 | __help__ = """ 56 | Help text... 57 | """ 58 | 59 | DEFAULT_NUM_CONNECTIONS = 5 60 | 61 | def get_size(bytes): 62 | size = bytes/1048576.0 63 | unit = "MB" 64 | if len(str(round(size))) > 3: 65 | size = size / 1024.0 66 | unit = "GB" 67 | return "%0.2f" % size, unit 68 | 69 | class ProgressBar(object): 70 | def __init__(self, segments, missing): 71 | self.segments = segments 72 | self.missing = missing 73 | self.segment_count = segments.qsize() 74 | 75 | digits = len(str(self.segment_count)) 76 | self._msg = ("Available: %%0%ds [%%s], " 77 | "Missing: %%0%ds [%%s], " 78 | "Total: %%0%ds [%%s]" % 79 | (digits,digits,digits)) 80 | 81 | def update(self): 82 | tnum = self.segment_count - self.segments.qsize() 83 | tpct = "%0.2f%%" % ((tnum/self.segment_count)*100.00) 84 | mnum = self.missing.qsize() 85 | mpct = "%0.2f%%" % ((mnum/self.segment_count)*100.00) 86 | anum = tnum-mnum 87 | apct = "%0.2f%%" % ((anum/self.segment_count)*100.00) 88 | msg = self._msg % (anum, apct, mnum, mpct, tnum, tpct) 89 | sys.stdout.write("\r%s" % msg) 90 | sys.stdout.flush() 91 | 92 | def finish(self): 93 | self.update() 94 | sys.stdout.write("\n") 95 | 96 | def main(nzb, num_connections, nntp_kwargs): 97 | threads = [] 98 | files = [] 99 | seg_count = 0 100 | bytes_total = 0 101 | segments = Queue.Queue() 102 | missing = Queue.Queue() 103 | 104 | # Listen for exit 105 | def signal_handler(signal, frame): 106 | sys.stdout.write('\n') 107 | sys.stdout.write("Stopping threads...") 108 | sys.stdout.flush() 109 | thread.stop_threads(threads) 110 | sys.stdout.write("done\n") 111 | sys.exit(0) 112 | 113 | # TODO: Listen to other signals 114 | signal.signal(signal.SIGINT, signal_handler) 115 | 116 | # Spawn some threads 117 | for i in range(num_connections): 118 | try: 119 | t = thread.SegmentCheckerThread(i, segments, missing, nntp_kwargs) 120 | t.setDaemon(True) 121 | t.start() 122 | threads.append(t) 123 | except: 124 | break 125 | 126 | print "Created %d threads" % (i+1) 127 | 128 | # Parse NZB and populate the Queue 129 | print "Parsing NZB: %s" % nzb 130 | for event, elem in iterparse(nzb, events=("start", "end")): 131 | if event == "start" and elem.tag.endswith('file'): 132 | files.append(elem.get('subject')) 133 | if event == "end" and elem.tag.endswith('segment'): 134 | bytes = int(elem.get('bytes',0)) 135 | bytes_total += bytes 136 | segments.put((files[-1], '<%s>' % elem.text, bytes)) 137 | seg_count += 1 138 | 139 | size, unit = get_size(bytes_total) 140 | print "Found %d files and %d segments totalling %s %s" % (len(files), seg_count, size, unit) 141 | 142 | pbar = ProgressBar(segments, missing) 143 | 144 | while not segments.empty(): 145 | pbar.update() 146 | time.sleep(0.1) 147 | 148 | pbar.finish() 149 | 150 | missing.join() 151 | 152 | num_missing = missing.qsize() 153 | if num_missing > 0: 154 | missing_bytes = 0 155 | print "Result: missing %d/%d segments; %0.2f%% complete" % (num_missing, seg_count, ((seg_count-num_missing)/seg_count * 100.00)) 156 | while not missing.empty(): 157 | f, seg, bytes = missing.get() 158 | missing_bytes += bytes 159 | print '\tfile="%s", segment="%s"' % (f, seg) 160 | 161 | size, unit = get_size(missing_bytes) 162 | print "Missing %s %s" % (size, unit) 163 | else: 164 | print "Result: all %d segments available" % seg_count 165 | 166 | thread.stop_threads(threads) 167 | 168 | def print_usage(): 169 | print __usage__ % __prog__ 170 | 171 | def run(): 172 | print "nzbverify version %s, Copyright (C) 2012 %s" % (__version__, __author__) 173 | 174 | num_connections = DEFAULT_NUM_CONNECTIONS 175 | config = None 176 | nntp_kwargs = { 177 | 'host': None, 178 | 'port': nntplib.NNTP_PORT, 179 | 'user': None, 180 | 'password': None, 181 | 'use_ssl': None, 182 | 'timeout': 10 183 | } 184 | 185 | # Parse command line options 186 | opts, args = getopt.getopt(sys.argv[1:], 's:u:P:n:c:eph', ["server=", "username=", "port=", "connections=", "config=", "ssl", "password", "help"]) 187 | for o, a in opts: 188 | if o in ("-h", "--help"): 189 | print __help__ 190 | print_usage() 191 | sys.exit(0) 192 | elif o in ("-s", "--server"): 193 | nntp_kwargs['host'] = a 194 | elif o in ("-u", "--username"): 195 | nntp_kwargs['user'] = a 196 | elif o in ("-p", "--password"): 197 | nntp_kwargs['password'] = getpass.getpass("Password: ") 198 | elif o in ("-e", "--ssl"): 199 | nntp_kwargs['use_ssl'] = True 200 | elif o in ("-P", "--port"): 201 | try: 202 | nntp_kwargs['port'] = int(a) 203 | except: 204 | print "Error: invalid port '%s'" % a 205 | sys.exit(0) 206 | elif o in ("-n", "--connections"): 207 | try: 208 | num_connections = int(a) 209 | except: 210 | print "Error: invalid number of connections '%s'" % a 211 | sys.exit(0) 212 | elif o in ("-c", "--config"): 213 | config = a 214 | 215 | # Get the NZB 216 | if len(args) < 1: 217 | print_usage() 218 | sys.exit(0) 219 | nzb = args[0] 220 | 221 | # See if we need to load certain NNTP details from config files 222 | # A host is required 223 | config = conf.get_config(config) 224 | if not nntp_kwargs['host'] and not config: 225 | print "Error: no server details provided" 226 | sys.exit(0) 227 | 228 | if config: 229 | credentials = config.authenticators(nntp_kwargs.get('host')) 230 | if not credentials: 231 | if not config.hosts: 232 | print "Error: Could not determine server details" 233 | sys.exit(0) 234 | 235 | # Just use the first entry 236 | host, credentials = config.hosts.items()[0] 237 | nntp_kwargs['host'] = host 238 | 239 | if not nntp_kwargs['user'] and not nntp_kwargs['password']: 240 | nntp_kwargs['user'] = credentials[0] 241 | nntp_kwargs['password'] = credentials[2] 242 | 243 | start = time.time() 244 | main(nzb, num_connections, nntp_kwargs) 245 | print "Verification took %s seconds" % (time.time() - start) 246 | --------------------------------------------------------------------------------