├── newsmangler ├── __init__.py ├── filewrap.py ├── fakepoll.py ├── article.py ├── common.py ├── yenc.py ├── asyncnntp.py └── postmangler.py ├── .gitignore ├── docs ├── TODO ├── sample.conf └── CHANGELOG ├── README.rst └── mangler.py /newsmangler/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.nzb 2 | *.pyc 3 | -------------------------------------------------------------------------------- /docs/TODO: -------------------------------------------------------------------------------- 1 | Poster 2 | ------ 3 | * Work out how to have the .NZB files generated before we start posting, so 4 | it can be posted along with the other files (for weird people). 5 | * Add part retrying when something goes BORK with a connection. 6 | * Add PAR2 generation. Read the par2cmdline source to work out how it decides 7 | how many blocks/files to generate for a given block/source size (so we know 8 | how many total files). Use popen to run the par2cmdline process in the 9 | background until it generates the files. Post them. The End. 10 | -------------------------------------------------------------------------------- /docs/sample.conf: -------------------------------------------------------------------------------- 1 | [posting] 2 | # The 'From' address to put on posts. If you change this then try to resume 3 | # a posting, many clients will get confused. Set it once and leave it! 4 | from: Bob 5 | 6 | # Default group you'd like to post to. If you'd like to post to more than 7 | # one group at a time, seperate the group names with commas. 8 | # default_group: alt.binaries.test,alt.binaries.test.yenc 9 | default_group: alt.binaries.test.yenc 10 | 11 | # Size of each article in bytes. 12 | article_size: 768000 13 | 14 | # String to prefix to each subject. 15 | subject_prefix: 16 | 17 | # Generate a .NZB for each post? 18 | generate_nzbs: 1 19 | 20 | # Space seperated list of filenames to skip when posting. 21 | skip_filenames: 22 | 23 | 24 | [aliases] 25 | # Group aliases in the form "short: long". 26 | abt: alt.binaries.test 27 | abty: alt.binaries.test.yenc 28 | 29 | 30 | [server] 31 | # Connection info for the server 32 | hostname: localhost 33 | port: 119 34 | 35 | # Authentication info for the server 36 | username: 37 | password: 38 | 39 | # Number of connections to open to the server 40 | connections: 2 41 | 42 | # How long to wait (in seconds) between connection attempts 43 | reconnect_delay: 5 44 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | **IMPORTANT NOTE** 3 | ================== 4 | This project is no longer being actively maintained. It still works fine but no 5 | new features will ever be appearing. Check out `my GoPostStuff project `_ 6 | for a new and much scarier version of the idea. 7 | 8 | 9 | 10 | newsmangler 11 | =========== 12 | 13 | newsmangler is a basic client for posting binaries to Usenet. The only notable 14 | feature is multiple connection support to efficiently utilize modern bandwidth. 15 | 16 | Installation 17 | ============ 18 | #. Download the source: ``git clone git://github.com/madcowfred/newsmangler.git`` 19 | (or download a .zip I guess). 20 | 21 | #. Copy sample.conf to ~/.newsmangler.conf, edit the options as appropriate. 22 | ``cp sample.conf ~/.newsmangler.conf`` 23 | ``nano ~/.newsmangler.conf`` 24 | 25 | #. Download and install the `yenc module `_ 26 | for greatly improved yEnc encoding speed. 27 | 28 | Usage 29 | ===== 30 | Make a directory containing the files you wish to post, the _directory name_ will 31 | be used as the post subject. For example, with a directory structure such as: 32 | 33 | test post please ignore/ 34 | - test.nfo 35 | - test.part1.rar 36 | - test.part2.rar 37 | 38 | And the command line: ``python mangler.py "test post please ignore"`` 39 | 40 | The files will post as: 41 | ``test post please ignore [1/3] - "test.nfo" yEnc (1/1)`` 42 | ``test post please ignore [2/3] - "test.part1.rar" yEnc (01/27)`` 43 | ``test post please ignore [3/3] - "test.part2.rar" yEnc (01/27)`` 44 | 45 | See ``python mangler.py --help`` for other options. 46 | -------------------------------------------------------------------------------- /newsmangler/filewrap.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | """Simple file wrapper to handle opening and closing on demand.""" 29 | 30 | import logging 31 | 32 | class FileWrap: 33 | def __init__(self, filepath, parts): 34 | self._filepath = filepath 35 | self._parts = parts 36 | 37 | self._file = None 38 | 39 | self.logger = logging.getLogger('mangler') 40 | 41 | def read_part(self, begin, end): 42 | self.logger.debug('%s read_part %d %d', self._filepath, begin, end) 43 | 44 | # Open the file if it's not already open 45 | if self._file is None: 46 | self.logger.debug('%s read_part open file', self._filepath) 47 | self._file = open(self._filepath, 'rb') 48 | 49 | # Seek to the right position and read the data 50 | self._file.seek(begin, 0) 51 | data = self._file.read(end - begin) 52 | 53 | # If this was the last part we should close the file 54 | self._parts -= 1 55 | if self._parts == 0: 56 | self.logger.debug('%s read_part close file', self._filepath) 57 | self._file.close() 58 | 59 | # Return the data 60 | return data 61 | -------------------------------------------------------------------------------- /newsmangler/fakepoll.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | "Fake poll() for systems that don't implement it (Windows, most notably)." 29 | 30 | import select 31 | import socket 32 | 33 | # Assume that they need constants 34 | select.POLLIN = 1 35 | select.POLLOUT = 2 36 | select.POLLNVAL = 4 37 | select.POLLPRI = 8 38 | select.POLLERR = 16 39 | select.POLLHUP = 32 40 | 41 | # --------------------------------------------------------------------------- 42 | 43 | class FakePoll: 44 | def __init__(self): 45 | self.FDs = {} 46 | 47 | # Register an FD for polling 48 | def register(self, fd, flags=None): 49 | if flags is None: 50 | self.FDs[fd] = select.POLLIN|select.POLLOUT|select.POLLNVAL 51 | else: 52 | self.FDs[fd] = flags 53 | 54 | # Unregister an FD 55 | def unregister(self, fd): 56 | del self.FDs[fd] 57 | 58 | # Poll (select!) for timeout seconds. Nasty. 59 | def poll(self, timeout): 60 | fds = self.FDs.keys() 61 | can_read, can_write = select.select(fds, fds, [], timeout)[:2] 62 | 63 | results = {} 64 | 65 | for fd in can_read: 66 | results[fd] = select.POLLIN 67 | for fd in can_write: 68 | if fd in results: 69 | results[fd] |= select.POLLOUT 70 | else: 71 | results[fd] = select.POLLOUT 72 | 73 | return results.items() 74 | 75 | # --------------------------------------------------------------------------- 76 | -------------------------------------------------------------------------------- /newsmangler/article.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012, freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | try: 29 | from collections import OrderedDict 30 | except ImportError: 31 | from ordereddict import OrderedDict 32 | from cStringIO import StringIO 33 | 34 | from newsmangler.yenc import yEncode 35 | 36 | class Article: 37 | def __init__(self, filewrap, begin, end, fileinfo, subject, partnum): 38 | self._filewrap = filewrap 39 | self._begin = begin 40 | self._end = end 41 | self._fileinfo = fileinfo 42 | self._subject = subject 43 | self._partnum = partnum 44 | 45 | self.headers = OrderedDict() 46 | self.postfile = StringIO() 47 | 48 | self.__article_size = 0 49 | 50 | def prepare(self): 51 | # Don't prepare again if we already did everything 52 | if self.__article_size > 0: 53 | self.postfile.seek(0, 0) 54 | return self.__article_size 55 | 56 | # Headers 57 | for k, v in self.headers.items(): 58 | self.postfile.write('%s: %s\r\n' % (k, v)) 59 | 60 | self.postfile.write('\r\n') 61 | 62 | # yEnc start 63 | line = '=ybegin part=%d total=%d line=128 size=%d name=%s\r\n' % ( 64 | self._partnum, self._fileinfo['parts'], self._fileinfo['filesize'], self._fileinfo['filename'] 65 | ) 66 | self.postfile.write(line) 67 | line = '=ypart begin=%d end=%d\r\n' % (self._begin + 1, self._end) 68 | self.postfile.write(line) 69 | 70 | # yEnc data 71 | data = self._filewrap.read_part(self._begin, self._end) 72 | partcrc = yEncode(self.postfile, data) 73 | 74 | # yEnc end 75 | line = '=yend size=%d part=%d pcrc32=%s\r\n' % (self._end - self._begin, self._partnum, partcrc) 76 | self.postfile.write(line) 77 | 78 | # And done writing for now 79 | self.postfile.write('.\r\n') 80 | self.__article_size = self.postfile.tell() 81 | self.postfile.seek(0, 0) 82 | 83 | return self.__article_size 84 | -------------------------------------------------------------------------------- /newsmangler/common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | """Various miscellaneous useful functions.""" 29 | 30 | NM_VERSION = '0.1.0git' 31 | 32 | import os 33 | import sys 34 | 35 | from ConfigParser import ConfigParser 36 | 37 | # --------------------------------------------------------------------------- 38 | # Parse our configuration file 39 | def ParseConfig(cfgfile='~/.newsmangler.conf'): 40 | configfile = os.path.expanduser(cfgfile) 41 | if not os.path.isfile(configfile): 42 | print 'ERROR: config file "%s" is missing!' % (configfile) 43 | sys.exit(1) 44 | 45 | c = ConfigParser() 46 | c.read(configfile) 47 | conf = {} 48 | for section in c.sections(): 49 | conf[section] = {} 50 | for option in c.options(section): 51 | v = c.get(section, option) 52 | if v.isdigit(): 53 | v = int(v) 54 | conf[section][option] = v 55 | 56 | return conf 57 | 58 | # --------------------------------------------------------------------------- 59 | # Come up with a 'safe' filename 60 | def SafeFilename(filename): 61 | safe_filename = os.path.basename(filename) 62 | for char in [' ', "\\", '|', '/', ':', '*', '?', '<', '>']: 63 | safe_filename = safe_filename.replace(char, '_') 64 | return safe_filename 65 | 66 | # --------------------------------------------------------------------------- 67 | # Return a nicely formatted size 68 | MB = 1024.0 * 1024 69 | def NiceSize(bytes): 70 | if bytes < 1024: 71 | return '%dB' % (bytes) 72 | elif bytes < MB: 73 | return '%.1fKB' % (bytes / 1024.0) 74 | else: 75 | return '%.1fMB' % (bytes / MB) 76 | 77 | # Return a nicely formatted time 78 | def NiceTime(seconds): 79 | hours, left = divmod(seconds, 60 * 60) 80 | mins, secs = divmod(left, 60) 81 | if hours: 82 | return '%dh %dm %ds' % (hours, mins, secs) 83 | else: 84 | return '%dm %ds' % (mins, secs) 85 | -------------------------------------------------------------------------------- /mangler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # --------------------------------------------------------------------------- 3 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions, and the following disclaimer. 11 | # * Redistributions in binary form must reproduce the above copyright notice, 12 | # this list of conditions, and the following disclaimer in the 13 | # documentation and/or other materials provided with the distribution. 14 | # * Neither the name of the author of this software nor the name of 15 | # contributors to this software may be used to endorse or promote products 16 | # derived from this software without specific prior written consent. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | # POSSIBILITY OF SUCH DAMAGE. 29 | 30 | """Posts stuff.""" 31 | 32 | import os 33 | import sys 34 | from ConfigParser import ConfigParser 35 | from optparse import OptionParser 36 | 37 | from newsmangler.common import ParseConfig 38 | from newsmangler.postmangler import PostMangler 39 | 40 | # --------------------------------------------------------------------------- 41 | 42 | def main(): 43 | # Parse our command line options 44 | parser = OptionParser(usage='usage: %prog [options] dir1 dir2 ... dirN') 45 | parser.add_option('-c', '--config', 46 | dest='config', 47 | help='Specify a different config file location', 48 | ) 49 | parser.add_option('-f', '--files', 50 | dest='files', 51 | help='Assume all arguments are filenames instead of directories, and use SUBJECT as the base subject', 52 | metavar='SUBJECT', 53 | ) 54 | parser.add_option('-g', '--group', 55 | dest='group', 56 | help='Post to a different group than the default', 57 | ) 58 | # parser.add_option('-p', '--par2', 59 | # dest='generate_par2', 60 | # action='store_true', 61 | # default=False, 62 | # help="Generate PAR2 files in the background if they don't exist already.", 63 | # ) 64 | parser.add_option('-d', '--debug', 65 | dest='debug', 66 | action='store_true', 67 | default=False, 68 | help="Enable debug logging", 69 | ) 70 | parser.add_option('--profile', 71 | dest='profile', 72 | action='store_true', 73 | default=False, 74 | help='Run with the hotshot profiler (measures execution time of functions)', 75 | ) 76 | 77 | (options, args) = parser.parse_args() 78 | 79 | # No args? We have nothing to do! 80 | if not args: 81 | parser.print_help() 82 | sys.exit(1) 83 | 84 | # Make sure at least one of the args exists 85 | postme = [] 86 | post_title = None 87 | if options.files: 88 | post_title = options.files 89 | for arg in args: 90 | if os.path.isfile(arg): 91 | postme.append(arg) 92 | else: 93 | print 'ERROR: "%s" does not exist or is not a file!' % (arg) 94 | else: 95 | for arg in args: 96 | if os.path.isdir(arg): 97 | postme.append(arg) 98 | else: 99 | print 'ERROR: "%s" does not exist or is not a file!' % (arg) 100 | 101 | if not postme: 102 | print 'ERROR: no valid arguments provided on command line!' 103 | sys.exit(1) 104 | 105 | # Parse our configuration file 106 | if options.config: 107 | conf = ParseConfig(options.config) 108 | else: 109 | conf = ParseConfig() 110 | 111 | # Make sure the group is ok 112 | if options.group: 113 | if '.' not in options.group: 114 | newsgroup = conf['aliases'].get(options.group) 115 | if not newsgroup: 116 | print 'ERROR: group alias "%s" does not exist!' % (options.group) 117 | sys.exit(1) 118 | else: 119 | newsgroup = options.group 120 | else: 121 | newsgroup = conf['posting']['default_group'] 122 | 123 | # Strip whitespace from the newsgroup list to obey RFC1036 124 | for c in (' \t'): 125 | newsgroup = newsgroup.replace(c, '') 126 | 127 | # And off we go 128 | poster = PostMangler(conf, options.debug) 129 | 130 | if options.profile: 131 | import hotshot 132 | prof = hotshot.Profile('profile.poster') 133 | prof.runcall(poster.post, newsgroup, postme, post_title=post_title) 134 | prof.close() 135 | 136 | import hotshot.stats 137 | stats = hotshot.stats.load('profile.poster') 138 | stats.strip_dirs() 139 | stats.sort_stats('time', 'calls') 140 | stats.print_stats(25) 141 | 142 | else: 143 | poster.post(newsgroup, postme, post_title=post_title) 144 | 145 | # --------------------------------------------------------------------------- 146 | 147 | if __name__ == '__main__': 148 | main() 149 | -------------------------------------------------------------------------------- /docs/CHANGELOG: -------------------------------------------------------------------------------- 1 | 2012-12-03: * Hopefully fix NZBs being generated with the segments tree in a 2 | strange order. 3 | 4 | 2012-04-01: * Fix some issues with the fake Poll implementation on Windows 5 | systems - apparently someone uses this instead of a good client. 6 | [by JackDandy] 7 | 8 | 2012-03-15: * Finish rewrite of how articles are handled: 9 | - Wrap article information in Article objects so we don't use 10 | awful magical lists to hand data around. 11 | - Don't read/encode article data until it's needed. This means 12 | we no longer read the entire set of files to post into memory 13 | at startup. 14 | - Helps lay the framework for tracking which posts failed and 15 | either retrying them straight away or later. 16 | * Made sure that copyright notices are up to date. 17 | * Add *.nzb to .gitignore. 18 | * Various misc code cleanups. 19 | 20 | 2012-03-12: * Completely rearrange directory structure to match what is 21 | expected of a Python app. 22 | * Improve error handling around connection teardown. 23 | * Combined BaseMangler and Poster into PostMangler. No real 24 | reason to have two separate classes. 25 | * Don't loop until a connection connects before doing anything, 26 | no posting will be done until one is idle anyway. 27 | 28 | 2012-03-08: * Remove leecher.py and classes/Leecher.py, there are plenty of 29 | great options for downloading files already. 30 | * Add -d/--debug option to actually display debug information. 31 | * Add some extra debug logging to classes/asyncNNTP. 32 | * Change the .nzb generation code to use ElementTree, no reason 33 | at all to do this manually. 34 | * Update README. 35 | 36 | 2012-01-13: * Ugly fix for an article that ends with a line consisting of a 37 | single space/tab character. 38 | * Fix unquoted ampersands in XML output breaking strict parsers. 39 | 40 | 2010-11-07: * Change asyncNNTP to extend asyncore.dispatcher channel methods 41 | properly. 42 | 43 | 2008-05-09: * Strip whitespace from the newsgroup list to obey RFC1036. 44 | 45 | 2005-11-11: * Change the Message-ID generation slightly to better match 46 | "expected" behaviour. (patch by Super-Fly) 47 | * Fix the =ybegin line so it has the correct line length. 48 | (patch by Super-Fly) 49 | 50 | 2005-11-06: * Bump the version to 0.02 to try and track down which version 51 | people are using. 52 | * Add the yEncode method we're using to the X-Newsposter header. 53 | * Require a newer ID string from yenc-freddie for it to be 54 | recognised. 55 | 56 | 2005-11-04: * Fix posted files count being incorrect if we end up skipping some 57 | things. 58 | 59 | 2005-10-25: * Modify some logic in yEncode_Python to follow the yEnc specs a 60 | little better. We now escape a leading period on a line, and 61 | never write more than linelen+1 bytes per line. 62 | 63 | 2005-10-21: * Add this file! 64 | * Add a log message detailing which yEncode version we're using. 65 | 66 | 2005-10-20: * Speed up article list generation a bit. 67 | * Add -fSUBJECT option for posting an arbitrary list of files 68 | instead of a directory/directories. 69 | * Use Psyco to speed up part encoding by 30-35% if it's available. 70 | * Use my modified yenc module to speed up part encoding by 65-70% 71 | if it's available. 72 | * Strip the directory name from the filename field of =ybegin when 73 | using files mode. 74 | * Reduce memory thrashing by not continually creating a new string 75 | for our data buffer. We now just use a pointer value and only 76 | read a new block of data in once we have exhausted the current 77 | one. 78 | * Add FakePoll class, emulates select.poll() on systems that do not 79 | have it (such as Windows). 80 | * Add -cCONFIG option to specify a different config file. 81 | 82 | 2005-10-19: * Print a status message once a second showing our progress. 83 | 84 | 2005-10-18: * Move some more logging to DEBUG level. 85 | * Don't set our posting start time until at least one thread is 86 | connected. This makes our posting speed more accurate. 87 | * Fix an invalid exception handler in asyncNNTP. 88 | * Only increment our byte count if we're posting a file, commands 89 | shouldn't count. 90 | 91 | 2005-10-17: * Disable our Date: header generator, let the server do it. 92 | * Modify the SO_SNDBUF of our sockets before we try to connect. 93 | 94 | 2005-10-14: * Add posting/skip_filenames option. 95 | * Clarify the comment for posting/default_group. 96 | * Fix the begin field of our =ybegin lines being off by one. This 97 | fixes decoding on NZB-O-Matic at least. 98 | * Write all groups out to the generated .nzb file if posting to 99 | more than one. 100 | -------------------------------------------------------------------------------- /newsmangler/yenc.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | """Useful functions for yEnc encoding/decoding.""" 29 | 30 | import re 31 | import zlib 32 | 33 | # --------------------------------------------------------------------------- 34 | 35 | HAVE_PSYCO = False 36 | HAVE_YENC = False 37 | HAVE_YENC_FRED = False 38 | 39 | # --------------------------------------------------------------------------- 40 | # Translation tables 41 | YDEC_TRANS = ''.join([chr((i + 256 - 42) % 256) for i in range(256)]) 42 | YENC_TRANS = ''.join([chr((i + 42) % 256) for i in range(256)]) 43 | 44 | YDEC_MAP = {} 45 | for i in range(256): 46 | YDEC_MAP[chr(i)] = chr((i + 256 - 64) % 256) 47 | 48 | # --------------------------------------------------------------------------- 49 | 50 | def yDecode(data): 51 | # unescape any escaped char (grr) 52 | data = re.sub(r'=(.)', yunescape, data) 53 | return data.translate(YDEC_TRANS) 54 | 55 | def yunescape(m): 56 | return YDEC_MAP[m.group(1)] 57 | 58 | def yEncode_C(postfile, data): 59 | # If we don't have my modified yenc module, we have to do the . quoting 60 | # ourselves. This is about 50% slower. 61 | if HAVE_YENC_FRED: 62 | yenced, tempcrc = _yenc.encode_string(data, escapedots=1)[:2] 63 | else: 64 | yenced, tempcrc = _yenc.encode_string(data)[:2] 65 | yenced = yenced.replace('\r\n.', '\r\n..') 66 | 67 | postfile.write(yenced) 68 | 69 | if not yenced.endswith('\r\n'): 70 | postfile.write('\r\n') 71 | 72 | return '%08x' % ((tempcrc ^ -1) & 2**32L - 1) 73 | 74 | def yEncode_Python(postfile, data, linelen=128): 75 | 'Encode data into yEnc format' 76 | 77 | translated = data.translate(YENC_TRANS) 78 | 79 | # escape =, NUL, LF, CR 80 | for i in (61, 0, 10, 13): 81 | j = '=%c' % (i + 64) 82 | translated = translated.replace(chr(i), j) 83 | 84 | # split the rest of it into lines 85 | lines = [] 86 | start = 0 87 | end = 0 88 | datalen = len(translated) 89 | 90 | while end < datalen: 91 | end = min(datalen, start + linelen) 92 | line = translated[start:end] 93 | 94 | # FIXME: line consisting entirely of a space/tab 95 | if start == end - 1: 96 | if line[0] in ('\x09', '\x20'): 97 | line = '=%c' % (ord(line[0]) + 64) 98 | else: 99 | # escape tab/space/period at the start of a line 100 | if line[0] in ('\x09', '\x20'): 101 | line = '=%c%s' % (ord(line[0]) + 64, line[1:-1]) 102 | end -= 1 103 | elif line[0] == '\x2e': 104 | line = '.%s' % (line) 105 | 106 | # escaped char on the end of the line 107 | if line[-1] == '=': 108 | line += translated[end] 109 | end += 1 110 | # escape tab/space at the end of a line 111 | elif line[-1] in ('\x09', '\x20'): 112 | line = '%s=%c' % (line[:-1], ord(line[-1]) + 64) 113 | 114 | postfile.write(line) 115 | postfile.write('\r\n') 116 | start = end 117 | 118 | return CRC32(data) 119 | 120 | # --------------------------------------------------------------------------- 121 | 122 | YSPLIT_RE = re.compile(r'(\S+)=') 123 | def ySplit(line): 124 | 'Split a =y* line into key/value pairs' 125 | fields = {} 126 | 127 | parts = YSPLIT_RE.split(line)[1:] 128 | if len(parts) % 2: 129 | return fields 130 | 131 | for i in range(0, len(parts), 2): 132 | key, value = parts[i], parts[i+1] 133 | fields[key] = value.strip() 134 | 135 | return fields 136 | 137 | # --------------------------------------------------------------------------- 138 | 139 | def yEncMode(): 140 | if HAVE_YENC_FRED: 141 | return 'yenc-fred' 142 | elif HAVE_YENC: 143 | return 'yenc-vanilla' 144 | elif HAVE_PSYCO: 145 | return 'python-psyco' 146 | else: 147 | return 'python-vanilla' 148 | 149 | # --------------------------------------------------------------------------- 150 | # Make a human readable CRC32 value 151 | def CRC32(data): 152 | return '%08x' % (zlib.crc32(data) & 2**32L - 1) 153 | 154 | # Come up with a 'safe' filename 155 | def SafeFilename(filename): 156 | safe_filename = os.path.basename(filename) 157 | for char in [' ', "\\", '|', '/', ':', '*', '?', '<', '>']: 158 | safe_filename = safe_filename.replace(char, '_') 159 | return safe_filename 160 | 161 | # --------------------------------------------------------------------------- 162 | # Use the _yenc C module if it's available. If not, try to use psyco to speed 163 | # up part encoding 25-30%. 164 | try: 165 | import _yenc 166 | except ImportError: 167 | try: 168 | import psyco 169 | except ImportError: 170 | pass 171 | else: 172 | HAVE_PSYCO = True 173 | psyco.bind(yEncode_Python) 174 | yEncode = yEncode_Python 175 | else: 176 | HAVE_YENC = True 177 | HAVE_YENC_FRED = ('Freddie mod' in _yenc.__doc__) 178 | yEncode = yEncode_C 179 | -------------------------------------------------------------------------------- /newsmangler/asyncnntp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | "A basic NNTP client using asyncore" 29 | 30 | import asyncore 31 | import errno 32 | import logging 33 | import re 34 | import select 35 | import socket 36 | import time 37 | 38 | # --------------------------------------------------------------------------- 39 | 40 | STATE_DISCONNECTED = 0 41 | STATE_CONNECTING = 1 42 | STATE_CONNECTED = 2 43 | 44 | MODE_AUTH = 0 45 | MODE_COMMAND = 1 46 | MODE_POST_INIT = 2 47 | MODE_POST_DATA = 3 48 | MODE_POST_DONE = 4 49 | MODE_DATA = 5 50 | 51 | POST_BUFFER_MIN = 16384 52 | POST_READ_SIZE = 262144 53 | 54 | MSGID_RE = re.compile(r'(<\S+@\S+>)') 55 | 56 | # --------------------------------------------------------------------------- 57 | 58 | class asyncNNTP(asyncore.dispatcher): 59 | def __init__(self, parent, connid, host, port, bindto, username, password): 60 | asyncore.dispatcher.__init__(self) 61 | 62 | self.logger = logging.getLogger('mangler') 63 | 64 | self.parent = parent 65 | self.connid = connid 66 | self.host = host 67 | self.port = port 68 | self.bindto = bindto 69 | self.username = username 70 | self.password = password 71 | 72 | self.reset() 73 | 74 | def reset(self): 75 | self._readbuf = '' 76 | self._writebuf = '' 77 | self._article = None 78 | self._pointer = 0 79 | 80 | self.reconnect_at = 0 81 | self.mode = MODE_AUTH 82 | self.state = STATE_DISCONNECTED 83 | 84 | def do_connect(self): 85 | # Create the socket 86 | self.create_socket(socket.AF_INET, socket.SOCK_STREAM) 87 | 88 | # Try to set our send buffer a bit larger 89 | for i in range(17, 13, -1): 90 | try: 91 | self.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 2**i) 92 | except socket.error: 93 | continue 94 | else: 95 | break 96 | self.logger.debug('%d: SO_SNDBUF is %s', self.connid, self.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) 97 | 98 | # If we have to bind our socket to an IP, do that 99 | #if self.bindto is not None: 100 | # self.bind((self.bindto, 0)) 101 | 102 | # Try to connect. This can blow up! 103 | try: 104 | self.connect((self.host, self.port)) 105 | except (socket.error, socket.gaierror), msg: 106 | self.really_close(msg) 107 | else: 108 | self.state = STATE_CONNECTING 109 | self.logger.debug('%d: connecting to %s:%s', self.connid, self.host, self.port) 110 | 111 | # ----------------------------------------------------------------------- 112 | # Check to see if it's time to reconnect yet 113 | def reconnect_check(self, now): 114 | if self.state == STATE_DISCONNECTED and now >= self.reconnect_at: 115 | self.do_connect() 116 | 117 | def add_channel(self, map=None): 118 | self.logger.debug('%d: adding FD %d to poller', self.connid, self._fileno) 119 | 120 | asyncore.dispatcher.add_channel(self, map) 121 | 122 | # Add ourselves to the poll object 123 | asyncore.poller.register(self._fileno) 124 | 125 | def del_channel(self, map=None): 126 | self.logger.debug('%d: removing FD %d from poller', self.connid, self._fileno) 127 | 128 | # Remove ourselves from the async map 129 | asyncore.dispatcher.del_channel(self, map) 130 | 131 | # Remove ourselves from the poll object 132 | if self._fileno is not None: 133 | try: 134 | asyncore.poller.unregister(self._fileno) 135 | except KeyError: 136 | pass 137 | 138 | def close(self): 139 | self.del_channel() 140 | if self.socket is not None: 141 | self.socket.close() 142 | 143 | # ----------------------------------------------------------------------- 144 | # We only want to be writable if we're connecting, or something is in our 145 | # buffer. 146 | def writable(self): 147 | return (not self.connected) or len(self._writebuf) 148 | 149 | # Send some data from our buffer when we can write 150 | def handle_write(self): 151 | #self.logger.debug('%d wants to write!', self._fileno) 152 | 153 | if not self.writable(): 154 | # We don't have any buffer, silly thing 155 | asyncore.poller.register(self._fileno, select.POLLIN) 156 | return 157 | 158 | sent = asyncore.dispatcher.send(self, self._writebuf[self._pointer:]) 159 | self._pointer += sent 160 | 161 | # We've run out of data 162 | if self._pointer == len(self._writebuf): 163 | self._writebuf = '' 164 | self._pointer = 0 165 | asyncore.poller.register(self._fileno, select.POLLIN) 166 | 167 | # If we're posting, we might need to read some more data from our file 168 | if self.mode == MODE_POST_DATA: 169 | self.parent._bytes += sent 170 | if len(self._writebuf) == 0: 171 | self.post_data() 172 | 173 | # ----------------------------------------------------------------------- 174 | # We want buffered output, duh 175 | def send(self, data): 176 | self._writebuf += data 177 | # We need to know about writable things now 178 | asyncore.poller.register(self._fileno) 179 | #self.logger.debug('%d has data!', self._fileno) 180 | 181 | # ----------------------------------------------------------------------- 182 | 183 | def handle_error(self): 184 | self.logger.exception('%d: unhandled exception!', self.connid) 185 | 186 | # ----------------------------------------------------------------------- 187 | 188 | def handle_connect(self): 189 | self.status = STATE_CONNECTED 190 | self.logger.debug('%d: connected!', self.connid) 191 | 192 | def handle_close(self): 193 | self.really_close() 194 | 195 | def really_close(self, error=None): 196 | self.mode = MODE_COMMAND 197 | self.status = STATE_DISCONNECTED 198 | 199 | self.close() 200 | self.reset() 201 | 202 | if error and hasattr(error, 'args'): 203 | self.logger.warning('%d: %s!', self.connid, error.args[1]) 204 | self.reconnect_at = time.time() + self.parent.conf['server']['reconnect_delay'] 205 | else: 206 | self.logger.warning('%d: Connection closed: %s', self.connid, error) 207 | 208 | # There is some data waiting to be read 209 | def handle_read(self): 210 | try: 211 | self._readbuf += self.recv(16384) 212 | except socket.error, msg: 213 | self.really_close(msg) 214 | return 215 | 216 | # Split the buffer into lines. Last line is always incomplete. 217 | lines = self._readbuf.split('\r\n') 218 | self._readbuf = lines.pop() 219 | 220 | # Do something useful here 221 | for line in lines: 222 | self.logger.debug('%d: < %s', self.connid, line) 223 | 224 | # Initial login stuff 225 | if self.mode == MODE_AUTH: 226 | resp = line.split(None, 1)[0] 227 | 228 | # Welcome... post, no post 229 | if resp in ('200', '201'): 230 | if self.username: 231 | text = 'AUTHINFO USER %s\r\n' % (self.username) 232 | self.send(text) 233 | self.logger.debug('%d: > AUTHINFO USER ********', self.connid) 234 | else: 235 | self.mode = MODE_COMMAND 236 | self.parent._idle.append(self) 237 | self.logger.debug('%d: ready.', self.connid) 238 | 239 | # Need password too 240 | elif resp in ('381'): 241 | if self.password: 242 | text = 'AUTHINFO PASS %s\r\n' % (self.password) 243 | self.send(text) 244 | self.logger.debug('%d: > AUTHINFO PASS ********', self.connid) 245 | else: 246 | self.really_close('need password!') 247 | 248 | # Auth ok 249 | elif resp in ('281'): 250 | self.mode = MODE_COMMAND 251 | self.parent._idle.append(self) 252 | self.logger.debug('%d: ready.', self.connid) 253 | 254 | # Auth failure 255 | elif resp in ('502'): 256 | self.really_close('authentication failure.') 257 | 258 | # Dunno 259 | else: 260 | self.logger.warning('%d: unknown response while MODE_AUTH - "%s"', 261 | self.connid, line) 262 | 263 | # Posting a file 264 | elif self.mode == MODE_POST_INIT: 265 | resp = line.split(None, 1)[0] 266 | # Posting is allowed 267 | if resp == '340': 268 | self.mode = MODE_POST_DATA 269 | 270 | # TODO: use the suggested message-ID, will require some rethinking as to how 271 | # messages are constructed 272 | m = MSGID_RE.search(line) 273 | if m: 274 | self.logger.debug('%d: changing Message-ID to %s', self.connid, m.group(1)) 275 | self._article.headers['Message-ID'] = m.group(1) 276 | 277 | # Prepare the article for posting 278 | article_size = self._article.prepare() 279 | self.parent.remember_msgid(article_size, self._article) 280 | 281 | self.post_data() 282 | 283 | # Posting is not allowed 284 | elif resp == '440': 285 | self.mode = MODE_COMMAND 286 | self.parent._idle.append(self) 287 | del self._article 288 | self.logger.warning('%d: posting not allowed!', self.connid) 289 | 290 | # WTF? 291 | else: 292 | self.logger.warning('%d: unknown response while MODE_POST_INIT - "%s"', 293 | self.connid, line) 294 | 295 | # Done posting 296 | elif self.mode == MODE_POST_DONE: 297 | resp = line.split(None, 1)[0] 298 | # Ok 299 | if resp == '240': 300 | #self.parent.post_success(self._article) 301 | 302 | self.mode = MODE_COMMAND 303 | self.parent._idle.append(self) 304 | 305 | # Not ok 306 | elif resp.startswith('44'): 307 | self.mode = MODE_COMMAND 308 | self.parent._idle.append(self) 309 | self.logger.warning('%d: posting failed - %s', self.connid, line) 310 | 311 | # WTF? 312 | else: 313 | self.logger.warning('%d: unknown response while MODE_POST_DONE - "%s"', 314 | self.connid, line) 315 | 316 | # Other stuff 317 | else: 318 | self.logger.warning('%d: unknown response from server - "%s"', 319 | self.connid, line) 320 | 321 | # ----------------------------------------------------------------------- 322 | # Guess what this does! 323 | def post_article(self, article): 324 | self.mode = MODE_POST_INIT 325 | self._article = article 326 | self.send('POST\r\n') 327 | self.logger.debug('%d: > POST', self.connid) 328 | 329 | def post_data(self): 330 | data = self._article.postfile.read(POST_READ_SIZE) 331 | if len(data) == 0: 332 | self.mode = MODE_POST_DONE 333 | self._article.postfile.close() 334 | self._article.postfile = None 335 | 336 | self.send(data) 337 | 338 | # --------------------------------------------------------------------------- 339 | -------------------------------------------------------------------------------- /newsmangler/postmangler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2005-2012 freddie@wafflemonster.org 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, 8 | # this list of conditions, and the following disclaimer. 9 | # * Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions, and the following disclaimer in the 11 | # documentation and/or other materials provided with the distribution. 12 | # * Neither the name of the author of this software nor the name of 13 | # contributors to this software may be used to endorse or promote products 14 | # derived from this software without specific prior written consent. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | # POSSIBILITY OF SUCH DAMAGE. 27 | 28 | """Main class for posting stuff.""" 29 | 30 | import asyncore 31 | import logging 32 | import os 33 | import select 34 | import sys 35 | import time 36 | 37 | from cStringIO import StringIO 38 | 39 | try: 40 | import xml.etree.cElementTree as ET 41 | except: 42 | import xml.etree.ElementTree as ET 43 | 44 | from newsmangler import asyncnntp 45 | from newsmangler import yenc 46 | from newsmangler.article import Article 47 | from newsmangler.common import * 48 | from newsmangler.filewrap import FileWrap 49 | 50 | # --------------------------------------------------------------------------- 51 | 52 | class PostMangler: 53 | def __init__(self, conf, debug=False): 54 | self.conf = conf 55 | 56 | self._conns = [] 57 | self._idle = [] 58 | 59 | # Create our logger 60 | self.logger = logging.getLogger('mangler') 61 | handler = logging.StreamHandler() 62 | formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s') 63 | handler.setFormatter(formatter) 64 | self.logger.addHandler(handler) 65 | if debug: 66 | self.logger.setLevel(logging.DEBUG) 67 | else: 68 | self.logger.setLevel(logging.INFO) 69 | 70 | # Create a poll object for async bits to use. If the user doesn't have 71 | # poll, we're going to have to fake it. 72 | try: 73 | asyncore.poller = select.poll() 74 | self.logger.info('Using poll() for sockets') 75 | except AttributeError: 76 | from newsmangler.fakepoll import FakePoll 77 | asyncore.poller = FakePoll() 78 | self.logger.info('Using FakePoll() for sockets') 79 | 80 | self.conf['posting']['skip_filenames'] = self.conf['posting'].get('skip_filenames', '').split() 81 | 82 | self._articles = [] 83 | self._files = {} 84 | self._msgids = {} 85 | 86 | self._current_dir = None 87 | self.newsgroup = None 88 | self.post_title = None 89 | 90 | # Some sort of useful logging junk about which yEncode we're using 91 | self.logger.info('Using %s module for yEnc', yenc.yEncMode()) 92 | 93 | # ----------------------------------------------------------------------- 94 | # Connect all of our connections 95 | def connect(self): 96 | for i in range(self.conf['server']['connections']): 97 | conn = asyncnntp.asyncNNTP(self, i, self.conf['server']['hostname'], 98 | self.conf['server']['port'], None, self.conf['server']['username'], 99 | self.conf['server']['password'], 100 | ) 101 | conn.do_connect() 102 | self._conns.append(conn) 103 | 104 | # ----------------------------------------------------------------------- 105 | # Poll our poll() object and do whatever is neccessary. Basically a combination 106 | # of asyncore.poll2() and asyncore.readwrite(), without all the frippery. 107 | def poll(self): 108 | results = asyncore.poller.poll(0) 109 | for fd, flags in results: 110 | obj = asyncore.socket_map.get(fd) 111 | if obj is None: 112 | self.logger.critical('Invalid FD for poll(): %d', fd) 113 | asyncore.poller.unregister(fd) 114 | continue 115 | 116 | try: 117 | if flags & (select.POLLIN | select.POLLPRI): 118 | obj.handle_read_event() 119 | if flags & select.POLLOUT: 120 | obj.handle_write_event() 121 | if flags & (select.POLLERR | select.POLLHUP | select.POLLNVAL): 122 | obj.handle_expt_event() 123 | except (asyncore.ExitNow, KeyboardInterrupt, SystemExit): 124 | raise 125 | except: 126 | obj.handle_error() 127 | 128 | # ----------------------------------------------------------------------- 129 | 130 | def post(self, newsgroup, postme, post_title=None): 131 | self.newsgroup = newsgroup 132 | self.post_title = post_title 133 | 134 | # Generate the list of articles we need to post 135 | self.generate_article_list(postme) 136 | 137 | # If we have no valid articles, bail 138 | if not self._articles: 139 | self.logger.warning('No valid articles to post!') 140 | return 141 | 142 | # Connect! 143 | self.connect() 144 | 145 | # And loop 146 | self._bytes = 0 147 | last_stuff = start = time.time() 148 | 149 | self.logger.info('Posting %d article(s)...', len(self._articles)) 150 | 151 | while 1: 152 | now = time.time() 153 | 154 | # Poll our sockets for events 155 | self.poll() 156 | 157 | # Possibly post some more parts now 158 | while self._idle and self._articles: 159 | conn = self._idle.pop(0) 160 | article = self._articles.pop(0) 161 | conn.post_article(article) 162 | 163 | # Do some stuff every now and then 164 | if now - last_stuff >= 0.5: 165 | last_stuff = now 166 | 167 | for conn in self._conns: 168 | conn.reconnect_check(now) 169 | 170 | if self._bytes: 171 | interval = time.time() - start 172 | speed = self._bytes / interval / 1024 173 | left = len(self._articles) + (len(self._conns) - len(self._idle)) 174 | print '%d article(s) remaining - %.1fKB/s \r' % (left, speed), 175 | sys.stdout.flush() 176 | 177 | # All done? 178 | if len(self._articles) == 0 and len(self._idle) == self.conf['server']['connections']: 179 | interval = time.time() - start 180 | speed = self._bytes / interval 181 | self.logger.info('Posting complete - %s in %s (%s/s)', 182 | NiceSize(self._bytes), NiceTime(interval), NiceSize(speed)) 183 | 184 | # If we have some msgids left over, we might have to generate 185 | # a .NZB 186 | if self.conf['posting']['generate_nzbs'] and self._msgids: 187 | self.generate_nzb() 188 | 189 | break 190 | 191 | # And sleep for a bit to try and cut CPU chompage 192 | time.sleep(0.01) 193 | 194 | # ----------------------------------------------------------------------- 195 | # Maybe remember the msgid for later 196 | def remember_msgid(self, article_size, article): 197 | if self.conf['posting']['generate_nzbs']: 198 | if self._current_dir != article._fileinfo['dirname']: 199 | if self._msgids: 200 | self.generate_nzb() 201 | self._msgids = {} 202 | 203 | self._current_dir = article._fileinfo['dirname'] 204 | 205 | subj = article._subject % (1) 206 | if subj not in self._msgids: 207 | self._msgids[subj] = [int(time.time())] 208 | #self._msgids[subj].append((article.headers['Message-ID'], article_size)) 209 | self._msgids[subj].append((article, article_size)) 210 | 211 | # ----------------------------------------------------------------------- 212 | # Generate the list of articles we need to post 213 | def generate_article_list(self, postme): 214 | # "files" mode is just one lot of files 215 | if self.post_title: 216 | self._gal_files(self.post_title, postme) 217 | # "dirs" mode could be a whole bunch 218 | else: 219 | for dirname in postme: 220 | dirname = os.path.abspath(dirname) 221 | if dirname: 222 | self._gal_files(os.path.basename(dirname), os.listdir(dirname), basepath=dirname) 223 | 224 | # Do the heavy lifting for generate_article_list 225 | def _gal_files(self, post_title, files, basepath=''): 226 | article_size = self.conf['posting']['article_size'] 227 | 228 | goodfiles = [] 229 | for filename in files: 230 | filepath = os.path.abspath(os.path.join(basepath, filename)) 231 | 232 | # Skip non-files and empty files 233 | if not os.path.isfile(filepath): 234 | continue 235 | if filename in self.conf['posting']['skip_filenames'] or filename == '.newsmangler': 236 | continue 237 | filesize = os.path.getsize(filepath) 238 | if filesize == 0: 239 | continue 240 | 241 | goodfiles.append((filepath, filename, filesize)) 242 | 243 | goodfiles.sort() 244 | 245 | # Do stuff with files 246 | n = 1 247 | for filepath, filename, filesize in goodfiles: 248 | parts, partial = divmod(filesize, article_size) 249 | if partial: 250 | parts += 1 251 | 252 | self._files[filepath] = FileWrap(filepath, parts) 253 | 254 | # Build a subject 255 | real_filename = os.path.split(filename)[1] 256 | 257 | temp = '%%0%sd' % (len(str(len(files)))) 258 | filenum = temp % (n) 259 | temp = '%%0%sd' % (len(str(parts))) 260 | subject = '%s [%s/%d] - "%s" yEnc (%s/%d)' % ( 261 | post_title, filenum, len(goodfiles), real_filename, temp, parts 262 | ) 263 | 264 | # Apply a subject prefix 265 | if self.conf['posting']['subject_prefix']: 266 | subject = '%s %s' % (self.conf['posting']['subject_prefix'], subject) 267 | 268 | # Now make up our parts 269 | fileinfo = { 270 | 'dirname': post_title, 271 | 'filename': real_filename, 272 | 'filepath': filepath, 273 | 'filesize': filesize, 274 | 'parts': parts, 275 | } 276 | 277 | for i in range(parts): 278 | partnum = i + 1 279 | begin = 0 + (i * article_size) 280 | end = min(filesize, partnum * article_size) 281 | 282 | # Build the article 283 | art = Article(self._files[filepath], begin, end, fileinfo, subject, partnum) 284 | art.headers['From'] = self.conf['posting']['from'] 285 | art.headers['Newsgroups'] = self.newsgroup 286 | art.headers['Subject'] = subject % (partnum) 287 | art.headers['Message-ID'] = '<%.5f.%d@%s>' % (time.time(), partnum, self.conf['server']['hostname']) 288 | art.headers['X-Newsposter'] = 'newsmangler %s (%s) - https://github.com/madcowfred/newsmangler\r\n' % ( 289 | NM_VERSION, yenc.yEncMode()) 290 | 291 | self._articles.append(art) 292 | 293 | n += 1 294 | 295 | # ----------------------------------------------------------------------- 296 | # Build an article for posting. 297 | def build_article(self, fileinfo, subject, partnum, begin, end): 298 | # Read the chunk of data from the file 299 | #f = self._files.get(fileinfo['filepath'], None) 300 | #if f is None: 301 | # self._files[fileinfo['filepath']] = f = open(fileinfo['filepath'], 'rb') 302 | 303 | #begin = f.tell() 304 | #data = f.read(self.conf['posting']['article_size']) 305 | #end = f.tell() 306 | 307 | # If that was the last part, close the file and throw it away 308 | #if partnum == fileinfo['parts']: 309 | # self._files[fileinfo['filepath']].close() 310 | # del self._files[fileinfo['filepath']] 311 | 312 | # Make a new article object and set headers 313 | art = Article(begin, end, fileinfo, subject, partnum) 314 | art.headers['From'] = self.conf['posting']['from'] 315 | art.headers['Newsgroups'] = self.newsgroup 316 | art.headers['Subject'] = subject % (partnum) 317 | art.headers['Message-ID'] = '<%.5f.%d@%s>' % (time.time(), partnum, self.conf['server']['hostname']) 318 | art.headers['X-Newsposter'] = 'newsmangler %s (%s) - https://github.com/madcowfred/newsmangler\r\n' % ( 319 | NM_VERSION, yenc.yEncMode()) 320 | 321 | self._articles.append(art) 322 | 323 | # ----------------------------------------------------------------------- 324 | # Generate a .NZB file! 325 | def generate_nzb(self): 326 | filename = 'newsmangler_%s.nzb' % (SafeFilename(self._current_dir)) 327 | 328 | self.logger.info('Begin generation of %s', filename) 329 | 330 | gentime = time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime()) 331 | root = ET.Element('nzb') 332 | root.append(ET.Comment('Generated by newsmangler v%s at %s' % (NM_VERSION, gentime))) 333 | 334 | for subject, msgids in self._msgids.items(): 335 | posttime = msgids.pop(0) 336 | 337 | # file 338 | f = ET.SubElement(root, 'file', 339 | { 340 | 'poster': self.conf['posting']['from'], 341 | 'date': str(posttime), 342 | 'subject': subject, 343 | } 344 | ) 345 | 346 | # newsgroups 347 | groups = ET.SubElement(f, 'groups') 348 | for newsgroup in self.newsgroup.split(','): 349 | group = ET.SubElement(groups, 'group') 350 | group.text = newsgroup 351 | 352 | # segments 353 | segments = ET.SubElement(f, 'segments') 354 | temp = [(m._partnum, m, article_size) for m, article_size in msgids] 355 | temp.sort() 356 | for partnum, article, article_size in temp: 357 | segment = ET.SubElement(segments, 'segment', 358 | { 359 | 'bytes': str(article_size), 360 | 'number': str(partnum), 361 | } 362 | ) 363 | segment.text = str(article.headers['Message-ID'][1:-1]) 364 | 365 | with open(filename, 'w') as nzbfile: 366 | ET.ElementTree(root).write(nzbfile, xml_declaration=True) 367 | 368 | self.logger.info('End generation of %s', filename) 369 | 370 | # --------------------------------------------------------------------------- 371 | --------------------------------------------------------------------------------