├── logs ├── papercut.log └── .cvsignore ├── .cvsignore ├── auth ├── .cvsignore ├── __init__.py ├── mysql.py ├── phpnuke_phpbb_mysql_users.py ├── postnuke_phpbb_mysql_users.py ├── phpbb_mysql_users.py ├── phorum_pgsql_users.py └── phorum_mysql_users.py ├── cache └── .cvsignore ├── storage ├── .cvsignore ├── forwarding_proxy.py ├── phorum_pgsql_fix.sql ├── p2p.py ├── phorum_mysql_fix.sql ├── phpbb_mysql_fix.sql ├── mime.py ├── mysql_storage.sql ├── __init__.py ├── strutil.py ├── mbox.py ├── maildir.py ├── mysql.py ├── phorum_mysql.py ├── phorum_pgsql.py └── phpnuke_phpbb_mysql.py ├── docs ├── draft-ietf-nntpext-base-15.txt └── draft-ietf-nntpext-tls-nntp-01.txt ├── TODO ├── util └── check_health.py ├── LICENSE ├── portable_locker.py ├── README ├── papercut_cache.py ├── settings.py └── INSTALL /logs/papercut.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.cvsignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /auth/.cvsignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /cache/.cvsignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /logs/.cvsignore: -------------------------------------------------------------------------------- 1 | *.log 2 | -------------------------------------------------------------------------------- /storage/.cvsignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /storage/forwarding_proxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpm/papercut/HEAD/storage/forwarding_proxy.py -------------------------------------------------------------------------------- /docs/draft-ietf-nntpext-base-15.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpm/papercut/HEAD/docs/draft-ietf-nntpext-base-15.txt -------------------------------------------------------------------------------- /storage/phorum_pgsql_fix.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE forums ADD nntp_group_name VARCHAR(30); 2 | ALTER TABLE forums ADD UNIQUE (nntp_group_name); 3 | -------------------------------------------------------------------------------- /auth/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: __init__.py,v 1.1 2002-04-04 23:10:20 jpm Exp $ 4 | -------------------------------------------------------------------------------- /storage/p2p.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: p2p.py,v 1.2 2002-04-03 23:07:22 jpm Exp $ 4 | import settings 5 | import anydbm 6 | 7 | class Papercut_Storage: 8 | """ 9 | Experimental Backend interface to implement the ideas brainstormed on the 10 | following page: http://webseitz.fluxent.com/wiki/PaperCut 11 | """ 12 | def __init__(self): 13 | # check for the p2p directories and dbm file now 14 | db = anydbm.open("p2p.dbm", "c") 15 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | TODO list: 2 | ---------- 3 | 4 | - Set the self.commands and self.extensions on the storage extensions, as some storages do not support all commands 5 | - MODE STREAM (it means several commands at the same time without waiting for responses) 6 | - Check more the patterns of searching (wildmat) -> backend.format_wildcards() -> Work in progress 7 | - Fork the server to the background automatically (using fork()?) 8 | - Make a command line option to make the server actually run on the foreground (-f option?) 9 | - Add a --verbose flag to replace the current __DEBUG__ flag -------------------------------------------------------------------------------- /storage/phorum_mysql_fix.sql: -------------------------------------------------------------------------------- 1 | # 2 | # Please change the values here as appropriate to your 3 | # setup (i.e. table name or size of the 'nntp_group_name' field) 4 | # 5 | # Warning: Do not change the field name to something else than 'nttp_group_name'! 6 | # 7 | ALTER TABLE forums ADD nntp_group_name VARCHAR(30) AFTER id; 8 | ALTER TABLE forums ADD UNIQUE (nntp_group_name); 9 | 10 | # 11 | # After dumping this file into MySQL you will need to manually update the contents 12 | # of the 'nttp_group_name' field to associate a table name / forum with a newsgroup to 13 | # be available on Papercut 14 | # -------------------------------------------------------------------------------- /storage/phpbb_mysql_fix.sql: -------------------------------------------------------------------------------- 1 | # 2 | # Please change the values here as appropriate to your 3 | # setup (i.e. table name or size of the 'nntp_group_name' field) 4 | # 5 | # Warning: Do not change the field name to something else than 'nttp_group_name'! 6 | # 7 | ALTER TABLE phpbb_forums ADD nntp_group_name VARCHAR(30) AFTER forum_name; 8 | ALTER TABLE phpbb_forums ADD UNIQUE (nntp_group_name); 9 | 10 | # 11 | # After dumping this file into MySQL you will need to manually update the contents 12 | # of the 'nttp_group_name' field to associate a table name / forum with a newsgroup to 13 | # be available on Papercut 14 | # -------------------------------------------------------------------------------- /util/check_health.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2004 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: check_health.py,v 1.1 2004-01-25 06:10:33 jpm Exp $ 4 | 5 | import settings 6 | from nntplib import NNTP 7 | 8 | s = NNTP(settings.nntp_hostname, settings.nntp_port) 9 | resp, groups = s.list() 10 | # check all of the groups, just in case 11 | for group_name, last, first, flag in groups: 12 | resp, count, first, last, name = s.group(group_name) 13 | print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last 14 | resp, subs = s.xhdr('subject', first + '-' + last) 15 | for id, sub in subs[-10:]: 16 | print id, sub 17 | s.quit() 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2002 Joao Prado Maia 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /portable_locker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # $Id: portable_locker.py,v 1.2 2002-10-03 01:05:24 jpm Exp $ 3 | 4 | # Note: this was originally from Python Cookbook, which was 5 | # probably taken from ASPN's Python Cookbook 6 | 7 | import os 8 | 9 | # needs win32all to work on Windows 10 | if os.name == 'nt': 11 | import win32con, win32file, pywintypes 12 | LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK 13 | LOCK_SH = 0 # the default 14 | LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY 15 | __overlapped = pywintypes.OVERLAPPED( ) 16 | 17 | def lock(fd, flags): 18 | hfile = win32file._get_osfhandle(fd.fileno( )) 19 | win32file.LockFileEx(hfile, flags, 0, 0xffff0000, __overlapped) 20 | 21 | def unlock(fd): 22 | hfile = win32file._get_osfhandle(fd.fileno( )) 23 | win32file.UnlockFileEx(hfile, 0, 0xffff0000, __overlapped) 24 | 25 | elif os.name == 'posix': 26 | import fcntl 27 | LOCK_EX = fcntl.LOCK_EX 28 | LOCK_SH = fcntl.LOCK_SH 29 | LOCK_NB = fcntl.LOCK_NB 30 | 31 | def lock(fd, flags): 32 | fcntl.flock(fd.fileno(), flags) 33 | 34 | def unlock(fd): 35 | fcntl.flock(fd.fileno(), fcntl.LOCK_UN) 36 | 37 | else: 38 | raise RuntimeError("portable_locker only defined for nt and posix platforms") 39 | 40 | -------------------------------------------------------------------------------- /auth/mysql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: mysql.py,v 1.3 2003-04-26 00:24:55 jpm Exp $ 4 | import MySQLdb 5 | import settings 6 | 7 | class Papercut_Auth: 8 | """ 9 | Authentication backend interface 10 | """ 11 | 12 | def __init__(self): 13 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 14 | self.cursor = self.conn.cursor() 15 | 16 | def is_valid_user(self, username, password): 17 | stmt = """ 18 | SELECT 19 | password 20 | FROM 21 | papercut_groups_auth 22 | WHERE 23 | username='%s' 24 | """ % (username) 25 | num_rows = self.cursor.execute(stmt) 26 | if num_rows == 0 or num_rows is None: 27 | settings.logEvent('Error - Authentication failed for username \'%s\' (user not found)' % (username)) 28 | return 0 29 | db_password = self.cursor.fetchone()[0] 30 | if db_password != password: 31 | settings.logEvent('Error - Authentication failed for username \'%s\' (incorrect password)' % (username)) 32 | return 0 33 | else: 34 | return 1 35 | 36 | -------------------------------------------------------------------------------- /auth/phpnuke_phpbb_mysql_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | import MySQLdb 4 | import settings 5 | import md5 6 | 7 | class Papercut_Auth: 8 | """ 9 | Authentication backend interface for the nuke port of phpBB (http://www.phpnuke.org) 10 | 11 | This backend module tries to authenticate the users against the nuke_users table. 12 | """ 13 | 14 | def __init__(self): 15 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 16 | self.cursor = self.conn.cursor() 17 | 18 | def is_valid_user(self, username, password): 19 | stmt = """ 20 | SELECT 21 | user_password 22 | FROM 23 | %susers 24 | WHERE 25 | username='%s' 26 | """ % (settings.nuke_table_prefix, username) 27 | num_rows = self.cursor.execute(stmt) 28 | if num_rows == 0 or num_rows is None: 29 | settings.logEvent('Error - Authentication failed for username \'%s\' (user not found)' % (username)) 30 | return 0 31 | db_password = self.cursor.fetchone()[0] 32 | if db_password != md5.new(password).hexdigest(): 33 | settings.logEvent('Error - Authentication failed for username \'%s\' (incorrect password)' % (username)) 34 | return 0 35 | else: 36 | return 1 37 | 38 | -------------------------------------------------------------------------------- /auth/postnuke_phpbb_mysql_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: postnuke_phpbb_mysql_users.py,v 1.1 2004-08-01 01:51:48 jpm Exp $ 4 | import MySQLdb 5 | import settings 6 | import md5 7 | 8 | class Papercut_Auth: 9 | """ 10 | Authentication backend interface for the phpBB web message board software (http://www.phpbb.com) when used inside PostNuke. 11 | 12 | This backend module tries to authenticate the users against the phpbb_users table. 13 | """ 14 | 15 | def __init__(self): 16 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 17 | self.cursor = self.conn.cursor() 18 | 19 | def is_valid_user(self, username, password): 20 | stmt = """ 21 | SELECT 22 | pn_pass 23 | FROM 24 | nuke_users 25 | WHERE 26 | pn_uname='%s' 27 | """ % (username) 28 | num_rows = self.cursor.execute(stmt) 29 | if num_rows == 0 or num_rows is None: 30 | settings.logEvent('Error - Authentication failed for username \'%s\' (user not found)' % (username)) 31 | return 0 32 | db_password = self.cursor.fetchone()[0] 33 | if db_password != md5.new(password).hexdigest(): 34 | settings.logEvent('Error - Authentication failed for username \'%s\' (incorrect password)' % (username)) 35 | return 0 36 | else: 37 | return 1 38 | 39 | -------------------------------------------------------------------------------- /storage/mime.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: mime.py,v 1.1 2002-02-03 06:15:25 jpm Exp $ 4 | import re 5 | import email 6 | 7 | def get_body(subpart): 8 | doubleline_regexp = re.compile("^\.\.", re.M) 9 | body = [] 10 | found = 0 11 | raw_headers = subpart.split('\r\n') 12 | for line in raw_headers: 13 | if not found and line == '': 14 | found = 1 15 | continue 16 | if found: 17 | body.append(doubleline_regexp.sub(".", line)) 18 | return "\r\n".join(body) 19 | 20 | def get_text_message(msg_string): 21 | msg = email.message_from_string(msg_string) 22 | cnt_type = msg.get_main_type() 23 | if cnt_type == 'text': 24 | # a simple mime based text/plain message (is this even possible?) 25 | body = get_body(msg_string) 26 | elif cnt_type == 'multipart': 27 | # needs to loop thru all parts and get the text version 28 | #print 'several parts here' 29 | text_parts = {} 30 | for part in msg.walk(): 31 | if part.get_main_type() == 'text': 32 | #print 'text based part' 33 | #print part.as_string() 34 | text_parts[part.get_params()[0][0]] = get_body(part.as_string()) 35 | if 'text/plain' in text_parts: 36 | return text_parts['text/plain'] 37 | elif 'text/html' in text_parts: 38 | return text_parts['text/html'] 39 | else: 40 | # not mime based 41 | body = get_body(msg_string) 42 | return body 43 | -------------------------------------------------------------------------------- /auth/phpbb_mysql_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: phpbb_mysql_users.py,v 1.4 2003-09-19 03:11:51 jpm Exp $ 4 | import MySQLdb 5 | import settings 6 | import md5 7 | 8 | class Papercut_Auth: 9 | """ 10 | Authentication backend interface for the phpBB web message board software (http://www.phpbb.com) 11 | 12 | This backend module tries to authenticate the users against the phpbb_users table. 13 | 14 | Many thanks to Chip McClure for the work on this file. 15 | """ 16 | 17 | def __init__(self): 18 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 19 | self.cursor = self.conn.cursor() 20 | 21 | def is_valid_user(self, username, password): 22 | stmt = """ 23 | SELECT 24 | user_password 25 | FROM 26 | %susers 27 | WHERE 28 | username='%s' 29 | """ % (settings.phpbb_table_prefix, username) 30 | num_rows = self.cursor.execute(stmt) 31 | if num_rows == 0 or num_rows is None: 32 | settings.logEvent('Error - Authentication failed for username \'%s\' (user not found)' % (username)) 33 | return 0 34 | db_password = self.cursor.fetchone()[0] 35 | if db_password != md5.new(password).hexdigest(): 36 | settings.logEvent('Error - Authentication failed for username \'%s\' (incorrect password)' % (username)) 37 | return 0 38 | else: 39 | return 1 40 | 41 | -------------------------------------------------------------------------------- /storage/mysql_storage.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS papercut_groups; 2 | CREATE TABLE papercut_groups ( 3 | id int(10) unsigned NOT NULL auto_increment, 4 | name varchar(50) NOT NULL default '', 5 | active smallint(6) NOT NULL default '0', 6 | description varchar(255) NOT NULL default '', 7 | table_name varchar(50) NOT NULL default '', 8 | PRIMARY KEY (id), 9 | KEY name (name), 10 | KEY active (active) 11 | ) TYPE=MyISAM; 12 | 13 | DROP TABLE IF EXISTS papercut_groups_auth; 14 | CREATE TABLE papercut_groups_auth ( 15 | id int(10) unsigned NOT NULL auto_increment, 16 | sess_id varchar(32) NOT NULL default '', 17 | name varchar(50) NOT NULL default '', 18 | username varchar(50) NOT NULL default '', 19 | password varchar(50) NOT NULL default '', 20 | PRIMARY KEY (id), 21 | KEY name (name), 22 | KEY username (username) 23 | ) TYPE=MyISAM; 24 | 25 | DROP TABLE IF EXISTS papercut_default_table; 26 | CREATE TABLE papercut_default_table ( 27 | id int(10) unsigned NOT NULL default '0', 28 | datestamp datetime NOT NULL default '0000-00-00 00:00:00', 29 | thread int(10) unsigned NOT NULL default '0', 30 | parent int(10) unsigned NOT NULL default '0', 31 | author varchar(255) NOT NULL default '', 32 | subject varchar(255) NOT NULL default '', 33 | message_id varchar(255) NOT NULL default '', 34 | bytes int(10) unsigned NOT NULL default '0', 35 | line_num int(10) unsigned NOT NULL default '0', 36 | host varchar(15) NOT NULL default '', 37 | body text NOT NULL, 38 | PRIMARY KEY (id), 39 | KEY author (author), 40 | KEY datestamp (datestamp), 41 | KEY subject (subject), 42 | KEY thread (thread), 43 | KEY parent (parent) 44 | ) TYPE=MyISAM; 45 | -------------------------------------------------------------------------------- /storage/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: __init__.py,v 1.4 2002-03-26 22:55:00 jpm Exp $ 4 | 5 | # 6 | # Papercut is a pretty dumb (some people might call it smart) server, because it 7 | # doesn't know or care where or how the Usenet articles are stored. The system 8 | # uses the concept of 'backends' to have access to the data being served by the 9 | # Usenet frontend. 10 | # 11 | # The 'Backends' of Papercut are the actual containers of the Usenet articles, 12 | # wherever they might be stored. The initial and proof of concept backend is 13 | # the Phorum (http://phorum.org) one, where the Usenet articles are actually 14 | # Phorum messages. 15 | # 16 | # If you want to create a new backend, please use the phorum_mysql.py file as 17 | # a guide for the implementation. You will need a lot of reading to understand 18 | # the NNTP protocol (i.e. how the NNTP responses should be sent back to the 19 | # user), so look under the 'docs' directory for the RFC documents. 20 | # 21 | # As a side note, Papercut is designed to be a simple as possible, so the actual 22 | # formatting of the responses are usually done on the backend itself. This is 23 | # for a reason - if Papercut had to format the information coming from the 24 | # backends unchanged, it would need to know 'too much', like the inner workings 25 | # of the MySQLdb module on the case of the Phorum backend and so on. 26 | # 27 | # Instead, Papercut expects a formatted return value from most (if not all) 28 | # methods of the backend module. This way we can abstract as much as possible 29 | # the data format of the articles, and have the main server code as simple and 30 | # fast as possible. 31 | # -------------------------------------------------------------------------------- /auth/phorum_pgsql_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: phorum_pgsql_users.py,v 1.3 2004-01-14 22:26:40 jpm Exp $ 4 | from pyPgSQL import PgSQL 5 | import settings 6 | import crypt 7 | import md5 8 | 9 | class Papercut_Auth: 10 | """ 11 | Authentication backend interface for the Phorum web message board software (http://phorum.org) 12 | 13 | This backend module tries to authenticate the users against the forums_auth table, which is 14 | used by Phorum to save its user based information, be it with a moderator level or not. 15 | """ 16 | 17 | def __init__(self): 18 | self.conn = PgSQL.connect(database=settings.dbname, user=settings.dbuser) 19 | self.cursor = self.conn.cursor() 20 | 21 | def is_valid_user(self, username, password): 22 | stmt = """ 23 | SELECT 24 | password 25 | FROM 26 | forums_auth 27 | WHERE 28 | username='%s' 29 | """ % (username) 30 | num_rows = self.cursor.execute(stmt) 31 | if num_rows == 0 or num_rows is None: 32 | settings.logEvent('Error - Authentication failed for username \'%s\' (user not found)' % (username)) 33 | return 0 34 | db_password = self.cursor.fetchone()[0] 35 | # somehow detect the version of phorum being used and guess the encryption type 36 | if len(db_password) == 32: 37 | result = (db_password != md5.new(password).hexdigest()) 38 | else: 39 | result = (db_password != crypt.crypt(password, password[:settings.PHP_CRYPT_SALT_LENGTH])) 40 | if result: 41 | settings.logEvent('Error - Authentication failed for username \'%s\' (incorrect password)' % (username)) 42 | return 0 43 | else: 44 | return 1 45 | -------------------------------------------------------------------------------- /auth/phorum_mysql_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: phorum_mysql_users.py,v 1.5 2004-01-14 22:26:40 jpm Exp $ 4 | import MySQLdb 5 | import settings 6 | import crypt 7 | import md5 8 | 9 | class Papercut_Auth: 10 | """ 11 | Authentication backend interface for the Phorum web message board software (http://phorum.org) 12 | 13 | This backend module tries to authenticate the users against the forums_auth table, which is 14 | used by Phorum to save its user based information, be it with a moderator level or not. 15 | """ 16 | 17 | def __init__(self): 18 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 19 | self.cursor = self.conn.cursor() 20 | 21 | def is_valid_user(self, username, password): 22 | stmt = """ 23 | SELECT 24 | password 25 | FROM 26 | forums_auth 27 | WHERE 28 | username='%s' 29 | """ % (username) 30 | num_rows = self.cursor.execute(stmt) 31 | if num_rows == 0 or num_rows is None: 32 | settings.logEvent('Error - Authentication failed for username \'%s\' (user not found)' % (username)) 33 | return 0 34 | db_password = self.cursor.fetchone()[0] 35 | # somehow detect the version of phorum being used and guess the encryption type 36 | if len(db_password) == 32: 37 | result = (db_password != md5.new(password).hexdigest()) 38 | else: 39 | result = (db_password != crypt.crypt(password, password[:settings.PHP_CRYPT_SALT_LENGTH])) 40 | if result: 41 | settings.logEvent('Error - Authentication failed for username \'%s\' (incorrect password)' % (username)) 42 | return 0 43 | else: 44 | return 1 45 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | -------------------- 2 | Papercut NNTP Server 3 | -------------------- 4 | 5 | Papercut is a news server written in 100% pure Python. It is intended to be 6 | extensible to the point where people can develop their own plug-ins and by 7 | that integrate the NNTP protocol to their applications. 8 | 9 | The server is compliant with most of the RFC0977 standards (when they make sense 10 | and are needed) and implements a lot of RFC1036 and RFC2980 extensions to the 11 | NNTP protocol. It was tested against Netscape News, Mozilla News and tin (under 12 | Solaris) and it works properly. 13 | 14 | The original need for this server was to integrate my PHP related web site 15 | forums with an NNTP gateway interface, so people could list and read the 16 | messages posted to the forums on their favorite News reader. The software on 17 | this case was Phorum (http://phorum.org) and the site is PHPBrasil.com 18 | (http://phpbrasil.com). At first it wasn't intended to support message posting, 19 | but it made sense to allow it after seeing how effective the tool was. 20 | 21 | The concept of storage modules was created exactly for this. I would create a Python 22 | class to handle the inner-workins of Phorum and MySQL and if I ever wanted to 23 | integrate the server with another type of software, I would just need to write 24 | a new storage module class. 25 | 26 | Anyway, back to the technical praise. Papercut is multi-threaded on Windows 27 | platforms and forking-based on UNIX platforms and should be reasonably fast 28 | (that means basically: 'it's pretty fast, but don't try serving 1000 connection 29 | at a time). The best thing about the application is that it is very simple to 30 | extend it. 31 | 32 | Papercut is licensed under the BSD license, which means you can sell it or do 33 | whatever you like with it. However, I ask that if you think Papercut is a good 34 | tool and you made a few enhancements to it or even fixed some bugs, please send 35 | me a patch or something. I will appreciate it :) 36 | 37 | -- Joao Prado Maia (jpm@pessoal.org) 38 | 39 | -------------------------------------------------------------------------------- /storage/strutil.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: strutil.py,v 1.3 2003-02-22 00:46:18 jpm Exp $ 4 | import time 5 | import re 6 | 7 | singleline_regexp = re.compile("^\.", re.M) 8 | 9 | def wrap(text, width=78): 10 | """Wraps text at a specified width. 11 | 12 | This is used on the PhorumMail feature, as to emulate completely the 13 | current Phorum behavior when it sends out copies of the posted 14 | articles. 15 | """ 16 | i = 0 17 | while i < len(text): 18 | if i + width + 1 > len(text): 19 | i = len(text) 20 | else: 21 | findnl = text.find('\n', i) 22 | findspc = text.rfind(' ', i, i+width+1) 23 | if findspc != -1: 24 | if findnl != -1 and findnl < findspc: 25 | i = findnl + 1 26 | else: 27 | text = text[:findspc] + '\n' + text[findspc+1:] 28 | i = findspc + 1 29 | else: 30 | findspc = text.find(' ', i) 31 | if findspc != -1: 32 | text = text[:findspc] + '\n' + text[findspc+1:] 33 | i = findspc + 1 34 | return text 35 | 36 | def get_formatted_time(time_tuple): 37 | """Formats the time tuple in a NNTP friendly way. 38 | 39 | Some newsreaders didn't like the date format being sent using leading 40 | zeros on the days, so we needed to hack our own little format. 41 | """ 42 | # days without leading zeros, please 43 | day = int(time.strftime('%d', time_tuple)) 44 | tmp1 = time.strftime('%a,', time_tuple) 45 | tmp2 = time.strftime('%b %Y %H:%M:%S %Z', time_tuple) 46 | return "%s %s %s" % (tmp1, day, tmp2) 47 | 48 | def format_body(text): 49 | """Formats the body of message being sent to the client. 50 | 51 | Since the NNTP protocol uses a single dot on a line to denote the end 52 | of the response, we need to substitute all leading dots on the body of 53 | the message with two dots. 54 | """ 55 | return singleline_regexp.sub("..", text) 56 | 57 | def format_wildcards(pattern): 58 | return pattern.replace('*', '.*').replace('?', '.*') 59 | 60 | def format_wildcards_sql(pattern): 61 | return pattern.replace('*', '%').replace('?', '%') 62 | -------------------------------------------------------------------------------- /papercut_cache.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: papercut_cache.py,v 1.7 2002-10-04 03:14:38 jpm Exp $ 4 | 5 | import binascii 6 | import md5 7 | import time 8 | import os 9 | import cPickle 10 | import portable_locker 11 | # papercut settings file 12 | import settings 13 | 14 | 15 | # methods that need to be cached 16 | cache_methods = ('get_XHDR', 'get_XGTITLE', 'get_LISTGROUP', 17 | 'get_XPAT', 'get_XOVER', 'get_BODY', 18 | 'get_HEAD', 'get_ARTICLE', 'get_STAT', 19 | 'get_LIST') 20 | 21 | 22 | class CallableWrapper: 23 | name = None 24 | thecallable = None 25 | cacheable_methods = () 26 | 27 | def __init__(self, name, thecallable, cacheable_methods): 28 | self.name = name 29 | self.thecallable = thecallable 30 | self.cacheable_methods = cacheable_methods 31 | 32 | def __call__(self, *args, **kwds): 33 | if self.name not in self.cacheable_methods: 34 | return self.thecallable(*args, **kwds) 35 | else: 36 | filename = self._get_filename(*args, **kwds) 37 | if os.path.exists(filename): 38 | # check the expiration 39 | expire, result = self._get_cached_result(filename) 40 | diff = time.time() - expire 41 | if diff > settings.nntp_cache_expire: 42 | # remove the file and run the method again 43 | return self._save_result(filename, *args, **kwds) 44 | else: 45 | return result 46 | else: 47 | return self._save_result(filename, *args, **kwds) 48 | 49 | def _get_cached_result(self, filename): 50 | inf = open(filename, 'rb') 51 | # get a lock on the file 52 | portable_locker.lock(inf, portable_locker.LOCK_SH) 53 | expire = cPickle.load(inf) 54 | result = cPickle.load(inf) 55 | # release the lock 56 | portable_locker.unlock(inf) 57 | inf.close() 58 | return (expire, result) 59 | 60 | def _save_result(self, filename, *args, **kwds): 61 | result = self.thecallable(*args, **kwds) 62 | # save the serialized result in the file 63 | outf = open(filename, 'w') 64 | # file write lock 65 | portable_locker.lock(outf, portable_locker.LOCK_EX) 66 | cPickle.dump(time.time(), outf) 67 | cPickle.dump(result, outf) 68 | # release the lock 69 | portable_locker.unlock(outf) 70 | outf.close() 71 | return result 72 | 73 | def _get_filename(self, *args, **kwds): 74 | arguments = '%s%s%s' % (self.name, args, kwds) 75 | return '%s%s' % (settings.nntp_cache_path, binascii.hexlify(md5.new(arguments).digest())) 76 | 77 | 78 | class Cache: 79 | backend = None 80 | cacheable_methods = () 81 | 82 | def __init__(self, storage_handle, cacheable_methods): 83 | self.backend = storage_handle.Papercut_Storage() 84 | self.cacheable_methods = cacheable_methods 85 | 86 | def __getattr__(self, name): 87 | result = getattr(self.backend, name) 88 | if callable(result): 89 | result = CallableWrapper(name, result, self.cacheable_methods) 90 | return result 91 | 92 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: settings.py,v 1.18 2004-08-01 01:03:22 jpm Exp $ 4 | import time 5 | import sys 6 | import os 7 | 8 | # 9 | # The following configuration settings should be pretty self-explanatory, but 10 | # please let me know if this is not complete or if more information / examples 11 | # are needed. 12 | # 13 | 14 | 15 | # what is the maximum number of concurrent connections that should be allowed 16 | max_connections = 20 17 | 18 | 19 | # 20 | # GENERAL PATH INFORMATION 21 | # 22 | 23 | # full path for where Papercut will store the log file 24 | log_path = "/home/papercut/logs/" 25 | # the actual log filename 26 | log_file = log_path + "papercut.log" 27 | 28 | 29 | # 30 | # HOSTNAME / PORT OF THE SERVER 31 | # 32 | 33 | # hostname that Papercut will bind against 34 | nntp_hostname = 'nntp.domain.com' 35 | # usually 119, but use 563 for an SSL server 36 | nntp_port = 119 37 | 38 | # type of server ('read-only' or 'read-write') 39 | server_type = 'read-write' 40 | 41 | 42 | # 43 | # NNTP AUTHENTICATION SUPPORT 44 | # 45 | 46 | # does the server need authentication ? ('yes' or 'no') 47 | nntp_auth = 'no' 48 | # backend that Papercut will use to authenticate the users 49 | auth_backend = '' 50 | # ONLY needed for phorum_mysql_users auth module 51 | PHP_CRYPT_SALT_LENGTH = 2 52 | 53 | 54 | # 55 | # CACHE SYSTEM 56 | # 57 | 58 | # the cache system may need a lot of diskspace ('yes' or 'no') 59 | nntp_cache = 'no' 60 | # cache expire (in seconds) 61 | nntp_cache_expire = 60 * 60 * 3 62 | # path to where the cached files should be kept 63 | nntp_cache_path = '/home/papercut/cache/' 64 | 65 | 66 | # 67 | # STORAGE MODULE 68 | # 69 | 70 | # backend that Papercut will use to get (and store) the actual articles content 71 | storage_backend = "phorum_mysql" 72 | 73 | # for the forwarding_proxy backend, set the next option to the remote nntp server 74 | forward_host = 'news.remotedomain.com' 75 | 76 | 77 | # 78 | # PHORUM STORAGE MODULE OPTIONS 79 | # 80 | 81 | # full path to the directory where the Phorum configuration files are stored 82 | phorum_settings_path = "/home/papercut/www/domain.com/phorum_settings/" 83 | # the version for the installed copy of Phorum 84 | phorum_version = "3.3.2a" 85 | 86 | # configuration values for 'storage/phorum_mysql.py' 87 | # database connection variables 88 | dbhost = "localhost" 89 | dbname = "phorum" 90 | dbuser = "anonymous" 91 | dbpass = "anonymous" 92 | 93 | 94 | # 95 | # PHPBB STORAGE MODULE OPTIONS 96 | # 97 | 98 | # the prefix for the phpBB tables 99 | phpbb_table_prefix = "phpbb_" 100 | 101 | 102 | # 103 | # PHPNUKE PHPBB STORAGE MODULE OPTIONS 104 | # 105 | 106 | # if you're running PHPNuke, set this for the nuke tables and phpbb_table_prefix 107 | # for the bb tables. 108 | nuke_table_prefix = "nuke_" 109 | 110 | # the prefix for the phpBB tables 111 | phpbb_table_prefix = "nuke_bb" 112 | 113 | 114 | # 115 | # MBOX STORAGE MODULE OPTIONS 116 | # 117 | 118 | # the full path for where the mbox files are stored in 119 | mbox_path = "/home/papercut/mboxes/" 120 | 121 | 122 | # check for the appropriate options 123 | if nntp_auth == 'yes' and auth_backend == '': 124 | sys.exit("Please configure the 'nntp_auth' and 'auth_backend' options correctly") 125 | 126 | # check for the trailing slash 127 | if phorum_settings_path[-1] != '/': 128 | phorum_settings_path = phorum_settings_path + '/' 129 | 130 | 131 | # helper function to log information 132 | def logEvent(msg): 133 | f = open(log_file, "a") 134 | f.write("[%s] %s\n" % (time.strftime("%a %b %d %H:%M:%S %Y", time.gmtime()), msg)) 135 | f.close() 136 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | -------------------- 2 | Papercut NNTP Server 3 | -------------------- 4 | 5 | For more information on what exactly Papercut is, and what are its main 6 | objectives / intended audience, please read the README file. 7 | 8 | 9 | Requirements: 10 | ------------- 11 | 12 | - Python 2.2 (it needs the email module to parse MIME based messages) 13 | - Database server 14 | - The Phorum backend needs MySQL or PostgreSQL as the storage driver) 15 | - MySQLdb python module (http://sf.net/projects/mysql-python) 16 | or 17 | - pyPgSQL python module (http://pypgsql.sourceforge.net/) 18 | - The phpBB backend needs MySQL 19 | - MySQLdb python module (http://sf.net/projects/mysql-python) 20 | - There are other storage modules, seek the source for details on those 21 | 22 | - Permission to add a new column to one of the Phorum tables 23 | 24 | 25 | Step by Step Instructions: 26 | -------------------------- 27 | 28 | 1) Unpack the distribution tarball where you intend to run it from 29 | 30 | $ cd /path/to/where/papercut/will/run 31 | $ gunzip papercut-X.tar.gz 32 | $ tar vxf papercut-X.tar 33 | [lots of stuff shows up on the console] 34 | $ cd papercut-X 35 | 36 | 2) Edit the settings.py configuration file 37 | 38 | - Change the 'log_path' variable to point to the appropriate directory on your 39 | server (i.e. /usr/local/papercut/logs/) 40 | 41 | - Modify the 'nntp_hostname' and 'nntp_port' variables to your appropriate 42 | server name and port number. (Note: if you want to run Papercut on port 199, 43 | you may need to be root depending on your system) 44 | 45 | - Choose your backend type by changing the 'backend_type' variable. (Note: as 46 | of version 0.7.1, there is only one backend -> phorum_mysql) 47 | 48 | - Finally, change the MySQL related variables so Papercut can connect to the 49 | appropriate database and get the content of the messages. 50 | 51 | 3) If your backend type is 'phorum_mysql': 52 | 53 | - You will need to add a new column under the main forum listing table to 54 | associate the name of a newsgroup to a table name. Since Phorum is totally 55 | dynamic on the number of forums it can create, we need an extra column to 56 | prevent problems. 57 | 58 | $ cd /path/to/where/papercut/will/run/ 59 | $ cd storage 60 | $ less phorum_mysql_fix.sql 61 | [read the information contained on the file] 62 | $ mysql -u username_here -p database_here < phorum_mysql_fix.sql 63 | [password will be requested now] 64 | 65 | - Now that the new column was created on the main forum listing table, you 66 | will need to edit it and enter the name of the newsgroup that you want for 67 | each forum. 68 | 69 | - After you finish editing the main forum table, you will need to go back to 70 | the settings.py file and configure the full path for the Phorum settings 71 | folder. That is, the folder where you keep the 'forums.php' configuration 72 | file and all other files that setup the options for each forum. 73 | 74 | It will usually have 'forums.php', '1.php', '2.php' and so on. The numbers 75 | on the filenames are actually the forum IDs on the main forum table. In any 76 | case, you will need to change the 'phorum_settings_path' variable on the 77 | settings.py file and put the full path to this folder. 78 | 79 | - You will also need to set the version of the installed copy of Phorum so 80 | Papercut can send the correct headers when sending out copies of the posted 81 | articles (also called PhorumMail for the Phorum lovers out there). Set the 82 | 'phorum_version' variable as appropriate on your case (i.e. '3.3.2a'). 83 | 84 | 85 | If you find any problems on this set of instructions, or if the instructions 86 | didn't work out for you, please let me know. 87 | -------------------------------------------------------------------------------- /storage/mbox.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: mbox.py,v 1.7 2004-08-01 01:51:48 jpm Exp $ 4 | 5 | import os 6 | import mailbox 7 | import settings 8 | import strutil 9 | import string 10 | 11 | 12 | class Papercut_Storage: 13 | """ 14 | Storage backend interface for mbox files 15 | """ 16 | mbox_dir = '' 17 | 18 | def __init__(self): 19 | self.mbox_dir = settings.mbox_path 20 | 21 | def get_mailbox(self, filename): 22 | return mailbox.PortableUnixMailbox(open(self.mbox_dir + filename)) 23 | 24 | def get_file_list(self): 25 | return os.listdir(self.mbox_dir) 26 | 27 | def get_group_list(self): 28 | groups = self.get_file_list() 29 | return ["papercut.mbox.%s" % k for k in groups] 30 | 31 | def group_exists(self, group_name): 32 | groups = self.get_group_list() 33 | found = False 34 | for name in groups: 35 | # group names are supposed to be case insensitive 36 | if string.lower(name) == string.lower(group_name): 37 | found = True 38 | break 39 | return found 40 | 41 | def get_first_article(self, group_name): 42 | return 1 43 | 44 | def get_group_stats(self, filename): 45 | total, max, min = self.get_mbox_stats(filename) 46 | return (total, min, max, filename) 47 | 48 | def get_mbox_stats(self, filename): 49 | mbox = self.get_mailbox(filename) 50 | dir(mbox) 51 | cnt = 0 52 | while mbox.next(): 53 | cnt = cnt + 1 54 | return (cnt-1, cnt, 1) 55 | 56 | def get_message_id(self, msg_num, group): 57 | return '<%s@%s>' % (msg_num, group) 58 | 59 | def get_NEWGROUPS(self, ts, group='%'): 60 | # XXX: eventually add some code in here to get the mboxes newer than the given timestamp 61 | return None 62 | 63 | def get_NEWNEWS(self, ts, group='*'): 64 | return '' 65 | 66 | def get_GROUP(self, group_name): 67 | result = self.get_mbox_stats(group_name.replace('papercut.mbox.', '')) 68 | return (result[0], result[2], result[1]) 69 | 70 | def get_LIST(self, username=""): 71 | result = self.get_file_list() 72 | if len(result) == 0: 73 | return "" 74 | else: 75 | groups = [] 76 | for mbox in result: 77 | total, maximum, minimum = self.get_mbox_stats(mbox) 78 | if settings.server_type == 'read-only': 79 | groups.append("papercut.mbox.%s %s %s n" % (mbox, maximum, minimum)) 80 | else: 81 | groups.append("papercut.mbox.%s %s %s y" % (mbox, maximum, minimum)) 82 | return "\r\n".join(groups) 83 | 84 | def get_STAT(self, group_name, id): 85 | # check if the message exists 86 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 87 | i = 0 88 | while mbox.next(): 89 | if i == int(id): 90 | return True 91 | i = i + 1 92 | return False 93 | 94 | def get_ARTICLE(self, group_name, id): 95 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 96 | i = 0 97 | while 1: 98 | msg = mbox.next() 99 | if msg is None: 100 | return None 101 | if i == int(id): 102 | return ("\r\n".join(["%s" % string.strip(k) for k in msg.headers]), msg.fp.read()) 103 | i = i + 1 104 | 105 | def get_LAST(self, group_name, current_id): 106 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 107 | if current_id == 1: 108 | return None 109 | else: 110 | i = 0 111 | while 1: 112 | msg = mbox.next() 113 | if msg is None: 114 | return None 115 | if (i+1) == current_id: 116 | return i 117 | i = i + 1 118 | 119 | def get_NEXT(self, group_name, current_id): 120 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 121 | print repr(current_id) 122 | i = 0 123 | while 1: 124 | msg = mbox.next() 125 | if msg is None: 126 | return None 127 | if i > current_id: 128 | return i 129 | i = i + 1 130 | 131 | def get_message(self, group_name, id): 132 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 133 | i = 0 134 | while 1: 135 | msg = mbox.next() 136 | if msg is None: 137 | return None 138 | if i == int(id): 139 | return msg 140 | i = i + 1 141 | 142 | def get_HEAD(self, group_name, id): 143 | msg = self.get_message(group_name, id) 144 | headers = [] 145 | headers.append("Path: %s" % (settings.nntp_hostname)) 146 | headers.append("From: %s" % (msg.get('from'))) 147 | headers.append("Newsgroups: %s" % (group_name)) 148 | headers.append("Date: %s" % (msg.get('date'))) 149 | headers.append("Subject: %s" % (msg.get('subject'))) 150 | headers.append("Message-ID: <%s@%s>" % (id, group_name)) 151 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, id)) 152 | return "\r\n".join(headers) 153 | 154 | def get_BODY(self, group_name, id): 155 | msg = self.get_message(group_name, id) 156 | if msg is None: 157 | return None 158 | else: 159 | return strutil.format_body(msg.fp.read()) 160 | 161 | def get_XOVER(self, group_name, start_id, end_id='ggg'): 162 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 163 | # don't count the first message 164 | mbox.next() 165 | i = 1 166 | overviews = [] 167 | while 1: 168 | msg = mbox.next() 169 | if msg is None: 170 | break 171 | author = msg.get('from') 172 | formatted_time = msg.get('date') 173 | message_id = msg.get('message-id') 174 | line_count = len(msg.fp.read().split('\n')) 175 | xref = 'Xref: %s %s:%s' % (settings.nntp_hostname, group_name, i) 176 | if msg.get('in-reply-to') is not None: 177 | reference = msg.get('in-reply-to') 178 | else: 179 | reference = "" 180 | # message_number subject author date message_id reference bytes lines xref 181 | overviews.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (i, msg.get('subject'), author, formatted_time, message_id, reference, len(strutil.format_body(msg.fp.read())), line_count, xref)) 182 | i = i + 1 183 | return "\r\n".join(overviews) 184 | 185 | def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): 186 | # no support for this right now 187 | return None 188 | 189 | def get_LISTGROUP(self, group_name): 190 | mbox = self.get_mailbox(group_name.replace('papercut.mbox.', '')) 191 | # don't count the first message 192 | mbox.next() 193 | i = 0 194 | ids = [] 195 | while 1: 196 | msg = mbox.next() 197 | if msg is None: 198 | break 199 | i = i + 1 200 | ids.append(i) 201 | return "\r\n".join(ids) 202 | 203 | def get_XGTITLE(self, pattern=None): 204 | # no support for this right now 205 | return None 206 | 207 | def get_XHDR(self, group_name, header, style, range): 208 | # no support for this right now 209 | return None 210 | 211 | def do_POST(self, group_name, lines, ip_address, username=''): 212 | # let's make the mbox storage always read-only for now 213 | return None 214 | -------------------------------------------------------------------------------- /storage/maildir.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2004 Scott Parish, Joao Prado Maia 3 | # See the LICENSE file for more information. 4 | # $Id: maildir.py,v 1.2 2004-08-01 01:51:48 jpm Exp $ 5 | 6 | # 7 | # Maildir backend for papercut 8 | # 9 | # Notes: 10 | # 11 | # Currently the numeric message ids are based off the number of 12 | # files in that group's directy. This means that if you change 13 | # a file name, or delete a file you are going to change ids, which 14 | # in turn is going to confuse nntp clients! 15 | # 16 | # To add a new group: 17 | # mkdir -p /home/papercut/maildir/my.new.group/{new,cur,tmp} 18 | # 19 | 20 | import dircache 21 | from fnmatch import fnmatch 22 | import glob 23 | import os 24 | import mailbox 25 | import rfc822 26 | import settings 27 | import socket 28 | import strutil 29 | import string 30 | import time 31 | 32 | 33 | def maildir_date_cmp(a, b): 34 | """compare maildir file names 'a' and 'b' for sort()""" 35 | a = os.path.basename(a) 36 | b = os.path.basename(b) 37 | a = int(a[: a.find(".")]) 38 | b = int(b[: b.find(".")]) 39 | return cmp(a, b) 40 | 41 | 42 | 43 | class Papercut_Storage: 44 | """ 45 | Storage backend interface for mbox files 46 | """ 47 | _proc_post_count = 0 48 | 49 | def __init__(self, group_prefix="papercut.maildir."): 50 | self.maildir_dir = settings.maildir_path 51 | self.group_prefix = group_prefix 52 | 53 | 54 | def _get_group_dir(self, group): 55 | return os.path.join(self.maildir_dir, group) 56 | 57 | 58 | def _groupname2group(self, group_name): 59 | return group_name.replace(self.group_prefix, '') 60 | 61 | 62 | def _group2groupname(self, group): 63 | return self.group_prefix + group 64 | 65 | 66 | def _new_to_cur(self, group): 67 | groupdir = self._get_group_dir(group) 68 | for f in dircache.listdir(os.path.join(groupdir, 'new')): 69 | ofp = os.path.join(groupdir, 'new', f) 70 | nfp = os.path.join(groupdir, 'cur', f + ":2,") 71 | os.rename(ofp, nfp) 72 | 73 | 74 | def get_groupname_list(self): 75 | groups = dircache.listdir(self.maildir_dir) 76 | return ["papercut.maildir.%s" % k for k in groups] 77 | 78 | 79 | def get_group_article_list(self, group): 80 | self._new_to_cur(group) 81 | groupdir = self._get_group_dir(group) 82 | articledir = os.path.join(self._get_group_dir(group), 'cur') 83 | articles = dircache.listdir(articledir) 84 | articles.sort(maildir_date_cmp) 85 | return articles 86 | 87 | 88 | def get_group_article_count(self, group): 89 | self._new_to_cur(group) 90 | articles = dircache.listdir(os.path.join(self.maildir_dir, group)) 91 | return len(articles) 92 | 93 | 94 | def group_exists(self, group_name): 95 | groupnames = self.get_groupname_list() 96 | found = False 97 | 98 | for name in groupnames: 99 | # group names are supposed to be case insensitive 100 | if string.lower(name) == string.lower(group_name): 101 | found = True 102 | break 103 | 104 | return found 105 | 106 | 107 | def get_first_article(self, group_name): 108 | return 1 109 | 110 | 111 | def get_group_stats(self, group_name): 112 | total, max, min = self.get_maildir_stats(group_name) 113 | return (total, min, max, group_name) 114 | 115 | 116 | def get_maildir_stats(self, group_name): 117 | cnt = len(self.get_group_article_list(group_name)) 118 | return cnt, cnt, 1 119 | 120 | 121 | def get_message_id(self, msg_num, group_name): 122 | msg_num = int(msg_num) 123 | group = self._groupname2group(group_name) 124 | return '<%s@%s>' % (self.get_group_article_list(group)[msg_num - 1], 125 | group_name) 126 | 127 | 128 | def get_NEWGROUPS(self, ts, group='%'): 129 | return None 130 | 131 | 132 | # UNTESTED 133 | def get_NEWNEWS(self, ts, group='*'): 134 | gpaths = glob.glob(os.path.join(self.maildir_dir, group)) 135 | articles = [] 136 | for gpath in gpaths: 137 | articles = dircache.listdir(os.path.join(gpath, "cur")) 138 | group = os.path.basename(gpath) 139 | group_name = self._group2groupname(group) 140 | 141 | for article in articles: 142 | apath = os.path.join(gpath, "cur", article) 143 | if os.path.getmtime(apath) < ts: 144 | continue 145 | 146 | articles.append("<%s@%s" % (article, group_name)) 147 | 148 | if len(articles) == 0: 149 | return '' 150 | else: 151 | return "\r\n".join(articles) 152 | 153 | 154 | def get_GROUP(self, group_name): 155 | group = self._groupname2group(group_name) 156 | result = self.get_maildir_stats(group) 157 | return (result[0], result[2], result[1]) 158 | 159 | 160 | def get_LIST(self, username=""): 161 | result = self.get_groupname_list() 162 | 163 | if len(result) == 0: 164 | return "" 165 | 166 | else: 167 | groups = [] 168 | mutable = ('y', 'n')[settings.server_type == 'read-only'] 169 | 170 | for group_name in result: 171 | group = self._groupname2group(group_name) 172 | total, maximum, minimum = self.get_maildir_stats(group) 173 | groups.append("%s %s %s %s" % (group_name, maximum, 174 | minimum, mutable)) 175 | return "\r\n".join(groups) 176 | 177 | 178 | def get_STAT(self, group_name, id): 179 | # check if the message exists 180 | id = int(id) 181 | group = self._groupname2group(group_name) 182 | 183 | return id <= self.get_group_article_count(group) 184 | 185 | 186 | def get_message(self, group_name, id): 187 | group = self._groupname2group(group_name) 188 | id = int(id) 189 | 190 | try: 191 | article = self.get_group_article_list(group)[id - 1] 192 | file = os.path.join(self.maildir_dir, group, "cur", article) 193 | return rfc822.Message(open(file)) 194 | 195 | except IndexError: 196 | return None 197 | 198 | 199 | def get_ARTICLE(self, group_name, id): 200 | msg = self.get_message(group_name, id) 201 | if not msg: 202 | return None 203 | return ("\r\n".join(["%s" % string.strip(k) for k in msg.headers]), msg.fp.read()) 204 | 205 | 206 | def get_LAST(self, group_name, current_id): 207 | if current_id <= 1: 208 | return None 209 | return current_id - 1 210 | 211 | 212 | def get_NEXT(self, group_name, current_id): 213 | group = self._groupname2group(group_name) 214 | if current_id >= self.get_group_article_count(group): 215 | return None 216 | return current_id + 1 217 | 218 | 219 | def get_HEAD(self, group_name, id): 220 | msg = self.get_message(group_name, id) 221 | headers = [] 222 | headers.append("Path: %s" % (settings.nntp_hostname)) 223 | headers.append("From: %s" % (msg.get('from'))) 224 | headers.append("Newsgroups: %s" % (group_name)) 225 | headers.append("Date: %s" % (msg.get('date'))) 226 | headers.append("Subject: %s" % (msg.get('subject'))) 227 | headers.append("Message-ID: <%s@%s>" % (id, group_name)) 228 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, 229 | group_name, id)) 230 | return "\r\n".join(headers) 231 | 232 | 233 | def get_BODY(self, group_name, id): 234 | msg = self.get_message(group_name, id) 235 | if msg is None: 236 | return None 237 | else: 238 | return strutil.format_body(msg.fp.read()) 239 | 240 | 241 | def get_XOVER(self, group_name, start_id, end_id='ggg'): 242 | group = self._groupname2group(group_name) 243 | start_id = int(start_id) 244 | if end_id == 'ggg': 245 | end_id = self.get_group_article_count(group) 246 | else: 247 | end_id = int(end_id) 248 | 249 | overviews = [] 250 | for id in range(start_id, end_id + 1): 251 | msg = self.get_message(group_name, id) 252 | 253 | if msg is None: 254 | break 255 | 256 | author = msg.get('from') 257 | formatted_time = msg.get('date') 258 | message_id = self.get_message_id(id, group_name) 259 | line_count = len(msg.fp.read().split('\n')) 260 | xref = 'Xref: %s %s:%d' % (settings.nntp_hostname, group_name, id) 261 | 262 | if msg.get('references') is not None: 263 | reference = msg.get('references') 264 | else: 265 | reference = "" 266 | # message_number subject author date 267 | # message_id reference bytes lines xref 268 | 269 | overviews.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % \ 270 | (id, msg.get('subject'), author, 271 | formatted_time, message_id, reference, 272 | len(strutil.format_body(msg.fp.read())), 273 | line_count, xref)) 274 | 275 | return "\r\n".join(overviews) 276 | 277 | 278 | # UNTESTED 279 | def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): 280 | group = self._groupname2group(group_name) 281 | header = header.upper() 282 | start_id = int(start_id) 283 | if end_id == 'ggg': 284 | end_id = self.get_group_article_count(group) 285 | else: 286 | end_id = int(end_id) 287 | 288 | hdrs = [] 289 | for id in range(start_id, end_id + 1): 290 | 291 | if header == 'MESSAGE-ID': 292 | msg_id = self.get_message_id(id, group_name) 293 | if fnmatch(msg_id, pattern): 294 | hdrs.append('%d %s' % (id, msg_id)) 295 | continue 296 | elif header == 'XREF': 297 | xref = '%s %s:%d' % (settings.nntp_hostname, group_name, id) 298 | if fnmatch(xref, pattern): 299 | hdrs.append('%d %s' % (id, xref)) 300 | continue 301 | 302 | msg = self.get_message(group_name, id) 303 | if header == 'BYTES': 304 | msg.fp.seek(0, 2) 305 | bytes = msg.fp.tell() 306 | if fnmatch(str(bytes), pattern): 307 | hdrs.append('%d %d' % (id, bytes)) 308 | elif header == 'LINES': 309 | lines = len(msg.fp.readlines()) 310 | if fnmatch(str(lines), pattern): 311 | hdrs.append('%d %d' % (id, lines)) 312 | else: 313 | hdr = msg.get(header) 314 | if hdr and fnmatch(hdr, pattern): 315 | hdrs.append('%d %s' % (id, hdr)) 316 | 317 | if len(hdrs): 318 | return "\r\n".join(hdrs) 319 | else: 320 | return "" 321 | 322 | 323 | def get_LISTGROUP(self, group_name): 324 | ids = range(1, self.get_group_article_count(group) + 1) 325 | ids = [str(id) for id in ids] 326 | return "\r\n".join(ids) 327 | 328 | def get_XGTITLE(self, pattern=None): 329 | # XXX no support for this right now 330 | return '' 331 | 332 | 333 | def get_XHDR(self, group_name, header, style, ranges): 334 | print group_name, header, style, ranges 335 | group = self._groupname2group(group_name) 336 | header = header.upper() 337 | 338 | if style == 'range': 339 | if len(ranges) == 2: 340 | range_end = int(ranges[1]) 341 | else: 342 | range_end = self.get_group_article_count(group) 343 | ids = range(int(ranges[0]), range_end + 1) 344 | else: 345 | ids = (int(ranges[0])) 346 | 347 | hdrs = [] 348 | for id in ids: 349 | if header == 'MESSAGE-ID': 350 | hdrs.append('%d %s' % \ 351 | (id, self.get_message_id(id, group_name))) 352 | continue 353 | elif header == 'XREF': 354 | hdrs.append('%d %s %s:%d' % (id, settings.nntp_hostname, 355 | group_name, id)) 356 | continue 357 | 358 | msg = self.get_message(group_name, id) 359 | if header == 'BYTES': 360 | msg.fp.seek(0, 2) 361 | hdrs.append('%d %d' % (id, msg.fp.tell())) 362 | elif header == 'LINES': 363 | hdrs.append('%d %d' % (id, len(msg.fp.readlines()))) 364 | else: 365 | hdr = msg.get(header) 366 | if hdr: 367 | hdrs.append('%d %s' % (id, hdr)) 368 | 369 | if len(hdrs) == 0: 370 | return "" 371 | else: 372 | return "\r\n".join(hdrs) 373 | 374 | 375 | def do_POST(self, group_name, body, ip_address, username=''): 376 | self._proc_post_count += 1 377 | count = self._proc_post_count 378 | 379 | ts = [int(x) for x in str(time.time()).split(".")] 380 | file = "%d.M%dP%dQ%d.%s" % (ts[0], ts[1], os.getpid(), 381 | count, socket.gethostname()) 382 | group = self._groupname2group(group_name) 383 | groupdir = self._get_group_dir(group) 384 | tfpath = os.path.join(self.maildir_dir, groupdir, "tmp", file) 385 | nfpath = os.path.join(self.maildir_dir, groupdir, "new", file) 386 | 387 | fd = open(tfpath, 'w') 388 | fd.write(body) 389 | fd.close 390 | 391 | os.rename(tfpath, nfpath) 392 | return 1 393 | 394 | -------------------------------------------------------------------------------- /storage/mysql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: mysql.py,v 1.44 2004-08-01 01:51:48 jpm Exp $ 4 | import MySQLdb 5 | import time 6 | import re 7 | import settings 8 | import strutil 9 | import mime 10 | 11 | # we don't need to compile the regexps everytime.. 12 | singleline_regexp = re.compile("^\.", re.M) 13 | from_regexp = re.compile("^From:(.*)", re.M) 14 | subject_regexp = re.compile("^Subject:(.*)", re.M) 15 | references_regexp = re.compile("^References:(.*)<(.*)>", re.M) 16 | 17 | class Papercut_Storage: 18 | """ 19 | Storage Backend interface for saving the article information in a MySQL database. 20 | 21 | This is not a storage to implement a web board -> nntp gateway, but a standalone nntp server. 22 | """ 23 | 24 | def __init__(self): 25 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 26 | self.cursor = self.conn.cursor() 27 | 28 | def quote_string(self, text): 29 | """Quotes strings the MySQL way.""" 30 | return text.replace("'", "\\'") 31 | 32 | def get_body(self, lines): 33 | pass 34 | 35 | def get_header(self, lines): 36 | pass 37 | 38 | def group_exists(self, group_name): 39 | stmt = """ 40 | SELECT 41 | COUNT(*) AS check 42 | FROM 43 | papercut_groups 44 | WHERE 45 | LOWER(name)=LOWER('%s')""" % (group_name) 46 | self.cursor.execute(stmt) 47 | return self.cursor.fetchone()[0] 48 | 49 | def article_exists(self, group_name, style, range): 50 | table_name = self.get_table_name(group_name) 51 | stmt = """ 52 | SELECT 53 | COUNT(*) AS check 54 | FROM 55 | %s 56 | WHERE 57 | """ % (table_name) 58 | if style == 'range': 59 | stmt = "%s id > %s" % (stmt, range[0]) 60 | if len(range) == 2: 61 | stmt = "%s AND id < %s" % (stmt, range[1]) 62 | else: 63 | stmt = "%s id = %s" % (stmt, range[0]) 64 | self.cursor.execute(stmt) 65 | return self.cursor.fetchone()[0] 66 | 67 | def get_first_article(self, group_name): 68 | table_name = self.get_table_name(group_name) 69 | stmt = """ 70 | SELECT 71 | IF(MIN(id) IS NULL, 0, MIN(id)) AS first_article 72 | FROM 73 | %s""" % (table_name) 74 | num_rows = self.cursor.execute(stmt) 75 | return self.cursor.fetchone()[0] 76 | 77 | def get_group_stats(self, group_name): 78 | total, max, min = self.get_table_stats(self.get_table_name(group_name)) 79 | return (total, min, max, group_name) 80 | 81 | def get_table_stats(self, table_name): 82 | stmt = """ 83 | SELECT 84 | COUNT(id) AS total, 85 | IF(MAX(id) IS NULL, 0, MAX(id)) AS maximum, 86 | IF(MIN(id) IS NULL, 0, MIN(id)) AS minimum 87 | FROM 88 | %s""" % (table_name) 89 | num_rows = self.cursor.execute(stmt) 90 | return self.cursor.fetchone() 91 | 92 | def get_table_name(self, group_name): 93 | stmt = """ 94 | SELECT 95 | table_name 96 | FROM 97 | papercut_groups 98 | WHERE 99 | name='%s'""" % (group_name.replace('*', '%')) 100 | self.cursor.execute(stmt) 101 | return self.cursor.fetchone()[0] 102 | 103 | def get_message_id(self, msg_num, group): 104 | return '<%s@%s>' % (msg_num, group) 105 | 106 | def get_NEWGROUPS(self, ts, group='%'): 107 | return None 108 | 109 | def get_NEWNEWS(self, ts, group='*'): 110 | stmt = """ 111 | SELECT 112 | name, 113 | table_name 114 | FROM 115 | papercut_groups 116 | WHERE 117 | name='%s' 118 | ORDER BY 119 | name ASC""" % (group_name.replace('*', '%')) 120 | self.cursor.execute(stmt) 121 | result = list(self.cursor.fetchall()) 122 | articles = [] 123 | for group, table in result: 124 | stmt = """ 125 | SELECT 126 | id 127 | FROM 128 | %s 129 | WHERE 130 | UNIX_TIMESTAMP(datestamp) >= %s""" % (table, ts) 131 | num_rows = self.cursor.execute(stmt) 132 | if num_rows == 0: 133 | continue 134 | ids = list(self.cursor.fetchall()) 135 | for id in ids: 136 | articles.append("<%s@%s>" % (id, group)) 137 | if len(articles) == 0: 138 | return '' 139 | else: 140 | return "\r\n".join(articles) 141 | 142 | def get_GROUP(self, group_name): 143 | table_name = self.get_table_name(group_name) 144 | result = self.get_table_stats(table_name) 145 | return (result[0], result[2], result[1]) 146 | 147 | def get_LIST(self, username=""): 148 | stmt = """ 149 | SELECT 150 | name, 151 | table_name 152 | FROM 153 | papercut_groups 154 | WHERE 155 | LENGTH(name) > 0 156 | ORDER BY 157 | name ASC""" 158 | self.cursor.execute(stmt) 159 | result = list(self.cursor.fetchall()) 160 | if len(result) == 0: 161 | return "" 162 | else: 163 | lists = [] 164 | for group_name, table in result: 165 | total, maximum, minimum = self.get_table_stats(table) 166 | if settings.server_type == 'read-only': 167 | lists.append("%s %s %s n" % (group_name, maximum, minimum)) 168 | else: 169 | lists.append("%s %s %s y" % (group_name, maximum, minimum)) 170 | return "\r\n".join(lists) 171 | 172 | def get_STAT(self, group_name, id): 173 | table_name = self.get_table_name(group_name) 174 | stmt = """ 175 | SELECT 176 | id 177 | FROM 178 | %s 179 | WHERE 180 | id=%s""" % (table_name, id) 181 | return self.cursor.execute(stmt) 182 | 183 | def get_ARTICLE(self, group_name, id): 184 | table_name = self.get_table_name(group_name) 185 | stmt = """ 186 | SELECT 187 | id, 188 | author, 189 | subject, 190 | UNIX_TIMESTAMP(datestamp) AS datestamp, 191 | body, 192 | parent 193 | FROM 194 | %s 195 | WHERE 196 | id=%s""" % (table_name, id) 197 | num_rows = self.cursor.execute(stmt) 198 | if num_rows == 0: 199 | return None 200 | result = list(self.cursor.fetchone()) 201 | headers = [] 202 | headers.append("Path: %s" % (settings.nntp_hostname)) 203 | headers.append("From: %s" % (result[1])) 204 | headers.append("Newsgroups: %s" % (group_name)) 205 | headers.append("Date: %s" % (strutil.get_formatted_time(time.localtime(result[3])))) 206 | headers.append("Subject: %s" % (result[2])) 207 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 208 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 209 | if result[5] != 0: 210 | headers.append("References: <%s@%s>" % (result[5], group_name)) 211 | return ("\r\n".join(headers), strutil.format_body(result[4])) 212 | 213 | def get_LAST(self, group_name, current_id): 214 | table_name = self.get_table_name(group_name) 215 | stmt = """ 216 | SELECT 217 | id 218 | FROM 219 | %s 220 | WHERE 221 | id < %s 222 | ORDER BY 223 | id DESC 224 | LIMIT 0, 1""" % (table_name, current_id) 225 | num_rows = self.cursor.execute(stmt) 226 | if num_rows == 0: 227 | return None 228 | return self.cursor.fetchone()[0] 229 | 230 | def get_NEXT(self, group_name, current_id): 231 | table_name = self.get_table_name(group_name) 232 | stmt = """ 233 | SELECT 234 | id 235 | FROM 236 | %s 237 | WHERE 238 | id > %s 239 | ORDER BY 240 | id ASC 241 | LIMIT 0, 1""" % (table_name, current_id) 242 | num_rows = self.cursor.execute(stmt) 243 | if num_rows == 0: 244 | return None 245 | return self.cursor.fetchone()[0] 246 | 247 | def get_HEAD(self, group_name, id): 248 | table_name = self.get_table_name(group_name) 249 | stmt = """ 250 | SELECT 251 | id, 252 | author, 253 | subject, 254 | UNIX_TIMESTAMP(datestamp) AS datestamp, 255 | parent 256 | FROM 257 | %s 258 | WHERE 259 | id=%s""" % (table_name, id) 260 | num_rows = self.cursor.execute(stmt) 261 | if num_rows == 0: 262 | return None 263 | result = list(self.cursor.fetchone()) 264 | headers = [] 265 | headers.append("Path: %s" % (settings.nntp_hostname)) 266 | headers.append("From: %s" % (result[1])) 267 | headers.append("Newsgroups: %s" % (group_name)) 268 | headers.append("Date: %s" % (strutil.get_formatted_time(time.localtime(result[3])))) 269 | headers.append("Subject: %s" % (result[2])) 270 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 271 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 272 | if result[4] != 0: 273 | headers.append("References: <%s@%s>" % (result[4], group_name)) 274 | return "\r\n".join(headers) 275 | 276 | def get_BODY(self, group_name, id): 277 | table_name = self.get_table_name(group_name) 278 | stmt = """ 279 | SELECT 280 | body 281 | FROM 282 | %s 283 | WHERE 284 | id=%s""" % (table_name, id) 285 | num_rows = self.cursor.execute(stmt) 286 | if num_rows == 0: 287 | return None 288 | else: 289 | return strutil.format_body(self.cursor.fetchone()[0]) 290 | 291 | def get_XOVER(self, group_name, start_id, end_id='ggg'): 292 | table_name = self.get_table_name(group_name) 293 | stmt = """ 294 | SELECT 295 | id, 296 | parent, 297 | author, 298 | subject, 299 | UNIX_TIMESTAMP(datestamp) AS datestamp, 300 | body, 301 | line_num, 302 | bytes 303 | FROM 304 | %s 305 | WHERE 306 | id >= %s""" % (table_name, start_id) 307 | if end_id != 'ggg': 308 | stmt = "%s AND id <= %s" % (stmt, end_id) 309 | self.cursor.execute(stmt) 310 | result = list(self.cursor.fetchall()) 311 | overviews = [] 312 | for row in result: 313 | message_id = "<%s@%s>" % (row[0], group_name) 314 | xref = 'Xref: %s %s:%s' % (settings.nntp_hostname, group_name, row[0]) 315 | if row[1] != 0: 316 | reference = "<%s@%s>" % (row[1], group_name) 317 | else: 318 | reference = "" 319 | # message_number subject author date message_id reference bytes lines xref 320 | overviews.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (row[0], row[3], row[2], strutil.get_formatted_time(time.localtime(row[4])), message_id, reference, row[7], row[6], xref)) 321 | return "\r\n".join(overviews) 322 | 323 | def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): 324 | table_name = self.get_table_name(group_name) 325 | stmt = """ 326 | SELECT 327 | id, 328 | parent, 329 | author, 330 | subject, 331 | UNIX_TIMESTAMP(datestamp) AS datestamp, 332 | bytes, 333 | line_num 334 | FROM 335 | %s 336 | WHERE 337 | id >= %s AND""" % (table_name, header, strutil.format_wildcards(pattern), start_id) 338 | if header.upper() == 'SUBJECT': 339 | stmt = "%s AND subject REGEXP '%s'" % (stmt, strutil.format_wildcards(pattern)) 340 | elif header.upper() == 'FROM': 341 | stmt = "%s AND (author REGEXP '%s' OR email REGEXP '%s')" % (stmt, strutil.format_wildcards(pattern), strutil.format_wildcards(pattern)) 342 | elif header.upper() == 'DATE': 343 | stmt = "%s AND %s" % (stmt, pattern) 344 | if end_id != 'ggg': 345 | stmt = "%s AND id <= %s" % (stmt, end_id) 346 | num_rows = self.cursor.execute(stmt) 347 | if num_rows == 0: 348 | return None 349 | result = list(self.cursor.fetchall()) 350 | hdrs = [] 351 | for row in result: 352 | if header.upper() == 'SUBJECT': 353 | hdrs.append('%s %s' % (row[0], row[3])) 354 | elif header.upper() == 'FROM': 355 | hdrs.append('%s %s' % (row[0], row[2])) 356 | elif header.upper() == 'DATE': 357 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[4])))) 358 | elif header.upper() == 'MESSAGE-ID': 359 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 360 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 361 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 362 | elif header.upper() == 'BYTES': 363 | hdrs.append('%s %s' % (row[0], row[5])) 364 | elif header.upper() == 'LINES': 365 | hdrs.append('%s %s' % (row[0], row[6])) 366 | elif header.upper() == 'XREF': 367 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 368 | if len(hdrs) == 0: 369 | return "" 370 | else: 371 | return "\r\n".join(hdrs) 372 | 373 | def get_LISTGROUP(self, group_name): 374 | table_name = self.get_table_name(group_name) 375 | stmt = """ 376 | SELECT 377 | id 378 | FROM 379 | %s 380 | ORDER BY 381 | id ASC""" % (table_name) 382 | self.cursor.execute(stmt) 383 | result = list(self.cursor.fetchall()) 384 | return "\r\n".join(["%s" % k for k in result]) 385 | 386 | def get_XGTITLE(self, pattern=None): 387 | stmt = """ 388 | SELECT 389 | name, 390 | description 391 | FROM 392 | papercut_groups 393 | WHERE 394 | LENGTH(name) > 0""" 395 | if pattern != None: 396 | stmt = stmt + """ AND 397 | name REGEXP '%s'""" % (strutil.format_wildcards(pattern)) 398 | stmt = stmt + """ 399 | ORDER BY 400 | name ASC""" 401 | self.cursor.execute(stmt) 402 | result = list(self.cursor.fetchall()) 403 | return "\r\n".join(["%s %s" % (k, v) for k, v in result]) 404 | 405 | def get_XHDR(self, group_name, header, style, range): 406 | table_name = self.get_table_name(group_name) 407 | stmt = """ 408 | SELECT 409 | id, 410 | parent, 411 | author, 412 | subject, 413 | UNIX_TIMESTAMP(datestamp) AS datestamp, 414 | bytes, 415 | line_num 416 | FROM 417 | %s 418 | WHERE 419 | """ % (table_name) 420 | if style == 'range': 421 | stmt = '%s id >= %s' % (stmt, range[0]) 422 | if len(range) == 2: 423 | stmt = '%s AND id <= %s' % (stmt, range[1]) 424 | else: 425 | stmt = '%s id = %s' % (stmt, range[0]) 426 | if self.cursor.execute(stmt) == 0: 427 | return None 428 | result = self.cursor.fetchall() 429 | hdrs = [] 430 | for row in result: 431 | if header.upper() == 'SUBJECT': 432 | hdrs.append('%s %s' % (row[0], row[3])) 433 | elif header.upper() == 'FROM': 434 | hdrs.append('%s %s' % (row[0], row[2])) 435 | elif header.upper() == 'DATE': 436 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[4])))) 437 | elif header.upper() == 'MESSAGE-ID': 438 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 439 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 440 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 441 | elif header.upper() == 'BYTES': 442 | hdrs.append('%s %s' % (row[0], row[6])) 443 | elif header.upper() == 'LINES': 444 | hdrs.append('%s %s' % (row[0], row[7])) 445 | elif header.upper() == 'XREF': 446 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 447 | if len(hdrs) == 0: 448 | return "" 449 | else: 450 | return "\r\n".join(hdrs) 451 | 452 | def do_POST(self, group_name, body, ip_address, username=''): 453 | table_name = self.get_table_name(group_name) 454 | author = from_regexp.search(body, 0).groups()[0].strip() 455 | subject = subject_regexp.search(body, 0).groups()[0].strip() 456 | if body.find('References') != -1: 457 | references = references_regexp.search(body, 0).groups() 458 | parent_id, void = references[-1].strip().split('@') 459 | stmt = """ 460 | SELECT 461 | IF(MAX(id) IS NULL, 1, MAX(id)+1) AS next_id 462 | FROM 463 | %s""" % (table_name) 464 | num_rows = self.cursor.execute(stmt) 465 | if num_rows == 0: 466 | new_id = 1 467 | else: 468 | new_id = self.cursor.fetchone()[0] 469 | stmt = """ 470 | SELECT 471 | id, 472 | thread 473 | FROM 474 | %s 475 | WHERE 476 | id=%s 477 | GROUP BY 478 | id""" % (table_name, parent_id) 479 | num_rows = self.cursor.execute(stmt) 480 | if num_rows == 0: 481 | return None 482 | parent_id, thread_id = self.cursor.fetchone() 483 | else: 484 | stmt = """ 485 | SELECT 486 | IF(MAX(id) IS NULL, 1, MAX(id)+1) AS next_id 487 | FROM 488 | %s""" % (table_name) 489 | self.cursor.execute(stmt) 490 | new_id = self.cursor.fetchone()[0] 491 | parent_id = 0 492 | thread_id = new_id 493 | body = mime.get_body(body) 494 | stmt = """ 495 | INSERT INTO 496 | %s 497 | ( 498 | id, 499 | datestamp, 500 | thread, 501 | parent, 502 | author, 503 | subject, 504 | host, 505 | body, 506 | bytes, 507 | line_num 508 | ) VALUES ( 509 | %s, 510 | NOW(), 511 | %s, 512 | %s, 513 | '%s', 514 | '%s', 515 | '%s', 516 | '%s', 517 | %s, 518 | %s 519 | ) 520 | """ % (table_name, new_id, thread_id, parent_id, self.quote_string(author), self.quote_string(subject), ip_address, self.quote_string(body), len(body), len(body.split('\n'))) 521 | if not self.cursor.execute(stmt): 522 | return None 523 | else: 524 | return 1 525 | -------------------------------------------------------------------------------- /docs/draft-ietf-nntpext-tls-nntp-01.txt: -------------------------------------------------------------------------------- 1 | Network Working Group J. Vinocur 2 | INTERNET DRAFT Cornell University 3 | Document: draft-ietf-nntpext-tls-nntp-01.txt C. Newman 4 | Sun Microsystems 5 | October 2003 6 | 7 | 8 | Using TLS with NNTP 9 | 10 | 11 | Status of this memo 12 | 13 | This document is an Internet-Draft and is in full conformance with 14 | all provisions of Section 10 of RFC 2026. 15 | 16 | Internet-Drafts are working documents of the Internet Engineering 17 | Task Force (IETF), its areas, and its working groups. Note that 18 | other groups may also distribute working documents as 19 | Internet-Drafts. 20 | 21 | Internet-Drafts are draft documents valid for a maximum of six 22 | months and may be updated, replaced, or obsoleted by other 23 | documents at any time. It is inappropriate to use Internet-Drafts 24 | as reference material or to cite them other than as "work in 25 | progress." 26 | 27 | The list of current Internet-Drafts can be accessed at 28 | http://www.ietf.org/ietf/1id-abstracts.html. 29 | 30 | The list of Internet-Draft Shadow Directories can be accessed at 31 | http://www.ietf.org/shadow.html. 32 | 33 | Copyright Notice 34 | 35 | Copyright (C) The Internet Society (2002). All Rights Reserved. 36 | 37 | Abstract 38 | 39 | This memo defines an extension to the Network News Transport 40 | Protocol [NNTP] to provide connection-based encryption (via 41 | Transport Layer Security [TLS]). The primary goal is to provide 42 | encryption for single-link confidentiality purposes, but data 43 | integrity and (optional) certificate-based peer entity 44 | authentication are also possible. 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | Vinocur & Newman Expires April 2004 [Page 1] 53 | 54 | Internet Draft TLS for NNTP October 2003 55 | 56 | 57 | Table of Contents 58 | 59 | 0. Changes from Previous Version ............................ 2 60 | 1. Introduction ............................................. 3 61 | 1.1. Conventions Used in this Document ................... 3 62 | 2. Advertising Capabilities with the Extensions Mechanism ... 3 63 | 3. STARTTLS Command ......................................... 4 64 | 3.1. Usage ............................................... 4 65 | 3.2. Description ......................................... 4 66 | 3.2.1. Processing After the STARTTLS Command .......... 5 67 | 3.2.2. Result of the STARTTLS Command ................. 6 68 | 3.3. Examples ............................................ 7 69 | 4. Augmented BNF Syntax for STARTTLS ........................ 8 70 | 5. Security Considerations .................................. 8 71 | 6. Acknowledgements ......................................... 10 72 | 7. Normative References ..................................... 10 73 | 8. Informative References ................................... 10 74 | 9. Authors' Addresses ....................................... 11 75 | 76 | 0. Changes from Previous Version 77 | 78 | New: 79 | o Text needed to comply with extensions framework guidelines: 80 | - Allows 483 to be returned for most commands 81 | - No pipelining 82 | - Not impacted by MODE READER 83 | o Examples section 84 | 85 | Changed: 86 | o Welcome banner is *not* reissued after STARTTLS 87 | o STARTTLS on an already-secure link gives 502 (not 580) 88 | o Failed negotiation gives 580 on the reestablished insecure link 89 | o Removed MULTIDOMAIN, need is resolved by RFC 3546 (a SHOULD) 90 | o Removed definition of 483, which is now included in base spec 91 | o Use HDR instead of PAT in the LIST EXTENSIONS example 92 | 93 | Clarified: 94 | o When the capability can be advertised 95 | o The specifc octet where encrypted session begins 96 | 97 | Other: 98 | o Reformatting to match base spec style 99 | o Assorted updates of phrasing and typographical varieties 100 | o Updated several references per new versions of documents 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | Vinocur & Newman Expires April 2004 [Page 2] 109 | 110 | Internet Draft TLS for NNTP October 2003 111 | 112 | 113 | 1. Introduction 114 | 115 | Historically, unencrypted NNTP [NNTP] connections were satisfactory 116 | for most purposes. However, sending passwords unencrypted over the 117 | network is no longer appropriate, and sometimes strong encryption 118 | is desired for the entire connection. 119 | 120 | The STARTTLS extension provides a way to use the popular TLS [TLS] 121 | service with the existing NNTP protocol. The current 122 | (unstandardized) use of TLS for NNTP is most commonly on a 123 | dedicated TCP port; this practice is discouraged for the reasons 124 | documented in section 7 of "Using TLS with IMAP, POP3 and ACAP" 125 | [TLS-IMAPPOP]. Therefore, this specification formalizes and 126 | extends the STARTTLS command already in occasional use by the 127 | installed base. 128 | 129 | 1.1. Conventions Used in this Document 130 | 131 | The key words "REQUIRED", "MUST", "MUST NOT", "SHOULD", "SHOULD 132 | NOT", "MAY", and "OPTIONAL" in this document are to be interpreted 133 | as described in "Key words for use in RFCs to Indicate Requirement 134 | Levels" [KEYWORDS]. 135 | 136 | Terms related to authentication are defined in "On Internet 137 | Authentication" [AUTH]. 138 | 139 | This document assumes you are familiar with NNTP [NNTP] and TLS 140 | [TLS]. 141 | 142 | In the examples, commands from the client are indicated with [C], 143 | and responses from the server are indicated with [S]. 144 | 145 | 2. Advertising Capabilities with the Extensions Mechanism 146 | 147 | The LIST EXTENSIONS command, documented in section 8 of [NNTP], 148 | provides a mechanism for clients to discover what extensions are 149 | available. 150 | 151 | A server supporting the STARTTLS command as defined in section 4 152 | will advertise the "STARTTLS" capability in response to the LIST 153 | EXTENSIONS command. However, this capability is not advertised 154 | after successful authentication [NNTP-AUTH], nor is it advertised 155 | once a TLS layer is active (see section 4.2.2). This capability 156 | may be advertised both before and after any use of MODE READER, 157 | with the same semantics. 158 | 159 | As the STARTTLS command is related to security, cached results of 160 | LIST EXTENSIONS from a previous session MUST NOT be used, as per 161 | 162 | 163 | 164 | Vinocur & Newman Expires April 2004 [Page 3] 165 | 166 | Internet Draft TLS for NNTP October 2003 167 | 168 | 169 | section 11.6 of [NNTP]. 170 | 171 | Example: 172 | [C] LIST EXTENSIONS 173 | [S] 202 Extensions supported: 174 | [S] OVER 175 | [S] HDR 176 | [S] LISTGROUP 177 | [S] STARTTLS 178 | [S] . 179 | 180 | Note that the STARTTLS command constitutes a mode change and thus 181 | clients MUST wait for completion prior to sending additional 182 | commands. 183 | 184 | 3. STARTTLS Command 185 | 186 | 3.1. Usage 187 | 188 | This command MUST NOT be pipelined. 189 | 190 | Syntax 191 | STARTTLS 192 | 193 | Responses 194 | 382 Continue with TLS negotiation 195 | 403 TLS temporarily not available 196 | 501 Command not supported or command syntax error 197 | 502 Command unavailable [1] 198 | 580 TLS negotiation failed 199 | 200 | [1] If a TLS layer is already active, or authentication has 201 | occurred, STARTTLS is not a valid command (see sections 4.2 and 202 | 4.2.2). 203 | 204 | Clients MUST support other response codes by processing them based 205 | on the first digit. However, the server MUST NOT return 483 in 206 | response to STARTTLS. (See section 3.2.1 of [NNTP].) 207 | 208 | 3.2. Description 209 | 210 | A client issues the STARTTLS command to request negotiation of TLS. 211 | The client MUST NOT send any additional commands on the socket 212 | until after it has received the server response to the command; 213 | this command MUST NOT be pipelined as per section 3.2 of [NNTP]. 214 | The STARTTLS command is usually used to request session encryption, 215 | although it can be used for client certificate authentication. 216 | 217 | 218 | 219 | 220 | Vinocur & Newman Expires April 2004 [Page 4] 221 | 222 | Internet Draft TLS for NNTP October 2003 223 | 224 | 225 | An NNTP server MAY require the client to perform a TLS negotiation 226 | before accepting any commands. In this case, the server SHOULD 227 | return the 483 encryption-required response code to every command 228 | other than HELP, LIST EXTENSIONS, QUIT, and any commands that 229 | establish encryption, such as STARTTLS; the server MUST NOT return 230 | 483 in response to these commands. Additionally, the client MAY 231 | decide to establish a security layer without first receiving a 483 232 | response. 233 | 234 | If the client receives a failure response to STARTTLS, the client 235 | must decide whether or not to continue the NNTP session. Such a 236 | decision is based on local policy. For instance, if TLS was being 237 | used for client authentication, the client might try to continue 238 | the session, in case the server allows it to do so even with no 239 | authentication. However, if TLS was being negotiated for 240 | encryption, a client that gets a failure response needs to decide 241 | whether to continue without TLS encryption, to wait and try again 242 | later, or to give up and notify the user of the error. 243 | 244 | After receiving a 382 response to a STARTTLS command, the client 245 | MUST start the TLS negotiation before giving any other NNTP 246 | commands. The TLS negotiation begins with the first octet 247 | following the CRLF of the 382 response. If, after having issued 248 | the STARTTLS command, the client finds out that some failure 249 | prevents it from actually starting a TLS handshake, then it SHOULD 250 | immediately close the connection. 251 | 252 | Servers MUST be able to understand backwards-compatible TLS Client 253 | Hello messages (provided that client_version is TLS 1.0 or later), 254 | and clients MAY use backwards-compatible Client Hello messages. 255 | Neither clients or servers are required to actually support Client 256 | Hello messages for anything other than TLS 1.0. However, the TLS 257 | extension for Server Name Indication [TLS-EXT] SHOULD be 258 | implemented by all clients; it also SHOULD be implemented by any 259 | server implementing STARTTLS that is known by multiple names 260 | (otherwise it is not possible for a server with several hostnames 261 | to present the correct certificate to the client). 262 | 263 | Although current use of TLS most often involves the dedication of 264 | port 563 for NNTP over TLS, the continued use of TLS on a separate 265 | port is discouraged for the reasons documented in section 7 of 266 | "Using TLS with IMAP, POP3 and ACAP" [TLS-IMAPPOP]. 267 | 268 | 3.2.1. Processing After the STARTTLS Command 269 | 270 | After the TLS handshake has been completed successfully, both 271 | parties MUST immediately decide whether or not to continue based on 272 | the authentication and privacy achieved. The NNTP client and 273 | 274 | 275 | 276 | Vinocur & Newman Expires April 2004 [Page 5] 277 | 278 | Internet Draft TLS for NNTP October 2003 279 | 280 | 281 | server may decide to move ahead even if the TLS negotiation ended 282 | with no authentication and/or no privacy because NNTP services are 283 | often performed without authentication or privacy, but some NNTP 284 | clients or servers may want to continue only if a particular level 285 | of authentication and/or privacy was achieved. 286 | 287 | If the NNTP client decides that the level of authentication or 288 | privacy is not high enough for it to continue, it SHOULD issue a 289 | QUIT command immediately after the TLS negotiation is complete. If 290 | the NNTP server decides that the level of authentication or privacy 291 | is not high enough for it to continue, it SHOULD do at least one of 292 | (1) close the connection, being aware that the client may interpret 293 | this behavior as a network problem and immediately reconnect and 294 | issue the same command sequence, or (2) keep the connection open 295 | and reply to NNTP commands from the client with the 483 response 296 | code (with a possible text string such as "Command refused due to 297 | lack of security"), however this behavior may tie up resources 298 | unacceptably. 299 | 300 | The decision of whether or not to believe the authenticity of the 301 | other party in a TLS negotiation is a local matter. However, some 302 | general rules for the decisions are: 303 | 304 | o The client MAY check that the identity presented in the server's 305 | certificate matches the intended server hostname or domain. 306 | This check is not required (and may fail in the absence of the 307 | TLS server_name extension [TLS-EXT], as described above), but if 308 | it is implemented and the match fails, the client SHOULD either 309 | request explicit user confirmation, or terminate the connection 310 | but allow the user to disable the check in the future. 311 | o Generally an NNTP server would want to accept any verifiable 312 | certificate from a client, however authentication can be done 313 | using the client certificate (perhaps in combination with the 314 | SASL EXTERNAL mechanism [NNTP-AUTH], although an implementation 315 | supporting STARTTLS is not required to support SASL in general 316 | or that mechanism in particular). The server MAY use 317 | information about the client certificate for identification of 318 | connections or posted articles (either in its logs or directly 319 | in posted articles). 320 | 321 | 3.2.2. Result of the STARTTLS Command 322 | 323 | If the TLS handshake fails in such a way that recovery is possible, 324 | the server will send a 580 response (without encryption), beginning 325 | with the first post-handshake octet. 326 | 327 | Upon successful completion of the TLS handshake, the NNTP protocol 328 | is reset to the initial state (the state in NNTP directly after the 329 | 330 | 331 | 332 | Vinocur & Newman Expires April 2004 [Page 6] 333 | 334 | Internet Draft TLS for NNTP October 2003 335 | 336 | 337 | connection is established). The server MUST discard any knowledge 338 | obtained from the client, such as the current newsgroup and article 339 | number, that was not obtained from the TLS negotiation itself; 340 | immediately after the TLS handshake, the server MUST NOT issue a 341 | welcome banner and MUST be prepared to accept commands from the 342 | client. The client MUST discard any knowledge obtained from the 343 | server, such as the list of NNTP service extensions, which was not 344 | obtained from the TLS negotiation itself. 345 | 346 | The extensions returned in response to a LIST EXTENSIONS command 347 | received after the TLS handshake MAY be different than the list 348 | returned before the TLS handshake. For example, an NNTP server 349 | supporting SASL [NNTP-AUTH] might not want to advertise support for 350 | a particular mechanism unless a client has sent an appropriate 351 | client certificate during a TLS handshake. 352 | 353 | Both the client and the server MUST know if there is a TLS session 354 | active. A client MUST NOT attempt to start a TLS session if a TLS 355 | session is already active. A server MUST NOT return the STARTTLS 356 | extension in response to a LIST EXTENSIONS command received after a 357 | TLS handshake has completed, and a server MUST respond with a 502 358 | response code if a STARTTLS command is received while a TLS session 359 | is already active. 360 | 361 | 3.3. Examples 362 | 363 | Example of a client being prompted to use encryption and 364 | negotiating it successfully (showing the removal of STARTTLS from 365 | the extensions list once a TLS layer is active), followed by an 366 | (inappropriate) attempt by the client to initiate another TLS 367 | negotiation: 368 | [C] LIST EXTENSIONS 369 | [S] 202 Extensions supported: 370 | [S] STARTTLS 371 | [S] OVER 372 | [S] . 373 | [C] GROUP local.confidential 374 | [S] 483 Encryption or stronger authentication required 375 | [C] STARTTLS 376 | [S] 382 Continue with TLS negotiation 377 | [TLS negotiation occurs here] 378 | [Following successful negotiation, traffic is via the TLS layer] 379 | [C] LIST EXTENSIONS 380 | [S] 202 Extensions supported: 381 | [S] OVER 382 | [S] . 383 | [C] STARTTLS 384 | [S] 502 STARTTLS not allowed with active TLS layer 385 | 386 | 387 | 388 | Vinocur & Newman Expires April 2004 [Page 7] 389 | 390 | Internet Draft TLS for NNTP October 2003 391 | 392 | 393 | Example of a request to begin TLS negotiation declined by the 394 | server: 395 | [C] STARTTLS 396 | [S] 403 TLS temporarily not available 397 | 398 | 4. Augmented BNF Syntax for STARTTLS 399 | 400 | This amends the formal syntax for NNTP [NNTP] to add the STARTTLS 401 | command. The syntax is defined using ABNF [ABNF], including the 402 | core rules from section 6 of [ABNF]. 403 | 404 | command /= starttls-command 405 | starttls-command = "STARTTLS" *WSP CRLF 406 | ; WSP and CRLF are defined in sec. 13 of [NNTP] 407 | 408 | 5. Security Considerations 409 | 410 | In general, the security considerations of the TLS protocol [TLS] 411 | and any implemented extensions [TLS-EXT] are applicable here; only 412 | the most important are highlighted specifically below. Also, this 413 | extension is not intended to cure the security considerations 414 | described in section 14 of [NNTP]; those considerations remain 415 | relevant to any NNTP implementation. 416 | 417 | Use of STARTTLS cannot protect protocol exchanges conducted prior 418 | to authentication. For this reason, the LIST EXTENSIONS command 419 | SHOULD be re-issued after successful negotiation of a security 420 | layer, and other protocol state SHOULD be re-negotiated as well. 421 | 422 | It should be noted that NNTP is not an end-to-end mechanism. Thus, 423 | if an NNTP client/server pair decide to add TLS privacy, they are 424 | securing the transport only for that link. Further, because 425 | delivery of a single piece of news may go between more than two 426 | NNTP servers, adding TLS privacy to one pair of servers does not 427 | mean that the entire NNTP chain has been made private. Further, 428 | just because an NNTP server can authenticate an NNTP client, it 429 | does not mean that the articles from the NNTP client were 430 | authenticated by the NNTP client when the client received them. 431 | 432 | Both the NNTP client and server must check the result of the TLS 433 | negotiation to see whether an acceptable degree of authentication 434 | and privacy was achieved. Ignoring this step completely 435 | invalidates using TLS for security. The decision about whether 436 | acceptable authentication or privacy was achieved is made locally, 437 | is implementation-dependent, and is beyond the scope of this 438 | document. 439 | 440 | The NNTP client and server should note carefully the result of the 441 | 442 | 443 | 444 | Vinocur & Newman Expires April 2004 [Page 8] 445 | 446 | Internet Draft TLS for NNTP October 2003 447 | 448 | 449 | TLS negotiation. If the negotiation results in no privacy, or if 450 | it results in privacy using algorithms or key lengths that are 451 | deemed not strong enough, or if the authentication is not good 452 | enough for either party, the client may choose to end the NNTP 453 | session with an immediate QUIT command, or the server may choose 454 | not to accept any more NNTP commands. 455 | 456 | The client and server should also be aware that the TLS protocol 457 | permits privacy and security capabilities to be renegotiated mid- 458 | connection (see section 7.4.1 of [TLS]). For example, one of the 459 | parties may desire minimal encryption after any authentication 460 | steps have been performed. This underscores the fact that security 461 | is not present simply because TLS has been negotiated; the nature 462 | of the established security layer must be considered. 463 | 464 | A man-in-the-middle attack can be launched by deleting the 382 465 | response from the server. This would cause the client not to try to 466 | start a TLS session. Another man-in-the-middle attack is to allow 467 | the server to announce its STARTTLS capability, but to alter the 468 | client's request to start TLS and the server's response. An NNTP 469 | client can partially protect against these attacks by recording the 470 | fact that a particular NNTP server offers TLS during one session 471 | and generating an alarm if it does not appear in the LIST 472 | EXTENSIONS response for a later session (of course, the STARTTLS 473 | extension would not be listed after a security layer is in place). 474 | 475 | If the TLS negotiation fails or if the client receives a 483 476 | response, the client has to decide what to do next. The client has 477 | to choose among three main options: to go ahead with the rest of 478 | the NNTP session, to retry TLS at a later time, or to give up and 479 | postpone newsreading activity. If a failure or error occurs, the 480 | client can assume that the server may be able to negotiate TLS in 481 | the future, and should try to negotiate TLS in a later session. 482 | However, if the client and server were only using TLS for 483 | authentication and no previous 480 response was received, the 484 | client may want to proceed with the NNTP session, in case some of 485 | the operations the client wanted to perform are accepted by the 486 | server even if the client is unauthenticated. 487 | 488 | Before the TLS handshake has begun, any protocol interactions are 489 | performed in the clear and may be modified by an active attacker. 490 | For this reason, clients and servers MUST discard any sensitive 491 | knowledge obtained prior to the start of the TLS handshake upon 492 | completion of the TLS handshake. 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | Vinocur & Newman Expires April 2004 [Page 9] 501 | 502 | Internet Draft TLS for NNTP October 2003 503 | 504 | 505 | 6. Acknowledgements 506 | 507 | A significant amount of the STARTTLS text was lifted from RFC 3207 508 | by Paul Hoffman. 509 | 510 | Special acknowledgement goes also to the people who commented 511 | privately on intermediate revisions of this document, as well as 512 | the members of the IETF NNTP Working Group for continual insight in 513 | discussion. 514 | 515 | 7. Normative References 516 | 517 | [ABNF] Crocker, D., Overell, P., "Augmented BNF for Syntax 518 | Specifications: ABNF", RFC 2234, November 1997. 519 | 520 | [AUTH] Haller, N., Atkinson, R., "On Internet Authentication", RFC 1704, 521 | October 1994. 522 | 523 | [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate 524 | Requirement Levels", RFC 2119, March 1997. 525 | 526 | [NNTP] Feather, C., "Network News Transport Protocol" 527 | (draft-ietf-nntpext-base-20.txt). 528 | 529 | [SMTP] Klensin, J., "Simple Mail Transport Protocol", RFC 2821, April 530 | 2001. 531 | 532 | [TLS] Dierks, T., Allen, C., "The TLS Protocol Version 1.0", RFC 2246, 533 | January 1999. 534 | 535 | [TLS-EXT] Blake-Wilson, S., Nystrom, M., Hopwood, D., Mikkelsen, J., 536 | Wright, T., "Transport Layer Security (TLS) Extensions", RFC 3546, June 537 | 2003. 538 | 539 | [TLS-IMAPPOP] Newman, C., "Using TLS with IMAP, POP3 and ACAP", RFC 540 | 2595, June 1999. 541 | 542 | 8. Informative References 543 | 544 | [HTTP] Fielding, R., Gettys, J., Mogul, J., Frystyk, H., Masinter, 545 | L., Leach, P., Berners-Lee, T., "Hypertext Transfer Protocol -- 546 | HTTP/1.1", RFC 2616, June 1999. 547 | 548 | [NNTP-AUTH] Vinocur, J., Newman, C., "NNTP Extension for 549 | Authentication", Work in Progress. 550 | 551 | 552 | 553 | 554 | 555 | 556 | Vinocur & Newman Expires April 2004 [Page 10] 557 | 558 | Internet Draft TLS for NNTP October 2003 559 | 560 | 561 | 9. Authors' Addresses 562 | 563 | Jeffrey M. Vinocur 564 | Department of Computer Science 565 | Upson Hall 566 | Cornell University 567 | Ithaca, NY 14853 568 | 569 | EMail: vinocur@cs.cornell.edu 570 | 571 | 572 | Chris Newman 573 | Sun Microsystems 574 | 1050 Lakes Drive, Suite 250 575 | West Covina, CA 91790 576 | 577 | EMail: cnewman@iplanet.com 578 | 579 | Full Copyright Statement 580 | 581 | Copyright (C) The Internet Society (2002). All Rights Reserved. 582 | 583 | This document and translations of it may be copied and furnished to 584 | others, and derivative works that comment on or otherwise explain 585 | it or assist in its implementation may be prepared, copied, 586 | published and distributed, in whole or in part, without restriction 587 | of any kind, provided that the above copyright notice and this 588 | paragraph are included on all such copies and derivative works. 589 | However, this document itself may not be modified in any way, such 590 | as by removing the copyright notice or references to the Internet 591 | Society or other Internet organizations, except as needed for the 592 | purpose of developing Internet standards in which case the 593 | procedures for copyrights defined in the Internet Standards process 594 | must be followed, or as required to translate it into languages 595 | other than English. 596 | 597 | The limited permissions granted above are perpetual and will not be 598 | revoked by the Internet Society or its successors or assigns. 599 | 600 | This document and the information contained herein is provided on 601 | an "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET 602 | ENGINEERING TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR 603 | IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF 604 | THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED 605 | WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. 606 | 607 | 608 | 609 | 610 | 611 | 612 | Vinocur & Newman Expires April 2004 [Page 11] 613 | 614 | 615 | -------------------------------------------------------------------------------- /storage/phorum_mysql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: phorum_mysql.py,v 1.47 2004-08-01 01:51:48 jpm Exp $ 4 | import MySQLdb 5 | import time 6 | from mimify import mime_encode_header, mime_decode_header 7 | import re 8 | import settings 9 | import mime 10 | import strutil 11 | import smtplib 12 | import md5 13 | 14 | # patch by Andreas Wegmann to fix the handling of unusual encodings of messages 15 | q_quote_multiline = re.compile("=\?(.*?)\?[qQ]\?(.*?)\?=.*?=\?\\1\?[qQ]\?(.*?)\?=", re.M | re.S) 16 | # we don't need to compile the regexps everytime.. 17 | doubleline_regexp = re.compile("^\.\.", re.M) 18 | singleline_regexp = re.compile("^\.", re.M) 19 | from_regexp = re.compile("^From:(.*)<(.*)>", re.M) 20 | subject_regexp = re.compile("^Subject:(.*)", re.M) 21 | references_regexp = re.compile("^References:(.*)<(.*)>", re.M) 22 | lines_regexp = re.compile("^Lines:(.*)", re.M) 23 | # phorum configuration files related regexps 24 | moderator_regexp = re.compile("(.*)PHORUM\['ForumModeration'\](.*)='(.*)';", re.M) 25 | url_regexp = re.compile("(.*)PHORUM\['forum_url'\](.*)='(.*)';", re.M) 26 | admin_regexp = re.compile("(.*)PHORUM\['admin_url'\](.*)='(.*)';", re.M) 27 | server_regexp = re.compile("(.*)PHORUM\['forum_url'\](.*)='(.*)http://(.*)/(.*)';", re.M) 28 | mail_code_regexp = re.compile("(.*)PHORUM\['PhorumMailCode'\](.*)=(.*)'(.*)';", re.M) 29 | 30 | class Papercut_Storage: 31 | """ 32 | Storage Backend interface for the Phorum web message board software (http://phorum.org) 33 | 34 | This is the interface for Phorum running on a MySQL database. For more information 35 | on the structure of the 'storage' package, please refer to the __init__.py 36 | available on the 'storage' sub-directory. 37 | """ 38 | 39 | def __init__(self): 40 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 41 | self.cursor = self.conn.cursor() 42 | 43 | def get_message_body(self, headers): 44 | """Parses and returns the most appropriate message body possible. 45 | 46 | The function tries to extract the plaintext version of a MIME based 47 | message, and if it is not available then it returns the html version. 48 | """ 49 | return mime.get_text_message(headers) 50 | 51 | def quote_string(self, text): 52 | """Quotes strings the MySQL way.""" 53 | return text.replace("'", "\\'") 54 | 55 | def group_exists(self, group_name): 56 | stmt = """ 57 | SELECT 58 | COUNT(*) AS total 59 | FROM 60 | forums 61 | WHERE 62 | LOWER(nntp_group_name)=LOWER('%s')""" % (group_name) 63 | self.cursor.execute(stmt) 64 | return self.cursor.fetchone()[0] 65 | 66 | def article_exists(self, group_name, style, range): 67 | table_name = self.get_table_name(group_name) 68 | stmt = """ 69 | SELECT 70 | COUNT(*) AS total 71 | FROM 72 | %s 73 | WHERE 74 | approved='Y'""" % (table_name) 75 | if style == 'range': 76 | stmt = "%s AND id > %s" % (stmt, range[0]) 77 | if len(range) == 2: 78 | stmt = "%s AND id < %s" % (stmt, range[1]) 79 | else: 80 | stmt = "%s AND id = %s" % (stmt, range[0]) 81 | self.cursor.execute(stmt) 82 | return self.cursor.fetchone()[0] 83 | 84 | def get_first_article(self, group_name): 85 | table_name = self.get_table_name(group_name) 86 | stmt = """ 87 | SELECT 88 | IF(MIN(id) IS NULL, 0, MIN(id)) AS first_article 89 | FROM 90 | %s 91 | WHERE 92 | approved='Y'""" % (table_name) 93 | num_rows = self.cursor.execute(stmt) 94 | return self.cursor.fetchone()[0] 95 | 96 | def get_group_stats(self, group_name): 97 | total, max, min = self.get_table_stats(self.get_table_name(group_name)) 98 | return (total, min, max, group_name) 99 | 100 | def get_table_stats(self, table_name): 101 | stmt = """ 102 | SELECT 103 | COUNT(id) AS total, 104 | IF(MAX(id) IS NULL, 0, MAX(id)) AS maximum, 105 | IF(MIN(id) IS NULL, 0, MIN(id)) AS minimum 106 | FROM 107 | %s 108 | WHERE 109 | approved='Y'""" % (table_name) 110 | num_rows = self.cursor.execute(stmt) 111 | return self.cursor.fetchone() 112 | 113 | def get_table_name(self, group_name): 114 | stmt = """ 115 | SELECT 116 | table_name 117 | FROM 118 | forums 119 | WHERE 120 | nntp_group_name='%s'""" % (group_name.replace('*', '%')) 121 | self.cursor.execute(stmt) 122 | return self.cursor.fetchone()[0] 123 | 124 | def get_message_id(self, msg_num, group): 125 | return '<%s@%s>' % (msg_num, group) 126 | 127 | def get_notification_emails(self, forum_id): 128 | # open the configuration file 129 | fp = open("%s%s.php" % (settings.phorum_settings_path, forum_id), "r") 130 | content = fp.read() 131 | fp.close() 132 | # get the value of the configuration variable 133 | recipients = [] 134 | mod_code = moderator_regexp.search(content, 0).groups() 135 | if mod_code[2] == 'r' or mod_code[2] == 'a': 136 | # get the moderator emails from the forum_auth table 137 | stmt = """ 138 | SELECT 139 | email 140 | FROM 141 | forums_auth, 142 | forums_moderators 143 | WHERE 144 | user_id=id AND 145 | forum_id=%s""" % (forum_id) 146 | self.cursor.execute(stmt) 147 | result = list(self.cursor.fetchall()) 148 | for row in result: 149 | recipients.append(row[0]) 150 | return recipients 151 | 152 | def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_author, msg_email, msg_subject, msg_body): 153 | msg_tpl = """From: Phorum <%(recipient)s> 154 | To: %(recipient)s 155 | Subject: Moderate for %(forum_name)s at %(phorum_server_hostname)s Message: %(msg_id)s. 156 | 157 | Subject: %(msg_subject)s 158 | Author: %(msg_author)s 159 | Message: %(phorum_url)s/read.php?f=%(forum_id)s&i=%(msg_id)s&t=%(thread_id)s&admview=1 160 | 161 | %(msg_body)s 162 | 163 | To delete this message use this URL: 164 | %(phorum_admin_url)s?page=easyadmin&action=del&type=quick&id=%(msg_id)s&num=1&thread=%(thread_id)s 165 | 166 | To edit this message use this URL: 167 | %(phorum_admin_url)s?page=edit&srcpage=easyadmin&id=%(msg_id)s&num=1&mythread=%(thread_id)s 168 | 169 | """ 170 | # get the forum_id for this group_name 171 | stmt = """ 172 | SELECT 173 | id, 174 | name 175 | FROM 176 | forums 177 | WHERE 178 | nntp_group_name='%s'""" % (group_name) 179 | self.cursor.execute(stmt) 180 | forum_id, forum_name = self.cursor.fetchone() 181 | # open the main configuration file 182 | fp = open("%sforums.php" % (settings.phorum_settings_path), "r") 183 | content = fp.read() 184 | fp.close() 185 | # regexps to get the content from the phorum configuration files 186 | phorum_url = url_regexp.search(content, 0).groups()[2] 187 | phorum_admin_url = admin_regexp.search(content, 0).groups()[2] 188 | phorum_server_hostname = server_regexp.search(content, 0).groups()[3] 189 | # connect to the SMTP server 190 | smtp = smtplib.SMTP('localhost') 191 | emails = self.get_notification_emails(forum_id) 192 | for recipient in emails: 193 | current_msg = msg_tpl % vars() 194 | smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg) 195 | 196 | # XXX: Coding blind here. I really don't know much about how Phorum works with 197 | # XXX: sending forum postings as emails, but it's here. Let's call this a 198 | # XXX: temporary implementation. Should work fine, I guess. 199 | phorum_mail_code = mail_code_regexp.search(content, 0).groups()[3] 200 | notification_mail_tpl = """Message-ID: <%(random_msgid)s@%(phorum_server_hostname)s> 201 | From: %(msg_author)s %(msg_email)s 202 | Subject: %(msg_subject)s 203 | To: %(forum_name)s <%(email_list)s> 204 | Return-Path: <%(email_return)s> 205 | Reply-To: %(email_return)s 206 | X-Phorum-%(phorum_mail_code)s-Version: Phorum %(phorum_version)s 207 | X-Phorum-%(phorum_mail_code)s-Forum: %(forum_name)s 208 | X-Phorum-%(phorum_mail_code)s-Thread: %(thread_id)s 209 | X-Phorum-%(phorum_mail_code)s-Parent: %(parent_id)s 210 | 211 | This message was sent from: %(forum_name)s. 212 | <%(phorum_url)s/read.php?f=%(forum_id)s&i=%(msg_id)s&t=%(thread_id)s> 213 | ---------------------------------------------------------------- 214 | 215 | %(msg_body)s 216 | 217 | ---------------------------------------------------------------- 218 | Sent using Papercut version %(__VERSION__)s 219 | """ 220 | stmt = """ 221 | SELECT 222 | email_list, 223 | email_return 224 | FROM 225 | forums 226 | WHERE 227 | LENGTH(email_list) > 0 AND 228 | id=%s""" % (forum_id) 229 | num_rows = self.cursor.execute(stmt) 230 | if num_rows == 1: 231 | email_list, email_return = self.cursor.fetchone() 232 | msg_body = strutil.wrap(msg_body) 233 | if len(msg_email) > 0: 234 | msg_email = '<%s>' % msg_email 235 | else: 236 | msg_email = '' 237 | random_msgid = md5.new(str(time.clock())).hexdigest() 238 | # this is pretty ugly, right ? 239 | from papercut import __VERSION__ 240 | phorum_version = settings.phorum_version 241 | current_msg = notification_mail_tpl % vars() 242 | smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg) 243 | smtp.quit() 244 | 245 | def get_NEWGROUPS(self, ts, group='%'): 246 | # since phorum doesn't record when each forum was created, we have no way of knowing this... 247 | return None 248 | 249 | def get_NEWNEWS(self, ts, group='*'): 250 | stmt = """ 251 | SELECT 252 | nntp_group_name, 253 | table_name 254 | FROM 255 | forums 256 | WHERE 257 | nntp_group_name='%s' 258 | ORDER BY 259 | nntp_group_name ASC""" % (group_name.replace('*', '%')) 260 | self.cursor.execute(stmt) 261 | result = list(self.cursor.fetchall()) 262 | articles = [] 263 | for group, table in result: 264 | stmt = """ 265 | SELECT 266 | id 267 | FROM 268 | %s 269 | WHERE 270 | approved='Y' AND 271 | UNIX_TIMESTAMP(datestamp) >= %s""" % (table, ts) 272 | num_rows = self.cursor.execute(stmt) 273 | if num_rows == 0: 274 | continue 275 | ids = list(self.cursor.fetchall()) 276 | for id in ids: 277 | articles.append("<%s@%s>" % (id, group)) 278 | if len(articles) == 0: 279 | return '' 280 | else: 281 | return "\r\n".join(articles) 282 | 283 | def get_GROUP(self, group_name): 284 | table_name = self.get_table_name(group_name) 285 | result = self.get_table_stats(table_name) 286 | return (result[0], result[2], result[1]) 287 | 288 | def get_LIST(self, username=""): 289 | stmt = """ 290 | SELECT 291 | nntp_group_name, 292 | table_name 293 | FROM 294 | forums 295 | WHERE 296 | LENGTH(nntp_group_name) > 0 297 | ORDER BY 298 | nntp_group_name ASC""" 299 | self.cursor.execute(stmt) 300 | result = list(self.cursor.fetchall()) 301 | if len(result) == 0: 302 | return "" 303 | else: 304 | lists = [] 305 | for group_name, table in result: 306 | total, maximum, minimum = self.get_table_stats(table) 307 | if settings.server_type == 'read-only': 308 | lists.append("%s %s %s n" % (group_name, maximum, minimum)) 309 | else: 310 | lists.append("%s %s %s y" % (group_name, maximum, minimum)) 311 | return "\r\n".join(lists) 312 | 313 | def get_STAT(self, group_name, id): 314 | table_name = self.get_table_name(group_name) 315 | stmt = """ 316 | SELECT 317 | id 318 | FROM 319 | %s 320 | WHERE 321 | approved='Y' AND 322 | id=%s""" % (table_name, id) 323 | return self.cursor.execute(stmt) 324 | 325 | def get_ARTICLE(self, group_name, id): 326 | table_name = self.get_table_name(group_name) 327 | stmt = """ 328 | SELECT 329 | A.id, 330 | author, 331 | email, 332 | subject, 333 | UNIX_TIMESTAMP(datestamp) AS datestamp, 334 | body, 335 | parent 336 | FROM 337 | %s A, 338 | %s_bodies B 339 | WHERE 340 | A.approved='Y' AND 341 | A.id=B.id AND 342 | A.id=%s""" % (table_name, table_name, id) 343 | num_rows = self.cursor.execute(stmt) 344 | if num_rows == 0: 345 | return None 346 | result = list(self.cursor.fetchone()) 347 | if len(result[2]) == 0: 348 | author = result[1] 349 | else: 350 | author = "%s <%s>" % (result[1], result[2]) 351 | formatted_time = strutil.get_formatted_time(time.localtime(result[4])) 352 | headers = [] 353 | headers.append("Path: %s" % (settings.nntp_hostname)) 354 | headers.append("From: %s" % (author)) 355 | headers.append("Newsgroups: %s" % (group_name)) 356 | headers.append("Date: %s" % (formatted_time)) 357 | headers.append("Subject: %s" % (result[3])) 358 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 359 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 360 | if result[6] != 0: 361 | headers.append("References: <%s@%s>" % (result[6], group_name)) 362 | return ("\r\n".join(headers), strutil.format_body(result[5])) 363 | 364 | def get_LAST(self, group_name, current_id): 365 | table_name = self.get_table_name(group_name) 366 | stmt = """ 367 | SELECT 368 | id 369 | FROM 370 | %s 371 | WHERE 372 | approved='Y' AND 373 | id < %s 374 | ORDER BY 375 | id DESC 376 | LIMIT 0, 1""" % (table_name, current_id) 377 | num_rows = self.cursor.execute(stmt) 378 | if num_rows == 0: 379 | return None 380 | return self.cursor.fetchone()[0] 381 | 382 | def get_NEXT(self, group_name, current_id): 383 | table_name = self.get_table_name(group_name) 384 | stmt = """ 385 | SELECT 386 | id 387 | FROM 388 | %s 389 | WHERE 390 | approved='Y' AND 391 | id > %s 392 | ORDER BY 393 | id ASC 394 | LIMIT 0, 1""" % (table_name, current_id) 395 | num_rows = self.cursor.execute(stmt) 396 | if num_rows == 0: 397 | return None 398 | return self.cursor.fetchone()[0] 399 | 400 | def get_HEAD(self, group_name, id): 401 | table_name = self.get_table_name(group_name) 402 | stmt = """ 403 | SELECT 404 | id, 405 | author, 406 | email, 407 | subject, 408 | UNIX_TIMESTAMP(datestamp) AS datestamp, 409 | parent 410 | FROM 411 | %s 412 | WHERE 413 | approved='Y' AND 414 | id=%s""" % (table_name, id) 415 | num_rows = self.cursor.execute(stmt) 416 | if num_rows == 0: 417 | return None 418 | result = list(self.cursor.fetchone()) 419 | if len(result[2]) == 0: 420 | author = result[1] 421 | else: 422 | author = "%s <%s>" % (result[1], result[2]) 423 | formatted_time = strutil.get_formatted_time(time.localtime(result[4])) 424 | headers = [] 425 | headers.append("Path: %s" % (settings.nntp_hostname)) 426 | headers.append("From: %s" % (author)) 427 | headers.append("Newsgroups: %s" % (group_name)) 428 | headers.append("Date: %s" % (formatted_time)) 429 | headers.append("Subject: %s" % (result[3])) 430 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 431 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 432 | if result[5] != 0: 433 | headers.append("References: <%s@%s>" % (result[5], group_name)) 434 | return "\r\n".join(headers) 435 | 436 | def get_BODY(self, group_name, id): 437 | table_name = self.get_table_name(group_name) 438 | stmt = """ 439 | SELECT 440 | B.body 441 | FROM 442 | %s A, 443 | %s_bodies B 444 | WHERE 445 | A.id=B.id AND 446 | A.approved='Y' AND 447 | B.id=%s""" % (table_name, table_name, id) 448 | num_rows = self.cursor.execute(stmt) 449 | if num_rows == 0: 450 | return None 451 | else: 452 | return strutil.format_body(self.cursor.fetchone()[0]) 453 | 454 | def get_XOVER(self, group_name, start_id, end_id='ggg'): 455 | table_name = self.get_table_name(group_name) 456 | stmt = """ 457 | SELECT 458 | A.id, 459 | parent, 460 | author, 461 | email, 462 | subject, 463 | UNIX_TIMESTAMP(datestamp) AS datestamp, 464 | B.body 465 | FROM 466 | %s A, 467 | %s_bodies B 468 | WHERE 469 | A.approved='Y' AND 470 | A.id=B.id AND 471 | A.id >= %s""" % (table_name, table_name, start_id) 472 | if end_id != 'ggg': 473 | stmt = "%s AND A.id <= %s" % (stmt, end_id) 474 | self.cursor.execute(stmt) 475 | result = list(self.cursor.fetchall()) 476 | overviews = [] 477 | for row in result: 478 | if row[3] == '': 479 | author = row[2] 480 | else: 481 | author = "%s <%s>" % (row[2], row[3]) 482 | formatted_time = strutil.get_formatted_time(time.localtime(row[5])) 483 | message_id = "<%s@%s>" % (row[0], group_name) 484 | line_count = len(row[6].split('\n')) 485 | xref = 'Xref: %s %s:%s' % (settings.nntp_hostname, group_name, row[0]) 486 | if row[1] != 0: 487 | reference = "<%s@%s>" % (row[1], group_name) 488 | else: 489 | reference = "" 490 | # message_number subject author date message_id reference bytes lines xref 491 | overviews.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (row[0], row[4], author, formatted_time, message_id, reference, len(strutil.format_body(row[6])), line_count, xref)) 492 | return "\r\n".join(overviews) 493 | 494 | def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): 495 | # XXX: need to actually check for the header values being passed as 496 | # XXX: not all header names map to column names on the tables 497 | table_name = self.get_table_name(group_name) 498 | stmt = """ 499 | SELECT 500 | A.id, 501 | parent, 502 | author, 503 | email, 504 | subject, 505 | UNIX_TIMESTAMP(datestamp) AS datestamp, 506 | B.body 507 | FROM 508 | %s A, 509 | %s_bodies B 510 | WHERE 511 | A.approved='Y' AND 512 | %s REGEXP '%s' AND 513 | A.id = B.id AND 514 | A.id >= %s""" % (table_name, table_name, header, strutil.format_wildcards(pattern), start_id) 515 | if end_id != 'ggg': 516 | stmt = "%s AND A.id <= %s" % (stmt, end_id) 517 | num_rows = self.cursor.execute(stmt) 518 | if num_rows == 0: 519 | return None 520 | result = list(self.cursor.fetchall()) 521 | hdrs = [] 522 | for row in result: 523 | if header.upper() == 'SUBJECT': 524 | hdrs.append('%s %s' % (row[0], row[4])) 525 | elif header.upper() == 'FROM': 526 | # XXX: totally broken with empty values for the email address 527 | hdrs.append('%s %s <%s>' % (row[0], row[2], row[3])) 528 | elif header.upper() == 'DATE': 529 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) 530 | elif header.upper() == 'MESSAGE-ID': 531 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 532 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 533 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 534 | elif header.upper() == 'BYTES': 535 | hdrs.append('%s %s' % (row[0], len(row[6]))) 536 | elif header.upper() == 'LINES': 537 | hdrs.append('%s %s' % (row[0], len(row[6].split('\n')))) 538 | elif header.upper() == 'XREF': 539 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 540 | if len(hdrs) == 0: 541 | return "" 542 | else: 543 | return "\r\n".join(hdrs) 544 | 545 | def get_LISTGROUP(self, group_name): 546 | table_name = self.get_table_name(group_name) 547 | stmt = """ 548 | SELECT 549 | id 550 | FROM 551 | %s 552 | WHERE 553 | approved='Y' 554 | ORDER BY 555 | id ASC""" % (table_name) 556 | self.cursor.execute(stmt) 557 | result = list(self.cursor.fetchall()) 558 | return "\r\n".join(["%s" % k for k in result]) 559 | 560 | def get_XGTITLE(self, pattern=None): 561 | stmt = """ 562 | SELECT 563 | nntp_group_name, 564 | description 565 | FROM 566 | forums 567 | WHERE 568 | LENGTH(nntp_group_name) > 0""" 569 | if pattern != None: 570 | stmt = stmt + """ AND 571 | nntp_group_name REGEXP '%s'""" % (strutil.format_wildcards(pattern)) 572 | stmt = stmt + """ 573 | ORDER BY 574 | nntp_group_name ASC""" 575 | self.cursor.execute(stmt) 576 | result = list(self.cursor.fetchall()) 577 | return "\r\n".join(["%s %s" % (k, v) for k, v in result]) 578 | 579 | def get_XHDR(self, group_name, header, style, range): 580 | table_name = self.get_table_name(group_name) 581 | stmt = """ 582 | SELECT 583 | A.id, 584 | parent, 585 | author, 586 | email, 587 | subject, 588 | UNIX_TIMESTAMP(datestamp) AS datestamp, 589 | B.body 590 | FROM 591 | %s A, 592 | %s_bodies B 593 | WHERE 594 | A.approved='Y' AND 595 | A.id = B.id AND """ % (table_name, table_name) 596 | if style == 'range': 597 | stmt = '%s A.id >= %s' % (stmt, range[0]) 598 | if len(range) == 2: 599 | stmt = '%s AND A.id <= %s' % (stmt, range[1]) 600 | else: 601 | stmt = '%s A.id = %s' % (stmt, range[0]) 602 | if self.cursor.execute(stmt) == 0: 603 | return None 604 | result = self.cursor.fetchall() 605 | hdrs = [] 606 | for row in result: 607 | if header.upper() == 'SUBJECT': 608 | hdrs.append('%s %s' % (row[0], row[4])) 609 | elif header.upper() == 'FROM': 610 | hdrs.append('%s %s <%s>' % (row[0], row[2], row[3])) 611 | elif header.upper() == 'DATE': 612 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) 613 | elif header.upper() == 'MESSAGE-ID': 614 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 615 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 616 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 617 | elif header.upper() == 'BYTES': 618 | hdrs.append('%s %s' % (row[0], len(row[6]))) 619 | elif header.upper() == 'LINES': 620 | hdrs.append('%s %s' % (row[0], len(row[6].split('\n')))) 621 | elif header.upper() == 'XREF': 622 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 623 | if len(hdrs) == 0: 624 | return "" 625 | else: 626 | return "\r\n".join(hdrs) 627 | 628 | def do_POST(self, group_name, lines, ip_address, username=''): 629 | table_name = self.get_table_name(group_name) 630 | body = self.get_message_body(lines) 631 | author, email = from_regexp.search(lines, 0).groups() 632 | subject = subject_regexp.search(lines, 0).groups()[0].strip() 633 | # patch by Andreas Wegmann to fix the handling of unusual encodings of messages 634 | lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines)) 635 | if lines.find('References') != -1: 636 | # get the 'modifystamp' value from the parent (if any) 637 | references = references_regexp.search(lines, 0).groups() 638 | parent_id, void = references[-1].strip().split('@') 639 | stmt = """ 640 | SELECT 641 | IF(MAX(id) IS NULL, 1, MAX(id)+1) AS next_id 642 | FROM 643 | %s""" % (table_name) 644 | num_rows = self.cursor.execute(stmt) 645 | if num_rows == 0: 646 | new_id = 1 647 | else: 648 | new_id = self.cursor.fetchone()[0] 649 | stmt = """ 650 | SELECT 651 | id, 652 | thread, 653 | modifystamp 654 | FROM 655 | %s 656 | WHERE 657 | approved='Y' AND 658 | id=%s 659 | GROUP BY 660 | id""" % (table_name, parent_id) 661 | num_rows = self.cursor.execute(stmt) 662 | if num_rows == 0: 663 | return None 664 | parent_id, thread_id, modifystamp = self.cursor.fetchone() 665 | else: 666 | stmt = """ 667 | SELECT 668 | IF(MAX(id) IS NULL, 1, MAX(id)+1) AS next_id, 669 | UNIX_TIMESTAMP() 670 | FROM 671 | %s""" % (table_name) 672 | self.cursor.execute(stmt) 673 | new_id, modifystamp = self.cursor.fetchone() 674 | parent_id = 0 675 | thread_id = new_id 676 | stmt = """ 677 | INSERT INTO 678 | %s 679 | ( 680 | id, 681 | datestamp, 682 | thread, 683 | parent, 684 | author, 685 | subject, 686 | email, 687 | host, 688 | email_reply, 689 | approved, 690 | msgid, 691 | modifystamp, 692 | userid 693 | ) VALUES ( 694 | %s, 695 | NOW(), 696 | %s, 697 | %s, 698 | '%s', 699 | '%s', 700 | '%s', 701 | '%s', 702 | 'N', 703 | 'Y', 704 | '', 705 | %s, 706 | 0 707 | ) 708 | """ % (table_name, new_id, thread_id, parent_id, self.quote_string(author.strip()), self.quote_string(subject), self.quote_string(email), ip_address, modifystamp) 709 | if not self.cursor.execute(stmt): 710 | return None 711 | else: 712 | # insert into the '*_bodies' table 713 | stmt = """ 714 | INSERT INTO 715 | %s_bodies 716 | ( 717 | id, 718 | body, 719 | thread 720 | ) VALUES ( 721 | %s, 722 | '%s', 723 | %s 724 | )""" % (table_name, new_id, self.quote_string(body), thread_id) 725 | if not self.cursor.execute(stmt): 726 | # delete from 'table_name' before returning.. 727 | stmt = """ 728 | DELETE FROM 729 | %s 730 | WHERE 731 | id=%s""" % (table_name, new_id) 732 | self.cursor.execute(stmt) 733 | return None 734 | else: 735 | # alert forum moderators 736 | self.send_notifications(group_name, new_id, thread_id, parent_id, author.strip(), email, subject, body) 737 | return 1 738 | -------------------------------------------------------------------------------- /storage/phorum_pgsql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002 Joao Prado Maia. See the LICENSE file for more information. 3 | # $Id: phorum_pgsql.py,v 1.13 2004-08-01 01:51:48 jpm Exp $ 4 | from pyPgSQL import PgSQL 5 | import time 6 | from mimify import mime_encode_header, mime_decode_header 7 | import re 8 | import settings 9 | import mime 10 | import strutil 11 | import smtplib 12 | import md5 13 | 14 | # patch by Andreas Wegmann to fix the handling of unusual encodings of messages 15 | q_quote_multiline = re.compile("=\?(.*?)\?[qQ]\?(.*?)\?=.*?=\?\\1\?[qQ]\?(.*?)\?=", re.M | re.S) 16 | 17 | # we don't need to compile the regexps everytime.. 18 | doubleline_regexp = re.compile("^\.\.", re.M) 19 | singleline_regexp = re.compile("^\.", re.M) 20 | from_regexp = re.compile("^From:(.*)<(.*)>", re.M) 21 | subject_regexp = re.compile("^Subject:(.*)", re.M) 22 | references_regexp = re.compile("^References:(.*)<(.*)>", re.M) 23 | lines_regexp = re.compile("^Lines:(.*)", re.M) 24 | # phorum configuration files related regexps 25 | moderator_regexp = re.compile("(.*)PHORUM\['ForumModeration'\](.*)='(.*)';", re.M) 26 | url_regexp = re.compile("(.*)PHORUM\['forum_url'\](.*)='(.*)';", re.M) 27 | admin_regexp = re.compile("(.*)PHORUM\['admin_url'\](.*)='(.*)';", re.M) 28 | server_regexp = re.compile("(.*)PHORUM\['forum_url'\](.*)='(.*)http://(.*)/(.*)';", re.M) 29 | mail_code_regexp = re.compile("(.*)PHORUM\['PhorumMailCode'\](.*)=(.*)'(.*)';", re.M) 30 | 31 | class Papercut_Storage: 32 | """ 33 | Storage Backend interface for the Phorum web message board software (http://phorum.org) 34 | 35 | This is the interface for Phorum running on a PostgreSQL database. For more information 36 | on the structure of the 'storage' package, please refer to the __init__.py 37 | available on the 'storage' sub-directory. 38 | """ 39 | 40 | def __init__(self): 41 | self.conn = PgSQL.connect(host=settings.dbhost, database=settings.dbname, user=settings.dbuser, password=settings.dbpass) 42 | self.cursor = self.conn.cursor() 43 | 44 | def get_message_body(self, headers): 45 | """Parses and returns the most appropriate message body possible. 46 | 47 | The function tries to extract the plaintext version of a MIME based 48 | message, and if it is not available then it returns the html version. 49 | """ 50 | return mime.get_text_message(headers) 51 | 52 | def group_exists(self, group_name): 53 | stmt = """ 54 | SELECT 55 | COUNT(*) AS total 56 | FROM 57 | forums 58 | WHERE 59 | LOWER(nntp_group_name)=LOWER('%s')""" % (group_name) 60 | self.cursor.execute(stmt) 61 | return self.cursor.fetchone()[0] 62 | 63 | def article_exists(self, group_name, style, range): 64 | table_name = self.get_table_name(group_name) 65 | stmt = """ 66 | SELECT 67 | COUNT(*) AS total 68 | FROM 69 | %s 70 | WHERE 71 | approved='Y'""" % (table_name) 72 | if style == 'range': 73 | stmt = "%s AND id > %s" % (stmt, range[0]) 74 | if len(range) == 2: 75 | stmt = "%s AND id < %s" % (stmt, range[1]) 76 | else: 77 | stmt = "%s AND id = %s" % (stmt, range[0]) 78 | self.cursor.execute(stmt) 79 | return self.cursor.fetchone()[0] 80 | 81 | def get_first_article(self, group_name): 82 | table_name = self.get_table_name(group_name) 83 | stmt = """ 84 | SELECT 85 | MIN(id) AS first_article 86 | FROM 87 | %s 88 | WHERE 89 | approved='Y'""" % (table_name) 90 | self.cursor.execute(stmt) 91 | minimum = self.cursor.fetchone()[0] 92 | if minimum is None: 93 | return 0 94 | else: 95 | return minimum 96 | 97 | def get_group_stats(self, group_name): 98 | total, max, min = self.get_table_stats(self.get_table_name(group_name)) 99 | return (total, min, max, group_name) 100 | 101 | def get_table_stats(self, table_name): 102 | stmt = """ 103 | SELECT 104 | COUNT(id) AS total, 105 | MAX(id) AS maximum, 106 | MIN(id) AS minimum 107 | FROM 108 | %s 109 | WHERE 110 | approved='Y'""" % (table_name) 111 | self.cursor.execute(stmt) 112 | total, maximum, minimum = self.cursor.fetchone() 113 | if maximum is None: 114 | maximum = 0 115 | if minimum is None: 116 | minimum = 0 117 | return (total, maximum, minimum) 118 | 119 | def get_table_name(self, group_name): 120 | stmt = """ 121 | SELECT 122 | table_name 123 | FROM 124 | forums 125 | WHERE 126 | nntp_group_name LIKE '%s'""" % (group_name.replace('*', '%')) 127 | self.cursor.execute(stmt) 128 | return self.cursor.fetchone()[0] 129 | 130 | def get_message_id(self, msg_num, group): 131 | return '<%s@%s>' % (msg_num, group) 132 | 133 | def get_notification_emails(self, forum_id): 134 | # open the configuration file 135 | fp = open("%s%s.php" % (settings.phorum_settings_path, forum_id), "r") 136 | content = fp.read() 137 | fp.close() 138 | # get the value of the configuration variable 139 | recipients = [] 140 | mod_code = moderator_regexp.search(content, 0).groups() 141 | if mod_code[2] == 'r' or mod_code[2] == 'a': 142 | # get the moderator emails from the forum_auth table 143 | stmt = """ 144 | SELECT 145 | email 146 | FROM 147 | forums_auth, 148 | forums_moderators 149 | WHERE 150 | user_id=id AND 151 | forum_id=%s""" % (forum_id) 152 | self.cursor.execute(stmt) 153 | result = list(self.cursor.fetchall()) 154 | for row in result: 155 | recipients.append(row[0].strip()) 156 | return recipients 157 | 158 | def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_author, msg_email, msg_subject, msg_body): 159 | msg_tpl = """From: Phorum <%(recipient)s> 160 | To: %(recipient)s 161 | Subject: Moderate for %(forum_name)s at %(phorum_server_hostname)s Message: %(msg_id)s. 162 | 163 | Subject: %(msg_subject)s 164 | Author: %(msg_author)s 165 | Message: %(phorum_url)s/read.php?f=%(forum_id)s&i=%(msg_id)s&t=%(thread_id)s&admview=1 166 | 167 | %(msg_body)s 168 | 169 | To delete this message use this URL: 170 | %(phorum_admin_url)s?page=easyadmin&action=del&type=quick&id=%(msg_id)s&num=1&thread=%(thread_id)s 171 | 172 | To edit this message use this URL: 173 | %(phorum_admin_url)s?page=edit&srcpage=easyadmin&id=%(msg_id)s&num=1&mythread=%(thread_id)s 174 | 175 | """ 176 | # get the forum_id for this group_name 177 | stmt = """ 178 | SELECT 179 | id, 180 | name 181 | FROM 182 | forums 183 | WHERE 184 | nntp_group_name='%s'""" % (group_name) 185 | self.cursor.execute(stmt) 186 | forum_id, forum_name = self.cursor.fetchone() 187 | forum_name.strip() 188 | # open the main configuration file 189 | fp = open("%sforums.php" % (settings.phorum_settings_path), "r") 190 | content = fp.read() 191 | fp.close() 192 | # regexps to get the content from the phorum configuration files 193 | phorum_url = url_regexp.search(content, 0).groups()[2] 194 | phorum_admin_url = admin_regexp.search(content, 0).groups()[2] 195 | phorum_server_hostname = server_regexp.search(content, 0).groups()[3] 196 | # connect to the SMTP server 197 | smtp = smtplib.SMTP('localhost') 198 | emails = self.get_notification_emails(forum_id) 199 | for recipient in emails: 200 | current_msg = msg_tpl % vars() 201 | smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg) 202 | 203 | # XXX: Coding blind here. I really don't know much about how Phorum works with 204 | # XXX: sending forum postings as emails, but it's here. Let's call this a 205 | # XXX: temporary implementation. Should work fine, I guess. 206 | phorum_mail_code = mail_code_regexp.search(content, 0).groups()[3] 207 | notification_mail_tpl = """Message-ID: <%(random_msgid)s@%(phorum_server_hostname)s> 208 | From: %(msg_author)s %(msg_email)s 209 | Subject: %(msg_subject)s 210 | To: %(forum_name)s <%(email_list)s> 211 | Return-Path: <%(email_return)s> 212 | Reply-To: %(email_return)s 213 | X-Phorum-%(phorum_mail_code)s-Version: Phorum %(phorum_version)s 214 | X-Phorum-%(phorum_mail_code)s-Forum: %(forum_name)s 215 | X-Phorum-%(phorum_mail_code)s-Thread: %(thread_id)s 216 | X-Phorum-%(phorum_mail_code)s-Parent: %(parent_id)s 217 | 218 | This message was sent from: %(forum_name)s. 219 | <%(phorum_url)s/read.php?f=%(forum_id)s&i=%(msg_id)s&t=%(thread_id)s> 220 | ---------------------------------------------------------------- 221 | 222 | %(msg_body)s 223 | 224 | ---------------------------------------------------------------- 225 | Sent using Papercut version %(__VERSION__)s 226 | """ 227 | stmt = """ 228 | SELECT 229 | email_list, 230 | email_return 231 | FROM 232 | forums 233 | WHERE 234 | LENGTH(email_list) > 0 AND 235 | id=%s""" % (forum_id) 236 | num_rows = self.cursor.execute(stmt) 237 | if num_rows == 1: 238 | email_list, email_return = self.cursor.fetchone() 239 | msg_body = strutil.wrap(msg_body) 240 | if len(msg_email) > 0: 241 | msg_email = '<%s>' % msg_email 242 | else: 243 | msg_email = '' 244 | random_msgid = md5.new(str(time.clock())).hexdigest() 245 | # this is pretty ugly, right ? 246 | from papercut import __VERSION__ 247 | phorum_version = settings.phorum_version 248 | current_msg = notification_mail_tpl % vars() 249 | smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg) 250 | smtp.quit() 251 | 252 | def get_NEWGROUPS(self, ts, group='%'): 253 | # since phorum doesn't record when each forum was created, we have no way of knowing this... 254 | return None 255 | 256 | def get_NEWNEWS(self, ts, group='*'): 257 | stmt = """ 258 | SELECT 259 | nntp_group_name, 260 | table_name 261 | FROM 262 | forums 263 | WHERE 264 | nntp_group_name='%s' 265 | ORDER BY 266 | nntp_group_name ASC""" % (group_name.replace('*', '%')) 267 | self.cursor.execute(stmt) 268 | result = list(self.cursor.fetchall()) 269 | articles = [] 270 | for group, table in result: 271 | stmt = """ 272 | SELECT 273 | id 274 | FROM 275 | %s 276 | WHERE 277 | approved='Y' AND 278 | DATE_PART('epoch', datestamp) >= %s""" % (table, ts) 279 | num_rows = self.cursor.execute(stmt) 280 | if num_rows == 0: 281 | continue 282 | ids = list(self.cursor.fetchall()) 283 | for id in ids: 284 | articles.append("<%s@%s>" % (id, group)) 285 | if len(articles) == 0: 286 | return '' 287 | else: 288 | return "\r\n".join(articles) 289 | 290 | def get_GROUP(self, group_name): 291 | table_name = self.get_table_name(group_name) 292 | result = self.get_table_stats(table_name) 293 | return (result[0], result[2], result[1]) 294 | 295 | def get_LIST(self, username=""): 296 | stmt = """ 297 | SELECT 298 | nntp_group_name, 299 | table_name 300 | FROM 301 | forums 302 | WHERE 303 | LENGTH(nntp_group_name) > 0 304 | ORDER BY 305 | nntp_group_name ASC""" 306 | self.cursor.execute(stmt) 307 | result = list(self.cursor.fetchall()) 308 | if len(result) == 0: 309 | return "" 310 | else: 311 | lists = [] 312 | for group_name, table in result: 313 | total, maximum, minimum = self.get_table_stats(table) 314 | if settings.server_type == 'read-only': 315 | lists.append("%s %s %s n" % (group_name, maximum, minimum)) 316 | else: 317 | lists.append("%s %s %s y" % (group_name, maximum, minimum)) 318 | return "\r\n".join(lists) 319 | 320 | def get_STAT(self, group_name, id): 321 | table_name = self.get_table_name(group_name) 322 | stmt = """ 323 | SELECT 324 | id 325 | FROM 326 | %s 327 | WHERE 328 | approved='Y' AND 329 | id=%s""" % (table_name, id) 330 | return self.cursor.execute(stmt) 331 | 332 | def get_ARTICLE(self, group_name, id): 333 | table_name = self.get_table_name(group_name) 334 | stmt = """ 335 | SELECT 336 | A.id, 337 | author, 338 | email, 339 | subject, 340 | DATE_PART('epoch', datestamp) AS datestamp, 341 | body, 342 | parent 343 | FROM 344 | %s A, 345 | %s_bodies B 346 | WHERE 347 | A.approved='Y' AND 348 | A.id=B.id AND 349 | A.id=%s""" % (table_name, table_name, id) 350 | num_rows = self.cursor.execute(stmt) 351 | if num_rows == 0: 352 | return None 353 | result = list(self.cursor.fetchone()) 354 | if len(result[2]) == 0: 355 | author = result[1].strip() 356 | else: 357 | author = "%s <%s>" % (result[1].strip(), result[2].strip()) 358 | formatted_time = strutil.get_formatted_time(time.localtime(result[4])) 359 | headers = [] 360 | headers.append("Path: %s" % (settings.nntp_hostname)) 361 | headers.append("From: %s" % (author)) 362 | headers.append("Newsgroups: %s" % (group_name)) 363 | headers.append("Date: %s" % (formatted_time)) 364 | headers.append("Subject: %s" % (result[3].strip())) 365 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 366 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 367 | if result[6] != 0: 368 | headers.append("References: <%s@%s>" % (result[6], group_name)) 369 | return ("\r\n".join(headers), strutil.format_body(result[5])) 370 | 371 | def get_LAST(self, group_name, current_id): 372 | table_name = self.get_table_name(group_name) 373 | stmt = """ 374 | SELECT 375 | id 376 | FROM 377 | %s 378 | WHERE 379 | approved='Y' AND 380 | id < %s 381 | ORDER BY 382 | id DESC 383 | LIMIT 1, 0""" % (table_name, current_id) 384 | num_rows = self.cursor.execute(stmt) 385 | if num_rows == 0: 386 | return None 387 | return self.cursor.fetchone()[0] 388 | 389 | def get_NEXT(self, group_name, current_id): 390 | table_name = self.get_table_name(group_name) 391 | stmt = """ 392 | SELECT 393 | id 394 | FROM 395 | %s 396 | WHERE 397 | approved='Y' AND 398 | id > %s 399 | ORDER BY 400 | id ASC 401 | LIMIT 1, 0""" % (table_name, current_id) 402 | num_rows = self.cursor.execute(stmt) 403 | if num_rows == 0: 404 | return None 405 | return self.cursor.fetchone()[0] 406 | 407 | def get_HEAD(self, group_name, id): 408 | table_name = self.get_table_name(group_name) 409 | stmt = """ 410 | SELECT 411 | id, 412 | author, 413 | email, 414 | subject, 415 | DATE_PART('epoch', datestamp) AS datestamp, 416 | parent 417 | FROM 418 | %s 419 | WHERE 420 | approved='Y' AND 421 | id=%s""" % (table_name, id) 422 | num_rows = self.cursor.execute(stmt) 423 | if num_rows == 0: 424 | return None 425 | result = list(self.cursor.fetchone()) 426 | if len(result[2]) == 0: 427 | author = result[1].strip() 428 | else: 429 | author = "%s <%s>" % (result[1].strip(), result[2].strip()) 430 | formatted_time = strutil.get_formatted_time(time.localtime(result[4])) 431 | headers = [] 432 | headers.append("Path: %s" % (settings.nntp_hostname)) 433 | headers.append("From: %s" % (author)) 434 | headers.append("Newsgroups: %s" % (group_name)) 435 | headers.append("Date: %s" % (formatted_time)) 436 | headers.append("Subject: %s" % (result[3].strip())) 437 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 438 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 439 | if result[5] != 0: 440 | headers.append("References: <%s@%s>" % (result[5], group_name)) 441 | return "\r\n".join(headers) 442 | 443 | def get_BODY(self, group_name, id): 444 | table_name = self.get_table_name(group_name) 445 | stmt = """ 446 | SELECT 447 | B.body 448 | FROM 449 | %s A, 450 | %s_bodies B 451 | WHERE 452 | A.id=B.id AND 453 | A.approved='Y' AND 454 | B.id=%s""" % (table_name, table_name, id) 455 | num_rows = self.cursor.execute(stmt) 456 | if num_rows == 0: 457 | return None 458 | else: 459 | return strutil.format_body(self.cursor.fetchone()[0]) 460 | 461 | def get_XOVER(self, group_name, start_id, end_id='ggg'): 462 | table_name = self.get_table_name(group_name) 463 | stmt = """ 464 | SELECT 465 | A.id, 466 | parent, 467 | author, 468 | email, 469 | subject, 470 | DATE_PART('epoch', datestamp) AS datestamp, 471 | B.body 472 | FROM 473 | %s A, 474 | %s_bodies B 475 | WHERE 476 | A.approved='Y' AND 477 | A.id=B.id AND 478 | A.id >= %s""" % (table_name, table_name, start_id) 479 | if end_id != 'ggg': 480 | stmt = "%s AND A.id <= %s" % (stmt, end_id) 481 | self.cursor.execute(stmt) 482 | result = list(self.cursor.fetchall()) 483 | overviews = [] 484 | for row in result: 485 | if row[3] == '': 486 | author = row[2].strip() 487 | else: 488 | author = "%s <%s>" % (row[2].strip(), row[3].strip()) 489 | formatted_time = strutil.get_formatted_time(time.localtime(row[5])) 490 | message_id = "<%s@%s>" % (row[0], group_name) 491 | line_count = len(row[6].split('\n')) 492 | xref = 'Xref: %s %s:%s' % (settings.nntp_hostname, group_name, row[0]) 493 | if row[1] != 0: 494 | reference = "<%s@%s>" % (row[1], group_name) 495 | else: 496 | reference = "" 497 | # message_number subject author date message_id reference bytes lines xref 498 | overviews.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (row[0], row[4].strip(), author, formatted_time, message_id, reference, len(strutil.format_body(row[6])), line_count, xref)) 499 | return "\r\n".join(overviews) 500 | 501 | def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): 502 | # XXX: need to actually check for the header values being passed as 503 | # XXX: not all header names map to column names on the tables 504 | table_name = self.get_table_name(group_name) 505 | stmt = """ 506 | SELECT 507 | A.id, 508 | parent, 509 | author, 510 | email, 511 | subject, 512 | DATE_PART('epoch', datestamp) AS datestamp, 513 | B.body 514 | FROM 515 | %s A, 516 | %s_bodies B 517 | WHERE 518 | A.approved='Y' AND 519 | %s LIKE '%s' AND 520 | A.id = B.id AND 521 | A.id >= %s""" % (table_name, table_name, header, strutil.format_wildcards_sql(pattern), start_id) 522 | if end_id != 'ggg': 523 | stmt = "%s AND A.id <= %s" % (stmt, end_id) 524 | num_rows = self.cursor.execute(stmt) 525 | if num_rows == 0: 526 | return None 527 | result = list(self.cursor.fetchall()) 528 | hdrs = [] 529 | for row in result: 530 | if header.upper() == 'SUBJECT': 531 | hdrs.append('%s %s' % (row[0], row[4].strip())) 532 | elif header.upper() == 'FROM': 533 | # XXX: totally broken with empty values for the email address 534 | hdrs.append('%s %s <%s>' % (row[0], row[2].strip(), row[3].strip())) 535 | elif header.upper() == 'DATE': 536 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) 537 | elif header.upper() == 'MESSAGE-ID': 538 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 539 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 540 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 541 | elif header.upper() == 'BYTES': 542 | hdrs.append('%s %s' % (row[0], len(row[6]))) 543 | elif header.upper() == 'LINES': 544 | hdrs.append('%s %s' % (row[0], len(row[6].split('\n')))) 545 | elif header.upper() == 'XREF': 546 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 547 | if len(hdrs) == 0: 548 | return "" 549 | else: 550 | return "\r\n".join(hdrs) 551 | 552 | def get_LISTGROUP(self, group_name): 553 | table_name = self.get_table_name(group_name) 554 | stmt = """ 555 | SELECT 556 | id 557 | FROM 558 | %s 559 | WHERE 560 | approved='Y' 561 | ORDER BY 562 | id ASC""" % (table_name) 563 | self.cursor.execute(stmt) 564 | result = list(self.cursor.fetchall()) 565 | return "\r\n".join(["%s" % k for k in result]) 566 | 567 | def get_XGTITLE(self, pattern=None): 568 | stmt = """ 569 | SELECT 570 | nntp_group_name, 571 | description 572 | FROM 573 | forums 574 | WHERE 575 | LENGTH(nntp_group_name) > 0""" 576 | if pattern != None: 577 | stmt = stmt + """ AND 578 | nntp_group_name LIKE '%s'""" % (strutil.format_wildcards_sql(pattern)) 579 | stmt = stmt + """ 580 | ORDER BY 581 | nntp_group_name ASC""" 582 | self.cursor.execute(stmt) 583 | result = list(self.cursor.fetchall()) 584 | return "\r\n".join(["%s %s" % (k, v) for k, v in result]) 585 | 586 | def get_XHDR(self, group_name, header, style, range): 587 | table_name = self.get_table_name(group_name) 588 | stmt = """ 589 | SELECT 590 | A.id, 591 | parent, 592 | author, 593 | email, 594 | subject, 595 | DATE_PART('epoch', datestamp) AS datestamp, 596 | B.body 597 | FROM 598 | %s A, 599 | %s_bodies B 600 | WHERE 601 | A.approved='Y' AND 602 | A.id = B.id AND """ % (table_name, table_name) 603 | if style == 'range': 604 | stmt = '%s A.id >= %s' % (stmt, range[0]) 605 | if len(range) == 2: 606 | stmt = '%s AND A.id <= %s' % (stmt, range[1]) 607 | else: 608 | stmt = '%s A.id = %s' % (stmt, range[0]) 609 | if self.cursor.execute(stmt) == 0: 610 | return None 611 | result = self.cursor.fetchall() 612 | hdrs = [] 613 | for row in result: 614 | if header.upper() == 'SUBJECT': 615 | hdrs.append('%s %s' % (row[0], row[4].strip())) 616 | elif header.upper() == 'FROM': 617 | hdrs.append('%s %s <%s>' % (row[0], row[2].strip(), row[3].strip())) 618 | elif header.upper() == 'DATE': 619 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) 620 | elif header.upper() == 'MESSAGE-ID': 621 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 622 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 623 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 624 | elif header.upper() == 'BYTES': 625 | hdrs.append('%s %s' % (row[0], len(row[6]))) 626 | elif header.upper() == 'LINES': 627 | hdrs.append('%s %s' % (row[0], len(row[6].split('\n')))) 628 | elif header.upper() == 'XREF': 629 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 630 | if len(hdrs) == 0: 631 | return "" 632 | else: 633 | return "\r\n".join(hdrs) 634 | 635 | def do_POST(self, group_name, lines, ip_address, username=''): 636 | table_name = self.get_table_name(group_name) 637 | body = self.get_message_body(lines) 638 | author, email = from_regexp.search(lines, 0).groups() 639 | subject = subject_regexp.search(lines, 0).groups()[0].strip() 640 | # patch by Andreas Wegmann to fix the handling of unusual encodings of messages 641 | lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines)) 642 | if lines.find('References') != -1: 643 | # get the 'modifystamp' value from the parent (if any) 644 | references = references_regexp.search(lines, 0).groups() 645 | parent_id, void = references[-1].strip().split('@') 646 | stmt = """ 647 | SELECT 648 | MAX(id) AS next_id 649 | FROM 650 | %s""" % (table_name) 651 | num_rows = self.cursor.execute(stmt) 652 | if num_rows == 0: 653 | new_id = 1 654 | else: 655 | new_id = self.cursor.fetchone()[0] 656 | if new_id is None: 657 | new_id = 1 658 | else: 659 | new_id = new_id + 1 660 | stmt = """ 661 | SELECT 662 | id, 663 | thread, 664 | modifystamp 665 | FROM 666 | %s 667 | WHERE 668 | approved='Y' AND 669 | id=%s 670 | GROUP BY 671 | id""" % (table_name, parent_id) 672 | num_rows = self.cursor.execute(stmt) 673 | if num_rows == 0: 674 | return None 675 | parent_id, thread_id, modifystamp = self.cursor.fetchone() 676 | else: 677 | stmt = """ 678 | SELECT 679 | MAX(id) AS next_id, 680 | DATE_PART('epoch', CURRENT_TIMESTAMP()) 681 | FROM 682 | %s""" % (table_name) 683 | self.cursor.execute(stmt) 684 | new_id, modifystamp = self.cursor.fetchone() 685 | if new_id is None: 686 | new_id = 1 687 | else: 688 | new_id = new_id + 1 689 | modifystamp = int(modifystamp) 690 | parent_id = 0 691 | thread_id = new_id 692 | stmt = """ 693 | INSERT INTO 694 | """ + table_name + """ 695 | ( 696 | id, 697 | datestamp, 698 | thread, 699 | parent, 700 | author, 701 | subject, 702 | email, 703 | host, 704 | email_reply, 705 | approved, 706 | msgid, 707 | modifystamp, 708 | userid 709 | ) VALUES ( 710 | %s, 711 | NOW(), 712 | %s, 713 | %s, 714 | '%s', 715 | '%s', 716 | '%s', 717 | '%s', 718 | 'N', 719 | 'Y', 720 | '', 721 | %s, 722 | 0 723 | ) 724 | """ 725 | if not self.cursor.execute(stmt, (new_id, thread_id, parent_id, author.strip(), subject.strip(), email.strip(), ip_address, modifystamp,)): 726 | return None 727 | else: 728 | # insert into the '*_bodies' table 729 | stmt = """ 730 | INSERT INTO 731 | """ + table_name + """_bodies 732 | ( 733 | id, 734 | body, 735 | thread 736 | ) VALUES ( 737 | %s, 738 | '%s', 739 | %s 740 | )""" 741 | if not self.cursor.execute(stmt, (new_id, body, thread_id,)): 742 | # delete from 'table_name' before returning.. 743 | stmt = """ 744 | DELETE FROM 745 | %s 746 | WHERE 747 | id=%s""" % (table_name, new_id) 748 | self.cursor.execute(stmt) 749 | return None 750 | else: 751 | # alert forum moderators 752 | self.send_notifications(group_name, new_id, thread_id, parent_id, author.strip(), email.strip(), subject.strip(), body) 753 | return 1 754 | -------------------------------------------------------------------------------- /storage/phpnuke_phpbb_mysql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2002, 2003, 2004 Joao Prado Maia. See the LICENSE file for more information. 3 | import MySQLdb 4 | import time 5 | from mimify import mime_encode_header, mime_decode_header 6 | import re 7 | import settings 8 | import md5 9 | import mime 10 | import strutil 11 | 12 | 13 | # patch by Andreas Wegmann to fix the handling of unusual encodings of messages 14 | q_quote_multiline = re.compile("=\?(.*?)\?[qQ]\?(.*?)\?=.*?=\?\\1\?[qQ]\?(.*?)\?=", re.M | re.S) 15 | 16 | # we don't need to compile the regexps everytime.. 17 | doubleline_regexp = re.compile("^\.\.", re.M) 18 | singleline_regexp = re.compile("^\.", re.M) 19 | from_regexp = re.compile("^From:(.*)<(.*)>", re.M) 20 | subject_regexp = re.compile("^Subject:(.*)", re.M) 21 | references_regexp = re.compile("^References:(.*)<(.*)>", re.M) 22 | lines_regexp = re.compile("^Lines:(.*)", re.M) 23 | 24 | class Papercut_Storage: 25 | """ 26 | Storage Backend interface for the nuke port of phpBB (http://www.phpnuke.org) 27 | 28 | This is the interface for PHPNuke/NukeBB running on a MySQL database. For more 29 | information on the structure of the 'storage' package, please refer to the 30 | __init__.py available on the 'storage' sub-directory. 31 | """ 32 | 33 | def __init__(self): 34 | self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) 35 | self.cursor = self.conn.cursor() 36 | 37 | def get_message_body(self, headers): 38 | """Parses and returns the most appropriate message body possible. 39 | 40 | The function tries to extract the plaintext version of a MIME based 41 | message, and if it is not available then it returns the html version. 42 | """ 43 | return mime.get_text_message(headers) 44 | 45 | def quote_string(self, text): 46 | """Quotes strings the MySQL way.""" 47 | return text.replace("'", "\\'") 48 | 49 | def make_bbcode_uid(self): 50 | return md5.new(str(time.clock())).hexdigest() 51 | 52 | def encode_ip(self, dotquad_ip): 53 | t = dotquad_ip.split('.') 54 | return '%02x%02x%02x%02x' % (int(t[0]), int(t[1]), int(t[2]), int(t[3])) 55 | 56 | def group_exists(self, group_name): 57 | stmt = """ 58 | SELECT 59 | COUNT(*) AS total 60 | FROM 61 | %sforums 62 | WHERE 63 | LOWER(nntp_group_name)=LOWER('%s')""" % (settings.phpbb_table_prefix, group_name) 64 | self.cursor.execute(stmt) 65 | return self.cursor.fetchone()[0] 66 | 67 | def article_exists(self, group_name, style, range): 68 | forum_id = self.get_forum(group_name) 69 | stmt = """ 70 | SELECT 71 | COUNT(*) AS total 72 | FROM 73 | %sposts 74 | WHERE 75 | forum_id=%s""" % (settings.phpbb_table_prefix, forum_id) 76 | if style == 'range': 77 | stmt = "%s AND post_id > %s" % (stmt, range[0]) 78 | if len(range) == 2: 79 | stmt = "%s AND post_id < %s" % (stmt, range[1]) 80 | else: 81 | stmt = "%s AND post_id = %s" % (stmt, range[0]) 82 | self.cursor.execute(stmt) 83 | return self.cursor.fetchone()[0] 84 | 85 | def get_first_article(self, group_name): 86 | forum_id = self.get_forum(group_name) 87 | stmt = """ 88 | SELECT 89 | IF(MIN(post_id) IS NULL, 0, MIN(post_id)) AS first_article 90 | FROM 91 | %sposts 92 | WHERE 93 | forum_id=%s""" % (settings.phpbb_table_prefix, forum_id) 94 | num_rows = self.cursor.execute(stmt) 95 | return self.cursor.fetchone()[0] 96 | 97 | def get_group_stats(self, group_name): 98 | total, max, min = self.get_forum_stats(self.get_forum(group_name)) 99 | return (total, min, max, group_name) 100 | 101 | def get_forum_stats(self, forum_id): 102 | stmt = """ 103 | SELECT 104 | COUNT(post_id) AS total, 105 | IF(MAX(post_id) IS NULL, 0, MAX(post_id)) AS maximum, 106 | IF(MIN(post_id) IS NULL, 0, MIN(post_id)) AS minimum 107 | FROM 108 | %sposts 109 | WHERE 110 | forum_id=%s""" % (settings.phpbb_table_prefix, forum_id) 111 | num_rows = self.cursor.execute(stmt) 112 | return self.cursor.fetchone() 113 | 114 | def get_forum(self, group_name): 115 | stmt = """ 116 | SELECT 117 | forum_id 118 | FROM 119 | %sforums 120 | WHERE 121 | nntp_group_name='%s'""" % (settings.phpbb_table_prefix, group_name) 122 | self.cursor.execute(stmt) 123 | return self.cursor.fetchone()[0] 124 | 125 | def get_message_id(self, msg_num, group): 126 | return '<%s@%s>' % (msg_num, group) 127 | 128 | def get_NEWGROUPS(self, ts, group='%'): 129 | # since phpBB doesn't record when each forum was created, we have no way of knowing this... 130 | return None 131 | 132 | def get_NEWNEWS(self, ts, group='*'): 133 | stmt = """ 134 | SELECT 135 | nntp_group_name, 136 | forum_id 137 | FROM 138 | %sforums 139 | WHERE 140 | nntp_group_name LIKE '%s' 141 | ORDER BY 142 | nntp_group_name ASC""" % (settings.phpbb_table_prefix, group.replace('*', '%')) 143 | self.cursor.execute(stmt) 144 | result = list(self.cursor.fetchall()) 145 | articles = [] 146 | for group_name, forum_id in result: 147 | stmt = """ 148 | SELECT 149 | post_id 150 | FROM 151 | %sposts 152 | WHERE 153 | forum_id=%s AND 154 | post_time >= %s""" % (settings.phpbb_table_prefix, forum_id, ts) 155 | num_rows = self.cursor.execute(stmt) 156 | if num_rows == 0: 157 | continue 158 | ids = list(self.cursor.fetchall()) 159 | for id in ids: 160 | articles.append("<%s@%s>" % (id, group_name)) 161 | if len(articles) == 0: 162 | return '' 163 | else: 164 | return "\r\n".join(articles) 165 | 166 | def get_GROUP(self, group_name): 167 | forum_id = self.get_forum(group_name) 168 | result = self.get_forum_stats(forum_id) 169 | return (result[0], result[2], result[1]) 170 | 171 | def get_LIST(self, username=""): 172 | stmt = """ 173 | SELECT 174 | nntp_group_name, 175 | forum_id 176 | FROM 177 | %sforums 178 | WHERE 179 | LENGTH(nntp_group_name) > 0 180 | ORDER BY 181 | nntp_group_name ASC""" % (settings.phpbb_table_prefix) 182 | self.cursor.execute(stmt) 183 | result = list(self.cursor.fetchall()) 184 | if len(result) == 0: 185 | return "" 186 | else: 187 | lists = [] 188 | for group_name, forum_id in result: 189 | total, maximum, minimum = self.get_forum_stats(forum_id) 190 | if settings.server_type == 'read-only': 191 | lists.append("%s %s %s n" % (group_name, maximum, minimum)) 192 | else: 193 | lists.append("%s %s %s y" % (group_name, maximum, minimum)) 194 | return "\r\n".join(lists) 195 | 196 | def get_STAT(self, group_name, id): 197 | forum_id = self.get_forum(group_name) 198 | stmt = """ 199 | SELECT 200 | post_id 201 | FROM 202 | %sposts 203 | WHERE 204 | forum_id=%s AND 205 | post_id=%s""" % (settings.phpbb_table_prefix, forum_id, id) 206 | return self.cursor.execute(stmt) 207 | 208 | def get_ARTICLE(self, group_name, id): 209 | forum_id = self.get_forum(group_name) 210 | prefix = settings.phpbb_table_prefix 211 | nuke_prefix = settings.nuke_table_prefix 212 | stmt = """ 213 | SELECT 214 | A.post_id, 215 | C.username, 216 | C.user_email, 217 | CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', E.topic_title) ELSE B.post_subject END, 218 | A.post_time, 219 | B.post_text, 220 | A.topic_id, 221 | A.post_username, 222 | MIN(D.post_id) 223 | FROM 224 | %sposts A, 225 | %sposts_text B 226 | INNER JOIN 227 | %sposts D 228 | ON 229 | D.topic_id=A.topic_id 230 | INNER JOIN 231 | %stopics E 232 | ON 233 | A.topic_id = E.topic_id 234 | LEFT JOIN 235 | %susers C 236 | ON 237 | A.poster_id=C.user_id 238 | WHERE 239 | A.forum_id=%s AND 240 | A.post_id=B.post_id AND 241 | A.post_id=%s 242 | GROUP BY 243 | D.topic_id""" % (prefix, prefix, prefix, prefix, nuke_prefix, forum_id, id) 244 | num_rows = self.cursor.execute(stmt) 245 | if num_rows == 0: 246 | return None 247 | result = list(self.cursor.fetchone()) 248 | # check if there is a registered user 249 | if result[7] == '': 250 | if len(result[2]) == 0: 251 | author = result[1] 252 | else: 253 | author = "%s <%s>" % (result[1], result[2]) 254 | else: 255 | author = result[7] 256 | formatted_time = strutil.get_formatted_time(time.localtime(result[4])) 257 | headers = [] 258 | headers.append("Path: %s" % (settings.nntp_hostname)) 259 | headers.append("From: %s" % (author)) 260 | headers.append("Newsgroups: %s" % (group_name)) 261 | headers.append("Date: %s" % (formatted_time)) 262 | headers.append("Subject: %s" % (result[3])) 263 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 264 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 265 | if result[8] != result[0]: 266 | headers.append("References: <%s@%s>" % (result[8], group_name)) 267 | return ("\r\n".join(headers), strutil.format_body(result[5])) 268 | 269 | def get_LAST(self, group_name, current_id): 270 | forum_id = self.get_forum(group_name) 271 | stmt = """ 272 | SELECT 273 | post_id 274 | FROM 275 | %sposts 276 | WHERE 277 | post_id < %s AND 278 | forum_id=%s 279 | ORDER BY 280 | post_id DESC 281 | LIMIT 0, 1""" % (settings.phpbb_table_prefix, current_id, forum_id) 282 | num_rows = self.cursor.execute(stmt) 283 | if num_rows == 0: 284 | return None 285 | return self.cursor.fetchone()[0] 286 | 287 | def get_NEXT(self, group_name, current_id): 288 | forum_id = self.get_forum(group_name) 289 | stmt = """ 290 | SELECT 291 | post_id 292 | FROM 293 | %sposts 294 | WHERE 295 | forum_id=%s AND 296 | post_id > %s 297 | ORDER BY 298 | post_id ASC 299 | LIMIT 0, 1""" % (settings.phpbb_table_prefix, forum_id, current_id) 300 | num_rows = self.cursor.execute(stmt) 301 | if num_rows == 0: 302 | return None 303 | return self.cursor.fetchone()[0] 304 | 305 | def get_HEAD(self, group_name, id): 306 | forum_id = self.get_forum(group_name) 307 | prefix = settings.phpbb_table_prefix 308 | nuke_prefix = settings.nuke_table_prefix 309 | stmt = """ 310 | SELECT 311 | A.post_id, 312 | C.username, 313 | C.user_email, 314 | CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', E.topic_title) ELSE B.post_subject END, 315 | A.post_time, 316 | A.topic_id, 317 | A.post_username, 318 | MIN(D.post_id) 319 | FROM 320 | %sposts A, 321 | %sposts_text B 322 | INNER JOIN 323 | %stopics E 324 | ON 325 | A.topic_id = E.topic_id 326 | INNER JOIN 327 | %sposts D 328 | ON 329 | D.topic_id=A.topic_id 330 | LEFT JOIN 331 | %susers C 332 | ON 333 | A.poster_id=C.user_id 334 | WHERE 335 | A.forum_id=%s AND 336 | A.post_id=B.post_id AND 337 | A.post_id=%s 338 | GROUP BY 339 | D.topic_id""" % (prefix, prefix, prefix, prefix, nuke_prefix, forum_id, id) 340 | num_rows = self.cursor.execute(stmt) 341 | if num_rows == 0: 342 | return None 343 | result = list(self.cursor.fetchone()) 344 | # check if there is a registered user 345 | if len(result[6]) == 0 or result[6] == '': 346 | if len(result[2]) == 0: 347 | author = result[1] 348 | else: 349 | author = "%s <%s>" % (result[1], result[2]) 350 | else: 351 | author = result[6] 352 | formatted_time = strutil.get_formatted_time(time.localtime(result[4])) 353 | headers = [] 354 | headers.append("Path: %s" % (settings.nntp_hostname)) 355 | headers.append("From: %s" % (author)) 356 | headers.append("Newsgroups: %s" % (group_name)) 357 | headers.append("Date: %s" % (formatted_time)) 358 | headers.append("Subject: %s" % (result[3])) 359 | headers.append("Message-ID: <%s@%s>" % (result[0], group_name)) 360 | headers.append("Xref: %s %s:%s" % (settings.nntp_hostname, group_name, result[0])) 361 | if result[7] != result[0]: 362 | headers.append("References: <%s@%s>" % (result[7], group_name)) 363 | return "\r\n".join(headers) 364 | 365 | def get_BODY(self, group_name, id): 366 | forum_id = self.get_forum(group_name) 367 | prefix = settings.phpbb_table_prefix 368 | stmt = """ 369 | SELECT 370 | B.post_text 371 | FROM 372 | %sposts A, 373 | %sposts_text B 374 | WHERE 375 | A.post_id=B.post_id AND 376 | A.forum_id=%s AND 377 | A.post_id=%s""" % (prefix, prefix, forum_id, id) 378 | num_rows = self.cursor.execute(stmt) 379 | if num_rows == 0: 380 | return None 381 | else: 382 | return strutil.format_body(self.cursor.fetchone()[0]) 383 | 384 | def get_XOVER(self, group_name, start_id, end_id='ggg'): 385 | forum_id = self.get_forum(group_name) 386 | prefix = settings.phpbb_table_prefix 387 | nuke_prefix = settings.nuke_table_prefix 388 | stmt = """ 389 | SELECT 390 | A.post_id, 391 | A.topic_id, 392 | C.username, 393 | C.user_email, 394 | CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', D.topic_title) ELSE B.post_subject END, 395 | A.post_time, 396 | B.post_text, 397 | A.post_username 398 | FROM 399 | %sposts A, 400 | %sposts_text B 401 | LEFT JOIN 402 | %susers C 403 | ON 404 | A.poster_id=C.user_id 405 | LEFT JOIN 406 | %stopics D 407 | ON 408 | A.topic_id = D.topic_id 409 | WHERE 410 | A.post_id=B.post_id AND 411 | A.forum_id=%s AND 412 | A.post_id >= %s""" % (prefix, prefix, nuke_prefix, prefix, forum_id, start_id) 413 | if end_id != 'ggg': 414 | stmt = "%s AND A.post_id <= %s" % (stmt, end_id) 415 | self.cursor.execute(stmt) 416 | result = list(self.cursor.fetchall()) 417 | overviews = [] 418 | for row in result: 419 | if row[7] == '': 420 | if row[3] == '': 421 | author = row[2] 422 | else: 423 | author = "%s <%s>" % (row[2], row[3]) 424 | else: 425 | author = row[7] 426 | formatted_time = strutil.get_formatted_time(time.localtime(row[5])) 427 | message_id = "<%s@%s>" % (row[0], group_name) 428 | line_count = len(row[6].split('\n')) 429 | xref = 'Xref: %s %s:%s' % (settings.nntp_hostname, group_name, row[0]) 430 | if row[1] != row[0]: 431 | reference = "<%s@%s>" % (row[1], group_name) 432 | else: 433 | reference = "" 434 | # message_number subject author date message_id reference bytes lines xref 435 | overviews.append("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (row[0], row[4], author, formatted_time, message_id, reference, len(strutil.format_body(row[6])), line_count, xref)) 436 | return "\r\n".join(overviews) 437 | 438 | def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): 439 | # XXX: need to actually check for the header values being passed as 440 | # XXX: not all header names map to column names on the tables 441 | forum_id = self.get_forum(group_name) 442 | prefix = settings.phpbb_table_prefix 443 | nuke_prefix = settings.nuke_table_prefix 444 | stmt = """ 445 | SELECT 446 | A.post_id, 447 | A.topic_id, 448 | C.username, 449 | C.user_email, 450 | CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', D.topic_title) ELSE B.post_subject END, 451 | A.post_time, 452 | B.post_text, 453 | A.post_username 454 | FROM 455 | %sposts A, 456 | %sposts_text B 457 | LEFT JOIN 458 | %susers C 459 | ON 460 | A.poster_id=C.user_id 461 | LEFT JOIN 462 | %stopics D 463 | ON 464 | A.topic_id = D.topic_id 465 | WHERE 466 | A.forum_id=%s AND 467 | %s REGEXP '%s' AND 468 | A.post_id = B.post_id AND 469 | A.post_id >= %s""" % (prefix, prefix, nuke_prefix, prefix, forum_id, header, strutil.format_wildcards(pattern), start_id) 470 | if end_id != 'ggg': 471 | stmt = "%s AND A.post_id <= %s" % (stmt, end_id) 472 | num_rows = self.cursor.execute(stmt) 473 | if num_rows == 0: 474 | return None 475 | result = list(self.cursor.fetchall()) 476 | hdrs = [] 477 | for row in result: 478 | if header.upper() == 'SUBJECT': 479 | hdrs.append('%s %s' % (row[0], row[4])) 480 | elif header.upper() == 'FROM': 481 | # XXX: totally broken with empty values for the email address 482 | hdrs.append('%s %s <%s>' % (row[0], row[2], row[3])) 483 | elif header.upper() == 'DATE': 484 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) 485 | elif header.upper() == 'MESSAGE-ID': 486 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 487 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 488 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 489 | elif header.upper() == 'BYTES': 490 | hdrs.append('%s %s' % (row[0], len(row[6]))) 491 | elif header.upper() == 'LINES': 492 | hdrs.append('%s %s' % (row[0], len(row[6].split('\n')))) 493 | elif header.upper() == 'XREF': 494 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 495 | if len(hdrs) == 0: 496 | return "" 497 | else: 498 | return "\r\n".join(hdrs) 499 | 500 | def get_LISTGROUP(self, group_name): 501 | forum_id = self.get_forum(group_name) 502 | stmt = """ 503 | SELECT 504 | post_id 505 | FROM 506 | %sposts 507 | WHERE 508 | forum_id=%s 509 | ORDER BY 510 | post_id ASC""" % (settings.phpbb_table_prefix, forum_id) 511 | self.cursor.execute(stmt) 512 | result = list(self.cursor.fetchall()) 513 | return "\r\n".join(["%s" % k for k in result]) 514 | 515 | def get_XGTITLE(self, pattern=None): 516 | stmt = """ 517 | SELECT 518 | nntp_group_name, 519 | forum_desc 520 | FROM 521 | %sforums 522 | WHERE 523 | LENGTH(nntp_group_name) > 0""" % (settings.phpbb_table_prefix) 524 | if pattern != None: 525 | stmt = stmt + """ AND 526 | nntp_group_name REGEXP '%s'""" % (strutil.format_wildcards(pattern)) 527 | stmt = stmt + """ 528 | ORDER BY 529 | nntp_group_name ASC""" 530 | self.cursor.execute(stmt) 531 | result = list(self.cursor.fetchall()) 532 | return "\r\n".join(["%s %s" % (k, v) for k, v in result]) 533 | 534 | def get_XHDR(self, group_name, header, style, range): 535 | forum_id = self.get_forum(group_name) 536 | prefix = settings.phpbb_table_prefix 537 | nuke_prefix = settings.nuke_table_prefix 538 | stmt = """ 539 | SELECT 540 | A.post_id, 541 | A.topic_id, 542 | D.username, 543 | D.user_email, 544 | CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END, 545 | A.post_time, 546 | B.post_text, 547 | A.post_username 548 | FROM 549 | %sposts A, 550 | %sposts_text B 551 | LEFT JOIN 552 | %stopics C 553 | ON 554 | A.topic_id = C.topic_id 555 | LEFT JOIN 556 | %susers D 557 | ON 558 | A.poster_id=D.user_id 559 | WHERE 560 | A.forum_id=%s AND 561 | A.post_id = B.post_id AND """ % (prefix, prefix, prefix, nuke_prefix, forum_id) 562 | if style == 'range': 563 | stmt = '%s A.post_id >= %s' % (stmt, range[0]) 564 | if len(range) == 2: 565 | stmt = '%s AND A.post_id <= %s' % (stmt, range[1]) 566 | else: 567 | stmt = '%s A.post_id = %s' % (stmt, range[0]) 568 | if self.cursor.execute(stmt) == 0: 569 | return None 570 | result = self.cursor.fetchall() 571 | hdrs = [] 572 | for row in result: 573 | if header.upper() == 'SUBJECT': 574 | hdrs.append('%s %s' % (row[0], row[4])) 575 | elif header.upper() == 'FROM': 576 | hdrs.append('%s %s <%s>' % (row[0], row[2], row[3])) 577 | elif header.upper() == 'DATE': 578 | hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) 579 | elif header.upper() == 'MESSAGE-ID': 580 | hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) 581 | elif (header.upper() == 'REFERENCES') and (row[1] != 0): 582 | hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) 583 | elif header.upper() == 'BYTES': 584 | hdrs.append('%s %s' % (row[0], len(row[6]))) 585 | elif header.upper() == 'LINES': 586 | hdrs.append('%s %s' % (row[0], len(row[6].split('\n')))) 587 | elif header.upper() == 'XREF': 588 | hdrs.append('%s %s %s:%s' % (row[0], settings.nntp_hostname, group_name, row[0])) 589 | if len(hdrs) == 0: 590 | return "" 591 | else: 592 | return "\r\n".join(hdrs) 593 | 594 | def do_POST(self, group_name, lines, ip_address, username=''): 595 | forum_id = self.get_forum(group_name) 596 | prefix = settings.phpbb_table_prefix 597 | nuke_prefix = settings.nuke_table_prefix 598 | # patch by Andreas Wegmann to fix the handling of unusual encodings of messages 599 | lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines)) 600 | body = self.get_message_body(lines) 601 | author, email = from_regexp.search(lines, 0).groups() 602 | subject = subject_regexp.search(lines, 0).groups()[0].strip() 603 | # get the authentication information now 604 | if username != '': 605 | stmt = """ 606 | SELECT 607 | user_id 608 | FROM 609 | %susers 610 | WHERE 611 | username='%s'""" % (nuke_prefix, username) 612 | num_rows = self.cursor.execute(stmt) 613 | if num_rows == 0: 614 | poster_id = -1 615 | else: 616 | poster_id = self.cursor.fetchone()[0] 617 | post_username = '' 618 | else: 619 | poster_id = -1 620 | post_username = author 621 | if lines.find('References') != -1: 622 | # get the 'modifystamp' value from the parent (if any) 623 | references = references_regexp.search(lines, 0).groups() 624 | parent_id, void = references[-1].strip().split('@') 625 | stmt = """ 626 | SELECT 627 | topic_id 628 | FROM 629 | %sposts 630 | WHERE 631 | post_id=%s 632 | GROUP BY 633 | post_id""" % (prefix, parent_id) 634 | num_rows = self.cursor.execute(stmt) 635 | if num_rows == 0: 636 | return None 637 | thread_id = self.cursor.fetchone()[0] 638 | else: 639 | # create a new topic 640 | stmt = """ 641 | INSERT INTO 642 | %stopics 643 | ( 644 | forum_id, 645 | topic_title, 646 | topic_poster, 647 | topic_time, 648 | topic_status, 649 | topic_vote, 650 | topic_type 651 | ) VALUES ( 652 | %s, 653 | '%s', 654 | %s, 655 | UNIX_TIMESTAMP(), 656 | 0, 657 | 0, 658 | 0 659 | )""" % (prefix, forum_id, self.quote_string(subject), poster_id) 660 | self.cursor.execute(stmt) 661 | thread_id = self.cursor.insert_id() 662 | stmt = """ 663 | INSERT INTO 664 | %sposts 665 | ( 666 | topic_id, 667 | forum_id, 668 | poster_id, 669 | post_time, 670 | poster_ip, 671 | post_username, 672 | enable_bbcode, 673 | enable_html, 674 | enable_smilies, 675 | enable_sig 676 | ) VALUES ( 677 | %s, 678 | %s, 679 | %s, 680 | UNIX_TIMESTAMP(), 681 | '%s', 682 | '%s', 683 | 1, 684 | 0, 685 | 1, 686 | 0 687 | )""" % (prefix, thread_id, forum_id, poster_id, self.encode_ip(ip_address), post_username) 688 | self.cursor.execute(stmt) 689 | new_id = self.cursor.insert_id() 690 | if not new_id: 691 | return None 692 | else: 693 | # insert into the '*posts_text' table 694 | stmt = """ 695 | INSERT INTO 696 | %sposts_text 697 | ( 698 | post_id, 699 | bbcode_uid, 700 | post_subject, 701 | post_text 702 | ) VALUES ( 703 | %s, 704 | '%s', 705 | '%s', 706 | '%s' 707 | )""" % (prefix, new_id, self.make_bbcode_uid(), self.quote_string(subject), self.quote_string(body)) 708 | if not self.cursor.execute(stmt): 709 | # delete from 'topics' and 'posts' tables before returning... 710 | stmt = """ 711 | DELETE FROM 712 | %stopics 713 | WHERE 714 | topic_id=%s""" % (prefix, thread_id) 715 | self.cursor.execute(stmt) 716 | stmt = """ 717 | DELETE FROM 718 | %sposts 719 | WHERE 720 | post_id=%s""" % (prefix, new_id) 721 | self.cursor.execute(stmt) 722 | return None 723 | else: 724 | if lines.find('References') != -1: 725 | # update the total number of posts in the forum 726 | stmt = """ 727 | UPDATE 728 | %sforums 729 | SET 730 | forum_posts=forum_posts+1, 731 | forum_last_post_id=%s 732 | WHERE 733 | forum_id=%s 734 | """ % (settings.phpbb_table_prefix, new_id, forum_id) 735 | self.cursor.execute(stmt) 736 | else: 737 | # update the total number of topics and posts in the forum 738 | stmt = """ 739 | UPDATE 740 | %sforums 741 | SET 742 | forum_topics=forum_topics+1, 743 | forum_posts=forum_posts+1, 744 | forum_last_post_id=%s 745 | WHERE 746 | forum_id=%s 747 | """ % (settings.phpbb_table_prefix, new_id, forum_id) 748 | self.cursor.execute(stmt) 749 | # update the user's post count, if this is indeed a real user 750 | if poster_id != -1: 751 | stmt = """ 752 | UPDATE 753 | %susers 754 | SET 755 | user_posts=user_posts+1 756 | WHERE 757 | user_id=%s""" % (nuke_prefix, poster_id) 758 | self.cursor.execute(stmt) 759 | # setup last post on the topic thread (Patricio Anguita ) 760 | stmt = """ 761 | UPDATE 762 | %stopics 763 | SET 764 | topic_replies=topic_replies+1, 765 | topic_last_post_id=%s 766 | WHERE 767 | topic_id=%s""" % (prefix, new_id, thread_id) 768 | self.cursor.execute(stmt) 769 | # if this is the first post on the thread.. (Patricio Anguita ) 770 | if lines.find('References') == -1: 771 | stmt = """ 772 | UPDATE 773 | %stopics 774 | SET 775 | topic_first_post_id=%s 776 | WHERE 777 | topic_id=%s AND 778 | topic_first_post_id=0""" % (prefix, new_id, thread_id) 779 | self.cursor.execute(stmt) 780 | return 1 781 | --------------------------------------------------------------------------------