├── .gitignore ├── README ├── bad.txt ├── cherrypy ├── LICENSE.txt ├── __init__.py ├── _cpchecker.py ├── _cpcompat.py ├── _cpconfig.py ├── _cpdispatch.py ├── _cperror.py ├── _cplogging.py ├── _cpmodpy.py ├── _cpnative_server.py ├── _cpreqbody.py ├── _cprequest.py ├── _cpserver.py ├── _cpthreadinglocal.py ├── _cptools.py ├── _cptree.py ├── _cpwsgi.py ├── _cpwsgi_server.py ├── cherryd ├── favicon.ico ├── lib │ ├── __init__.py │ ├── auth.py │ ├── auth_basic.py │ ├── auth_digest.py │ ├── caching.py │ ├── covercp.py │ ├── cpstats.py │ ├── cptools.py │ ├── encoding.py │ ├── gctools.py │ ├── http.py │ ├── httpauth.py │ ├── httputil.py │ ├── jsontools.py │ ├── profiler.py │ ├── reprconf.py │ ├── sessions.py │ ├── static.py │ └── xmlrpcutil.py ├── process │ ├── __init__.py │ ├── plugins.py │ ├── servers.py │ ├── win32.py │ └── wspbus.py └── wsgiserver │ ├── __init__.py │ ├── ssl_builtin.py │ ├── ssl_pyopenssl.py │ ├── wsgiserver2.py │ └── wsgiserver3.py ├── db.py ├── html ├── browse.tmp.htm ├── main.tmp.htm ├── media │ └── jquery-1.7.1.min.js └── settings.tmp.htm ├── mako ├── __init__.py ├── _ast_util.py ├── ast.py ├── cache.py ├── codegen.py ├── exceptions.py ├── ext │ ├── __init__.py │ ├── autohandler.py │ ├── babelplugin.py │ ├── beaker_cache.py │ ├── preprocessors.py │ ├── pygmentplugin.py │ └── turbogears.py ├── filters.py ├── lexer.py ├── lookup.py ├── parsetree.py ├── pygen.py ├── pyparser.py ├── runtime.py ├── template.py └── util.py ├── nntplib_ssl.py ├── omniverse.py ├── parse.py ├── settings.py ├── threads.py └── web.py /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore my config.ini until I decide on a better way to implement. 2 | # no need to share my usenet login ;) 3 | config.ini 4 | settings.ini 5 | 6 | # Compiled source # 7 | ################### 8 | *.com 9 | *.class 10 | *.dll 11 | *.exe 12 | *.o 13 | *.so 14 | 15 | # Packages # 16 | ############ 17 | # it's better to unpack these files and commit the raw source 18 | # git has its own built in compression methods 19 | *.7z 20 | *.dmg 21 | *.gz 22 | *.iso 23 | *.jar 24 | *.rar 25 | *.tar 26 | *.zip 27 | 28 | # Logs and databases # 29 | ###################### 30 | log/ 31 | *.log 32 | db/ 33 | *.sql 34 | *.sqlite 35 | 36 | # OS generated files # 37 | ###################### 38 | .DS_Store* 39 | ehthumbs.db 40 | Icon? 41 | Thumbs.db 42 | Desktop.ini 43 | 44 | # Python related files # 45 | ######################## 46 | *.py[co] 47 | 48 | # Packages 49 | *.egg 50 | *.egg-info 51 | dist 52 | build 53 | eggs 54 | parts 55 | bin 56 | var 57 | sdist 58 | develop-eggs 59 | .installed.cfg 60 | 61 | # Installer logs 62 | pip-log.txt 63 | 64 | # Unit test / coverage reports 65 | .coverage 66 | .tox 67 | 68 | #Translations 69 | *.mo 70 | 71 | #Mr Developer 72 | .mr.developer.cfg -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | OmniVerse - Automatic downloader for comic books (cbr, cbz, etc) in the tradition of Sickbeard. 2 | 3 | * If you've installed a previous version, please uninstall before installing the current version. There have been changes to the database structure that are incompatible with the current version. 4 | 5 | * Requires Python. Currently being developed and tested using Python 2.7. Some users have reported no problems using 2.6. If you find an issue that is the result of an earlier version of Python, please report it and I'll do my best to accomodate. 6 | 7 | * To run the program, use python to run omniverse.py. The first time it runs navigate to the settings page (typically http://localhost:8085/settings for a first time installation) and setup your newsgroup provider information and select what newsgroups you want archived. Currently there is only space for one newsgroup provider but a future feature will allow multiple servers. 8 | 9 | * If you need to bind to a different ip address or port for the web interface, there are command options for that: --host=[ip address] and --port=[port number] 10 | 11 | * Quit the program (CTRL+C) and restart the server so that your new settings will take affect. 12 | 13 | * Once the server gets started, you should see logging information about what headers are being downloaded. IMPORTANT: The subject lines may be NSFW or NSFL for that matter. The same goes for the web interface and the produced log file. As I'm sure we all know, there's plenty of spam on usenet. One my goals is to write a better spam/porn filter and then those messages should go away. 14 | 15 | * To access the web interface, browse to http://localhost:8085. From there you can browse the database of available comic books and create a nzb file to feed to sabnzb. 16 | 17 | * To quit the server and shut everything down use CTRL+C. It should quit cleanly and return to the command-line. If after a couple of minutes it doesn't, use process explorer to kill. 18 | 19 | * If you see anything wrong or have trouble with your particular flavor of usenet provider please submit an issue through github (see the tab marked Issues on the omniverse main page of github). 20 | 21 | * Also, if you use a block account this project works by downloading headers so depending on your provider this program may impact your monthly or block quota. I don't have exact numbers yet but I don't think its a large amount. 22 | 23 | * Another thing to make sure of is to take a connection away from Sabnzb+. For example, if you provider gives you 20 connections, you'll want to change the connections to 19 in Sabnzb+ so there is no conflict. 24 | 25 | Phew. I don't think I left out anything. Thanks for everyone's encouragement. Be sure to follow me on twitter (http://twitter.com/omniverse2) so you can send me any comments, rants, raves, or suggestions. 26 | 27 | Thanks again everyone! 28 | 404 29 | -------------------------------------------------------------------------------- /bad.txt: -------------------------------------------------------------------------------- 1 | \bhot sexy\b 2 | \bporn\b 3 | \bpussy\b 4 | \bfucked\b 5 | \bSexy Emos\b -------------------------------------------------------------------------------- /cherrypy/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2004-2011, CherryPy Team (team@cherrypy.org) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of the CherryPy Team nor the names of its contributors 13 | may be used to endorse or promote products derived from this software 14 | without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /cherrypy/_cpcompat.py: -------------------------------------------------------------------------------- 1 | """Compatibility code for using CherryPy with various versions of Python. 2 | 3 | CherryPy 3.2 is compatible with Python versions 2.3+. This module provides a 4 | useful abstraction over the differences between Python versions, sometimes by 5 | preferring a newer idiom, sometimes an older one, and sometimes a custom one. 6 | 7 | In particular, Python 2 uses str and '' for byte strings, while Python 3 8 | uses str and '' for unicode strings. We will call each of these the 'native 9 | string' type for each version. Because of this major difference, this module 10 | provides new 'bytestr', 'unicodestr', and 'nativestr' attributes, as well as 11 | two functions: 'ntob', which translates native strings (of type 'str') into 12 | byte strings regardless of Python version, and 'ntou', which translates native 13 | strings to unicode strings. This also provides a 'BytesIO' name for dealing 14 | specifically with bytes, and a 'StringIO' name for dealing with native strings. 15 | It also provides a 'base64_decode' function with native strings as input and 16 | output. 17 | """ 18 | import os 19 | import re 20 | import sys 21 | 22 | if sys.version_info >= (3, 0): 23 | py3k = True 24 | bytestr = bytes 25 | unicodestr = str 26 | nativestr = unicodestr 27 | basestring = (bytes, str) 28 | def ntob(n, encoding='ISO-8859-1'): 29 | """Return the given native string as a byte string in the given encoding.""" 30 | # In Python 3, the native string type is unicode 31 | return n.encode(encoding) 32 | def ntou(n, encoding='ISO-8859-1'): 33 | """Return the given native string as a unicode string with the given encoding.""" 34 | # In Python 3, the native string type is unicode 35 | return n 36 | def tonative(n, encoding='ISO-8859-1'): 37 | """Return the given string as a native string in the given encoding.""" 38 | # In Python 3, the native string type is unicode 39 | if isinstance(n, bytes): 40 | return n.decode(encoding) 41 | return n 42 | # type("") 43 | from io import StringIO 44 | # bytes: 45 | from io import BytesIO as BytesIO 46 | else: 47 | # Python 2 48 | py3k = False 49 | bytestr = str 50 | unicodestr = unicode 51 | nativestr = bytestr 52 | basestring = basestring 53 | def ntob(n, encoding='ISO-8859-1'): 54 | """Return the given native string as a byte string in the given encoding.""" 55 | # In Python 2, the native string type is bytes. Assume it's already 56 | # in the given encoding, which for ISO-8859-1 is almost always what 57 | # was intended. 58 | return n 59 | def ntou(n, encoding='ISO-8859-1'): 60 | """Return the given native string as a unicode string with the given encoding.""" 61 | # In Python 2, the native string type is bytes. 62 | # First, check for the special encoding 'escape'. The test suite uses this 63 | # to signal that it wants to pass a string with embedded \uXXXX escapes, 64 | # but without having to prefix it with u'' for Python 2, but no prefix 65 | # for Python 3. 66 | if encoding == 'escape': 67 | return unicode( 68 | re.sub(r'\\u([0-9a-zA-Z]{4})', 69 | lambda m: unichr(int(m.group(1), 16)), 70 | n.decode('ISO-8859-1'))) 71 | # Assume it's already in the given encoding, which for ISO-8859-1 is almost 72 | # always what was intended. 73 | return n.decode(encoding) 74 | def tonative(n, encoding='ISO-8859-1'): 75 | """Return the given string as a native string in the given encoding.""" 76 | # In Python 2, the native string type is bytes. 77 | if isinstance(n, unicode): 78 | return n.encode(encoding) 79 | return n 80 | try: 81 | # type("") 82 | from cStringIO import StringIO 83 | except ImportError: 84 | # type("") 85 | from StringIO import StringIO 86 | # bytes: 87 | BytesIO = StringIO 88 | 89 | try: 90 | set = set 91 | except NameError: 92 | from sets import Set as set 93 | 94 | try: 95 | # Python 3.1+ 96 | from base64 import decodebytes as _base64_decodebytes 97 | except ImportError: 98 | # Python 3.0- 99 | # since CherryPy claims compability with Python 2.3, we must use 100 | # the legacy API of base64 101 | from base64 import decodestring as _base64_decodebytes 102 | 103 | def base64_decode(n, encoding='ISO-8859-1'): 104 | """Return the native string base64-decoded (as a native string).""" 105 | if isinstance(n, unicodestr): 106 | b = n.encode(encoding) 107 | else: 108 | b = n 109 | b = _base64_decodebytes(b) 110 | if nativestr is unicodestr: 111 | return b.decode(encoding) 112 | else: 113 | return b 114 | 115 | try: 116 | # Python 2.5+ 117 | from hashlib import md5 118 | except ImportError: 119 | from md5 import new as md5 120 | 121 | try: 122 | # Python 2.5+ 123 | from hashlib import sha1 as sha 124 | except ImportError: 125 | from sha import new as sha 126 | 127 | try: 128 | sorted = sorted 129 | except NameError: 130 | def sorted(i): 131 | i = i[:] 132 | i.sort() 133 | return i 134 | 135 | try: 136 | reversed = reversed 137 | except NameError: 138 | def reversed(x): 139 | i = len(x) 140 | while i > 0: 141 | i -= 1 142 | yield x[i] 143 | 144 | try: 145 | # Python 3 146 | from urllib.parse import urljoin, urlencode 147 | from urllib.parse import quote, quote_plus 148 | from urllib.request import unquote, urlopen 149 | from urllib.request import parse_http_list, parse_keqv_list 150 | except ImportError: 151 | # Python 2 152 | from urlparse import urljoin 153 | from urllib import urlencode, urlopen 154 | from urllib import quote, quote_plus 155 | from urllib import unquote 156 | from urllib2 import parse_http_list, parse_keqv_list 157 | 158 | try: 159 | from threading import local as threadlocal 160 | except ImportError: 161 | from cherrypy._cpthreadinglocal import local as threadlocal 162 | 163 | try: 164 | dict.iteritems 165 | # Python 2 166 | iteritems = lambda d: d.iteritems() 167 | copyitems = lambda d: d.items() 168 | except AttributeError: 169 | # Python 3 170 | iteritems = lambda d: d.items() 171 | copyitems = lambda d: list(d.items()) 172 | 173 | try: 174 | dict.iterkeys 175 | # Python 2 176 | iterkeys = lambda d: d.iterkeys() 177 | copykeys = lambda d: d.keys() 178 | except AttributeError: 179 | # Python 3 180 | iterkeys = lambda d: d.keys() 181 | copykeys = lambda d: list(d.keys()) 182 | 183 | try: 184 | dict.itervalues 185 | # Python 2 186 | itervalues = lambda d: d.itervalues() 187 | copyvalues = lambda d: d.values() 188 | except AttributeError: 189 | # Python 3 190 | itervalues = lambda d: d.values() 191 | copyvalues = lambda d: list(d.values()) 192 | 193 | try: 194 | # Python 3 195 | import builtins 196 | except ImportError: 197 | # Python 2 198 | import __builtin__ as builtins 199 | 200 | try: 201 | # Python 2. We have to do it in this order so Python 2 builds 202 | # don't try to import the 'http' module from cherrypy.lib 203 | from Cookie import SimpleCookie, CookieError 204 | from httplib import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected 205 | from BaseHTTPServer import BaseHTTPRequestHandler 206 | except ImportError: 207 | # Python 3 208 | from http.cookies import SimpleCookie, CookieError 209 | from http.client import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected 210 | from http.server import BaseHTTPRequestHandler 211 | 212 | try: 213 | # Python 2. We have to do it in this order so Python 2 builds 214 | # don't try to import the 'http' module from cherrypy.lib 215 | from httplib import HTTPSConnection 216 | except ImportError: 217 | try: 218 | # Python 3 219 | from http.client import HTTPSConnection 220 | except ImportError: 221 | # Some platforms which don't have SSL don't expose HTTPSConnection 222 | HTTPSConnection = None 223 | 224 | try: 225 | # Python 2 226 | xrange = xrange 227 | except NameError: 228 | # Python 3 229 | xrange = range 230 | 231 | import threading 232 | if hasattr(threading.Thread, "daemon"): 233 | # Python 2.6+ 234 | def get_daemon(t): 235 | return t.daemon 236 | def set_daemon(t, val): 237 | t.daemon = val 238 | else: 239 | def get_daemon(t): 240 | return t.isDaemon() 241 | def set_daemon(t, val): 242 | t.setDaemon(val) 243 | 244 | try: 245 | from email.utils import formatdate 246 | def HTTPDate(timeval=None): 247 | return formatdate(timeval, usegmt=True) 248 | except ImportError: 249 | from rfc822 import formatdate as HTTPDate 250 | 251 | try: 252 | # Python 3 253 | from urllib.parse import unquote as parse_unquote 254 | def unquote_qs(atom, encoding, errors='strict'): 255 | return parse_unquote(atom.replace('+', ' '), encoding=encoding, errors=errors) 256 | except ImportError: 257 | # Python 2 258 | from urllib import unquote as parse_unquote 259 | def unquote_qs(atom, encoding, errors='strict'): 260 | return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) 261 | 262 | try: 263 | # Prefer simplejson, which is usually more advanced than the builtin module. 264 | import simplejson as json 265 | json_decode = json.JSONDecoder().decode 266 | json_encode = json.JSONEncoder().iterencode 267 | except ImportError: 268 | if py3k: 269 | # Python 3.0: json is part of the standard library, 270 | # but outputs unicode. We need bytes. 271 | import json 272 | json_decode = json.JSONDecoder().decode 273 | _json_encode = json.JSONEncoder().iterencode 274 | def json_encode(value): 275 | for chunk in _json_encode(value): 276 | yield chunk.encode('utf8') 277 | elif sys.version_info >= (2, 6): 278 | # Python 2.6: json is part of the standard library 279 | import json 280 | json_decode = json.JSONDecoder().decode 281 | json_encode = json.JSONEncoder().iterencode 282 | else: 283 | json = None 284 | def json_decode(s): 285 | raise ValueError('No JSON library is available') 286 | def json_encode(s): 287 | raise ValueError('No JSON library is available') 288 | 289 | try: 290 | import cPickle as pickle 291 | except ImportError: 292 | # In Python 2, pickle is a Python version. 293 | # In Python 3, pickle is the sped-up C version. 294 | import pickle 295 | 296 | try: 297 | os.urandom(20) 298 | import binascii 299 | def random20(): 300 | return binascii.hexlify(os.urandom(20)).decode('ascii') 301 | except (AttributeError, NotImplementedError): 302 | import random 303 | # os.urandom not available until Python 2.4. Fall back to random.random. 304 | def random20(): 305 | return sha('%s' % random.random()).hexdigest() 306 | 307 | try: 308 | from _thread import get_ident as get_thread_ident 309 | except ImportError: 310 | from thread import get_ident as get_thread_ident 311 | 312 | try: 313 | # Python 3 314 | next = next 315 | except NameError: 316 | # Python 2 317 | def next(i): 318 | return i.next() 319 | -------------------------------------------------------------------------------- /cherrypy/_cpconfig.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration system for CherryPy. 3 | 4 | Configuration in CherryPy is implemented via dictionaries. Keys are strings 5 | which name the mapped value, which may be of any type. 6 | 7 | 8 | Architecture 9 | ------------ 10 | 11 | CherryPy Requests are part of an Application, which runs in a global context, 12 | and configuration data may apply to any of those three scopes: 13 | 14 | Global 15 | Configuration entries which apply everywhere are stored in 16 | cherrypy.config. 17 | 18 | Application 19 | Entries which apply to each mounted application are stored 20 | on the Application object itself, as 'app.config'. This is a two-level 21 | dict where each key is a path, or "relative URL" (for example, "/" or 22 | "/path/to/my/page"), and each value is a config dict. Usually, this 23 | data is provided in the call to tree.mount(root(), config=conf), 24 | although you may also use app.merge(conf). 25 | 26 | Request 27 | Each Request object possesses a single 'Request.config' dict. 28 | Early in the request process, this dict is populated by merging global 29 | config entries, Application entries (whose path equals or is a parent 30 | of Request.path_info), and any config acquired while looking up the 31 | page handler (see next). 32 | 33 | 34 | Declaration 35 | ----------- 36 | 37 | Configuration data may be supplied as a Python dictionary, as a filename, 38 | or as an open file object. When you supply a filename or file, CherryPy 39 | uses Python's builtin ConfigParser; you declare Application config by 40 | writing each path as a section header:: 41 | 42 | [/path/to/my/page] 43 | request.stream = True 44 | 45 | To declare global configuration entries, place them in a [global] section. 46 | 47 | You may also declare config entries directly on the classes and methods 48 | (page handlers) that make up your CherryPy application via the ``_cp_config`` 49 | attribute. For example:: 50 | 51 | class Demo: 52 | _cp_config = {'tools.gzip.on': True} 53 | 54 | def index(self): 55 | return "Hello world" 56 | index.exposed = True 57 | index._cp_config = {'request.show_tracebacks': False} 58 | 59 | .. note:: 60 | 61 | This behavior is only guaranteed for the default dispatcher. 62 | Other dispatchers may have different restrictions on where 63 | you can attach _cp_config attributes. 64 | 65 | 66 | Namespaces 67 | ---------- 68 | 69 | Configuration keys are separated into namespaces by the first "." in the key. 70 | Current namespaces: 71 | 72 | engine 73 | Controls the 'application engine', including autoreload. 74 | These can only be declared in the global config. 75 | 76 | tree 77 | Grafts cherrypy.Application objects onto cherrypy.tree. 78 | These can only be declared in the global config. 79 | 80 | hooks 81 | Declares additional request-processing functions. 82 | 83 | log 84 | Configures the logging for each application. 85 | These can only be declared in the global or / config. 86 | 87 | request 88 | Adds attributes to each Request. 89 | 90 | response 91 | Adds attributes to each Response. 92 | 93 | server 94 | Controls the default HTTP server via cherrypy.server. 95 | These can only be declared in the global config. 96 | 97 | tools 98 | Runs and configures additional request-processing packages. 99 | 100 | wsgi 101 | Adds WSGI middleware to an Application's "pipeline". 102 | These can only be declared in the app's root config ("/"). 103 | 104 | checker 105 | Controls the 'checker', which looks for common errors in 106 | app state (including config) when the engine starts. 107 | Global config only. 108 | 109 | The only key that does not exist in a namespace is the "environment" entry. 110 | This special entry 'imports' other config entries from a template stored in 111 | cherrypy._cpconfig.environments[environment]. It only applies to the global 112 | config, and only when you use cherrypy.config.update. 113 | 114 | You can define your own namespaces to be called at the Global, Application, 115 | or Request level, by adding a named handler to cherrypy.config.namespaces, 116 | app.namespaces, or app.request_class.namespaces. The name can 117 | be any string, and the handler must be either a callable or a (Python 2.5 118 | style) context manager. 119 | """ 120 | 121 | import cherrypy 122 | from cherrypy._cpcompat import set, basestring 123 | from cherrypy.lib import reprconf 124 | 125 | # Deprecated in CherryPy 3.2--remove in 3.3 126 | NamespaceSet = reprconf.NamespaceSet 127 | 128 | def merge(base, other): 129 | """Merge one app config (from a dict, file, or filename) into another. 130 | 131 | If the given config is a filename, it will be appended to 132 | the list of files to monitor for "autoreload" changes. 133 | """ 134 | if isinstance(other, basestring): 135 | cherrypy.engine.autoreload.files.add(other) 136 | 137 | # Load other into base 138 | for section, value_map in reprconf.as_dict(other).items(): 139 | if not isinstance(value_map, dict): 140 | raise ValueError( 141 | "Application config must include section headers, but the " 142 | "config you tried to merge doesn't have any sections. " 143 | "Wrap your config in another dict with paths as section " 144 | "headers, for example: {'/': config}.") 145 | base.setdefault(section, {}).update(value_map) 146 | 147 | 148 | class Config(reprconf.Config): 149 | """The 'global' configuration data for the entire CherryPy process.""" 150 | 151 | def update(self, config): 152 | """Update self from a dict, file or filename.""" 153 | if isinstance(config, basestring): 154 | # Filename 155 | cherrypy.engine.autoreload.files.add(config) 156 | reprconf.Config.update(self, config) 157 | 158 | def _apply(self, config): 159 | """Update self from a dict.""" 160 | if isinstance(config.get("global", None), dict): 161 | if len(config) > 1: 162 | cherrypy.checker.global_config_contained_paths = True 163 | config = config["global"] 164 | if 'tools.staticdir.dir' in config: 165 | config['tools.staticdir.section'] = "global" 166 | reprconf.Config._apply(self, config) 167 | 168 | def __call__(self, *args, **kwargs): 169 | """Decorator for page handlers to set _cp_config.""" 170 | if args: 171 | raise TypeError( 172 | "The cherrypy.config decorator does not accept positional " 173 | "arguments; you must use keyword arguments.") 174 | def tool_decorator(f): 175 | if not hasattr(f, "_cp_config"): 176 | f._cp_config = {} 177 | for k, v in kwargs.items(): 178 | f._cp_config[k] = v 179 | return f 180 | return tool_decorator 181 | 182 | 183 | Config.environments = environments = { 184 | "staging": { 185 | 'engine.autoreload_on': False, 186 | 'checker.on': False, 187 | 'tools.log_headers.on': False, 188 | 'request.show_tracebacks': False, 189 | 'request.show_mismatched_params': False, 190 | }, 191 | "production": { 192 | 'engine.autoreload_on': False, 193 | 'checker.on': False, 194 | 'tools.log_headers.on': False, 195 | 'request.show_tracebacks': False, 196 | 'request.show_mismatched_params': False, 197 | 'log.screen': False, 198 | }, 199 | "embedded": { 200 | # For use with CherryPy embedded in another deployment stack. 201 | 'engine.autoreload_on': False, 202 | 'checker.on': False, 203 | 'tools.log_headers.on': False, 204 | 'request.show_tracebacks': False, 205 | 'request.show_mismatched_params': False, 206 | 'log.screen': False, 207 | 'engine.SIGHUP': None, 208 | 'engine.SIGTERM': None, 209 | }, 210 | "test_suite": { 211 | 'engine.autoreload_on': False, 212 | 'checker.on': False, 213 | 'tools.log_headers.on': False, 214 | 'request.show_tracebacks': True, 215 | 'request.show_mismatched_params': True, 216 | 'log.screen': False, 217 | }, 218 | } 219 | 220 | 221 | def _server_namespace_handler(k, v): 222 | """Config handler for the "server" namespace.""" 223 | atoms = k.split(".", 1) 224 | if len(atoms) > 1: 225 | # Special-case config keys of the form 'server.servername.socket_port' 226 | # to configure additional HTTP servers. 227 | if not hasattr(cherrypy, "servers"): 228 | cherrypy.servers = {} 229 | 230 | servername, k = atoms 231 | if servername not in cherrypy.servers: 232 | from cherrypy import _cpserver 233 | cherrypy.servers[servername] = _cpserver.Server() 234 | # On by default, but 'on = False' can unsubscribe it (see below). 235 | cherrypy.servers[servername].subscribe() 236 | 237 | if k == 'on': 238 | if v: 239 | cherrypy.servers[servername].subscribe() 240 | else: 241 | cherrypy.servers[servername].unsubscribe() 242 | else: 243 | setattr(cherrypy.servers[servername], k, v) 244 | else: 245 | setattr(cherrypy.server, k, v) 246 | Config.namespaces["server"] = _server_namespace_handler 247 | 248 | def _engine_namespace_handler(k, v): 249 | """Backward compatibility handler for the "engine" namespace.""" 250 | engine = cherrypy.engine 251 | if k == 'autoreload_on': 252 | if v: 253 | engine.autoreload.subscribe() 254 | else: 255 | engine.autoreload.unsubscribe() 256 | elif k == 'autoreload_frequency': 257 | engine.autoreload.frequency = v 258 | elif k == 'autoreload_match': 259 | engine.autoreload.match = v 260 | elif k == 'reload_files': 261 | engine.autoreload.files = set(v) 262 | elif k == 'deadlock_poll_freq': 263 | engine.timeout_monitor.frequency = v 264 | elif k == 'SIGHUP': 265 | engine.listeners['SIGHUP'] = set([v]) 266 | elif k == 'SIGTERM': 267 | engine.listeners['SIGTERM'] = set([v]) 268 | elif "." in k: 269 | plugin, attrname = k.split(".", 1) 270 | plugin = getattr(engine, plugin) 271 | if attrname == 'on': 272 | if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): 273 | plugin.subscribe() 274 | return 275 | elif (not v) and hasattr(getattr(plugin, 'unsubscribe', None), '__call__'): 276 | plugin.unsubscribe() 277 | return 278 | setattr(plugin, attrname, v) 279 | else: 280 | setattr(engine, k, v) 281 | Config.namespaces["engine"] = _engine_namespace_handler 282 | 283 | 284 | def _tree_namespace_handler(k, v): 285 | """Namespace handler for the 'tree' config namespace.""" 286 | if isinstance(v, dict): 287 | for script_name, app in v.items(): 288 | cherrypy.tree.graft(app, script_name) 289 | cherrypy.engine.log("Mounted: %s on %s" % (app, script_name or "/")) 290 | else: 291 | cherrypy.tree.graft(v, v.script_name) 292 | cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/")) 293 | Config.namespaces["tree"] = _tree_namespace_handler 294 | 295 | 296 | -------------------------------------------------------------------------------- /cherrypy/_cpmodpy.py: -------------------------------------------------------------------------------- 1 | """Native adapter for serving CherryPy via mod_python 2 | 3 | Basic usage: 4 | 5 | ########################################## 6 | # Application in a module called myapp.py 7 | ########################################## 8 | 9 | import cherrypy 10 | 11 | class Root: 12 | @cherrypy.expose 13 | def index(self): 14 | return 'Hi there, Ho there, Hey there' 15 | 16 | 17 | # We will use this method from the mod_python configuration 18 | # as the entry point to our application 19 | def setup_server(): 20 | cherrypy.tree.mount(Root()) 21 | cherrypy.config.update({'environment': 'production', 22 | 'log.screen': False, 23 | 'show_tracebacks': False}) 24 | 25 | ########################################## 26 | # mod_python settings for apache2 27 | # This should reside in your httpd.conf 28 | # or a file that will be loaded at 29 | # apache startup 30 | ########################################## 31 | 32 | # Start 33 | DocumentRoot "/" 34 | Listen 8080 35 | LoadModule python_module /usr/lib/apache2/modules/mod_python.so 36 | 37 | 38 | PythonPath "sys.path+['/path/to/my/application']" 39 | SetHandler python-program 40 | PythonHandler cherrypy._cpmodpy::handler 41 | PythonOption cherrypy.setup myapp::setup_server 42 | PythonDebug On 43 | 44 | # End 45 | 46 | The actual path to your mod_python.so is dependent on your 47 | environment. In this case we suppose a global mod_python 48 | installation on a Linux distribution such as Ubuntu. 49 | 50 | We do set the PythonPath configuration setting so that 51 | your application can be found by from the user running 52 | the apache2 instance. Of course if your application 53 | resides in the global site-package this won't be needed. 54 | 55 | Then restart apache2 and access http://127.0.0.1:8080 56 | """ 57 | 58 | import logging 59 | import sys 60 | 61 | import cherrypy 62 | from cherrypy._cpcompat import BytesIO, copyitems, ntob 63 | from cherrypy._cperror import format_exc, bare_error 64 | from cherrypy.lib import httputil 65 | 66 | 67 | # ------------------------------ Request-handling 68 | 69 | 70 | 71 | def setup(req): 72 | from mod_python import apache 73 | 74 | # Run any setup functions defined by a "PythonOption cherrypy.setup" directive. 75 | options = req.get_options() 76 | if 'cherrypy.setup' in options: 77 | for function in options['cherrypy.setup'].split(): 78 | atoms = function.split('::', 1) 79 | if len(atoms) == 1: 80 | mod = __import__(atoms[0], globals(), locals()) 81 | else: 82 | modname, fname = atoms 83 | mod = __import__(modname, globals(), locals(), [fname]) 84 | func = getattr(mod, fname) 85 | func() 86 | 87 | cherrypy.config.update({'log.screen': False, 88 | "tools.ignore_headers.on": True, 89 | "tools.ignore_headers.headers": ['Range'], 90 | }) 91 | 92 | engine = cherrypy.engine 93 | if hasattr(engine, "signal_handler"): 94 | engine.signal_handler.unsubscribe() 95 | if hasattr(engine, "console_control_handler"): 96 | engine.console_control_handler.unsubscribe() 97 | engine.autoreload.unsubscribe() 98 | cherrypy.server.unsubscribe() 99 | 100 | def _log(msg, level): 101 | newlevel = apache.APLOG_ERR 102 | if logging.DEBUG >= level: 103 | newlevel = apache.APLOG_DEBUG 104 | elif logging.INFO >= level: 105 | newlevel = apache.APLOG_INFO 106 | elif logging.WARNING >= level: 107 | newlevel = apache.APLOG_WARNING 108 | # On Windows, req.server is required or the msg will vanish. See 109 | # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html. 110 | # Also, "When server is not specified...LogLevel does not apply..." 111 | apache.log_error(msg, newlevel, req.server) 112 | engine.subscribe('log', _log) 113 | 114 | engine.start() 115 | 116 | def cherrypy_cleanup(data): 117 | engine.exit() 118 | try: 119 | # apache.register_cleanup wasn't available until 3.1.4. 120 | apache.register_cleanup(cherrypy_cleanup) 121 | except AttributeError: 122 | req.server.register_cleanup(req, cherrypy_cleanup) 123 | 124 | 125 | class _ReadOnlyRequest: 126 | expose = ('read', 'readline', 'readlines') 127 | def __init__(self, req): 128 | for method in self.expose: 129 | self.__dict__[method] = getattr(req, method) 130 | 131 | 132 | recursive = False 133 | 134 | _isSetUp = False 135 | def handler(req): 136 | from mod_python import apache 137 | try: 138 | global _isSetUp 139 | if not _isSetUp: 140 | setup(req) 141 | _isSetUp = True 142 | 143 | # Obtain a Request object from CherryPy 144 | local = req.connection.local_addr 145 | local = httputil.Host(local[0], local[1], req.connection.local_host or "") 146 | remote = req.connection.remote_addr 147 | remote = httputil.Host(remote[0], remote[1], req.connection.remote_host or "") 148 | 149 | scheme = req.parsed_uri[0] or 'http' 150 | req.get_basic_auth_pw() 151 | 152 | try: 153 | # apache.mpm_query only became available in mod_python 3.1 154 | q = apache.mpm_query 155 | threaded = q(apache.AP_MPMQ_IS_THREADED) 156 | forked = q(apache.AP_MPMQ_IS_FORKED) 157 | except AttributeError: 158 | bad_value = ("You must provide a PythonOption '%s', " 159 | "either 'on' or 'off', when running a version " 160 | "of mod_python < 3.1") 161 | 162 | threaded = options.get('multithread', '').lower() 163 | if threaded == 'on': 164 | threaded = True 165 | elif threaded == 'off': 166 | threaded = False 167 | else: 168 | raise ValueError(bad_value % "multithread") 169 | 170 | forked = options.get('multiprocess', '').lower() 171 | if forked == 'on': 172 | forked = True 173 | elif forked == 'off': 174 | forked = False 175 | else: 176 | raise ValueError(bad_value % "multiprocess") 177 | 178 | sn = cherrypy.tree.script_name(req.uri or "/") 179 | if sn is None: 180 | send_response(req, '404 Not Found', [], '') 181 | else: 182 | app = cherrypy.tree.apps[sn] 183 | method = req.method 184 | path = req.uri 185 | qs = req.args or "" 186 | reqproto = req.protocol 187 | headers = copyitems(req.headers_in) 188 | rfile = _ReadOnlyRequest(req) 189 | prev = None 190 | 191 | try: 192 | redirections = [] 193 | while True: 194 | request, response = app.get_serving(local, remote, scheme, 195 | "HTTP/1.1") 196 | request.login = req.user 197 | request.multithread = bool(threaded) 198 | request.multiprocess = bool(forked) 199 | request.app = app 200 | request.prev = prev 201 | 202 | # Run the CherryPy Request object and obtain the response 203 | try: 204 | request.run(method, path, qs, reqproto, headers, rfile) 205 | break 206 | except cherrypy.InternalRedirect: 207 | ir = sys.exc_info()[1] 208 | app.release_serving() 209 | prev = request 210 | 211 | if not recursive: 212 | if ir.path in redirections: 213 | raise RuntimeError("InternalRedirector visited the " 214 | "same URL twice: %r" % ir.path) 215 | else: 216 | # Add the *previous* path_info + qs to redirections. 217 | if qs: 218 | qs = "?" + qs 219 | redirections.append(sn + path + qs) 220 | 221 | # Munge environment and try again. 222 | method = "GET" 223 | path = ir.path 224 | qs = ir.query_string 225 | rfile = BytesIO() 226 | 227 | send_response(req, response.output_status, response.header_list, 228 | response.body, response.stream) 229 | finally: 230 | app.release_serving() 231 | except: 232 | tb = format_exc() 233 | cherrypy.log(tb, 'MOD_PYTHON', severity=logging.ERROR) 234 | s, h, b = bare_error() 235 | send_response(req, s, h, b) 236 | return apache.OK 237 | 238 | 239 | def send_response(req, status, headers, body, stream=False): 240 | # Set response status 241 | req.status = int(status[:3]) 242 | 243 | # Set response headers 244 | req.content_type = "text/plain" 245 | for header, value in headers: 246 | if header.lower() == 'content-type': 247 | req.content_type = value 248 | continue 249 | req.headers_out.add(header, value) 250 | 251 | if stream: 252 | # Flush now so the status and headers are sent immediately. 253 | req.flush() 254 | 255 | # Set response body 256 | if isinstance(body, basestring): 257 | req.write(body) 258 | else: 259 | for seg in body: 260 | req.write(seg) 261 | 262 | 263 | 264 | # --------------- Startup tools for CherryPy + mod_python --------------- # 265 | 266 | 267 | import os 268 | import re 269 | try: 270 | import subprocess 271 | def popen(fullcmd): 272 | p = subprocess.Popen(fullcmd, shell=True, 273 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 274 | close_fds=True) 275 | return p.stdout 276 | except ImportError: 277 | def popen(fullcmd): 278 | pipein, pipeout = os.popen4(fullcmd) 279 | return pipeout 280 | 281 | 282 | def read_process(cmd, args=""): 283 | fullcmd = "%s %s" % (cmd, args) 284 | pipeout = popen(fullcmd) 285 | try: 286 | firstline = pipeout.readline() 287 | if (re.search(ntob("(not recognized|No such file|not found)"), firstline, 288 | re.IGNORECASE)): 289 | raise IOError('%s must be on your system path.' % cmd) 290 | output = firstline + pipeout.read() 291 | finally: 292 | pipeout.close() 293 | return output 294 | 295 | 296 | class ModPythonServer(object): 297 | 298 | template = """ 299 | # Apache2 server configuration file for running CherryPy with mod_python. 300 | 301 | DocumentRoot "/" 302 | Listen %(port)s 303 | LoadModule python_module modules/mod_python.so 304 | 305 | 306 | SetHandler python-program 307 | PythonHandler %(handler)s 308 | PythonDebug On 309 | %(opts)s 310 | 311 | """ 312 | 313 | def __init__(self, loc="/", port=80, opts=None, apache_path="apache", 314 | handler="cherrypy._cpmodpy::handler"): 315 | self.loc = loc 316 | self.port = port 317 | self.opts = opts 318 | self.apache_path = apache_path 319 | self.handler = handler 320 | 321 | def start(self): 322 | opts = "".join([" PythonOption %s %s\n" % (k, v) 323 | for k, v in self.opts]) 324 | conf_data = self.template % {"port": self.port, 325 | "loc": self.loc, 326 | "opts": opts, 327 | "handler": self.handler, 328 | } 329 | 330 | mpconf = os.path.join(os.path.dirname(__file__), "cpmodpy.conf") 331 | f = open(mpconf, 'wb') 332 | try: 333 | f.write(conf_data) 334 | finally: 335 | f.close() 336 | 337 | response = read_process(self.apache_path, "-k start -f %s" % mpconf) 338 | self.ready = True 339 | return response 340 | 341 | def stop(self): 342 | os.popen("apache -k stop") 343 | self.ready = False 344 | 345 | -------------------------------------------------------------------------------- /cherrypy/_cpnative_server.py: -------------------------------------------------------------------------------- 1 | """Native adapter for serving CherryPy via its builtin server.""" 2 | 3 | import logging 4 | import sys 5 | 6 | import cherrypy 7 | from cherrypy._cpcompat import BytesIO 8 | from cherrypy._cperror import format_exc, bare_error 9 | from cherrypy.lib import httputil 10 | from cherrypy import wsgiserver 11 | 12 | 13 | class NativeGateway(wsgiserver.Gateway): 14 | 15 | recursive = False 16 | 17 | def respond(self): 18 | req = self.req 19 | try: 20 | # Obtain a Request object from CherryPy 21 | local = req.server.bind_addr 22 | local = httputil.Host(local[0], local[1], "") 23 | remote = req.conn.remote_addr, req.conn.remote_port 24 | remote = httputil.Host(remote[0], remote[1], "") 25 | 26 | scheme = req.scheme 27 | sn = cherrypy.tree.script_name(req.uri or "/") 28 | if sn is None: 29 | self.send_response('404 Not Found', [], ['']) 30 | else: 31 | app = cherrypy.tree.apps[sn] 32 | method = req.method 33 | path = req.path 34 | qs = req.qs or "" 35 | headers = req.inheaders.items() 36 | rfile = req.rfile 37 | prev = None 38 | 39 | try: 40 | redirections = [] 41 | while True: 42 | request, response = app.get_serving( 43 | local, remote, scheme, "HTTP/1.1") 44 | request.multithread = True 45 | request.multiprocess = False 46 | request.app = app 47 | request.prev = prev 48 | 49 | # Run the CherryPy Request object and obtain the response 50 | try: 51 | request.run(method, path, qs, req.request_protocol, headers, rfile) 52 | break 53 | except cherrypy.InternalRedirect: 54 | ir = sys.exc_info()[1] 55 | app.release_serving() 56 | prev = request 57 | 58 | if not self.recursive: 59 | if ir.path in redirections: 60 | raise RuntimeError("InternalRedirector visited the " 61 | "same URL twice: %r" % ir.path) 62 | else: 63 | # Add the *previous* path_info + qs to redirections. 64 | if qs: 65 | qs = "?" + qs 66 | redirections.append(sn + path + qs) 67 | 68 | # Munge environment and try again. 69 | method = "GET" 70 | path = ir.path 71 | qs = ir.query_string 72 | rfile = BytesIO() 73 | 74 | self.send_response( 75 | response.output_status, response.header_list, 76 | response.body) 77 | finally: 78 | app.release_serving() 79 | except: 80 | tb = format_exc() 81 | #print tb 82 | cherrypy.log(tb, 'NATIVE_ADAPTER', severity=logging.ERROR) 83 | s, h, b = bare_error() 84 | self.send_response(s, h, b) 85 | 86 | def send_response(self, status, headers, body): 87 | req = self.req 88 | 89 | # Set response status 90 | req.status = str(status or "500 Server Error") 91 | 92 | # Set response headers 93 | for header, value in headers: 94 | req.outheaders.append((header, value)) 95 | if (req.ready and not req.sent_headers): 96 | req.sent_headers = True 97 | req.send_headers() 98 | 99 | # Set response body 100 | for seg in body: 101 | req.write(seg) 102 | 103 | 104 | class CPHTTPServer(wsgiserver.HTTPServer): 105 | """Wrapper for wsgiserver.HTTPServer. 106 | 107 | wsgiserver has been designed to not reference CherryPy in any way, 108 | so that it can be used in other frameworks and applications. 109 | Therefore, we wrap it here, so we can apply some attributes 110 | from config -> cherrypy.server -> HTTPServer. 111 | """ 112 | 113 | def __init__(self, server_adapter=cherrypy.server): 114 | self.server_adapter = server_adapter 115 | 116 | server_name = (self.server_adapter.socket_host or 117 | self.server_adapter.socket_file or 118 | None) 119 | 120 | wsgiserver.HTTPServer.__init__( 121 | self, server_adapter.bind_addr, NativeGateway, 122 | minthreads=server_adapter.thread_pool, 123 | maxthreads=server_adapter.thread_pool_max, 124 | server_name=server_name) 125 | 126 | self.max_request_header_size = self.server_adapter.max_request_header_size or 0 127 | self.max_request_body_size = self.server_adapter.max_request_body_size or 0 128 | self.request_queue_size = self.server_adapter.socket_queue_size 129 | self.timeout = self.server_adapter.socket_timeout 130 | self.shutdown_timeout = self.server_adapter.shutdown_timeout 131 | self.protocol = self.server_adapter.protocol_version 132 | self.nodelay = self.server_adapter.nodelay 133 | 134 | ssl_module = self.server_adapter.ssl_module or 'pyopenssl' 135 | if self.server_adapter.ssl_context: 136 | adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) 137 | self.ssl_adapter = adapter_class( 138 | self.server_adapter.ssl_certificate, 139 | self.server_adapter.ssl_private_key, 140 | self.server_adapter.ssl_certificate_chain) 141 | self.ssl_adapter.context = self.server_adapter.ssl_context 142 | elif self.server_adapter.ssl_certificate: 143 | adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) 144 | self.ssl_adapter = adapter_class( 145 | self.server_adapter.ssl_certificate, 146 | self.server_adapter.ssl_private_key, 147 | self.server_adapter.ssl_certificate_chain) 148 | 149 | 150 | -------------------------------------------------------------------------------- /cherrypy/_cpserver.py: -------------------------------------------------------------------------------- 1 | """Manage HTTP servers with CherryPy.""" 2 | 3 | import warnings 4 | 5 | import cherrypy 6 | from cherrypy.lib import attributes 7 | from cherrypy._cpcompat import basestring, py3k 8 | 9 | # We import * because we want to export check_port 10 | # et al as attributes of this module. 11 | from cherrypy.process.servers import * 12 | 13 | 14 | class Server(ServerAdapter): 15 | """An adapter for an HTTP server. 16 | 17 | You can set attributes (like socket_host and socket_port) 18 | on *this* object (which is probably cherrypy.server), and call 19 | quickstart. For example:: 20 | 21 | cherrypy.server.socket_port = 80 22 | cherrypy.quickstart() 23 | """ 24 | 25 | socket_port = 8080 26 | """The TCP port on which to listen for connections.""" 27 | 28 | _socket_host = '127.0.0.1' 29 | def _get_socket_host(self): 30 | return self._socket_host 31 | def _set_socket_host(self, value): 32 | if value == '': 33 | raise ValueError("The empty string ('') is not an allowed value. " 34 | "Use '0.0.0.0' instead to listen on all active " 35 | "interfaces (INADDR_ANY).") 36 | self._socket_host = value 37 | socket_host = property(_get_socket_host, _set_socket_host, 38 | doc="""The hostname or IP address on which to listen for connections. 39 | 40 | Host values may be any IPv4 or IPv6 address, or any valid hostname. 41 | The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if 42 | your hosts file prefers IPv6). The string '0.0.0.0' is a special 43 | IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' 44 | is the similar IN6ADDR_ANY for IPv6. The empty string or None are 45 | not allowed.""") 46 | 47 | socket_file = None 48 | """If given, the name of the UNIX socket to use instead of TCP/IP. 49 | 50 | When this option is not None, the `socket_host` and `socket_port` options 51 | are ignored.""" 52 | 53 | socket_queue_size = 5 54 | """The 'backlog' argument to socket.listen(); specifies the maximum number 55 | of queued connections (default 5).""" 56 | 57 | socket_timeout = 10 58 | """The timeout in seconds for accepted connections (default 10).""" 59 | 60 | shutdown_timeout = 5 61 | """The time to wait for HTTP worker threads to clean up.""" 62 | 63 | protocol_version = 'HTTP/1.1' 64 | """The version string to write in the Status-Line of all HTTP responses, 65 | for example, "HTTP/1.1" (the default). Depending on the HTTP server used, 66 | this should also limit the supported features used in the response.""" 67 | 68 | thread_pool = 10 69 | """The number of worker threads to start up in the pool.""" 70 | 71 | thread_pool_max = -1 72 | """The maximum size of the worker-thread pool. Use -1 to indicate no limit.""" 73 | 74 | max_request_header_size = 500 * 1024 75 | """The maximum number of bytes allowable in the request headers. If exceeded, 76 | the HTTP server should return "413 Request Entity Too Large".""" 77 | 78 | max_request_body_size = 100 * 1024 * 1024 79 | """The maximum number of bytes allowable in the request body. If exceeded, 80 | the HTTP server should return "413 Request Entity Too Large".""" 81 | 82 | instance = None 83 | """If not None, this should be an HTTP server instance (such as 84 | CPWSGIServer) which cherrypy.server will control. Use this when you need 85 | more control over object instantiation than is available in the various 86 | configuration options.""" 87 | 88 | ssl_context = None 89 | """When using PyOpenSSL, an instance of SSL.Context.""" 90 | 91 | ssl_certificate = None 92 | """The filename of the SSL certificate to use.""" 93 | 94 | ssl_certificate_chain = None 95 | """When using PyOpenSSL, the certificate chain to pass to 96 | Context.load_verify_locations.""" 97 | 98 | ssl_private_key = None 99 | """The filename of the private key to use with SSL.""" 100 | 101 | if py3k: 102 | ssl_module = 'builtin' 103 | """The name of a registered SSL adaptation module to use with the builtin 104 | WSGI server. Builtin options are: 'builtin' (to use the SSL library built 105 | into recent versions of Python). You may also register your 106 | own classes in the wsgiserver.ssl_adapters dict.""" 107 | else: 108 | ssl_module = 'pyopenssl' 109 | """The name of a registered SSL adaptation module to use with the builtin 110 | WSGI server. Builtin options are 'builtin' (to use the SSL library built 111 | into recent versions of Python) and 'pyopenssl' (to use the PyOpenSSL 112 | project, which you must install separately). You may also register your 113 | own classes in the wsgiserver.ssl_adapters dict.""" 114 | 115 | statistics = False 116 | """Turns statistics-gathering on or off for aware HTTP servers.""" 117 | 118 | nodelay = True 119 | """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" 120 | 121 | wsgi_version = (1, 0) 122 | """The WSGI version tuple to use with the builtin WSGI server. 123 | The provided options are (1, 0) [which includes support for PEP 3333, 124 | which declares it covers WSGI version 1.0.1 but still mandates the 125 | wsgi.version (1, 0)] and ('u', 0), an experimental unicode version. 126 | You may create and register your own experimental versions of the WSGI 127 | protocol by adding custom classes to the wsgiserver.wsgi_gateways dict.""" 128 | 129 | def __init__(self): 130 | self.bus = cherrypy.engine 131 | self.httpserver = None 132 | self.interrupt = None 133 | self.running = False 134 | 135 | def httpserver_from_self(self, httpserver=None): 136 | """Return a (httpserver, bind_addr) pair based on self attributes.""" 137 | if httpserver is None: 138 | httpserver = self.instance 139 | if httpserver is None: 140 | from cherrypy import _cpwsgi_server 141 | httpserver = _cpwsgi_server.CPWSGIServer(self) 142 | if isinstance(httpserver, basestring): 143 | # Is anyone using this? Can I add an arg? 144 | httpserver = attributes(httpserver)(self) 145 | return httpserver, self.bind_addr 146 | 147 | def start(self): 148 | """Start the HTTP server.""" 149 | if not self.httpserver: 150 | self.httpserver, self.bind_addr = self.httpserver_from_self() 151 | ServerAdapter.start(self) 152 | start.priority = 75 153 | 154 | def _get_bind_addr(self): 155 | if self.socket_file: 156 | return self.socket_file 157 | if self.socket_host is None and self.socket_port is None: 158 | return None 159 | return (self.socket_host, self.socket_port) 160 | def _set_bind_addr(self, value): 161 | if value is None: 162 | self.socket_file = None 163 | self.socket_host = None 164 | self.socket_port = None 165 | elif isinstance(value, basestring): 166 | self.socket_file = value 167 | self.socket_host = None 168 | self.socket_port = None 169 | else: 170 | try: 171 | self.socket_host, self.socket_port = value 172 | self.socket_file = None 173 | except ValueError: 174 | raise ValueError("bind_addr must be a (host, port) tuple " 175 | "(for TCP sockets) or a string (for Unix " 176 | "domain sockets), not %r" % value) 177 | bind_addr = property(_get_bind_addr, _set_bind_addr, 178 | doc='A (host, port) tuple for TCP sockets or a str for Unix domain sockets.') 179 | 180 | def base(self): 181 | """Return the base (scheme://host[:port] or sock file) for this server.""" 182 | if self.socket_file: 183 | return self.socket_file 184 | 185 | host = self.socket_host 186 | if host in ('0.0.0.0', '::'): 187 | # 0.0.0.0 is INADDR_ANY and :: is IN6ADDR_ANY. 188 | # Look up the host name, which should be the 189 | # safest thing to spit out in a URL. 190 | import socket 191 | host = socket.gethostname() 192 | 193 | port = self.socket_port 194 | 195 | if self.ssl_certificate: 196 | scheme = "https" 197 | if port != 443: 198 | host += ":%s" % port 199 | else: 200 | scheme = "http" 201 | if port != 80: 202 | host += ":%s" % port 203 | 204 | return "%s://%s" % (scheme, host) 205 | 206 | -------------------------------------------------------------------------------- /cherrypy/_cpthreadinglocal.py: -------------------------------------------------------------------------------- 1 | # This is a backport of Python-2.4's threading.local() implementation 2 | 3 | """Thread-local objects 4 | 5 | (Note that this module provides a Python version of thread 6 | threading.local class. Depending on the version of Python you're 7 | using, there may be a faster one available. You should always import 8 | the local class from threading.) 9 | 10 | Thread-local objects support the management of thread-local data. 11 | If you have data that you want to be local to a thread, simply create 12 | a thread-local object and use its attributes: 13 | 14 | >>> mydata = local() 15 | >>> mydata.number = 42 16 | >>> mydata.number 17 | 42 18 | 19 | You can also access the local-object's dictionary: 20 | 21 | >>> mydata.__dict__ 22 | {'number': 42} 23 | >>> mydata.__dict__.setdefault('widgets', []) 24 | [] 25 | >>> mydata.widgets 26 | [] 27 | 28 | What's important about thread-local objects is that their data are 29 | local to a thread. If we access the data in a different thread: 30 | 31 | >>> log = [] 32 | >>> def f(): 33 | ... items = mydata.__dict__.items() 34 | ... items.sort() 35 | ... log.append(items) 36 | ... mydata.number = 11 37 | ... log.append(mydata.number) 38 | 39 | >>> import threading 40 | >>> thread = threading.Thread(target=f) 41 | >>> thread.start() 42 | >>> thread.join() 43 | >>> log 44 | [[], 11] 45 | 46 | we get different data. Furthermore, changes made in the other thread 47 | don't affect data seen in this thread: 48 | 49 | >>> mydata.number 50 | 42 51 | 52 | Of course, values you get from a local object, including a __dict__ 53 | attribute, are for whatever thread was current at the time the 54 | attribute was read. For that reason, you generally don't want to save 55 | these values across threads, as they apply only to the thread they 56 | came from. 57 | 58 | You can create custom local objects by subclassing the local class: 59 | 60 | >>> class MyLocal(local): 61 | ... number = 2 62 | ... initialized = False 63 | ... def __init__(self, **kw): 64 | ... if self.initialized: 65 | ... raise SystemError('__init__ called too many times') 66 | ... self.initialized = True 67 | ... self.__dict__.update(kw) 68 | ... def squared(self): 69 | ... return self.number ** 2 70 | 71 | This can be useful to support default values, methods and 72 | initialization. Note that if you define an __init__ method, it will be 73 | called each time the local object is used in a separate thread. This 74 | is necessary to initialize each thread's dictionary. 75 | 76 | Now if we create a local object: 77 | 78 | >>> mydata = MyLocal(color='red') 79 | 80 | Now we have a default number: 81 | 82 | >>> mydata.number 83 | 2 84 | 85 | an initial color: 86 | 87 | >>> mydata.color 88 | 'red' 89 | >>> del mydata.color 90 | 91 | And a method that operates on the data: 92 | 93 | >>> mydata.squared() 94 | 4 95 | 96 | As before, we can access the data in a separate thread: 97 | 98 | >>> log = [] 99 | >>> thread = threading.Thread(target=f) 100 | >>> thread.start() 101 | >>> thread.join() 102 | >>> log 103 | [[('color', 'red'), ('initialized', True)], 11] 104 | 105 | without affecting this thread's data: 106 | 107 | >>> mydata.number 108 | 2 109 | >>> mydata.color 110 | Traceback (most recent call last): 111 | ... 112 | AttributeError: 'MyLocal' object has no attribute 'color' 113 | 114 | Note that subclasses can define slots, but they are not thread 115 | local. They are shared across threads: 116 | 117 | >>> class MyLocal(local): 118 | ... __slots__ = 'number' 119 | 120 | >>> mydata = MyLocal() 121 | >>> mydata.number = 42 122 | >>> mydata.color = 'red' 123 | 124 | So, the separate thread: 125 | 126 | >>> thread = threading.Thread(target=f) 127 | >>> thread.start() 128 | >>> thread.join() 129 | 130 | affects what we see: 131 | 132 | >>> mydata.number 133 | 11 134 | 135 | >>> del mydata 136 | """ 137 | 138 | # Threading import is at end 139 | 140 | class _localbase(object): 141 | __slots__ = '_local__key', '_local__args', '_local__lock' 142 | 143 | def __new__(cls, *args, **kw): 144 | self = object.__new__(cls) 145 | key = 'thread.local.' + str(id(self)) 146 | object.__setattr__(self, '_local__key', key) 147 | object.__setattr__(self, '_local__args', (args, kw)) 148 | object.__setattr__(self, '_local__lock', RLock()) 149 | 150 | if args or kw and (cls.__init__ is object.__init__): 151 | raise TypeError("Initialization arguments are not supported") 152 | 153 | # We need to create the thread dict in anticipation of 154 | # __init__ being called, to make sure we don't call it 155 | # again ourselves. 156 | dict = object.__getattribute__(self, '__dict__') 157 | currentThread().__dict__[key] = dict 158 | 159 | return self 160 | 161 | def _patch(self): 162 | key = object.__getattribute__(self, '_local__key') 163 | d = currentThread().__dict__.get(key) 164 | if d is None: 165 | d = {} 166 | currentThread().__dict__[key] = d 167 | object.__setattr__(self, '__dict__', d) 168 | 169 | # we have a new instance dict, so call out __init__ if we have 170 | # one 171 | cls = type(self) 172 | if cls.__init__ is not object.__init__: 173 | args, kw = object.__getattribute__(self, '_local__args') 174 | cls.__init__(self, *args, **kw) 175 | else: 176 | object.__setattr__(self, '__dict__', d) 177 | 178 | class local(_localbase): 179 | 180 | def __getattribute__(self, name): 181 | lock = object.__getattribute__(self, '_local__lock') 182 | lock.acquire() 183 | try: 184 | _patch(self) 185 | return object.__getattribute__(self, name) 186 | finally: 187 | lock.release() 188 | 189 | def __setattr__(self, name, value): 190 | lock = object.__getattribute__(self, '_local__lock') 191 | lock.acquire() 192 | try: 193 | _patch(self) 194 | return object.__setattr__(self, name, value) 195 | finally: 196 | lock.release() 197 | 198 | def __delattr__(self, name): 199 | lock = object.__getattribute__(self, '_local__lock') 200 | lock.acquire() 201 | try: 202 | _patch(self) 203 | return object.__delattr__(self, name) 204 | finally: 205 | lock.release() 206 | 207 | 208 | def __del__(): 209 | threading_enumerate = enumerate 210 | __getattribute__ = object.__getattribute__ 211 | 212 | def __del__(self): 213 | key = __getattribute__(self, '_local__key') 214 | 215 | try: 216 | threads = list(threading_enumerate()) 217 | except: 218 | # if enumerate fails, as it seems to do during 219 | # shutdown, we'll skip cleanup under the assumption 220 | # that there is nothing to clean up 221 | return 222 | 223 | for thread in threads: 224 | try: 225 | __dict__ = thread.__dict__ 226 | except AttributeError: 227 | # Thread is dying, rest in peace 228 | continue 229 | 230 | if key in __dict__: 231 | try: 232 | del __dict__[key] 233 | except KeyError: 234 | pass # didn't have anything in this thread 235 | 236 | return __del__ 237 | __del__ = __del__() 238 | 239 | from threading import currentThread, enumerate, RLock 240 | -------------------------------------------------------------------------------- /cherrypy/_cptree.py: -------------------------------------------------------------------------------- 1 | """CherryPy Application and Tree objects.""" 2 | 3 | import os 4 | import sys 5 | 6 | import cherrypy 7 | from cherrypy._cpcompat import ntou, py3k 8 | from cherrypy import _cpconfig, _cplogging, _cprequest, _cpwsgi, tools 9 | from cherrypy.lib import httputil 10 | 11 | 12 | class Application(object): 13 | """A CherryPy Application. 14 | 15 | Servers and gateways should not instantiate Request objects directly. 16 | Instead, they should ask an Application object for a request object. 17 | 18 | An instance of this class may also be used as a WSGI callable 19 | (WSGI application object) for itself. 20 | """ 21 | 22 | root = None 23 | """The top-most container of page handlers for this app. Handlers should 24 | be arranged in a hierarchy of attributes, matching the expected URI 25 | hierarchy; the default dispatcher then searches this hierarchy for a 26 | matching handler. When using a dispatcher other than the default, 27 | this value may be None.""" 28 | 29 | config = {} 30 | """A dict of {path: pathconf} pairs, where 'pathconf' is itself a dict 31 | of {key: value} pairs.""" 32 | 33 | namespaces = _cpconfig.NamespaceSet() 34 | toolboxes = {'tools': cherrypy.tools} 35 | 36 | log = None 37 | """A LogManager instance. See _cplogging.""" 38 | 39 | wsgiapp = None 40 | """A CPWSGIApp instance. See _cpwsgi.""" 41 | 42 | request_class = _cprequest.Request 43 | response_class = _cprequest.Response 44 | 45 | relative_urls = False 46 | 47 | def __init__(self, root, script_name="", config=None): 48 | self.log = _cplogging.LogManager(id(self), cherrypy.log.logger_root) 49 | self.root = root 50 | self.script_name = script_name 51 | self.wsgiapp = _cpwsgi.CPWSGIApp(self) 52 | 53 | self.namespaces = self.namespaces.copy() 54 | self.namespaces["log"] = lambda k, v: setattr(self.log, k, v) 55 | self.namespaces["wsgi"] = self.wsgiapp.namespace_handler 56 | 57 | self.config = self.__class__.config.copy() 58 | if config: 59 | self.merge(config) 60 | 61 | def __repr__(self): 62 | return "%s.%s(%r, %r)" % (self.__module__, self.__class__.__name__, 63 | self.root, self.script_name) 64 | 65 | script_name_doc = """The URI "mount point" for this app. A mount point is that portion of 66 | the URI which is constant for all URIs that are serviced by this 67 | application; it does not include scheme, host, or proxy ("virtual host") 68 | portions of the URI. 69 | 70 | For example, if script_name is "/my/cool/app", then the URL 71 | "http://www.example.com/my/cool/app/page1" might be handled by a 72 | "page1" method on the root object. 73 | 74 | The value of script_name MUST NOT end in a slash. If the script_name 75 | refers to the root of the URI, it MUST be an empty string (not "/"). 76 | 77 | If script_name is explicitly set to None, then the script_name will be 78 | provided for each call from request.wsgi_environ['SCRIPT_NAME']. 79 | """ 80 | def _get_script_name(self): 81 | if self._script_name is None: 82 | # None signals that the script name should be pulled from WSGI environ. 83 | return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") 84 | return self._script_name 85 | def _set_script_name(self, value): 86 | if value: 87 | value = value.rstrip("/") 88 | self._script_name = value 89 | script_name = property(fget=_get_script_name, fset=_set_script_name, 90 | doc=script_name_doc) 91 | 92 | def merge(self, config): 93 | """Merge the given config into self.config.""" 94 | _cpconfig.merge(self.config, config) 95 | 96 | # Handle namespaces specified in config. 97 | self.namespaces(self.config.get("/", {})) 98 | 99 | def find_config(self, path, key, default=None): 100 | """Return the most-specific value for key along path, or default.""" 101 | trail = path or "/" 102 | while trail: 103 | nodeconf = self.config.get(trail, {}) 104 | 105 | if key in nodeconf: 106 | return nodeconf[key] 107 | 108 | lastslash = trail.rfind("/") 109 | if lastslash == -1: 110 | break 111 | elif lastslash == 0 and trail != "/": 112 | trail = "/" 113 | else: 114 | trail = trail[:lastslash] 115 | 116 | return default 117 | 118 | def get_serving(self, local, remote, scheme, sproto): 119 | """Create and return a Request and Response object.""" 120 | req = self.request_class(local, remote, scheme, sproto) 121 | req.app = self 122 | 123 | for name, toolbox in self.toolboxes.items(): 124 | req.namespaces[name] = toolbox 125 | 126 | resp = self.response_class() 127 | cherrypy.serving.load(req, resp) 128 | cherrypy.engine.publish('acquire_thread') 129 | cherrypy.engine.publish('before_request') 130 | 131 | return req, resp 132 | 133 | def release_serving(self): 134 | """Release the current serving (request and response).""" 135 | req = cherrypy.serving.request 136 | 137 | cherrypy.engine.publish('after_request') 138 | 139 | try: 140 | req.close() 141 | except: 142 | cherrypy.log(traceback=True, severity=40) 143 | 144 | cherrypy.serving.clear() 145 | 146 | def __call__(self, environ, start_response): 147 | return self.wsgiapp(environ, start_response) 148 | 149 | 150 | class Tree(object): 151 | """A registry of CherryPy applications, mounted at diverse points. 152 | 153 | An instance of this class may also be used as a WSGI callable 154 | (WSGI application object), in which case it dispatches to all 155 | mounted apps. 156 | """ 157 | 158 | apps = {} 159 | """ 160 | A dict of the form {script name: application}, where "script name" 161 | is a string declaring the URI mount point (no trailing slash), and 162 | "application" is an instance of cherrypy.Application (or an arbitrary 163 | WSGI callable if you happen to be using a WSGI server).""" 164 | 165 | def __init__(self): 166 | self.apps = {} 167 | 168 | def mount(self, root, script_name="", config=None): 169 | """Mount a new app from a root object, script_name, and config. 170 | 171 | root 172 | An instance of a "controller class" (a collection of page 173 | handler methods) which represents the root of the application. 174 | This may also be an Application instance, or None if using 175 | a dispatcher other than the default. 176 | 177 | script_name 178 | A string containing the "mount point" of the application. 179 | This should start with a slash, and be the path portion of the 180 | URL at which to mount the given root. For example, if root.index() 181 | will handle requests to "http://www.example.com:8080/dept/app1/", 182 | then the script_name argument would be "/dept/app1". 183 | 184 | It MUST NOT end in a slash. If the script_name refers to the 185 | root of the URI, it MUST be an empty string (not "/"). 186 | 187 | config 188 | A file or dict containing application config. 189 | """ 190 | if script_name is None: 191 | raise TypeError( 192 | "The 'script_name' argument may not be None. Application " 193 | "objects may, however, possess a script_name of None (in " 194 | "order to inpect the WSGI environ for SCRIPT_NAME upon each " 195 | "request). You cannot mount such Applications on this Tree; " 196 | "you must pass them to a WSGI server interface directly.") 197 | 198 | # Next line both 1) strips trailing slash and 2) maps "/" -> "". 199 | script_name = script_name.rstrip("/") 200 | 201 | if isinstance(root, Application): 202 | app = root 203 | if script_name != "" and script_name != app.script_name: 204 | raise ValueError("Cannot specify a different script name and " 205 | "pass an Application instance to cherrypy.mount") 206 | script_name = app.script_name 207 | else: 208 | app = Application(root, script_name) 209 | 210 | # If mounted at "", add favicon.ico 211 | if (script_name == "" and root is not None 212 | and not hasattr(root, "favicon_ico")): 213 | favicon = os.path.join(os.getcwd(), os.path.dirname(__file__), 214 | "favicon.ico") 215 | root.favicon_ico = tools.staticfile.handler(favicon) 216 | 217 | if config: 218 | app.merge(config) 219 | 220 | self.apps[script_name] = app 221 | 222 | return app 223 | 224 | def graft(self, wsgi_callable, script_name=""): 225 | """Mount a wsgi callable at the given script_name.""" 226 | # Next line both 1) strips trailing slash and 2) maps "/" -> "". 227 | script_name = script_name.rstrip("/") 228 | self.apps[script_name] = wsgi_callable 229 | 230 | def script_name(self, path=None): 231 | """The script_name of the app at the given path, or None. 232 | 233 | If path is None, cherrypy.request is used. 234 | """ 235 | if path is None: 236 | try: 237 | request = cherrypy.serving.request 238 | path = httputil.urljoin(request.script_name, 239 | request.path_info) 240 | except AttributeError: 241 | return None 242 | 243 | while True: 244 | if path in self.apps: 245 | return path 246 | 247 | if path == "": 248 | return None 249 | 250 | # Move one node up the tree and try again. 251 | path = path[:path.rfind("/")] 252 | 253 | def __call__(self, environ, start_response): 254 | # If you're calling this, then you're probably setting SCRIPT_NAME 255 | # to '' (some WSGI servers always set SCRIPT_NAME to ''). 256 | # Try to look up the app using the full path. 257 | env1x = environ 258 | if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): 259 | env1x = _cpwsgi.downgrade_wsgi_ux_to_1x(environ) 260 | path = httputil.urljoin(env1x.get('SCRIPT_NAME', ''), 261 | env1x.get('PATH_INFO', '')) 262 | sn = self.script_name(path or "/") 263 | if sn is None: 264 | start_response('404 Not Found', []) 265 | return [] 266 | 267 | app = self.apps[sn] 268 | 269 | # Correct the SCRIPT_NAME and PATH_INFO environ entries. 270 | environ = environ.copy() 271 | if not py3k: 272 | if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): 273 | # Python 2/WSGI u.0: all strings MUST be of type unicode 274 | enc = environ[ntou('wsgi.url_encoding')] 275 | environ[ntou('SCRIPT_NAME')] = sn.decode(enc) 276 | environ[ntou('PATH_INFO')] = path[len(sn.rstrip("/")):].decode(enc) 277 | else: 278 | # Python 2/WSGI 1.x: all strings MUST be of type str 279 | environ['SCRIPT_NAME'] = sn 280 | environ['PATH_INFO'] = path[len(sn.rstrip("/")):] 281 | else: 282 | if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): 283 | # Python 3/WSGI u.0: all strings MUST be full unicode 284 | environ['SCRIPT_NAME'] = sn 285 | environ['PATH_INFO'] = path[len(sn.rstrip("/")):] 286 | else: 287 | # Python 3/WSGI 1.x: all strings MUST be ISO-8859-1 str 288 | environ['SCRIPT_NAME'] = sn.encode('utf-8').decode('ISO-8859-1') 289 | environ['PATH_INFO'] = path[len(sn.rstrip("/")):].encode('utf-8').decode('ISO-8859-1') 290 | return app(environ, start_response) 291 | -------------------------------------------------------------------------------- /cherrypy/_cpwsgi_server.py: -------------------------------------------------------------------------------- 1 | """WSGI server interface (see PEP 333). This adds some CP-specific bits to 2 | the framework-agnostic wsgiserver package. 3 | """ 4 | import sys 5 | 6 | import cherrypy 7 | from cherrypy import wsgiserver 8 | 9 | 10 | class CPWSGIServer(wsgiserver.CherryPyWSGIServer): 11 | """Wrapper for wsgiserver.CherryPyWSGIServer. 12 | 13 | wsgiserver has been designed to not reference CherryPy in any way, 14 | so that it can be used in other frameworks and applications. Therefore, 15 | we wrap it here, so we can set our own mount points from cherrypy.tree 16 | and apply some attributes from config -> cherrypy.server -> wsgiserver. 17 | """ 18 | 19 | def __init__(self, server_adapter=cherrypy.server): 20 | self.server_adapter = server_adapter 21 | self.max_request_header_size = self.server_adapter.max_request_header_size or 0 22 | self.max_request_body_size = self.server_adapter.max_request_body_size or 0 23 | 24 | server_name = (self.server_adapter.socket_host or 25 | self.server_adapter.socket_file or 26 | None) 27 | 28 | self.wsgi_version = self.server_adapter.wsgi_version 29 | s = wsgiserver.CherryPyWSGIServer 30 | s.__init__(self, server_adapter.bind_addr, cherrypy.tree, 31 | self.server_adapter.thread_pool, 32 | server_name, 33 | max = self.server_adapter.thread_pool_max, 34 | request_queue_size = self.server_adapter.socket_queue_size, 35 | timeout = self.server_adapter.socket_timeout, 36 | shutdown_timeout = self.server_adapter.shutdown_timeout, 37 | ) 38 | self.protocol = self.server_adapter.protocol_version 39 | self.nodelay = self.server_adapter.nodelay 40 | 41 | if sys.version_info >= (3, 0): 42 | ssl_module = self.server_adapter.ssl_module or 'builtin' 43 | else: 44 | ssl_module = self.server_adapter.ssl_module or 'pyopenssl' 45 | if self.server_adapter.ssl_context: 46 | adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) 47 | self.ssl_adapter = adapter_class( 48 | self.server_adapter.ssl_certificate, 49 | self.server_adapter.ssl_private_key, 50 | self.server_adapter.ssl_certificate_chain) 51 | self.ssl_adapter.context = self.server_adapter.ssl_context 52 | elif self.server_adapter.ssl_certificate: 53 | adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) 54 | self.ssl_adapter = adapter_class( 55 | self.server_adapter.ssl_certificate, 56 | self.server_adapter.ssl_private_key, 57 | self.server_adapter.ssl_certificate_chain) 58 | 59 | self.stats['Enabled'] = getattr(self.server_adapter, 'statistics', False) 60 | 61 | def error_log(self, msg="", level=20, traceback=False): 62 | cherrypy.engine.log(msg, level, traceback) 63 | 64 | -------------------------------------------------------------------------------- /cherrypy/cherryd: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | """The CherryPy daemon.""" 3 | 4 | import sys 5 | 6 | import cherrypy 7 | from cherrypy.process import plugins, servers 8 | from cherrypy import Application 9 | 10 | def start(configfiles=None, daemonize=False, environment=None, 11 | fastcgi=False, scgi=False, pidfile=None, imports=None, 12 | cgi=False): 13 | """Subscribe all engine plugins and start the engine.""" 14 | sys.path = [''] + sys.path 15 | for i in imports or []: 16 | exec("import %s" % i) 17 | 18 | for c in configfiles or []: 19 | cherrypy.config.update(c) 20 | # If there's only one app mounted, merge config into it. 21 | if len(cherrypy.tree.apps) == 1: 22 | for app in cherrypy.tree.apps.values(): 23 | if isinstance(app, Application): 24 | app.merge(c) 25 | 26 | engine = cherrypy.engine 27 | 28 | if environment is not None: 29 | cherrypy.config.update({'environment': environment}) 30 | 31 | # Only daemonize if asked to. 32 | if daemonize: 33 | # Don't print anything to stdout/sterr. 34 | cherrypy.config.update({'log.screen': False}) 35 | plugins.Daemonizer(engine).subscribe() 36 | 37 | if pidfile: 38 | plugins.PIDFile(engine, pidfile).subscribe() 39 | 40 | if hasattr(engine, "signal_handler"): 41 | engine.signal_handler.subscribe() 42 | if hasattr(engine, "console_control_handler"): 43 | engine.console_control_handler.subscribe() 44 | 45 | if (fastcgi and (scgi or cgi)) or (scgi and cgi): 46 | cherrypy.log.error("You may only specify one of the cgi, fastcgi, and " 47 | "scgi options.", 'ENGINE') 48 | sys.exit(1) 49 | elif fastcgi or scgi or cgi: 50 | # Turn off autoreload when using *cgi. 51 | cherrypy.config.update({'engine.autoreload_on': False}) 52 | # Turn off the default HTTP server (which is subscribed by default). 53 | cherrypy.server.unsubscribe() 54 | 55 | addr = cherrypy.server.bind_addr 56 | if fastcgi: 57 | f = servers.FlupFCGIServer(application=cherrypy.tree, 58 | bindAddress=addr) 59 | elif scgi: 60 | f = servers.FlupSCGIServer(application=cherrypy.tree, 61 | bindAddress=addr) 62 | else: 63 | f = servers.FlupCGIServer(application=cherrypy.tree, 64 | bindAddress=addr) 65 | s = servers.ServerAdapter(engine, httpserver=f, bind_addr=addr) 66 | s.subscribe() 67 | 68 | # Always start the engine; this will start all other services 69 | try: 70 | engine.start() 71 | except: 72 | # Assume the error has been logged already via bus.log. 73 | sys.exit(1) 74 | else: 75 | engine.block() 76 | 77 | 78 | if __name__ == '__main__': 79 | from optparse import OptionParser 80 | 81 | p = OptionParser() 82 | p.add_option('-c', '--config', action="append", dest='config', 83 | help="specify config file(s)") 84 | p.add_option('-d', action="store_true", dest='daemonize', 85 | help="run the server as a daemon") 86 | p.add_option('-e', '--environment', dest='environment', default=None, 87 | help="apply the given config environment") 88 | p.add_option('-f', action="store_true", dest='fastcgi', 89 | help="start a fastcgi server instead of the default HTTP server") 90 | p.add_option('-s', action="store_true", dest='scgi', 91 | help="start a scgi server instead of the default HTTP server") 92 | p.add_option('-x', action="store_true", dest='cgi', 93 | help="start a cgi server instead of the default HTTP server") 94 | p.add_option('-i', '--import', action="append", dest='imports', 95 | help="specify modules to import") 96 | p.add_option('-p', '--pidfile', dest='pidfile', default=None, 97 | help="store the process id in the given file") 98 | p.add_option('-P', '--Path', action="append", dest='Path', 99 | help="add the given paths to sys.path") 100 | options, args = p.parse_args() 101 | 102 | if options.Path: 103 | for p in options.Path: 104 | sys.path.insert(0, p) 105 | 106 | start(options.config, options.daemonize, 107 | options.environment, options.fastcgi, options.scgi, 108 | options.pidfile, options.imports, options.cgi) 109 | 110 | -------------------------------------------------------------------------------- /cherrypy/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/four-o-four/omniverse/19219fb41fa83e7ce66b1864ba564a88333e0397/cherrypy/favicon.ico -------------------------------------------------------------------------------- /cherrypy/lib/__init__.py: -------------------------------------------------------------------------------- 1 | """CherryPy Library""" 2 | 3 | # Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 4 | from cherrypy.lib.reprconf import unrepr, modules, attributes 5 | 6 | class file_generator(object): 7 | """Yield the given input (a file object) in chunks (default 64k). (Core)""" 8 | 9 | def __init__(self, input, chunkSize=65536): 10 | self.input = input 11 | self.chunkSize = chunkSize 12 | 13 | def __iter__(self): 14 | return self 15 | 16 | def __next__(self): 17 | chunk = self.input.read(self.chunkSize) 18 | if chunk: 19 | return chunk 20 | else: 21 | if hasattr(self.input, 'close'): 22 | self.input.close() 23 | raise StopIteration() 24 | next = __next__ 25 | 26 | def file_generator_limited(fileobj, count, chunk_size=65536): 27 | """Yield the given file object in chunks, stopping after `count` 28 | bytes has been emitted. Default chunk size is 64kB. (Core) 29 | """ 30 | remaining = count 31 | while remaining > 0: 32 | chunk = fileobj.read(min(chunk_size, remaining)) 33 | chunklen = len(chunk) 34 | if chunklen == 0: 35 | return 36 | remaining -= chunklen 37 | yield chunk 38 | 39 | def set_vary_header(response, header_name): 40 | "Add a Vary header to a response" 41 | varies = response.headers.get("Vary", "") 42 | varies = [x.strip() for x in varies.split(",") if x.strip()] 43 | if header_name not in varies: 44 | varies.append(header_name) 45 | response.headers['Vary'] = ", ".join(varies) 46 | -------------------------------------------------------------------------------- /cherrypy/lib/auth.py: -------------------------------------------------------------------------------- 1 | import cherrypy 2 | from cherrypy.lib import httpauth 3 | 4 | 5 | def check_auth(users, encrypt=None, realm=None): 6 | """If an authorization header contains credentials, return True, else False.""" 7 | request = cherrypy.serving.request 8 | if 'authorization' in request.headers: 9 | # make sure the provided credentials are correctly set 10 | ah = httpauth.parseAuthorization(request.headers['authorization']) 11 | if ah is None: 12 | raise cherrypy.HTTPError(400, 'Bad Request') 13 | 14 | if not encrypt: 15 | encrypt = httpauth.DIGEST_AUTH_ENCODERS[httpauth.MD5] 16 | 17 | if hasattr(users, '__call__'): 18 | try: 19 | # backward compatibility 20 | users = users() # expect it to return a dictionary 21 | 22 | if not isinstance(users, dict): 23 | raise ValueError("Authentication users must be a dictionary") 24 | 25 | # fetch the user password 26 | password = users.get(ah["username"], None) 27 | except TypeError: 28 | # returns a password (encrypted or clear text) 29 | password = users(ah["username"]) 30 | else: 31 | if not isinstance(users, dict): 32 | raise ValueError("Authentication users must be a dictionary") 33 | 34 | # fetch the user password 35 | password = users.get(ah["username"], None) 36 | 37 | # validate the authorization by re-computing it here 38 | # and compare it with what the user-agent provided 39 | if httpauth.checkResponse(ah, password, method=request.method, 40 | encrypt=encrypt, realm=realm): 41 | request.login = ah["username"] 42 | return True 43 | 44 | request.login = False 45 | return False 46 | 47 | def basic_auth(realm, users, encrypt=None, debug=False): 48 | """If auth fails, raise 401 with a basic authentication header. 49 | 50 | realm 51 | A string containing the authentication realm. 52 | 53 | users 54 | A dict of the form: {username: password} or a callable returning a dict. 55 | 56 | encrypt 57 | callable used to encrypt the password returned from the user-agent. 58 | if None it defaults to a md5 encryption. 59 | 60 | """ 61 | if check_auth(users, encrypt): 62 | if debug: 63 | cherrypy.log('Auth successful', 'TOOLS.BASIC_AUTH') 64 | return 65 | 66 | # inform the user-agent this path is protected 67 | cherrypy.serving.response.headers['www-authenticate'] = httpauth.basicAuth(realm) 68 | 69 | raise cherrypy.HTTPError(401, "You are not authorized to access that resource") 70 | 71 | def digest_auth(realm, users, debug=False): 72 | """If auth fails, raise 401 with a digest authentication header. 73 | 74 | realm 75 | A string containing the authentication realm. 76 | users 77 | A dict of the form: {username: password} or a callable returning a dict. 78 | """ 79 | if check_auth(users, realm=realm): 80 | if debug: 81 | cherrypy.log('Auth successful', 'TOOLS.DIGEST_AUTH') 82 | return 83 | 84 | # inform the user-agent this path is protected 85 | cherrypy.serving.response.headers['www-authenticate'] = httpauth.digestAuth(realm) 86 | 87 | raise cherrypy.HTTPError(401, "You are not authorized to access that resource") 88 | -------------------------------------------------------------------------------- /cherrypy/lib/auth_basic.py: -------------------------------------------------------------------------------- 1 | # This file is part of CherryPy 2 | # -*- coding: utf-8 -*- 3 | # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 4 | 5 | __doc__ = """This module provides a CherryPy 3.x tool which implements 6 | the server-side of HTTP Basic Access Authentication, as described in :rfc:`2617`. 7 | 8 | Example usage, using the built-in checkpassword_dict function which uses a dict 9 | as the credentials store:: 10 | 11 | userpassdict = {'bird' : 'bebop', 'ornette' : 'wayout'} 12 | checkpassword = cherrypy.lib.auth_basic.checkpassword_dict(userpassdict) 13 | basic_auth = {'tools.auth_basic.on': True, 14 | 'tools.auth_basic.realm': 'earth', 15 | 'tools.auth_basic.checkpassword': checkpassword, 16 | } 17 | app_config = { '/' : basic_auth } 18 | 19 | """ 20 | 21 | __author__ = 'visteya' 22 | __date__ = 'April 2009' 23 | 24 | import binascii 25 | from cherrypy._cpcompat import base64_decode 26 | import cherrypy 27 | 28 | 29 | def checkpassword_dict(user_password_dict): 30 | """Returns a checkpassword function which checks credentials 31 | against a dictionary of the form: {username : password}. 32 | 33 | If you want a simple dictionary-based authentication scheme, use 34 | checkpassword_dict(my_credentials_dict) as the value for the 35 | checkpassword argument to basic_auth(). 36 | """ 37 | def checkpassword(realm, user, password): 38 | p = user_password_dict.get(user) 39 | return p and p == password or False 40 | 41 | return checkpassword 42 | 43 | 44 | def basic_auth(realm, checkpassword, debug=False): 45 | """A CherryPy tool which hooks at before_handler to perform 46 | HTTP Basic Access Authentication, as specified in :rfc:`2617`. 47 | 48 | If the request has an 'authorization' header with a 'Basic' scheme, this 49 | tool attempts to authenticate the credentials supplied in that header. If 50 | the request has no 'authorization' header, or if it does but the scheme is 51 | not 'Basic', or if authentication fails, the tool sends a 401 response with 52 | a 'WWW-Authenticate' Basic header. 53 | 54 | realm 55 | A string containing the authentication realm. 56 | 57 | checkpassword 58 | A callable which checks the authentication credentials. 59 | Its signature is checkpassword(realm, username, password). where 60 | username and password are the values obtained from the request's 61 | 'authorization' header. If authentication succeeds, checkpassword 62 | returns True, else it returns False. 63 | 64 | """ 65 | 66 | if '"' in realm: 67 | raise ValueError('Realm cannot contain the " (quote) character.') 68 | request = cherrypy.serving.request 69 | 70 | auth_header = request.headers.get('authorization') 71 | if auth_header is not None: 72 | try: 73 | scheme, params = auth_header.split(' ', 1) 74 | if scheme.lower() == 'basic': 75 | username, password = base64_decode(params).split(':', 1) 76 | if checkpassword(realm, username, password): 77 | if debug: 78 | cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') 79 | request.login = username 80 | return # successful authentication 81 | except (ValueError, binascii.Error): # split() error, base64.decodestring() error 82 | raise cherrypy.HTTPError(400, 'Bad Request') 83 | 84 | # Respond with 401 status and a WWW-Authenticate header 85 | cherrypy.serving.response.headers['www-authenticate'] = 'Basic realm="%s"' % realm 86 | raise cherrypy.HTTPError(401, "You are not authorized to access that resource") 87 | 88 | -------------------------------------------------------------------------------- /cherrypy/lib/gctools.py: -------------------------------------------------------------------------------- 1 | import gc 2 | import inspect 3 | import os 4 | import sys 5 | import time 6 | 7 | try: 8 | import objgraph 9 | except ImportError: 10 | objgraph = None 11 | 12 | import cherrypy 13 | from cherrypy import _cprequest, _cpwsgi 14 | from cherrypy.process.plugins import SimplePlugin 15 | 16 | 17 | class ReferrerTree(object): 18 | """An object which gathers all referrers of an object to a given depth.""" 19 | 20 | peek_length = 40 21 | 22 | def __init__(self, ignore=None, maxdepth=2, maxparents=10): 23 | self.ignore = ignore or [] 24 | self.ignore.append(inspect.currentframe().f_back) 25 | self.maxdepth = maxdepth 26 | self.maxparents = maxparents 27 | 28 | def ascend(self, obj, depth=1): 29 | """Return a nested list containing referrers of the given object.""" 30 | depth += 1 31 | parents = [] 32 | 33 | # Gather all referrers in one step to minimize 34 | # cascading references due to repr() logic. 35 | refs = gc.get_referrers(obj) 36 | self.ignore.append(refs) 37 | if len(refs) > self.maxparents: 38 | return [("[%s referrers]" % len(refs), [])] 39 | 40 | try: 41 | ascendcode = self.ascend.__code__ 42 | except AttributeError: 43 | ascendcode = self.ascend.im_func.func_code 44 | for parent in refs: 45 | if inspect.isframe(parent) and parent.f_code is ascendcode: 46 | continue 47 | if parent in self.ignore: 48 | continue 49 | if depth <= self.maxdepth: 50 | parents.append((parent, self.ascend(parent, depth))) 51 | else: 52 | parents.append((parent, [])) 53 | 54 | return parents 55 | 56 | def peek(self, s): 57 | """Return s, restricted to a sane length.""" 58 | if len(s) > (self.peek_length + 3): 59 | half = self.peek_length // 2 60 | return s[:half] + '...' + s[-half:] 61 | else: 62 | return s 63 | 64 | def _format(self, obj, descend=True): 65 | """Return a string representation of a single object.""" 66 | if inspect.isframe(obj): 67 | filename, lineno, func, context, index = inspect.getframeinfo(obj) 68 | return "" % func 69 | 70 | if not descend: 71 | return self.peek(repr(obj)) 72 | 73 | if isinstance(obj, dict): 74 | return "{" + ", ".join(["%s: %s" % (self._format(k, descend=False), 75 | self._format(v, descend=False)) 76 | for k, v in obj.items()]) + "}" 77 | elif isinstance(obj, list): 78 | return "[" + ", ".join([self._format(item, descend=False) 79 | for item in obj]) + "]" 80 | elif isinstance(obj, tuple): 81 | return "(" + ", ".join([self._format(item, descend=False) 82 | for item in obj]) + ")" 83 | 84 | r = self.peek(repr(obj)) 85 | if isinstance(obj, (str, int, float)): 86 | return r 87 | return "%s: %s" % (type(obj), r) 88 | 89 | def format(self, tree): 90 | """Return a list of string reprs from a nested list of referrers.""" 91 | output = [] 92 | def ascend(branch, depth=1): 93 | for parent, grandparents in branch: 94 | output.append((" " * depth) + self._format(parent)) 95 | if grandparents: 96 | ascend(grandparents, depth + 1) 97 | ascend(tree) 98 | return output 99 | 100 | 101 | def get_instances(cls): 102 | return [x for x in gc.get_objects() if isinstance(x, cls)] 103 | 104 | 105 | class RequestCounter(SimplePlugin): 106 | 107 | def start(self): 108 | self.count = 0 109 | 110 | def before_request(self): 111 | self.count += 1 112 | 113 | def after_request(self): 114 | self.count -=1 115 | request_counter = RequestCounter(cherrypy.engine) 116 | request_counter.subscribe() 117 | 118 | 119 | def get_context(obj): 120 | if isinstance(obj, _cprequest.Request): 121 | return "path=%s;stage=%s" % (obj.path_info, obj.stage) 122 | elif isinstance(obj, _cprequest.Response): 123 | return "status=%s" % obj.status 124 | elif isinstance(obj, _cpwsgi.AppResponse): 125 | return "PATH_INFO=%s" % obj.environ.get('PATH_INFO', '') 126 | elif hasattr(obj, "tb_lineno"): 127 | return "tb_lineno=%s" % obj.tb_lineno 128 | return "" 129 | 130 | 131 | class GCRoot(object): 132 | """A CherryPy page handler for testing reference leaks.""" 133 | 134 | classes = [(_cprequest.Request, 2, 2, 135 | "Should be 1 in this request thread and 1 in the main thread."), 136 | (_cprequest.Response, 2, 2, 137 | "Should be 1 in this request thread and 1 in the main thread."), 138 | (_cpwsgi.AppResponse, 1, 1, 139 | "Should be 1 in this request thread only."), 140 | ] 141 | 142 | def index(self): 143 | return "Hello, world!" 144 | index.exposed = True 145 | 146 | def stats(self): 147 | output = ["Statistics:"] 148 | 149 | for trial in range(10): 150 | if request_counter.count > 0: 151 | break 152 | time.sleep(0.5) 153 | else: 154 | output.append("\nNot all requests closed properly.") 155 | 156 | # gc_collect isn't perfectly synchronous, because it may 157 | # break reference cycles that then take time to fully 158 | # finalize. Call it thrice and hope for the best. 159 | gc.collect() 160 | gc.collect() 161 | unreachable = gc.collect() 162 | if unreachable: 163 | if objgraph is not None: 164 | final = objgraph.by_type('Nondestructible') 165 | if final: 166 | objgraph.show_backrefs(final, filename='finalizers.png') 167 | 168 | trash = {} 169 | for x in gc.garbage: 170 | trash[type(x)] = trash.get(type(x), 0) + 1 171 | if trash: 172 | output.insert(0, "\n%s unreachable objects:" % unreachable) 173 | trash = [(v, k) for k, v in trash.items()] 174 | trash.sort() 175 | for pair in trash: 176 | output.append(" " + repr(pair)) 177 | 178 | # Check declared classes to verify uncollected instances. 179 | # These don't have to be part of a cycle; they can be 180 | # any objects that have unanticipated referrers that keep 181 | # them from being collected. 182 | allobjs = {} 183 | for cls, minobj, maxobj, msg in self.classes: 184 | allobjs[cls] = get_instances(cls) 185 | 186 | for cls, minobj, maxobj, msg in self.classes: 187 | objs = allobjs[cls] 188 | lenobj = len(objs) 189 | if lenobj < minobj or lenobj > maxobj: 190 | if minobj == maxobj: 191 | output.append( 192 | "\nExpected %s %r references, got %s." % 193 | (minobj, cls, lenobj)) 194 | else: 195 | output.append( 196 | "\nExpected %s to %s %r references, got %s." % 197 | (minobj, maxobj, cls, lenobj)) 198 | 199 | for obj in objs: 200 | if objgraph is not None: 201 | ig = [id(objs), id(inspect.currentframe())] 202 | fname = "graph_%s_%s.png" % (cls.__name__, id(obj)) 203 | objgraph.show_backrefs( 204 | obj, extra_ignore=ig, max_depth=4, too_many=20, 205 | filename=fname, extra_info=get_context) 206 | output.append("\nReferrers for %s (refcount=%s):" % 207 | (repr(obj), sys.getrefcount(obj))) 208 | t = ReferrerTree(ignore=[objs], maxdepth=3) 209 | tree = t.ascend(obj) 210 | output.extend(t.format(tree)) 211 | 212 | return "\n".join(output) 213 | stats.exposed = True 214 | 215 | -------------------------------------------------------------------------------- /cherrypy/lib/http.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | warnings.warn('cherrypy.lib.http has been deprecated and will be removed ' 3 | 'in CherryPy 3.3 use cherrypy.lib.httputil instead.', 4 | DeprecationWarning) 5 | 6 | from cherrypy.lib.httputil import * 7 | 8 | -------------------------------------------------------------------------------- /cherrypy/lib/jsontools.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cherrypy 3 | from cherrypy._cpcompat import basestring, ntou, json, json_encode, json_decode 4 | 5 | def json_processor(entity): 6 | """Read application/json data into request.json.""" 7 | if not entity.headers.get(ntou("Content-Length"), ntou("")): 8 | raise cherrypy.HTTPError(411) 9 | 10 | body = entity.fp.read() 11 | try: 12 | cherrypy.serving.request.json = json_decode(body.decode('utf-8')) 13 | except ValueError: 14 | raise cherrypy.HTTPError(400, 'Invalid JSON document') 15 | 16 | def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], 17 | force=True, debug=False, processor = json_processor): 18 | """Add a processor to parse JSON request entities: 19 | The default processor places the parsed data into request.json. 20 | 21 | Incoming request entities which match the given content_type(s) will 22 | be deserialized from JSON to the Python equivalent, and the result 23 | stored at cherrypy.request.json. The 'content_type' argument may 24 | be a Content-Type string or a list of allowable Content-Type strings. 25 | 26 | If the 'force' argument is True (the default), then entities of other 27 | content types will not be allowed; "415 Unsupported Media Type" is 28 | raised instead. 29 | 30 | Supply your own processor to use a custom decoder, or to handle the parsed 31 | data differently. The processor can be configured via 32 | tools.json_in.processor or via the decorator method. 33 | 34 | Note that the deserializer requires the client send a Content-Length 35 | request header, or it will raise "411 Length Required". If for any 36 | other reason the request entity cannot be deserialized from JSON, 37 | it will raise "400 Bad Request: Invalid JSON document". 38 | 39 | You must be using Python 2.6 or greater, or have the 'simplejson' 40 | package importable; otherwise, ValueError is raised during processing. 41 | """ 42 | request = cherrypy.serving.request 43 | if isinstance(content_type, basestring): 44 | content_type = [content_type] 45 | 46 | if force: 47 | if debug: 48 | cherrypy.log('Removing body processors %s' % 49 | repr(request.body.processors.keys()), 'TOOLS.JSON_IN') 50 | request.body.processors.clear() 51 | request.body.default_proc = cherrypy.HTTPError( 52 | 415, 'Expected an entity of content type %s' % 53 | ', '.join(content_type)) 54 | 55 | for ct in content_type: 56 | if debug: 57 | cherrypy.log('Adding body processor for %s' % ct, 'TOOLS.JSON_IN') 58 | request.body.processors[ct] = processor 59 | 60 | def json_handler(*args, **kwargs): 61 | value = cherrypy.serving.request._json_inner_handler(*args, **kwargs) 62 | return json_encode(value) 63 | 64 | def json_out(content_type='application/json', debug=False, handler=json_handler): 65 | """Wrap request.handler to serialize its output to JSON. Sets Content-Type. 66 | 67 | If the given content_type is None, the Content-Type response header 68 | is not set. 69 | 70 | Provide your own handler to use a custom encoder. For example 71 | cherrypy.config['tools.json_out.handler'] = , or 72 | @json_out(handler=function). 73 | 74 | You must be using Python 2.6 or greater, or have the 'simplejson' 75 | package importable; otherwise, ValueError is raised during processing. 76 | """ 77 | request = cherrypy.serving.request 78 | if debug: 79 | cherrypy.log('Replacing %s with JSON handler' % request.handler, 80 | 'TOOLS.JSON_OUT') 81 | request._json_inner_handler = request.handler 82 | request.handler = handler 83 | if content_type is not None: 84 | if debug: 85 | cherrypy.log('Setting Content-Type to %s' % content_type, 'TOOLS.JSON_OUT') 86 | cherrypy.serving.response.headers['Content-Type'] = content_type 87 | 88 | -------------------------------------------------------------------------------- /cherrypy/lib/profiler.py: -------------------------------------------------------------------------------- 1 | """Profiler tools for CherryPy. 2 | 3 | CherryPy users 4 | ============== 5 | 6 | You can profile any of your pages as follows:: 7 | 8 | from cherrypy.lib import profiler 9 | 10 | class Root: 11 | p = profile.Profiler("/path/to/profile/dir") 12 | 13 | def index(self): 14 | self.p.run(self._index) 15 | index.exposed = True 16 | 17 | def _index(self): 18 | return "Hello, world!" 19 | 20 | cherrypy.tree.mount(Root()) 21 | 22 | You can also turn on profiling for all requests 23 | using the ``make_app`` function as WSGI middleware. 24 | 25 | CherryPy developers 26 | =================== 27 | 28 | This module can be used whenever you make changes to CherryPy, 29 | to get a quick sanity-check on overall CP performance. Use the 30 | ``--profile`` flag when running the test suite. Then, use the ``serve()`` 31 | function to browse the results in a web browser. If you run this 32 | module from the command line, it will call ``serve()`` for you. 33 | 34 | """ 35 | 36 | 37 | def new_func_strip_path(func_name): 38 | """Make profiler output more readable by adding ``__init__`` modules' parents""" 39 | filename, line, name = func_name 40 | if filename.endswith("__init__.py"): 41 | return os.path.basename(filename[:-12]) + filename[-12:], line, name 42 | return os.path.basename(filename), line, name 43 | 44 | try: 45 | import profile 46 | import pstats 47 | pstats.func_strip_path = new_func_strip_path 48 | except ImportError: 49 | profile = None 50 | pstats = None 51 | 52 | import os, os.path 53 | import sys 54 | import warnings 55 | 56 | from cherrypy._cpcompat import BytesIO 57 | 58 | _count = 0 59 | 60 | class Profiler(object): 61 | 62 | def __init__(self, path=None): 63 | if not path: 64 | path = os.path.join(os.path.dirname(__file__), "profile") 65 | self.path = path 66 | if not os.path.exists(path): 67 | os.makedirs(path) 68 | 69 | def run(self, func, *args, **params): 70 | """Dump profile data into self.path.""" 71 | global _count 72 | c = _count = _count + 1 73 | path = os.path.join(self.path, "cp_%04d.prof" % c) 74 | prof = profile.Profile() 75 | result = prof.runcall(func, *args, **params) 76 | prof.dump_stats(path) 77 | return result 78 | 79 | def statfiles(self): 80 | """:rtype: list of available profiles. 81 | """ 82 | return [f for f in os.listdir(self.path) 83 | if f.startswith("cp_") and f.endswith(".prof")] 84 | 85 | def stats(self, filename, sortby='cumulative'): 86 | """:rtype stats(index): output of print_stats() for the given profile. 87 | """ 88 | sio = BytesIO() 89 | if sys.version_info >= (2, 5): 90 | s = pstats.Stats(os.path.join(self.path, filename), stream=sio) 91 | s.strip_dirs() 92 | s.sort_stats(sortby) 93 | s.print_stats() 94 | else: 95 | # pstats.Stats before Python 2.5 didn't take a 'stream' arg, 96 | # but just printed to stdout. So re-route stdout. 97 | s = pstats.Stats(os.path.join(self.path, filename)) 98 | s.strip_dirs() 99 | s.sort_stats(sortby) 100 | oldout = sys.stdout 101 | try: 102 | sys.stdout = sio 103 | s.print_stats() 104 | finally: 105 | sys.stdout = oldout 106 | response = sio.getvalue() 107 | sio.close() 108 | return response 109 | 110 | def index(self): 111 | return """ 112 | CherryPy profile data 113 | 114 | 115 | 116 | 117 | 118 | """ 119 | index.exposed = True 120 | 121 | def menu(self): 122 | yield "

Profiling runs

" 123 | yield "

Click on one of the runs below to see profiling data.

" 124 | runs = self.statfiles() 125 | runs.sort() 126 | for i in runs: 127 | yield "%s
" % (i, i) 128 | menu.exposed = True 129 | 130 | def report(self, filename): 131 | import cherrypy 132 | cherrypy.response.headers['Content-Type'] = 'text/plain' 133 | return self.stats(filename) 134 | report.exposed = True 135 | 136 | 137 | class ProfileAggregator(Profiler): 138 | 139 | def __init__(self, path=None): 140 | Profiler.__init__(self, path) 141 | global _count 142 | self.count = _count = _count + 1 143 | self.profiler = profile.Profile() 144 | 145 | def run(self, func, *args): 146 | path = os.path.join(self.path, "cp_%04d.prof" % self.count) 147 | result = self.profiler.runcall(func, *args) 148 | self.profiler.dump_stats(path) 149 | return result 150 | 151 | 152 | class make_app: 153 | def __init__(self, nextapp, path=None, aggregate=False): 154 | """Make a WSGI middleware app which wraps 'nextapp' with profiling. 155 | 156 | nextapp 157 | the WSGI application to wrap, usually an instance of 158 | cherrypy.Application. 159 | 160 | path 161 | where to dump the profiling output. 162 | 163 | aggregate 164 | if True, profile data for all HTTP requests will go in 165 | a single file. If False (the default), each HTTP request will 166 | dump its profile data into a separate file. 167 | 168 | """ 169 | if profile is None or pstats is None: 170 | msg = ("Your installation of Python does not have a profile module. " 171 | "If you're on Debian, try `sudo apt-get install python-profiler`. " 172 | "See http://www.cherrypy.org/wiki/ProfilingOnDebian for details.") 173 | warnings.warn(msg) 174 | 175 | self.nextapp = nextapp 176 | self.aggregate = aggregate 177 | if aggregate: 178 | self.profiler = ProfileAggregator(path) 179 | else: 180 | self.profiler = Profiler(path) 181 | 182 | def __call__(self, environ, start_response): 183 | def gather(): 184 | result = [] 185 | for line in self.nextapp(environ, start_response): 186 | result.append(line) 187 | return result 188 | return self.profiler.run(gather) 189 | 190 | 191 | def serve(path=None, port=8080): 192 | if profile is None or pstats is None: 193 | msg = ("Your installation of Python does not have a profile module. " 194 | "If you're on Debian, try `sudo apt-get install python-profiler`. " 195 | "See http://www.cherrypy.org/wiki/ProfilingOnDebian for details.") 196 | warnings.warn(msg) 197 | 198 | import cherrypy 199 | cherrypy.config.update({'server.socket_port': int(port), 200 | 'server.thread_pool': 10, 201 | 'environment': "production", 202 | }) 203 | cherrypy.quickstart(Profiler(path)) 204 | 205 | 206 | if __name__ == "__main__": 207 | serve(*tuple(sys.argv[1:])) 208 | 209 | -------------------------------------------------------------------------------- /cherrypy/lib/xmlrpcutil.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import cherrypy 4 | from cherrypy._cpcompat import ntob 5 | 6 | def get_xmlrpclib(): 7 | try: 8 | import xmlrpc.client as x 9 | except ImportError: 10 | import xmlrpclib as x 11 | return x 12 | 13 | def process_body(): 14 | """Return (params, method) from request body.""" 15 | try: 16 | return get_xmlrpclib().loads(cherrypy.request.body.read()) 17 | except Exception: 18 | return ('ERROR PARAMS', ), 'ERRORMETHOD' 19 | 20 | 21 | def patched_path(path): 22 | """Return 'path', doctored for RPC.""" 23 | if not path.endswith('/'): 24 | path += '/' 25 | if path.startswith('/RPC2/'): 26 | # strip the first /rpc2 27 | path = path[5:] 28 | return path 29 | 30 | 31 | def _set_response(body): 32 | # The XML-RPC spec (http://www.xmlrpc.com/spec) says: 33 | # "Unless there's a lower-level error, always return 200 OK." 34 | # Since Python's xmlrpclib interprets a non-200 response 35 | # as a "Protocol Error", we'll just return 200 every time. 36 | response = cherrypy.response 37 | response.status = '200 OK' 38 | response.body = ntob(body, 'utf-8') 39 | response.headers['Content-Type'] = 'text/xml' 40 | response.headers['Content-Length'] = len(body) 41 | 42 | 43 | def respond(body, encoding='utf-8', allow_none=0): 44 | xmlrpclib = get_xmlrpclib() 45 | if not isinstance(body, xmlrpclib.Fault): 46 | body = (body,) 47 | _set_response(xmlrpclib.dumps(body, methodresponse=1, 48 | encoding=encoding, 49 | allow_none=allow_none)) 50 | 51 | def on_error(*args, **kwargs): 52 | body = str(sys.exc_info()[1]) 53 | xmlrpclib = get_xmlrpclib() 54 | _set_response(xmlrpclib.dumps(xmlrpclib.Fault(1, body))) 55 | 56 | -------------------------------------------------------------------------------- /cherrypy/process/__init__.py: -------------------------------------------------------------------------------- 1 | """Site container for an HTTP server. 2 | 3 | A Web Site Process Bus object is used to connect applications, servers, 4 | and frameworks with site-wide services such as daemonization, process 5 | reload, signal handling, drop privileges, PID file management, logging 6 | for all of these, and many more. 7 | 8 | The 'plugins' module defines a few abstract and concrete services for 9 | use with the bus. Some use tool-specific channels; see the documentation 10 | for each class. 11 | """ 12 | 13 | from cherrypy.process.wspbus import bus 14 | from cherrypy.process import plugins, servers 15 | -------------------------------------------------------------------------------- /cherrypy/process/win32.py: -------------------------------------------------------------------------------- 1 | """Windows service. Requires pywin32.""" 2 | 3 | import os 4 | import win32api 5 | import win32con 6 | import win32event 7 | import win32service 8 | import win32serviceutil 9 | 10 | from cherrypy.process import wspbus, plugins 11 | 12 | 13 | class ConsoleCtrlHandler(plugins.SimplePlugin): 14 | """A WSPBus plugin for handling Win32 console events (like Ctrl-C).""" 15 | 16 | def __init__(self, bus): 17 | self.is_set = False 18 | plugins.SimplePlugin.__init__(self, bus) 19 | 20 | def start(self): 21 | if self.is_set: 22 | self.bus.log('Handler for console events already set.', level=40) 23 | return 24 | 25 | result = win32api.SetConsoleCtrlHandler(self.handle, 1) 26 | if result == 0: 27 | self.bus.log('Could not SetConsoleCtrlHandler (error %r)' % 28 | win32api.GetLastError(), level=40) 29 | else: 30 | self.bus.log('Set handler for console events.', level=40) 31 | self.is_set = True 32 | 33 | def stop(self): 34 | if not self.is_set: 35 | self.bus.log('Handler for console events already off.', level=40) 36 | return 37 | 38 | try: 39 | result = win32api.SetConsoleCtrlHandler(self.handle, 0) 40 | except ValueError: 41 | # "ValueError: The object has not been registered" 42 | result = 1 43 | 44 | if result == 0: 45 | self.bus.log('Could not remove SetConsoleCtrlHandler (error %r)' % 46 | win32api.GetLastError(), level=40) 47 | else: 48 | self.bus.log('Removed handler for console events.', level=40) 49 | self.is_set = False 50 | 51 | def handle(self, event): 52 | """Handle console control events (like Ctrl-C).""" 53 | if event in (win32con.CTRL_C_EVENT, win32con.CTRL_LOGOFF_EVENT, 54 | win32con.CTRL_BREAK_EVENT, win32con.CTRL_SHUTDOWN_EVENT, 55 | win32con.CTRL_CLOSE_EVENT): 56 | self.bus.log('Console event %s: shutting down bus' % event) 57 | 58 | # Remove self immediately so repeated Ctrl-C doesn't re-call it. 59 | try: 60 | self.stop() 61 | except ValueError: 62 | pass 63 | 64 | self.bus.exit() 65 | # 'First to return True stops the calls' 66 | return 1 67 | return 0 68 | 69 | 70 | class Win32Bus(wspbus.Bus): 71 | """A Web Site Process Bus implementation for Win32. 72 | 73 | Instead of time.sleep, this bus blocks using native win32event objects. 74 | """ 75 | 76 | def __init__(self): 77 | self.events = {} 78 | wspbus.Bus.__init__(self) 79 | 80 | def _get_state_event(self, state): 81 | """Return a win32event for the given state (creating it if needed).""" 82 | try: 83 | return self.events[state] 84 | except KeyError: 85 | event = win32event.CreateEvent(None, 0, 0, 86 | "WSPBus %s Event (pid=%r)" % 87 | (state.name, os.getpid())) 88 | self.events[state] = event 89 | return event 90 | 91 | def _get_state(self): 92 | return self._state 93 | def _set_state(self, value): 94 | self._state = value 95 | event = self._get_state_event(value) 96 | win32event.PulseEvent(event) 97 | state = property(_get_state, _set_state) 98 | 99 | def wait(self, state, interval=0.1, channel=None): 100 | """Wait for the given state(s), KeyboardInterrupt or SystemExit. 101 | 102 | Since this class uses native win32event objects, the interval 103 | argument is ignored. 104 | """ 105 | if isinstance(state, (tuple, list)): 106 | # Don't wait for an event that beat us to the punch ;) 107 | if self.state not in state: 108 | events = tuple([self._get_state_event(s) for s in state]) 109 | win32event.WaitForMultipleObjects(events, 0, win32event.INFINITE) 110 | else: 111 | # Don't wait for an event that beat us to the punch ;) 112 | if self.state != state: 113 | event = self._get_state_event(state) 114 | win32event.WaitForSingleObject(event, win32event.INFINITE) 115 | 116 | 117 | class _ControlCodes(dict): 118 | """Control codes used to "signal" a service via ControlService. 119 | 120 | User-defined control codes are in the range 128-255. We generally use 121 | the standard Python value for the Linux signal and add 128. Example: 122 | 123 | >>> signal.SIGUSR1 124 | 10 125 | control_codes['graceful'] = 128 + 10 126 | """ 127 | 128 | def key_for(self, obj): 129 | """For the given value, return its corresponding key.""" 130 | for key, val in self.items(): 131 | if val is obj: 132 | return key 133 | raise ValueError("The given object could not be found: %r" % obj) 134 | 135 | control_codes = _ControlCodes({'graceful': 138}) 136 | 137 | 138 | def signal_child(service, command): 139 | if command == 'stop': 140 | win32serviceutil.StopService(service) 141 | elif command == 'restart': 142 | win32serviceutil.RestartService(service) 143 | else: 144 | win32serviceutil.ControlService(service, control_codes[command]) 145 | 146 | 147 | class PyWebService(win32serviceutil.ServiceFramework): 148 | """Python Web Service.""" 149 | 150 | _svc_name_ = "Python Web Service" 151 | _svc_display_name_ = "Python Web Service" 152 | _svc_deps_ = None # sequence of service names on which this depends 153 | _exe_name_ = "pywebsvc" 154 | _exe_args_ = None # Default to no arguments 155 | 156 | # Only exists on Windows 2000 or later, ignored on windows NT 157 | _svc_description_ = "Python Web Service" 158 | 159 | def SvcDoRun(self): 160 | from cherrypy import process 161 | process.bus.start() 162 | process.bus.block() 163 | 164 | def SvcStop(self): 165 | from cherrypy import process 166 | self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) 167 | process.bus.exit() 168 | 169 | def SvcOther(self, control): 170 | process.bus.publish(control_codes.key_for(control)) 171 | 172 | 173 | if __name__ == '__main__': 174 | win32serviceutil.HandleCommandLine(PyWebService) 175 | -------------------------------------------------------------------------------- /cherrypy/wsgiserver/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', 2 | 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', 3 | 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', 4 | 'WorkerThread', 'ThreadPool', 'SSLAdapter', 5 | 'CherryPyWSGIServer', 6 | 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', 7 | 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class'] 8 | 9 | import sys 10 | if sys.version_info < (3, 0): 11 | from wsgiserver2 import * 12 | else: 13 | # Le sigh. Boo for backward-incompatible syntax. 14 | exec('from .wsgiserver3 import *') 15 | -------------------------------------------------------------------------------- /cherrypy/wsgiserver/ssl_builtin.py: -------------------------------------------------------------------------------- 1 | """A library for integrating Python's builtin ``ssl`` library with CherryPy. 2 | 3 | The ssl module must be importable for SSL functionality. 4 | 5 | To use this module, set ``CherryPyWSGIServer.ssl_adapter`` to an instance of 6 | ``BuiltinSSLAdapter``. 7 | """ 8 | 9 | try: 10 | import ssl 11 | except ImportError: 12 | ssl = None 13 | 14 | try: 15 | from _pyio import DEFAULT_BUFFER_SIZE 16 | except ImportError: 17 | try: 18 | from io import DEFAULT_BUFFER_SIZE 19 | except ImportError: 20 | DEFAULT_BUFFER_SIZE = -1 21 | 22 | import sys 23 | 24 | from cherrypy import wsgiserver 25 | 26 | 27 | class BuiltinSSLAdapter(wsgiserver.SSLAdapter): 28 | """A wrapper for integrating Python's builtin ssl module with CherryPy.""" 29 | 30 | certificate = None 31 | """The filename of the server SSL certificate.""" 32 | 33 | private_key = None 34 | """The filename of the server's private key file.""" 35 | 36 | def __init__(self, certificate, private_key, certificate_chain=None): 37 | if ssl is None: 38 | raise ImportError("You must install the ssl module to use HTTPS.") 39 | self.certificate = certificate 40 | self.private_key = private_key 41 | self.certificate_chain = certificate_chain 42 | 43 | def bind(self, sock): 44 | """Wrap and return the given socket.""" 45 | return sock 46 | 47 | def wrap(self, sock): 48 | """Wrap and return the given socket, plus WSGI environ entries.""" 49 | try: 50 | s = ssl.wrap_socket(sock, do_handshake_on_connect=True, 51 | server_side=True, certfile=self.certificate, 52 | keyfile=self.private_key, ssl_version=ssl.PROTOCOL_SSLv23) 53 | except ssl.SSLError: 54 | e = sys.exc_info()[1] 55 | if e.errno == ssl.SSL_ERROR_EOF: 56 | # This is almost certainly due to the cherrypy engine 57 | # 'pinging' the socket to assert it's connectable; 58 | # the 'ping' isn't SSL. 59 | return None, {} 60 | elif e.errno == ssl.SSL_ERROR_SSL: 61 | if e.args[1].endswith('http request'): 62 | # The client is speaking HTTP to an HTTPS server. 63 | raise wsgiserver.NoSSLError 64 | elif e.args[1].endswith('unknown protocol'): 65 | # The client is speaking some non-HTTP protocol. 66 | # Drop the conn. 67 | return None, {} 68 | raise 69 | return s, self.get_environ(s) 70 | 71 | # TODO: fill this out more with mod ssl env 72 | def get_environ(self, sock): 73 | """Create WSGI environ entries to be merged into each request.""" 74 | cipher = sock.cipher() 75 | ssl_environ = { 76 | "wsgi.url_scheme": "https", 77 | "HTTPS": "on", 78 | 'SSL_PROTOCOL': cipher[1], 79 | 'SSL_CIPHER': cipher[0] 80 | ## SSL_VERSION_INTERFACE string The mod_ssl program version 81 | ## SSL_VERSION_LIBRARY string The OpenSSL program version 82 | } 83 | return ssl_environ 84 | 85 | if sys.version_info >= (3, 0): 86 | def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): 87 | return wsgiserver.CP_makefile(sock, mode, bufsize) 88 | else: 89 | def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): 90 | return wsgiserver.CP_fileobject(sock, mode, bufsize) 91 | 92 | -------------------------------------------------------------------------------- /cherrypy/wsgiserver/ssl_pyopenssl.py: -------------------------------------------------------------------------------- 1 | """A library for integrating pyOpenSSL with CherryPy. 2 | 3 | The OpenSSL module must be importable for SSL functionality. 4 | You can obtain it from http://pyopenssl.sourceforge.net/ 5 | 6 | To use this module, set CherryPyWSGIServer.ssl_adapter to an instance of 7 | SSLAdapter. There are two ways to use SSL: 8 | 9 | Method One 10 | ---------- 11 | 12 | * ``ssl_adapter.context``: an instance of SSL.Context. 13 | 14 | If this is not None, it is assumed to be an SSL.Context instance, 15 | and will be passed to SSL.Connection on bind(). The developer is 16 | responsible for forming a valid Context object. This approach is 17 | to be preferred for more flexibility, e.g. if the cert and key are 18 | streams instead of files, or need decryption, or SSL.SSLv3_METHOD 19 | is desired instead of the default SSL.SSLv23_METHOD, etc. Consult 20 | the pyOpenSSL documentation for complete options. 21 | 22 | Method Two (shortcut) 23 | --------------------- 24 | 25 | * ``ssl_adapter.certificate``: the filename of the server SSL certificate. 26 | * ``ssl_adapter.private_key``: the filename of the server's private key file. 27 | 28 | Both are None by default. If ssl_adapter.context is None, but .private_key 29 | and .certificate are both given and valid, they will be read, and the 30 | context will be automatically created from them. 31 | """ 32 | 33 | import socket 34 | import threading 35 | import time 36 | 37 | from cherrypy import wsgiserver 38 | 39 | try: 40 | from OpenSSL import SSL 41 | from OpenSSL import crypto 42 | except ImportError: 43 | SSL = None 44 | 45 | 46 | class SSL_fileobject(wsgiserver.CP_fileobject): 47 | """SSL file object attached to a socket object.""" 48 | 49 | ssl_timeout = 3 50 | ssl_retry = .01 51 | 52 | def _safe_call(self, is_reader, call, *args, **kwargs): 53 | """Wrap the given call with SSL error-trapping. 54 | 55 | is_reader: if False EOF errors will be raised. If True, EOF errors 56 | will return "" (to emulate normal sockets). 57 | """ 58 | start = time.time() 59 | while True: 60 | try: 61 | return call(*args, **kwargs) 62 | except SSL.WantReadError: 63 | # Sleep and try again. This is dangerous, because it means 64 | # the rest of the stack has no way of differentiating 65 | # between a "new handshake" error and "client dropped". 66 | # Note this isn't an endless loop: there's a timeout below. 67 | time.sleep(self.ssl_retry) 68 | except SSL.WantWriteError: 69 | time.sleep(self.ssl_retry) 70 | except SSL.SysCallError, e: 71 | if is_reader and e.args == (-1, 'Unexpected EOF'): 72 | return "" 73 | 74 | errnum = e.args[0] 75 | if is_reader and errnum in wsgiserver.socket_errors_to_ignore: 76 | return "" 77 | raise socket.error(errnum) 78 | except SSL.Error, e: 79 | if is_reader and e.args == (-1, 'Unexpected EOF'): 80 | return "" 81 | 82 | thirdarg = None 83 | try: 84 | thirdarg = e.args[0][0][2] 85 | except IndexError: 86 | pass 87 | 88 | if thirdarg == 'http request': 89 | # The client is talking HTTP to an HTTPS server. 90 | raise wsgiserver.NoSSLError() 91 | 92 | raise wsgiserver.FatalSSLAlert(*e.args) 93 | except: 94 | raise 95 | 96 | if time.time() - start > self.ssl_timeout: 97 | raise socket.timeout("timed out") 98 | 99 | def recv(self, *args, **kwargs): 100 | buf = [] 101 | r = super(SSL_fileobject, self).recv 102 | while True: 103 | data = self._safe_call(True, r, *args, **kwargs) 104 | buf.append(data) 105 | p = self._sock.pending() 106 | if not p: 107 | return "".join(buf) 108 | 109 | def sendall(self, *args, **kwargs): 110 | return self._safe_call(False, super(SSL_fileobject, self).sendall, 111 | *args, **kwargs) 112 | 113 | def send(self, *args, **kwargs): 114 | return self._safe_call(False, super(SSL_fileobject, self).send, 115 | *args, **kwargs) 116 | 117 | 118 | class SSLConnection: 119 | """A thread-safe wrapper for an SSL.Connection. 120 | 121 | ``*args``: the arguments to create the wrapped ``SSL.Connection(*args)``. 122 | """ 123 | 124 | def __init__(self, *args): 125 | self._ssl_conn = SSL.Connection(*args) 126 | self._lock = threading.RLock() 127 | 128 | for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read', 129 | 'renegotiate', 'bind', 'listen', 'connect', 'accept', 130 | 'setblocking', 'fileno', 'close', 'get_cipher_list', 131 | 'getpeername', 'getsockname', 'getsockopt', 'setsockopt', 132 | 'makefile', 'get_app_data', 'set_app_data', 'state_string', 133 | 'sock_shutdown', 'get_peer_certificate', 'want_read', 134 | 'want_write', 'set_connect_state', 'set_accept_state', 135 | 'connect_ex', 'sendall', 'settimeout', 'gettimeout'): 136 | exec("""def %s(self, *args): 137 | self._lock.acquire() 138 | try: 139 | return self._ssl_conn.%s(*args) 140 | finally: 141 | self._lock.release() 142 | """ % (f, f)) 143 | 144 | def shutdown(self, *args): 145 | self._lock.acquire() 146 | try: 147 | # pyOpenSSL.socket.shutdown takes no args 148 | return self._ssl_conn.shutdown() 149 | finally: 150 | self._lock.release() 151 | 152 | 153 | class pyOpenSSLAdapter(wsgiserver.SSLAdapter): 154 | """A wrapper for integrating pyOpenSSL with CherryPy.""" 155 | 156 | context = None 157 | """An instance of SSL.Context.""" 158 | 159 | certificate = None 160 | """The filename of the server SSL certificate.""" 161 | 162 | private_key = None 163 | """The filename of the server's private key file.""" 164 | 165 | certificate_chain = None 166 | """Optional. The filename of CA's intermediate certificate bundle. 167 | 168 | This is needed for cheaper "chained root" SSL certificates, and should be 169 | left as None if not required.""" 170 | 171 | def __init__(self, certificate, private_key, certificate_chain=None): 172 | if SSL is None: 173 | raise ImportError("You must install pyOpenSSL to use HTTPS.") 174 | 175 | self.context = None 176 | self.certificate = certificate 177 | self.private_key = private_key 178 | self.certificate_chain = certificate_chain 179 | self._environ = None 180 | 181 | def bind(self, sock): 182 | """Wrap and return the given socket.""" 183 | if self.context is None: 184 | self.context = self.get_context() 185 | conn = SSLConnection(self.context, sock) 186 | self._environ = self.get_environ() 187 | return conn 188 | 189 | def wrap(self, sock): 190 | """Wrap and return the given socket, plus WSGI environ entries.""" 191 | return sock, self._environ.copy() 192 | 193 | def get_context(self): 194 | """Return an SSL.Context from self attributes.""" 195 | # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473 196 | c = SSL.Context(SSL.SSLv23_METHOD) 197 | c.use_privatekey_file(self.private_key) 198 | if self.certificate_chain: 199 | c.load_verify_locations(self.certificate_chain) 200 | c.use_certificate_file(self.certificate) 201 | return c 202 | 203 | def get_environ(self): 204 | """Return WSGI environ entries to be merged into each request.""" 205 | ssl_environ = { 206 | "HTTPS": "on", 207 | # pyOpenSSL doesn't provide access to any of these AFAICT 208 | ## 'SSL_PROTOCOL': 'SSLv2', 209 | ## SSL_CIPHER string The cipher specification name 210 | ## SSL_VERSION_INTERFACE string The mod_ssl program version 211 | ## SSL_VERSION_LIBRARY string The OpenSSL program version 212 | } 213 | 214 | if self.certificate: 215 | # Server certificate attributes 216 | cert = open(self.certificate, 'rb').read() 217 | cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) 218 | ssl_environ.update({ 219 | 'SSL_SERVER_M_VERSION': cert.get_version(), 220 | 'SSL_SERVER_M_SERIAL': cert.get_serial_number(), 221 | ## 'SSL_SERVER_V_START': Validity of server's certificate (start time), 222 | ## 'SSL_SERVER_V_END': Validity of server's certificate (end time), 223 | }) 224 | 225 | for prefix, dn in [("I", cert.get_issuer()), 226 | ("S", cert.get_subject())]: 227 | # X509Name objects don't seem to have a way to get the 228 | # complete DN string. Use str() and slice it instead, 229 | # because str(dn) == "" 230 | dnstr = str(dn)[18:-2] 231 | 232 | wsgikey = 'SSL_SERVER_%s_DN' % prefix 233 | ssl_environ[wsgikey] = dnstr 234 | 235 | # The DN should be of the form: /k1=v1/k2=v2, but we must allow 236 | # for any value to contain slashes itself (in a URL). 237 | while dnstr: 238 | pos = dnstr.rfind("=") 239 | dnstr, value = dnstr[:pos], dnstr[pos + 1:] 240 | pos = dnstr.rfind("/") 241 | dnstr, key = dnstr[:pos], dnstr[pos + 1:] 242 | if key and value: 243 | wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key) 244 | ssl_environ[wsgikey] = value 245 | 246 | return ssl_environ 247 | 248 | def makefile(self, sock, mode='r', bufsize=-1): 249 | if SSL and isinstance(sock, SSL.ConnectionType): 250 | timeout = sock.gettimeout() 251 | f = SSL_fileobject(sock, mode, bufsize) 252 | f.ssl_timeout = timeout 253 | return f 254 | else: 255 | return wsgiserver.CP_fileobject(sock, mode, bufsize) 256 | 257 | -------------------------------------------------------------------------------- /db.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import sys 3 | import threading 4 | import Queue 5 | import logging 6 | 7 | ## local modules 8 | import threads 9 | 10 | DB_FILE = "db\\headers.db" 11 | 12 | def sql_alphanumeric(s): 13 | for idx in range(len(s)): 14 | if s[idx].isalnum(): 15 | return s[idx:] 16 | 17 | class MultiThreadOK(threads.MyThread): 18 | def __init__(self, db, autostart=False): 19 | super(MultiThreadOK, self).__init__(name="Database Thread") 20 | self.db=db 21 | self.CANCEL = False 22 | self.reqs=Queue.Queue() 23 | if autostart: 24 | self.start() 25 | def run(self): 26 | cnx = sqlite3.connect(self.db) 27 | cnx.text_factory = sqlite3.OptimizedUnicode 28 | cnx.create_function("alphanumeric", 1, sql_alphanumeric) 29 | cursor = cnx.cursor() 30 | while not self.CANCEL: 31 | try: 32 | req, arg, res = self.reqs.get(timeout=10) 33 | except Queue.Empty: 34 | continue 35 | 36 | if req=='--close--': break 37 | if req=='--commit--': 38 | cnx.commit() 39 | continue 40 | 41 | #logging.getLogger().info("DB: Executing " + repr(req) + " (" + repr(arg) + ")") 42 | 43 | cursor.execute(req, arg) 44 | if res: 45 | for rec in cursor: 46 | res.put(rec) 47 | res.put('--no more--') 48 | cnx.close() 49 | def cancel(self): 50 | logging.getLogger().info("%s will finish canceling after processing approximately %d items." % (self.name, self.reqs.qsize())) 51 | self.CANCEL = True 52 | def execute(self, req, arg=None, res=None): 53 | self.reqs.put((req, arg or tuple(), res)) 54 | def select(self, req, arg=None): 55 | res=Queue.Queue() 56 | self.execute(req, arg, res) 57 | while True: 58 | rec=res.get() 59 | if rec=='--no more--': break 60 | yield rec 61 | def close(self): 62 | self.execute('--close--') 63 | def commit(self): 64 | self.execute('--commit--') 65 | 66 | handler = MultiThreadOK(DB_FILE) 67 | 68 | def setup(): 69 | 70 | try: 71 | connection = sqlite3.connect(DB_FILE) 72 | connection.execute( 73 | 'CREATE TABLE IF NOT EXISTS articles ' 74 | '(subject TEXT PRIMARY KEY, ' 75 | 'parts TEXT, ' 76 | 'total_parts INT, ' 77 | 'complete TINYINT, ' 78 | 'filename TEXT, ' 79 | 'groups TEXT, ' 80 | 'poster TEXT, ' 81 | 'date_posted INT, ' 82 | 'size INT, ' 83 | 'yenc TINYINT)' 84 | ) 85 | 86 | connection.execute( 87 | 'CREATE INDEX IF NOT EXISTS filename_asc ON articles (filename ASC)') 88 | 89 | except sqlite3.Error, e: 90 | logging.getLogger().exception("Database cannot be found or created. Cannot continue, quiting.") 91 | #TODO a more graceful exit. 92 | sys.exit(1) 93 | 94 | handler.start() 95 | 96 | def connect(): 97 | return handler -------------------------------------------------------------------------------- /html/browse.tmp.htm: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | <%inherit file="/main.tmp.htm" /> 3 | 4 | 8 | 9 | 26 | 27 |
28 | Search: 29 |
30 | 31 |
32 | 33 | 34 | % for result in result_set: 35 | ${makerow(result)} 36 | % endfor 37 |
SubjectFilename
38 | 39 |
40 | 41 | % for page in range(pg, pg+11): 42 | ${page} 43 | % endfor 44 | 45 | <%def name="makerow(result)"> 46 | <% 47 | (rowid, subject, parts, total_parts, complete, filename, groups, poster, date_posted, size, yenc) = result 48 | %> 49 | 50 | % if complete != 1: 51 | 52 | % else: 53 | 54 | % endif 55 | ${subject} 56 | ${filename} 57 | ${total_parts} 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /html/main.tmp.htm: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 4 | 5 | 6 | 7 | 8 | 9 | Browse | Configure Settings

10 | 11 | ${self.body()} 12 | 13 | -------------------------------------------------------------------------------- /html/settings.tmp.htm: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | <%inherit file="/main.tmp.htm" /> 3 | 4 | 87 | 88 |
89 |
90 |
91 | % if is_ssl: 92 |
93 | % else: 94 |
95 | %endif 96 |
97 |
98 | 99 | 100 |
 
101 |
102 | % for group in groups: 103 | ${group}
104 | % endfor 105 |
106 | 107 |
108 | -------------------------------------------------------------------------------- /mako/__init__.py: -------------------------------------------------------------------------------- 1 | # mako/__init__.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | 8 | __version__ = '0.6.2' 9 | 10 | -------------------------------------------------------------------------------- /mako/ast.py: -------------------------------------------------------------------------------- 1 | # mako/ast.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """utilities for analyzing expressions and blocks of Python 8 | code, as well as generating Python from AST nodes""" 9 | 10 | from mako import exceptions, pyparser, util 11 | import re 12 | 13 | class PythonCode(object): 14 | """represents information about a string containing Python code""" 15 | def __init__(self, code, **exception_kwargs): 16 | self.code = code 17 | 18 | # represents all identifiers which are assigned to at some point in the code 19 | self.declared_identifiers = set() 20 | 21 | # represents all identifiers which are referenced before their assignment, if any 22 | self.undeclared_identifiers = set() 23 | 24 | # note that an identifier can be in both the undeclared and declared lists. 25 | 26 | # using AST to parse instead of using code.co_varnames, 27 | # code.co_names has several advantages: 28 | # - we can locate an identifier as "undeclared" even if 29 | # its declared later in the same block of code 30 | # - AST is less likely to break with version changes 31 | # (for example, the behavior of co_names changed a little bit 32 | # in python version 2.5) 33 | if isinstance(code, basestring): 34 | expr = pyparser.parse(code.lstrip(), "exec", **exception_kwargs) 35 | else: 36 | expr = code 37 | 38 | f = pyparser.FindIdentifiers(self, **exception_kwargs) 39 | f.visit(expr) 40 | 41 | class ArgumentList(object): 42 | """parses a fragment of code as a comma-separated list of expressions""" 43 | def __init__(self, code, **exception_kwargs): 44 | self.codeargs = [] 45 | self.args = [] 46 | self.declared_identifiers = set() 47 | self.undeclared_identifiers = set() 48 | if isinstance(code, basestring): 49 | if re.match(r"\S", code) and not re.match(r",\s*$", code): 50 | # if theres text and no trailing comma, insure its parsed 51 | # as a tuple by adding a trailing comma 52 | code += "," 53 | expr = pyparser.parse(code, "exec", **exception_kwargs) 54 | else: 55 | expr = code 56 | 57 | f = pyparser.FindTuple(self, PythonCode, **exception_kwargs) 58 | f.visit(expr) 59 | 60 | class PythonFragment(PythonCode): 61 | """extends PythonCode to provide identifier lookups in partial control statements 62 | 63 | e.g. 64 | for x in 5: 65 | elif y==9: 66 | except (MyException, e): 67 | etc. 68 | """ 69 | def __init__(self, code, **exception_kwargs): 70 | m = re.match(r'^(\w+)(?:\s+(.*?))?:\s*(#|$)', code.strip(), re.S) 71 | if not m: 72 | raise exceptions.CompileException( 73 | "Fragment '%s' is not a partial control statement" % 74 | code, **exception_kwargs) 75 | if m.group(3): 76 | code = code[:m.start(3)] 77 | (keyword, expr) = m.group(1,2) 78 | if keyword in ['for','if', 'while']: 79 | code = code + "pass" 80 | elif keyword == 'try': 81 | code = code + "pass\nexcept:pass" 82 | elif keyword == 'elif' or keyword == 'else': 83 | code = "if False:pass\n" + code + "pass" 84 | elif keyword == 'except': 85 | code = "try:pass\n" + code + "pass" 86 | else: 87 | raise exceptions.CompileException( 88 | "Unsupported control keyword: '%s'" % 89 | keyword, **exception_kwargs) 90 | super(PythonFragment, self).__init__(code, **exception_kwargs) 91 | 92 | 93 | class FunctionDecl(object): 94 | """function declaration""" 95 | def __init__(self, code, allow_kwargs=True, **exception_kwargs): 96 | self.code = code 97 | expr = pyparser.parse(code, "exec", **exception_kwargs) 98 | 99 | f = pyparser.ParseFunc(self, **exception_kwargs) 100 | f.visit(expr) 101 | if not hasattr(self, 'funcname'): 102 | raise exceptions.CompileException( 103 | "Code '%s' is not a function declaration" % code, 104 | **exception_kwargs) 105 | if not allow_kwargs and self.kwargs: 106 | raise exceptions.CompileException( 107 | "'**%s' keyword argument not allowed here" % 108 | self.argnames[-1], **exception_kwargs) 109 | 110 | def get_argument_expressions(self, include_defaults=True): 111 | """return the argument declarations of this FunctionDecl as a printable list.""" 112 | 113 | namedecls = [] 114 | defaults = [d for d in self.defaults] 115 | kwargs = self.kwargs 116 | varargs = self.varargs 117 | argnames = [f for f in self.argnames] 118 | argnames.reverse() 119 | for arg in argnames: 120 | default = None 121 | if kwargs: 122 | arg = "**" + arg 123 | kwargs = False 124 | elif varargs: 125 | arg = "*" + arg 126 | varargs = False 127 | else: 128 | default = len(defaults) and defaults.pop() or None 129 | if include_defaults and default: 130 | namedecls.insert(0, "%s=%s" % 131 | (arg, 132 | pyparser.ExpressionGenerator(default).value() 133 | ) 134 | ) 135 | else: 136 | namedecls.insert(0, arg) 137 | return namedecls 138 | 139 | class FunctionArgs(FunctionDecl): 140 | """the argument portion of a function declaration""" 141 | 142 | def __init__(self, code, **kwargs): 143 | super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, **kwargs) 144 | -------------------------------------------------------------------------------- /mako/cache.py: -------------------------------------------------------------------------------- 1 | # mako/cache.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | from mako import exceptions, util 8 | 9 | _cache_plugins = util.PluginLoader("mako.cache") 10 | 11 | register_plugin = _cache_plugins.register 12 | register_plugin("beaker", "mako.ext.beaker_cache", "BeakerCacheImpl") 13 | 14 | 15 | class Cache(object): 16 | """Represents a data content cache made available to the module 17 | space of a specific :class:`.Template` object. 18 | 19 | As of Mako 0.6, :class:`.Cache` by itself is mostly a 20 | container for a :class:`.CacheImpl` object, which implements 21 | a fixed API to provide caching services; specific subclasses exist to 22 | implement different 23 | caching strategies. Mako includes a backend that works with 24 | the Beaker caching system. Beaker itself then supports 25 | a number of backends (i.e. file, memory, memcached, etc.) 26 | 27 | The construction of a :class:`.Cache` is part of the mechanics 28 | of a :class:`.Template`, and programmatic access to this 29 | cache is typically via the :attr:`.Template.cache` attribute. 30 | 31 | """ 32 | 33 | impl = None 34 | """Provide the :class:`.CacheImpl` in use by this :class:`.Cache`. 35 | 36 | This accessor allows a :class:`.CacheImpl` with additional 37 | methods beyond that of :class:`.Cache` to be used programmatically. 38 | 39 | """ 40 | 41 | id = None 42 | """Return the 'id' that identifies this cache. 43 | 44 | This is a value that should be globally unique to the 45 | :class:`.Template` associated with this cache, and can 46 | be used by a caching system to name a local container 47 | for data specific to this template. 48 | 49 | """ 50 | 51 | starttime = None 52 | """Epochal time value for when the owning :class:`.Template` was 53 | first compiled. 54 | 55 | A cache implementation may wish to invalidate data earlier than 56 | this timestamp; this has the effect of the cache for a specific 57 | :class:`.Template` starting clean any time the :class:`.Template` 58 | is recompiled, such as when the original template file changed on 59 | the filesystem. 60 | 61 | """ 62 | 63 | def __init__(self, template, *args): 64 | # check for a stale template calling the 65 | # constructor 66 | if isinstance(template, basestring) and args: 67 | return 68 | self.template = template 69 | self.id = template.module.__name__ 70 | self.starttime = template.module._modified_time 71 | self._def_regions = {} 72 | self.impl = self._load_impl(self.template.cache_impl) 73 | 74 | def _load_impl(self, name): 75 | return _cache_plugins.load(name)(self) 76 | 77 | def get_or_create(self, key, creation_function, **kw): 78 | """Retrieve a value from the cache, using the given creation function 79 | to generate a new value.""" 80 | 81 | if not self.template.cache_enabled: 82 | return creation_function() 83 | 84 | return self.impl.get_or_create(key, 85 | creation_function, 86 | **self._get_cache_kw(kw)) 87 | 88 | def set(self, key, value, **kw): 89 | """Place a value in the cache. 90 | 91 | :param key: the value's key. 92 | :param value: the value 93 | :param \**kw: cache configuration arguments. 94 | 95 | """ 96 | 97 | self.impl.set(key, value, **self._get_cache_kw(kw)) 98 | 99 | put = set 100 | """A synonym for :meth:`.Cache.set`. 101 | 102 | This is here for backwards compatibility. 103 | 104 | """ 105 | 106 | def get(self, key, **kw): 107 | """Retrieve a value from the cache. 108 | 109 | :param key: the value's key. 110 | :param \**kw: cache configuration arguments. The 111 | backend is configured using these arguments upon first request. 112 | Subsequent requests that use the same series of configuration 113 | values will use that same backend. 114 | 115 | """ 116 | return self.impl.get(key, **self._get_cache_kw(kw)) 117 | 118 | def invalidate(self, key, **kw): 119 | """Invalidate a value in the cache. 120 | 121 | :param key: the value's key. 122 | :param \**kw: cache configuration arguments. The 123 | backend is configured using these arguments upon first request. 124 | Subsequent requests that use the same series of configuration 125 | values will use that same backend. 126 | 127 | """ 128 | self.impl.invalidate(key, **self._get_cache_kw(kw)) 129 | 130 | def invalidate_body(self): 131 | """Invalidate the cached content of the "body" method for this template. 132 | 133 | """ 134 | self.invalidate('render_body', __M_defname='render_body') 135 | 136 | def invalidate_def(self, name): 137 | """Invalidate the cached content of a particular <%def> within this template.""" 138 | 139 | self.invalidate('render_%s' % name, __M_defname='render_%s' % name) 140 | 141 | def invalidate_closure(self, name): 142 | """Invalidate a nested <%def> within this template. 143 | 144 | Caching of nested defs is a blunt tool as there is no 145 | management of scope - nested defs that use cache tags 146 | need to have names unique of all other nested defs in the 147 | template, else their content will be overwritten by 148 | each other. 149 | 150 | """ 151 | 152 | self.invalidate(name, __M_defname=name) 153 | 154 | def _get_cache_kw(self, kw): 155 | defname = kw.pop('__M_defname', None) 156 | if not defname: 157 | tmpl_kw = self.template.cache_args.copy() 158 | tmpl_kw.update(kw) 159 | return tmpl_kw 160 | elif defname in self._def_regions: 161 | return self._def_regions[defname] 162 | else: 163 | tmpl_kw = self.template.cache_args.copy() 164 | tmpl_kw.update(kw) 165 | self._def_regions[defname] = tmpl_kw 166 | return tmpl_kw 167 | 168 | class CacheImpl(object): 169 | """Provide a cache implementation for use by :class:`.Cache`.""" 170 | 171 | def __init__(self, cache): 172 | self.cache = cache 173 | 174 | def get_or_create(self, key, creation_function, **kw): 175 | """Retrieve a value from the cache, using the given creation function 176 | to generate a new value. 177 | 178 | This function *must* return a value, either from 179 | the cache, or via the given creation function. 180 | If the creation function is called, the newly 181 | created value should be populated into the cache 182 | under the given key before being returned. 183 | 184 | :param key: the value's key. 185 | :param creation_function: function that when called generates 186 | a new value. 187 | :param \**kw: cache configuration arguments. 188 | 189 | """ 190 | raise NotImplementedError() 191 | 192 | def set(self, key, value, **kw): 193 | """Place a value in the cache. 194 | 195 | :param key: the value's key. 196 | :param value: the value 197 | :param \**kw: cache configuration arguments. 198 | 199 | """ 200 | raise NotImplementedError() 201 | 202 | def get(self, key, **kw): 203 | """Retrieve a value from the cache. 204 | 205 | :param key: the value's key. 206 | :param \**kw: cache configuration arguments. 207 | 208 | """ 209 | raise NotImplementedError() 210 | 211 | def invalidate(self, key, **kw): 212 | """Invalidate a value in the cache. 213 | 214 | :param key: the value's key. 215 | :param \**kw: cache configuration arguments. 216 | 217 | """ 218 | raise NotImplementedError() 219 | -------------------------------------------------------------------------------- /mako/exceptions.py: -------------------------------------------------------------------------------- 1 | # mako/exceptions.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """exception classes""" 8 | 9 | import traceback, sys, re 10 | from mako import util 11 | 12 | class MakoException(Exception): 13 | pass 14 | 15 | class RuntimeException(MakoException): 16 | pass 17 | 18 | def _format_filepos(lineno, pos, filename): 19 | if filename is None: 20 | return " at line: %d char: %d" % (lineno, pos) 21 | else: 22 | return " in file '%s' at line: %d char: %d" % (filename, lineno, pos) 23 | 24 | 25 | class CompileException(MakoException): 26 | def __init__(self, message, source, lineno, pos, filename): 27 | MakoException.__init__(self, message + _format_filepos(lineno, pos, filename)) 28 | self.lineno =lineno 29 | self.pos = pos 30 | self.filename = filename 31 | self.source = source 32 | 33 | class SyntaxException(MakoException): 34 | def __init__(self, message, source, lineno, pos, filename): 35 | MakoException.__init__(self, message + _format_filepos(lineno, pos, filename)) 36 | self.lineno =lineno 37 | self.pos = pos 38 | self.filename = filename 39 | self.source = source 40 | 41 | class UnsupportedError(MakoException): 42 | """raised when a retired feature is used.""" 43 | 44 | class TemplateLookupException(MakoException): 45 | pass 46 | 47 | class TopLevelLookupException(TemplateLookupException): 48 | pass 49 | 50 | class RichTraceback(object): 51 | """Pulls the current exception from the sys traceback and extracts 52 | Mako-specific template information. 53 | 54 | See the usage examples in :ref:`handling_exceptions`. 55 | 56 | """ 57 | def __init__(self, error=None, traceback=None): 58 | self.source, self.lineno = "", 0 59 | 60 | if error is None or traceback is None: 61 | t, value, tback = sys.exc_info() 62 | 63 | if error is None: 64 | error = value or t 65 | 66 | if traceback is None: 67 | traceback = tback 68 | 69 | self.error = error 70 | self.records = self._init(traceback) 71 | 72 | if isinstance(self.error, (CompileException, SyntaxException)): 73 | import mako.template 74 | self.source = self.error.source 75 | self.lineno = self.error.lineno 76 | self._has_source = True 77 | 78 | self._init_message() 79 | 80 | @property 81 | def errorname(self): 82 | return util.exception_name(self.error) 83 | 84 | def _init_message(self): 85 | """Find a unicode representation of self.error""" 86 | try: 87 | self.message = unicode(self.error) 88 | except UnicodeError: 89 | try: 90 | self.message = str(self.error) 91 | except UnicodeEncodeError: 92 | # Fallback to args as neither unicode nor 93 | # str(Exception(u'\xe6')) work in Python < 2.6 94 | self.message = self.error.args[0] 95 | if not isinstance(self.message, unicode): 96 | self.message = unicode(self.message, 'ascii', 'replace') 97 | 98 | def _get_reformatted_records(self, records): 99 | for rec in records: 100 | if rec[6] is not None: 101 | yield (rec[4], rec[5], rec[2], rec[6]) 102 | else: 103 | yield tuple(rec[0:4]) 104 | 105 | @property 106 | def traceback(self): 107 | """return a list of 4-tuple traceback records (i.e. normal python 108 | format) with template-corresponding lines remapped to the originating 109 | template. 110 | 111 | """ 112 | return list(self._get_reformatted_records(self.records)) 113 | 114 | @property 115 | def reverse_records(self): 116 | return reversed(self.records) 117 | 118 | @property 119 | def reverse_traceback(self): 120 | """return the same data as traceback, except in reverse order. 121 | """ 122 | 123 | return list(self._get_reformatted_records(self.reverse_records)) 124 | 125 | def _init(self, trcback): 126 | """format a traceback from sys.exc_info() into 7-item tuples, 127 | containing the regular four traceback tuple items, plus the original 128 | template filename, the line number adjusted relative to the template 129 | source, and code line from that line number of the template.""" 130 | 131 | import mako.template 132 | mods = {} 133 | rawrecords = traceback.extract_tb(trcback) 134 | new_trcback = [] 135 | for filename, lineno, function, line in rawrecords: 136 | if not line: 137 | line = '' 138 | try: 139 | (line_map, template_lines) = mods[filename] 140 | except KeyError: 141 | try: 142 | info = mako.template._get_module_info(filename) 143 | module_source = info.code 144 | template_source = info.source 145 | template_filename = info.template_filename or filename 146 | except KeyError: 147 | # A normal .py file (not a Template) 148 | if not util.py3k: 149 | try: 150 | fp = open(filename, 'rb') 151 | encoding = util.parse_encoding(fp) 152 | fp.close() 153 | except IOError: 154 | encoding = None 155 | if encoding: 156 | line = line.decode(encoding) 157 | else: 158 | line = line.decode('ascii', 'replace') 159 | new_trcback.append((filename, lineno, function, line, 160 | None, None, None, None)) 161 | continue 162 | 163 | template_ln = module_ln = 1 164 | line_map = {} 165 | for line in module_source.split("\n"): 166 | match = re.match(r'\s*# SOURCE LINE (\d+)', line) 167 | if match: 168 | template_ln = int(match.group(1)) 169 | module_ln += 1 170 | line_map[module_ln] = template_ln 171 | template_lines = [line for line in 172 | template_source.split("\n")] 173 | mods[filename] = (line_map, template_lines) 174 | 175 | template_ln = line_map[lineno] 176 | if template_ln <= len(template_lines): 177 | template_line = template_lines[template_ln - 1] 178 | else: 179 | template_line = None 180 | new_trcback.append((filename, lineno, function, 181 | line, template_filename, template_ln, 182 | template_line, template_source)) 183 | if not self.source: 184 | for l in range(len(new_trcback)-1, 0, -1): 185 | if new_trcback[l][5]: 186 | self.source = new_trcback[l][7] 187 | self.lineno = new_trcback[l][5] 188 | break 189 | else: 190 | if new_trcback: 191 | try: 192 | # A normal .py file (not a Template) 193 | fp = open(new_trcback[-1][0], 'rb') 194 | encoding = util.parse_encoding(fp) 195 | fp.seek(0) 196 | self.source = fp.read() 197 | fp.close() 198 | if encoding: 199 | self.source = self.source.decode(encoding) 200 | except IOError: 201 | self.source = '' 202 | self.lineno = new_trcback[-1][1] 203 | return new_trcback 204 | 205 | 206 | def text_error_template(lookup=None): 207 | """Provides a template that renders a stack trace in a similar format to 208 | the Python interpreter, substituting source template filenames, line 209 | numbers and code for that of the originating source template, as 210 | applicable. 211 | 212 | """ 213 | import mako.template 214 | return mako.template.Template(r""" 215 | <%page args="error=None, traceback=None"/> 216 | <%! 217 | from mako.exceptions import RichTraceback 218 | %>\ 219 | <% 220 | tback = RichTraceback(error=error, traceback=traceback) 221 | %>\ 222 | Traceback (most recent call last): 223 | % for (filename, lineno, function, line) in tback.traceback: 224 | File "${filename}", line ${lineno}, in ${function or '?'} 225 | ${line | trim} 226 | % endfor 227 | ${tback.errorname}: ${tback.message} 228 | """) 229 | 230 | def html_error_template(): 231 | """Provides a template that renders a stack trace in an HTML format, 232 | providing an excerpt of code as well as substituting source template 233 | filenames, line numbers and code for that of the originating source 234 | template, as applicable. 235 | 236 | The template's default encoding_errors value is 'htmlentityreplace'. the 237 | template has two options. With the full option disabled, only a section of 238 | an HTML document is returned. with the css option disabled, the default 239 | stylesheet won't be included. 240 | 241 | """ 242 | import mako.template 243 | return mako.template.Template(r""" 244 | <%! 245 | from mako.exceptions import RichTraceback 246 | %> 247 | <%page args="full=True, css=True, error=None, traceback=None"/> 248 | % if full: 249 | 250 | 251 | Mako Runtime Error 252 | % endif 253 | % if css: 254 | 266 | % endif 267 | % if full: 268 | 269 | 270 | % endif 271 | 272 |

Error !

273 | <% 274 | tback = RichTraceback(error=error, traceback=traceback) 275 | src = tback.source 276 | line = tback.lineno 277 | if src: 278 | lines = src.split('\n') 279 | else: 280 | lines = None 281 | %> 282 |

${tback.errorname}: ${tback.message|h}

283 | 284 | % if lines: 285 |
286 |
287 | % for index in range(max(0, line-4),min(len(lines), line+5)): 288 | % if index + 1 == line: 289 |
${index + 1} ${lines[index] | h}
290 | % else: 291 |
${index + 1} ${lines[index] | h}
292 | % endif 293 | % endfor 294 |
295 |
296 | % endif 297 | 298 |
299 | % for (filename, lineno, function, line) in tback.reverse_traceback: 300 |
${filename}, line ${lineno}:
301 |
${line | h}
302 | % endfor 303 |
304 | 305 | % if full: 306 | 307 | 308 | % endif 309 | """, output_encoding=sys.getdefaultencoding(), encoding_errors='htmlentityreplace') 310 | -------------------------------------------------------------------------------- /mako/ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/four-o-four/omniverse/19219fb41fa83e7ce66b1864ba564a88333e0397/mako/ext/__init__.py -------------------------------------------------------------------------------- /mako/ext/autohandler.py: -------------------------------------------------------------------------------- 1 | # ext/autohandler.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """adds autohandler functionality to Mako templates. 8 | 9 | requires that the TemplateLookup class is used with templates. 10 | 11 | usage: 12 | 13 | <%! 14 | from mako.ext.autohandler import autohandler 15 | %> 16 | <%inherit file="${autohandler(template, context)}"/> 17 | 18 | 19 | or with custom autohandler filename: 20 | 21 | <%! 22 | from mako.ext.autohandler import autohandler 23 | %> 24 | <%inherit file="${autohandler(template, context, name='somefilename')}"/> 25 | 26 | """ 27 | 28 | import posixpath, os, re 29 | 30 | def autohandler(template, context, name='autohandler'): 31 | lookup = context.lookup 32 | _template_uri = template.module._template_uri 33 | if not lookup.filesystem_checks: 34 | try: 35 | return lookup._uri_cache[(autohandler, _template_uri, name)] 36 | except KeyError: 37 | pass 38 | 39 | tokens = re.findall(r'([^/]+)', posixpath.dirname(_template_uri)) + [name] 40 | while len(tokens): 41 | path = '/' + '/'.join(tokens) 42 | if path != _template_uri and _file_exists(lookup, path): 43 | if not lookup.filesystem_checks: 44 | return lookup._uri_cache.setdefault( 45 | (autohandler, _template_uri, name), path) 46 | else: 47 | return path 48 | if len(tokens) == 1: 49 | break 50 | tokens[-2:] = [name] 51 | 52 | if not lookup.filesystem_checks: 53 | return lookup._uri_cache.setdefault( 54 | (autohandler, _template_uri, name), None) 55 | else: 56 | return None 57 | 58 | def _file_exists(lookup, path): 59 | psub = re.sub(r'^/', '',path) 60 | for d in lookup.directories: 61 | if os.path.exists(d + '/' + psub): 62 | return True 63 | else: 64 | return False 65 | 66 | -------------------------------------------------------------------------------- /mako/ext/babelplugin.py: -------------------------------------------------------------------------------- 1 | # ext/babelplugin.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """gettext message extraction via Babel: http://babel.edgewall.org/""" 8 | from StringIO import StringIO 9 | 10 | from babel.messages.extract import extract_python 11 | 12 | from mako import lexer, parsetree 13 | 14 | def extract(fileobj, keywords, comment_tags, options): 15 | """Extract messages from Mako templates. 16 | 17 | :param fileobj: the file-like object the messages should be extracted from 18 | :param keywords: a list of keywords (i.e. function names) that should be 19 | recognized as translation functions 20 | :param comment_tags: a list of translator tags to search for and include 21 | in the results 22 | :param options: a dictionary of additional options (optional) 23 | :return: an iterator over ``(lineno, funcname, message, comments)`` tuples 24 | :rtype: ``iterator`` 25 | """ 26 | encoding = options.get('input_encoding', options.get('encoding', None)) 27 | 28 | template_node = lexer.Lexer(fileobj.read(), 29 | input_encoding=encoding).parse() 30 | for extracted in extract_nodes(template_node.get_children(), 31 | keywords, comment_tags, options): 32 | yield extracted 33 | 34 | def extract_nodes(nodes, keywords, comment_tags, options): 35 | """Extract messages from Mako's lexer node objects 36 | 37 | :param nodes: an iterable of Mako parsetree.Node objects to extract from 38 | :param keywords: a list of keywords (i.e. function names) that should be 39 | recognized as translation functions 40 | :param comment_tags: a list of translator tags to search for and include 41 | in the results 42 | :param options: a dictionary of additional options (optional) 43 | :return: an iterator over ``(lineno, funcname, message, comments)`` tuples 44 | :rtype: ``iterator`` 45 | """ 46 | translator_comments = [] 47 | in_translator_comments = False 48 | 49 | for node in nodes: 50 | child_nodes = None 51 | if in_translator_comments and isinstance(node, parsetree.Text) and \ 52 | not node.content.strip(): 53 | # Ignore whitespace within translator comments 54 | continue 55 | 56 | if isinstance(node, parsetree.Comment): 57 | value = node.text.strip() 58 | if in_translator_comments: 59 | translator_comments.extend(_split_comment(node.lineno, value)) 60 | continue 61 | for comment_tag in comment_tags: 62 | if value.startswith(comment_tag): 63 | in_translator_comments = True 64 | translator_comments.extend(_split_comment(node.lineno, 65 | value)) 66 | continue 67 | 68 | if isinstance(node, parsetree.DefTag): 69 | code = node.function_decl.code 70 | child_nodes = node.nodes 71 | elif isinstance(node, parsetree.BlockTag): 72 | code = node.body_decl.code 73 | child_nodes = node.nodes 74 | elif isinstance(node, parsetree.CallTag): 75 | code = node.code.code 76 | child_nodes = node.nodes 77 | elif isinstance(node, parsetree.PageTag): 78 | code = node.body_decl.code 79 | elif isinstance(node, parsetree.CallNamespaceTag): 80 | attribs = ', '.join(['%s=%s' % (key, val) 81 | for key, val in node.attributes.iteritems()]) 82 | code = '{%s}' % attribs 83 | child_nodes = node.nodes 84 | elif isinstance(node, parsetree.ControlLine): 85 | if node.isend: 86 | translator_comments = [] 87 | in_translator_comments = False 88 | continue 89 | code = node.text 90 | elif isinstance(node, parsetree.Code): 91 | # <% and <%! blocks would provide their own translator comments 92 | translator_comments = [] 93 | in_translator_comments = False 94 | 95 | code = node.code.code 96 | elif isinstance(node, parsetree.Expression): 97 | code = node.code.code 98 | else: 99 | translator_comments = [] 100 | in_translator_comments = False 101 | continue 102 | 103 | # Comments don't apply unless they immediately preceed the message 104 | if translator_comments and \ 105 | translator_comments[-1][0] < node.lineno - 1: 106 | translator_comments = [] 107 | else: 108 | translator_comments = \ 109 | [comment[1] for comment in translator_comments] 110 | 111 | if isinstance(code, unicode): 112 | code = code.encode('ascii', 'backslashreplace') 113 | code = StringIO(code) 114 | for lineno, funcname, messages, python_translator_comments \ 115 | in extract_python(code, keywords, comment_tags, options): 116 | yield (node.lineno + (lineno - 1), funcname, messages, 117 | translator_comments + python_translator_comments) 118 | 119 | translator_comments = [] 120 | in_translator_comments = False 121 | 122 | if child_nodes: 123 | for extracted in extract_nodes(child_nodes, keywords, comment_tags, 124 | options): 125 | yield extracted 126 | 127 | 128 | def _split_comment(lineno, comment): 129 | """Return the multiline comment at lineno split into a list of comment line 130 | numbers and the accompanying comment line""" 131 | return [(lineno + index, line) for index, line in 132 | enumerate(comment.splitlines())] 133 | -------------------------------------------------------------------------------- /mako/ext/beaker_cache.py: -------------------------------------------------------------------------------- 1 | """Provide a :class:`.CacheImpl` for the Beaker caching system.""" 2 | 3 | from mako import exceptions 4 | 5 | from mako.cache import CacheImpl 6 | 7 | _beaker_cache = None 8 | class BeakerCacheImpl(CacheImpl): 9 | """A :class:`.CacheImpl` provided for the Beaker caching system. 10 | 11 | This plugin is used by default, based on the default 12 | value of ``'beaker'`` for the ``cache_impl`` parameter of the 13 | :class:`.Template` or :class:`.TemplateLookup` classes. 14 | 15 | """ 16 | 17 | def __init__(self, cache): 18 | global _beaker_cache 19 | if _beaker_cache is None: 20 | try: 21 | from beaker import cache as beaker_cache 22 | except ImportError, e: 23 | raise exceptions.RuntimeException( 24 | "the Beaker package is required to use cache " 25 | "functionality.") 26 | 27 | if 'manager' in cache.template.cache_args: 28 | _beaker_cache = cache.template.cache_args['manager'] 29 | else: 30 | _beaker_cache = beaker_cache.CacheManager() 31 | super(BeakerCacheImpl, self).__init__(cache) 32 | 33 | def _get_cache(self, **kw): 34 | expiretime = kw.pop('timeout', None) 35 | if 'dir' in kw: 36 | kw['data_dir'] = kw.pop('dir') 37 | elif self.cache.template.module_directory: 38 | kw['data_dir'] = self.cache.template.module_directory 39 | 40 | if 'manager' in kw: 41 | kw.pop('manager') 42 | 43 | if kw.get('type') == 'memcached': 44 | kw['type'] = 'ext:memcached' 45 | 46 | if 'region' in kw: 47 | region = kw.pop('region') 48 | cache = _beaker_cache.get_cache_region(self.cache.id, region, **kw) 49 | else: 50 | cache = _beaker_cache.get_cache(self.cache.id, **kw) 51 | cache_args = {'starttime':self.cache.starttime} 52 | if expiretime: 53 | cache_args['expiretime'] = expiretime 54 | return cache, cache_args 55 | 56 | def get_or_create(self, key, creation_function, **kw): 57 | cache, kw = self._get_cache(**kw) 58 | return cache.get(key, createfunc=creation_function, **kw) 59 | 60 | def put(self, key, value, **kw): 61 | cache, kw = self._get_cache(**kw) 62 | cache.put(key, value, **kw) 63 | 64 | def get(self, key, **kw): 65 | cache, kw = self._get_cache(**kw) 66 | return cache.get(key, **kw) 67 | 68 | def invalidate(self, key, **kw): 69 | cache, kw = self._get_cache(**kw) 70 | cache.remove_value(key, **kw) 71 | -------------------------------------------------------------------------------- /mako/ext/preprocessors.py: -------------------------------------------------------------------------------- 1 | # ext/preprocessors.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """preprocessing functions, used with the 'preprocessor' 8 | argument on Template, TemplateLookup""" 9 | 10 | import re 11 | 12 | def convert_comments(text): 13 | """preprocess old style comments. 14 | 15 | example: 16 | 17 | from mako.ext.preprocessors import convert_comments 18 | t = Template(..., preprocessor=preprocess_comments)""" 19 | return re.sub(r'(?<=\n)\s*#[^#]', "##", text) 20 | 21 | -------------------------------------------------------------------------------- /mako/ext/pygmentplugin.py: -------------------------------------------------------------------------------- 1 | # ext/pygmentplugin.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | import re 8 | try: 9 | set 10 | except NameError: 11 | from sets import Set as set 12 | 13 | from pygments.lexers.web import \ 14 | HtmlLexer, XmlLexer, JavascriptLexer, CssLexer 15 | from pygments.lexers.agile import PythonLexer 16 | from pygments.lexer import Lexer, DelegatingLexer, RegexLexer, bygroups, \ 17 | include, using, this 18 | from pygments.token import Error, Punctuation, \ 19 | Text, Comment, Operator, Keyword, Name, String, Number, Other, Literal 20 | from pygments.util import html_doctype_matches, looks_like_xml 21 | 22 | class MakoLexer(RegexLexer): 23 | name = 'Mako' 24 | aliases = ['mako'] 25 | filenames = ['*.mao'] 26 | 27 | tokens = { 28 | 'root': [ 29 | (r'(\s*)(\%)(\s*end(?:\w+))(\n|\Z)', 30 | bygroups(Text, Comment.Preproc, Keyword, Other)), 31 | (r'(\s*)(\%(?!%))([^\n]*)(\n|\Z)', 32 | bygroups(Text, Comment.Preproc, using(PythonLexer), Other)), 33 | (r'(\s*)(##[^\n]*)(\n|\Z)', 34 | bygroups(Text, Comment.Preproc, Other)), 35 | (r'''(?s)<%doc>.*?''', Comment.Preproc), 36 | (r'(<%)([\w\.\:]+)', bygroups(Comment.Preproc, Name.Builtin), 'tag'), 37 | (r'()', bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), 38 | (r'<%(?=([\w\.\:]+))', Comment.Preproc, 'ondeftags'), 39 | (r'(<%(?:!?))(.*?)(%>)(?s)', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), 40 | (r'(\$\{)(.*?)(\})', 41 | bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), 42 | (r'''(?sx) 43 | (.+?) # anything, followed by: 44 | (?: 45 | (?<=\n)(?=%(?!%)|\#\#) | # an eval or comment line 46 | (?=\#\*) | # multiline comment 47 | (?=', Comment.Preproc, '#pop'), 67 | (r'\s+', Text), 68 | ], 69 | 'attr': [ 70 | ('".*?"', String, '#pop'), 71 | ("'.*?'", String, '#pop'), 72 | (r'[^\s>]+', String, '#pop'), 73 | ], 74 | } 75 | 76 | 77 | class MakoHtmlLexer(DelegatingLexer): 78 | name = 'HTML+Mako' 79 | aliases = ['html+mako'] 80 | 81 | def __init__(self, **options): 82 | super(MakoHtmlLexer, self).__init__(HtmlLexer, MakoLexer, 83 | **options) 84 | 85 | class MakoXmlLexer(DelegatingLexer): 86 | name = 'XML+Mako' 87 | aliases = ['xml+mako'] 88 | 89 | def __init__(self, **options): 90 | super(MakoXmlLexer, self).__init__(XmlLexer, MakoLexer, 91 | **options) 92 | 93 | class MakoJavascriptLexer(DelegatingLexer): 94 | name = 'JavaScript+Mako' 95 | aliases = ['js+mako', 'javascript+mako'] 96 | 97 | def __init__(self, **options): 98 | super(MakoJavascriptLexer, self).__init__(JavascriptLexer, 99 | MakoLexer, **options) 100 | 101 | class MakoCssLexer(DelegatingLexer): 102 | name = 'CSS+Mako' 103 | aliases = ['css+mako'] 104 | 105 | def __init__(self, **options): 106 | super(MakoCssLexer, self).__init__(CssLexer, MakoLexer, 107 | **options) 108 | -------------------------------------------------------------------------------- /mako/ext/turbogears.py: -------------------------------------------------------------------------------- 1 | # ext/turbogears.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | import re, inspect 8 | from mako.lookup import TemplateLookup 9 | from mako.template import Template 10 | 11 | class TGPlugin(object): 12 | """TurboGears compatible Template Plugin.""" 13 | 14 | def __init__(self, extra_vars_func=None, options=None, extension='mak'): 15 | self.extra_vars_func = extra_vars_func 16 | self.extension = extension 17 | if not options: 18 | options = {} 19 | 20 | # Pull the options out and initialize the lookup 21 | lookup_options = {} 22 | for k, v in options.iteritems(): 23 | if k.startswith('mako.'): 24 | lookup_options[k[5:]] = v 25 | elif k in ['directories', 'filesystem_checks', 'module_directory']: 26 | lookup_options[k] = v 27 | self.lookup = TemplateLookup(**lookup_options) 28 | 29 | self.tmpl_options = {} 30 | # transfer lookup args to template args, based on those available 31 | # in getargspec 32 | for kw in inspect.getargspec(Template.__init__)[0]: 33 | if kw in lookup_options: 34 | self.tmpl_options[kw] = lookup_options[kw] 35 | 36 | def load_template(self, templatename, template_string=None): 37 | """Loads a template from a file or a string""" 38 | if template_string is not None: 39 | return Template(template_string, **self.tmpl_options) 40 | # Translate TG dot notation to normal / template path 41 | if '/' not in templatename: 42 | templatename = '/' + templatename.replace('.', '/') + '.' + self.extension 43 | 44 | # Lookup template 45 | return self.lookup.get_template(templatename) 46 | 47 | def render(self, info, format="html", fragment=False, template=None): 48 | if isinstance(template, basestring): 49 | template = self.load_template(template) 50 | 51 | # Load extra vars func if provided 52 | if self.extra_vars_func: 53 | info.update(self.extra_vars_func()) 54 | 55 | return template.render(**info) 56 | 57 | -------------------------------------------------------------------------------- /mako/filters.py: -------------------------------------------------------------------------------- 1 | # mako/filters.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | 8 | import re, urllib, htmlentitydefs, codecs 9 | from StringIO import StringIO 10 | from mako import util 11 | 12 | xml_escapes = { 13 | '&' : '&', 14 | '>' : '>', 15 | '<' : '<', 16 | '"' : '"', # also " in html-only 17 | "'" : ''' # also ' in html-only 18 | } 19 | 20 | # XXX: " is valid in HTML and XML 21 | # ' is not valid HTML, but is valid XML 22 | 23 | def legacy_html_escape(string): 24 | """legacy HTML escape for non-unicode mode.""" 25 | 26 | return re.sub(r'([&<"\'>])', lambda m: xml_escapes[m.group()], string) 27 | 28 | try: 29 | import markupsafe 30 | html_escape = markupsafe.escape 31 | except ImportError: 32 | html_escape = legacy_html_escape 33 | 34 | 35 | def xml_escape(string): 36 | return re.sub(r'([&<"\'>])', lambda m: xml_escapes[m.group()], string) 37 | 38 | def url_escape(string): 39 | # convert into a list of octets 40 | string = string.encode("utf8") 41 | return urllib.quote_plus(string) 42 | 43 | def url_unescape(string): 44 | text = urllib.unquote_plus(string) 45 | if not is_ascii_str(text): 46 | text = text.decode("utf8") 47 | return text 48 | 49 | def trim(string): 50 | return string.strip() 51 | 52 | 53 | class Decode(object): 54 | def __getattr__(self, key): 55 | def decode(x): 56 | if isinstance(x, unicode): 57 | return x 58 | elif not isinstance(x, str): 59 | return unicode(str(x), encoding=key) 60 | else: 61 | return unicode(x, encoding=key) 62 | return decode 63 | decode = Decode() 64 | 65 | 66 | _ASCII_re = re.compile(r'\A[\x00-\x7f]*\Z') 67 | 68 | def is_ascii_str(text): 69 | return isinstance(text, str) and _ASCII_re.match(text) 70 | 71 | ################################################################ 72 | 73 | class XMLEntityEscaper(object): 74 | def __init__(self, codepoint2name, name2codepoint): 75 | self.codepoint2entity = dict([(c, u'&%s;' % n) 76 | for c,n in codepoint2name.iteritems()]) 77 | self.name2codepoint = name2codepoint 78 | 79 | def escape_entities(self, text): 80 | """Replace characters with their character entity references. 81 | 82 | Only characters corresponding to a named entity are replaced. 83 | """ 84 | return unicode(text).translate(self.codepoint2entity) 85 | 86 | def __escape(self, m): 87 | codepoint = ord(m.group()) 88 | try: 89 | return self.codepoint2entity[codepoint] 90 | except (KeyError, IndexError): 91 | return '&#x%X;' % codepoint 92 | 93 | 94 | __escapable = re.compile(r'["&<>]|[^\x00-\x7f]') 95 | 96 | def escape(self, text): 97 | """Replace characters with their character references. 98 | 99 | Replace characters by their named entity references. 100 | Non-ASCII characters, if they do not have a named entity reference, 101 | are replaced by numerical character references. 102 | 103 | The return value is guaranteed to be ASCII. 104 | """ 105 | return self.__escapable.sub(self.__escape, unicode(text) 106 | ).encode('ascii') 107 | 108 | # XXX: This regexp will not match all valid XML entity names__. 109 | # (It punts on details involving involving CombiningChars and Extenders.) 110 | # 111 | # .. __: http://www.w3.org/TR/2000/REC-xml-20001006#NT-EntityRef 112 | __characterrefs = re.compile(r'''& (?: 113 | \#(\d+) 114 | | \#x([\da-f]+) 115 | | ( (?!\d) [:\w] [-.:\w]+ ) 116 | ) ;''', 117 | re.X | re.UNICODE) 118 | 119 | def __unescape(self, m): 120 | dval, hval, name = m.groups() 121 | if dval: 122 | codepoint = int(dval) 123 | elif hval: 124 | codepoint = int(hval, 16) 125 | else: 126 | codepoint = self.name2codepoint.get(name, 0xfffd) 127 | # U+FFFD = "REPLACEMENT CHARACTER" 128 | if codepoint < 128: 129 | return chr(codepoint) 130 | return unichr(codepoint) 131 | 132 | def unescape(self, text): 133 | """Unescape character references. 134 | 135 | All character references (both entity references and numerical 136 | character references) are unescaped. 137 | """ 138 | return self.__characterrefs.sub(self.__unescape, text) 139 | 140 | 141 | _html_entities_escaper = XMLEntityEscaper(htmlentitydefs.codepoint2name, 142 | htmlentitydefs.name2codepoint) 143 | 144 | html_entities_escape = _html_entities_escaper.escape_entities 145 | html_entities_unescape = _html_entities_escaper.unescape 146 | 147 | 148 | def htmlentityreplace_errors(ex): 149 | """An encoding error handler. 150 | 151 | This python `codecs`_ error handler replaces unencodable 152 | characters with HTML entities, or, if no HTML entity exists for 153 | the character, XML character references. 154 | 155 | >>> u'The cost was \u20ac12.'.encode('latin1', 'htmlentityreplace') 156 | 'The cost was €12.' 157 | """ 158 | if isinstance(ex, UnicodeEncodeError): 159 | # Handle encoding errors 160 | bad_text = ex.object[ex.start:ex.end] 161 | text = _html_entities_escaper.escape(bad_text) 162 | return (unicode(text), ex.end) 163 | raise ex 164 | 165 | codecs.register_error('htmlentityreplace', htmlentityreplace_errors) 166 | 167 | 168 | # TODO: options to make this dynamic per-compilation will be added in a later release 169 | DEFAULT_ESCAPES = { 170 | 'x':'filters.xml_escape', 171 | 'h':'filters.html_escape', 172 | 'u':'filters.url_escape', 173 | 'trim':'filters.trim', 174 | 'entity':'filters.html_entities_escape', 175 | 'unicode':'unicode', 176 | 'decode':'decode', 177 | 'str':'str', 178 | 'n':'n' 179 | } 180 | 181 | if util.py3k: 182 | DEFAULT_ESCAPES.update({ 183 | 'unicode':'str' 184 | }) 185 | 186 | NON_UNICODE_ESCAPES = DEFAULT_ESCAPES.copy() 187 | NON_UNICODE_ESCAPES['h'] = 'filters.legacy_html_escape' 188 | 189 | -------------------------------------------------------------------------------- /mako/pygen.py: -------------------------------------------------------------------------------- 1 | # mako/pygen.py 2 | # Copyright (C) 2006-2012 the Mako authors and contributors 3 | # 4 | # This module is part of Mako and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """utilities for generating and formatting literal Python code.""" 8 | 9 | import re, string 10 | from StringIO import StringIO 11 | from mako import exceptions 12 | 13 | class PythonPrinter(object): 14 | def __init__(self, stream): 15 | # indentation counter 16 | self.indent = 0 17 | 18 | # a stack storing information about why we incremented 19 | # the indentation counter, to help us determine if we 20 | # should decrement it 21 | self.indent_detail = [] 22 | 23 | # the string of whitespace multiplied by the indent 24 | # counter to produce a line 25 | self.indentstring = " " 26 | 27 | # the stream we are writing to 28 | self.stream = stream 29 | 30 | # a list of lines that represents a buffered "block" of code, 31 | # which can be later printed relative to an indent level 32 | self.line_buffer = [] 33 | 34 | self.in_indent_lines = False 35 | 36 | self._reset_multi_line_flags() 37 | 38 | def write(self, text): 39 | self.stream.write(text) 40 | 41 | def write_indented_block(self, block): 42 | """print a line or lines of python which already contain indentation. 43 | 44 | The indentation of the total block of lines will be adjusted to that of 45 | the current indent level.""" 46 | self.in_indent_lines = False 47 | for l in re.split(r'\r?\n', block): 48 | self.line_buffer.append(l) 49 | 50 | def writelines(self, *lines): 51 | """print a series of lines of python.""" 52 | for line in lines: 53 | self.writeline(line) 54 | 55 | def writeline(self, line): 56 | """print a line of python, indenting it according to the current 57 | indent level. 58 | 59 | this also adjusts the indentation counter according to the 60 | content of the line. 61 | 62 | """ 63 | 64 | if not self.in_indent_lines: 65 | self._flush_adjusted_lines() 66 | self.in_indent_lines = True 67 | 68 | decreased_indent = False 69 | 70 | if (line is None or 71 | re.match(r"^\s*#",line) or 72 | re.match(r"^\s*$", line) 73 | ): 74 | hastext = False 75 | else: 76 | hastext = True 77 | 78 | is_comment = line and len(line) and line[0] == '#' 79 | 80 | # see if this line should decrease the indentation level 81 | if (not decreased_indent and 82 | not is_comment and 83 | (not hastext or self._is_unindentor(line)) 84 | ): 85 | 86 | if self.indent > 0: 87 | self.indent -=1 88 | # if the indent_detail stack is empty, the user 89 | # probably put extra closures - the resulting 90 | # module wont compile. 91 | if len(self.indent_detail) == 0: 92 | raise exceptions.SyntaxException( 93 | "Too many whitespace closures") 94 | self.indent_detail.pop() 95 | 96 | if line is None: 97 | return 98 | 99 | # write the line 100 | self.stream.write(self._indent_line(line) + "\n") 101 | 102 | # see if this line should increase the indentation level. 103 | # note that a line can both decrase (before printing) and 104 | # then increase (after printing) the indentation level. 105 | 106 | if re.search(r":[ \t]*(?:#.*)?$", line): 107 | # increment indentation count, and also 108 | # keep track of what the keyword was that indented us, 109 | # if it is a python compound statement keyword 110 | # where we might have to look for an "unindent" keyword 111 | match = re.match(r"^\s*(if|try|elif|while|for)", line) 112 | if match: 113 | # its a "compound" keyword, so we will check for "unindentors" 114 | indentor = match.group(1) 115 | self.indent +=1 116 | self.indent_detail.append(indentor) 117 | else: 118 | indentor = None 119 | # its not a "compound" keyword. but lets also 120 | # test for valid Python keywords that might be indenting us, 121 | # else assume its a non-indenting line 122 | m2 = re.match(r"^\s*(def|class|else|elif|except|finally)", line) 123 | if m2: 124 | self.indent += 1 125 | self.indent_detail.append(indentor) 126 | 127 | def close(self): 128 | """close this printer, flushing any remaining lines.""" 129 | self._flush_adjusted_lines() 130 | 131 | def _is_unindentor(self, line): 132 | """return true if the given line is an 'unindentor', 133 | relative to the last 'indent' event received. 134 | 135 | """ 136 | 137 | # no indentation detail has been pushed on; return False 138 | if len(self.indent_detail) == 0: 139 | return False 140 | 141 | indentor = self.indent_detail[-1] 142 | 143 | # the last indent keyword we grabbed is not a 144 | # compound statement keyword; return False 145 | if indentor is None: 146 | return False 147 | 148 | # if the current line doesnt have one of the "unindentor" keywords, 149 | # return False 150 | match = re.match(r"^\s*(else|elif|except|finally).*\:", line) 151 | if not match: 152 | return False 153 | 154 | # whitespace matches up, we have a compound indentor, 155 | # and this line has an unindentor, this 156 | # is probably good enough 157 | return True 158 | 159 | # should we decide that its not good enough, heres 160 | # more stuff to check. 161 | #keyword = match.group(1) 162 | 163 | # match the original indent keyword 164 | #for crit in [ 165 | # (r'if|elif', r'else|elif'), 166 | # (r'try', r'except|finally|else'), 167 | # (r'while|for', r'else'), 168 | #]: 169 | # if re.match(crit[0], indentor) and re.match(crit[1], keyword): 170 | # return True 171 | 172 | #return False 173 | 174 | def _indent_line(self, line, stripspace=''): 175 | """indent the given line according to the current indent level. 176 | 177 | stripspace is a string of space that will be truncated from the 178 | start of the line before indenting.""" 179 | 180 | return re.sub(r"^%s" % stripspace, self.indentstring 181 | * self.indent, line) 182 | 183 | def _reset_multi_line_flags(self): 184 | """reset the flags which would indicate we are in a backslashed 185 | or triple-quoted section.""" 186 | 187 | self.backslashed, self.triplequoted = False, False 188 | 189 | def _in_multi_line(self, line): 190 | """return true if the given line is part of a multi-line block, 191 | via backslash or triple-quote.""" 192 | 193 | # we are only looking for explicitly joined lines here, not 194 | # implicit ones (i.e. brackets, braces etc.). this is just to 195 | # guard against the possibility of modifying the space inside of 196 | # a literal multiline string with unfortunately placed 197 | # whitespace 198 | 199 | current_state = (self.backslashed or self.triplequoted) 200 | 201 | if re.search(r"\\$", line): 202 | self.backslashed = True 203 | else: 204 | self.backslashed = False 205 | 206 | triples = len(re.findall(r"\"\"\"|\'\'\'", line)) 207 | if triples == 1 or triples % 2 != 0: 208 | self.triplequoted = not self.triplequoted 209 | 210 | return current_state 211 | 212 | def _flush_adjusted_lines(self): 213 | stripspace = None 214 | self._reset_multi_line_flags() 215 | 216 | for entry in self.line_buffer: 217 | if self._in_multi_line(entry): 218 | self.stream.write(entry + "\n") 219 | else: 220 | entry = entry.expandtabs() 221 | if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry): 222 | stripspace = re.match(r"^([ \t]*)", entry).group(1) 223 | self.stream.write(self._indent_line(entry, stripspace) + "\n") 224 | 225 | self.line_buffer = [] 226 | self._reset_multi_line_flags() 227 | 228 | 229 | def adjust_whitespace(text): 230 | """remove the left-whitespace margin of a block of Python code.""" 231 | 232 | state = [False, False] 233 | (backslashed, triplequoted) = (0, 1) 234 | 235 | def in_multi_line(line): 236 | start_state = (state[backslashed] or state[triplequoted]) 237 | 238 | if re.search(r"\\$", line): 239 | state[backslashed] = True 240 | else: 241 | state[backslashed] = False 242 | 243 | def match(reg, t): 244 | m = re.match(reg, t) 245 | if m: 246 | return m, t[len(m.group(0)):] 247 | else: 248 | return None, t 249 | 250 | while line: 251 | if state[triplequoted]: 252 | m, line = match(r"%s" % state[triplequoted], line) 253 | if m: 254 | state[triplequoted] = False 255 | else: 256 | m, line = match(r".*?(?=%s|$)" % state[triplequoted], line) 257 | else: 258 | m, line = match(r'#', line) 259 | if m: 260 | return start_state 261 | 262 | m, line = match(r"\"\"\"|\'\'\'", line) 263 | if m: 264 | state[triplequoted] = m.group(0) 265 | continue 266 | 267 | m, line = match(r".*?(?=\"\"\"|\'\'\'|#|$)", line) 268 | 269 | return start_state 270 | 271 | def _indent_line(line, stripspace = ''): 272 | return re.sub(r"^%s" % stripspace, '', line) 273 | 274 | lines = [] 275 | stripspace = None 276 | 277 | for line in re.split(r'\r?\n', text): 278 | if in_multi_line(line): 279 | lines.append(line) 280 | else: 281 | line = line.expandtabs() 282 | if stripspace is None and re.search(r"^[ \t]*[^# \t]", line): 283 | stripspace = re.match(r"^([ \t]*)", line).group(1) 284 | lines.append(_indent_line(line, stripspace)) 285 | return "\n".join(lines) 286 | -------------------------------------------------------------------------------- /nntplib_ssl.py: -------------------------------------------------------------------------------- 1 | import nntplib 2 | import re 3 | import socket 4 | 5 | socket.setdefaulttimeout(120) 6 | 7 | NNTP_SSL_PORT = 563 8 | CRLF = '\r\n' 9 | 10 | class NNTP_SSL(nntplib.NNTP): 11 | """NNTP client class over SSL connection 12 | 13 | Instantiate with: NNTP_SSL(hostname, port=NNTP_SSL_PORT, user=None, 14 | password=None, readermode=None, usenetrc=True, 15 | keyfile=None, certfile=None) 16 | 17 | - host: hostname to connect to 18 | - port: port to connect to (default the standard NNTP port) 19 | - user: username to authenticate with 20 | - password: password to use with username 21 | - readermode: if true, send 'mode reader' command after 22 | connecting. 23 | - keyfile: PEM formatted file that contains your private key 24 | - certfile: PEM formatted certificate chain file 25 | 26 | See the methods of the parent class NNTP for more documentation. 27 | """ 28 | 29 | def __init__(self, host, port = NNTP_SSL_PORT, user=None, password=None, 30 | readermode=None, usenetrc=True, 31 | keyfile = None, certfile = None): 32 | self.host = host 33 | self.port = port 34 | self.keyfile = keyfile 35 | self.certfile = certfile 36 | self.buffer = "" 37 | msg = "getaddrinfo returns an empty list" 38 | self.sock = None 39 | 40 | try: 41 | results = socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM) 42 | except socket.gaierror, e: 43 | raise IOError(e) 44 | 45 | for res in results: 46 | af, socktype, proto, canonname, sa = res 47 | try: 48 | self.sock = socket.socket(af, socktype, proto) 49 | self.sock.connect(sa) 50 | except socket.error, msg: 51 | if self.sock: 52 | self.sock.close() 53 | self.sock = None 54 | continue 55 | break 56 | if not self.sock: 57 | raise socket.error, msg 58 | self.file = self.sock.makefile('rb') 59 | try: socket.ssl 60 | except AttributeError: self.sslobj=None 61 | else: self.sslobj = socket.ssl(self.sock, self.keyfile, self.certfile) 62 | self.debugging = 0 63 | self.welcome = self.getresp() 64 | 65 | # 'mode reader' is sometimes necessary to enable 'reader' mode. 66 | # However, the order in which 'mode reader' and 'authinfo' need to 67 | # arrive differs between some NNTP servers. Try to send 68 | # 'mode reader', and if it fails with an authorization failed 69 | # error, try again after sending authinfo. 70 | readermode_afterauth = 0 71 | if readermode: 72 | try: 73 | self.welcome = self.shortcmd('mode reader') 74 | except NNTPPermanentError: 75 | # error 500, probably 'not implemented' 76 | pass 77 | except NNTPTemporaryError, e: 78 | if user and e.response[:3] == '480': 79 | # Need authorization before 'mode reader' 80 | readermode_afterauth = 1 81 | else: 82 | raise 83 | # If no login/password was specified, try to get them from ~/.netrc 84 | # Presume that if .netc has an entry, NNRP authentication is required. 85 | try: 86 | if usenetrc and not user: 87 | import netrc 88 | credentials = netrc.netrc() 89 | auth = credentials.authenticators(host) 90 | if auth: 91 | user = auth[0] 92 | password = auth[2] 93 | except IOError: 94 | pass 95 | # Perform NNRP authentication if needed. 96 | if user: 97 | resp = self.shortcmd('authinfo user '+user) 98 | if resp[:3] == '381': 99 | if not password: 100 | raise NNTPReplyError(resp) 101 | else: 102 | resp = self.shortcmd( 103 | 'authinfo pass '+password) 104 | if resp[:3] != '281': 105 | raise NNTPPermanentError(resp) 106 | if readermode_afterauth: 107 | try: 108 | self.welcome = self.shortcmd('mode reader') 109 | except NNTPPermanentError: 110 | # error 500, probably 'not implemented' 111 | pass 112 | 113 | def _fillBuffer(self): 114 | localbuf = self.sslobj.read() 115 | if len(localbuf) == 0: 116 | raise nntplib.NNTPProtocolError('-ERR EOF') 117 | self.buffer += localbuf 118 | 119 | def getline(self): 120 | line = "" 121 | renewline = re.compile(r'.*?\n') 122 | match = renewline.match(self.buffer) 123 | while not match: 124 | self._fillBuffer() 125 | match = renewline.match(self.buffer) 126 | line = match.group(0) 127 | self.buffer = renewline.sub('' ,self.buffer, 1) 128 | if self.debugging > 1: print '*get*', repr(line) 129 | 130 | if not line: raise EOFError 131 | if line[-2:] == CRLF: line = line[:-2] 132 | elif line[-1:] in CRLF: line = line[:-1] 133 | return line 134 | 135 | def putline(self, line): 136 | if self.debugging > 1: print '*put*', repr(line) 137 | line += CRLF 138 | bytes = len(line) 139 | while bytes > 0: 140 | sent = self.sslobj.write(line) 141 | if sent == bytes: 142 | break # avoid copy 143 | line = line[sent:] 144 | bytes = bytes - sent 145 | 146 | def quit(self): 147 | """Process a QUIT command and close the socket. Returns: 148 | - resp: server response if successful""" 149 | try: 150 | resp = self.shortcmd('QUIT') 151 | except nntplib.NNTPProtocolError, val: 152 | resp = val 153 | self.sock.close() 154 | del self.sslobj, self.sock 155 | return resp 156 | -------------------------------------------------------------------------------- /parse.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | re_parts = re.compile(r"\((?P\d+)/(?P\d+)\)") 4 | re_yenc = re.compile(r"yEnc") 5 | re_size = re.compile(r"(?P\d+KB)") 6 | 7 | # selecting the filename, perferring to grab everything inside of quotes. some posters do not use 8 | # quotes and don't consistently delimit what is filename and what is descriptive subject. 9 | re_filename = re.compile(r"\"??(?P[^\"]*?(\.part\d{1,2}\.rar|\.vol\d{1,2}\+\d{1,2}|\.cbr|\.cbz|\.pdf|\.rar|\.par2|\.par|\.zip|\.sfv|\.nfo|\.nzb)+)\"??", re.I) 10 | re_filename_wo_quotes = re.compile("") 11 | re_similar = re.compile(r"\)(\d+)/\d+\(") 12 | re_x_of_y = re.compile(r"\d{1,3} of \d{1,3} - ") 13 | 14 | with open('bad.txt', 'r') as f: 15 | lines = f.readlines() 16 | re_bad = [] 17 | 18 | for line in f.readlines(): 19 | re_bad.append(re.compile(line, re.I)) 20 | 21 | def subject_to_totals(subject): 22 | 23 | matches = re_parts.search(subject) 24 | if matches: 25 | return (matches.group("part"), matches.group("total_parts")) 26 | else: 27 | return (None, None) 28 | 29 | def subject_to_yenc(subject): 30 | 31 | matches = re.search(r"yEnc", subject) 32 | if matches: return True 33 | else: return False 34 | 35 | def subject_to_size(subject): 36 | 37 | matches = re_size.search(subject) 38 | if matches: 39 | return matches.group("size") 40 | else: 41 | return None 42 | 43 | def subject_to_filename(subject): 44 | 45 | matches = re_filename.search(subject) 46 | if matches: 47 | # remove any text such as "1 of 8" 48 | return re_x_of_y.sub("", matches.group("filename")).strip() 49 | else: 50 | return None 51 | 52 | def subject_to_similar(subject): 53 | # my trick to subtitute right to left by reversing the string, make the substitution, 54 | # then reverse back. 55 | re.sub 56 | return re_similar.sub(")/1(", subject[::-1], 1)[::-1] 57 | 58 | def bad_filter(text): 59 | for r in re_bad: 60 | if r.search(text): 61 | return True 62 | return False 63 | 64 | 65 | if __name__ == "__main__": 66 | assert(subject_to_filename('(1/17) "Heavy Metal Vol.9.rar" - 713.49 MB - Heavy Metal (Complete Comic) Vol.9 yEnc (1/131)') == 67 | "Heavy Metal Vol.9.rar") 68 | assert(subject_to_filename('(1/17) "Heavy Metal Vol.9.part1.rar" - 713.49 MB - Heavy Metal (Complete Comic) Vol.9 yEnc (1/261)') == 69 | "Heavy Metal Vol.9.part1.rar") 70 | 71 | assert(subject_to_filename('(Kroost 6) [4/7] - "Kroost 6.cbr.vol01+2.PAR2" yEnc (1/)') == 72 | "Kroost 6.cbr.vol01+2.PAR2") 73 | assert(subject_to_filename("(I'M BA-A-A-A-ACK!!!!) Kull The Destroyer #25 ||1978 || FBScan || [4/5] - \"Kull The Destroyer 025 (1978) (FBScan).CBR.vol1+2.PAR2\"") == 74 | "Kull The Destroyer 025 (1978) (FBScan).CBR.vol1+2.PAR2") 75 | 76 | 77 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | # Settings Manager 2 | 3 | import ConfigParser 4 | import os 5 | import logging 6 | 7 | # todo a little thread safety would help 8 | 9 | _shared_settings = ConfigParser.RawConfigParser() 10 | 11 | 12 | class NoSectionError(ConfigParser.NoSectionError): 13 | def __init__(self, *args, **kwargs): 14 | ConfigParser.NoSectionError.__init__(self, *args, **kwargs) 15 | 16 | class NoOptionError(ConfigParser.NoOptionError): 17 | def __init__(self, *args, **kwargs): 18 | ConfigParser.NoOptionError.__init__(self, *args, **kwargs) 19 | 20 | class SettingsError(ConfigParser.Error): 21 | def __init__(self, *args, **kwargs): 22 | ConfigParser.Error.__init__(self, *args, **kwargs) 23 | 24 | 25 | # creates a path to a file resource relative to the current working directory 26 | # todo refactor into new module 27 | def build_path(*args): 28 | return os.path.join(os.path.abspath("."), *args) 29 | 30 | def load(filename): 31 | _shared_settings.read(build_path(filename)); 32 | 33 | def save(filename): 34 | logging.getLogger().info("Saving settings.") 35 | try: 36 | fobj = open(build_path(filename), "wb") 37 | _shared_settings.write(fobj) 38 | except IOErrror, e: 39 | logging.getLogger().exception("Unable to save settings.") 40 | 41 | def get(option): 42 | (section, option) = option.split(":") 43 | if _shared_settings.has_section(section): 44 | try: 45 | return _shared_settings.get(section=section, option=option) 46 | except ConfigParser.NoOptionError, error: 47 | raise NoOptionError(section, option) 48 | else: 49 | raise NoSectionError("No section named %s." % section) 50 | 51 | def remove(option): 52 | (section, option) = option.split(":") 53 | _shared_settings.remove_option(section, option) 54 | 55 | # parses the group name/last message id pairing 56 | # ie, "(rec.anime:3100)" -> ("rec.anime" : "3100") 57 | def group_parse(group_option): 58 | return tuple(group_option[1:-1].split(':')) 59 | 60 | def set(option, value): 61 | (section, option) = option.split(":") 62 | 63 | try: 64 | _shared_settings.add_section(section) 65 | except ConfigParser.DuplicateSectionError: 66 | pass 67 | 68 | return _shared_settings.set(section, option, value) 69 | -------------------------------------------------------------------------------- /threads.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | class LockedIterator(object): 4 | 5 | def __init__(self, it): 6 | self.lock = threading.Lock() 7 | self.it = it.__iter__() 8 | 9 | def __iter__(self): return self 10 | 11 | def next(self): 12 | self.lock.acquire() 13 | try: 14 | return self.it.next() 15 | finally: 16 | self.lock.release() 17 | 18 | class MyThread(threading.Thread): 19 | pass -------------------------------------------------------------------------------- /web.py: -------------------------------------------------------------------------------- 1 | import cherrypy 2 | import db 3 | import logging 4 | import threading 5 | import json 6 | import cgi 7 | import time 8 | 9 | import settings 10 | 11 | from omniverse import ArticleProducer 12 | 13 | def get_setting(option, section = "NNTP"): 14 | try: 15 | return settings.get(section + ":" + option) 16 | except (settings.NoSectionError, settings.NoOptionError), error: 17 | return "" 18 | 19 | class SettingsPages: 20 | 21 | @cherrypy.expose 22 | def index(self): 23 | 24 | groups = [] 25 | while 1: 26 | try: 27 | (group, last) = settings.group_parse(settings.get("NNTP:server.0.group.%d" % len(groups))) 28 | groups.append(group) 29 | except (settings.NoOptionError, settings.NoSectionError), e: 30 | break 31 | 32 | data= { 33 | "host" : get_setting("server.0.host"), 34 | "port" : get_setting("server.0.port"), 35 | "is_ssl" : "checked" if get_setting("server.0.is_ssl") == "1" else "", 36 | "username" : get_setting("server.0.username"), 37 | "password" : get_setting("server.0.password"), 38 | "groups" : groups 39 | } 40 | return load_template('./html/settings.tmp.htm').render(**data) 41 | 42 | @cherrypy.expose 43 | def save(self, **kwargs): 44 | 45 | host = kwargs['host'] 46 | port = kwargs['port'] 47 | is_ssl = "1" if kwargs['is_ssl'] == "checked" else "0" 48 | username = kwargs['username'] 49 | password = kwargs['password'] 50 | groups = kwargs['groups'].split(',') 51 | 52 | settings.set("NNTP:server.0.host", host) 53 | settings.set("NNTP:server.0.port", port) 54 | settings.set("NNTP:server.0.is_ssl", is_ssl) 55 | settings.set("NNTP:server.0.username", username) 56 | settings.set("NNTP:server.0.password", password) 57 | 58 | old_group_values = {} 59 | 60 | idx = 0 61 | while 1: 62 | try: 63 | (group, last) = settings.group_parse(settings.get("NNTP:server.0.group.%d" % idx)) 64 | settings.remove("NNTP:server.0.group.%d" % idx) 65 | 66 | old_group_values[group] = last 67 | idx += 1 68 | except settings.NoOptionError, e: 69 | break 70 | 71 | if groups[0]: 72 | for idx, group in zip(range(len(groups)), groups): 73 | settings.set("NNTP:server.0.group.%d" % idx, "(%s:%s)" % (group, old_group_values.get(group, "0"))) 74 | 75 | settings.set("NNTP:server.0.enabled", 1) 76 | return "Settings Updated." 77 | 78 | class GroupWorker(threading.Thread): 79 | 80 | def __init__(self, conn, keyword): 81 | threading.Thread.__init__(self) 82 | self.groups = None 83 | self.conn = conn 84 | self.keyword = keyword 85 | 86 | def run(self): 87 | (resp, groups) = self.conn.list() 88 | self.conn.quit() 89 | groups = [(group, int(last) - int(first)) for (group, last, first, flag) in groups if self.keyword in group] 90 | groups.sort(lambda x, y: y[1] - x[1]) 91 | groups = ["%s (%d)" % (group, size) for (group, size) in groups] 92 | self.groups = groups 93 | 94 | class GroupPages: 95 | 96 | def __init__(self): 97 | self._workers = {} 98 | 99 | @cherrypy.expose 100 | def list(self, host, port, is_ssl, username, password, keyword="comics"): 101 | 102 | is_ssl = is_ssl == "checked" 103 | 104 | uid = (host, port, is_ssl, username, password, keyword) 105 | 106 | cherrypy.response.headers['Content-type'] = 'application/json' 107 | 108 | try: 109 | worker = self._workers[uid] 110 | if worker.groups is not None: 111 | if len(worker.groups) > 0: 112 | return json.dumps({'status' : 'done', 'result' : worker.groups}) 113 | else: 114 | return json.dumps({'status' : 'error', 'result' : 'No Newsgroups Found.'}) 115 | 116 | return json.dumps({'status' : 'working', 'result' : '' }) 117 | 118 | except KeyError, e: 119 | try: 120 | port = int(port) 121 | except ValueError, e: 122 | return json.dumps({'status' : 'error', 'result' : "Settings Error: Invalid Port."}) 123 | 124 | try: 125 | conn = ArticleProducer().connect(host=host, port=port, is_ssl=is_ssl, username=username, password=password, retry=0) 126 | except IOError, e: 127 | return json.dumps({'status' : 'error', 'result' : 'Settings Error: %s' % str(e)}) 128 | 129 | self._workers[uid] = GroupWorker(conn, keyword) 130 | self._workers[uid].start() 131 | return json.dumps({'status' : 'working', 'result' : '' }) 132 | 133 | class NzbPages: 134 | 135 | @cherrypy.expose 136 | def save(self, ids): 137 | 138 | # nzb spec can be found here: http://docs.newzbin2.es/index.php/Newzbin:NZB_Specs 139 | # refactor refactor refactor 140 | 141 | if type(ids) is not type([]): 142 | ids = [ids] 143 | 144 | nzb_filename = "omniverse_%d" % int(time.time()) 145 | 146 | cherrypy.response.headers['Content-type'] = 'application/xml+nzb' 147 | cherrypy.response.headers['Content-disposition'] = 'attachment; filename="%s.nzb"' % nzb_filename 148 | connection = db.connect() 149 | 150 | pieces = [] 151 | pieces.append('') 152 | pieces.append('') 153 | pieces.append('') 154 | pieces.append('') 155 | pieces.append('\t%s' % nzb_filename) 156 | pieces.append('\tComics') 157 | pieces.append('') 158 | 159 | if ids: 160 | for _id in ids: 161 | (rowid, subject, parts, total_parts, complete, 162 | filename, groups, poster, date_posted, size, yenc) = connection.select( 163 | "SELECT rowid, subject, parts, total_parts, complete, filename, groups, poster, date_posted, size, yenc from articles WHERE rowid = ?", (_id,)).next() 164 | 165 | pieces.append('' % ( 166 | cgi.escape(poster, True), 167 | date_posted, 168 | cgi.escape(subject, True))) 169 | 170 | pieces.append('\t') 171 | for group in groups.split(','): 172 | pieces.append('\t\t%s' % group.strip()) 173 | pieces.append('\t') 174 | 175 | parts = eval(parts) 176 | 177 | pieces.append('\t') 178 | for part in parts.keys(): 179 | idx = parts[part] 180 | pieces.append('\t\t%s' % ( 181 | int(size) / int(total_parts), 182 | idx, 183 | cgi.escape(part[1:-1], True))) 184 | pieces.append('\t') 185 | 186 | 187 | pieces.append('') 188 | pieces.append('') 189 | 190 | output = "\n".join(pieces) 191 | 192 | try: 193 | import xml.parsers.expat 194 | 195 | parser = xml.parsers.expat.ParserCreate() 196 | parser.Parse(output) 197 | except Exception, e: 198 | logging.getLogger().exception("Failed to parse nzb data.") 199 | 200 | return output 201 | 202 | class RootPages: 203 | 204 | # child pages 205 | groups = GroupPages() 206 | settings = SettingsPages() 207 | nzb = NzbPages() 208 | 209 | @cherrypy.expose 210 | def index(self, pg=1, sz=500, query=""): 211 | return self.browse(pg, sz, query) 212 | 213 | @cherrypy.expose 214 | def status(self): 215 | connection = db.connect() 216 | num_rows = connection.select("SELECT COUNT(*) as num_rows FROM articles WHERE filename LIKE '%cbr' OR filename LIKE '%cbz'").next() 217 | return "Number of records: %d" % num_rows 218 | 219 | @cherrypy.expose 220 | def browse(self, pg=1, sz=500, query=""): 221 | 222 | try: 223 | # size per page 224 | sz = max(500, int(sz)) 225 | 226 | #current page number 227 | pg = max(1, int(pg)) 228 | except ValueError: 229 | logging.getLogger().exception("Bad GET value.") 230 | sz = 500 231 | pg = 1 232 | 233 | connection = db.connect() 234 | 235 | result_set = connection.select( 236 | "SELECT rowid, * FROM articles WHERE filename LIKE ? ORDER BY alphanumeric(filename) LIMIT ? OFFSET ?", 237 | ("%" + query + "%", sz, (pg - 1) * sz)) 238 | 239 | return load_template('./html/browse.tmp.htm').render(result_set = result_set, pg = pg, sz = sz, query = query) 240 | 241 | def load_template(filename): 242 | 243 | from mako.template import Template 244 | from mako.lookup import TemplateLookup 245 | 246 | return Template( 247 | filename=filename,output_encoding='utf-8',encoding_errors='replace', 248 | lookup= TemplateLookup( 249 | directories=['./html'], 250 | output_encoding='utf-8', 251 | input_encoding='utf-8', 252 | encoding_errors='replace')) 253 | --------------------------------------------------------------------------------