',
8 | 'OPTIONS': {
9 | 'DB': 1,
10 | 'PASSWORD': 'yadayada',
11 | 'PARSER_CLASS': 'redis.connection.HiredisParser'
12 | },
13 | },
14 | }
15 |
16 | # When using unix domain sockets
17 | # Note: ``LOCATION`` needs to be the same as the ``unixsocket`` setting
18 | # in your redis.conf
19 | CACHES = {
20 | 'default': {
21 | 'BACKEND': 'redis_cache.RedisCache',
22 | 'LOCATION': '/path/to/socket/file',
23 | 'OPTIONS': {
24 | 'DB': 1,
25 | 'PASSWORD': 'yadayada',
26 | 'PARSER_CLASS': 'redis.connection.HiredisParser'
27 | },
28 | },
29 | }
30 | """
31 | from torngas.cache.backends.base import BaseCache, InvalidCacheBackendError
32 | from torngas.utils.storage import SortedDict
33 | from torngas.utils.strtools import safestr, safeunicode
34 | from tornado.util import import_object
35 | from torngas.exception import ConfigError
36 |
37 | try:
38 | import cPickle as pickle
39 | except ImportError:
40 | import pickle
41 |
42 | try:
43 | import redis
44 | except ImportError:
45 | raise InvalidCacheBackendError(
46 | "Redis cache backend requires the 'redis-py' library")
47 | from redis.connection import UnixDomainSocketConnection, Connection
48 | from redis.connection import DefaultParser
49 |
50 | import sys
51 |
52 | PY3 = (sys.version_info >= (3,))
53 |
54 | if PY3:
55 | bytes_type = bytes
56 | else:
57 | bytes_type = str
58 |
59 |
60 | def python_2_unicode_compatible(klass):
61 | """
62 | A decorator that defines __unicode__ and __str__ methods under Python 2.
63 | Under Python 3 it does nothing.
64 |
65 | To support Python 2 and 3 with a single code base, define a __str__ method
66 | returning text and apply this decorator to the class.
67 |
68 | Backported from Django 1.5+.
69 | """
70 | if not PY3:
71 | klass.__unicode__ = klass.__str__
72 | klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
73 | return klass
74 |
75 |
76 | @python_2_unicode_compatible
77 | class CacheKey(object):
78 | """
79 | A stub string class that we can use to check if a key was created already.
80 | """
81 |
82 | def __init__(self, key):
83 | self._key = key
84 |
85 | def __eq__(self, other):
86 | return self._key == other
87 |
88 | def __str__(self):
89 | return safeunicode(self._key)
90 |
91 | def __repr__(self):
92 | return repr(self._key)
93 |
94 | def __hash__(self):
95 | return hash(self._key)
96 |
97 |
98 | class CacheConnectionPool(object):
99 | def __init__(self):
100 | self._connection_pools = {}
101 |
102 | def get_connection_pool(self, host='127.0.0.1', port=6379, db=1,
103 | password=None, parser_class=None,
104 | unix_socket_path=None):
105 | connection_identifier = (host, port, db, parser_class, unix_socket_path)
106 | if not self._connection_pools.get(connection_identifier):
107 | connection_class = (
108 | unix_socket_path and UnixDomainSocketConnection or Connection
109 | )
110 | kwargs = {
111 | 'db': db,
112 | 'password': password,
113 | 'connection_class': connection_class,
114 | 'parser_class': parser_class,
115 | }
116 | if unix_socket_path is None:
117 | kwargs.update({
118 | 'host': host,
119 | 'port': port,
120 | })
121 | else:
122 | kwargs['path'] = unix_socket_path
123 | self._connection_pools[connection_identifier] = redis.ConnectionPool(**kwargs)
124 | return self._connection_pools[connection_identifier]
125 |
126 |
127 | pool = CacheConnectionPool()
128 |
129 |
130 | class CacheClass(BaseCache):
131 | def __init__(self, server, params):
132 | """
133 | Connect to Redis, and set up cache backend.
134 | """
135 | self._init(server, params)
136 |
137 | def _init(self, server, params):
138 | super(CacheClass, self).__init__(params)
139 | self._server = server
140 | self._params = params
141 |
142 | unix_socket_path = None
143 | if ':' in self.server:
144 | host, port = self.server.rsplit(':', 1)
145 | try:
146 | port = int(port)
147 | except (ValueError, TypeError):
148 | raise ConfigError("port value must be an integer")
149 | else:
150 | host, port = None, None
151 | unix_socket_path = self.server
152 |
153 | kwargs = {
154 | 'db': self.db,
155 | 'password': self.password,
156 | 'host': host,
157 | 'port': port,
158 | 'unix_socket_path': unix_socket_path,
159 | }
160 | connection_pool = pool.get_connection_pool(
161 | parser_class=self.parser_class,
162 | **kwargs
163 | )
164 | self._client = redis.Redis(
165 | connection_pool=connection_pool,
166 | **kwargs
167 | )
168 |
169 | @property
170 | def server(self):
171 | return self._server or "127.0.0.1:6379"
172 |
173 | @property
174 | def params(self):
175 | return self._params or {}
176 |
177 | @property
178 | def options(self):
179 | return self.params.get('OPTIONS', {})
180 |
181 | @property
182 | def db(self):
183 | _db = self.params.get('db', self.options.get('DB', 1))
184 | try:
185 | _db = int(_db)
186 | except (ValueError, TypeError):
187 | raise ConfigError("db value must be an integer")
188 | return _db
189 |
190 | @property
191 | def password(self):
192 | return self.params.get('password', self.options.get('PASSWORD', None))
193 |
194 | @property
195 | def parser_class(self):
196 | cls = self.options.get('PARSER_CLASS', None)
197 | if cls is None:
198 | return DefaultParser
199 | mod_path, cls_name = cls.rsplit('.', 1)
200 | try:
201 | mod = import_object(mod_path)
202 | parser_class = getattr(mod, cls_name)
203 | except (AttributeError, ImportError):
204 | raise ConfigError("Could not find parser class '%s'" % parser_class)
205 | return parser_class
206 |
207 | def __getstate__(self):
208 | return {'params': self._params, 'server': self._server}
209 |
210 | def __setstate__(self, state):
211 | self._init(**state)
212 |
213 | def make_key(self, key, version=None):
214 | """
215 | Returns the utf-8 encoded bytestring of the given key as a CacheKey
216 | instance to be able to check if it was "made" before.
217 | """
218 | if not isinstance(key, CacheKey):
219 | key = CacheKey(key)
220 | return key
221 |
222 | def add(self, key, value, timeout=None, version=None):
223 | """
224 | Add a value to the cache, failing if the key already exists.
225 |
226 | Returns ``True`` if the object was added, ``False`` if not.
227 | """
228 | return self.set(key, value, timeout, _add_only=True)
229 |
230 | def get(self, key, default=None, version=None):
231 | """
232 | Retrieve a value from the cache.
233 |
234 | Returns unpickled value if key is found, the default if not.
235 | """
236 | key = self.make_key(key, version=version)
237 | value = self._client.get(key)
238 | if value is None:
239 | return default
240 | try:
241 | result = int(value)
242 | except (ValueError, TypeError):
243 | result = self.unpickle(value)
244 | return result
245 |
246 | def _set(self, key, value, timeout, client, _add_only=False):
247 | """
248 | 向redis设置值
249 | :param key: 键
250 | :param value: 值
251 | :param timeout: 过期时间
252 | :param client: 连接客户端对象
253 | :param _add_only: 如果为true，仅当redis中不存在键时才add，否则不作任何操作
254 | :return:
255 | """
256 | if timeout == 0:
257 | if _add_only:
258 | return client.setnx(key, value)
259 | return client.set(key, value)
260 | elif timeout > 0:
261 | if _add_only:
262 | added = client.setnx(key, value)
263 | if added:
264 | client.expire(key, timeout)
265 | return added
266 | return client.setex(key, value, timeout)
267 | else:
268 | return False
269 |
270 | def set(self, key, value, timeout=None, version=None, client=None, _add_only=False):
271 | """
272 | Persist a value to the cache, and set an optional expiration time.
273 | """
274 | if not client:
275 | client = self._client
276 | key = self.make_key(key, version=version)
277 | if timeout is None:
278 | timeout = self.default_timeout
279 |
280 | # If ``value`` is not an int, then pickle it
281 | #pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
282 | if not isinstance(value, int) or isinstance(value, bool):
283 | result = self._set(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL), int(timeout), client, _add_only)
284 | else:
285 | result = self._set(key, value, int(timeout), client, _add_only)
286 | # result is a boolean
287 | return result
288 |
289 | def delete(self, key, version=None):
290 | """
291 | Remove a key from the cache.
292 | """
293 | self._client.delete(self.make_key(key, version=version))
294 |
295 | def delete_many(self, keys, version=None):
296 | """
297 | Remove multiple keys at once.
298 | """
299 | if keys:
300 | keys = map(lambda key: self.make_key(key, version=version), keys)
301 | self._client.delete(*keys)
302 |
303 | def clear(self):
304 | """
305 | Flush all cache keys.
306 | """
307 | # TODO : potential data loss here, should we only delete keys based on the correct version ?
308 | self._client.flushdb()
309 |
310 | def unpickle(self, value):
311 | """
312 | Unpickles the given value.
313 | """
314 | value = safestr(value)
315 | return pickle.loads(value)
316 |
317 | def get_many(self, keys, version=None):
318 | """
319 | Retrieve many keys.
320 | """
321 | if not keys:
322 | return {}
323 | recovered_data = SortedDict()
324 | new_keys = list(map(lambda key: self.make_key(key, version=version), keys))
325 | map_keys = dict(zip(new_keys, keys))
326 | results = self._client.mget(new_keys)
327 | for key, value in zip(new_keys, results):
328 | if value is None:
329 | continue
330 | try:
331 | value = int(value)
332 | except (ValueError, TypeError):
333 | value = self.unpickle(value)
334 | if isinstance(value, bytes_type):
335 | value = safeunicode(value)
336 | recovered_data[map_keys[key]] = value
337 | return recovered_data
338 |
339 | def set_many(self, data, timeout=None, version=None):
340 | """
341 | Set a bunch of values in the cache at once from a dict of key/value
342 | pairs. This is much more efficient than calling set() multiple times.
343 |
344 | If timeout is given, that timeout will be used for the key; otherwise
345 | the default cache timeout will be used.
346 | """
347 | pipeline = self._client.pipeline()
348 | for key, value in data.items():
349 | self.set(key, value, timeout, version=version, client=pipeline)
350 | pipeline.execute()
351 |
352 |
353 | def incr(self, key, delta=1, version=None):
354 | """
355 | Add delta to value in the cache. If the key does not exist, raise a
356 | ValueError exception.
357 | """
358 | key = self.make_key(key, version=version)
359 | exists = self._client.exists(key)
360 | if not exists:
361 | raise ValueError("Key '%s' not found" % key)
362 | try:
363 | value = self._client.incr(key, delta)
364 | except redis.ResponseError:
365 | value = self.get(key) + 1
366 | self.set(key, value)
367 | return value
368 |
369 |
370 | class RedisCache(CacheClass):
371 | """
372 | A subclass that is supposed to be used on Django >= 1.3.
373 | """
374 |
375 | def make_key(self, key, version=None):
376 | if not isinstance(key, CacheKey):
377 | key = CacheKey(super(CacheClass, self).make_key(key, version))
378 | return key
379 |
380 | def incr_version(self, key, delta=1, version=None):
381 | """
382 | Adds delta to the cache version for the supplied key. Returns the
383 | new version.
384 |
385 | Note: In Redis 2.0 you cannot rename a volitle key, so we have to move
386 | the value from the old key to the new key and maintain the ttl.
387 | """
388 | if version is None:
389 | version = self.version
390 | old_key = self.make_key(key, version)
391 | value = self.get(old_key, version=version)
392 | ttl = self._client.ttl(old_key)
393 | if value is None:
394 | raise ValueError("Key '%s' not found" % key)
395 | new_key = self.make_key(key, version=version + delta)
396 | # TODO: See if we can check the version of Redis, since 2.2 will be able
397 | # to rename volitile keys.
398 | self.set(new_key, value, timeout=ttl)
399 | self.delete(old_key)
400 | return version + delta
--------------------------------------------------------------------------------
/torngas/decorators/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/decorators/multithreading.py:
--------------------------------------------------------------------------------
1 | """
2 | multithreading support for tornado
3 | come from whirlwind
4 | """
5 | from __future__ import with_statement
6 | from tornado.web import *
7 | import threading
8 |
9 | from Queue import Queue
10 |
11 |
12 | def threadedhandler(method):
13 | @asynchronous
14 | def wrapper(self, *args, **kwargs):
15 | self._is_threaded = True
16 | self._auto_finish = False
17 | action = ThreadedHandler(method, self, *args, **kwargs)
18 | ThreadPool.instance().add_task(action.do_work)
19 |
20 | return wrapper
21 |
22 |
23 | class ThreadedHandler():
24 | def __init__(self, method, handler, *args, **kwargs):
25 | self._method = method
26 | self._handler = handler
27 | self._args = args
28 | self._kwargs = kwargs
29 |
30 |
31 | def do_work(self):
32 | try:
33 | # TODO: handle handlers that return a value.
34 | # (think tornado considers that a json response)
35 | self._method(self._handler, *self._args, **self._kwargs)
36 | if not self._handler._is_torngas_finished:
37 | self._handler.finish()
38 | except Exception, e:
39 | self._handle_request_exception(e)
40 |
41 |
42 | def threadedfunc(method):
43 | @asynchronous
44 | def wrapper(*args, **kwargs):
45 | action = ThreadedFunction(method, *args, **kwargs)
46 | ThreadPool.instance().add_task(action.do_work)
47 |
48 | return wrapper
49 |
50 |
51 | class ThreadedFunction():
52 | def __init__(self, method, *args, **kwargs):
53 | self._method = method
54 | self._args = args
55 | self._kwargs = kwargs
56 |
57 | def do_work(self):
58 | try:
59 | self._method(*self._args, **self._kwargs)
60 | except Exception, e:
61 | raise
62 |
63 |
64 | class ThreadPool():
65 | """
66 | Pool of threads consuming tasks from a queue
67 |
68 | Note: I'm not crazy about the fixed threadpool implementation.
69 | TODO: should have a max_threads argument, then we can build up to that as needed and
70 | reap unused threads.
71 | -dustin
72 | """
73 |
74 | def __init__(self, num_threads=10):
75 | self.tasks = Queue(num_threads)
76 | for _ in range(num_threads): ThreadPoolWorker(self.tasks)
77 |
78 | """
79 | Submits a task to the threadpool
80 | callback will be called once the task completes.
81 | """
82 |
83 | def add_task(self, func, callback=None):
84 | """Add a task to the queue"""
85 | self.tasks.put((func, callback))
86 |
87 | def wait_completion(self):
88 | """Wait for completion of all the tasks in the queue"""
89 | self.tasks.join()
90 |
91 | '''
92 | Returns the global threadpool. Use this in almost all cases.
93 | '''
94 | _instance_lock = threading.Lock()
95 |
96 | @classmethod
97 | def instance(cls):
98 |
99 | if not hasattr(cls, "_instance"):
100 | with ThreadPool._instance_lock:
101 | #singleon
102 | cls._instance = cls()
103 | return cls._instance
104 |
105 |
106 | class ThreadPoolWorker(threading.Thread):
107 | """Thread executing tasks from a given tasks queue"""
108 |
109 | def __init__(self, tasks):
110 | threading.Thread.__init__(self)
111 | self.tasks = tasks
112 | self.daemon = True
113 | self.start()
114 |
115 | def run(self):
116 | while True:
117 | func, callback = self.tasks.get()
118 | try:
119 | func()
120 | except Exception, e:
121 | print e
122 | if callback:
123 | callback()
124 | self.tasks.task_done()
--------------------------------------------------------------------------------
/torngas/decorators/whitelist.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | """
3 | Torngas Whitelist Module
4 |
5 | """
6 | from tornado import web
7 | import types
8 | from torngas.utils import lazyimport
9 |
10 | settings_module = lazyimport('torngas.helpers.settings_helper')
11 |
12 |
13 | def whitelisted(argument=None):
14 | """
15 | 白名单，如果在参数中列出可访问的ip或在配置文件中列出，则被标记的请求方法仅可允许白名单ip访问
16 | :param argument: whitelist ip list
17 | :return:bool
18 | """
19 |
20 | def is_whitelisted(remote_ip, whitelist):
21 | if remote_ip in whitelist:
22 | return True
23 | else:
24 | return False
25 |
26 | if type(argument) is types.FunctionType:
27 |
28 | def wrapper(self, *args, **kwargs):
29 | white_setting = settings_module.settings.WHITELIST
30 | if white_setting:
31 | if is_whitelisted(self.request.remote_ip,
32 | settings_module.settings.WHITELIST):
33 | return argument(self, *args, **kwargs)
34 | raise web.HTTPError(403)
35 | else:
36 | raise web.HTTPError(403)
37 |
38 | return wrapper
39 |
40 | else:
41 | if isinstance(argument, str):
42 | argument = [argument]
43 |
44 | elif not isinstance(argument, list):
45 | raise ValueError('whitelisted requires no parameters or '
46 | 'a string or list')
47 |
48 | def argument_wrapper(method):
49 |
50 | def validate(self, *args, **kwargs):
51 | if is_whitelisted(self.request.remote_ip, argument):
52 | return method(self, *args, **kwargs)
53 | raise web.HTTPError(403)
54 |
55 | return validate
56 |
57 | return argument_wrapper
58 |
--------------------------------------------------------------------------------
/torngas/dispatch/__init__.py:
--------------------------------------------------------------------------------
1 | """Multi-consumer multi-producer dispatching mechanism
2 |
3 | Originally based on pydispatch (BSD) http://pypi.python.org/pypi/PyDispatcher/2.0.1
4 | See license.txt for original license.
5 |
6 | Heavily modified for Django's purposes.
7 | """
8 |
9 | from torngas.dispatch.dispatcher import Signal, receiver
10 | from torngas.dispatch import signals
11 |
--------------------------------------------------------------------------------
/torngas/dispatch/dispatcher.py:
--------------------------------------------------------------------------------
1 | import weakref
2 | import threading
3 |
4 | from torngas.dispatch import saferef
5 |
6 |
7 | WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref)
8 |
9 |
10 | def _make_id(target):
11 | if hasattr(target, '__func__'):
12 | return (id(target.__self__), id(target.__func__))
13 | return id(target)
14 |
15 |
16 | class Signal(object):
17 | """
18 | Base class for all signals
19 |
20 | Internal attributes:
21 |
22 | receivers
23 | { receriverkey (id) : weakref(receiver) }
24 | """
25 |
26 | def __init__(self, providing_args=None):
27 | """
28 | Create a new signal.
29 |
30 | providing_args
31 | A list of the arguments this signal can pass along in a send() call.
32 | """
33 | self.receivers = []
34 | if providing_args is None:
35 | providing_args = []
36 | self.providing_args = set(providing_args)
37 | self.lock = threading.Lock()
38 |
39 | def connect(self, receiver, sender=None, weak=True, dispatch_uid=None):
40 | """
41 | Connect receiver to sender for signal.
42 |
43 | Arguments:
44 |
45 | receiver
46 | A function or an instance method which is to receive signals.
47 | Receivers must be hashable objects.
48 |
49 | If weak is True, then receiver must be weak-referencable (more
50 | precisely saferef.safeRef() must be able to create a reference
51 | to the receiver).
52 |
53 | Receivers must be able to accept keyword arguments.
54 |
55 | If receivers have a dispatch_uid attribute, the receiver will
56 | not be added if another receiver already exists with that
57 | dispatch_uid.
58 |
59 | sender
60 | The sender to which the receiver should respond. Must either be
61 | of type Signal, or None to receive events from any sender.
62 |
63 | weak
64 | Whether to use weak references to the receiver. By default, the
65 | module will attempt to use weak references to the receiver
66 | objects. If this parameter is false, then strong references will
67 | be used.
68 |
69 | dispatch_uid
70 | An identifier used to uniquely identify a particular instance of
71 | a receiver. This will usually be a string, though it may be
72 | anything hashable.
73 | """
74 | from torngas.helpers.settings_helper import settings
75 |
76 | # If DEBUG is on, check that we got a good receiver
77 | if settings.TORNADO_CONF.debug:
78 | import inspect
79 |
80 | assert callable(receiver), "Signal receivers must be callable."
81 |
82 | # Check for **kwargs
83 | # Not all callables are inspectable with getargspec, so we'll
84 | # try a couple different ways but in the end fall back on assuming
85 | # it is -- we don't want to prevent registration of valid but weird
86 | # callables.
87 | try:
88 | argspec = inspect.getargspec(receiver)
89 | except TypeError:
90 | try:
91 | argspec = inspect.getargspec(receiver.__call__)
92 | except (TypeError, AttributeError):
93 | argspec = None
94 | if argspec:
95 | assert argspec[2] is not None, \
96 | "Signal receivers must accept keyword arguments (**kwargs)."
97 |
98 | if dispatch_uid:
99 | lookup_key = (dispatch_uid, _make_id(sender))
100 | else:
101 | lookup_key = (_make_id(receiver), _make_id(sender))
102 |
103 | if weak:
104 | receiver = saferef.safeRef(receiver, onDelete=self._remove_receiver)
105 |
106 | with self.lock:
107 | for r_key, _ in self.receivers:
108 | if r_key == lookup_key:
109 | break
110 | else:
111 | self.receivers.append((lookup_key, receiver))
112 |
113 | def disconnect(self, receiver=None, sender=None, weak=True, dispatch_uid=None):
114 | """
115 | Disconnect receiver from sender for signal.
116 |
117 | If weak references are used, disconnect need not be called. The receiver
118 | will be remove from dispatch automatically.
119 |
120 | Arguments:
121 |
122 | receiver
123 | The registered receiver to disconnect. May be none if
124 | dispatch_uid is specified.
125 |
126 | sender
127 | The registered sender to disconnect
128 |
129 | weak
130 | The weakref state to disconnect
131 |
132 | dispatch_uid
133 | the unique identifier of the receiver to disconnect
134 | """
135 | if dispatch_uid:
136 | lookup_key = (dispatch_uid, _make_id(sender))
137 | else:
138 | lookup_key = (_make_id(receiver), _make_id(sender))
139 |
140 | with self.lock:
141 | for index in xrange(len(self.receivers)):
142 | (r_key, _) = self.receivers[index]
143 | if r_key == lookup_key:
144 | del self.receivers[index]
145 | break
146 |
147 | def has_listeners(self, sender=None):
148 | return bool(self._live_receivers(_make_id(sender)))
149 |
150 | def send(self, sender, **named):
151 | """
152 | Send signal from sender to all connected receivers.
153 |
154 | If any receiver raises an error, the error propagates back through send,
155 | terminating the dispatch loop, so it is quite possible to not have all
156 | receivers called if a raises an error.
157 |
158 | Arguments:
159 |
160 | sender
161 | The sender of the signal Either a specific object or None.
162 |
163 | named
164 | Named arguments which will be passed to receivers.
165 |
166 | Returns a list of tuple pairs [(receiver, response), ... ].
167 | """
168 | responses = []
169 | if not self.receivers:
170 | return responses
171 |
172 | for receiver in self._live_receivers(_make_id(sender)):
173 | response = receiver(signal=self, sender=sender, **named)
174 | responses.append((receiver, response))
175 | return responses
176 |
177 | def send_robust(self, sender, **named):
178 | """
179 | Send signal from sender to all connected receivers catching errors.
180 |
181 | Arguments:
182 |
183 | sender
184 | The sender of the signal. Can be any python object (normally one
185 | registered with a connect if you actually want something to
186 | occur).
187 |
188 | named
189 | Named arguments which will be passed to receivers. These
190 | arguments must be a subset of the argument names defined in
191 | providing_args.
192 |
193 | Return a list of tuple pairs [(receiver, response), ... ]. May raise
194 | DispatcherKeyError.
195 |
196 | If any receiver raises an error (specifically any subclass of
197 | Exception), the error instance is returned as the result for that
198 | receiver.
199 | """
200 | responses = []
201 | if not self.receivers:
202 | return responses
203 |
204 | # Call each receiver with whatever arguments it can accept.
205 | # Return a list of tuple pairs [(receiver, response), ... ].
206 | for receiver in self._live_receivers(_make_id(sender)):
207 | try:
208 | response = receiver(signal=self, sender=sender, **named)
209 | except Exception as err:
210 | responses.append((receiver, err))
211 | else:
212 | responses.append((receiver, response))
213 | return responses
214 |
215 | def _live_receivers(self, senderkey):
216 | """
217 | Filter sequence of receivers to get resolved, live receivers.
218 |
219 | This checks for weak references and resolves them, then returning only
220 | live receivers.
221 | """
222 | none_senderkey = _make_id(None)
223 | receivers = []
224 |
225 | for (receiverkey, r_senderkey), receiver in self.receivers:
226 | if r_senderkey == none_senderkey or r_senderkey == senderkey:
227 | if isinstance(receiver, WEAKREF_TYPES):
228 | # Dereference the weak reference.
229 | receiver = receiver()
230 | if receiver is not None:
231 | receivers.append(receiver)
232 | else:
233 | receivers.append(receiver)
234 | return receivers
235 |
236 | def _remove_receiver(self, receiver):
237 | """
238 | Remove dead receivers from connections.
239 | """
240 |
241 | with self.lock:
242 | to_remove = []
243 | for key, connected_receiver in self.receivers:
244 | if connected_receiver == receiver:
245 | to_remove.append(key)
246 | for key in to_remove:
247 | last_idx = len(self.receivers) - 1
248 | # enumerate in reverse order so that indexes are valid even
249 | # after we delete some items
250 | for idx, (r_key, _) in enumerate(reversed(self.receivers)):
251 | if r_key == key:
252 | del self.receivers[last_idx - idx]
253 |
254 |
255 | def receiver(signal, **kwargs):
256 | """
257 | A decorator for connecting receivers to signals. Used by passing in the
258 | signal (or list of signals) and keyword arguments to connect::
259 |
260 | @receiver(post_save, sender=MyModel)
261 | def signal_receiver(sender, **kwargs):
262 | ...
263 |
264 | @receiver([post_save, post_delete], sender=MyModel)
265 | def signals_receiver(sender, **kwargs):
266 | ...
267 |
268 | """
269 |
270 | def _decorator(func):
271 | if isinstance(signal, (list, tuple)):
272 | for s in signal:
273 | s.connect(func, **kwargs)
274 | else:
275 | signal.connect(func, **kwargs)
276 | return func
277 |
278 | return _decorator
279 |
--------------------------------------------------------------------------------
/torngas/dispatch/saferef.py:
--------------------------------------------------------------------------------
1 | """
2 | torngas saferef from django
3 | "Safe weakrefs", originally from pyDispatcher.
4 |
5 | Provides a way to safely weakref any function, including bound methods (which
6 | aren't handled by the core weakref module).
7 | """
8 |
9 | import traceback
10 | import weakref
11 |
12 | def safeRef(target, onDelete = None):
13 | """Return a *safe* weak reference to a callable target
14 |
15 | target -- the object to be weakly referenced, if it's a
16 | bound method reference, will create a BoundMethodWeakref,
17 | otherwise creates a simple weakref.
18 | onDelete -- if provided, will have a hard reference stored
19 | to the callable to be called after the safe reference
20 | goes out of scope with the reference object, (either a
21 | weakref or a BoundMethodWeakref) as argument.
22 | """
23 | if hasattr(target, '__self__'):
24 | if target.__self__ is not None:
25 | # Turn a bound method into a BoundMethodWeakref instance.
26 | # Keep track of these instances for lookup by disconnect().
27 | assert hasattr(target, '__func__'), """safeRef target %r has __self__, but no __func__, don't know how to create reference"""%( target,)
28 | reference = get_bound_method_weakref(
29 | target=target,
30 | onDelete=onDelete
31 | )
32 | return reference
33 | if callable(onDelete):
34 | return weakref.ref(target, onDelete)
35 | else:
36 | return weakref.ref( target )
37 |
38 | class BoundMethodWeakref(object):
39 | """'Safe' and reusable weak references to instance methods
40 |
41 | BoundMethodWeakref objects provide a mechanism for
42 | referencing a bound method without requiring that the
43 | method object itself (which is normally a transient
44 | object) is kept alive. Instead, the BoundMethodWeakref
45 | object keeps weak references to both the object and the
46 | function which together define the instance method.
47 |
48 | Attributes:
49 | key -- the identity key for the reference, calculated
50 | by the class's calculateKey method applied to the
51 | target instance method
52 | deletionMethods -- sequence of callable objects taking
53 | single argument, a reference to this object which
54 | will be called when *either* the target object or
55 | target function is garbage collected (i.e. when
56 | this object becomes invalid). These are specified
57 | as the onDelete parameters of safeRef calls.
58 | weakSelf -- weak reference to the target object
59 | weakFunc -- weak reference to the target function
60 |
61 | Class Attributes:
62 | _allInstances -- class attribute pointing to all live
63 | BoundMethodWeakref objects indexed by the class's
64 | calculateKey(target) method applied to the target
65 | objects. This weak value dictionary is used to
66 | short-circuit creation so that multiple references
67 | to the same (object, function) pair produce the
68 | same BoundMethodWeakref instance.
69 |
70 | """
71 |
72 | _allInstances = weakref.WeakValueDictionary()
73 |
74 | def __new__( cls, target, onDelete=None, *arguments,**named ):
75 | """Create new instance or return current instance
76 |
77 | Basically this method of construction allows us to
78 | short-circuit creation of references to already-
79 | referenced instance methods. The key corresponding
80 | to the target is calculated, and if there is already
81 | an existing reference, that is returned, with its
82 | deletionMethods attribute updated. Otherwise the
83 | new instance is created and registered in the table
84 | of already-referenced methods.
85 | """
86 | key = cls.calculateKey(target)
87 | current =cls._allInstances.get(key)
88 | if current is not None:
89 | current.deletionMethods.append( onDelete)
90 | return current
91 | else:
92 | base = super( BoundMethodWeakref, cls).__new__( cls )
93 | cls._allInstances[key] = base
94 | base.__init__( target, onDelete, *arguments,**named)
95 | return base
96 |
97 | def __init__(self, target, onDelete=None):
98 | """Return a weak-reference-like instance for a bound method
99 |
100 | target -- the instance-method target for the weak
101 | reference, must have __self__ and __func__ attributes
102 | and be reconstructable via:
103 | target.__func__.__get__( target.__self__ )
104 | which is true of built-in instance methods.
105 | onDelete -- optional callback which will be called
106 | when this weak reference ceases to be valid
107 | (i.e. either the object or the function is garbage
108 | collected). Should take a single argument,
109 | which will be passed a pointer to this object.
110 | """
111 | def remove(weak, self=self):
112 | """Set self.isDead to true when method or instance is destroyed"""
113 | methods = self.deletionMethods[:]
114 | del self.deletionMethods[:]
115 | try:
116 | del self.__class__._allInstances[ self.key ]
117 | except KeyError:
118 | pass
119 | for function in methods:
120 | try:
121 | if callable( function ):
122 | function( self )
123 | except Exception as e:
124 | try:
125 | traceback.print_exc()
126 | except AttributeError:
127 | print('Exception during saferef %s cleanup function %s: %s' % (
128 | self, function, e)
129 | )
130 | self.deletionMethods = [onDelete]
131 | self.key = self.calculateKey( target )
132 | self.weakSelf = weakref.ref(target.__self__, remove)
133 | self.weakFunc = weakref.ref(target.__func__, remove)
134 | self.selfName = str(target.__self__)
135 | self.funcName = str(target.__func__.__name__)
136 |
137 | def calculateKey( cls, target ):
138 | """Calculate the reference key for this reference
139 |
140 | Currently this is a two-tuple of the id()'s of the
141 | target object and the target function respectively.
142 | """
143 | return (id(target.__self__),id(target.__func__))
144 | calculateKey = classmethod( calculateKey )
145 |
146 | def __str__(self):
147 | """Give a friendly representation of the object"""
148 | return """%s( %s.%s )"""%(
149 | self.__class__.__name__,
150 | self.selfName,
151 | self.funcName,
152 | )
153 |
154 | __repr__ = __str__
155 |
156 | def __hash__(self):
157 | return hash(self.key)
158 |
159 | def __bool__( self ):
160 | """Whether we are still a valid reference"""
161 | return self() is not None
162 |
163 | def __nonzero__(self): # Python 2 compatibility
164 | return type(self).__bool__(self)
165 |
166 | def __eq__(self, other):
167 | """Compare with another reference"""
168 | if not isinstance(other, self.__class__):
169 | return self.__class__ == type(other)
170 | return self.key == other.key
171 |
172 | def __call__(self):
173 | """Return a strong reference to the bound method
174 |
175 | If the target cannot be retrieved, then will
176 | return None, otherwise returns a bound instance
177 | method for our object and function.
178 |
179 | Note:
180 | You may call this method any number of times,
181 | as it does not invalidate the reference.
182 | """
183 | target = self.weakSelf()
184 | if target is not None:
185 | function = self.weakFunc()
186 | if function is not None:
187 | return function.__get__(target)
188 | return None
189 |
190 | class BoundNonDescriptorMethodWeakref(BoundMethodWeakref):
191 | """A specialized BoundMethodWeakref, for platforms where instance methods
192 | are not descriptors.
193 |
194 | It assumes that the function name and the target attribute name are the
195 | same, instead of assuming that the function is a descriptor. This approach
196 | is equally fast, but not 100% reliable because functions can be stored on an
197 | attribute named differenty than the function's name such as in:
198 |
199 | class A: pass
200 | def foo(self): return "foo"
201 | A.bar = foo
202 |
203 | But this shouldn't be a common use case. So, on platforms where methods
204 | aren't descriptors (such as Jython) this implementation has the advantage
205 | of working in the most cases.
206 | """
207 | def __init__(self, target, onDelete=None):
208 | """Return a weak-reference-like instance for a bound method
209 |
210 | target -- the instance-method target for the weak
211 | reference, must have __self__ and __func__ attributes
212 | and be reconstructable via:
213 | target.__func__.__get__( target.__self__ )
214 | which is true of built-in instance methods.
215 | onDelete -- optional callback which will be called
216 | when this weak reference ceases to be valid
217 | (i.e. either the object or the function is garbage
218 | collected). Should take a single argument,
219 | which will be passed a pointer to this object.
220 | """
221 | assert getattr(target.__self__, target.__name__) == target, \
222 | ("method %s isn't available as the attribute %s of %s" %
223 | (target, target.__name__, target.__self__))
224 | super(BoundNonDescriptorMethodWeakref, self).__init__(target, onDelete)
225 |
226 | def __call__(self):
227 | """Return a strong reference to the bound method
228 |
229 | If the target cannot be retrieved, then will
230 | return None, otherwise returns a bound instance
231 | method for our object and function.
232 |
233 | Note:
234 | You may call this method any number of times,
235 | as it does not invalidate the reference.
236 | """
237 | target = self.weakSelf()
238 | if target is not None:
239 | function = self.weakFunc()
240 | if function is not None:
241 | # Using partial() would be another option, but it erases the
242 | # "signature" of the function. That is, after a function is
243 | # curried, the inspect module can't be used to determine how
244 | # many arguments the function expects, nor what keyword
245 | # arguments it supports, and pydispatcher needs this
246 | # information.
247 | return getattr(target, function.__name__)
248 | return None
249 |
250 | def get_bound_method_weakref(target, onDelete):
251 | """Instantiates the appropiate BoundMethodWeakRef, depending on the details of
252 | the underlying class method implementation"""
253 | if hasattr(target, '__get__'):
254 | # target method is a descriptor, so the default implementation works:
255 | return BoundMethodWeakref(target=target, onDelete=onDelete)
256 | else:
257 | # no luck, use the alternative implementation:
258 | return BoundNonDescriptorMethodWeakref(target=target, onDelete=onDelete)
259 |
--------------------------------------------------------------------------------
/torngas/dispatch/signals.py:
--------------------------------------------------------------------------------
1 | from torngas.dispatch import Signal
2 |
3 | call_started = Signal()
4 | handler_started = Signal()
5 | handler_finished = Signal()
6 | call_finished = Signal()
7 | # got_request_exception = Signal(providing_args=["request"])
--------------------------------------------------------------------------------
/torngas/exception.py:
--------------------------------------------------------------------------------
1 | """
2 | torngas exception module
3 | """
4 | try:
5 | from exceptions import Exception, StandardError, Warning
6 | except ImportError:
7 | # Python 3
8 | StandardError = Exception
9 |
10 |
11 | class TorngasError(StandardError):
12 | """Exception related to operation with torngas."""
13 |
14 |
15 | class ArgumentError(TorngasError):
16 | """Arguments error"""
17 |
18 |
19 | class ConfigError(TorngasError):
20 | """raise config error"""
21 | # def __repr__(self):
22 | # return 'Configuration for %s is missing or invalid' % self.args[0]
23 |
24 |
25 | class UrlError(TorngasError):
26 | """route write error"""
27 |
28 |
29 | from tornado.web import HTTPError
30 |
31 |
32 | class APIError(HTTPError):
33 | """API error handling exception
34 |
35 | API server always returns formatted JSON to client even there is
36 | an internal server error.
37 | """
38 |
39 | def __init__(self, status_code, log_message=None, *args, **kwargs):
40 | super(APIError, self).__init__(status_code, log_message, *args, **kwargs)
41 |
42 |
43 |
--------------------------------------------------------------------------------
/torngas/global_settings.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf-8-*-
2 | import os
3 | ############
4 | # 中间件 #
5 | ############
6 | MIDDLEWARE_CLASSES = (
7 | 'torngas.middleware.SessionMiddleware',
8 | )
9 |
10 | ############
11 | # 加载的应用 #
12 | ############
13 | INSTALLED_APPS = (
14 | 'Main',
15 |
16 | )
17 |
18 |
19 | ############
20 | #应用的html模板路径
21 | ###########
22 | APPS_TEMPLATES_DIR = {
23 | 'Main': 'Main/templates'
24 |
25 | }
26 |
27 | ###########
28 | # 缓存配置 #
29 | ###########
30 | CACHES = {
31 | 'default': {
32 | 'BACKEND': 'torngas.cache.backends.localcache.LocMemCache',
33 | 'LOCATION': 'process_cache',
34 | 'OPTIONS': {
35 | 'MAX_ENTRIES': 10000,
36 | 'CULL_FREQUENCY': 3
37 | }
38 | },
39 | 'session_loccache': {
40 | 'BACKEND': 'torngas.cache.backends.localcache.LocMemCache',
41 | 'LOCATION': 'process_session',
42 | 'OPTIONS': {
43 | 'MAX_ENTRIES': 10000,
44 | 'CULL_FREQUENCY': 3
45 | }
46 |
47 | },
48 | 'memcache': {
49 | 'BACKEND': 'torngas.cache.backends.memcached.MemcachedCache',
50 | 'LOCATION': [
51 | '127.0.0.1:11211'
52 | ],
53 | 'TIMEOUT': 300
54 | },
55 | 'dummy': {
56 | 'BACKEND': 'torngas.cache.backends.dummy.DummyCache'
57 | },
58 | 'filebased': {
59 | 'BACKEND': 'torngas.cache.backends.filebased.FileBasedCache',
60 | 'LOCATION': '.'
61 | },
62 | 'redis_cache': {
63 | 'BACKEND': 'torngas.cache.backends.rediscache.RedisCache',
64 | 'LOCATION': '127.0.0.1:6379',
65 | 'TIMEOUT': 3,
66 | 'OPTIONS': {
67 | 'DB': 0,
68 | # 'PASSWORD': 'yadayada',
69 | 'PARSER_CLASS': 'redis.connection.DefaultParser'
70 | },
71 | 'KEY_PREFIX': '',
72 | 'VERSION': 1
73 | },
74 |
75 | }
76 |
77 |
78 | #################
79 | #本地化翻译文件地址#
80 | #################
81 | TRANSLATIONS_CONF = {
82 | 'translations_dir': os.path.join(os.path.dirname(__file__), 'translations'),
83 | 'locale_default': 'zh_CN',
84 | 'use_accept_language': True
85 | }
86 |
87 | #tornado全局配置
88 | TORNADO_CONF = {
89 | "static_path": "/var/www/static",
90 | "xsrf_cookies": True,
91 | "debug": True,
92 | "xheaders": True,
93 | "login_url": '/login',
94 | "cookie_secret": "bXZ/gDAbQA+zaTxdqJwxKa8OZTbuZE/ok3doaow9N4Q="
95 | #安全起见，可以定期生成新的cookie 秘钥，生成方法：
96 | #base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)
97 | }
98 |
99 | #白名单未开启，如需使用，请用元祖列出白名单ip
100 | WHITELIST = False
101 | #######
102 | # WHITELIST = (
103 | # '127.0.0.1',
104 | # '127.0.0.2',
105 | # )
106 |
107 | #tornado日志功能配置
108 | LOG_CONFIG = {
109 | 'path': '../log', #日志记录路径
110 | 'level': 'info', #日志级别
111 | 'filesize': 1000 * 1000 * 1000, #日志文件大小限制
112 | 'backup_num': 5, #最多保留文件数
113 | 'log_to_stderr': True
114 | }
115 |
116 | IPV4_ONLY = True
117 |
118 | #开启session支持
119 | SESSION = {
120 | 'session_cache_alias': 'session_loccache', # 'session_loccache',对应cache配置
121 | 'session_name': '__TORNADOID',
122 | 'cookie_domain': '',
123 | 'cookie_path': '/',
124 | 'expires': 0, # 24 * 60 * 60, # 24 hours in seconds,0代表浏览器会话过期
125 | 'ignore_change_ip': False,
126 | 'httponly': True,
127 | 'secure': False,
128 | 'secret_key': 'fLjUfxqXtfNoIldA0A0J',
129 | 'session_version': 'v1'
130 | }
131 |
132 | #配置模版引擎
133 | #引入相应的TemplateLoader即可
134 | #若使用自带的请给予None
135 | #支持mako和jinja2
136 | #mako设置为torngas.template.MakoTemplateLoader
137 | TEMPLATE_ENGINE = 'torngas.template.Jinja2TemplateLoader'
138 |
139 | TEMPLATE_CONFIG = {
140 | ########### mako 配置项 使用mako时生效###########
141 | #模版路径由torngas.handler中commonhandler重写，无需指定，模版将存在于每个应用的根目录下
142 | 'filesystem_checks': False, #通用选项
143 | 'cache_directory': '/tmp/_tmpl_cache', #模版编译文件目录,通用选项
144 | 'collection_size': 50, #暂存入内存的模版项，可以提高性能，mako选项,详情见mako文档
145 | 'cache_size': 0, #类似于mako的collection_size，设定为-1为不清理缓存，0则每次都会重编译模板
146 | 'format_exceptions': True, #格式化异常输出，mako专用
147 | 'autoescape': False #默认转义设定，jinja2专用
148 | ########### end ##################
149 | }
150 |
151 |
152 |
--------------------------------------------------------------------------------
/torngas/handlers/__init__.py:
--------------------------------------------------------------------------------
1 | from common_handler import CommonHandler, WebHandler
2 | from api_handler import ApiHandler
--------------------------------------------------------------------------------
/torngas/handlers/api_handler.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | import json
3 | from torngas.exception import APIError
4 | from common_handler import CommonHandler
5 |
6 |
7 | class ApiHandler(CommonHandler):
8 | def get_format(self):
9 | format = self.get_argument('format', None)
10 | if not format:
11 | accept = self.request.headers.get('Accept')
12 | if accept:
13 | if 'javascript' in accept:
14 | format = 'jsonp'
15 | else:
16 | format = 'json'
17 | return format or 'json'
18 |
19 |
20 | def write_api(self, obj, nofail=False):
21 | format = self.get_format()
22 | if format == 'json':
23 | self.set_header("Content-Type", "application/json; charset=UTF-8")
24 | self.write(json.dumps(obj))
25 | elif format == 'jsonp':
26 | self.set_header("Content-Type", "application/javascript")
27 | callback = self.get_argument('callback', 'callback')
28 | self.write('%s(%s);' % (callback, json.dumps(obj)))
29 | elif nofail:
30 | self.write(json.dumps(obj))
31 | else:
32 | raise APIError(400, 'Unknown response format requested: %s' % format)
33 |
34 | #根据场景可实现个性化的api错误处理
35 | # def write_error(self, status_code, **kwargs):
36 | # errortext = 'Internal error'
37 | # error_code = status_code
38 | # import traceback
39 | #
40 | # self.logger.error(traceback.format_exc())
41 | # if kwargs.get('error_code'):
42 | # error_code = kwargs.get('error_code')
43 | # exc_info = kwargs.get('exc_info')
44 | # if exc_info:
45 | # errortext = getattr(exc_info[1], 'log_message', errortext)
46 | # self.write_api({'error_code': error_code,
47 | # 'error_info': errortext,
48 | # 'description': self.request.path},
49 | # nofail=True)
50 |
--------------------------------------------------------------------------------
/torngas/handlers/common_handler.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf-8-*-
2 | import base64
3 | import hmac
4 | import hashlib
5 | import time
6 | import threading
7 | import re
8 | from urllib import unquote
9 | from tornado.escape import utf8
10 | import tornado
11 | from torngas import exception
12 | from torngas.mixin.handler_mixin import UncaughtExceptionMixin, FlashMessageMixIn
13 | from torngas.helpers import settings_helper, logger_helper
14 | from torngas.dispatch import signals
15 |
16 |
17 | class CommonHandler(tornado.web.RequestHandler):
18 | def __init__(self, application, request, **kwargs):
19 | super(CommonHandler, self).__init__(application, request, **kwargs)
20 | self._is_threaded = False
21 | self._is_torngas_finished = False
22 |
23 |
24 | def initialize(self, **kwargs):
25 | self.appname = kwargs.get('app_name', None)
26 |
27 |
28 | def prepare(self):
29 | signals.handler_started.send(sender=self.__class__)
30 | self.application.middleware_manager.run_request_hooks(self)
31 |
32 | def reverse_url(self, name, *args):
33 | return super(CommonHandler, self).reverse_url(self.appname + '-' + name, *args)
34 |
35 | def create_post_token(self):
36 | """返回一个当前时间戳的16进制哈希码，用来做post 请求的验证token"""
37 | timestamp = utf8(str(int(time.time())))
38 | value = base64.b64encode(utf8(timestamp))
39 | hashtxt = hmac.new(utf8(value), digestmod=hashlib.sha1)
40 | return utf8(hashtxt.hexdigest())
41 |
42 |
43 | @property
44 | def logger(self):
45 | return logger_helper.logger.getlogger
46 |
47 | @property
48 | def cache(self):
49 | return self.application.cache
50 |
51 | def finish(self, chunk=None):
52 |
53 | signals.handler_finished.send(sender=self.__class__)
54 | self._is_torngas_finished = True
55 | self.application.middleware_manager.run_response_hooks(self)
56 | if self._is_threaded:
57 | self._chunk = chunk
58 | tornado.ioloop.IOLoop.instance().add_callback(self.threaded_finish_callback)
59 | return
60 |
61 | super(CommonHandler, self).finish(chunk)
62 |
63 |
64 | def threaded_finish_callback(self):
65 | """
66 | 如果使用多线程回调装饰器，此方法将起作用
67 | :return:
68 | """
69 | if self.application.settings.get('debug', False):
70 | print "In the finish callback thread is ", str(threading.currentThread())
71 | super(CommonHandler, self).finish(self._chunk)
72 | self._chunk = None
73 |
74 | def get_arguments_dict(self):
75 | params = {}
76 | for key in self.request.arguments:
77 | values = self.get_arguments(key)
78 | k = unquote(key)
79 | if len(values) == 1:
80 | params[k] = values[0]
81 | else:
82 | params[k] = values
83 |
84 | return params
85 |
86 | def get_argument(self, name, default=[], strip=True):
87 | value = super(CommonHandler, self).get_argument(name, default, strip)
88 | if value == default:
89 | return value
90 | return unquote(value)
91 |
92 |
93 | def get_user_locale(self):
94 |
95 | if settings_helper.settings.TRANSLATIONS_CONF.use_accept_language:
96 | return None
97 |
98 | return tornado.locale.get(settings_helper.settings.TRANSLATIONS_CONF.locale_default)
99 |
100 | def _cleanup_param(self, val, strip=True):
101 | # Get rid of any weird control chars
102 | value = re.sub(r"[\x00-\x08\x0e-\x1f]", " ", val)
103 | value = tornado.web._unicode(value)
104 | if strip: value = value.strip()
105 | return unquote(value)
106 |
107 | def write(self, chunk, status=None):
108 | if status:
109 | self.set_status(status)
110 |
111 | super(CommonHandler, self).write(chunk)
112 |
113 |
114 | class WebHandler(UncaughtExceptionMixin, CommonHandler, FlashMessageMixIn):
115 | def get_template_path(self):
116 | templdir_settings = settings_helper.settings.APPS_TEMPLATES_DIR
117 | if not templdir_settings:
118 | raise exception.ConfigError('config {0} section no exist!'.format(templdir_settings))
119 | if len(templdir_settings):
120 | apptmpl_dir = templdir_settings.get(self.appname, None)
121 | print apptmpl_dir
122 | return ''.join([self.application.project_path, apptmpl_dir, '/']) if apptmpl_dir else None
123 | else:
124 | return None
125 |
126 |
127 | def create_template_loader(self, template_path):
128 |
129 | loader = self.application.tmpl
130 | if loader is None:
131 | return super(CommonHandler, self).create_template_loader(template_path)
132 | else:
133 | app_name = self.appname
134 | return loader(template_path, app_name=app_name)
135 |
136 |
137 | class ErrorHandler(CommonHandler, UncaughtExceptionMixin):
138 | """raise 404 error if url is not found.
139 | fixed tornado.web.RequestHandler HTTPError bug.
140 | """
141 |
142 | def prepare(self):
143 | super(ErrorHandler, self).prepare()
144 | self.set_status(404)
145 | raise tornado.web.HTTPError(404)
146 |
147 |
148 | tornado.web.ErrorHandler = ErrorHandler
--------------------------------------------------------------------------------
/torngas/helpers/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/helpers/logger_helper.py:
--------------------------------------------------------------------------------
1 | #coding=utf8
2 | import os, logging
3 | from torngas.helpers import settings_helper
4 | from tornado.options import options
5 | from tornado.log import LogFormatter
6 | from datetime import datetime
7 |
8 | NAME_PREFIX = 'torngas'
9 |
10 |
11 | class logger():
12 | def __init__(self):
13 |
14 | self.config = settings_helper.settings.LOG_CONFIG
15 | self.dirpath = self.get_dirpath()
16 | self.load_config()
17 |
18 | def get_dirpath(self):
19 | return os.path.join(os.path.abspath(self.config["path"]), str(datetime.now().date()))
20 |
21 | def get_abspath(self, file_prefix=NAME_PREFIX):
22 | return os.path.join(self.dirpath, '{0}_port[{1}].log'.format(file_prefix, options.port))
23 |
24 | @property
25 | def getlogger(self, name_prefix=NAME_PREFIX):
26 | rootlogger = logging.getLogger(name_prefix)
27 | fn, lno, func = rootlogger.findCaller()
28 | if fn.endswith('.py'):
29 | file_prefix = os.path.splitext(os.path.split(fn)[1])[0]
30 | else:
31 | file_prefix = os.path.split(fn)[1]
32 | file_path = self.get_abspath(file_prefix)
33 | logger = logging.getLogger(name_prefix + '.' + file_prefix)
34 | if os.path.exists(self.dirpath) and self.get_dirpath() == self.dirpath and \
35 | os.path.isfile(file_path):
36 | return logger
37 |
38 | else:
39 | new_dirpath = self.get_dirpath()
40 | if not os.path.exists(new_dirpath):
41 | os.makedirs(new_dirpath)
42 | self.dirpath = new_dirpath
43 | self.set_handler(logger, file_path)
44 |
45 | return logger
46 |
47 |
48 | def set_handler(self, logger=None, file_path=None):
49 | if not logger:
50 | logger = logging.getLogger(NAME_PREFIX)
51 | if not file_path:
52 | file_path = self.get_abspath()
53 | logging.getLogger().handlers = []
54 | logger.setLevel(getattr(logging, options.logging.upper()))
55 | logger.handlers = []
56 |
57 | if options.log_file_prefix:
58 | channel = logging.handlers.RotatingFileHandler(
59 | filename=file_path,
60 | maxBytes=options.log_file_max_size,
61 | backupCount=options.log_file_num_backups)
62 |
63 | channel.setFormatter(LogFormatter(color=False))
64 | logger.addHandler(channel)
65 |
66 | if (options.log_to_stderr or
67 | (options.log_to_stderr is None and not logger.handlers)):
68 | channel = logging.StreamHandler()
69 |
70 | channel.setFormatter(LogFormatter())
71 | logger.addHandler(channel)
72 |
73 |
74 | def load_config(self):
75 | if not os.path.exists(self.dirpath):
76 | os.makedirs(self.dirpath)
77 | options.log_file_prefix = self.get_abspath()
78 | options.logging = self.config["level"]
79 | options.log_to_stderr = self.config["log_to_stderr"]
80 | options.log_file_max_size = self.config["filesize"]
81 | options.log_file_num_backups = self.config["backup_num"]
82 |
83 |
84 | logger = logger()
85 |
--------------------------------------------------------------------------------
/torngas/helpers/route_helper.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | from torngas import exception
3 | from tornado.web import url as urlspec
4 | from tornado.util import import_object
5 |
6 |
7 | class RouteLoader(object):
8 | """
9 | 路由加载器，将路由加载进tornado的路由系统中，
10 | path_prefix:为模块前缀，这样路由可以省去写前缀
11 | path:由于设计为子应用形式，路由最终路径为 /path/你的路由，比如blog应用下的/index,会被解析为/blog/index,
12 | 如果不希望在路由前加/path，则为单个路由设置path='/'，path为必填参数
13 | app_name:设置为子应用的模块名，大小写必须相同，根据此设置来找模版位置，必填
14 | """
15 |
16 | def __init__(self, path_prefix, path=None, app_name=None):
17 | if not path:
18 | raise exception.UrlError('path arg not found!')
19 | if not app_name:
20 | raise exception.UrlError('app_name arg not found!')
21 | self.path_prefix = path_prefix
22 | self.path = path if path != '/' else ''
23 | self.app_name = app_name
24 |
25 | def urlhelper(self, *urllist):
26 | """
27 | 路由列表list
28 | """
29 | urls = []
30 | for u in urllist:
31 | handler_path = '.'.join([self.path_prefix, u.get('handler_path')])
32 | pattern = u.get('pattern')
33 | if pattern.endswith('/'):
34 | pattern += '?'
35 | else:
36 | pattern += '/?'
37 | path = u.get('path', None)
38 | if path:
39 | if path != '/':
40 | pattern = path + pattern
41 | else:
42 | pattern = self.path + pattern
43 | kw = dict(u.get('kwargs', {}))
44 | kw['app_name'] = self.app_name
45 | url_name = self.app_name + '-' + u.get('name')
46 | urls.append(urlspec(pattern, import_object(handler_path), kwargs=kw, name=url_name))
47 |
48 | return urls
49 |
50 |
51 | class url(object):
52 | """
53 |
54 | :param name:路由的名字，设计为必填项。这样能更好的管理路由，方便使用reverse_url生成路由
55 | :param pattern:路由表达式
56 | :param process_setting:路由的handler，view，path设置
57 | :param kwargs:额外参数提供
58 | :return:dict，路由字典
59 | """
60 |
61 | def __call__(self, name=None, pattern=None, process_setting='', kwargs=None):
62 | p_list = process_setting.split(',')
63 | setting = dict()
64 |
65 | def p_list2dict(p):
66 | tmp = p.split('=')
67 | setting[tmp[0]] = tmp[1]
68 |
69 | [p_list2dict(p) for p in p_list]
70 |
71 | view = setting.get('view')
72 | handler = setting.get('handler')
73 | path = setting.get('path', None)
74 |
75 | if not pattern or not view or not handler or not name:
76 | raise exception.ArgumentError('url argument error!')
77 |
78 | handler_path = '.'.join([view, handler])
79 | return dict(
80 | pattern=pattern,
81 | handler_path=handler_path,
82 | name=name,
83 | path=path,
84 | kwargs=kwargs or {}
85 | )
86 |
87 |
88 | url = url()
89 |
90 |
--------------------------------------------------------------------------------
/torngas/helpers/settings_helper.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | import os
3 | from tornado.util import import_object
4 | from tornado.options import options
5 | from torngas.exception import ConfigError
6 | from torngas.utils.storage import storage
7 | import warnings
8 |
9 |
10 | class Settings(object):
11 | def __init__(self):
12 | pass
13 |
14 | def get_settings(self, name):
15 | """
16 |
17 | :param name: 配置名
18 | :return:配置项
19 | """
20 | if not hasattr(self, '._config'):
21 | global_setttings = import_object('torngas.global_settings')
22 | if self._get_settings_env():
23 | try:
24 | settings_module = import_object(self._get_settings_env())
25 | except ImportError:
26 | settings_module = global_setttings
27 | warnings.warn('config import error. but now,using global settings.')
28 | else:
29 | settings_module = global_setttings
30 | self._config = settings_module
31 |
32 | if hasattr(self._config, name):
33 | return getattr(self._config, name)
34 | elif hasattr(self._config, name):
35 | return getattr(self._config, name)
36 | else:
37 | raise ConfigError('config "%s" not exist!' % name)
38 |
39 |
40 | def _get_settings_env(self):
41 | try:
42 | if options.config == 'devel':
43 | settings_env = os.environ["TORNGAS_DEV_SETTINGS_MODULE"]
44 | elif options.config == 'functest':
45 | settings_env = os.environ["TORNGAS_TEST_SETTINGS_MODULE"]
46 | elif options.config == 'production':
47 | settings_env = os.environ["TORNGAS_ONLINE_SETTINGS_MODULE"]
48 | else:
49 | settings_env = os.environ["TORNGAS_DEV_SETTINGS_MODULE"]
50 | return settings_env
51 | except KeyError:
52 | warnings.warn('need a app level settings file. but now,using global settings.')
53 |
54 |
55 | def __getattr__(self, item):
56 | setting = self.get_settings(item)
57 | return storage(setting) if type(setting) is dict else setting
58 |
59 |
60 | settings = Settings()
61 |
62 | if __name__ == "__main__":
63 | os.environ.setdefault("TORNGAS_ONLINE_SETTINGS_MODULE", "aquis_app.settings")
64 |
65 | print settings.APPS_TEMPLATES_DIR
66 | print settings.CACHES['default']
--------------------------------------------------------------------------------
/torngas/initserver.py:
--------------------------------------------------------------------------------
1 | #!/usr/local/bin/python
2 | #coding=utf-8
3 | import os
4 | import logging
5 | import tornado.httpserver
6 | import tornado.ioloop
7 | import tornado.options
8 | import tornado.web
9 | from tornado.options import define, options
10 | from tornado.log import enable_pretty_logging
11 | from tornado.util import import_object
12 | from torngas.utils import lazyimport
13 | from torngas.exception import ConfigError
14 |
15 |
16 | application_module = lazyimport('torngas.application')
17 | settings_module = lazyimport('torngas.helpers.settings_helper')
18 | logger_module = lazyimport('torngas.helpers.logger_helper')
19 |
20 | define("port", default=8000, help="run server on it", type=int)
21 | define("config", default='devel', help="if run as online ,use online,settings type:online, test, dev", type=str)
22 | define("address", default='localhost', help='listen host,default:localhost', type=str)
23 |
24 |
25 | class Server(object):
26 | def __init__(self, project_path=None, settings=settings_module.settings, application=None):
27 | self.application = application
28 | self.settings = settings
29 | self.proj_path = project_path
30 | self.urls = []
31 |
32 | def load_application(self):
33 | #加载app，进行初始化配置,如无ap参数，则使用内置app初始化
34 | logger_module.logger.load_config()
35 | tornado.options.parse_command_line()
36 | #tornado把默认的根logger加了handler
37 | #把根logger的handler去除，然后重新绑定在tornado的logger下
38 | logging.getLogger().handlers = []
39 | enable_pretty_logging(None, logging.getLogger('tornado'))
40 | #加载本地化配置
41 | tornado.locale.load_translations(self.settings.TRANSLATIONS_CONF.translations_dir)
42 | #初始化app
43 | if not self.application:
44 | self.application = application_module.AppApplication(handlers=self.urls,
45 | settings=self.settings.TORNADO_CONF)
46 |
47 | self.application.project_path = self.proj_path if self.proj_path.endswith('/') else self.proj_path + '/'
48 | self.application.tmpl = import_object(self.settings.TEMPLATE_ENGINE) if self.settings.TEMPLATE_ENGINE else None
49 | return self.application
50 |
51 | def load_urls(self):
52 | #加载app
53 |
54 | if self.settings.INSTALLED_APPS:
55 | for app in self.settings.INSTALLED_APPS:
56 | app_urls = import_object(app + '.urls.urls')
57 | self.urls.extend(app_urls)
58 |
59 | else:
60 | raise ConfigError('load urls error,INSTALLED_APPS not found!')
61 |
62 | return self.urls
63 |
64 |
65 | def server_start(self):
66 | #服务启动
67 | if self.settings.IPV4_ONLY:
68 | import socket
69 | from tornado.netutil import bind_sockets
70 |
71 | sockets = bind_sockets(options.port, options.address, family=socket.AF_INET)
72 | else:
73 | sockets = bind_sockets(options.port, options.address)
74 |
75 | self.print_settings_info()
76 | http_server = tornado.httpserver.HTTPServer(self.application)
77 | http_server.add_sockets(sockets)
78 | tornado.ioloop.IOLoop.instance().start()
79 | print 'tornado server started. listen port: %s ,host address: %s' % (options.port, options.address)
80 |
81 | def print_settings_info(self):
82 |
83 | print 'tornado version: %s' % tornado.version
84 | print 'project path: %s' % self.proj_path
85 | print 'load middleware: %s' % list(self.settings.MIDDLEWARE_CLASSES).__str__()
86 | print 'debug open: %s' % self.settings.TORNADO_CONF.debug
87 | print 'load subApp:\n %s' % self.settings.INSTALLED_APPS.__str__()
88 | print 'IPV4_Only: %s' % self.settings.IPV4_ONLY
89 | print 'template engine: %s' % self.settings.TEMPLATE_ENGINE
90 |
91 |
92 |
93 |
94 |
95 |
--------------------------------------------------------------------------------
/torngas/middleware/__init__.py:
--------------------------------------------------------------------------------
1 | #-*- coding=utf8 -*-
2 | from middleware_manager import MiddlewareManager
3 | from session_middleware import SessionMiddleware
4 | from middleware_manager import BaseMiddleware
5 |
--------------------------------------------------------------------------------
/torngas/middleware/middleware_manager.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | from tornado.util import import_object
3 | from torngas.utils import lazyimport
4 | from torngas.helpers.logger_helper import logger
5 |
6 | settings_module = lazyimport('torngas.helpers.settings_helper')
7 |
8 |
9 | class MiddlewareManager():
10 | def __init__(self):
11 | self.request_middleware = []
12 | self.response_middleware = []
13 | self.exception_middleware = []
14 | self.call_middleware = []
15 | self.endcall_middleware = []
16 | self.init_middleware = []
17 | self.load()
18 |
19 | def run_init_hooks(self, app):
20 | self.__run_hooks('init', self.init_middleware, app)
21 |
22 | def run_call_hooks(self, request):
23 | self.__run_hooks('call', self.call_middleware, request)
24 |
25 | def run_endcall_hooks(self, request):
26 | self.__run_hooks('endcall', self.endcall_middleware, request)
27 |
28 | def run_request_hooks(self, req_handler):
29 | self.__run_hooks('request', self.request_middleware, req_handler)
30 |
31 | def run_response_hooks(self, req_handler):
32 |
33 | self.__run_hooks('response', self.response_middleware, req_handler)
34 |
35 | def __run_hooks(self, type, middleware_classes, process_object):
36 | for middleware_class in middleware_classes:
37 | if (type == 'init'):
38 | middleware_class.process_init(process_object)
39 | try:
40 | if (type == 'request'):
41 | middleware_class.process_request(process_object)
42 |
43 | elif (type == 'response'):
44 | middleware_class.process_response(process_object)
45 |
46 | elif (type == 'call'):
47 | middleware_class.process_call(process_object)
48 |
49 | elif (type == 'endcall'):
50 | middleware_class.process_endcall(process_object)
51 |
52 | except Exception, ex:
53 |
54 | if hasattr(middleware_class, 'process_exception'):
55 |
56 | middleware_class.process_exception(process_object, ex)
57 | else:
58 | raise
59 |
60 | def load(self):
61 | if hasattr(settings_module.settings, 'MIDDLEWARE_CLASSES') and len(
62 | settings_module.settings.MIDDLEWARE_CLASSES) > 0:
63 | for mclass in settings_module.settings.MIDDLEWARE_CLASSES:
64 |
65 | try:
66 | cls = import_object(mclass)
67 | except ImportError, ex:
68 | logger.getlogger.error('middleware error. module __import__ failed,msg:', ex)
69 |
70 | try:
71 | inst = cls()
72 | except Exception, ex:
73 | logger.getlogger.error('middleware error. cant instantiate cls(),msg:', ex)
74 |
75 | if hasattr(inst, 'process_init'):
76 | self.init_middleware.append(inst)
77 |
78 | if hasattr(inst, 'process_request'):
79 | self.request_middleware.append(inst)
80 |
81 | if hasattr(inst, 'process_response'):
82 | self.response_middleware.append(inst)
83 |
84 | if hasattr(inst, 'process_call'):
85 | self.call_middleware.append(inst)
86 |
87 | if hasattr(inst, 'process_endcall'):
88 | self.endcall_middleware.append(inst)
89 | self.response_middleware.reverse()
90 | self.endcall_middleware.reverse()
91 |
92 |
93 | class BaseMiddleware(object):
94 | """
95 | 编写中间件需继承BaseMiddleware并实现其中任何一个方法即可
96 |
97 | """
98 |
99 | def process_init(self, application):
100 | """
101 |
102 | :param application: 应用程序对象，此方法在tornado启动时执行一次
103 | :return:None
104 | """
105 | pass
106 |
107 | def process_call(self, request):
108 | """
109 | 在请求进入application时调用，参数为请求对象，此时还未匹配路由
110 | :param request: 请求对象
111 | :return:
112 | """
113 | pass
114 |
115 | def process_request(self, handler):
116 | """
117 | 匹配路由后，执行处理handler时调用
118 | :param handler: handler对象
119 | :return:
120 | """
121 | pass
122 |
123 | def process_exception(self, req_handler, exception):
124 | """
125 | 在除了proecss_init方法外其他方法发生异常时调用
126 | :param req_handler: 如果在call方法发生异常，则返回request对象，其他方法返回handler对象
127 | :param exception: 异常栈对象
128 | :return:
129 | """
130 | pass
131 |
132 | def process_response(self, handler):
133 | """
134 | 请求结束后响应时调用，此时还未触发模版呈现
135 | :param handler: handler对象
136 | :return:
137 | """
138 | pass
139 |
140 | def process_endcall(self, handler):
141 | """
142 | 请求结束后调用，此时已完成响应并呈现用户，一般用来处理收尾操作，清理缓存对象，断开连接等
143 | :param handler: handler对象
144 | :return:
145 | """
146 | pass
--------------------------------------------------------------------------------
/torngas/middleware/session_middleware.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf-8-*-
2 | """
3 | session中间件,支持disk，进程内缓存cache,memcache
4 |
5 | """
6 | import os, time
7 |
8 | try:
9 | import cPickle as pickle
10 | except ImportError:
11 | import pickle
12 | try:
13 | import hashlib
14 |
15 | sha1 = hashlib.sha1
16 | except ImportError:
17 | import sha
18 |
19 | sha1 = sha.new
20 | import hmac, re
21 | from torngas.utils.storage import storage
22 | from torngas.utils.strtools import safestr
23 | from torngas.helpers.logger_helper import logger
24 | from torngas.utils import lazyimport
25 | from middleware_manager import BaseMiddleware
26 | from torngas import Null
27 |
28 | settings_module = lazyimport('torngas.helpers.settings_helper')
29 | cache_module = lazyimport('torngas.cache')
30 | rx = re.compile('^[0-9a-fA-F]+$')
31 |
32 |
33 | class SessionMiddleware(BaseMiddleware):
34 | def process_init(self, application):
35 | self._cachestore = cache_module.get_cache(settings_module.settings.SESSION.session_cache_alias)
36 |
37 |
38 | def process_request(self, handler):
39 | session = SessionManager(handler, self._cachestore, settings_module.settings.SESSION)
40 | session.load_session()
41 | handler.session = session
42 |
43 | def process_exception(self, ex_object, exception):
44 | self.session = Null()
45 | logger.getlogger.error("session middleware error:{0}".format(exception.message))
46 | pass
47 |
48 |
49 | def process_response(self, handler):
50 | handler.session.save()
51 | del handler.session
52 |
53 | def process_endcall(self, handler):
54 | pass
55 |
56 |
57 | """
58 | sessioin过期策略分为三种情形：
59 | 1.固定时间过期，例如10天内没有访问则过期，timeout=xxx
60 | *cookie策略：每次访问时设置cookie为过期时间
61 | *缓存策略：每次访问设置缓存失效时间为固定过期时间
62 | 2.会话过期，关闭浏览器即过期timeout=0
63 | *cookie策略：岁浏览器关闭而过期
64 | *缓存策略：设置缓存失效期为1天，每次访问更新失效期，如果浏览器关闭，则一天后被清除
65 |
66 | 3.永不过期(记住我)
67 | *cookie策略：timeout1年
68 | *缓存策略：1年
69 | """
70 | _DAY1 = 24 * 60 * 60
71 | _DAY30 = _DAY1 * 30
72 | _VERIFICATION_KEY = '__VERIFID'
73 | __all__ = [
74 | 'Session', 'SessionExpired',
75 | 'Store', 'DiskStore', 'DBStore', 'SimpleCacheStore'
76 | ]
77 |
78 | session_parameters = storage({
79 | 'session_name': '__TORNADOID',
80 | 'cookie_domain': None,
81 | 'cookie_path': '/',
82 | 'expires': 0, #24 * 60 * 60, # 24 hours in seconds
83 | 'ignore_change_ip': False,
84 | 'httponly': True,
85 | 'secure': False,
86 | 'secret_key': 'fLjUfxqXtfNoIldA0A0J',
87 | 'session_version': 'V1.6'
88 | })
89 |
90 |
91 | class SessionManager(object):
92 | _killed = False
93 |
94 | def __init__(self, handler, store, config=session_parameters):
95 |
96 | self._get_cookie = handler.get_cookie
97 | self._set_cookie = handler.set_cookie
98 | self.remote_ip = handler.request.remote_ip
99 | self.store = store
100 |
101 | self.config = storage(config)
102 | self._data = {}
103 |
104 | def __contains__(self, key):
105 | return key in self._data
106 |
107 | def __setitem__(self, key, value):
108 | self._data[key] = value
109 |
110 | def __getitem__(self, key):
111 | return self._data.get(key, None)
112 |
113 |
114 | def __delitem__(self, key):
115 | del self._data[key]
116 |
117 | def get(self, key, default=None):
118 | return self._data.get(key, default)
119 |
120 | def load_session(self):
121 | """
122 | 加载session
123 | :return:
124 | """
125 | self.sessionid = self._get_cookie(self.config.session_name)
126 |
127 | if self.sessionid and not self._valid_session_id(self.sessionid):
128 | self.sessionid = None
129 | self.expired()
130 |
131 | if self.sessionid:
132 | if self.sessionid in self.store:
133 | expires, _data = self.store.get(self.sessionid)
134 | self._data.update(_data)
135 | self.config.expires = expires
136 | self._validate_ip()
137 | hmac_verif = self._get_cookie(_VERIFICATION_KEY)
138 | if hmac_verif != self._generate_hmac(self.sessionid):
139 | self.expired()
140 |
141 | if not self.sessionid:
142 | self.sessionid = self._create_sessionid()
143 |
144 | self._data['remote_ip'] = self.remote_ip
145 |
146 |
147 | def save(self):
148 | if not self._killed:
149 |
150 | httponly = self.config.httponly
151 | secure = self.config.secure
152 | expires = self.config.expires#单位是秒
153 | cache_expires = expires
154 | if expires == 0:
155 | #过期时间为0时，对于tornado来说，是会话有效期，关闭浏览器失效，但是
156 | #对于cache缓存而言，无法及时捕获会话结束状态，鉴于此，将cache的缓存设置为一天
157 | #cache在每次请求后会清理过期的缓存
158 | cache_expires = _DAY1
159 |
160 | if not secure:
161 | secure = ''
162 |
163 | if not httponly:
164 | httponly = ''
165 | set_expire = 0 if expires == 0 else time.time() + expires
166 | self._set_cookie(
167 | self.config.session_name,
168 | self.sessionid,
169 | domain=self.config.cookie_domain or '',
170 | expires=set_expire,
171 | path=self.config.cookie_path,
172 | secure=secure,
173 | httponly=httponly)
174 | self._set_cookie(_VERIFICATION_KEY, self._generate_hmac(self.sessionid),
175 | domain=self.config.cookie_domain or '',
176 | expires=set_expire,
177 | path=self.config.cookie_path,
178 | secure=secure,
179 | httponly=httponly)
180 | self.store.set(self.sessionid, ( expires, self._data), cache_expires)
181 |
182 | else:
183 | self._set_cookie(self.config.session_name, self.sessionid, expires=-1)
184 | self._set_cookie(_VERIFICATION_KEY, self._generate_hmac(self.sessionid), expires=-1)
185 | del self.store[self.sessionid]
186 |
187 |
188 | def _valid_session_id(self, sessionid):
189 | """
190 | 验证sessionid格式
191 | :return:bool
192 | """
193 |
194 | if sessionid:
195 | sessionid = sessionid.split('|')[0]
196 |
197 | return rx.match(sessionid)
198 |
199 | def expired(self):
200 | """
201 | 强制过期
202 | :return:None
203 | """
204 | self._killed = True
205 | self.save()
206 |
207 | def _create_sessionid(self):
208 | while True:
209 | rand = os.urandom(16)
210 | now = time.time()
211 | secret_key = self.config.secret_key
212 | session_id = sha1("%s%s%s%s" % (rand, now, safestr(self.remote_ip), secret_key))
213 | session_id = session_id.hexdigest()
214 | if session_id not in self.store:
215 | break
216 | return str(session_id).upper() + '|' + self.config.session_version
217 |
218 | def _generate_hmac(self, session_id):
219 | return hmac.new(session_id, self.config.secret_key, hashlib.sha1).hexdigest()
220 |
221 |
222 | def _validate_ip(self):
223 | if self.sessionid and self._data.get('remote_ip', None) != self.remote_ip:
224 | if not self.config.ignore_change_ip:
225 | return self.expired()
226 |
227 | def set_expire(self, expires):
228 | self.config.expires = expires
229 | self.save()
230 |
231 |
232 | if __name__ == '__main__':
233 | import doctest
234 |
235 | doctest.testmod()
236 |
237 |
238 |
239 |
240 |
--------------------------------------------------------------------------------
/torngas/mixin/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/mixin/handler_mixin.py:
--------------------------------------------------------------------------------
1 | #coding-utf8
2 | import httplib
3 | import os
4 | from tornado.escape import json_encode, json_decode
5 | from tornado.util import import_object
6 | from torngas.template import Jinja2TemplateLoader
7 |
8 |
9 | class UncaughtExceptionMixin(object):
10 | def get_error_html(self, status_code, **kwargs):
11 |
12 | def get_snippet(fp, target_line, num_lines):
13 | if fp.endswith('.html'):
14 | fp = os.path.join(self.get_template_path(), fp)
15 |
16 | half_lines = (num_lines / 2)
17 | try:
18 | with open(fp) as f:
19 | all_lines = [line for line in f]
20 |
21 | return ''.join(all_lines[target_line - half_lines:target_line + half_lines])
22 | except Exception, ex:
23 | self.logger.error(ex)
24 |
25 | return ''
26 |
27 | if self.application.settings.get('debug', False) is False:
28 | full_message = kwargs.get('exception', None)
29 | if not full_message or unicode(full_message) == '':
30 | full_message = 'Sky is falling!'
31 |
32 | return "%(code)d: %(message)s%(code)d: %(message)s
%(full_message)s" % {
33 | "code": status_code,
34 | "message": httplib.responses[status_code],
35 | "full_message": full_message,
36 | }
37 |
38 | else:
39 | exception = kwargs.get('exception', None)
40 | resource = os.path.split(os.path.dirname(__file__))[0]
41 | loader = self.application.tmpl
42 |
43 | if loader is None or int(status_code) == 404:
44 | tmpl_file = '/resource/exception.html'
45 | elif loader is Jinja2TemplateLoader:
46 | tmpl_file = '/resource/exception.j2'
47 | else:
48 | tmpl_file = '/resource/exception.mako'
49 | import traceback
50 | import sys
51 | import tornado
52 |
53 | return self.render_string(resource + tmpl_file, get_snippet=get_snippet,
54 | exception=exception, traceback=traceback, sys=sys, tornado=tornado,
55 | status_code=status_code, os=os, kwargs=kwargs)
56 |
57 | def write_error(self, status_code, **kwargs):
58 |
59 | if 'exc_info' in kwargs:
60 | exc_info = kwargs.pop('exc_info')
61 | kwargs['exception'] = exc_info[1]
62 | self.finish(self.get_error_html(status_code, **kwargs))
63 | return
64 |
65 |
66 | class FlashMessageMixIn(object):
67 | """
68 | Store a message between requests which the user needs to see.
69 |
70 | views
71 | -------
72 |
73 | self.flash("Welcome back, %s" % username, 'success')
74 |
75 | base.html
76 | ------------
77 |
78 | {% set messages = handler.get_flashed_messages() %}
79 | {% if messages %}
80 |
81 | {% for category, msg in messages %}
82 | {{ msg }}
83 | {% end %}
84 |
85 | {% end %}
86 | """
87 | _flash_name = "__flhMsg"
88 |
89 | def flash(self, message, category='message'):
90 | messages = self.messages()
91 | messages.append((category, message))
92 | self.set_secure_cookie(self._flash_name, json_encode(messages))
93 |
94 | def messages(self):
95 | messages = self.get_secure_cookie(self._flash_name)
96 | messages = json_decode(messages) if messages else []
97 | return messages
98 |
99 | def get_flashed_messages(self):
100 | messages = self.messages()
101 | self.clear_cookie(self._flash_name)
102 | return messages
103 |
104 |
105 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/models/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/models/main_models.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | #your models module write here
3 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/templates/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | welcome torngas!
5 |
6 |
7 | {{ welcome }}
8 |
9 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/urls.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | from torngas.helpers.route_helper import url, RouteLoader
3 |
4 | route = RouteLoader(path_prefix='Main.views', path='/', app_name='Main')
5 |
6 | urls = route.urlhelper(
7 | url('Index', r'/', 'view=main_handler,handler=Main')
8 | )
9 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/views/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/Main/views/main_handler.py:
--------------------------------------------------------------------------------
1 | from torngas.handlers import WebHandler
2 |
3 |
4 | class BaseHandler(WebHandler):
5 | """
6 | do some your base things
7 | """
8 |
9 |
10 | class Main(BaseHandler):
11 | def get(self):
12 | welcome = "hello word!"
13 | self.render("index.html", welcome=welcome)
14 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/app/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'mengqingyun'
2 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/app/settings_devel.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf-8-*-
2 | import os
3 | ############
4 | # 中间件 #
5 | ############
6 | MIDDLEWARE_CLASSES = (
7 | 'torngas.middleware.SessionMiddleware',
8 | )
9 |
10 | ############
11 | # 加载的应用 #
12 | ############
13 | INSTALLED_APPS = (
14 | 'Main',
15 | )
16 |
17 |
18 | ############
19 | #应用的html模板路径
20 | ###########
21 | APPS_TEMPLATES_DIR = {
22 | 'Main': 'Main/templates'
23 | }
24 |
25 | ###########
26 | # 缓存配置 #
27 | ###########
28 | CACHES = {
29 | 'default': {
30 | 'BACKEND': 'torngas.cache.backends.localcache.LocMemCache',
31 | 'LOCATION': 'process_cache',
32 | 'OPTIONS': {
33 | 'MAX_ENTRIES': 10000,
34 | 'CULL_FREQUENCY': 3
35 | }
36 | },
37 | 'session_loccache': {
38 | 'BACKEND': 'torngas.cache.backends.localcache.LocMemCache',
39 | 'LOCATION': 'process_session',
40 | 'OPTIONS': {
41 | 'MAX_ENTRIES': 10000,
42 | 'CULL_FREQUENCY': 3
43 | }
44 |
45 | },
46 | 'memcache': {
47 | 'BACKEND': 'torngas.cache.backends.memcached.MemcachedCache',
48 | 'LOCATION': [
49 | '192.168.1.107:11211'
50 | ],
51 | 'TIMEOUT': 300
52 | },
53 | 'dummy': {
54 | 'BACKEND': 'torngas.cache.backends.dummy.DummyCache'
55 | },
56 | 'filebased': {
57 | 'BACKEND': 'torngas.cache.backends.filebased.FileBasedCache',
58 | 'LOCATION': '.'
59 | },
60 | 'redis_cache': {
61 | 'BACKEND': 'torngas.cache.backends.rediscache.RedisCache',
62 | 'LOCATION': '192.168.1.107:6379',
63 | 'TIMEOUT': 3,
64 | 'OPTIONS': {
65 | 'DB': 0,
66 | # 'PASSWORD': 'yadayada',
67 | 'PARSER_CLASS': 'redis.connection.DefaultParser'
68 | },
69 | 'KEY_PREFIX': '',
70 | 'VERSION': 1
71 | },
72 |
73 | }
74 |
75 |
76 | #################
77 | #本地化翻译文件地址#
78 | #################
79 | TRANSLATIONS_CONF = {
80 | 'translations_dir': os.path.join(os.path.dirname(__file__), 'translations'),
81 | 'locale_default': 'zh_CN',
82 | 'use_accept_language': True
83 | }
84 |
85 | #tornado全局配置
86 | TORNADO_CONF = {
87 | "static_path": os.path.join(os.path.dirname(__file__), "../static"),
88 | "xsrf_cookies": True,
89 | "debug": True,
90 | "xheaders": True,
91 | "login_url": '/login',
92 | "permanent_session_lifetime": 0,
93 | "cookie_secret": "bXZ/gDAbQA+zaTxdqJwxKa8OZTbuZE/ok3doaow9N4Q="
94 | #安全起见，可以定期生成新的cookie 秘钥，生成方法：
95 | #base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)
96 | }
97 |
98 | #白名单未开启，如需使用，请用元祖列出白名单ip
99 | WHITELIST = False
100 | #######
101 | # WHITELIST = (
102 | # '127.0.0.1',
103 | # '127.0.0.2',
104 | # )
105 |
106 | #tornado日志功能配置
107 | LOG_CONFIG = {
108 | 'path': '../log', #日志记录路径
109 | 'level': 'info', #日志级别
110 | 'filesize': 1000 * 1000 * 1000, #日志文件大小限制
111 | 'backup_num': 5, #最多保留文件数
112 | 'log_to_stderr': True
113 | }
114 |
115 | IPV4_ONLY = True
116 |
117 | #开启session支持
118 | SESSION = {
119 | 'session_cache_alias': 'session_loccache', # 'session_loccache',
120 | 'session_name': '__TORNADOID',
121 | 'cookie_domain': '',
122 | 'cookie_path': '/',
123 | 'expires': 0, # 24 * 60 * 60, # 24 hours in seconds
124 | 'ignore_change_ip': False,
125 | # 'expired_message': 'Session expired',
126 | 'httponly': True,
127 | 'secure': False,
128 | 'secret_key': 'fLjUfxqXtfNoIldA0A0J',
129 | 'session_version': 'V1'
130 | }
131 |
132 | #配置模版引擎
133 | #引入相应的TemplateLoader即可
134 | #若使用自带的请给予None
135 | #支持mako和jinja2
136 | #mako设置为torngas.template.MakoTemplateLoader
137 | TEMPLATE_ENGINE = 'torngas.template.Jinja2TemplateLoader'
138 |
139 | TEMPLATE_CONFIG = {
140 | ########### mako 配置项 使用mako时生效###########
141 | #模版路径由torngas.handler中commonhandler重写，无需指定，模版将存在于每个应用的根目录下
142 | 'filesystem_checks': False, #通用选项
143 | 'cache_directory': '../_tmpl_cache', #模版编译文件目录,通用选项
144 | 'collection_size': 50, #暂存入内存的模版项，可以提高性能，mako选项,详情见mako文档
145 | 'cache_size': 0, #类似于mako的collection_size，设定为-1为不清理缓存，0则每次都会重编译模板
146 | 'format_exceptions': True, #格式化异常输出，mako专用
147 | 'autoescape': False #默认转义设定，jinja2专用
148 | ########### end ##################
149 | }
150 |
151 |
152 |
--------------------------------------------------------------------------------
/torngas/resource/app_template/app/settings_functest.py:
--------------------------------------------------------------------------------
1 | """
2 | write your func test settings
3 | """
4 | from settings_devel import *
--------------------------------------------------------------------------------
/torngas/resource/app_template/app/settings_production.py:
--------------------------------------------------------------------------------
1 | """
2 | write your production settings
3 | """
4 | from settings_functest import *
--------------------------------------------------------------------------------
/torngas/resource/app_template/runserver.py:
--------------------------------------------------------------------------------
1 | #!/usr/local/bin/python
2 | #coding=utf-8
3 | import os
4 | from torngas.initserver import Server
5 |
6 | PROJECT_NAME = "app"
7 | PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
8 | os.environ.setdefault("TORNGAS_ONLINE_SETTINGS_MODULE", "%s.settings_production" % PROJECT_NAME)
9 | os.environ.setdefault("TORNGAS_DEV_SETTINGS_MODULE", "%s.settings_devel" % PROJECT_NAME)
10 | os.environ.setdefault("TORNGAS_TEST_SETTINGS_MODULE", "%s.settings_functest" % PROJECT_NAME)
11 | if __name__ == '__main__':
12 | server = Server(PROJECT_PATH, application=None)
13 | server.load_urls()
14 | server.load_application()
15 | server.server_start()
16 |
17 |
--------------------------------------------------------------------------------
/torngas/resource/exception.html:
--------------------------------------------------------------------------------
1 | {% set type, value, tback = sys.exc_info() %}
2 |
3 |
5 |
6 |
7 |
8 | HTTP Status {{ status_code }} » Tornado v{{ tornado.version }}
9 |
10 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
HTTP Status {{ status_code }}
108 |
109 | {% if exception %}
110 | {% set traceback_list = traceback.extract_tb(tback) %}
111 | Application raised {{ exception.__class__.__name__ }}: {{ exception }}.
112 | {% set filepath, line, method, code = traceback_list[-1] %}
113 |
114 |
115 |
116 |
123 |
124 |
125 | {% set extension = os.path.splitext(filepath)[1][1:] %}
126 | {% if extension in ['py', 'html', 'htm'] %}
127 |
128 | {{ escape(get_snippet(filepath, line, 10)) }}
129 |
130 | {% else %}
131 | Cannot load file, type not supported.
132 | {% end %}
133 | |
134 |
135 |
136 |
137 |
138 | Full Traceback
139 | click each row to view full detail and source code snippet.
140 |
141 |
142 | {% for filepath, line, method, code in traceback_list %}
143 |
144 |
151 |
152 |
153 | {% set extension = os.path.splitext(filepath)[1][1:] %}
154 | {% if extension in ['py', 'html', 'htm'] %}
155 |
156 | {{ escape(get_snippet(filepath, line, 10)) }}
157 |
158 | {% else %}
159 | Cannot load file, type not supported.
160 | {% end %}
161 | |
162 |
163 |
164 |
165 | {% end %}
166 |
Request Headers
169 |
170 |
171 |
172 | {% for hk, hv in handler.request.headers.iteritems() %}
173 |
174 | | {{ hk }} |
175 | {{ hv }} |
176 |
177 | {% end %}
178 |
179 |
180 |
181 |
182 |
183 | Response Headers
184 |
185 |
186 |
187 | {% for hk, hv in handler._headers.iteritems() %}
188 |
189 | | {{ hk }} |
190 | {{ hv }} |
191 |
192 | {% end %}
193 |
194 |
195 | {% end %}
196 |
197 |
198 |
199 |
200 |
201 |
204 |
205 |
206 |
208 |
209 |
210 |
211 |
214 |
222 |
223 |
--------------------------------------------------------------------------------
/torngas/resource/exception.j2:
--------------------------------------------------------------------------------
1 |
2 |
3 | {% set type, value, tback = sys.exc_info() %}
4 |
5 |
7 |
8 |
9 |
10 | HTTP Status {{ status_code }} » Tornado v{{ tornado.version }}
11 |
12 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
HTTP Status {{ status_code }}
110 |
111 | {% if exception %}
112 | {% set traceback_list = traceback.extract_tb(tback) %}
113 | Application raised {{ exception.__class__.__name__ }}: {{ exception }}.
114 | {% set filepath, line, method, code = traceback_list[-1] %}
115 |
116 |
117 |
118 |
125 |
126 |
127 | {% set extension = os.path.splitext(filepath)[1][1:] %}
128 | {% if extension in ['py', 'html', 'htm'] %}
129 |
130 | {{ get_snippet(filepath, line, 10)|e }}
131 |
132 | {% else %}
133 | Cannot load file, type not supported.
134 | {% endif %}
135 | |
136 |
137 |
138 |
139 |
140 | Full Traceback
141 | click each row to view full detail and source code snippet.
142 |
143 |
144 | {% for filepath, line, method, code in traceback_list %}
145 |
146 |
153 |
154 |
155 | {% set extension = os.path.splitext(filepath)[1][1:] %}
156 | {% if extension in ['py', 'html', 'htm'] %}
157 |
158 | {{ get_snippet(filepath, line, 10)|e }}
159 |
160 | {% else %}
161 | Cannot load file, type not supported.
162 | {% endif %}
163 | |
164 |
165 |
166 |
167 | {% endfor %}
168 |
Request Headers
171 |
172 |
173 |
174 | {% for hk, hv in handler.request.headers.iteritems() %}
175 |
176 | | {{ hk }} |
177 | {{ hv }} |
178 |
179 | {% endfor %}
180 |
181 |
182 |
183 |
184 |
185 | Response Headers
186 |
187 |
188 |
189 | {% for hk, hv in handler._headers.iteritems() %}
190 |
191 | | {{ hk }} |
192 | {{ hv }} |
193 |
194 | {% endfor %}
195 |
196 |
197 | {% endif %}
198 |
199 |
200 |
201 |
202 |
203 |
206 |
207 |
208 |
210 |
211 |
212 |
213 |
216 |
224 |
225 |
--------------------------------------------------------------------------------
/torngas/resource/exception.mako:
--------------------------------------------------------------------------------
1 | <%
2 | import traceback
3 | import sys
4 | import os
5 | import tornado
6 | type, value, tback = sys.exc_info()
7 | %>
8 |
9 |
10 |
11 |
12 |
13 |
14 | HTTP Status ${status_code} » Tornado v${ tornado.version }
15 |
16 |
107 |
108 |
109 |
110 |
111 | % if status_code!=404:
112 | <%
113 | style='background:#ffc;'
114 | %>
115 | % endif
116 |
117 |
HTTP Status ${ status_code }
118 |
119 | % if exception :
120 | <%
121 | traceback_list = traceback.extract_tb(tback)
122 | %>
123 | Application raised ${ exception.__class__.__name__ }: ${ exception }.
124 | <%
125 | filepath, line, method, code = traceback_list[-1]
126 | %>
127 |
128 |
129 |
130 |
141 |
142 |
143 | <%
144 | extension = os.path.splitext(filepath)[1][1:]
145 | %>
146 |
147 | % if extension in ['py', 'html', 'htm']:
148 |
149 | ${get_snippet(filepath, line, 10)|h }
150 |
151 |
152 | % else:
153 | Cannot load file, type not supported.
154 | % endif
155 | |
156 |
157 |
158 |
159 |
160 | Full Traceback
161 | click each row to view full detail and source code snippet.
162 |
163 |
164 | % for filepath, line, method, code in traceback_list :
165 |
166 |
173 |
174 |
175 | % if os.path.splitext(filepath)[1][1:] in ['py', 'html', 'htm'] :
176 |
177 | ${ get_snippet(filepath, line, 10)|h }
178 |
179 | % else:
180 | Cannot load file, type not supported.
181 | % endif
182 | |
183 |
184 |
185 |
186 | % endfor
187 |
Request Headers
190 |
191 |
192 |
193 | % for hk, hv in handler.request.headers.iteritems() :
194 |
195 | | ${hk} |
196 | ${ hv } |
197 |
198 | % endfor
199 |
200 |
201 |
202 |
203 |
204 | Response Headers
205 |
206 |
207 |
208 | % for hk, hv in handler._headers.iteritems() :
209 |
210 | | ${ hk } |
211 | ${ hv } |
212 |
213 | % endfor
214 |
215 |
216 | % endif
217 |
218 |
219 |
220 |
221 |
224 |
225 |
226 |
228 |
229 |
230 |
231 |
234 |
242 |
243 |
--------------------------------------------------------------------------------
/torngas/template/__init__.py:
--------------------------------------------------------------------------------
1 | from mako_loader import MakoTemplateLoader
2 | from jinja2_loader import Jinja2TemplateLoader
--------------------------------------------------------------------------------
/torngas/template/jinja2_loader.py:
--------------------------------------------------------------------------------
1 | import os
2 | from jinja2 import Environment, FileSystemLoader, FileSystemBytecodeCache
3 | from tornado.template import Loader
4 | from torngas.helpers.settings_helper import settings
5 |
6 | _CACHE = FileSystemBytecodeCache()
7 | _LOADER = FileSystemLoader([])
8 | _JINJA_ENV = Environment(bytecode_cache=_CACHE,
9 | autoescape=settings.TEMPLATE_CONFIG.autoescape,
10 | cache_size=settings.TEMPLATE_CONFIG.cache_size,
11 | auto_reload=settings.TEMPLATE_CONFIG.filesystem_checks,
12 | loader=_LOADER)
13 |
14 |
15 | class Jinja2TemplateLoader(Loader):
16 | def __init__(self, root_directory='', app_name='', **kwargs):
17 | super(Jinja2TemplateLoader, self).__init__(root_directory, **kwargs)
18 | path = os.path.abspath(root_directory)
19 | _JINJA_ENV.loader.searchpath = [path]
20 |
21 | cache_dir = os.path.abspath(
22 | os.path.join(settings.TEMPLATE_CONFIG.cache_directory, app_name))
23 | if not os.path.exists(cache_dir):
24 | os.makedirs(cache_dir)
25 | _CACHE.directory = cache_dir
26 |
27 |
28 | def load(self, name):
29 | with self.lock:
30 | if os.path.isabs(name):
31 | path, file = os.path.split(name)
32 | _JINJA_ENV.loader.searchpath = [path]
33 | template = _JINJA_ENV.get_template(file)
34 | else:
35 | template = _JINJA_ENV.get_template(name)
36 | template.generate = template.render
37 | return template
38 |
39 |
40 | def reset(self):
41 | if hasattr(_JINJA_ENV, 'bytecode_cache') and _JINJA_ENV.bytecode_cache:
42 | _JINJA_ENV.bytecode_cache.clear()
--------------------------------------------------------------------------------
/torngas/template/mako_loader.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from mako.lookup import TemplateLookup
4 | from tornado.template import Loader
5 | from torngas.helpers.settings_helper import settings
6 |
7 |
8 | class MakoTemplateLoader(Loader):
9 | def __init__(self, root_directory, app_name, **kwargs):
10 | super(MakoTemplateLoader, self).__init__(root_directory, **kwargs)
11 | path = os.path.abspath(root_directory)
12 | self._lookup = TemplateLookup(directories=path, input_encoding='utf-8',
13 | output_encoding='utf-8',
14 | encoding_errors='replace',
15 | filesystem_checks=settings.TEMPLATE_CONFIG.filesystem_checks,
16 | module_directory=os.path.abspath(
17 | os.path.join(settings.TEMPLATE_CONFIG.cache_directory, app_name)),
18 | default_filters=['decode.utf8'],
19 | collection_size=settings.TEMPLATE_CONFIG.collection_size,
20 | format_exceptions=settings.TEMPLATE_CONFIG.format_exceptions)
21 |
22 | def load(self, name):
23 | with self.lock:
24 | if os.path.isabs(name):
25 | path, file = os.path.split(name)
26 | self._lookup.directories = [path]
27 | template = self._lookup.get_template(file)
28 | else:
29 | template = self._lookup.get_template(name)
30 | template.generate = template.render
31 |
32 | return template
33 |
34 | def reset(self):
35 | pass
--------------------------------------------------------------------------------
/torngas/utils/__init__.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf8-*-
2 | """
3 | utils提供以下工具集
4 | "Storage", "storage", "storify",
5 | "Counter", "counter",
6 | "iters",
7 | "rstrips", "lstrips", "strips",
8 | "safeunicode", "safestr", "utf8",
9 | "TimeoutError", "timelimit",
10 | "re_compile", "re_subm",
11 | "group", "uniq", "iterview",
12 | "IterBetter", "iterbetter",
13 | "safeiter", "safewrite",
14 | "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd",
15 | "requeue", "restack",
16 | "listget", "intget", "datestr",
17 | "numify", "denumify", "commify", "dateify",
18 | "tryall",
19 | "autoassign",
20 | "to36",
21 | """
22 | from tornado.util import import_object
23 | class LazyImport:
24 | """lazy import module"""
25 | def __init__(self,module_name):
26 | self.module_name=module_name
27 | self.module=None
28 | def __getattr__(self,func_name):
29 | if self.module is None:
30 | self.module=import_object(self.module_name)
31 | return getattr(self.module,func_name)
32 |
33 | lazyimport=LazyImport
--------------------------------------------------------------------------------
/torngas/utils/funcutils.py:
--------------------------------------------------------------------------------
1 | from torngas.utils.iterutils import dictincr
2 | import traceback
3 |
4 |
5 | def tryall(context, prefix=None):
6 | """
7 | Tries a series of functions and prints their results.
8 | `context` is a dictionary mapping names to values;
9 | the value will only be tried if it's callable.
10 |
11 | >>> tryall(dict(j=lambda: True))
12 | j: True
13 | ----------------------------------------
14 | results:
15 | True: 1
16 |
17 | For example, you might have a file `test/stuff.py`
18 | with a series of functions testing various things in it.
19 | At the bottom, have a line:
20 |
21 | if __name__ == "__main__": tryall(globals())
22 |
23 | Then you can run `python test/stuff.py` and get the results of
24 | all the tests.
25 | """
26 | context = context.copy() # vars() would update
27 | results = {}
28 | for (key, value) in context.iteritems():
29 | if not hasattr(value, '__call__'):
30 | continue
31 | if prefix and not key.startswith(prefix):
32 | continue
33 | print key + ':',
34 | try:
35 | r = value()
36 | dictincr(results, r)
37 | print r
38 | except:
39 | print 'ERROR'
40 | dictincr(results, 'ERROR')
41 | print ' ' + '\n '.join(traceback.format_exc().split('\n'))
42 |
43 | print '-' * 40
44 | print 'results:'
45 | for (key, value) in results.iteritems():
46 | print ' ' * 2, str(key) + ':', value
47 |
48 |
49 | def autoassign(self, locals):
50 | """
51 | Automatically assigns local variables to `self`.
52 |
53 | >>> self = storage()
54 | >>> autoassign(self, dict(a=1, b=2))
55 | >>> self
56 |
57 |
58 | Generally used in `__init__` methods, as in:
59 |
60 | def __init__(self, foo, bar, baz=1): autoassign(self, locals())
61 | """
62 | for (key, value) in locals.iteritems():
63 | if key == 'self':
64 | continue
65 | setattr(self, key, value)
66 |
67 |
68 | import json, re
69 | import locale
70 |
71 |
72 | def strip_html(data):
73 | if not data:
74 | return
75 | p = re.compile(r'<[^<]*?/?>')
76 | return p.sub('', data)
77 |
78 |
79 | def add_commas(val, as_data_type='int', the_locale=locale.LC_ALL):
80 | locale.setlocale(the_locale, "")
81 | if as_data_type == 'int':
82 | return locale.format('%d', int(val), True)
83 | elif as_data_type == 'float':
84 | return locale.format('%f', float(val), True)
85 | else:
86 | return val
87 |
88 |
89 | def get_time_string(str):
90 | if str == "N/A":
91 | return str
92 |
93 | parts = str.split("/")
94 | isPM = parts[0].find('am') == -1
95 | if not isPM:
96 | parts[0] = parts[0].replace("am", "")
97 |
98 | parts[1] = parts[1].replace("c", "")
99 | if (len(parts[0]) >= 3):
100 | if (len(parts[0]) == 4):
101 | parts[0] = parts[0][0:2] + ":" + parts[0][2:]
102 | else:
103 | parts[0] = parts[0][:1] + ":" + parts[0][1:]
104 | if (len(parts[1]) >= 3):
105 | if (len(parts[1]) == 4):
106 | parts[1] = parts[1][0:2] + ":" + parts[1][2:]
107 | else:
108 | parts[1] = parts[1][:1] + ":" + parts[1][1:]
109 |
110 | if isPM:
111 | time = parts[0] + "/" + parts[1] + "c"
112 | else:
113 | time = parts[0] + "am/" + parts[1] + "c"
114 |
115 | return time
116 |
117 |
118 | class Pluralizer():
119 | #
120 | # (pattern, search, replace) regex english plural rules tuple
121 | #
122 | rule_tuple = (
123 | ('[ml]ouse$', '([ml])ouse$', '\\1ice'),
124 | ('child$', 'child$', 'children'),
125 | ('booth$', 'booth$', 'booths'),
126 | ('foot$', 'foot$', 'feet'),
127 | ('ooth$', 'ooth$', 'eeth'),
128 | ('l[eo]af$', 'l([eo])af$', 'l\\1aves'),
129 | ('sis$', 'sis$', 'ses'),
130 | ('man$', 'man$', 'men'),
131 | ('ife$', 'ife$', 'ives'),
132 | ('eau$', 'eau$', 'eaux'),
133 | ('lf$', 'lf$', 'lves'),
134 | ('[sxz]$', '$', 'es'),
135 | ('[^aeioudgkprt]h$', '$', 'es'),
136 | ('(qu|[^aeiou])y$', 'y$', 'ies'),
137 | ('$', '$', 's')
138 | )
139 |
140 | def regex_rules(self, rules=rule_tuple):
141 | for line in rules:
142 | pattern, search, replace = line
143 | yield lambda word: re.search(pattern, word) and re.sub(search, replace, word)
144 |
145 | def plural(self, noun):
146 | for rule in self.regex_rules():
147 | result = rule(noun)
148 | if result:
149 | return result
150 |
151 |
152 | if __name__ == "__main__":
153 | import doctest
154 |
155 | doctest.testmod()
156 |
157 |
--------------------------------------------------------------------------------
/torngas/utils/iterutils.py:
--------------------------------------------------------------------------------
1 | import sys ,traceback ,time
2 | from torngas.utils.storage import storage
3 |
4 |
5 | class Counter(storage):
6 | """Keeps count of how many times something is added.
7 |
8 | >>> c = counter()
9 | >>> c.add('x')
10 | >>> c.add('x')
11 | >>> c.add('x')
12 | >>> c.add('x')
13 | >>> c.add('x')
14 | >>> c.add('y')
15 | >>> c
16 |
17 | >>> c.most()
18 | ['x']
19 | """
20 |
21 | def add(self, n):
22 | self.setdefault(n, 0)
23 | self[n] += 1
24 |
25 | def most(self):
26 | """Returns the keys with maximum count."""
27 | m = max(self.itervalues())
28 | return [k for k, v in self.iteritems() if v == m]
29 |
30 | def least(self):
31 | """Returns the keys with mininum count."""
32 | m = min(self.itervalues())
33 | return [k for k, v in self.iteritems() if v == m]
34 |
35 | def percent(self, key):
36 | """Returns what percentage a certain key is of all entries.
37 |
38 | >>> c = counter()
39 | >>> c.add('x')
40 | >>> c.add('x')
41 | >>> c.add('x')
42 | >>> c.add('y')
43 | >>> c.percent('x')
44 | 0.75
45 | >>> c.percent('y')
46 | 0.25
47 | """
48 | return float(self[key]) / sum(self.values())
49 |
50 | def sorted_keys(self):
51 | """Returns keys sorted by value.
52 |
53 | >>> c = counter()
54 | >>> c.add('x')
55 | >>> c.add('x')
56 | >>> c.add('y')
57 | >>> c.sorted_keys()
58 | ['x', 'y']
59 | """
60 | return sorted(self.keys(), key=lambda k: self[k], reverse=True)
61 |
62 | def sorted_values(self):
63 | """Returns values sorted by value.
64 |
65 | >>> c = counter()
66 | >>> c.add('x')
67 | >>> c.add('x')
68 | >>> c.add('y')
69 | >>> c.sorted_values()
70 | [2, 1]
71 | """
72 | return [self[k] for k in self.sorted_keys()]
73 |
74 | def sorted_items(self):
75 | """Returns items sorted by value.
76 |
77 | >>> c = counter()
78 | >>> c.add('x')
79 | >>> c.add('x')
80 | >>> c.add('y')
81 | >>> c.sorted_items()
82 | [('x', 2), ('y', 1)]
83 | """
84 | return [(k, self[k]) for k in self.sorted_keys()]
85 |
86 | def __repr__(self):
87 | return ''
88 |
89 |
90 | counter = Counter
91 |
92 | iters = [list, tuple]
93 | import __builtin__
94 |
95 | if hasattr(__builtin__, 'set'):
96 | iters.append(set)
97 | if hasattr(__builtin__, 'frozenset'):
98 | iters.append(set)
99 | if sys.version_info < (2, 6): # sets module deprecated in 2.6
100 | try:
101 | from sets import Set
102 |
103 | iters.append(Set)
104 | except ImportError:
105 | pass
106 |
107 |
108 | class _hack(tuple): pass
109 |
110 |
111 | iters = _hack(iters)
112 | iters.__doc__ = """
113 | A list of iterable items (like lists, but not strings). Includes whichever
114 | of lists, tuples, sets, and Sets are available in this version of Python.
115 | """
116 |
117 | def group(seq, size):
118 | """
119 | Returns an iterator over a series of lists of length size from iterable.
120 |
121 | >>> list(group([1,2,3,4], 2))
122 | [[1, 2], [3, 4]]
123 | >>> list(group([1,2,3,4,5], 2))
124 | [[1, 2], [3, 4], [5]]
125 | """
126 |
127 | def take(seq, n):
128 | for i in xrange(n):
129 | yield seq.next()
130 |
131 | if not hasattr(seq, 'next'):
132 | seq = iter(seq)
133 | while True:
134 | x = list(take(seq, size))
135 | if x:
136 | yield x
137 | else:
138 | break
139 |
140 |
141 | def uniq(seq, key=None):
142 | """
143 | Removes duplicate elements from a list while preserving the order of the rest.
144 |
145 | >>> uniq([9,0,2,1,0])
146 | [9, 0, 2, 1]
147 |
148 | The value of the optional `key` parameter should be a function that
149 | takes a single argument and returns a key to test the uniqueness.
150 |
151 | >>> uniq(["Foo", "foo", "bar"], key=lambda s: s.lower())
152 | ['Foo', 'bar']
153 | """
154 | key = key or (lambda x: x)
155 | seen = set()
156 | result = []
157 | for v in seq:
158 | k = key(v)
159 | if k in seen:
160 | continue
161 | seen.add(k)
162 | result.append(v)
163 | return result
164 |
165 |
166 | def iterview(x):
167 | """
168 | Takes an iterable `x` and returns an iterator over it
169 | which prints its progress to stderr as it iterates through.
170 | """
171 | WIDTH = 70
172 |
173 | def plainformat(n, lenx):
174 | return '%5.1f%% (%*d/%d)' % ((float(n) / lenx) * 100, len(str(lenx)), n, lenx)
175 |
176 | def bars(size, n, lenx):
177 | val = int((float(n) * size) / lenx + 0.5)
178 | if size - val:
179 | spacing = ">" + (" " * (size - val))[1:]
180 | else:
181 | spacing = ""
182 | return "[%s%s]" % ("=" * val, spacing)
183 |
184 | def eta(elapsed, n, lenx):
185 | if n == 0:
186 | return '--:--:--'
187 | if n == lenx:
188 | secs = int(elapsed)
189 | else:
190 | secs = int((elapsed / n) * (lenx - n))
191 | mins, secs = divmod(secs, 60)
192 | hrs, mins = divmod(mins, 60)
193 |
194 | return '%02d:%02d:%02d' % (hrs, mins, secs)
195 |
196 | def format(starttime, n, lenx):
197 | out = plainformat(n, lenx) + ' '
198 | if n == lenx:
199 | end = ' '
200 | else:
201 | end = ' ETA '
202 | end += eta(time.time() - starttime, n, lenx)
203 | out += bars(WIDTH - len(out) - len(end), n, lenx)
204 | out += end
205 | return out
206 |
207 | starttime = time.time()
208 | lenx = len(x)
209 | for n, y in enumerate(x):
210 | sys.stderr.write('\r' + format(starttime, n, lenx))
211 | yield y
212 | sys.stderr.write('\r' + format(starttime, n + 1, lenx) + '\n')
213 |
214 |
215 | class IterBetter:
216 | """
217 | Returns an object that can be used as an iterator
218 | but can also be used via __getitem__ (although it
219 | cannot go backwards -- that is, you cannot request
220 | `iterbetter[0]` after requesting `iterbetter[1]`).
221 |
222 | >>> import itertools
223 | >>> c = iterbetter(itertools.count())
224 | >>> c[1]
225 | 1
226 | >>> c[5]
227 | 5
228 | >>> c[3]
229 | Traceback (most recent call last):
230 | ...
231 | IndexError: already passed 3
232 |
233 | For boolean test, IterBetter peeps at first value in the itertor without effecting the iteration.
234 |
235 | >>> c = iterbetter(iter(range(5)))
236 | >>> bool(c)
237 | True
238 | >>> list(c)
239 | [0, 1, 2, 3, 4]
240 | >>> c = iterbetter(iter([]))
241 | >>> bool(c)
242 | False
243 | >>> list(c)
244 | []
245 | """
246 |
247 | def __init__(self, iterator):
248 | self.i, self.c = iterator, 0
249 |
250 | def __iter__(self):
251 | if hasattr(self, "_head"):
252 | yield self._head
253 |
254 | while 1:
255 | yield self.i.next()
256 | self.c += 1
257 |
258 | def __getitem__(self, i):
259 | #todo: slices
260 | if i < self.c:
261 | raise IndexError, "already passed " + str(i)
262 | try:
263 | while i > self.c:
264 | self.i.next()
265 | self.c += 1
266 | # now self.c == i
267 | self.c += 1
268 | return self.i.next()
269 | except StopIteration:
270 | raise IndexError, str(i)
271 |
272 | def __nonzero__(self):
273 | if hasattr(self, "__len__"):
274 | return len(self) != 0
275 | elif hasattr(self, "_head"):
276 | return True
277 | else:
278 | try:
279 | self._head = self.i.next()
280 | except StopIteration:
281 | return False
282 | else:
283 | return True
284 |
285 |
286 | iterbetter = IterBetter
287 |
288 |
289 | def safeiter(it, cleanup=None, ignore_errors=True):
290 | """Makes an iterator safe by ignoring the exceptions occured during the iteration.
291 | """
292 |
293 | def next():
294 | while True:
295 | try:
296 | return it.next()
297 | except StopIteration:
298 | raise
299 | except:
300 | traceback.print_exc()
301 |
302 | it = iter(it)
303 | while True:
304 | yield next()
305 |
306 |
307 | def dictreverse(mapping):
308 | """
309 | Returns a new dictionary with keys and values swapped.
310 |
311 | >>> dictreverse({1: 2, 3: 4})
312 | {2: 1, 4: 3}
313 | """
314 | return dict([(value, key) for (key, value) in mapping.iteritems()])
315 |
316 |
317 | def dictfind(dictionary, element):
318 | """
319 | Returns a key whose value in `dictionary` is `element`
320 | or, if none exists, None.
321 |
322 | >>> d = {1:2, 3:4}
323 | >>> dictfind(d, 4)
324 | 3
325 | >>> dictfind(d, 5)
326 | """
327 | for (key, value) in dictionary.iteritems():
328 | if element is value:
329 | return key
330 |
331 |
332 | def dictfindall(dictionary, element):
333 | """
334 | Returns the keys whose values in `dictionary` are `element`
335 | or, if none exists, [].
336 |
337 | >>> d = {1:4, 3:4}
338 | >>> dictfindall(d, 4)
339 | [1, 3]
340 | >>> dictfindall(d, 5)
341 | []
342 | """
343 | res = []
344 | for (key, value) in dictionary.iteritems():
345 | if element is value:
346 | res.append(key)
347 | return res
348 |
349 |
350 | def dictincr(dictionary, element):
351 | """
352 | Increments `element` in `dictionary`,
353 | setting it to one if it doesn't exist.
354 |
355 | >>> d = {1:2, 3:4}
356 | >>> dictincr(d, 1)
357 | 3
358 | >>> d[1]
359 | 3
360 | >>> dictincr(d, 5)
361 | 1
362 | >>> d[5]
363 | 1
364 | """
365 | dictionary.setdefault(element, 0)
366 | dictionary[element] += 1
367 | return dictionary[element]
368 |
369 |
370 | def dictadd(*dicts):
371 | """
372 | Returns a dictionary consisting of the keys in the argument dictionaries.
373 | If they share a key, the value from the last argument is used.
374 |
375 | >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
376 | {1: 0, 2: 1, 3: 1}
377 | """
378 | result = {}
379 | for dct in dicts:
380 | result.update(dct)
381 | return result
382 |
383 |
384 | def requeue(queue, index=-1):
385 | """Returns the element at index after moving it to the beginning of the queue.
386 |
387 | >>> x = [1, 2, 3, 4]
388 | >>> requeue(x)
389 | 4
390 | >>> x
391 | [4, 1, 2, 3]
392 | """
393 | x = queue.pop(index)
394 | queue.insert(0, x)
395 | return x
396 |
397 |
398 | def restack(stack, index=0):
399 | """Returns the element at index after moving it to the top of stack.
400 |
401 | >>> x = [1, 2, 3, 4]
402 | >>> restack(x)
403 | 1
404 | >>> x
405 | [2, 3, 4, 1]
406 | """
407 | x = stack.pop(index)
408 | stack.append(x)
409 | return x
410 |
411 |
412 | def listget(lst, ind, default=None):
413 | """
414 | Returns `lst[ind]` if it exists, `default` otherwise.
415 |
416 | >>> listget(['a'], 0)
417 | 'a'
418 | >>> listget(['a'], 1)
419 | >>> listget(['a'], 1, 'b')
420 | 'b'
421 | """
422 | if len(lst) - 1 < ind:
423 | return default
424 | return lst[ind]
425 |
426 |
427 |
--------------------------------------------------------------------------------
/torngas/utils/numtools.py:
--------------------------------------------------------------------------------
1 |
2 | def intget(integer, default=None):
3 | """
4 | Returns `integer` as an int or `default` if it can't.
5 |
6 | >>> intget('3')
7 | 3
8 | >>> intget('3a')
9 | >>> intget('3a', 0)
10 | 0
11 | """
12 | try:
13 | return int(integer)
14 | except (TypeError, ValueError):
15 | return default
16 |
17 |
18 |
19 | def numify(string):
20 | """
21 | Removes all non-digit characters from `string`.
22 |
23 | >>> numify('800-555-1212')
24 | '8005551212'
25 | >>> numify('800.555.1212')
26 | '8005551212'
27 |
28 | """
29 | return ''.join([c for c in str(string) if c.isdigit()])
30 |
31 |
32 | def denumify(string, pattern):
33 | """
34 | Formats `string` according to `pattern`, where the letter X gets replaced
35 | by characters from `string`.
36 |
37 | >>> denumify("8005551212", "(XXX) XXX-XXXX")
38 | '(800) 555-1212'
39 |
40 | """
41 | out = []
42 | for c in pattern:
43 | if c == "X":
44 | out.append(string[0])
45 | string = string[1:]
46 | else:
47 | out.append(c)
48 | return ''.join(out)
49 |
50 |
51 | def commify(n):
52 | """
53 | Add commas to an integer `n`.
54 |
55 | >>> commify(1)
56 | '1'
57 | >>> commify(123)
58 | '123'
59 | >>> commify(1234)
60 | '1,234'
61 | >>> commify(1234567890)
62 | '1,234,567,890'
63 | >>> commify(123.0)
64 | '123.0'
65 | >>> commify(1234.5)
66 | '1,234.5'
67 | >>> commify(1234.56789)
68 | '1,234.56789'
69 | >>> commify('%.2f' % 1234.5)
70 | '1,234.50'
71 | >>> commify(None)
72 | >>>
73 |
74 | """
75 | if n is None: return None
76 | n = str(n)
77 | if '.' in n:
78 | dollars, cents = n.split('.')
79 | else:
80 | dollars, cents = n, None
81 |
82 | r = []
83 | for i, c in enumerate(str(dollars)[::-1]):
84 | if i and (not (i % 3)):
85 | r.insert(0, ',')
86 | r.insert(0, c)
87 | out = ''.join(r)
88 | if cents:
89 | out += '.' + cents
90 | return out
91 |
92 |
93 | def nthstr(n):
94 | """
95 | Formats an ordinal.
96 | Doesn't handle negative numbers.
97 |
98 | >>> nthstr(1)
99 | '1st'
100 | >>> nthstr(0)
101 | '0th'
102 | >>> [nthstr(x) for x in [2, 3, 4, 5, 10, 11, 12, 13, 14, 15]]
103 | ['2nd', '3rd', '4th', '5th', '10th', '11th', '12th', '13th', '14th', '15th']
104 | >>> [nthstr(x) for x in [91, 92, 93, 94, 99, 100, 101, 102]]
105 | ['91st', '92nd', '93rd', '94th', '99th', '100th', '101st', '102nd']
106 | >>> [nthstr(x) for x in [111, 112, 113, 114, 115]]
107 | ['111th', '112th', '113th', '114th', '115th']
108 |
109 | """
110 |
111 | assert n >= 0
112 | if n % 100 in [11, 12, 13]: return '%sth' % n
113 | return {1: '%sst', 2: '%snd', 3: '%srd'}.get(n % 10, '%sth') % n
114 |
115 |
116 | def cond(predicate, consequence, alternative=None):
117 | """
118 | Function replacement for if-else to use in expressions.
119 |
120 | >>> x = 2
121 | >>> cond(x % 2 == 0, "even", "odd")
122 | 'even'
123 | >>> cond(x % 2 == 0, "even", "odd") + '_row'
124 | 'even_row'
125 | """
126 | if predicate:
127 | return consequence
128 | else:
129 | return alternative
130 |
--------------------------------------------------------------------------------
/torngas/utils/storage.py:
--------------------------------------------------------------------------------
1 | #-*-coding=utf-8-*-
2 | try:
3 | set
4 | except NameError:
5 | from sets import Set as set
6 | from torngas.utils.strtools import safeunicode
7 | import warnings
8 | import sys
9 |
10 | PY3 = (sys.version_info >= (3,))
11 |
12 |
13 | class Storage(dict):
14 | """
15 | from web.py
16 | 对字典进行扩展，使其支持通过 dict.a形式访问以代替dict['a']
17 | """
18 |
19 | def __getattr__(self, key):
20 | try:
21 | return self[key]
22 | except KeyError, k:
23 | raise AttributeError, k
24 |
25 | def __setattr__(self, key, value):
26 | self[key] = value
27 |
28 | def __delattr__(self, key):
29 | try:
30 | del self[key]
31 | except KeyError, k:
32 | raise AttributeError, k
33 |
34 | def __repr__(self):
35 | return ''
36 |
37 |
38 | storage = Storage
39 |
40 |
41 | def storify(mapping, *requireds, **defaults):
42 | """
43 | Creates a `storage` object from dictionary `mapping`, raising `KeyError` if
44 | d doesn't have all of the keys in `requireds` and using the default
45 | values for keys found in `defaults`.
46 |
47 | For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of
48 | `storage({'a':1, 'b':2, 'c':3})`.
49 |
50 | If a `storify` value is a list (e.g. multiple values in a form submission),
51 | `storify` returns the last element of the list, unless the key appears in
52 | `defaults` as a list. Thus:
53 |
54 | >>> storify({'a':[1, 2]}).a
55 | 2
56 | >>> storify({'a':[1, 2]}, a=[]).a
57 | [1, 2]
58 | >>> storify({'a':1}, a=[]).a
59 | [1]
60 | >>> storify({}, a=[]).a
61 | []
62 |
63 | Similarly, if the value has a `value` attribute, `storify will return _its_
64 | value, unless the key appears in `defaults` as a dictionary.
65 |
66 | >>> storify({'a':storage(value=1)}).a
67 | 1
68 | >>> storify({'a':storage(value=1)}, a={}).a
69 |
70 | >>> storify({}, a={}).a
71 | {}
72 |
73 | Optionally, keyword parameter `_unicode` can be passed to convert all values to unicode.
74 |
75 | >>> storify({'x': 'a'}, _unicode=True)
76 |
77 | >>> storify({'x': storage(value='a')}, x={}, _unicode=True)
78 | }>
79 | >>> storify({'x': storage(value='a')}, _unicode=True)
80 |
81 | """
82 | _unicode = defaults.pop('_unicode', False)
83 |
84 | # if _unicode is callable object, use it convert a string to unicode.
85 | to_unicode = safeunicode
86 | if _unicode is not False and hasattr(_unicode, "__call__"):
87 | to_unicode = _unicode
88 |
89 | def unicodify(s):
90 | if _unicode and isinstance(s, str):
91 | return to_unicode(s)
92 | else:
93 | return s
94 |
95 | def getvalue(x):
96 | if hasattr(x, 'file') and hasattr(x, 'value'):
97 | return x.value
98 | elif hasattr(x, 'value'):
99 | return unicodify(x.value)
100 | else:
101 | return unicodify(x)
102 |
103 | stor = Storage()
104 | for key in requireds + tuple(mapping.keys()):
105 | value = mapping[key]
106 | if isinstance(value, list):
107 | if isinstance(defaults.get(key), list):
108 | value = [getvalue(x) for x in value]
109 | else:
110 | value = value[-1]
111 | if not isinstance(defaults.get(key), dict):
112 | value = getvalue(value)
113 | if isinstance(defaults.get(key), list) and not isinstance(value, list):
114 | value = [value]
115 | setattr(stor, key, value)
116 |
117 | for (key, value) in defaults.iteritems():
118 | result = value
119 | if hasattr(stor, key):
120 | result = stor[key]
121 | if value == () and not isinstance(result, tuple):
122 | result = (result,)
123 | setattr(stor, key, result)
124 |
125 | return stor
126 |
127 |
128 | class SortedDict(dict):
129 | """
130 | A dictionary that keeps its keys in the order in which they're inserted.
131 | """
132 |
133 | def __new__(cls, *args, **kwargs):
134 | instance = super(SortedDict, cls).__new__(cls, *args, **kwargs)
135 | instance.keyOrder = []
136 | return instance
137 |
138 | def __init__(self, data=None):
139 | if data is None or isinstance(data, dict):
140 | data = data or []
141 | super(SortedDict, self).__init__(data)
142 | self.keyOrder = list(data) if data else []
143 | else:
144 | super(SortedDict, self).__init__()
145 | super_set = super(SortedDict, self).__setitem__
146 | for key, value in data:
147 | # Take the ordering from first key
148 | if key not in self:
149 | self.keyOrder.append(key)
150 | # But override with last value in data (dict() does this)
151 | super_set(key, value)
152 |
153 | def __deepcopy__(self, memo):
154 | return self.__class__([(key, copy.deepcopy(value, memo))
155 | for key, value in self.items()])
156 |
157 | def __copy__(self):
158 | # The Python's default copy implementation will alter the state
159 | # of self. The reason for this seems complex but is likely related to
160 | # subclassing dict.
161 | return self.copy()
162 |
163 | def __setitem__(self, key, value):
164 | if key not in self:
165 | self.keyOrder.append(key)
166 | super(SortedDict, self).__setitem__(key, value)
167 |
168 | def __delitem__(self, key):
169 | super(SortedDict, self).__delitem__(key)
170 | self.keyOrder.remove(key)
171 |
172 | def __iter__(self):
173 | return iter(self.keyOrder)
174 |
175 | def __reversed__(self):
176 | return reversed(self.keyOrder)
177 |
178 | def pop(self, k, *args):
179 | result = super(SortedDict, self).pop(k, *args)
180 | try:
181 | self.keyOrder.remove(k)
182 | except ValueError:
183 | # Key wasn't in the dictionary in the first place. No problem.
184 | pass
185 | return result
186 |
187 | def popitem(self):
188 | result = super(SortedDict, self).popitem()
189 | self.keyOrder.remove(result[0])
190 | return result
191 |
192 | def _iteritems(self):
193 | for key in self.keyOrder:
194 | yield key, self[key]
195 |
196 | def _iterkeys(self):
197 | for key in self.keyOrder:
198 | yield key
199 |
200 | def _itervalues(self):
201 | for key in self.keyOrder:
202 | yield self[key]
203 |
204 | if PY3:
205 | items = _iteritems
206 | keys = _iterkeys
207 | values = _itervalues
208 | else:
209 | iteritems = _iteritems
210 | iterkeys = _iterkeys
211 | itervalues = _itervalues
212 |
213 | def items(self):
214 | return [(k, self[k]) for k in self.keyOrder]
215 |
216 | def keys(self):
217 | return self.keyOrder[:]
218 |
219 | def values(self):
220 | return [self[k] for k in self.keyOrder]
221 |
222 | def update(self, dict_):
223 | for k, v in dict_.iteritems():
224 | self[k] = v
225 |
226 | def setdefault(self, key, default):
227 | if key not in self:
228 | self.keyOrder.append(key)
229 | return super(SortedDict, self).setdefault(key, default)
230 |
231 | def value_for_index(self, index):
232 | """Returns the value of the item at the given zero-based index."""
233 | # This, and insert() are deprecated because they cannot be implemented
234 | # using collections.OrderedDict (Python 2.7 and up), which we'll
235 | # eventually switch to
236 | warnings.warn(
237 | "SortedDict.value_for_index is deprecated", PendingDeprecationWarning,
238 | stacklevel=2
239 | )
240 | return self[self.keyOrder[index]]
241 |
242 | def insert(self, index, key, value):
243 | """Inserts the key, value pair before the item with the given index."""
244 | warnings.warn(
245 | "SortedDict.insert is deprecated", PendingDeprecationWarning,
246 | stacklevel=2
247 | )
248 | if key in self.keyOrder:
249 | n = self.keyOrder.index(key)
250 | del self.keyOrder[n]
251 | if n < index:
252 | index -= 1
253 | self.keyOrder.insert(index, key)
254 | super(SortedDict, self).__setitem__(key, value)
255 |
256 | def copy(self):
257 | """Returns a copy of this object."""
258 | # This way of initializing the copy means it works for subclasses, too.
259 | return self.__class__(self)
260 |
261 | def __repr__(self):
262 | """
263 | Replaces the normal dict.__repr__ with a version that returns the keys
264 | in their sorted order.
265 | """
266 | return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.iteritems()])
267 |
268 | def clear(self):
269 | super(SortedDict, self).clear()
270 | self.keyOrder = []
271 |
272 |
273 | sorteddict = SortedDict
--------------------------------------------------------------------------------
/torngas/utils/strtools.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | iters = [list, tuple]
3 |
4 | def to36(q):
5 | """
6 | Converts an integer to base 36 (a useful scheme for human-sayable IDs).
7 |
8 | >>> to36(35)
9 | 'z'
10 | >>> to36(119292)
11 | '2k1o'
12 | >>> int(to36(939387374), 36)
13 | 939387374
14 | >>> to36(0)
15 | '0'
16 | >>> to36(-393)
17 | Traceback (most recent call last):
18 | ...
19 | ValueError: must supply a positive integer
20 |
21 | """
22 | if q < 0: raise ValueError, "must supply a positive integer"
23 | letters = "0123456789abcdefghijklmnopqrstuvwxyz"
24 | converted = []
25 | while q != 0:
26 | q, r = divmod(q, 36)
27 | converted.insert(0, letters[r])
28 | return "".join(converted) or '0'
29 |
30 |
31 | def _strips(direction, text, remove):
32 | if isinstance(remove, iters):
33 | for subr in remove:
34 | text = _strips(direction, text, subr)
35 | return text
36 |
37 | if direction == 'l':
38 | if text.startswith(remove):
39 | return text[len(remove):]
40 | elif direction == 'r':
41 | if text.endswith(remove):
42 | return text[:-len(remove)]
43 | else:
44 | raise ValueError, "Direction needs to be r or l."
45 | return text
46 |
47 |
48 | def rstrips(text, remove):
49 | """
50 | removes the string `remove` from the right of `text`
51 |
52 | >>> rstrips("foobar", "bar")
53 | 'foo'
54 |
55 | """
56 | return _strips('r', text, remove)
57 |
58 |
59 | def lstrips(text, remove):
60 | """
61 | removes the string `remove` from the left of `text`
62 |
63 | >>> lstrips("foobar", "foo")
64 | 'bar'
65 | >>> lstrips('http://foo.org/', ['http://', 'https://'])
66 | 'foo.org/'
67 | >>> lstrips('FOOBARBAZ', ['FOO', 'BAR'])
68 | 'BAZ'
69 | >>> lstrips('FOOBARBAZ', ['BAR', 'FOO'])
70 | 'BARBAZ'
71 |
72 | """
73 | return _strips('l', text, remove)
74 |
75 |
76 | def strips(text, remove):
77 | """
78 | removes the string `remove` from the both sides of `text`
79 |
80 | >>> strips("foobarfoo", "foo")
81 | 'bar'
82 |
83 | """
84 | return rstrips(lstrips(text, remove), remove)
85 |
86 |
87 | def safeunicode(obj, encoding='utf-8'):
88 | r"""
89 | Converts any given object to unicode string.
90 |
91 | >>> safeunicode('hello')
92 | u'hello'
93 | >>> safeunicode(2)
94 | u'2'
95 | >>> safeunicode('\xe1\x88\xb4')
96 | u'\u1234'
97 | """
98 | t = type(obj)
99 | if t is unicode:
100 | return obj
101 | elif t is str:
102 | return obj.decode(encoding)
103 | elif t in [int, float, bool]:
104 | return unicode(obj)
105 | elif hasattr(obj, '__unicode__') or isinstance(obj, unicode):
106 | return unicode(obj)
107 | else:
108 | return str(obj).decode(encoding)
109 |
110 |
111 | def safestr(obj, encoding='utf-8'):
112 | r"""
113 | Converts any given object to utf-8 encoded string.
114 |
115 | >>> safestr('hello')
116 | 'hello'
117 | >>> safestr(u'\u1234')
118 | '\xe1\x88\xb4'
119 | >>> safestr(2)
120 | '2'
121 | """
122 | if isinstance(obj, unicode):
123 | return obj.encode(encoding)
124 | elif isinstance(obj, str):
125 | return obj
126 | elif hasattr(obj, 'next'): # iterator
127 | return itertools.imap(safestr, obj)
128 | else:
129 | return str(obj)
130 |
131 | # for backward-compatibility
132 | utf8 = safestr
133 |
134 | import re
135 | re_compile=re.compile
136 |
137 | class _re_subm_proxy:
138 | def __init__(self):
139 | self.match = None
140 |
141 | def __call__(self, match):
142 | self.match = match
143 | return ''
144 |
145 |
146 | def re_subm(pat, repl, string):
147 | """
148 | Like re.sub, but returns the replacement _and_ the match object.
149 |
150 | >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball')
151 | >>> t
152 | 'foooooolish'
153 | >>> m.groups()
154 | ('oooooo',)
155 | """
156 | compiled_pat = re_compile(pat)
157 | proxy = _re_subm_proxy()
158 | compiled_pat.sub(proxy.__call__, string)
159 | return compiled_pat.sub(repl, string), proxy.match
160 |
161 | r_url = re_compile('(? 0:
82 | self.can_read.release()
83 | t -= 1
84 |
85 | @contextlib.contextmanager
86 | def writer(self):
87 | self.writer_enters()
88 | try:
89 | yield
90 | finally:
91 | self.writer_leaves()
92 |
--------------------------------------------------------------------------------
/torngas/utils/timetools.py:
--------------------------------------------------------------------------------
1 | import re, sys, time, threading, itertools, traceback, os
2 | try:
3 | import datetime
4 | except ImportError:
5 | pass
6 |
7 |
8 | class TimeoutError(Exception): pass
9 |
10 |
11 | def timelimit(timeout):
12 | """
13 | A decorator to limit a function to `timeout` seconds, raising `TimeoutError`
14 | if it takes longer.
15 |
16 | >>> import time
17 | >>> def meaningoflife():
18 | ... time.sleep(.2)
19 | ... return 42
20 | >>>
21 | >>> timelimit(.1)(meaningoflife)()
22 | Traceback (most recent call last):
23 | ...
24 | TimeoutError: took too long
25 | >>> timelimit(1)(meaningoflife)()
26 | 42
27 |
28 | _Caveat:_ The function isn't stopped after `timeout` seconds but continues
29 | executing in a separate thread. (There seems to be no way to kill a thread.)
30 |
31 | inspired by
32 | """
33 |
34 | def _1(function):
35 | def _2(*args, **kw):
36 | class Dispatch(threading.Thread):
37 | def __init__(self):
38 | threading.Thread.__init__(self)
39 | self.result = None
40 | self.error = None
41 |
42 | self.setDaemon(True)
43 | self.start()
44 |
45 | def run(self):
46 | try:
47 | self.result = function(*args, **kw)
48 | except:
49 | self.error = sys.exc_info()
50 |
51 | c = Dispatch()
52 | c.join(timeout)
53 | if c.isAlive():
54 | raise TimeoutError, 'took too long'
55 | if c.error:
56 | raise c.error[0], c.error[1]
57 | return c.result
58 |
59 | return _2
60 |
61 | return _1
62 |
63 |
64 | def datestr(then, now=None):
65 | """
66 | Converts a (UTC) datetime object to a nice string representation.
67 |
68 | >>> from datetime import datetime, timedelta
69 | >>> d = datetime(1970, 5, 1)
70 | >>> datestr(d, now=d)
71 | '0 microseconds ago'
72 | >>> for t, v in {
73 | ... timedelta(microseconds=1): '1 microsecond ago',
74 | ... timedelta(microseconds=2): '2 microseconds ago',
75 | ... -timedelta(microseconds=1): '1 microsecond from now',
76 | ... -timedelta(microseconds=2): '2 microseconds from now',
77 | ... timedelta(microseconds=2000): '2 milliseconds ago',
78 | ... timedelta(seconds=2): '2 seconds ago',
79 | ... timedelta(seconds=2*60): '2 minutes ago',
80 | ... timedelta(seconds=2*60*60): '2 hours ago',
81 | ... timedelta(days=2): '2 days ago',
82 | ... }.iteritems():
83 | ... assert datestr(d, now=d+t) == v
84 | >>> datestr(datetime(1970, 1, 1), now=d)
85 | 'January 1'
86 | >>> datestr(datetime(1969, 1, 1), now=d)
87 | 'January 1, 1969'
88 | >>> datestr(datetime(1970, 6, 1), now=d)
89 | 'June 1, 1970'
90 | >>> datestr(None)
91 | ''
92 | """
93 |
94 | def agohence(n, what, divisor=None):
95 | if divisor: n = n // divisor
96 |
97 | out = str(abs(n)) + ' ' + what # '2 day'
98 | if abs(n) != 1: out += 's' # '2 days'
99 | out += ' ' # '2 days '
100 | if n < 0:
101 | out += 'from now'
102 | else:
103 | out += 'ago'
104 | return out # '2 days ago'
105 |
106 | oneday = 24 * 60 * 60
107 |
108 | if not then: return ""
109 | if not now: now = datetime.datetime.utcnow()
110 | if type(now).__name__ == "DateTime":
111 | now = datetime.datetime.fromtimestamp(now)
112 | if type(then).__name__ == "DateTime":
113 | then = datetime.datetime.fromtimestamp(then)
114 | elif type(then).__name__ == "date":
115 | then = datetime.datetime(then.year, then.month, then.day)
116 |
117 | delta = now - then
118 | deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06)
119 | deltadays = abs(deltaseconds) // oneday
120 | if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor
121 |
122 | if deltadays:
123 | if abs(deltadays) < 4:
124 | return agohence(deltadays, 'day')
125 |
126 | try:
127 | out = then.strftime('%B %e') # e.g. 'June 3'
128 | except ValueError:
129 | # %e doesn't work on Windows.
130 | out = then.strftime('%B %d') # e.g. 'June 03'
131 |
132 | if then.year != now.year or deltadays < 0:
133 | out += ', %s' % then.year
134 | return out
135 |
136 | if int(deltaseconds):
137 | if abs(deltaseconds) > (60 * 60):
138 | return agohence(deltaseconds, 'hour', 60 * 60)
139 | elif abs(deltaseconds) > 60:
140 | return agohence(deltaseconds, 'minute', 60)
141 | else:
142 | return agohence(deltaseconds, 'second')
143 |
144 | deltamicroseconds = delta.microseconds
145 | if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity
146 | if abs(deltamicroseconds) > 1000:
147 | return agohence(deltamicroseconds, 'millisecond', 1000)
148 |
149 | return agohence(deltamicroseconds, 'microsecond')
150 |
151 | from numtools import denumify
152 | def dateify(datestring):
153 | """
154 | Formats a numified `datestring` properly.
155 | """
156 | return denumify(datestring, "XXXX-XX-XX XX:XX:XX")
157 |
158 |
--------------------------------------------------------------------------------