├── .gitignore ├── README.MD └── multithreadtornado ├── bin └── serv.py ├── biz └── hello.py ├── etc ├── dev_log.conf ├── includes_dev.json └── webapp_dev.json ├── lib ├── configer.py ├── path.py ├── route.py ├── tools.py └── util │ ├── __init__.py │ └── cache.py └── var ├── app.log └── http.log /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .pyc 3 | .log 4 | 5 | ### Example user template template 6 | ### Example user template 7 | 8 | # IntelliJ project files 9 | .idea 10 | *.iml 11 | out 12 | gen 13 | # Created by .ignore support plugin (hsz.mobi) 14 | 15 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | ICEworld 2 | ========= 3 | A multithread implementation based on Tornado 4 | 5 | ###### a fast restful dev framework based on tornado. 6 | 7 | ###### Take advantage of a fashionable C10K framework Tornado(ioloop). 8 | 9 | ###### ioloop throws requests to threadpool, threadpool notifies ioloop when it is done. 10 | 11 | ###### Before deployment, `tornado` and `futures` are required. 12 | ```bash 13 | sudo easy_install tornado 14 | sudo easy_install futures 15 | sudo easy_install lxml 16 | ``` 17 | #### Quick Start 18 | 1.在“biz”目录中创建一个py文件，文件名任意但最好不要跟第三方库冲突 19 | 20 | 2.使用 "Router.route" 装饰器注册函数到路由表中，仿造示例即可 21 | 22 | 3.到“bin”目录下，使用命令"python serv.py" 启动工程，用浏览器访问步骤二中注册的路径可看到效果 23 | 24 | 25 | 基于tornado改装的多线程业务处理模型框架，自带跨域请求，json/xml参数解析，缓存和路由优化。适合多人合作的service系统后台搭建！ 26 | >疑问请联系: 27 | 28 | 29 | ***no rights reserved, enjoy!!*** 30 | -------------------------------------------------------------------------------- /multithreadtornado/bin/serv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.6 2 | #coding=utf-8 3 | #app path build-ups 4 | # author Rowland 5 | # edit 2014-03-19 14:17:30 6 | 7 | import os 8 | import sys 9 | import json 10 | import getopt 11 | reload(sys) 12 | sys.setdefaultencoding('utf-8') 13 | sys.path.append(os.path.join( 14 | os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 15 | 'lib')) 16 | 17 | import tornado.ioloop 18 | import tornado.web 19 | from tornado.log import access_log 20 | 21 | from tools import Log, XMLUtils 22 | from lxml import etree 23 | 24 | import route 25 | from configer import * 26 | 27 | class Xroute(tornado.web.RequestHandler): 28 | '''通用预处理''' 29 | def prepare(self): 30 | # 获得正确的客户端ip 31 | ip = self.request.headers.get("X-Real-Ip", self.request.remote_ip) 32 | ip = self.request.headers.get("X-Forwarded-For", ip) 33 | ip = ip.split(',')[0].strip() 34 | self.request.remote_ip = ip 35 | self.json_args = {} #initialize args 36 | # 允许跨域请求 37 | req_origin = self.request.headers.get("Origin") 38 | if req_origin: 39 | self.set_header("Access-Control-Allow-Origin", req_origin) 40 | self.set_header("Access-Control-Allow-Credentials", "true") 41 | self.set_header("Allow", "GET, HEAD, POST") 42 | if self.request.method == "OPTIONS": 43 | self.set_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") 44 | self.set_header("Access-Control-Allow-Headers", "Accept, Cache-Control, Content-Type") 45 | self.finish() 46 | return 47 | else: 48 | self.set_header("Cache-Control", "no-cache") 49 | # json格式请求 50 | if self.request.headers.get('Content-Type', '').find("application/json") >= 0: 51 | try: 52 | self.json_args = json.loads(self.request.body) 53 | except Exception as ex: 54 | self.send_error(400) 55 | # xml格式请求 56 | elif self.request.headers.get('Content-Type', '').find('application/xml') >= 0: 57 | try: 58 | xu = XMLUtils() 59 | xml = self.request.body 60 | root = etree.fromstring(xml) 61 | root_dict = xu.parseElement(root) 62 | self.json_args = root_dict 63 | except Exception, e: 64 | self.send_error(501) 65 | # 普通参数请求 66 | elif self.request.arguments: 67 | self.json_args = dict((k, v[-1]) for k, v in self.request.arguments.items()) 68 | 69 | @tornado.web.asynchronous 70 | def get(self, path): 71 | route.Router.get(path, self) 72 | 73 | @tornado.web.asynchronous 74 | def post(self, path): 75 | route.Router.post(path, self) 76 | 77 | @tornado.web.asynchronous 78 | def put(self, path): 79 | route.Router.put(path, self) 80 | 81 | @tornado.web.asynchronous 82 | def delete(self, path): 83 | route.Router.delete(path, self) 84 | 85 | @tornado.web.asynchronous 86 | def options(self, path): 87 | route.Router.options(path, self) 88 | 89 | 90 | def get_application(): 91 | """获取应用程序对象 92 | """ 93 | return tornado.web.Application([(r"^(/[^\.|]*)(?!\.\w+)$", Xroute)], 94 | log_function=log_request) 95 | 96 | 97 | def log_request(handler): 98 | """http日志函数 99 | """ 100 | if handler.get_status() < 400: 101 | log_method = access_log.info 102 | elif handler.get_status() < 500: 103 | log_method = access_log.warning 104 | else: 105 | log_method = access_log.error 106 | req = handler.request 107 | log_method('"%s %s" %d %s %.6f', 108 | req.method, req.uri, handler.get_status(), 109 | req.remote_ip, req.request_time() ) 110 | 111 | def init_application(conf_file): 112 | """初始化应用程序 113 | """ 114 | 115 | cpff = ConfigParserFromFile() 116 | conf_file | E(cpff.parseall) | E(conf.setup) 117 | 118 | 119 | if __name__=="__main__": 120 | # init 121 | os.chdir(os.path.join(os.path.dirname(__file__), '..')) 122 | port = 8888 123 | includes = None 124 | opts, argvs = getopt.getopt(sys.argv[1:], "c:p:h") 125 | for op, value in opts: 126 | if op == '-c': 127 | includes = value 128 | path._ETC_PATH = os.path.dirname(os.path.abspath(value)) 129 | elif op == '-p': 130 | port = int(value) 131 | elif op == '-h': 132 | print u'''使用参数启动: 133 | usage: [-p|-c] 134 | -p [prot] ******启动端口,默认端口:%d 135 | -c ******加载配置文件 136 | ''' % port 137 | sys.exit(0) 138 | if not includes: 139 | includes = os.path.join(path._ETC_PATH, 'includes_dev.json') 140 | print "no configuration found!,will use [%s] instead" % includes 141 | # main 142 | init_application(includes) 143 | application = get_application() 144 | application.listen(port) 145 | logger = Log().getLog() 146 | logger.info("starting..., listen [%d], configurated by (%s)", port, includes) 147 | tornado.ioloop.IOLoop.instance().start() 148 | 149 | 150 | -------------------------------------------------------------------------------- /multithreadtornado/biz/hello.py: -------------------------------------------------------------------------------- 1 | #encoding=utf-8 2 | 3 | import time, random 4 | from route import Router 5 | from util.cache import Cache 6 | 7 | 8 | class HelloTest(object): 9 | ''' 10 | 正则捕获的参数会自动注入到方法的参数列表中(从第二个开始)如下面例子中who 11 | 第一个参数为tornado.web.handler对象（request） 12 | ''' 13 | @Router.route(url=r"/", method=Router._GET) 14 | def home(self, req): 15 | return "hello world" 16 | 17 | @Router.route(url=r"/hello/([a-z]+)", method=Router._GET | Router._POST) 18 | def test(self, req, who): 19 | #http://localhost:8888/hello/billy 20 | return "Hi," + who 21 | 22 | @Router.route(url=r"/greetings/([a-z]+)", method=Router._GET) 23 | def test2(self, req, who): 24 | #http://localhost:8888/greetings/rowland 25 | raise Exception("error") 26 | 27 | @Router.route(url=r"/book/([a-z]+)/(\d+)", method=Router._GET | Router._POST) 28 | def test3(self, req, categories, bookid): 29 | #http://localhost:8888/book/medicine/49875 30 | return "You are looking for a " + categories + " book\n" + "book No. " + bookid 31 | 32 | @Router.route(url=r"/json", method=Router._GET | Router._POST) 33 | def test4(self, req): 34 | #http://localhost:8888/json 35 | #print req.request.body 36 | who = req.json_args.get("who", "default") 37 | age = req.json_args.get("age", 0) 38 | person = {} 39 | person['who'] = who 40 | person['age'] = int(age) 41 | return person 42 | 43 | @Router.route(url=r"/pi/is", method=Router._GET | Router._POST) 44 | def test5(self, req): 45 | return self.pi() 46 | 47 | @Cache.lru(ttl=20) 48 | def pi(self): 49 | n = 3000000 50 | return sum((1 if random.random() ** 2 + random.random() ** 2 < 1 else 0 for i in xrange(n))) * 4.0 / n 51 | -------------------------------------------------------------------------------- /multithreadtornado/etc/dev_log.conf: -------------------------------------------------------------------------------- 1 | ############################################################## 2 | [loggers] 3 | keys=root,tornado.access,simple 4 | 5 | [logger_root] 6 | level=DEBUG 7 | handlers=console_handler 8 | qualname=root 9 | 10 | [logger_simple] 11 | level=DEBUG 12 | handlers=simple_handler,console_handler 13 | qualname=simple 14 | propagate=0 15 | 16 | [logger_tornado.access] 17 | level=DEBUG 18 | handlers=http_handler,console_handler 19 | qualname=tornado.access 20 | propagate=0 21 | 22 | ############################################################## 23 | [handlers] 24 | keys=console_handler,simple_handler,http_handler 25 | 26 | [handler_console_handler] 27 | class=StreamHandler 28 | level=DEBUG 29 | formatter=simple_formatter 30 | args=(sys.stdout,) 31 | 32 | [handler_simple_handler] 33 | class=handlers.TimedRotatingFileHandler 34 | level=DEBUG 35 | formatter=simple_formatter 36 | args=('var/app.log', 'midnight') 37 | 38 | [handler_http_handler] 39 | class=handlers.TimedRotatingFileHandler 40 | level=DEBUG 41 | formatter=http_formatter 42 | args=('var/http.log', 'midnight') 43 | 44 | ############################################################### 45 | [formatters] 46 | keys=simple_formatter,http_formatter 47 | 48 | [formatter_simple_formatter] 49 | format=[%(asctime)s][%(threadName)s][%(filename)s:%(lineno)s] %(levelname)s: %(message)s 50 | datefmt=%y-%m-%d %H:%M:%S 51 | 52 | [formatter_http_formatter] 53 | format=[%(asctime)s] %(message)s 54 | datefmt=%y-%m-%d %H:%M:%S 55 | -------------------------------------------------------------------------------- /multithreadtornado/etc/includes_dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "$includes": [ 3 | "webapp_dev.json" 4 | ], 5 | "logging": { 6 | "config_file": "dev_log.conf", 7 | "default_logger": "simple" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /multithreadtornado/etc/webapp_dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "redis":{ 3 | "standalone":{ 4 | "host":"localhost", 5 | "port":6379 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /multithreadtornado/lib/configer.py: -------------------------------------------------------------------------------- 1 | #coding: utf-8 2 | 3 | import os, sys, re 4 | import json 5 | from abc import * 6 | import path 7 | import traceback 8 | 9 | 10 | class ConfigParser(object): 11 | ''' 12 | This abstract class provides a strategy of how to get those configurations 13 | through a file or remote config ? 14 | ''' 15 | @abstractmethod 16 | def parseall(self, *args): 17 | pass 18 | 19 | 20 | class E(object): 21 | def __init__(self, func): 22 | self.func = func 23 | 24 | def __ror__(self, inputs): 25 | return self.func(inputs) 26 | 27 | 28 | class Configer(object): 29 | ''' 30 | This class will hold configurations and registered setups(functions) 31 | It can determine when to setup them 32 | ''' 33 | config = {} 34 | setups = [] 35 | 36 | def register_my_setup(self, **deco): 37 | def foo(func): 38 | location = deco.get('look') 39 | level = deco.get('level', 99999) 40 | self.setups.append({ 41 | 'func': func, 42 | 'location': location, 43 | 'level': level 44 | }) 45 | return func 46 | 47 | return foo 48 | 49 | def setup(self, own_cfg, onlevel=0): 50 | ''' 51 | Call all(or specific level) setup functions which registered via using 52 | "Configer.register_my_setup" decorator. 53 | If "onlevel" has been set, only the matched setup fucntions will be 54 | loaded(or hot reloaded). 55 | BE CAREFUL! The registed setup function shall apply reload logic in case 56 | of a runtime-hot-reloaded callback hit. 57 | ''' 58 | self.setups.sort(key=lambda x: x['level']) 59 | self.config.update(own_cfg) 60 | 61 | for s in Configer.setups: 62 | func = s['func'] 63 | location = s['location'] 64 | try: 65 | if location: 66 | func(self.config[location]) 67 | else: 68 | func() 69 | except Exception: 70 | traceback.print_exc() 71 | sys.exit(1) 72 | 73 | 74 | class ConfigParserFromFile(ConfigParser): 75 | ''' 76 | via Config Files 77 | ''' 78 | def parseall(self, fullpath): 79 | etc = path._ETC_PATH 80 | cfg = {} 81 | with open(fullpath, 'r') as f: 82 | raw = f.read() 83 | #去掉多行注释 84 | raw_escape_comment = re.sub(r'[\s\t\n]+/\*[\s\S]+?\*/', '', raw) 85 | cfg = json.loads(raw_escape_comment) 86 | if cfg.get('$includes'): 87 | for include in cfg['$includes']: 88 | icfg = self.parseall(os.path.join(etc, include)) 89 | cfg.update(icfg) 90 | return cfg 91 | 92 | conf = Configer() 93 | -------------------------------------------------------------------------------- /multithreadtornado/lib/path.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #coding=utf-8 3 | #author Rowland 4 | #edit 2014-03-19 14:15:54 5 | import os 6 | import sys 7 | 8 | 9 | _HOME_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 10 | 11 | _BIN_PATH = os.path.join(_HOME_PATH, 'bin') 12 | _BIZ_PATH = os.path.join(_HOME_PATH, 'biz') 13 | _ETC_PATH = os.path.join(_HOME_PATH, 'etc') 14 | _LIB_PATH = os.path.join(_HOME_PATH, 'lib') 15 | 16 | _path = () 17 | _path += _BIN_PATH, _BIZ_PATH, _ETC_PATH 18 | map(sys.path.append, _path) 19 | 20 | -------------------------------------------------------------------------------- /multithreadtornado/lib/route.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #coding=utf-8 3 | # author Rowland 4 | # edit 2014-03-19 14:16:32 5 | # 虚拟路由总管，拦截请求转发请求到业务线程池 6 | # edit 2014-12-15 18:02:09 7 | # 修改映射表数据结构为单向链表（方便进行优化算法），增加URL冲突检测，优化性能精简代码，高负载下随机拒绝服务 8 | # edit 2015年4月15日14:45:58 9 | # 修改setup规则为装饰器自动注册，方便后续提供热加载策略 10 | 11 | import os 12 | import inspect 13 | import sys 14 | import re 15 | import random 16 | import json 17 | from itertools import groupby 18 | 19 | import tornado.ioloop 20 | import tornado.web 21 | from concurrent import futures 22 | 23 | import path, tools 24 | from configer import conf 25 | 26 | MAX_WORKERS = 16 27 | 28 | executor = futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 29 | 30 | 31 | @conf.register_my_setup(level=2) 32 | def set_up(): 33 | ''' erase all nodes 34 | this function maybe called for hot deployment 35 | ''' 36 | Router.mapper = [] 37 | Router.mapper_sentry = {} 38 | Router.last_sentry = {} 39 | global logger 40 | logger = tools.Log().getLog() 41 | #automic scan dirs and initial all node 42 | files_list = os.listdir(path._BIZ_PATH) 43 | files_list = set([x[:x.rfind(".")] for x in files_list if x.endswith(".py")]) 44 | map(__import__, files_list) 45 | Router.pre_check() 46 | 47 | 48 | def _call_wrap(call, params): 49 | handler = params[0] 50 | try: 51 | #logger.info('request: %s %s', handler.request.path, handler.json_args or {}) 52 | ret = call(*params) 53 | # stringify result 54 | if isinstance(ret, dict): 55 | ret = json.dumps(ret) 56 | else: 57 | ret = str(ret) 58 | tornado.ioloop.IOLoop.instance().add_callback(lambda: params[0].finish(ret)) 59 | except Exception, ex: 60 | logger.exception(ex) 61 | tornado.ioloop.IOLoop.instance().add_callback(lambda: params[0].send_error()) 62 | 63 | 64 | class Router(object): 65 | '''dispather and decortor''' 66 | _GET = 0x001 67 | _POST = 0x002 68 | _PUT = 0x004 69 | _DELETE = 0x008 70 | _OPTIONS = 0x010 71 | mapper = [] 72 | mapper_sentry = {} 73 | last_sentry = {} 74 | 75 | @classmethod 76 | def check_redefined_node(cls, sentry, url_exp, method): 77 | if not sentry: 78 | return None 79 | if sentry['eUrl'] == url_exp and sentry['method'] & method: 80 | return sentry 81 | else: 82 | return Router.check_redefined_node(sentry['next'], url_exp, method) 83 | 84 | @classmethod 85 | def lookup_suitable_node(cls, prev, sentry, url, method, assert_wrong_method=False): 86 | if not sentry: 87 | if assert_wrong_method: 88 | raise tornado.web.HTTPError(405) 89 | raise tornado.web.HTTPError(404) 90 | matcher = sentry['eUrl'].match(url) 91 | m = sentry['method'] 92 | wrong = assert_wrong_method 93 | if matcher: 94 | if m & method: #hit! 95 | if prev: 96 | prev['next'] = sentry['next'] 97 | sentry['next'] = Router.mapper_sentry 98 | Router.mapper_sentry = sentry 99 | return sentry, matcher 100 | else: 101 | wrong = True 102 | return Router.lookup_suitable_node(sentry, sentry['next'], url, method, wrong) 103 | 104 | @classmethod 105 | def pre_check(cls): 106 | check_mapper_list = filter(lambda (x, y): len(y) > 1, [ 107 | (key, list(items)) for key, items in 108 | groupby(Router.mapper, 109 | lambda x: x) 110 | ] 111 | ) 112 | if check_mapper_list: 113 | for check_mapper in check_mapper_list: 114 | logger.fatal('Definition conflict : FUNCTION[%s]', check_mapper[0]) 115 | sys.exit(1) 116 | 117 | @classmethod 118 | def route(cls, **deco): 119 | def foo(func): 120 | url = deco.get('url') or '/' 121 | eUrl = re.compile('^' + url + '$', re.IGNORECASE) 122 | method = deco.get('method') or Router._GET 123 | if Router.check_redefined_node(Router.mapper_sentry, eUrl, method): 124 | logger.fatal('Definition conflict : URL[%s]', url) 125 | sys.exit(1) 126 | else: 127 | mapper_node = { 128 | 'eUrl': eUrl, 129 | 'method': method, 130 | 'callName': func.__name__, 131 | 'className': inspect.stack()[1][3], 132 | 'moduleName': func.__module__, 133 | 'next': {}, 134 | } 135 | Router.mapper.append( 136 | '.'.join([mapper_node['moduleName'], mapper_node['className'], mapper_node['callName']])) 137 | # Yes, I used linked list here 138 | # Any better way to contain urls? 139 | # Disadvantage: have to visit the urls list from head to end to 140 | # determind 404 141 | if Router.mapper_sentry: 142 | Router.last_sentry['next'] = mapper_node 143 | Router.last_sentry = Router.last_sentry['next'] 144 | else: 145 | Router.mapper_sentry = mapper_node 146 | Router.last_sentry = Router.mapper_sentry 147 | return func 148 | 149 | return foo 150 | 151 | @classmethod 152 | def get(cls, path, reqhandler): 153 | Router.emit(path, reqhandler, Router._GET) 154 | 155 | @classmethod 156 | def post(cls, path, reqhandler): 157 | Router.emit(path, reqhandler, Router._POST) 158 | 159 | @classmethod 160 | def put(cls, path, reqhandler): 161 | Router.emit(path, reqhandler, Router._PUT) 162 | 163 | @classmethod 164 | def delete(cls, path, reqhandler): 165 | Router.emit(path, reqhandler, Router._DELETE) 166 | 167 | @classmethod 168 | def options(cls, path, reqhandler): 169 | Router.emit(path, reqhandler, Router._OPTIONS) 170 | 171 | @classmethod 172 | def verify_passport(cls): 173 | capacity = 0 if len(executor._threads) == 0 else executor._work_queue.qsize() / float( 174 | len(executor._threads)) 175 | if 2 > capacity >= 1.0: 176 | #随机拒绝请求 177 | return False if (random.random() + 1) > capacity else True 178 | elif capacity >= 2: 179 | return False 180 | else: 181 | return True 182 | 183 | @classmethod 184 | def emit(cls, path, reqhandler, method_flag): 185 | #logger.info('request coming![%s][%s]', path, method_flag) 186 | if not Router.verify_passport(): 187 | logger.warn("server is under high pressure ,[free thread:%d] [queue size:%d] [request refused %s]", 188 | len(executor._threads), 189 | executor._work_queue.qsize(), 190 | path) 191 | raise tornado.web.HTTPError(502) 192 | return 193 | mapper_node, m = Router.lookup_suitable_node(None, Router.mapper_sentry, path, method_flag) 194 | if mapper_node and m: 195 | params = (reqhandler,) 196 | for items in m.groups(): 197 | params += (items,) 198 | callName = mapper_node['callName'] 199 | className = mapper_node['className'] 200 | moduleName = mapper_node['moduleName'] 201 | module = __import__(moduleName) 202 | clazz = getattr(module, className) 203 | try: 204 | obj = clazz() 205 | except Exception, e: 206 | logger.exception("error occured when creating instance of %s" % className) 207 | raise tornado.web.HTTPError(500) 208 | call = getattr(obj, callName) 209 | executor.submit(_call_wrap, call, params) 210 | 211 | -------------------------------------------------------------------------------- /multithreadtornado/lib/tools.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | # author Rowland 3 | # edit 2014-03-19 14:16:46 4 | import os 5 | import logging 6 | import logging.config 7 | from configer import conf 8 | import path 9 | 10 | 11 | @conf.register_my_setup(look='logging', level=1) 12 | def set_up(cfg): 13 | log_path = os.path.join(path._ETC_PATH, cfg['config_file']) 14 | logging.config.fileConfig(log_path) 15 | Log.logger = logging.getLogger(cfg['default_logger']) 16 | 17 | 18 | class Log(): 19 | logger = None 20 | 21 | def getLog(self): 22 | return Log.logger 23 | 24 | 25 | class XMLUtils(object): 26 | 27 | def parseElement(self, e): 28 | ret = {} 29 | if e.text: 30 | ret['text'] = e.text 31 | if e.attrib: 32 | ret['attr'] = e.attrib 33 | for i in e.iterchildren(): 34 | ret.update({i.tag: self.parseElement(i)}) 35 | return ret 36 | -------------------------------------------------------------------------------- /multithreadtornado/lib/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nikoloss/iceworld/ab59f0686bdc14c26951f71acdb43cbeb276ae6a/multithreadtornado/lib/util/__init__.py -------------------------------------------------------------------------------- /multithreadtornado/lib/util/cache.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #coding=utf-8 3 | # author Rowland 4 | import threading 5 | import time 6 | 7 | CACHE_SIZE = 999 8 | 9 | 10 | class Cache(object): 11 | ''' 12 | lru缓存采用了lru算法，数据结构为单向链表，命中的节点会被直接提升为头结点 13 | 受python递归栈约束，故将缓存容量设置为999，要使用更大值需设置递归深度值 14 | ''' 15 | _lock = threading.Lock() 16 | _sentry = {} 17 | _index = [] 18 | _count = 0 19 | 20 | @classmethod 21 | def fifo(cls, **kw): 22 | raise Exception('not implemented') 23 | 24 | @classmethod 25 | def lookup_node(cls, prev, sentry, key): 26 | if not sentry or not key: 27 | return None 28 | if sentry['key'] == key: 29 | with Cache._lock: 30 | if prev: 31 | prev['next'] = sentry['next'] 32 | sentry['next'] = Cache._sentry 33 | Cache._sentry = sentry 34 | return sentry 35 | return Cache.lookup_node(sentry, sentry['next'], key) 36 | 37 | @classmethod 38 | def set_node(cls, node): 39 | if not node: 40 | return 41 | with Cache._lock: 42 | node['next'] = Cache._sentry 43 | Cache._sentry = node 44 | 45 | @classmethod 46 | def lru(cls, **kw): 47 | ttl = kw.get('ttl', 10) 48 | 49 | def deco_func(func): 50 | def deco_args(*argv, **kwargv): 51 | #TODO 优化方法+参数的签名，TTL过期时间判定 52 | argvs = map(str, argv) 53 | if argv: 54 | try: 55 | if getattr(argv[0], func.__name__): 56 | argvs = map(str, argv[1:]) 57 | except: 58 | pass 59 | kv = str(kwargv) 60 | key = func.__module__ + '|' + argv[0].__class__.__name__ + '|' + func.__name__ + '|' + ''.join( 61 | argvs) + '|' + kv 62 | #print key 63 | v_node = Cache.lookup_node(None, Cache._sentry, key) 64 | if not v_node: 65 | #miss 66 | v = func(*argv, **kwargv) 67 | v_node = { 68 | 'key': key, 69 | 'value': v, 70 | 'timestamp': time.time() 71 | } 72 | Cache.set_node(v_node) 73 | return v_node['value'] 74 | 75 | return deco_args 76 | 77 | return deco_func 78 | -------------------------------------------------------------------------------- /multithreadtornado/var/app.log: -------------------------------------------------------------------------------- 1 | <<<<<<< HEAD 2 | [16-07-03 18:20:42][MainThread][serv.py:145] INFO: starting..., listen [8888], configurated by (/Users/luoran/work/pyspace/iceworld/multithreadtornado/etc/includes_dev.json) 3 | [16-07-03 18:21:32][MainThread][serv.py:145] INFO: starting..., listen [8888], configurated by (/Users/luoran/work/pyspace/iceworld/multithreadtornado/etc/includes_dev.json) 4 | [16-07-03 18:23:03][MainThread][serv.py:145] INFO: starting..., listen [8888], configurated by (/Users/luoran/work/pyspace/iceworld/multithreadtornado/etc/includes_dev.json) 5 | ======= 6 | 占位 7 | >>>>>>> 7c87ea4c495db9d9cf40b4b5fd400c00eac77b90 8 | -------------------------------------------------------------------------------- /multithreadtornado/var/http.log: -------------------------------------------------------------------------------- 1 | <<<<<<< HEAD 2 | [16-07-03 18:20:50] "GET /" 404 ::1 0.000715 3 | [16-07-03 18:20:50] "GET /favicon.ico" 404 ::1 0.000338 4 | [16-07-03 18:21:35] "GET /" 404 ::1 0.000784 5 | [16-07-03 18:23:07] "GET /" 200 ::1 0.001038 6 | [16-07-03 18:23:11] "GET /" 304 ::1 0.000913 7 | [16-07-03 18:23:13] "GET /hello/bill" 404 ::1 0.000408 8 | ======= 9 | 占位 10 | >>>>>>> 7c87ea4c495db9d9cf40b4b5fd400c00eac77b90 11 | --------------------------------------------------------------------------------