├── .gitignore ├── backend ├── biz │ ├── Demo.py │ └── __init__.py ├── etc │ ├── dev_log.conf │ └── etc.json ├── lib │ ├── __init__.py │ ├── autoconf.py │ ├── device.py │ ├── eloop.py │ ├── http.py │ ├── log.py │ ├── path.py │ └── router.py ├── serv.py └── var │ └── zanwei.txt ├── gateway ├── biz │ ├── __init__.py │ ├── core.py │ ├── delegate.py │ └── zbus.py ├── etc │ ├── dev_log.conf │ └── etc.json ├── lib │ ├── __init__.py │ ├── autoconf.py │ ├── gen.py │ ├── log.py │ └── path.py ├── serv.py └── var │ └── zanwei.txt ├── readme.md └── router ├── etc ├── dev_log.conf └── etc.json ├── lib ├── __init__.py ├── autoconf.py ├── const.py ├── eloop.py ├── log.py └── path.py ├── router.py ├── serv ├── __init__.py ├── device.py ├── errors.py └── trie.py └── var └── zanwei.txt /.gitignore: -------------------------------------------------------------------------------- 1 | ### Example user template template 2 | ### Example user template 3 | 4 | # IntelliJ project files 5 | .pyc 6 | .idea 7 | *.iml 8 | out 9 | gen 10 | # Created by .ignore support plugin (hsz.mobi) 11 | -------------------------------------------------------------------------------- /backend/biz/Demo.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | from lib.router import Router 3 | 4 | 5 | class Hello(object): 6 | @Router.routine(url='api/hello/(\w+)', method=Router.POST | Router.GET, timeout=5) 7 | def test(self, params, name): 8 | '''params是表单数据没有则为空字典，name为正则表达式捕获的参数， 9 | 如果有多个正则参数会依次注入到函数的参数列表中 10 | ''' 11 | age = params.get('age', '13') 12 | return 'hello! My name is ' + name + ', I\'m ' + age + ' years old' 13 | -------------------------------------------------------------------------------- /backend/biz/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'luor' 2 | -------------------------------------------------------------------------------- /backend/etc/dev_log.conf: -------------------------------------------------------------------------------- 1 | ############################################################## 2 | [loggers] 3 | keys=root,backend 4 | 5 | [logger_root] 6 | level=DEBUG 7 | handlers=console_handler 8 | qualname=root 9 | 10 | [logger_backend] 11 | level=DEBUG 12 | handlers=backend_handler,console_handler 13 | qualname=backend 14 | propagate=0 15 | 16 | 17 | ############################################################## 18 | [handlers] 19 | keys=console_handler,backend_handler 20 | 21 | [handler_console_handler] 22 | class=StreamHandler 23 | level=DEBUG 24 | formatter=application_formatter 25 | args=(sys.stdout,) 26 | 27 | [handler_backend_handler] 28 | class=handlers.TimedRotatingFileHandler 29 | level=DEBUG 30 | formatter=application_formatter 31 | args=('var/backend.log', 'midnight') 32 | 33 | ############################################################### 34 | [formatters] 35 | keys=application_formatter 36 | 37 | [formatter_application_formatter] 38 | format=[%(asctime)s][%(threadName)s][%(filename)s:%(lineno)s] %(levelname)s: %(message)s 39 | datefmt=%y-%m-%d %H:%M:%S 40 | 41 | -------------------------------------------------------------------------------- /backend/etc/etc.json: -------------------------------------------------------------------------------- 1 | { 2 | "connect": "127.0.0.1:9022", 3 | "logging": { 4 | "conf": "dev_log.conf" 5 | } 6 | } -------------------------------------------------------------------------------- /backend/lib/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'luor' 2 | -------------------------------------------------------------------------------- /backend/lib/autoconf.py: -------------------------------------------------------------------------------- 1 | #coding: utf-8 2 | 3 | import os, sys, re 4 | import json 5 | from abc import * 6 | import traceback 7 | 8 | 9 | class ConfigParser(object): 10 | ''' 11 | This abstract class provides a strategy of how to get those configurations 12 | through a file or remote config ? 13 | ''' 14 | @abstractmethod 15 | def parseall(self, *args): 16 | pass 17 | 18 | 19 | class E(object): 20 | def __init__(self, func): 21 | self.func = func 22 | 23 | def __ror__(self, inputs): 24 | return self.func(inputs) 25 | 26 | 27 | class Configer(object): 28 | ''' 29 | This class will hold configurations and registered setups(functions) 30 | It can determine when to setup them 31 | ''' 32 | config = {} 33 | setups = [] 34 | 35 | 36 | def register_my_setup(self, **deco): 37 | def foo(func): 38 | location = deco.get('look') 39 | level = deco.get('level', 99999) 40 | self.setups.append({ 41 | 'func': func, 42 | 'location': location, 43 | 'level': level 44 | }) 45 | return func 46 | 47 | return foo 48 | 49 | def setup(self, own_cfg, onlevel=0): 50 | ''' 51 | Call all(or specific level) setup functions which registered via using 52 | "Configer.register_my_setup" decorator. 53 | If "onlevel" has been set, only the matched setup fucntions will be 54 | loaded(or hot reloaded). 55 | BE CAREFUL! The registed setup function shall apply reload logic in case 56 | of a runtime-hot-reloaded callback hit. 57 | ''' 58 | self.setups.sort(key=lambda x: x['level']) 59 | self.config.update(own_cfg) 60 | 61 | for s in Configer.setups: 62 | func = s['func'] 63 | location = s['location'] 64 | try: 65 | if location: 66 | func(self.config[location]) 67 | else: 68 | func() 69 | except Exception: 70 | traceback.print_exc() 71 | sys.exit(1) 72 | 73 | def on_change(self): 74 | pass 75 | 76 | 77 | class ConfigParserFromFile(ConfigParser): 78 | ''' 79 | via Config Files 80 | ''' 81 | def parseall(self, fullpath): 82 | etc = os.path.dirname(fullpath) 83 | cfg = {} 84 | 85 | with open(fullpath, 'r') as f: 86 | raw = f.read() 87 | #去掉多行注释 88 | raw_escape_comment = re.sub(r'/\*[\s\S]+?\*/', '', raw) 89 | cfg = json.loads(raw_escape_comment) 90 | if cfg.get('$includes'): 91 | for include in cfg['$includes']: 92 | icfg = self.parseall(os.path.join(etc, include)) 93 | cfg.update(icfg) 94 | return cfg 95 | 96 | 97 | conf_drawer = Configer() -------------------------------------------------------------------------------- /backend/lib/device.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import zmq 3 | import http 4 | from collections import deque 5 | from eloop import Handler, IOLoop 6 | from log import app_log 7 | from router import Router 8 | 9 | try: 10 | import ujson as json 11 | except ImportError: 12 | import json 13 | 14 | from concurrent import futures 15 | 16 | MAX_WORKERS = 8 17 | executor = futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) 18 | 19 | 20 | def on_thread(**conf): 21 | global executor 22 | t_executor = conf.get('executor') or executor 23 | 24 | def _deco_func(func): 25 | def _deco_params(*args, **kwargs): 26 | t_executor.submit(func, *args, **kwargs) 27 | 28 | return _deco_params 29 | 30 | return _deco_func 31 | 32 | 33 | class Exporter(Handler): 34 | def __init__(self, sock, ioloop=IOLoop.instance()): 35 | self._ioloop = ioloop 36 | self._sock = sock 37 | self._buffer = deque(maxlen=100) 38 | self._flag = zmq.POLLIN 39 | self._ioloop.add_handler(self._sock, self.handle, self._flag) 40 | 41 | def send(self, frame): 42 | try: 43 | self._sock.send_multipart(frame, zmq.NOBLOCK) 44 | except zmq.Again: 45 | self._buffer.append(frame) 46 | self._flag |= zmq.POLLOUT 47 | self._ioloop.update_handler(self._sock, self._flag) 48 | except Exception as e: 49 | app_log.exception(e) 50 | 51 | def _handle_send(self): 52 | try: 53 | frame = self._buffer.popleft() 54 | self._sock.send_multipart(frame, zmq.NOBLOCK) 55 | except IndexError: 56 | self._flag &= (~zmq.POLLOUT) 57 | self._ioloop.update_handler(self._sock, self._flag) 58 | except Exception as e: 59 | app_log.exception(e) 60 | self._flag &= (~zmq.POLLOUT) 61 | self._ioloop.update_handler(self._sock, self._flag) 62 | 63 | def _handle_recv(self): 64 | source_id, url, seed_id, method, url_params, params = self._sock.recv_multipart() 65 | try: 66 | params = json.loads(params) 67 | Router.dispatch(source_id, url, seed_id, method, params, json.loads(url_params), self.on_wrap) 68 | except http.HttpMethodNotAllowed as e: 69 | self._ioloop.add_callback(self.send, ['rep', source_id, seed_id, e.state, e.content]) 70 | except Exception as e: 71 | self._ioloop.add_callback(self.send, ['rep', source_id, seed_id, '500', '(500): internal error']) 72 | app_log.exception(e) 73 | 74 | @on_thread() 75 | def on_wrap(self, func, source_id, seed_id): 76 | try: 77 | ret = func() 78 | if isinstance(ret, unicode): 79 | ret = ret.encode() 80 | elif not isinstance(ret, str): 81 | ret = json.dumps(ret) 82 | self._ioloop.add_callback(self.send, ['rep', source_id, seed_id, '200', ret]) 83 | except Exception as e: 84 | self._ioloop.add_callback(self.send, ['rep', source_id, seed_id, '500', '(500): internal error']) 85 | app_log.exception(e) 86 | -------------------------------------------------------------------------------- /backend/lib/eloop.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import time 3 | import thread 4 | import threading 5 | import functools 6 | import zmq 7 | import numbers 8 | import heapq 9 | from log import app_log as log 10 | 11 | 12 | class Handler(object): 13 | def handle(self, event): 14 | if event & zmq.POLLIN: 15 | self._handle_recv() 16 | elif event & zmq.POLLOUT: 17 | self._handle_send() 18 | 19 | def _handle_recv(self): 20 | raise NotImplementedError 21 | 22 | def _handle_send(self): 23 | raise NotImplementedError 24 | 25 | 26 | class Waker(Handler): 27 | def __init__(self): 28 | self._ctx = zmq.Context() 29 | self._reader = self._ctx.socket(zmq.PULL) 30 | self._writer = self._ctx.socket(zmq.PUSH) 31 | self._reader.bind('inproc://IOLOOPWAKER') 32 | self._writer.connect('inproc://IOLOOPWAKER') 33 | 34 | def fileno(self): 35 | return self._reader 36 | 37 | def _handle_recv(self): 38 | try: 39 | self._reader.recv(zmq.NOBLOCK) 40 | except zmq.ZMQError: 41 | pass 42 | 43 | def wake_up(self): 44 | try: 45 | self._writer.send(b'x', zmq.NOBLOCK) 46 | except zmq.ZMQError: 47 | pass 48 | 49 | 50 | class Timeout(object): 51 | __slots__ = ['deadline', 'callback', 'cancelled'] 52 | 53 | def __init__(self, deadline, callback, *args, **kwargs): 54 | if not isinstance(deadline, numbers.Real): 55 | raise TypeError("Unsupported deadline %r" % deadline) 56 | self.deadline = deadline 57 | self.callback = functools.partial(callback, *args, **kwargs) 58 | self.cancelled = False 59 | # IOLoop.instance().add_callback(IOLoop.instance().add_timeout, self) 60 | 61 | def cancel(self): 62 | self.cancelled = True 63 | 64 | def __le__(self, other): 65 | return self.deadline <= other.deadline 66 | 67 | def __lt__(self, other): 68 | return self.deadline < other.deadline 69 | 70 | 71 | class IOLoop(object): 72 | _instance_lock = threading.Lock() 73 | _local = threading.local() 74 | 75 | @staticmethod 76 | def instance(): 77 | """Returns a global `IOLoop` instance. 78 | """ 79 | if not hasattr(IOLoop, "_instance"): 80 | with IOLoop._instance_lock: 81 | if not hasattr(IOLoop, "_instance"): 82 | # New instance after double check 83 | IOLoop._instance = IOLoop() 84 | return IOLoop._instance 85 | 86 | @staticmethod 87 | def initialized(): 88 | """Returns true if the singleton instance has been created.""" 89 | return hasattr(IOLoop, "_instance") 90 | 91 | def __init__(self): 92 | self._handlers = {} 93 | self._callbacks = [] 94 | self._callback_lock = threading.Lock() 95 | self._timeouts = [] 96 | self._poller = zmq.Poller() 97 | self._idle_timeout = 5000.0 98 | self._thread_ident = -1 99 | self._waker = Waker() 100 | self.add_handler(self._waker.fileno(), self._waker.handle, zmq.POLLIN) 101 | self._idel_call = lambda :None 102 | 103 | def set_idel_call(self, idel_call): 104 | self._idel_call = idel_call 105 | 106 | def add_handler(self, fd, handler, flag): 107 | self._handlers[fd] = handler 108 | self._poller.register(fd, flag) 109 | 110 | def update_handler(self, fd, flag): 111 | self._poller.modify(fd, flag) 112 | 113 | def remove_handler(self, handler): 114 | fd = handler.fileno() 115 | self._handlers.pop(fd) 116 | self._poller.unregister(fd) 117 | 118 | def add_callback(self, callback, *args, **kwargs): 119 | with self._callback_lock: 120 | is_empty = not self._callbacks 121 | self._callbacks.append(functools.partial(callback, *args, **kwargs)) 122 | if is_empty and self._thread_ident != thread.get_ident(): 123 | self._waker.wake_up() 124 | 125 | def _run_callback(self, callback): 126 | try: 127 | callback() 128 | except Exception, e: 129 | log.exception(e) 130 | 131 | def add_timeout(self, timeout): 132 | heapq.heappush(self._timeouts, timeout) 133 | 134 | def start(self): 135 | 136 | self._thread_ident = thread.get_ident() 137 | 138 | while True: 139 | 140 | poll_time = self._idle_timeout 141 | 142 | with self._callback_lock: 143 | callbacks = self._callbacks 144 | self._callbacks = [] 145 | 146 | for callback in callbacks: 147 | self._run_callback(callback) 148 | # 为什么把超时列表放到callbacks执行之后读取? 149 | # 因为: 150 | # 1.add_timeout的动作也是通过add_callback来完成的,callbacks执行可能会影响到timeouts长度 151 | # 2.callback在执行的时候也会耽误一些时间, 在callbacks执行之后判断timeout才是比较准确的 152 | due_timeouts = [] 153 | now = time.time() 154 | while self._timeouts: 155 | lastest_timeout = heapq.heappop(self._timeouts) 156 | if not lastest_timeout.cancelled: 157 | if lastest_timeout.deadline <= now: 158 | due_timeouts.append(lastest_timeout) 159 | else: 160 | # 拿多了, 推进去, 顺便把poll()的时间确定出来 161 | heapq.heappush(self._timeouts, lastest_timeout) 162 | poll_time = lastest_timeout.deadline - time.time() # 这个值有可能是负数, 163 | poll_time = max(0.0, poll_time) # 为负数的话变为0 164 | break 165 | for timeout in due_timeouts: 166 | self._run_callback(timeout.callback) 167 | 168 | if self._callbacks: 169 | poll_time = 0.0 170 | 171 | sockets = dict(self._poller.poll(poll_time)) 172 | if sockets: 173 | for sock, event in sockets.iteritems(): 174 | handler = self._handlers[sock] 175 | try: 176 | handler(event) 177 | except Exception as e: 178 | log.exception(e) 179 | else: 180 | try: 181 | self._idel_call() 182 | except Exception as e: 183 | log.exception(e) 184 | -------------------------------------------------------------------------------- /backend/lib/http.py: -------------------------------------------------------------------------------- 1 | class HttpMethodNotAllowed(Exception): 2 | state = '405' 3 | content = '(405): method is forbidden' -------------------------------------------------------------------------------- /backend/lib/log.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import os 3 | 4 | import logging 5 | from logging.config import fileConfig 6 | import path 7 | from autoconf import conf_drawer 8 | 9 | app_log = logging.getLogger('backend') 10 | 11 | 12 | @conf_drawer.register_my_setup(look='logging', level=1) 13 | def setup(log_conf): 14 | log_path = os.path.join(path.ETC_PATH, log_conf['conf']) 15 | fileConfig(log_path) 16 | -------------------------------------------------------------------------------- /backend/lib/path.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import os 3 | 4 | HOME_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 5 | BIZ_PATH = os.path.join(HOME_PATH, 'biz') 6 | ETC_PATH = os.path.join(HOME_PATH, 'etc') 7 | LIB_PATH = os.path.join(HOME_PATH, 'lib') 8 | -------------------------------------------------------------------------------- /backend/lib/router.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import sys, inspect 3 | from lib.log import app_log 4 | import http 5 | import functools 6 | 7 | 8 | 9 | class Router(object): 10 | services = {} 11 | GET = 1 12 | POST = 1 << 1 13 | PUT = 1 << 2 14 | DELETE = 1 << 3 15 | 16 | @staticmethod 17 | def routine(**kwargs): 18 | 19 | def _register(func): 20 | url = kwargs['url'] 21 | method = kwargs.get('method', Router.GET | Router.POST) 22 | timeout = str(kwargs.get('timeout', 0)) 23 | if Router.services.get(url): 24 | app_log.fatal('Url Conflict: [%s]', url) 25 | sys.exit(1) 26 | Router.services[url] = { 27 | 'method': method, 28 | 'timeout': timeout or '', 29 | 'key': inspect.stack()[1][3], 30 | 'func': func 31 | } 32 | return func 33 | return _register 34 | 35 | @staticmethod 36 | def dispatch(source_id, url, seed_id, method, params, url_params, callwrap): 37 | service = Router.services.get(url) 38 | key = service['key'] 39 | func = service['func'] 40 | if not int(method) & service['method']: 41 | raise http.HttpMethodNotAllowed() 42 | klass = func.func_globals[key] 43 | instance = klass() 44 | if url_params: 45 | ifunc = functools.partial(func, instance, params, *url_params) 46 | else: 47 | ifunc = functools.partial(func, instance, params) 48 | callwrap(ifunc, source_id, seed_id) 49 | 50 | @staticmethod 51 | def register_urls(exporter): 52 | for serv_ident, service in Router.services.iteritems(): 53 | exporter.send(['train', serv_ident, service['timeout'], serv_ident]) -------------------------------------------------------------------------------- /backend/serv.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | 3 | import zmq 4 | import getopt, functools 5 | 6 | from lib.log import app_log 7 | from lib import path 8 | from lib.autoconf import * 9 | from lib.eloop import IOLoop 10 | from lib.device import Exporter 11 | from lib.router import Router 12 | 13 | HEARTBEAT_CC = 3.0 # 心跳周期 14 | 15 | exporter = None 16 | context = zmq.Context() 17 | 18 | 19 | @conf_drawer.register_my_setup(look='connect') 20 | def init(conf_file): 21 | global exporter 22 | #automic scan dirs 23 | files_list = os.listdir(path.BIZ_PATH) 24 | files_list = set([x[:x.rfind(".")] for x in files_list if x.endswith(".py")]) 25 | map(__import__, ['biz.' + x for x in files_list]) 26 | sock = context.socket(zmq.DEALER) 27 | sock.connect('tcp://' + conf_file) 28 | exporter = Exporter(sock) 29 | Router.register_urls(exporter) 30 | IOLoop.instance().set_idel_call(functools.partial(exporter.send, ['echo'])) 31 | 32 | 33 | 34 | 35 | if __name__ == '__main__': 36 | opts, argvs = getopt.getopt(sys.argv[1:], "c:f:b:") 37 | includes = None 38 | for op, value in opts: 39 | if op == '-c': 40 | includes = value 41 | if not includes: 42 | includes = os.path.join(path.ETC_PATH, 'etc.json') 43 | print "no configuration found!,will use [%s] instead" % includes 44 | includes = os.path.join(path.ETC_PATH, 'etc.json') 45 | cpff = ConfigParserFromFile() 46 | includes | E(cpff.parseall) | E(conf_drawer.setup) 47 | 48 | app_log.info('backend server started!') 49 | IOLoop.instance().start() -------------------------------------------------------------------------------- /backend/var/zanwei.txt: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /gateway/biz/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'luor' 2 | -------------------------------------------------------------------------------- /gateway/biz/core.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import tornado.web 3 | import sys 4 | 5 | 6 | class Application(tornado.web.Application): 7 | handlers = [] 8 | 9 | @classmethod 10 | def register(cls, **kwargs): 11 | path = kwargs.get('path') 12 | 13 | def deco(handler): 14 | clazz = handler 15 | Application.handlers.append((path, clazz)) 16 | return handler 17 | 18 | return deco 19 | 20 | def __init__(self, **kwargs): 21 | settings = { 22 | 'xsrf_cookies': False, 23 | 'autoreload': True 24 | } 25 | kwargs.update(settings) 26 | if Application.handlers: 27 | super(Application, self).__init__(Application.handlers, **kwargs) 28 | else: 29 | sys.exit(1) 30 | -------------------------------------------------------------------------------- /gateway/biz/delegate.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import urllib 3 | import tornado.web 4 | 5 | try: 6 | import json as json 7 | except ImportError: 8 | import json 9 | from biz.zbus import ZBus, ZRequest 10 | from core import Application 11 | 12 | 13 | @Application.register(path=r"^/([^\.|]*)(?!\.\w+)$") 14 | class Xroute(tornado.web.RequestHandler): 15 | '''doorlet''' 16 | GET = 1 17 | POST = 1 << 1 18 | PUT = 1 << 2 19 | DELETE = 1 << 3 20 | 21 | def prepare(self): 22 | # 获得正确的客户端ip 23 | ip = self.request.headers.get("X-Real-Ip", self.request.remote_ip) 24 | ip = self.request.headers.get("X-Forwarded-For", ip) 25 | ip = ip.split(',')[0].strip() 26 | self.request.remote_ip = ip 27 | # 允许跨域请求 28 | req_origin = self.request.headers.get("Origin") 29 | if req_origin: 30 | self.set_header("Access-Control-Allow-Origin", req_origin) 31 | self.set_header("Access-Control-Allow-Credentials", "true") 32 | self.set_header("Allow", "GET, HEAD, POST") 33 | if self.request.method == "OPTIONS": 34 | self.set_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") 35 | self.set_header("Access-Control-Allow-Headers", "Accept, Cache-Control, Content-Type") 36 | self.finish() 37 | return 38 | else: 39 | self.set_header("Cache-Control", "no-cache") 40 | # 分析请求参数 41 | self.dict_args = {} 42 | # json格式请求 43 | if self.request.headers.get('Content-Type', '').find("application/json") >= 0: 44 | try: 45 | self.dict_args = json.loads(self.request.body) 46 | if not self.json_args.get('ck'): 47 | # 如果没有ck，尝试从cookie里面加载usercheck 48 | self.json_args['ck'] = urllib.unquote(self.get_cookie('usercheck', '')) 49 | return 50 | except Exception as ex: 51 | self.send_error(400) 52 | # 普通参数请求 53 | else: 54 | self.dict_args = dict((k, v[-1]) for k, v in self.request.arguments.items()) 55 | return 56 | 57 | @tornado.web.asynchronous 58 | def get(self, path): 59 | req = ZRequest(path, str(Xroute.GET), self.dict_args) 60 | ZBus.instance().send(req, self.handle_zresponse) 61 | 62 | @tornado.web.asynchronous 63 | def post(self, path): 64 | req = ZRequest(path, str(Xroute.POST), self.dict_args) 65 | ZBus.instance().send(req, self.handle_zresponse) 66 | 67 | @tornado.web.asynchronous 68 | def put(self, path): 69 | req = ZRequest(path, str(Xroute.PUT), self.dict_args) 70 | ZBus.instance().send(req, self.handle_zresponse) 71 | 72 | @tornado.web.asynchronous 73 | def delete(self, path): 74 | req = ZRequest(path, str(Xroute.DELETE), self.dict_args) 75 | ZBus.instance().send(req, self.handle_zresponse) 76 | 77 | def handle_zresponse(self, zresponse): 78 | state_code = zresponse.state 79 | if state_code: 80 | self.set_status(state_code) 81 | self.finish(zresponse.content) -------------------------------------------------------------------------------- /gateway/biz/zbus.py: -------------------------------------------------------------------------------- 1 | # coding:utf8 2 | import zmq 3 | from zmq.eventloop.zmqstream import ZMQStream 4 | from lib.gen import Gen 5 | from tornado.log import app_log 6 | from lib.autoconf import conf_drawer 7 | 8 | try: 9 | import ujson as json 10 | except ImportError: 11 | import json 12 | 13 | 14 | @conf_drawer.register_my_setup(look='router') 15 | def setup(dist): 16 | ZBus.instance().connect(dist) 17 | 18 | 19 | class ZBus(object): 20 | def __init__(self): 21 | self._context = zmq.Context() 22 | self._callback = {} 23 | self._zstream = None 24 | 25 | @staticmethod 26 | def instance(): 27 | if not hasattr(ZBus, '_instance'): 28 | ZBus._instance = ZBus() 29 | return ZBus._instance 30 | 31 | @staticmethod 32 | def initialized(): 33 | return hasattr(ZBus, '_instance') 34 | 35 | def connect(self, dist): 36 | if self._zstream: 37 | self._zstream.close() 38 | self._zsock = self._context.socket(zmq.XREQ) 39 | self._zsock.connect('tcp://{dist}'.format(dist=dist)) 40 | self._zstream = ZMQStream(self._zsock) 41 | self._zstream.on_recv(self.on_recv) 42 | 43 | def send(self, request, callback): 44 | self._callback[request.seed_id] = callback 45 | self._zstream.send_multipart(request.box()) 46 | 47 | def on_recv(self, frame): 48 | response = ZResponse(frame) 49 | callback = self._callback.pop(response.seed_id) if self._callback.get(response.seed_id) else None 50 | if callback and callable(callback): 51 | callback(response) 52 | 53 | 54 | class ZRequest(object): 55 | def __init__(self, path, method, dict_args, gen=Gen.global_id): 56 | self.seed_id = gen() 57 | self.path = str(path) 58 | self.method = method 59 | self.params = json.dumps(dict_args) 60 | 61 | def box(self): 62 | return [self.seed_id, self.path, self.method, self.params] 63 | 64 | 65 | class ZResponse(object): 66 | def __init__(self, frame): 67 | self.seed_id, self.state, self.content = frame 68 | self.state = int(self.state) if self.state else None 69 | -------------------------------------------------------------------------------- /gateway/etc/dev_log.conf: -------------------------------------------------------------------------------- 1 | ############################################################## 2 | [loggers] 3 | keys=root,tornado.access,tornado.application, fadeaway 4 | 5 | [logger_root] 6 | level=DEBUG 7 | handlers=console_handler 8 | qualname=root 9 | 10 | [logger_tornado.application] 11 | level=DEBUG 12 | handlers=tornado.application_handler,console_handler 13 | qualname=tornado.application 14 | propagate=0 15 | 16 | [logger_tornado.access] 17 | level=DEBUG 18 | handlers=http_handler 19 | qualname=tornado.access 20 | propagate=0 21 | 22 | [logger_fadeaway] 23 | level=DEBUG 24 | handlers=http_handler 25 | qualname=fadeaway 26 | propagate=0 27 | 28 | ############################################################## 29 | [handlers] 30 | keys=console_handler,tornado.application_handler,http_handler 31 | 32 | [handler_console_handler] 33 | class=StreamHandler 34 | level=DEBUG 35 | formatter=tornado.application_formatter 36 | args=(sys.stdout,) 37 | 38 | [handler_tornado.application_handler] 39 | class=handlers.TimedRotatingFileHandler 40 | level=DEBUG 41 | formatter=tornado.application_formatter 42 | args=('var/application.log', 'midnight') 43 | 44 | [handler_http_handler] 45 | class=handlers.TimedRotatingFileHandler 46 | level=DEBUG 47 | formatter=http_formatter 48 | args=('var/http.log', 'midnight') 49 | 50 | ############################################################### 51 | [formatters] 52 | keys=tornado.application_formatter,http_formatter 53 | 54 | [formatter_tornado.application_formatter] 55 | format=[%(asctime)s][%(threadName)s][%(filename)s:%(lineno)s] %(levelname)s: %(message)s 56 | datefmt=%y-%m-%d %H:%M:%S 57 | 58 | [formatter_http_formatter] 59 | format=[%(asctime)s] %(message)s 60 | datefmt=%y-%m-%d %H:%M:%S 61 | -------------------------------------------------------------------------------- /gateway/etc/etc.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "conf": "dev_log.conf" 4 | }, 5 | 6 | /* 7 | * router 8 | */ 9 | "router": "localhost:9021" 10 | } -------------------------------------------------------------------------------- /gateway/lib/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'luor' 2 | -------------------------------------------------------------------------------- /gateway/lib/autoconf.py: -------------------------------------------------------------------------------- 1 | #coding: utf-8 2 | 3 | import os, sys, re 4 | import json 5 | from abc import * 6 | import traceback 7 | 8 | 9 | class ConfigParser(object): 10 | ''' 11 | This abstract class provides a strategy of how to get those configurations 12 | through a file or remote config ? 13 | ''' 14 | @abstractmethod 15 | def parseall(self, *args): 16 | pass 17 | 18 | 19 | class E(object): 20 | def __init__(self, func): 21 | self.func = func 22 | 23 | def __ror__(self, inputs): 24 | return self.func(inputs) 25 | 26 | 27 | class Configer(object): 28 | ''' 29 | This class will hold configurations and registered setups(functions) 30 | It can determine when to setup them 31 | ''' 32 | config = {} 33 | setups = [] 34 | 35 | 36 | def register_my_setup(self, **deco): 37 | def foo(func): 38 | location = deco.get('look') 39 | level = deco.get('level', 99999) 40 | self.setups.append({ 41 | 'func': func, 42 | 'location': location, 43 | 'level': level 44 | }) 45 | return func 46 | 47 | return foo 48 | 49 | def setup(self, own_cfg, onlevel=0): 50 | ''' 51 | Call all(or specific level) setup functions which registered via using 52 | "Configer.register_my_setup" decorator. 53 | If "onlevel" has been set, only the matched setup fucntions will be 54 | loaded(or hot reloaded). 55 | BE CAREFUL! The registed setup function shall apply reload logic in case 56 | of a runtime-hot-reloaded callback hit. 57 | ''' 58 | self.setups.sort(key=lambda x: x['level']) 59 | self.config.update(own_cfg) 60 | 61 | for s in Configer.setups: 62 | func = s['func'] 63 | location = s['location'] 64 | try: 65 | if location: 66 | func(self.config[location]) 67 | else: 68 | func() 69 | except Exception: 70 | traceback.print_exc() 71 | sys.exit(1) 72 | 73 | def on_change(self): 74 | pass 75 | 76 | 77 | class ConfigParserFromFile(ConfigParser): 78 | ''' 79 | via Config Files 80 | ''' 81 | def parseall(self, fullpath): 82 | etc = os.path.dirname(fullpath) 83 | cfg = {} 84 | 85 | with open(fullpath, 'r') as f: 86 | raw = f.read() 87 | #去掉多行注释 88 | raw_escape_comment = re.sub(r'/\*[\s\S]+?\*/', '', raw) 89 | cfg = json.loads(raw_escape_comment) 90 | if cfg.get('$includes'): 91 | for include in cfg['$includes']: 92 | icfg = self.parseall(os.path.join(etc, include)) 93 | cfg.update(icfg) 94 | return cfg 95 | 96 | 97 | conf_drawer = Configer() -------------------------------------------------------------------------------- /gateway/lib/gen.py: -------------------------------------------------------------------------------- 1 | #coding: utf8 2 | import uuid 3 | import time 4 | 5 | 6 | class Gen(object): 7 | 8 | node_id = str(uuid.getnode()) 9 | gtime = 0 10 | gid = 0 11 | 12 | @staticmethod 13 | def global_id(): 14 | now = int(time.time()) 15 | if Gen.gtime == now: 16 | Gen.gid += 1 17 | else: 18 | Gen.gid = 0 19 | Gen.gtime = now 20 | return ''.join([Gen.node_id, str(now), str(Gen.gid)]) -------------------------------------------------------------------------------- /gateway/lib/log.py: -------------------------------------------------------------------------------- 1 | #coding: utf8 2 | import os 3 | from logging.config import fileConfig 4 | import path 5 | from autoconf import conf_drawer 6 | 7 | 8 | @conf_drawer.register_my_setup(look='logging', level=1) 9 | def setup(log_conf): 10 | log_path = os.path.join(path.ETC_PATH, log_conf['conf']) 11 | fileConfig(log_path) -------------------------------------------------------------------------------- /gateway/lib/path.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import os 3 | from autoconf import conf_drawer 4 | 5 | HOME_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 6 | 7 | BIZ_PATH = os.path.join(HOME_PATH, 'biz') 8 | ETC_PATH = os.path.join(HOME_PATH, 'etc') 9 | LIB_PATH = os.path.join(HOME_PATH, 'lib') 10 | 11 | 12 | @conf_drawer.register_my_setup() 13 | def all_beautiful_memories_begin(): 14 | import log 15 | files_list = os.listdir(BIZ_PATH) 16 | files_list = set(['biz.' + x[:x.rfind(".")] for x in files_list if x.endswith(".py")]) 17 | map(__import__, files_list) 18 | -------------------------------------------------------------------------------- /gateway/serv.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | 3 | import getopt 4 | import tornado.ioloop 5 | import tornado.web 6 | from biz.core import Application 7 | from zmq.eventloop.ioloop import ZMQIOLoop 8 | from lib.autoconf import * 9 | from lib import path 10 | from tornado.log import app_log 11 | 12 | loop = ZMQIOLoop() 13 | loop.install() 14 | 15 | 16 | if __name__ == "__main__": 17 | # init 18 | port = 8888 19 | includes = None 20 | opts, argvs = getopt.getopt(sys.argv[1:], "c:p:") 21 | for op, value in opts: 22 | if op == '-c': 23 | includes = value 24 | elif op == '-p': 25 | port = int(value) 26 | if not includes: 27 | includes = os.path.join(path.ETC_PATH, 'etc.json') 28 | print "no configuration found!,will use [%s] instead" % includes 29 | cpff = ConfigParserFromFile() 30 | includes | E(cpff.parseall) | E(conf_drawer.setup) 31 | app = Application() 32 | app.listen(port) 33 | app_log.info('starting...') 34 | tornado.ioloop.IOLoop.instance().start() 35 | -------------------------------------------------------------------------------- /gateway/var/zanwei.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nikoloss/cellnest/2ef17c88632d512c740463da176bc69bf21435ba/gateway/var/zanwei.txt -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Cell Nest 2 | === 3 | ## Introduction 4 | 构建分布式的巢状http服务。一个请求的过程如下： 5 | 6 | 1.gateway利用tornado框架来分析http请求中的参数和路径，并绑定全局唯一request_id，接着投递给router， 7 | 8 | 2.router通过匹配url转发表确定出一个cell service(backend)，将来源(gateway)同时写入数据包，并将数据包投递给backend 9 | 10 | 3.backend启动之时会向router汇报url转发表，同时通过心跳维护与router的连接，backend在接受router的请求之后以多线程模型展开 11 | 业务处理，处理完毕之后回复router 12 | 13 | 4.router拆数据包得到原始gateway信息从而确定是哪个gateway节点来源，于是将报文回复给gateway节点 14 | 15 | 5.gateway接收router回复的response，通过数据包中的request_id来回复具体的socket fd，自此 一个完整的请求流程处理完毕 16 | 17 | ## Why? 18 | 为什么要这么做？随着软件规模的上升跟解耦，服务会越来越离散跟分裂，这样有助于在不影响一部分服务的同时，开发另一部分新 19 | 服务，然而这些服务终究是“服务”需要提供统一的接口供外部调用，同时保证在一部分服务的宕机下不会影响整个系统，同时还要解决依赖风暴。于是一个既方便开发又方便部署的方案就被提上了案头。使用此项目，只需要扩展backend，新的backend进程在启动之时会向路由节点汇报自己所有的url，路由节点会动态的更新自己的转发表。从而实现不宕机动态部署(启发式服务发现)。同时也可以直接杀掉backend进程，路由节点会自动删除此服务对应的转发表为不可用(通过心跳实现) 20 | 21 | ## Quick Start 22 | 部署之前需要tornado, futures, zmq模块，先安装依赖 23 | 然后依次启动gateway, route, backend。backend是重点需要关注的，一个backend也就是一个nest cell。 24 | 首先进入backend目录，然后 25 | 26 | 1.在“biz”目录中创建一个py文件，文件名任意但最好不要跟第三方库冲突 27 | 28 | 2.使用 "Router.routine" 装饰器注册函数到路由表中，仿造示例即可 29 | 30 | 3.到主目录下，使用命令"python serv.py" 启动工程，用浏览器访问步骤二中注册的路径可看到效果(例如访问demo的路径就是http://localhost:8888/api/hello/billy?age=12) 31 | 32 | 33 | ## License 34 | Due to benefit from zeromq, licensed under the GNU Lesser 35 | General Public License V3 plus, respect. 36 | 37 | ## Feedback 38 | * mailto(rowland.lan@163.com) or (rowland.lancer@gmail.com) 39 | * QQ(623135465) 40 | * 知乎(http://www.zhihu.com/people/luo-ran-22) 41 | -------------------------------------------------------------------------------- /router/etc/dev_log.conf: -------------------------------------------------------------------------------- 1 | ############################################################## 2 | [loggers] 3 | keys=root,router 4 | 5 | [logger_root] 6 | level=DEBUG 7 | handlers=console_handler 8 | qualname=root 9 | 10 | [logger_router] 11 | level=DEBUG 12 | handlers=router_handler,console_handler 13 | qualname=router 14 | propagate=0 15 | 16 | 17 | ############################################################## 18 | [handlers] 19 | keys=console_handler,router_handler 20 | 21 | [handler_console_handler] 22 | class=StreamHandler 23 | level=DEBUG 24 | formatter=application_formatter 25 | args=(sys.stdout,) 26 | 27 | [handler_router_handler] 28 | class=handlers.TimedRotatingFileHandler 29 | level=DEBUG 30 | formatter=application_formatter 31 | args=('var/router.log', 'midnight') 32 | 33 | ############################################################### 34 | [formatters] 35 | keys=application_formatter 36 | 37 | [formatter_application_formatter] 38 | format=[%(asctime)s][%(threadName)s][%(filename)s:%(lineno)s] %(levelname)s: %(message)s 39 | datefmt=%y-%m-%d %H:%M:%S 40 | 41 | -------------------------------------------------------------------------------- /router/etc/etc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nikoloss/cellnest/2ef17c88632d512c740463da176bc69bf21435ba/router/etc/etc.json -------------------------------------------------------------------------------- /router/lib/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'luor' 2 | -------------------------------------------------------------------------------- /router/lib/autoconf.py: -------------------------------------------------------------------------------- 1 | #coding: utf-8 2 | 3 | import os, sys, re 4 | import json 5 | from abc import * 6 | import traceback 7 | 8 | 9 | class ConfigParser(object): 10 | ''' 11 | This abstract class provides a strategy of how to get those configurations 12 | through a file or remote config ? 13 | ''' 14 | @abstractmethod 15 | def parseall(self, *args): 16 | pass 17 | 18 | 19 | class E(object): 20 | def __init__(self, func): 21 | self.func = func 22 | 23 | def __ror__(self, inputs): 24 | return self.func(inputs) 25 | 26 | 27 | class Configer(object): 28 | ''' 29 | This class will hold configurations and registered setups(functions) 30 | It can determine when to setup them 31 | ''' 32 | config = {} 33 | setups = [] 34 | 35 | 36 | def register_my_setup(self, **deco): 37 | def foo(func): 38 | location = deco.get('look') 39 | level = deco.get('level', 99999) 40 | self.setups.append({ 41 | 'func': func, 42 | 'location': location, 43 | 'level': level 44 | }) 45 | return func 46 | 47 | return foo 48 | 49 | def setup(self, own_cfg, onlevel=0): 50 | ''' 51 | Call all(or specific level) setup functions which registered via using 52 | "Configer.register_my_setup" decorator. 53 | If "onlevel" has been set, only the matched setup fucntions will be 54 | loaded(or hot reloaded). 55 | BE CAREFUL! The registed setup function shall apply reload logic in case 56 | of a runtime-hot-reloaded callback hit. 57 | ''' 58 | self.setups.sort(key=lambda x: x['level']) 59 | self.config.update(own_cfg) 60 | 61 | for s in Configer.setups: 62 | func = s['func'] 63 | location = s['location'] 64 | try: 65 | if location: 66 | func(self.config[location]) 67 | else: 68 | func() 69 | except Exception: 70 | traceback.print_exc() 71 | sys.exit(1) 72 | 73 | def on_change(self): 74 | pass 75 | 76 | 77 | class ConfigParserFromFile(ConfigParser): 78 | ''' 79 | via Config Files 80 | ''' 81 | def parseall(self, fullpath): 82 | etc = os.path.dirname(fullpath) 83 | cfg = {} 84 | 85 | with open(fullpath, 'r') as f: 86 | raw = f.read() 87 | #去掉多行注释 88 | raw_escape_comment = re.sub(r'/\*[\s\S]+?\*/', '', raw) 89 | cfg = json.loads(raw_escape_comment) 90 | if cfg.get('$includes'): 91 | for include in cfg['$includes']: 92 | icfg = self.parseall(os.path.join(etc, include)) 93 | cfg.update(icfg) 94 | return cfg 95 | 96 | 97 | conf_drawer = Configer() -------------------------------------------------------------------------------- /router/lib/const.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | 3 | 4 | class HttpStateCode(object): 5 | 6 | pass -------------------------------------------------------------------------------- /router/lib/eloop.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import time 3 | import thread 4 | import threading 5 | import functools 6 | import zmq 7 | import numbers 8 | import heapq 9 | from log import app_log as log 10 | 11 | 12 | class Handler(object): 13 | def handle(self, event): 14 | if event & zmq.POLLIN: 15 | self._handle_recv() 16 | elif event & zmq.POLLOUT: 17 | self._handle_send() 18 | 19 | def _handle_recv(self): 20 | raise NotImplementedError 21 | 22 | def _handle_send(self): 23 | raise NotImplementedError 24 | 25 | 26 | class Waker(Handler): 27 | def __init__(self): 28 | self._ctx = zmq.Context() 29 | self._reader = self._ctx.socket(zmq.PULL) 30 | self._writer = self._ctx.socket(zmq.PUSH) 31 | self._reader.bind('inproc://IOLOOPWAKER') 32 | self._writer.connect('inproc://IOLOOPWAKER') 33 | 34 | def fileno(self): 35 | return self._reader 36 | 37 | def _handle_recv(self): 38 | try: 39 | self._reader.recv(zmq.NOBLOCK) 40 | except zmq.ZMQError: 41 | pass 42 | 43 | def wake_up(self): 44 | try: 45 | self._writer.send(b'x', zmq.NOBLOCK) 46 | except zmq.ZMQError: 47 | pass 48 | 49 | 50 | class Timeout(object): 51 | __slots__ = ['deadline', 'callback', 'cancelled'] 52 | 53 | def __init__(self, deadline, callback, *args, **kwargs): 54 | if not isinstance(deadline, numbers.Real): 55 | raise TypeError("Unsupported deadline %r" % deadline) 56 | self.deadline = deadline 57 | self.callback = functools.partial(callback, *args, **kwargs) 58 | self.cancelled = False 59 | # IOLoop.instance().add_callback(IOLoop.instance().add_timeout, self) 60 | 61 | def cancel(self): 62 | self.cancelled = True 63 | 64 | def __le__(self, other): 65 | return self.deadline <= other.deadline 66 | 67 | def __lt__(self, other): 68 | return self.deadline < other.deadline 69 | 70 | 71 | class IOLoop(object): 72 | _instance_lock = threading.Lock() 73 | _local = threading.local() 74 | 75 | @staticmethod 76 | def instance(): 77 | """Returns a global `IOLoop` instance. 78 | """ 79 | if not hasattr(IOLoop, "_instance"): 80 | with IOLoop._instance_lock: 81 | if not hasattr(IOLoop, "_instance"): 82 | # New instance after double check 83 | IOLoop._instance = IOLoop() 84 | return IOLoop._instance 85 | 86 | @staticmethod 87 | def initialized(): 88 | """Returns true if the singleton instance has been created.""" 89 | return hasattr(IOLoop, "_instance") 90 | 91 | def __init__(self): 92 | self._handlers = {} 93 | self._callbacks = [] 94 | self._callback_lock = threading.Lock() 95 | self._timeouts = [] 96 | self._poller = zmq.Poller() 97 | self._idle_timeout = 3600.0 98 | self._thread_ident = -1 99 | self._waker = Waker() 100 | self.add_handler(self._waker.fileno(), self._waker.handle, zmq.POLLIN) 101 | 102 | def add_handler(self, fd, handler, flag): 103 | self._handlers[fd] = handler 104 | self._poller.register(fd, flag) 105 | 106 | def update_handler(self, fd, flag): 107 | self._poller.modify(fd, flag) 108 | 109 | def remove_handler(self, handler): 110 | fd = handler.fileno() 111 | self._handlers.pop(fd) 112 | self._poller.unregister(fd) 113 | 114 | def add_callback(self, callback, *args, **kwargs): 115 | with self._callback_lock: 116 | is_empty = not self._callbacks 117 | self._callbacks.append(functools.partial(callback, *args, **kwargs)) 118 | if is_empty and self._thread_ident != thread.get_ident(): 119 | self._waker.wake_up() 120 | 121 | def _run_callback(self, callback): 122 | try: 123 | callback() 124 | except Exception, e: 125 | log.exception(e) 126 | 127 | def add_timeout(self, timeout): 128 | heapq.heappush(self._timeouts, timeout) 129 | 130 | def start(self): 131 | 132 | self._thread_ident = thread.get_ident() 133 | 134 | while True: 135 | 136 | poll_time = self._idle_timeout 137 | 138 | with self._callback_lock: 139 | callbacks = self._callbacks 140 | self._callbacks = [] 141 | 142 | for callback in callbacks: 143 | self._run_callback(callback) 144 | # 为什么把超时列表放到callbacks执行之后读取? 145 | # 因为: 146 | # 1.add_timeout的动作也是通过add_callback来完成的,callbacks执行可能会影响到timeouts长度 147 | # 2.callback在执行的时候也会耽误一些时间, 在callbacks执行之后判断timeout才是比较准确的 148 | due_timeouts = [] 149 | now = time.time() 150 | while self._timeouts: 151 | lastest_timeout = heapq.heappop(self._timeouts) 152 | if not lastest_timeout.cancelled: 153 | if lastest_timeout.deadline <= now: 154 | due_timeouts.append(lastest_timeout) 155 | else: 156 | # 拿多了, 推进去, 顺便把poll()的时间确定出来 157 | heapq.heappush(self._timeouts, lastest_timeout) 158 | poll_time = lastest_timeout.deadline - time.time() # 这个值有可能是负数, 159 | poll_time = max(0.0, poll_time) # 为负数的话变为0 160 | break 161 | for timeout in due_timeouts: 162 | self._run_callback(timeout.callback) 163 | 164 | if self._callbacks: 165 | poll_time = 0.0 166 | 167 | sockets = dict(self._poller.poll(poll_time * 1000)) 168 | if sockets: 169 | for sock, event in sockets.iteritems(): 170 | handler = self._handlers[sock] 171 | try: 172 | handler(event) 173 | except Exception as e: 174 | log.exception(e) 175 | -------------------------------------------------------------------------------- /router/lib/log.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import os 3 | 4 | import logging 5 | from logging.config import fileConfig 6 | import path 7 | from autoconf import conf_drawer 8 | 9 | app_log = logging.getLogger('router') 10 | 11 | 12 | @conf_drawer.register_my_setup(look='logging', level=1) 13 | def setup(log_conf): 14 | log_path = os.path.join(path.ETC_PATH, log_conf['conf']) 15 | fileConfig(log_path) 16 | -------------------------------------------------------------------------------- /router/lib/path.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import os 3 | 4 | HOME_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 5 | 6 | ETC_PATH = os.path.join(HOME_PATH, 'etc') 7 | LIB_PATH = os.path.join(HOME_PATH, 'lib') 8 | -------------------------------------------------------------------------------- /router/router.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import zmq 3 | import getopt 4 | from serv import device, trie 5 | from lib.eloop import IOLoop 6 | from lib import path 7 | from lib.autoconf import * 8 | from lib.log import app_log 9 | 10 | front_dist = "tcp://*:{front}" 11 | backend_dist = "tcp://*:{backend}" 12 | context = zmq.Context() 13 | front_handler = None 14 | backend_handler = None 15 | serv_node_trie = trie.ServNode('ROOT') 16 | 17 | #===========TEST============== 18 | serv_node = device.Server('', 'identify', '15') 19 | trie.train(serv_node_trie, ['ok', 'test'], serv_node) 20 | 21 | 22 | 23 | @conf_drawer.register_my_setup(look='router') 24 | def setup(conf): 25 | global front_dist, backend_dist, front_handler, backend_handler 26 | front_dist = front_dist.format(front=conf['front']) 27 | backend_dist = backend_dist.format(backend=conf['backend']) 28 | front_sock = context.socket(zmq.ROUTER) 29 | front_sock.bind(front_dist) 30 | backend_sock = context.socket(zmq.ROUTER) 31 | backend_sock.bind(backend_dist) 32 | front_handler = device.Front(front_sock, serv_node_trie) 33 | backend_handler = device.Backend(backend_sock, serv_node_trie) 34 | device.connect(front_handler, 'on_recv', backend_handler, 'send') 35 | device.connect(backend_handler, 'on_response', front_handler, 'send') 36 | 37 | 38 | if __name__ == '__main__': 39 | opts, argvs = getopt.getopt(sys.argv[1:], "c:f:b:") 40 | includes = None 41 | for op, value in opts: 42 | if op == '-c': 43 | includes = value 44 | if not includes: 45 | includes = os.path.join(path.ETC_PATH, 'etc.json') 46 | print "no configuration found!,will use [%s] instead" % includes 47 | includes = os.path.join(path.ETC_PATH, 'etc.json') 48 | cpff = ConfigParserFromFile() 49 | includes | E(cpff.parseall) | E(conf_drawer.setup) 50 | 51 | app_log.info('checked! All Green...') 52 | IOLoop.instance().start() 53 | -------------------------------------------------------------------------------- /router/serv/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'luor' 2 | -------------------------------------------------------------------------------- /router/serv/device.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import zmq 3 | import time 4 | from collections import deque 5 | from lib.eloop import Handler, IOLoop, Timeout 6 | from trie import search_server, train 7 | from lib.log import app_log 8 | from lib.autoconf import E 9 | 10 | try: 11 | import ujson as json 12 | except ImportError: 13 | import json 14 | 15 | BUFFER_SZ = 100 16 | HEARTBEAT_CC = 5 17 | timeouts = {} 18 | serv_hb = {} 19 | 20 | 21 | def connect(subject, sub_func, object, obj_func): 22 | sub_function = getattr(subject, sub_func) 23 | obj_function = getattr(object, obj_func) 24 | 25 | def _deco_func(sfunc, ofunc): 26 | def _deco_params(*args, **kwargs): 27 | sfunc(*args, **kwargs) | E(ofunc) 28 | 29 | return _deco_params 30 | 31 | sub_function = _deco_func(sub_function, obj_function) 32 | setattr(subject, sub_func, sub_function) 33 | 34 | 35 | class Server(object): 36 | def __init__(self, timeout_conf, serv_ident, function_id): 37 | self.timeout_conf = int(timeout_conf) if timeout_conf else None 38 | self.serv_ident = serv_ident 39 | self.function_id = function_id 40 | 41 | 42 | class Front(Handler): 43 | def __init__(self, sock, root, ioloop=IOLoop.instance()): 44 | self._ioloop = ioloop 45 | self._sock = sock 46 | self._root = root 47 | self._buffer = deque(maxlen=BUFFER_SZ) 48 | self._flag = zmq.POLLIN 49 | self._ioloop.add_handler(self._sock, self.handle, self._flag) 50 | 51 | def send(self, frame): 52 | try: 53 | self._sock.send_multipart(frame, zmq.NOBLOCK) 54 | except zmq.Again: 55 | self._buffer.append(frame) 56 | self._flag |= zmq.POLLOUT 57 | self._ioloop.update_handler(self._sock, self._flag) 58 | except Exception as e: 59 | app_log.exception(e) 60 | 61 | def on_timeout(self, source_ident, seed_id): 62 | self.send([source_ident, seed_id, '408', 'timeout']) 63 | if timeouts.get(seed_id): 64 | timeouts.pop(seed_id) 65 | 66 | def on_recv(self, frame): 67 | ret = None 68 | source_ident, seed_id, path, method, params = frame 69 | url_fragments = path.split('/') 70 | url_params = [] 71 | servers = search_server(self._root, url_fragments, url_params) 72 | if not servers: 73 | self.send([source_ident, seed_id, '404', 'resource is not found!']) 74 | return 75 | to_be_del = [] 76 | for server in servers: 77 | been = (time.time() - serv_hb[server.serv_ident]) 78 | if been > 2 * HEARTBEAT_CC: 79 | app_log.warn('[%s] has lost contact for %f minute', server.function_id, been) 80 | if been > 10 * HEARTBEAT_CC: 81 | app_log.info('[%s] removed!', server.function_id) 82 | # servers.remove(server) 83 | to_be_del.append(server) 84 | else: 85 | if server.timeout_conf: 86 | timeout = Timeout( 87 | time.time() + max(server.timeout_conf, 1), 88 | self.on_timeout, 89 | source_ident, 90 | seed_id 91 | ) 92 | self._ioloop.add_timeout(timeout) 93 | ret = [server.serv_ident, source_ident, server.function_id, seed_id, method, json.dumps(url_params), params] 94 | break 95 | else: 96 | self.send([source_ident, seed_id, '502', 'resource temporary unavailable']) 97 | if to_be_del: 98 | map(servers.remove, to_be_del) 99 | return ret 100 | 101 | 102 | def _handle_recv(self): 103 | frame = self._sock.recv_multipart() 104 | self.on_recv(frame) 105 | 106 | def _handle_send(self): 107 | try: 108 | frame = self._buffer.popleft() 109 | self._sock.send_multipart(frame, zmq.NOBLOCK) 110 | except IndexError: 111 | self._flag &= (~zmq.POLLOUT) 112 | self._ioloop.update_handler(self._sock, self._flag) 113 | except Exception as e: 114 | app_log.exception(e) 115 | self._flag &= (~zmq.POLLOUT) 116 | self._ioloop.update_handler(self._sock, self._flag) 117 | 118 | 119 | class Backend(Handler): 120 | def __init__(self, sock, root, ioloop=IOLoop.instance()): 121 | self._ioloop = ioloop 122 | self._sock = sock 123 | self._root = root 124 | self._buffer = deque(maxlen=BUFFER_SZ) 125 | self._flag = zmq.POLLIN 126 | self._ioloop.add_handler(self._sock, self.handle, self._flag) 127 | 128 | def send(self, frame): 129 | if not frame: 130 | return 131 | try: 132 | self._sock.send_multipart(frame, zmq.NOBLOCK) 133 | except zmq.Again: 134 | self._buffer.append(frame) 135 | self._flag |= zmq.POLLOUT 136 | self._ioloop.update_handler(self._sock, self._flag) 137 | except Exception as e: 138 | app_log.exception(e) 139 | 140 | def on_recv(self, frame): 141 | if len(frame) > 1: 142 | server_id = frame[0] 143 | # any response will be regarded as heartbeat 144 | serv_hb[server_id] = time.time() 145 | flag = frame[1] 146 | if flag == 'rep': 147 | self.on_response(frame) 148 | elif flag == 'train': 149 | self.on_train(frame) 150 | else: 151 | app_log.warn('Unknown response: %s', str(frame)) 152 | 153 | def on_response(self, frame): 154 | server_id, flag, source_id, seed_id, state, content = frame 155 | return [source_id, seed_id, state, content] 156 | 157 | def on_train(self, frame): 158 | server_id, flag, path, timeout_conf, function_id = frame 159 | serv_node = Server(timeout_conf, server_id, function_id) 160 | train(self._root, path.split('/'), serv_node) 161 | 162 | def _handle_recv(self): 163 | frame = self._sock.recv_multipart() 164 | self.on_recv(frame) 165 | 166 | def _handle_send(self): 167 | try: 168 | frame = self._buffer.popleft() 169 | self._sock.send_multipart(frame, zmq.NOBLOCK) 170 | except IndexError: 171 | self._flag &= (~zmq.POLLOUT) 172 | self._ioloop.update_handler(self._sock, self._flag) 173 | except Exception as e: 174 | app_log.exception(e) 175 | self._flag &= (~zmq.POLLOUT) 176 | self._ioloop.update_handler(self._sock, self._flag) 177 | -------------------------------------------------------------------------------- /router/serv/errors.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | 3 | 4 | class ServerErrs(Exception): 5 | state = '500' 6 | content = 'Gateway (500):internal error' 7 | 8 | 9 | class ServerGone(ServerErrs): 10 | state = '502' 11 | content = 'Gateway (502):resource is temporary unavailable' 12 | -------------------------------------------------------------------------------- /router/serv/trie.py: -------------------------------------------------------------------------------- 1 | # coding: utf8 2 | import re 3 | 4 | 5 | class ServNode(object): 6 | __slots__ = ['kw', 'reg', 'servers', 'leafs'] 7 | def __init__(self, kw): 8 | self.kw = kw 9 | self.reg = re.compile(kw) # regex 10 | self.servers = [] 11 | self.leafs = [] 12 | 13 | 14 | def train(current_node, words, server): 15 | if not words: 16 | current_node.servers.append(server) 17 | current_node.servers.reverse() 18 | return 19 | else: 20 | for serv_node in current_node.leafs: 21 | if words[0] == serv_node.kw: 22 | train(serv_node, words[1:], server) 23 | break 24 | else: 25 | serv_node = ServNode(words[0]) 26 | current_node.leafs.append(serv_node) 27 | train(serv_node, words[1:], server) 28 | 29 | 30 | def search_server(current_node, words, params=[]): 31 | if not words: 32 | return current_node.servers 33 | for serv_node in current_node.leafs: 34 | m = re.match(serv_node.reg, words[0]) 35 | if m: 36 | params += m.groups() 37 | return search_server(serv_node, words[1:], params) 38 | else: 39 | return 40 | 41 | 42 | if __name__ == '__main__': 43 | begin = ServNode('ROOT') 44 | urls = [ 45 | ['path', 'com', '([a-zA-Z]+)', '(\d+)'], 46 | ['path', 'com', '400', 'ok'], 47 | ['other', 'st'] 48 | ] 49 | # train(begin, urls[0], 'serv#1') 50 | # train(begin, urls[0], 'serv#12') 51 | # train(begin, urls[1], 'serv#2') 52 | # train(begin, urls[2], 'serv#3') 53 | params = [] 54 | 55 | node = search_server(begin, ['path', 'tt', 'www', '71'], params) 56 | print '===============================' 57 | if node: 58 | print '[result] = ', node, params 59 | -------------------------------------------------------------------------------- /router/var/zanwei.txt: -------------------------------------------------------------------------------- 1 | 1 --------------------------------------------------------------------------------