├── .gitignore ├── README.md ├── cli.py ├── conf ├── __init__.py ├── conf.ini ├── debug.ini └── pre.ini ├── controllers ├── Plan.py └── Test.py ├── library ├── Classes.py ├── Decorate.py ├── G.py ├── Middleware.py ├── MyElasticsearch.py ├── MyMongo.py ├── MyRabbitmq.py ├── MyRedis.py └── Utils.py ├── mapper ├── UserDO.py └── __init__.py ├── models ├── Base.py └── Test.py ├── pyclient └── execute.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.pyc 3 | conf/__pycache__/ 4 | controllers/__pycache__/ 5 | library/__pycache__/ 6 | logs/ 7 | mapper/__pycache__/ 8 | dao/__pycache__/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 注意：最近更新的代码中有部分类有个改动，使用方法截图中的代码已经不适用，具体使用方法见controllers目录下的两个demo文件即可 2 | 3 | 这是一个基于python3.5的脚本开发脚手架，集成了mysql(sqlalchemy), rabbitmq(pika), redis(redis)等常用的中间件，快速开发业务。 4 | 具体的使用方法见 controllers 目录下的 Test.py 文件 5 | 6 | **配置环境** 7 | 此脚手架使用openstack的 oslo.config 作为配置管理，目前区分三个环境，分别是线上环境(conf)，开发环境(debug)，预览环境(pre)。 指定环境有三种方式： 8 | 9 | - 方式一：使用 --config-file=conf/xxx.ini 参数来读取指定配置。 10 | - 方式二：设置 UNITYMOB_ENVIRON 环境变量，变量值分别为 conf, debug, pre ，分别对应上述三个环境。 11 | - 方式三：如果不指定配置文件，也没有设置环境变量，则默认使用conf环境的配置。 12 | 注意：如果同时设置了方式一和方式二，方式一的优先级级别最高。 13 | 14 | **使用方法** 15 | - python3.5 cli.py --executer=test/get_all_user 16 | - test_rabbitmq为一个rabbitmq 的worker端：python3.5 cli.py --executer=test/test_rabbitmq 17 | 18 | - 同时也支持给方法传参数，方法如下： 19 | python3.5 cli.py --executer=plan --args=one,two --kwargs=three:hello 20 | 21 | **使用方法截图** 22 | ![Alt text](https://github.com/xiaowan/pyadmin/blob/master/snapshot/logincontroller.jpeg) 23 | -------------------------------------------------------------------------------- /cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import re 4 | import sys 5 | 6 | from pyclient.execute import main 7 | 8 | if __name__ == '__main__': 9 | sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) 10 | sys.exit(main()) -------------------------------------------------------------------------------- /conf/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from oslo_config import cfg 4 | from oslo_log import log as logging 5 | from os.path import join, dirname 6 | from os import environ 7 | 8 | conf = cfg.CONF 9 | log = logging.getLogger(__name__) 10 | logging.register_options(conf) 11 | 12 | # common 13 | default_opts = [ 14 | cfg.StrOpt(name="environ", default="UNITYMOB_ENVIRON"), # 环境变量key 15 | cfg.StrOpt(name="env", default="conf"), # 环境变量key 16 | cfg.ListOpt(name="args", default=[]), # 外部位置参数 17 | cfg.DictOpt(name="kwargs", default={}), # 外部字典参数 18 | cfg.StrOpt(name="executer", default=None) # 逻辑执行者 19 | ] 20 | conf.register_cli_opts(default_opts) 21 | 22 | # sqlalchemy 23 | sqlalchemy = cfg.OptGroup(name='sqlalchemy', title="MySQL ORM 相关配置") 24 | conf.register_group(sqlalchemy) 25 | conf.register_cli_opts([ 26 | cfg.BoolOpt('echo', default=True), 27 | cfg.BoolOpt('autoflush', default=True), 28 | cfg.IntOpt('pool_size', 10), 29 | cfg.IntOpt('pool_recycle', 3600) 30 | ], sqlalchemy) 31 | 32 | # email 33 | email = cfg.OptGroup(name='email', title="邮件服务相关配置") 34 | conf.register_group(email) 35 | conf.register_cli_opts([ 36 | cfg.StrOpt("host", default="smtp.mxhichina.com"), 37 | cfg.IntOpt("port", default=465), 38 | cfg.StrOpt("user", default="zybi@jollycorp.com"), 39 | cfg.StrOpt("password", default="jolly@2015") 40 | ], email) 41 | 42 | # rabbitmq 43 | rabbitmq = cfg.OptGroup(name='rabbitmq', title="RabbitMq 相关配置") 44 | conf.register_group(rabbitmq) 45 | conf.register_cli_opts([ 46 | cfg.StrOpt('dsn', default=''), 47 | ], rabbitmq) 48 | 49 | # mysql 50 | mysql = cfg.OptGroup(name='mysql', title="MySQL DSN配置") 51 | conf.register_group(mysql) 52 | conf.register_cli_opts([ 53 | cfg.StrOpt('unitymob', default='localhost'), 54 | cfg.StrOpt('jolly_brands', default='localhost'), 55 | ], mysql) 56 | 57 | # facebook 58 | facebook = cfg.OptGroup(name='facebook', title="FaceBook 相关配置") 59 | conf.register_group(facebook) 60 | conf.register_cli_opts([ 61 | cfg.StrOpt('app_id', default=''), 62 | cfg.StrOpt('app_secret', default=''), 63 | cfg.StrOpt('access_token', default=''), 64 | ], facebook) 65 | 66 | # redis 67 | redis = cfg.OptGroup(name='redis', title="Redis 相关配置") 68 | conf.register_group(redis) 69 | conf.register_cli_opts([ 70 | cfg.StrOpt('host', default='127.0.0.1'), 71 | cfg.IntOpt('port', default=6379), 72 | cfg.StrOpt('password', default='unitymob'), 73 | ], redis) 74 | 75 | env = environ.get(conf.environ, 'conf') 76 | env = env if env in ['debug', 'pre', 'conf'] else 'conf' 77 | conf(default_config_files=[join(dirname(__file__), '.'.join([env, 'ini']))]) 78 | 79 | logging.setup(conf, "pyclient") -------------------------------------------------------------------------------- /conf/conf.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | environ= UNITYMOB_ENVIRON 3 | env = conf 4 | 5 | [sqlalchemy] 6 | echo = False 7 | autoflush = True 8 | pool_size = 100 9 | pool_recycle = 3600 10 | 11 | [email] 12 | host = smtp.exmail.qq.com 13 | port = 465 14 | user = username 15 | password = password 16 | 17 | [mysql] 18 | unitymob = 'mysql://username:password@.0.0.0.0/db?charset=utf8mb4' 19 | 20 | [rabbitmq] 21 | dsn = 'amqp://username:password@0.0.0.0:5672/vhosts' 22 | 23 | [redis] 24 | host = '127.0.0.1' 25 | port = 6379 26 | password = password 27 | -------------------------------------------------------------------------------- /conf/debug.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | environ= UNITYMOB_ENVIRON 3 | env=debug 4 | 5 | [sqlalchemy] 6 | echo = False 7 | autoflush = True 8 | pool_size = 100 9 | pool_recycle = 3600 10 | 11 | [email] 12 | host = smtp.exmail.qq.com 13 | port = 465 14 | user = user 15 | password = password 16 | 17 | [mysql] 18 | unitymob = 'mysql://root:root@127.0.0.1/authdb?charset=utf8mb4' 19 | 20 | [rabbitmq] 21 | dsn = 'amqp://username:password@0.0.0.0:5672/unitymob' 22 | 23 | [redis] 24 | host = '127.0.0.1' 25 | port = 6379 26 | password = unitymob 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /conf/pre.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | environ= UNITYMOB_ENVIRON 3 | env=pre 4 | 5 | [sqlalchemy] 6 | echo = False 7 | autoflush = True 8 | pool_size = 100 9 | pool_recycle = 3600 10 | 11 | [email] 12 | host = smtp.exmail.qq.com 13 | port = 465 14 | user = username 15 | password = password 16 | 17 | [mysql] 18 | unitymob = 'mysql://username:password@0.0.0.0/db?charset=utf8mb4' 19 | 20 | [rabbitmq] 21 | dsn = 'amqp://username:password@0.0.0.0:5672/vhosts' 22 | 23 | [redis] 24 | host = '127.0.0.1' 25 | port = 6379 26 | password = password 27 | -------------------------------------------------------------------------------- /controllers/Plan.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from library.Classes import Classes 4 | 5 | 6 | class PlanController(Classes.BaseMinix): 7 | default_method = 'test' 8 | 9 | def test(self, one, two, three="hello"): 10 | print(one) 11 | print(two) 12 | print(three) 13 | -------------------------------------------------------------------------------- /controllers/Test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.5 2 | 3 | from conf import conf 4 | from library.Classes import Classes 5 | from models.Test import TestModel 6 | from library.Middleware import rabbitmqWorkerFactory, Clear, TimeExpense 7 | import time 8 | 9 | 10 | class TestController(Classes.BaseMinix): 11 | default_method = 'get_all_user' # 该属性为默认执行该类的业务处理方法 12 | 13 | def __init__(self): 14 | self.testModel = TestModel.getInstance() 15 | super().__init__() 16 | 17 | def prepare(self): 18 | print("先于执行业务方法执行的逻辑") 19 | 20 | @Clear 21 | def get_all_user(self): 22 | """ 23 | Clear装饰器可手动控制资源的释放 24 | 如不使用该装饰器则默认由框架自动管理 25 | """ 26 | users = self.testModel.get_all_user() 27 | for user in users: 28 | print(user.nickname + '---' + user.loginname) 29 | 30 | def test_rabbitmq(self): 31 | @rabbitmqWorkerFactory(conf.rabbitmq.dsn, 'exchange_name', 'queue_name') 32 | def consum_mq(ch, method, properties, body): 33 | """ 在这里处理队列信息 """ 34 | print(body) 35 | 36 | @TimeExpense 37 | def test_time_expense(self): 38 | """ 方法执行时间函数 """ 39 | time.sleep(3) 40 | 41 | def test_params(self, one, two, three="hello"): 42 | print(one) 43 | print(two) 44 | print(three) 45 | -------------------------------------------------------------------------------- /library/Classes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from .G import G 4 | from .Decorate import DI 5 | import logging 6 | 7 | LOGGER = logging.getLogger(__file__) 8 | 9 | 10 | class Classes(object): 11 | class SingletonMinix(object): 12 | """ 单例 """ 13 | _instance = None 14 | 15 | @classmethod 16 | def getInstance(cls): 17 | if cls._instance is None: 18 | cls._instance = cls() 19 | return cls._instance 20 | 21 | @DI(g=G.getInstance()) 22 | class BaseMinix(SingletonMinix): 23 | """ 所有业务相关类的基类 """ 24 | 25 | @property 26 | def session(self): 27 | return self.g.session 28 | 29 | @property 30 | def rabbitmq(self): 31 | return self.g.rabbitmq 32 | 33 | @property 34 | def utils(self): 35 | return self.g.utils 36 | 37 | @property 38 | def log(self): 39 | return self.g.log 40 | 41 | @property 42 | def conf(self): 43 | return self.g.conf 44 | 45 | @property 46 | def mongo(self): 47 | return self.g.mongo 48 | 49 | @property 50 | def es(self): 51 | return self.g.es 52 | 53 | @property 54 | def redis(self): 55 | return self.g.redis 56 | 57 | def prepare(self): 58 | """ 如果有需要预先执行的逻辑，写在prepare方法中 """ 59 | pass 60 | -------------------------------------------------------------------------------- /library/Decorate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | 工具类装饰器 5 | """ 6 | 7 | import sys 8 | import time 9 | from oslo_utils import timeutils 10 | from conf import log 11 | 12 | from sqlalchemy.orm.session import Session 13 | from mapper import UnitymobSession 14 | 15 | 16 | def DI(**kwargs): 17 | """ 注入装饰 """ 18 | 19 | def outer(cls): 20 | for x in kwargs: 21 | setattr(cls, x, kwargs.get(x)) 22 | return cls 23 | 24 | return outer 25 | 26 | 27 | def TimeExpense(func): 28 | """ 统计 job 执行时长 """ 29 | 30 | def _deco(*args, **kwargs): 31 | with timeutils.StopWatch() as w: 32 | func(*args, **kwargs) 33 | log.info("脚本运行结束,共计耗时 : {expense} 秒".format(expense=int(w.elapsed()))) 34 | sys.stdout.flush() 35 | 36 | return _deco 37 | 38 | 39 | def Transaction(name=None): 40 | """ 41 | 声明式事务,该方法只能使用在对方法上 42 | 特性 : 43 | a. 支持直接传入session对象(暂无实现) 44 | b. 传入session对象对应的类书属性名称 45 | 如果出现exception ,直接上抛异常给调用方 46 | """ 47 | 48 | def outer(func): 49 | def _deco(self, *args, **kwargs): 50 | if name is not None and hasattr(self, name): 51 | session = getattr(self, name) 52 | if isinstance(session, Session): 53 | try: 54 | res = func(self, *args, **kwargs) 55 | session.commit() 56 | return res 57 | except Exception as e: 58 | session.rollback() 59 | raise e 60 | finally: 61 | pass 62 | 63 | return _deco 64 | 65 | return outer 66 | 67 | 68 | def session_init(): 69 | def outer(func): 70 | def _deco(self, *args, **kwargs): 71 | try: 72 | session = UnitymobSession() 73 | setattr(self, 'session', session) 74 | res = func(self, *args, **kwargs) 75 | return res 76 | except Exception as e: 77 | raise e 78 | finally: 79 | session.close() 80 | 81 | return _deco 82 | 83 | return outer 84 | -------------------------------------------------------------------------------- /library/G.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import threading 4 | 5 | from conf import conf, log 6 | from library.Utils import Utils 7 | from library.MyRabbitmq import MyRabbitmq 8 | from library.MyRedis import MyRedis 9 | from library.MyMongo import MyMongo 10 | from library.MyElasticsearch import MyElasticsearch 11 | 12 | from mapper import UnitymobSession 13 | 14 | 15 | class G(object): 16 | """ 全局类 """ 17 | _instance = None 18 | _cleard = False 19 | 20 | def __init__(self): 21 | self.thread_local = threading.local() 22 | self.conf = conf 23 | self.utils = Utils 24 | self.log = log 25 | 26 | @property 27 | def is_cleard(self): 28 | """ 请求结束，是否已经清理结尾 """ 29 | return self._cleard 30 | 31 | @classmethod 32 | def getInstance(cls): 33 | if cls._instance is None: 34 | cls._instance = cls() 35 | return cls._instance 36 | 37 | @property 38 | def session(self): 39 | if not hasattr(self.thread_local, '_session'): 40 | self.thread_local._session = UnitymobSession() 41 | return self.thread_local._session 42 | 43 | @property 44 | def rabbitmq(self): 45 | """ rabbitmq 操作句柄 """ 46 | return MyRabbitmq.getInstance(conf.rabbitmq.dsn) 47 | 48 | @property 49 | def redis(self): 50 | return MyRedis.getInstance(conf.redis.host, conf.redis.port, conf.redis.password, False) 51 | 52 | @property 53 | def mongo(self): 54 | return MyMongo.getInstance(conf.mongo.dsn).get_mongodb_client 55 | 56 | @property 57 | def es(self): 58 | return MyElasticsearch.getInstance(conf.elasticsearch.dsn).get_es_client 59 | 60 | def close(self): 61 | """ 善后清理工作 """ 62 | # mysql 63 | if hasattr(self.thread_local, '_session'): 64 | self.thread_local._session.close() 65 | 66 | # rabbitmq 67 | del self.rabbitmq.channel 68 | 69 | self._cleard = True 70 | -------------------------------------------------------------------------------- /library/Middleware.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | 各种中间件装饰器 5 | """ 6 | 7 | import sys 8 | import json 9 | import time 10 | import threading 11 | import traceback 12 | import pika 13 | import paho.mqtt.client as mqtt 14 | from oslo_context import context 15 | from conf import conf, log 16 | from library.Utils import Utils 17 | from library.G import G 18 | from library.Decorate import TimeExpense 19 | from facebookads.api import FacebookAdsApi 20 | 21 | 22 | def Clear(func): 23 | """ 负责清理中间遇到的所有资源问题，目前已经不使用 """ 24 | 25 | def _deco(*args, **kwargs): 26 | try: 27 | func(*args, **kwargs) 28 | except Exception: 29 | raise 30 | finally: 31 | G.getInstance().close() 32 | 33 | return _deco 34 | 35 | 36 | def ApiInit(func): 37 | """ Facebook Api 初始化装饰器 """ 38 | 39 | def _deco(*args, **kwargs): 40 | access_token = None 41 | try: 42 | body = json.loads(args[3].decode()) 43 | access_token = body.get('access_token') 44 | except Exception as ex: 45 | access_token = conf.facebook.access_token 46 | 47 | finally: 48 | FacebookAdsApi.init(conf.facebook.app_id, conf.facebook.app_secret, access_token) 49 | func(*args, **kwargs) 50 | 51 | return _deco 52 | 53 | 54 | def rabbitmqWorkerFactory(dsn='', exchange='', queue=''): 55 | """ rabbitmq """ 56 | print("执行rabbitmq 预初始工作") 57 | 58 | def outer(func): 59 | print("开始连接rabbitmq") 60 | # connection mq 61 | connection = pika.BlockingConnection(pika.URLParameters(dsn)) 62 | channel = connection.channel() 63 | 64 | # declare exchange 65 | channel.exchange_declare(exchange=exchange, exchange_type='direct', durable=True) 66 | 67 | # declare queue 68 | result = channel.queue_declare(queue=queue, durable=True) 69 | queue_name = result.method.queue 70 | 71 | channel.queue_bind(exchange=exchange, queue=queue_name) 72 | 73 | # config for rabbitmq'worker 74 | channel.basic_qos(prefetch_count=1) 75 | 76 | @TimeExpense 77 | def _deco(ch, method, properties, body): 78 | try: 79 | context.RequestContext() 80 | log.info("开始执行业务方法") 81 | func(ch, method, properties, body) 82 | except Exception as e: 83 | print(e) 84 | if conf.env == 'conf': 85 | mail_title = Utils.currentTime() + " " + sys.argv[0] + "执行异常,异常内容见邮件" 86 | Utils.sendMail(mail_title, traceback.format_exc(), [ 87 | '邮箱地址' 88 | ]) 89 | else: 90 | traceback.print_exc() 91 | log.info("业务方法异常") 92 | 93 | finally: 94 | log.info("队列消息确认消费") 95 | ch.basic_ack(delivery_tag=method.delivery_tag) 96 | 97 | sys.stdout.flush() 98 | channel.basic_consume(_deco, queue=queue, no_ack=False) 99 | 100 | channel.start_consuming() 101 | 102 | class Heartbeat(threading.Thread): 103 | def __init__(self, connection): 104 | super(Heartbeat, self).__init__() 105 | self.lock = threading.Lock() 106 | self.connection = connection 107 | self.quitflag = False 108 | self.stopflag = True 109 | self.setDaemon(True) 110 | 111 | def run(self): 112 | while not self.quitflag: 113 | time.sleep(10) 114 | self.lock.acquire() 115 | if self.stopflag: 116 | self.lock.release() 117 | continue 118 | try: 119 | self.connection.process_data_events() 120 | except Exception as ex: 121 | self.lock.release() 122 | return 123 | self.lock.release() 124 | 125 | def startHeartbeat(self): 126 | self.lock.acquire() 127 | if self.quitflag == True: 128 | self.lock.release() 129 | return 130 | self.stopflag = False 131 | self.lock.release() 132 | 133 | heartbeat = Heartbeat(connection) 134 | heartbeat.start() 135 | heartbeat.startHeartbeat() 136 | 137 | return outer 138 | 139 | 140 | def mqttWorkerFactory(hostname='', port=0, username='', password='', topic='#'): 141 | """ mqtt 工厂方法 """ 142 | 143 | def outer(func): 144 | print("开始连接mqtt协议...") 145 | 146 | # connection mq 147 | 148 | def on_connect(client, userdata, flags, rc): 149 | print("Connected with result code " + str(rc)) 150 | client.subscribe(topic) 151 | 152 | @TimeExpense 153 | def _deco(client, userdata, msg): 154 | try: 155 | print("\n" + Utils.currentTime() + " 开始执行业务方法") 156 | func(client, userdata, msg) 157 | except Exception as e: 158 | print("业务方法异常") 159 | 160 | client = mqtt.Client() 161 | client.reinitialise(client_id="xxx", clean_session=False, userdata=None) 162 | client.on_connect = on_connect 163 | client.on_message = _deco 164 | 165 | try: 166 | client.username_pw_set(username=username, password=password) 167 | client.connect(hostname, port, 60) 168 | client.loop_forever() 169 | except Exception as ex: 170 | raise ex 171 | print("mqtt服务异常") 172 | 173 | return outer 174 | -------------------------------------------------------------------------------- /library/MyElasticsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | from elasticsearch import Elasticsearch 5 | 6 | 7 | class MyElasticsearch(object): 8 | """ elasticsearch 连接 """ 9 | _instance = None 10 | 11 | def __init__(self, dsn=None): 12 | self.dsn = dsn if isinstance(dsn, list) else [dsn] 13 | self._client = None 14 | 15 | @classmethod 16 | def getInstance(cls, dsn=None): 17 | if cls._instance is None: 18 | cls._instance = cls(dsn=dsn) 19 | return cls._instance 20 | 21 | @property 22 | def get_es_client(self): 23 | if self._client is None: 24 | self._client = Elasticsearch(self.dsn) 25 | return self._client 26 | -------------------------------------------------------------------------------- /library/MyMongo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from pymongo import MongoClient 4 | 5 | 6 | class MyMongo(object): 7 | """ mongodb 连接 """ 8 | 9 | _instance = None 10 | 11 | def __init__(self, dsn=None): 12 | self.dsn = dsn 13 | self._mongodb_client = None 14 | 15 | @classmethod 16 | def getInstance(cls, dsn=None): 17 | if cls._instance is None: 18 | cls._instance = cls(dsn=dsn) 19 | return cls._instance 20 | 21 | def close(self): 22 | if self._mongodb_client is not None: 23 | self._mongodb_client.close() 24 | 25 | @property 26 | def get_mongodb_client(self): 27 | if self._mongodb_client is None: 28 | self._mongodb_client = MongoClient(self.dsn) 29 | return self._mongodb_client 30 | -------------------------------------------------------------------------------- /library/MyRabbitmq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | 自定义rabbitmq 5 | """ 6 | 7 | import pika 8 | import json 9 | 10 | 11 | class MyRabbitmq(object): 12 | _instance = None 13 | _connection = None 14 | _channel = None 15 | 16 | @classmethod 17 | def getInstance(cls, dsn): 18 | if cls._instance is None: 19 | cls._instance = cls(dsn) 20 | return cls._instance 21 | 22 | def __init__(self, dsn): 23 | self.dsn = dsn 24 | 25 | @property 26 | def connection(self): 27 | if self._connection is None: 28 | self._connection = pika.BlockingConnection(pika.URLParameters(self.dsn)) 29 | else: 30 | if self._connection.is_closed: 31 | self._connection = pika.BlockingConnection(pika.URLParameters(self.dsn)) 32 | 33 | return self._connection 34 | 35 | @property 36 | def channel(self): 37 | if self._channel is None: 38 | try: 39 | self._channel = self.connection.channel() 40 | except Exception as ex: 41 | self._connection = None 42 | self._channel = self.connection.channel() 43 | 44 | return self._channel 45 | 46 | @channel.deleter 47 | def channel(self): 48 | if self._channel is not None: 49 | try: 50 | if not self._channel.is_closed: 51 | self._channel.close() 52 | except Exception as ex: 53 | pass 54 | finally: 55 | self._channel = None 56 | 57 | def publish(self, exchange=None, routing_key=None, message={}): 58 | """ 发布消息 """ 59 | return self.channel.basic_publish(exchange=exchange, routing_key=routing_key, body=json.dumps(message)) 60 | 61 | def close(self): 62 | """ 关闭连接 """ 63 | self.connection.close() 64 | -------------------------------------------------------------------------------- /library/MyRedis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | __author__ = "yishan@jollycorp.com" 4 | 5 | import redis 6 | 7 | class MyRedis(object): 8 | """ redis config """ 9 | 10 | _instance = None 11 | 12 | redis 13 | 14 | def __init__(self, host, port, password, decode_responses): 15 | self.redis = redis.StrictRedis(host=host, port=port, password=password, decode_responses=decode_responses) 16 | 17 | @classmethod 18 | def getInstance(cls, host, port, password, decode_responses): 19 | if cls._instance is None: 20 | cls._instance = cls(host, port, password, decode_responses) 21 | return cls._instance 22 | 23 | def get(self, key): 24 | return self.redis.get(key) 25 | 26 | def set(self, name, value): 27 | return self.redis.set(name, value) 28 | 29 | def incr(self, name, amount=1): 30 | return self.redis.incr(name, amount) 31 | 32 | def decr(self, name, amount=1): 33 | return self.redis.decr(name, amount) 34 | 35 | def keys(self, pattern='*'): 36 | return self.redis.keys(pattern='*') 37 | 38 | def delete(self, *names): 39 | return self.redis.delete(*names) 40 | 41 | def getPipe(self, transaction=True, shard_hint=None): 42 | return self.redis.pipeline(transaction=transaction, shard_hint=shard_hint) 43 | 44 | def close(self): 45 | """ 官方暂无实现 """ 46 | pass -------------------------------------------------------------------------------- /library/Utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import pika 5 | import time 6 | import pytz 7 | import datetime 8 | import hashlib 9 | import smtplib 10 | from email.mime.text import MIMEText 11 | from email.header import Header 12 | 13 | from mapper import UnitymobSession 14 | from contextlib import contextmanager 15 | 16 | from .MyRedis import MyRedis 17 | from .MyRabbitmq import MyRabbitmq 18 | from conf import conf 19 | 20 | 21 | class Utils(object): 22 | @staticmethod 23 | def rabbitmqWorkerFactory(dsn='', exchange='', queue='', callback=None): 24 | # connection mq 25 | connection = pika.BlockingConnection(pika.URLParameters(dsn)) 26 | channel = connection.channel() 27 | 28 | # declare exchange 29 | channel.exchange_declare(exchange=exchange, type='direct', durable=True) 30 | 31 | # declare queue 32 | result = channel.queue_declare(queue=queue, durable=True) 33 | queue_name = result.method.queue 34 | 35 | channel.queue_bind(exchange=exchange, queue=queue_name) 36 | 37 | # config for rabbitmq'worker 38 | channel.basic_qos(prefetch_count=1) 39 | channel.basic_consume(callback, queue=queue, no_ack=False) 40 | 41 | channel.start_consuming() 42 | 43 | @staticmethod 44 | def utc_to_local(utc_time_str, utc_format='%Y-%m-%dT%H:%M:%S%z'): 45 | local_format = "%Y-%m-%d %H:%M:%S" 46 | time_str = Utils.utc_to_str(utc_time_str=utc_time_str, utc_format=utc_format, local_format=local_format) 47 | return int(time.mktime(time.strptime(time_str, local_format))) 48 | 49 | @staticmethod 50 | def utc_to_str(utc_time_str, utc_format='%Y-%m-%dT%H:%M:%S%z', local_format="%Y-%m-%d %H:%M:%S"): 51 | local_tz = pytz.timezone('Asia/Chongqing') 52 | utc_dt = datetime.datetime.strptime(utc_time_str, utc_format) 53 | local_dt = utc_dt.replace(tzinfo=pytz.utc).astimezone(local_tz) 54 | time_str = local_dt.strftime(local_format) 55 | return time_str 56 | 57 | @staticmethod 58 | def theOtherDay(separation=1): 59 | """ 计算当前时间前几天的时间 """ 60 | return datetime.date.today() + datetime.timedelta(days=separation) 61 | 62 | @staticmethod 63 | def strtimeToUTC(utc_time_str, utc_format='%Y-%m-%dT%H:%M:%S%z'): 64 | return datetime.datetime.strptime(utc_time_str, utc_format) 65 | 66 | @staticmethod 67 | def currentTime(format='%Y-%m-%d %H:%M:%S'): 68 | return time.strftime(format, time.localtime(int(time.time()))) 69 | 70 | @staticmethod 71 | @contextmanager 72 | def rabbitmq(): 73 | """ redis context """ 74 | mq = MyRabbitmq(conf.rabbitmq.dsn) 75 | try: 76 | yield mq 77 | finally: 78 | mq.close() 79 | 80 | @staticmethod 81 | def getFileNameByPath(filePath): 82 | return os.path.split(filePath)[-1] 83 | 84 | @staticmethod 85 | def md5(str): 86 | m2 = hashlib.md5() 87 | m2.update("{str}".format(str=str).encode("utf-8")) 88 | return m2.hexdigest() 89 | 90 | @staticmethod 91 | def sendMail(title="", msg="", receiver=[]): 92 | """ 93 | 发送邮件 94 | :param title: 标题 95 | :param msg: 邮件内容 96 | :param receiver: 列表 所有邮件接受者 97 | :return: void 98 | """ 99 | email = conf.email 100 | try: 101 | message = MIMEText(msg, 'plain', 'utf-8') 102 | message['from'] = email.user 103 | message['to'] = ';'.join(receiver) 104 | message['subject'] = Header(title, 'utf-8') 105 | 106 | smtp = smtplib.SMTP_SSL(email.host, email.port) 107 | smtp.set_debuglevel(0) 108 | smtp.login(email.user, email.password) 109 | smtp.sendmail(email.user, receiver, message.as_string()) 110 | smtp.close() 111 | except Exception as e: 112 | print("报警邮件发送异常,生无可恋...") 113 | 114 | @staticmethod 115 | def debug(data): 116 | print(" ================== start debug ================== ") 117 | print(data) 118 | print(" ================== end debug ================== ") 119 | exit() 120 | -------------------------------------------------------------------------------- /mapper/UserDO.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.5 2 | 3 | from . import BaseDO 4 | from sqlalchemy import Column 5 | from sqlalchemy.types import * 6 | 7 | 8 | class UserDO(BaseDO): 9 | __tablename__ = 'py_user' 10 | 11 | nickname = Column(VARCHAR, default=None) 12 | loginname = Column(VARCHAR, default=None) 13 | avatar = Column(VARCHAR, default=None) 14 | is_valid = Column(VARCHAR, default=None) 15 | -------------------------------------------------------------------------------- /mapper/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from io import StringIO 4 | from os import linesep 5 | from conf import conf 6 | from sqlalchemy.orm import sessionmaker 7 | from sqlalchemy import create_engine 8 | from sqlalchemy.ext.declarative import declarative_base 9 | from sqlalchemy import Column, String, TIMESTAMP, Integer, TEXT 10 | import time 11 | 12 | 13 | sa = conf.sqlalchemy 14 | 15 | db_engine = create_engine(conf.mysql.unitymob, echo=sa.echo, 16 | pool_size=sa.pool_size, 17 | pool_recycle=sa.pool_recycle) 18 | 19 | UnitymobSession = sessionmaker(bind=db_engine, autoflush=sa.autoflush) 20 | 21 | 22 | class OriginalMapper(object): 23 | __table_args__ = { 24 | 'mysql_charset': 'utf8mb4' 25 | } 26 | 27 | @property 28 | def dict(self): 29 | return {c.name: getattr(self, c.name, None) for c in self.__table__.columns} 30 | 31 | def __str__(self) -> str: 32 | string = StringIO() 33 | string.writelines("============={classname}=============".format(classname=self.__class__.__name__) + linesep ) 34 | for x in self.__dict__.keys(): 35 | if x[0:1] == '_': continue 36 | string.writelines("{key} 的值为 {value}".format(key=x, value=self.__dict__[x]) + linesep ) 37 | string.seek(0) 38 | desc = string.getvalue() 39 | string.close() 40 | return desc 41 | 42 | class BaseMapper(OriginalMapper): 43 | id = Column(Integer, primary_key=True, autoincrement=True) 44 | create_time = Column(Integer) 45 | 46 | BaseDO = declarative_base(cls=BaseMapper) 47 | 48 | -------------------------------------------------------------------------------- /models/Base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.5 2 | 3 | from library.Decorate import Transaction 4 | from library.Classes import Classes 5 | 6 | 7 | class BaseModel(Classes.BaseMinix): 8 | 9 | @Transaction(name="session") 10 | def save(self, obj): 11 | """ 保存对象，支持批量写入""" 12 | if isinstance(obj, list): 13 | res = self.session.add_all(obj) 14 | else: 15 | res = self.session.add(obj) 16 | self.session.flush() 17 | return res 18 | -------------------------------------------------------------------------------- /models/Test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.5 2 | 3 | from .Base import BaseModel 4 | from mapper.UserDO import UserDO 5 | 6 | 7 | class TestModel(BaseModel): 8 | def get_all_user(self): 9 | return self.session.query(UserDO).all() 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /pyclient/execute.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from conf import conf 4 | from library.G import G 5 | from contextlib import closing 6 | 7 | 8 | class Execute(object): 9 | 10 | def __init__(self): 11 | self._executor = None 12 | self._method = None 13 | self._analyze_executor(conf.executer) 14 | self._args = conf.args 15 | self._kwargs = conf.kwargs 16 | 17 | def _analyze_executor(self, invoker): 18 | """ 分析执行者 """ 19 | _invoker = invoker.split('/', 1) 20 | if len(_invoker) == 1: 21 | self._executor = _invoker[0] 22 | 23 | if len(_invoker) == 2: 24 | self._executor = _invoker[0] 25 | self._method = _invoker[1] 26 | 27 | def run(self): 28 | """ 开始执行 """ 29 | invoker_prefix = str(self._executor).capitalize() 30 | controller = "{invoker}Controller".format(invoker=invoker_prefix) 31 | 32 | executor = __import__("controllers.{invoker}".format(invoker=invoker_prefix), fromlist=controller) 33 | executor_obj = eval("executor.{controller}.getInstance()".format(controller=controller)) 34 | 35 | if self._method is None: 36 | if hasattr(executor_obj, 'default_method'): 37 | self._method = executor_obj.default_method.lower() 38 | 39 | try: 40 | executor_obj.prepare() 41 | method = getattr(executor_obj, self._method) 42 | method(*self._args, **self._kwargs) 43 | except Exception: 44 | raise 45 | finally: 46 | if not G.getInstance().is_cleard: 47 | with closing(G.getInstance()): 48 | pass 49 | 50 | 51 | def main(): 52 | try: 53 | Execute().run() 54 | return 0 55 | except Exception as ex: 56 | print(ex) 57 | finally: 58 | return 1 59 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | amqp==1.4.9 2 | elasticsearch2==2.5.0 3 | facebook-business==3.2.3 4 | mysqlclient==1.3.7 5 | oslo.config==6.2.1 6 | oslo.context==2.20.0 7 | oslo.db==4.20.0 8 | oslo.i18n==3.20.0 9 | oslo.log==3.38.1 10 | oslo.serialization==2.20.0 11 | oslo.utils==3.36.2 12 | paho-mqtt==1.3.1 13 | pika==0.10.0 14 | pymongo==3.6.1 15 | PyMySQL==0.7.11 16 | pyparsing==2.2.0 17 | redis==2.10.5 18 | requests==2.18.4 19 | SQLAlchemy==1.2.0 20 | tornado==4.5.3 21 | --------------------------------------------------------------------------------