├── .gitignore ├── LICENSE ├── README.md ├── core ├── __init__.py ├── server.py └── service.py ├── examples ├── rpyc_util.py ├── schedule_client.py ├── schedule_server.py └── web_server.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.pyc 3 | examples/*.sqlite* 4 | examples/*.log -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 玖亖伍 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 基于apscheduler和rpyc的调度系统(schedule-system)示例 2 | 3 | ## 目录结构及说明 4 | ```text 5 | ./ 6 | ├── .gitignore ----------------------------------- git忽略文件 7 | ├── core/ ---------------------------------------- 调度核心封装 8 | │ ├── __init__.py ------------------------------ 构成包 9 | │ ├── server.py -------------------------------- rpyc服务器 10 | │ └── service.py ------------------------------- rpyc接口服务 11 | ├── examples/ ------------------------------------ 使用示例 12 | │ ├── jobstore.sqlite -------------------------- 作业存储文件 13 | │ ├── logger.log ------------------------------- 日志文件 14 | │ ├── schedule_client.py ----------------------- 调度示例-客户端 15 | │ ├── schedule_server.py ----------------------- 调度示例-服务端 16 | │ └── web_server.py ---------------------------- Web服务器示例 17 | ├── LICENSE -------------------------------------- 项目许可证 18 | ├── README.md ------------------------------------ 项目说明文件 19 | └── requirements.txt ----------------------------- pip依赖包 20 | ``` -------------------------------------------------------------------------------- /core/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 调度 3 | ''' 4 | from .server import schedule_start 5 | from .service import SchedulerService -------------------------------------------------------------------------------- /core/server.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 调度服务器 3 | ''' 4 | import logging 5 | import time 6 | from datetime import datetime, timedelta 7 | 8 | from rpyc.utils.server import ThreadedServer 9 | from rpyc.utils.helpers import classpartial 10 | from apscheduler.schedulers.background import BackgroundScheduler 11 | from apscheduler.executors.pool import ( 12 | ThreadPoolExecutor, 13 | ProcessPoolExecutor 14 | ) 15 | from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore 16 | from apscheduler.events import ( 17 | EVENT_JOB_EXECUTED, 18 | EVENT_JOB_ERROR, 19 | EVENT_JOB_ADDED, 20 | EVENT_JOB_SUBMITTED, 21 | EVENT_JOB_REMOVED 22 | ) 23 | 24 | from .service import SchedulerService 25 | 26 | 27 | def event_listener(event, scheduler): 28 | code = getattr(event, 'code') 29 | job_id = getattr(event, 'job_id') 30 | jobstore = getattr(event, 'jobstore') 31 | event_codes = { 32 | EVENT_JOB_EXECUTED: 'JOB_EXECUTED', 33 | EVENT_JOB_ERROR: 'JOB_ERROR', 34 | EVENT_JOB_ADDED: 'JOB_ADDED', 35 | EVENT_JOB_SUBMITTED: 'JOB_SUBMITTED', 36 | EVENT_JOB_REMOVED: 'JOB_REMOVED' 37 | } 38 | event_name = None 39 | job = None 40 | need_record = False 41 | if code in event_codes: 42 | need_record = True 43 | event_name = event_codes[code] 44 | job = scheduler.get_job(job_id) 45 | if code == EVENT_JOB_SUBMITTED: 46 | time.sleep(0.05) 47 | elif code in [EVENT_JOB_EXECUTED, EVENT_JOB_ERROR]: 48 | # 执行完成和出错不可能同时出现，故延时一样 49 | time.sleep(0.1) 50 | print('*' * 80) 51 | print(job_id) 52 | print(event_name, datetime.now()) 53 | # print(job) 54 | ''' 55 | if hasattr(event, 'exception') and event.exception: 56 | error, *_ = event.exception.args 57 | tmp = re.sub(r'\([\w.]+\)\s*ORA-\d+:\s*', '', error) 58 | if bool(tmp): 59 | error = tmp 60 | # TODO: log error 61 | ''' 62 | 63 | def modify_logger(logger, log_file): 64 | # refer: https://docs.python.org/3.5/library/logging.html#logrecord-attributes 65 | formatter = logging.Formatter( 66 | fmt='\n'.join([ 67 | '[%(name)s] %(asctime)s.%(msecs)d', 68 | '\t%(pathname)s [line: %(lineno)d]', 69 | '\t%(processName)s[%(process)d] => %(threadName)s[%(thread)d] => %(module)s.%(filename)s:%(funcName)s()', 70 | '\t%(levelname)s: %(message)s\n' 71 | ]), 72 | datefmt='%Y-%m-%d %H:%M:%S' 73 | ) 74 | # stream_handler = logging.StreamHandler() 75 | # stream_handler.setFormatter(formatter) 76 | # logger.addHandler(stream_handler) 77 | 78 | file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8') 79 | file_handler.setFormatter(formatter) 80 | logger.addHandler(file_handler) 81 | 82 | logger.setLevel(logging.DEBUG) 83 | 84 | return logger 85 | 86 | def get_scheduler(store_path=None, log_file=None): 87 | if store_path is None: 88 | store_path = r'jobstore.sqlite' 89 | if log_file is None: 90 | log_file = r'logger.log' 91 | scheduler = BackgroundScheduler({'apscheduler.timezone': 'Asia/Shanghai'}) 92 | jobstores = { 93 | 'default': SQLAlchemyJobStore(url='sqlite:///{0}'.format(store_path)) 94 | } 95 | executors = { 96 | 'default': ThreadPoolExecutor(20), 97 | 'processpool': ProcessPoolExecutor(5) 98 | } 99 | job_defaults = { 100 | 'coalesce': False, 101 | 'max_instances': 1 102 | } 103 | scheduler.configure(jobstores=jobstores, executors=executors) 104 | # 事件记录 105 | scheduler.add_listener( 106 | lambda event: event_listener(event, scheduler), 107 | EVENT_JOB_EXECUTED | EVENT_JOB_ERROR | EVENT_JOB_ADDED | EVENT_JOB_SUBMITTED | EVENT_JOB_REMOVED 108 | ) 109 | # 日志定制 110 | scheduler._logger = modify_logger(scheduler._logger, log_file=log_file) 111 | return scheduler 112 | 113 | def get_server(listen_config, scheduler, SchedulerServiceClass=None): 114 | # 额外构造函数参数 115 | ser_args = ['test args'] 116 | ser_kwargs = {} 117 | # 传递Service构造函数参数 118 | SSC = SchedulerServiceClass if SchedulerServiceClass is not None else SchedulerService 119 | service = classpartial(SSC, scheduler, *ser_args, **ser_kwargs) 120 | # 允许属性访问 121 | protocol_config = {'allow_public_attrs': True} 122 | # 实例化RPYC服务器 123 | server = ThreadedServer(service, protocol_config=protocol_config, **listen_config) 124 | return server 125 | 126 | def schedule_start(listen_config=None, store_path=None, log_file=None, SchedulerServiceClass=None): 127 | # 实例化调度器 128 | scheduler = get_scheduler(store_path=store_path, log_file=log_file) 129 | # 启动调度 130 | scheduler.start() 131 | # 监听配置 132 | if listen_config is None: 133 | listen_config = { 134 | 'port': 12345, 135 | 'hostname': '0.0.0.0' 136 | } 137 | # 实例化服务器 138 | server = get_server(listen_config, scheduler, SchedulerServiceClass=SchedulerServiceClass) 139 | print('rpyc server running at [{hostname}:{port}]'.format(**listen_config)) 140 | try: 141 | # 启动RPYC服务器 142 | server.start() 143 | except (KeyboardInterrupt, SystemExit): 144 | pass 145 | finally: 146 | # 停止调度 147 | scheduler.shutdown() 148 | # 停止RPYC服务器 149 | server.close() 150 | 151 | if __name__ == '__main__': 152 | schedule_start() 153 | -------------------------------------------------------------------------------- /core/service.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 对外接口服务 3 | ''' 4 | import sys 5 | 6 | import rpyc 7 | 8 | 9 | class SchedulerService(rpyc.Service): 10 | def __init__(self, scheduler, args=None, kwargs=None): 11 | self.scheduler = scheduler 12 | print(args, kwargs) 13 | 14 | def on_connect(self, conn): 15 | print('rpyc connect') 16 | pass 17 | 18 | def on_disconnect(self, conn): 19 | print('rpyc disconnect') 20 | pass 21 | 22 | def exposed_add_job(self, func, *args, **kwargs): 23 | return self.scheduler.add_job(func, *args, **kwargs) 24 | 25 | def exposed_modify_job(self, job_id, jobstore=None, **changes): 26 | return self.scheduler.modify_job(job_id, jobstore, **changes) 27 | 28 | def exposed_reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args): 29 | return self.scheduler.reschedule_job(job_id, jobstore, trigger, **trigger_args) 30 | 31 | def exposed_pause_job(self, job_id, jobstore=None): 32 | return self.scheduler.pause_job(job_id, jobstore) 33 | 34 | def exposed_resume_job(self, job_id, jobstore=None): 35 | return self.scheduler.resume_job(job_id, jobstore) 36 | 37 | def exposed_remove_job(self, job_id, jobstore=None): 38 | self.scheduler.remove_job(job_id, jobstore) 39 | 40 | def exposed_remove_all_jobs(self, jobstore=None): 41 | return self.scheduler.remove_all_jobs(jobstore) 42 | 43 | def exposed_get_job(self, job_id, jobstore=None): 44 | return self.scheduler.get_job(job_id, jobstore) 45 | 46 | def exposed_get_jobs(self, jobstore=None): 47 | return self.scheduler.get_jobs(jobstore) 48 | 49 | def exposed_print_jobs(self, jobstore=None): 50 | return self.scheduler.print_jobs(jobstore, out=sys.stdout) 51 | -------------------------------------------------------------------------------- /examples/rpyc_util.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | import json 3 | 4 | import rpyc 5 | from rpyc.core.protocol import Connection 6 | 7 | 8 | class RpycUtil(object): 9 | """RpycUtil-Rpyc客户端工具""" 10 | def __init__(self, host, port): 11 | super(RpycUtil, self).__init__() 12 | self.host = host 13 | self.port = port 14 | self.conn = None 15 | 16 | def connect(self): 17 | self.disconnect() 18 | client_config={ 19 | "allow_all_attrs": True 20 | } 21 | self.conn = rpyc.connect(self.host, self.port, config=client_config) 22 | 23 | def disconnect(self): 24 | if isinstance(self.conn, Connection): 25 | if not self.conn.closed: 26 | self.conn.close() 27 | self.conn = None 28 | 29 | def ensure_connected(func): 30 | @wraps(func) 31 | def func_wrapper(*args): 32 | self = args[0] 33 | if not isinstance(self.conn, Connection) or self.conn.closed: 34 | self.connect() 35 | print('running {0}()'.format(func.__name__)) 36 | # 如果和rpyc服务端断开了，则自动重新连接 37 | try: 38 | return func(*args) 39 | except Exception as e: 40 | if isinstance(e, EOFError): 41 | self.connect() 42 | return func(*args) 43 | else: 44 | raise e 45 | return func_wrapper 46 | 47 | @ensure_connected 48 | def add_job_json(self, params={}): 49 | # 将dict转化成对应的json格式的str, 避免rpyc传递后变成netref对象，导致直接使用时不能被序列化 50 | job_params = json.dumps(params, ensure_ascii=False) 51 | return self.conn.root.add_job_json(job_params) 52 | 53 | @ensure_connected 54 | def get_jobs_json(self): 55 | ret = self.conn.root.get_jobs_json() 56 | return ret 57 | 58 | @ensure_connected 59 | def remove_job(self, job_id): 60 | ret = self.conn.root.remove_job(job_id) 61 | return ret -------------------------------------------------------------------------------- /examples/schedule_client.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import rpyc 4 | from apscheduler.job import Job 5 | from apscheduler.jobstores.base import JobLookupError 6 | 7 | if __name__ == '__main__': 8 | conn = rpyc.connect('localhost', 54345) 9 | # 添加job 10 | job01 = conn.root.api_demo_hello(args=['Hello, World']) 11 | job02 = conn.root.api_add_demo_job() 12 | # 显示所有job 13 | print(conn.root.get_jobs()) 14 | # 显示job01的id 15 | print(job01.id) 16 | # 移除jib01 17 | try: 18 | conn.root.remove_job(job01.id) 19 | except JobLookupError: 20 | pass 21 | time.sleep(5) 22 | print(job02.id) 23 | # 查询job 24 | job03 = conn.root.get_job(job02.id) 25 | if bool(job03): 26 | print(job03, job03.id) 27 | # 删除job03 28 | job03.remove() 29 | # 移除所有job 30 | conn.root.remove_all_jobs() 31 | -------------------------------------------------------------------------------- /examples/schedule_server.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import json 4 | import time 5 | from datetime import datetime, timedelta 6 | 7 | import requests 8 | from apscheduler.job import Job 9 | sys.path.insert(0, '..') 10 | from core import schedule_start, SchedulerService 11 | 12 | 13 | def demo_job(): 14 | print('job start', datetime.now()) 15 | time.sleep(2) 16 | print('job stop', datetime.now()) 17 | return datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') 18 | 19 | def get_weather(): 20 | url01 = 'http://weather.hao.360.cn/sed_api_weather_info.php?app=clockWeather&_jsonp=callback' 21 | url02 = 'http://tq.360.cn/api/weatherquery/querys?app=tq360&code={code}' 22 | idx = len('callback(') 23 | resp01 = requests.get(url01) 24 | data01 = json.loads(resp01.text[idx + 1:-2:]) 25 | # TODO ... 26 | return 27 | 28 | class CustomScheduler(SchedulerService): 29 | '''CustomScheduler-自定义SchedulerService''' 30 | def exposed_api_add_demo_job(self): 31 | return self.scheduler.add_job( 32 | demo_job, 33 | trigger='interval', 34 | # minutes=1, 35 | seconds=5, 36 | next_run_time=datetime.now() + timedelta(seconds=2), 37 | args=[], 38 | replace_existing=True, 39 | id='add_demo_job' 40 | ) 41 | 42 | def exposed_api_demo_hello(self, args=None): 43 | print(args) 44 | date_time = datetime.now() + timedelta(seconds=5) 45 | now_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') 46 | print(now_str) 47 | return self.scheduler.add_job( 48 | print, 49 | trigger='date', 50 | next_run_time=date_time, 51 | args=[now_str.center(50, '~')] 52 | ) 53 | 54 | def exposed_get_jobs_json(self, jobstore=None): 55 | result = [] 56 | for job_item in self.scheduler.get_jobs(jobstore): 57 | item_data = { 58 | 'id': job_item.id, 59 | 'name': job_item.name, 60 | 'kwargs': job_item.kwargs, 61 | 'next_run_time': job_item.next_run_time.strftime('%Y-%m-%d %H:%M:%S'), 62 | 'pending': job_item.pending 63 | } 64 | result.append(item_data) 65 | return json.dumps(result) 66 | 67 | def exposed_add_job_json(self, job_params, jobstore=None): 68 | params = json.loads(job_params) 69 | new_job = self.scheduler.add_job( 70 | get_weather, 71 | trigger='date', 72 | next_run_time=datetime.now() + timedelta(seconds=1), 73 | **params 74 | ) 75 | if isinstance(new_job, Job): 76 | return new_job.id 77 | 78 | server_params = dict( 79 | listen_config={ 80 | 'port': 54345, 81 | 'hostname': '0.0.0.0' 82 | }, 83 | store_path = r'jobstore.sqlite', 84 | log_file = r'logger.log', 85 | SchedulerServiceClass=CustomScheduler 86 | ) 87 | 88 | if __name__ == '__main__': 89 | schedule_start(**server_params) 90 | -------------------------------------------------------------------------------- /examples/web_server.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, jsonify, render_template_string 2 | 3 | from rpyc_util import RpycUtil 4 | 5 | 6 | app = Flask(__name__) 7 | app.config['JSON_AS_ASCII'] = False 8 | 9 | rpyc = RpycUtil('localhost', 54345) 10 | 11 | @app.route('/') 12 | def index(): 13 | params = dict( 14 | args=[], 15 | replace_existing=True, 16 | id='get_weather_info' 17 | ) 18 | result = rpyc.add_job_json(params) 19 | print(repr(result)) 20 | return "Hello World" 21 | 22 | if __name__=='__main__': 23 | run_cfg = dict( 24 | debug=True, 25 | host='0.0.0.0', 26 | port=5665 27 | ) 28 | app.run(**run_cfg) 29 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # core 2 | rpyc 3 | apscheduler 4 | # examples 5 | flask 6 | requests --------------------------------------------------------------------------------