├── example ├── __init__.py ├── queue_job.py └── queue_task.py ├── requirements.txt ├── .gitignore ├── zyqueue ├── __init__.py ├── register.py ├── utils.py ├── bin │ └── zyqueue ├── admin.py ├── decorators.py └── workers.py ├── setup.py └── README.md /example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gearman==2.0.2 2 | rq==0.5.5 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .settings/org.eclipse.core.resources.prefs 3 | .pydevproject 4 | conf/.settings.py.swp 5 | *.swp 6 | .DS_Store 7 | .project 8 | static/pics 9 | -------------------------------------------------------------------------------- /zyqueue/__init__.py: -------------------------------------------------------------------------------- 1 | # zyqueue 2 | from admin import queue_admin 3 | from decorators import QueueJob 4 | from workers import QueueWorkerServer 5 | 6 | __version__ = '0.0.1' 7 | VERSION = tuple(map(int, __version__.split('.'))) 8 | -------------------------------------------------------------------------------- /zyqueue/register.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: register.py 9 | 创 建 者: ZengDuju 10 | 创建日期: 2015-10-15 11 | ''' 12 | from collections import defaultdict 13 | from functools import wraps 14 | 15 | tree = lambda: defaultdict(tree) 16 | 17 | 18 | class Register(object): 19 | """不同服务端注册器 20 | """ 21 | _registered = tree() 22 | _servers = ['redis', 'rabbitmq', 'gearman'] # 注册的服务 23 | _function = ['submit', 'worker', 'admin'] # 服务需要的方法功能 24 | 25 | def __init__(self, server, function): 26 | """初始化 27 | """ 28 | self.server = server 29 | self.function = function 30 | 31 | def __call__(self, _func): 32 | self._registered[self.server][self.function] = _func 33 | 34 | @wraps(_func) 35 | def wrapped(*args, **kwargs): 36 | """修饰器 37 | """ 38 | return _func(*args, **kwargs) 39 | return wrapped 40 | 41 | @classmethod 42 | def get_registered(cls): 43 | """获取注册方法 44 | """ 45 | return cls._registered 46 | 47 | @classmethod 48 | def get_reg_server(cls): 49 | """获取已注册的服务 50 | """ 51 | return cls._servers 52 | -------------------------------------------------------------------------------- /example/queue_job.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: queue_job.py 9 | 创 建 者: ZengDuju 10 | 创建日期: 2015-10-10 11 | ''' 12 | from queue_task import queue_task_redis, queue_task_gearman, queue_task_rabbitmq 13 | 14 | 15 | redis_url = "redis://192.168.6.184:6389" 16 | gearman_host = "192.168.6.7:18888" 17 | job_data = {'test1': 'redis', 'test2': 'gearman'} 18 | 19 | for i in range(1): 20 | job_data['num'] = i 21 | # queue_task_test.submit(server='redis', connection=redis_url, job_data=job_data, queue='default') 22 | # queue_task_test.submit(server='redis', connection=redis_url, job_data=job_data, queue='high') 23 | # queue_task_test.submit(server='redis', connection=redis_url, job_data=job_data, queue='low') 24 | # queue_task_test.submit(server='gearman', connection=gearman_host, job_data=job_data) 25 | # queue_task_test.submit(server='rabbitmq', connection='192.168.6.7', job_data=job_data, exchange='zyqueue_rmq', exchange_type='direct', routing_keys='route1') 26 | 27 | queue_task_redis.submit(job_data=job_data, queue='default') 28 | queue_task_redis.submit(job_data=job_data, queue='high') 29 | queue_task_redis.submit(job_data=job_data, queue='low') 30 | # queue_task_gearman.submit(job_data=job_data) 31 | queue_task_rabbitmq.submit(job_data=job_data, exchange='zyqueue_rmq', exchange_type='direct', routing_keys='route1') 32 | -------------------------------------------------------------------------------- /example/queue_task.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: queue_task.py 9 | 创 建 者: ZengDuju 10 | 创建日期: 2015-10-10 11 | ''' 12 | import logging 13 | 14 | from zyqueue import QueueJob 15 | 16 | 17 | # @QueueJob(server='gearman', connection='192.168.6.7:18888') 18 | def queue_task_gearman(worker, job): 19 | """gearman tast execute 20 | """ 21 | try: 22 | logging.info("task execute success! job data: %s" % (','.join(["{}: {}".format(key, value) for key, value in job.data.iteritems()]))) 23 | except Exception, e: 24 | logging.error(msg="task execute failed! error: %s" % (e)) 25 | return True 26 | 27 | 28 | @QueueJob(server='redis', connection='redis://192.168.6.184:6389') 29 | def queue_task_redis(worker, job): 30 | """gearman tast execute 31 | """ 32 | try: 33 | logging.info("task execute success! job data: %s" % (','.join(["{}: {}".format(key, value) for key, value in job.data.iteritems()]))) 34 | except Exception, e: 35 | logging.error(msg="task execute failed! error: %s" % (e)) 36 | return True 37 | 38 | 39 | @QueueJob(server='rabbitmq', connection='192.168.6.7', exchange='zyqueue_rmq', exchange_type='direct', queue="zyqueue_test", routing_keys='route1') 40 | def queue_task_rabbitmq(worker, job): 41 | """gearman tast execute 42 | """ 43 | try: 44 | logging.info("task execute success! job data: %s" % (','.join(["{}: {}".format(key, value) for key, value in job.data.iteritems()]))) 45 | except Exception, e: 46 | logging.error(msg="task execute failed! error: %s" % (e)) 47 | return True 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import os 3 | import sys 4 | from distutils.core import setup 5 | from distutils.command.install_data import install_data 6 | from distutils.command.install import INSTALL_SCHEMES 7 | # from zyqueue import __version__ 8 | 9 | # perform the setup action 10 | 11 | packages, data_files = [], [] 12 | 13 | cmdclasses = {'install_data': install_data} 14 | 15 | for scheme in INSTALL_SCHEMES.values(): 16 | scheme['data'] = scheme['purelib'] 17 | 18 | 19 | def fullsplit(path, result=None): 20 | """ 21 | Split a pathname into components (the opposite of os.path.join) in a 22 | platform-neutral way. 23 | """ 24 | if result is None: 25 | result = [] 26 | head, tail = os.path.split(path) 27 | if head == '': 28 | return [tail] + result 29 | if head == path: 30 | return result 31 | return fullsplit(head, [tail] + result) 32 | 33 | 34 | def is_not_module(filename): 35 | """check filename 36 | """ 37 | return os.path.splitext(filename)[1] not in ['.py', '.pyc', '.pyo'] 38 | 39 | for zyqueue_dir in ['zyqueue']: 40 | for dirpath, dirnames, filenames in os.walk(zyqueue_dir): 41 | # Ignore dirnames that start with '.' 42 | for i, dirname in enumerate(dirnames): 43 | if dirname.startswith('.'): 44 | del dirnames[i] 45 | if '__init__.py' in filenames: 46 | packages.append('.'.join(fullsplit(dirpath))) 47 | data = [f for f in filenames if is_not_module(f)] 48 | if data: 49 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]]) 50 | elif filenames: 51 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]]) 52 | data_files.append(['.', ['README.md']]) 53 | 54 | 55 | setup_args = { 56 | 'name': 'zyqueue', 57 | # 'version': __version__, 58 | 'version': '0.0.1', 59 | 'description': 'ireader queue client', 60 | 'long_description': open('README.md').read(), 61 | 'author': 'WangLichao,Zengduju', 62 | 'author_email': "zengduju@zhangyue.com", 63 | 'packages': packages, 64 | 'data_files': data_files, 65 | # 'include_package_data': True, 66 | 'scripts': ["zyqueue/bin/zyqueue"], 67 | } 68 | 69 | setup(**setup_args) 70 | -------------------------------------------------------------------------------- /zyqueue/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: utils.py 9 | 创 建 者: ZengDuju 10 | 创建日期: 2015-10-08 11 | ''' 12 | import json 13 | import decimal 14 | 15 | from gearman import DataEncoder 16 | 17 | 18 | def pretty_table(fields, data): 19 | """生成表格形式输出 20 | """ 21 | widths = _compute_widths(fields, data) 22 | lines = [] 23 | if len(data) == 0: 24 | return "" 25 | # Add Border 26 | bits = [] 27 | bits.append("+") 28 | for field in fields: 29 | bits.append('-' * widths[field]) 30 | bits.append("+") 31 | bits.append("\n") 32 | border = "".join(bits) 33 | lines.append(border) 34 | # Add Hearder 35 | bits = [] 36 | bits.append("|") 37 | for field in fields: 38 | bits.append(('{:^' + str(widths[field]) + '}').format(field)) 39 | bits.append("|") 40 | bits.append("\n") 41 | hearder = "".join(bits) 42 | lines.append(hearder) 43 | lines.append(border) 44 | # Add Rows 45 | for line in data: 46 | bits = [] 47 | bits.append("|") 48 | for field in fields: 49 | bits.append(('{:^' + str(widths[field]) + '}').format(line[field])) 50 | bits.append("|") 51 | bits.append("\n") 52 | lines.append("".join(bits)) 53 | lines.append(border) 54 | return ("").join(lines) 55 | 56 | 57 | def _compute_widths(fields, rows): 58 | """计算宽度 59 | """ 60 | widths = {field: len(field) + 2 for field in fields} 61 | for row in rows: 62 | for field in fields: 63 | widths[field] = max(widths[field], len(str(row[field])) + 2) 64 | return widths 65 | 66 | 67 | class _JSONEncoder(json.JSONEncoder): 68 | """JSON编码 69 | """ 70 | 71 | def default(self, obj): 72 | """添加对decimal的支持 73 | """ 74 | if isinstance(obj, decimal.Decimal): 75 | return str(obj) 76 | return super(_JSONEncoder, self).default(obj) 77 | 78 | 79 | class JSONDataEncoder(DataEncoder): 80 | """JSON解码 81 | """ 82 | 83 | @classmethod 84 | def encode(cls, encodable_object): 85 | """编码 86 | """ 87 | return json.dumps(encodable_object, cls=_JSONEncoder) 88 | 89 | @classmethod 90 | def decode(cls, decodable_string): 91 | """解码 92 | """ 93 | return json.loads(decodable_string) 94 | 95 | 96 | class Storage(dict): 97 | """增加data属性, 统一不同server任务数据 98 | """ 99 | 100 | @property 101 | def data(self): 102 | """增加data属性 103 | """ 104 | return self['data'] 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [ZyQueue 1.0.0 文档](http://techblog.ireader.com.cn/blog/zyqueue/) 2 | 3 | 任务队列 4 | 5 | --- 6 | 7 | ## ZyQueue简介 ## 8 | zyqueue是一个专注于实时处理任务队列的分布式多进程任务调度系统。它使用上与celery非常相似，通过装饰器模式将队列的生产者与消费者进行关联，使用上简单易用，同时支持gearman，redis，rabbitmq等常用队列做中间调度层。且易于扩展其他队列服务做为中间调度层。 9 | 10 | 11 | 简单：ZyQueue易于使用和维护，并且不需要使用配置文件。下面是一个最简单的任务应用实现： 12 | 13 | ```python 14 | @QueueJob(server='gearman', connection='192.168.6.7:18888') 15 | def queue_task(worker, job): 16 | """gearman tast execute 17 | """ 18 | print "task execute success! job data: {}s"。format(','.join(["{}: {}".format( 19 | key, 20 | value 21 | ) for key, value in job.data.iteritems()])) 22 | return True 23 | ``` 24 | 25 | 高可用性：ZyQueue保持了中间调度层的各种高可用特性。 26 | 27 | 快速：单个 ZyQueue 进程每分钟可数以百万计的任务至队列。 28 | 29 | 灵活：ZyQueue可以使用任意支持的队列服务作为中间调度层，并可轻松扩展定制新的服务作为中间调度层。 30 | 31 | ZyQueue目前支持 32 | 中间调度层：gearman，redis，rabbitmq 33 | 多进程：父子进程模式 34 | 35 | 依赖包：gearman，rq，redis，arrow 36 | 37 | ## ZyQueue初步 ## 38 | 39 | ### 安装 ### 40 | python setup.py install 41 | 42 | ### 应用 ### 43 | ZyQueue使用简单。首先我们会需要创建一个独立模块来编写我们所有的队列任务。 44 | 创建tasks.py 45 | ```python 46 | from zyqueue import QueueJob 47 | 48 | @QueueJob(server='gearman', connection='192.168.6.7:18888') 49 | def queue_task(worker, job): 50 | """gearman tast execute 51 | """ 52 | print "task execute success! job data: %s" % (','.join(["{}: {}".format( 53 | key, 54 | value 55 | ) for key, value in job.data.iteritems()])) 56 | return True 57 | ``` 58 | 修饰器的server参数确定使用哪种中间调度层，connection指定中间调度层的URL，如此便定义好了一个任务，称为queue_task。打印出任务数据。任务的参数被固定为worker和job，其中job为包装后的任务数据，通过job.data可以取出任务数据字典。 59 | 60 | ### 运行ZyQueue职程服务器 ### 61 | ``` 62 | zyqueue --max_workers 1 --task_file tasks.py 63 | ``` 64 | max_worker指定启动多少个进程来监听定义的任务，task_file指定定义好的任务模块。 65 | 想要查看完整的命令行参数列表，执行命令： 66 | ``` 67 | zyqueue --help 68 | ``` 69 | ### 提交任务 ### 70 | 可以调用submit()方法来提交任务： 71 | ```python 72 | from tasks import queue_task 73 | 74 | gearman_host = "192.168.6.7:18888" 75 | job_data = {'test1': 'redis', 'test2': 'gearman'} 76 | queue_task.submit(job_data=job_data) 77 | ``` 78 | 这个任务将有之前运行的职程服务器执行，并且可以查看职程的控制台输出来验证。 79 | 80 | 81 | ### 中间调度层 ### 82 | #### Gearman #### 83 | 需要安装和启动Gearman服务。可以通过： 84 | ``` 85 | zyqueue -S gearman -C 192.168.6.7:18888 --status 86 | zyqueue -S gearman -C 192.168.6.7:18888 --workers 87 | ``` 88 | 查看Gearman服务状态信息。任务定义和提交任务见上两节。 89 | #### Redis #### 90 | 需安装和启动Redis服务。可以通过： 91 | ``` 92 | zyqueue -S redis -C redis://192.168.6.184:6389 --workers 93 | ``` 94 | 查看Redis服务状态信息。任务定义和提交与Gearman一致。 95 | #### RabbitMQ #### 96 | 需要安装和启动RabbitMQ。 97 | 任务定义： 98 | ```python 99 | @QueueJob(server='rabbitmq', connection='192.168.6.7', exchange='zyqueue_rmq', 100 | exchange_type='direct', queue="zyqueue_test", routing_keys='route1') 101 | def queue_task(worker, job): 102 | """gearman tast execute 103 | """ 104 | print "task execute success! job data: %s" % (','.join(["{}: {}".format( 105 | key, 106 | value 107 | ) for key, value in job.data.iteritems()])) 108 | return True 109 | ``` 110 | 任务提交： 111 | ```python 112 | queue_task.submit(server='rabbitmq', connection='192.168.6.7', job_data=job_data, 113 | exchange='zyqueue_rmq', exchange_type='direct', routing_keys='route1') 114 | ``` 115 | 与Gearman和Redis中间调度层不同的是RabbitMQ需要加上自有的一些队列参数，以完善RabbitMQ队列的功能使用。 116 | -------------------------------------------------------------------------------- /zyqueue/bin/zyqueue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: zyqueue 9 | 创 建 者: ZengDuju 10 | 创建日期: 2015-09-15 11 | ''' 12 | import sys 13 | import os 14 | import argparse 15 | import logging 16 | 17 | import zyqueue 18 | 19 | 20 | def setup_logging(options): 21 | """log 配置 22 | """ 23 | if options.debug: 24 | level = logging.DEBUG 25 | else: 26 | level = logging.INFO 27 | 28 | log_format = "%(asctime)s %(levelname)s:%(name)s: %(message)s" 29 | logging.basicConfig(level=level, format=log_format, stream=sys.stdout) 30 | 31 | 32 | def queue_serve(options): 33 | """queue worker serve start 34 | """ 35 | server = zyqueue.QueueWorkerServer(options, 36 | use_sighandler=True, 37 | verbose=True) 38 | server.serve_forever() 39 | 40 | 41 | def main(): 42 | """主函数 43 | """ 44 | parser = argparse.ArgumentParser(description='Zhangyue queue client.') 45 | # parser.add_argument("-S", "--server", 46 | # help="Queue Server, Redis or Gearman", 47 | # required=True) 48 | # common 49 | common_args = parser.add_argument_group('Common arguments') 50 | # common_args.add_argument("-C", "--connection", 51 | # help="Queue server address to use for connection") 52 | common_args.add_argument("--status", 53 | help="Status for the queue server", 54 | action="store_true") 55 | common_args.add_argument("--workers", 56 | help="Workers for the queue server", 57 | action="store_true") 58 | common_args.add_argument("--max_workers", 59 | help="The maximum number of queue worker", 60 | type=int) 61 | common_args.add_argument("--task_file", help="The queue task file") 62 | common_args.add_argument("--debug", 63 | help="Debug log level", 64 | action="store_true") 65 | # gearman 66 | gearman_args = parser.add_argument_group('Gearman command arguments') 67 | gearman_args.add_argument("--shutdown", 68 | help="Shutdown gearman server", 69 | action="store_true") 70 | gearman_args.add_argument("--gmversion", 71 | help="The version number of the gearman server", 72 | action="store_true") 73 | # RQ 74 | rq_args = parser.add_argument_group('RQ command arguments') 75 | rq_args.add_argument("--queue", help="RQ queue info") 76 | rq_args.add_argument("--queues", 77 | help="All RQ queues infos", 78 | action="store_true") 79 | rq_args.add_argument("--jobs", 80 | help="All RQ jobs infos of one queue") 81 | rq_args.add_argument("--cancel_job", 82 | help="Cancels the job with the given job ID") 83 | rq_args.add_argument("--requeue_job", 84 | help="Requeues the job with the given job ID") 85 | rq_args.add_argument("--requeue_all", 86 | help="Requeue all failed jobs", 87 | action="store_true") 88 | rq_args.add_argument("--empty_queue", help="Empty given queues.") 89 | rq_args.add_argument("--compact_queue", help="Compact given queues.") 90 | 91 | options = parser.parse_args() 92 | setup_logging(options) 93 | 94 | if is_check_sever_status(options): 95 | queue_info = zyqueue.queue_admin(options) 96 | print queue_info 97 | sys.exit(0) 98 | # 检查task file是否存在 99 | isfile = os.path.isfile(options.task_file) 100 | if not isfile: 101 | print "Error: job file '{}' does not exist!".format(options.task_file) 102 | sys.exit(0) 103 | queue_serve(options) 104 | return 105 | 106 | 107 | def is_check_sever_status(options): 108 | """判断命令参数是否为查询服务状态信息 109 | """ 110 | for key in ('status', 'workers', 'shutdown', 'gmversion', 111 | 'queue', 'queues', 'jobs', 'cancel_job', 112 | 'requeue_job', 'requeue_all', 'empty_queue', 113 | 'compact_queue'): 114 | if getattr(options, key): 115 | return True 116 | return False 117 | 118 | 119 | if __name__ == "__main__": 120 | main() 121 | -------------------------------------------------------------------------------- /zyqueue/admin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | ''' 5 | Copyright (c) 2014,掌阅科技 6 | All rights reserved. 7 | 8 | 摘 要: admin.py 9 | 创 建 者: ZengDuju 10 | 创建日期: 2015-10-13 11 | ''' 12 | import arrow 13 | 14 | from gearman import GearmanAdminClient 15 | from redis import Redis 16 | from rq import (Queue, Worker, cancel_job, get_failed_queue, pop_connection, 17 | push_connection, requeue_job) 18 | 19 | from .utils import pretty_table 20 | from .register import Register 21 | 22 | 23 | def queue_admin(options): 24 | """队列管理, 信息查询 25 | """ 26 | return Register.get_registered()[options.server.lower()]['admin'](options) 27 | 28 | 29 | @Register('gearman', 'admin') 30 | def get_gminfo(options): 31 | """获取gearman服务信息 32 | """ 33 | gm_client = GearmanAdminClient([options.connection]) 34 | if options.status: 35 | status = gm_client.get_status() 36 | return pretty_table(['task', 'queued', 'running', 'workers'], status) 37 | elif options.workers: 38 | workers = gm_client.get_workers() 39 | return pretty_table(['ip', 'client_id', 'tasks', 'file_descriptor'], 40 | workers) 41 | elif options.shutdown: 42 | shutdown_info = gm_client.send_shutdown() 43 | return shutdown_info 44 | elif options.gmversion: 45 | version = gm_client.get_version() 46 | return version 47 | 48 | 49 | @Register('redis', 'admin') 50 | def get_rqinfo(options): 51 | """获取rq队列信息 52 | """ 53 | redis_conn = Redis.from_url(options.connection) 54 | push_connection(redis_conn) 55 | # RQ队列信息获取操作 56 | if options.status: 57 | workers = Worker.all() 58 | queues = Queue.all() 59 | return workers, queues 60 | if options.queue: 61 | queue = Queue(options.queue) 62 | return queue 63 | if options.cancel_job: 64 | cancel_job(options.cancel_job) 65 | return 'OK' 66 | if options.requeue_job: 67 | requeue_job(options.requeue_job) 68 | return 'OK' 69 | if options.requeue_all: 70 | return requeue_all() 71 | if options.empty_queue: 72 | empty_queue(options.empty_queue) 73 | return 'OK' 74 | if options.compact_queue: 75 | compact_queue(options.compact_queue) 76 | return 'OK' 77 | if options.queues: 78 | return list_queues() 79 | if options.jobs: 80 | return list_jobs(options.jobs) 81 | if options.workers: 82 | return list_workers() 83 | pop_connection() 84 | 85 | 86 | def requeue_all(): 87 | """rq admin method 88 | """ 89 | fq = get_failed_queue() 90 | job_ids = fq.job_ids 91 | count = len(job_ids) 92 | for job_id in job_ids: 93 | requeue_job(job_id) 94 | return dict(status='OK', count=count) 95 | 96 | 97 | def empty_queue(queue_name): 98 | """rq admin method 99 | """ 100 | q = Queue(queue_name) 101 | q.empty() 102 | return dict(status='OK') 103 | 104 | 105 | def compact_queue(queue_name): 106 | """rq admin method 107 | """ 108 | q = Queue(queue_name) 109 | q.compact() 110 | return dict(status='OK') 111 | 112 | 113 | def list_queues(): 114 | """rq admin method 115 | """ 116 | queues = serialize_queues(sorted(Queue.all())) 117 | return pretty_table(['name', 'count'], queues) 118 | 119 | 120 | def list_jobs(queue_name): 121 | """rq admin method 122 | """ 123 | queue = Queue(queue_name) 124 | # total_items = queue.count 125 | jobs = [serialize_job(job) for job in queue.get_jobs()] 126 | return pretty_table(['origin', 127 | 'id', 128 | 'description', 129 | 'enqueued_at', 130 | 'created_at', 131 | 'ended_at', 132 | 'result', 133 | 'exc_info'], jobs) 134 | 135 | 136 | def list_workers(): 137 | """rq admin method 138 | """ 139 | 140 | def serialize_queue_names(worker): 141 | """rq admin method 142 | """ 143 | return [q.name for q in worker.queues] 144 | 145 | workers = [dict(name=worker.name, 146 | queues=serialize_queue_names(worker), 147 | state=worker.get_state()) 148 | for worker in Worker.all()] 149 | return pretty_table(['state', 'queues', 'name'], workers) 150 | 151 | 152 | def serialize_queues(queues): 153 | """rq admin method 154 | """ 155 | return [dict(name=q.name, 156 | count=q.count) 157 | for q in queues] 158 | 159 | 160 | def serialize_date(dt): 161 | """rq admin method 162 | """ 163 | if dt is None: 164 | return None 165 | return arrow.get(dt).to('UTC').datetime.isoformat() 166 | 167 | 168 | def serialize_job(job): 169 | """rq admin method 170 | """ 171 | return dict( 172 | id=job.id, 173 | created_at=serialize_date(job.created_at), 174 | enqueued_at=serialize_date(job.enqueued_at), 175 | ended_at=serialize_date(job.ended_at), 176 | origin=job.origin, 177 | result=job._result, 178 | exc_info=job.exc_info, 179 | description=job.description) 180 | -------------------------------------------------------------------------------- /zyqueue/decorators.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # pylint: disable=unused-argument 4 | 5 | ''' 6 | Copyright (c) 2014,掌阅科技 7 | All rights reserved. 8 | 9 | 摘 要: decorators.py 10 | 创 建 者: ZengDuju 11 | 创建日期: 2015-10-09 12 | ''' 13 | import os 14 | import sys 15 | import logging 16 | import traceback 17 | from functools import wraps 18 | from collections import defaultdict 19 | 20 | import gearman 21 | from gearman.constants import PRIORITY_NONE 22 | from gearman.errors import ExceededConnectionAttempts, ServerUnavailable 23 | from rq import Queue 24 | from redis import Redis 25 | import ujson as json 26 | import pika 27 | 28 | from .utils import JSONDataEncoder, Storage 29 | from .register import Register 30 | 31 | 32 | class Task(object): 33 | 34 | """gearman任务对象封装 35 | Attributes: 36 | task: 任务名字 37 | callback: 任务回调函数 38 | verbose: 是否输出详细错误日志 39 | """ 40 | 41 | def __init__(self, task, callback, 42 | rabbitmq_kwargs=None, 43 | verbose=False): 44 | """初始化操作 45 | """ 46 | self.task = task 47 | self.callback = callback 48 | self.verbose = verbose 49 | 50 | # rabbitmq 任务专用参数 51 | self.rabbitmq_kwargs = rabbitmq_kwargs 52 | 53 | def __call__(self, worker, job): 54 | try: 55 | return self.callback(worker, job) 56 | except Exception, e: 57 | if self.verbose: 58 | logging.error('WORKER FAILED: %s, %s\n%s', 59 | self.task, 60 | e, 61 | traceback.format_exc()) 62 | worker.shutdown() # 关闭worker使任务重新回到队列中 63 | raise 64 | 65 | 66 | class GearmanJobDataTypeError(Exception): 67 | """错误异常封装 68 | """ 69 | 70 | def __init__(self, errmsg): 71 | self.errmsg = errmsg 72 | 73 | def __str__(self): 74 | return "gearman task data type error,msg is :%s" % self.errmsg 75 | 76 | 77 | class GearmanClient(gearman.GearmanClient): 78 | """扩展gearmanclient的编码方式 79 | """ 80 | data_encoder = JSONDataEncoder 81 | 82 | 83 | @Register('gearman', 'submit') 84 | def submit_gearman_job(connection, job_data, 85 | job_name='', timeout=None, priority=PRIORITY_NONE, 86 | background=True, wait_until_complete=False, 87 | max_retries=0, **kwargs): 88 | """提交任务至gearman队列 89 | """ 90 | # 多余的未使用kwargs参数是为了兼容不同方法参数而添加的 91 | gearman_addr = connection.split(',') 92 | gearman_client = GearmanClient(gearman_addr) 93 | try: 94 | if isinstance(job_data, list): 95 | submit_job_data_list = [dict(task=job_name, data=data, priority=priority) for data in job_data] 96 | gearman_client.submit_multiple_jobs(submit_job_data_list, 97 | background=background, 98 | wait_until_complete=wait_until_complete, 99 | max_retries=max_retries, 100 | poll_timeout=timeout) 101 | elif isinstance(job_data, dict): 102 | gearman_client.submit_job(job_name, 103 | job_data, 104 | priority=priority, 105 | background=background, 106 | wait_until_complete=wait_until_complete, 107 | max_retries=max_retries, 108 | poll_timeout=timeout) 109 | else: 110 | # 任务类型错误 111 | raise GearmanJobDataTypeError('type can only be list and dict') 112 | except ServerUnavailable as e: 113 | error_str = "gearman 连接失败: ServerUnavailable: {}, job_name: {}".format(e, job_name) 114 | logging.error("gearman 连接失败. ServerUnavailable: {}, job_name: {}", exc_info=True) 115 | except ExceededConnectionAttempts as e: 116 | error_str = "gearman 任务添加失败: ExceededConnectionAttempts: {}, job_name: {}".format(e, job_name) 117 | logging.error(error_str, exc_info=True) 118 | 119 | 120 | @Register('redis', 'submit') 121 | def submit_rq_job(connection, job_data, 122 | queue='default', timeout=None, func=None, **kwargs): 123 | """提交任务至redis队列 124 | """ 125 | redis_conn = Redis.from_url(connection) 126 | q = Queue(queue, connection=redis_conn) 127 | job_data = Storage({'data': job_data}) 128 | q.enqueue_call(func, ["placeholder", job_data], timeout=timeout) 129 | 130 | 131 | @Register('rabbitmq', 'submit') 132 | def submit_rmq_job(connection, job_data, 133 | exchange='', exchange_type='', 134 | routing_keys='', 135 | **kwargs): 136 | """提交任务至rabbitmq队列 137 | """ 138 | rmq_conn = pika.BlockingConnection(pika.ConnectionParameters(connection)) 139 | channel = rmq_conn.channel() 140 | 141 | # 定义交换机 142 | channel.exchange_declare(exchange=exchange, type=exchange_type) 143 | body = json.dumps(job_data) 144 | #将消息发送到交换机 145 | for routing_key in routing_keys.split(','): 146 | channel.basic_publish(exchange=exchange, 147 | routing_key=routing_key, 148 | body=body) 149 | rmq_conn.close() 150 | 151 | 152 | class QueueJob(object): 153 | """job添加修饰器 154 | """ 155 | _tasks = defaultdict(set) 156 | _brokers = set() 157 | 158 | def __init__(self, 159 | server='', 160 | connection='', 161 | exchange='', 162 | exchange_type='', 163 | queue='', 164 | routing_keys=''): 165 | """初始化 166 | """ 167 | # 中间人参数 168 | self.server = server.lower() 169 | self.connection = connection.lower() 170 | self._brokers.add((self.server, self.connection)) 171 | 172 | # RabbitMQ参数 173 | self.rabbitmq_kwargs = {'queue': queue, 174 | 'exchange': exchange, 175 | 'exchange_type': exchange_type, 176 | 'routing_keys': routing_keys} 177 | 178 | def __call__(self, _func): 179 | """增加submit方法 180 | """ 181 | job_name = _func.__name__ 182 | self._tasks[(self.server, self.connection)].add(Task( 183 | job_name, 184 | _func, 185 | rabbitmq_kwargs=self.rabbitmq_kwargs 186 | )) 187 | 188 | @wraps(_func) 189 | def submit(job_data, **kwargs): 190 | """ 191 | Args: 192 | job_data支持list和dict两种结构 193 | """ 194 | kwargs['func'] = _func 195 | kwargs['job_name'] = job_name 196 | if self.server in Register.get_reg_server(): 197 | # 不同server共用参数加上自有参数 198 | Register.get_registered()[self.server]['submit'](self.connection, job_data, **kwargs) 199 | else: 200 | error_str = "queue 任务添加失败: 不支持指定server: {}, job_name: {}".format(self.server, job_name) 201 | logging.error(error_str, exc_info=True) 202 | _func.submit = submit 203 | return _func 204 | 205 | @classmethod 206 | def get_tasks(cls): 207 | """获取全部task 208 | """ 209 | return cls._tasks 210 | 211 | @classmethod 212 | def get_brokers(cls): 213 | """获取全部中间人 214 | """ 215 | return cls._brokers 216 | 217 | 218 | def load(task_file): 219 | """加载目录 220 | """ 221 | path, file_name = os.path.split(task_file) 222 | sys.path.append(path) 223 | __import__(file_name.replace('.py', '')) 224 | 225 | # 通过修饰器获取配置 226 | tasks = QueueJob.get_tasks() 227 | brokers = QueueJob.get_brokers() 228 | return tasks, brokers 229 | -------------------------------------------------------------------------------- /zyqueue/workers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # pylint: disable=unused-argument 4 | 5 | ''' 6 | Copyright (c) 2014,掌阅科技 7 | All rights reserved. 8 | 9 | 摘 要: workers.py 10 | 创 建 者: ZengDuju 11 | 创建日期: 2015-10-12 12 | ''' 13 | import os 14 | import sys 15 | import time 16 | import signal 17 | from multiprocessing import Process, Queue, cpu_count, active_children 18 | from Queue import Empty 19 | import logging 20 | import ujson as json 21 | from collections import defaultdict 22 | 23 | import gearman 24 | from redis import Redis 25 | from rq import Connection 26 | from rq import Worker as RqWorker 27 | from rq import Queue as RqQueue 28 | import pika 29 | 30 | from .utils import JSONDataEncoder, Storage 31 | from .decorators import load 32 | from .register import Register 33 | 34 | __all__ = ['QueueWorkerServer'] 35 | listen = ['high', 'default', 'low'] 36 | 37 | QUEUE_WORKERS = defaultdict(set) # 记录所有进程的信息 38 | 39 | 40 | class GearmanWorker(gearman.GearmanWorker): 41 | """接收数据使用JSON格式 42 | """ 43 | data_encoder = JSONDataEncoder 44 | 45 | def after_poll(self, any_activity): 46 | """回调 47 | """ 48 | # logging.info('callback: any_activity=%s', any_activity) 49 | if any_activity: 50 | pass 51 | return True 52 | 53 | 54 | class QueueWorkerServer(object): 55 | """多进程启动worker 56 | """ 57 | 58 | def __init__(self, 59 | options, 60 | use_sighandler=True, 61 | verbose=False): 62 | self.tasks, self.brokers = load(task_file=options.task_file) 63 | # self.connection = options.connection 64 | self.verbose = verbose 65 | self.id_prefix = '' 66 | if options.max_workers: 67 | self.max_workers = int(options.max_workers) 68 | else: 69 | self.max_workers = cpu_count() 70 | if use_sighandler: 71 | self._setup_sighandler() 72 | 73 | file_name = os.path.split(options.task_file)[1] 74 | self.id_prefix = "{}.".format(file_name) 75 | 76 | def start_process(self, doneq, server, connection, process_counter): 77 | target_kwargs = {} 78 | target_kwargs['doneq'] = doneq 79 | target_kwargs['tasks'] = self.tasks.get((server, connection), []) 80 | target = Register.get_registered()[server]['worker'] 81 | target_kwargs['connection'] = connection 82 | client_id = '{}.{}{}'.format(server, self.id_prefix, process_counter) 83 | target_kwargs['client_id'] = client_id 84 | proc = Process(target=target, kwargs=target_kwargs) 85 | proc.start() 86 | if self.verbose: 87 | logging.info("Server: %s. Num workers: %s of %s", server, process_counter, self.max_workers) 88 | return proc 89 | 90 | def serve_forever(self): 91 | """启动多进程服务处理队列任务 92 | """ 93 | global QUEUE_WORKERS 94 | QUEUE_WORKERS = defaultdict(set) 95 | # 队列初始化 96 | doneq = Queue() 97 | # 记录创建的进程数,保证每个进程id唯一 98 | process_counter = 0 99 | try: 100 | while True: 101 | for server, connection in self.brokers: 102 | while len(QUEUE_WORKERS[(server, connection)]) < self.max_workers: 103 | QUEUE_WORKERS[(server, connection)].add(self.start_process(doneq, server, connection, process_counter)) 104 | process_counter += 1 105 | try: 106 | r = doneq.get(True, 5) 107 | except Empty: 108 | r = None 109 | if r is not None: 110 | if isinstance(r, gearman.errors.ServerUnavailable): 111 | if self.verbose: 112 | logging.info("Reconnecting.") 113 | time.sleep(2) 114 | elif r is True: 115 | logging.info('Normal process exit (May actually be a problem)') 116 | time.sleep(0.1) 117 | for server, connection in self.brokers: 118 | QUEUE_WORKERS[(server, connection)] = set([w for w in active_children() if w in QUEUE_WORKERS[(server, connection)]]) 119 | except KeyboardInterrupt: 120 | logging.error('EXIT. RECEIVED INTERRUPT') 121 | 122 | @staticmethod 123 | def _setup_sighandler(): 124 | """设置中断信号 125 | """ 126 | signal.signal(signal.SIGINT, _interrupt_handler) 127 | signal.signal(signal.SIGTERM, _interrupt_handler) 128 | 129 | 130 | def _interrupt_handler(signum, frame): 131 | """获取信号后响应 132 | """ 133 | global QUEUE_WORKERS 134 | logging.info('get signal: signum=%s, frame=%s', signum, frame.f_exc_value) 135 | if signum in (signal.SIGTERM, signal.SIGINT): 136 | for workers in QUEUE_WORKERS.values(): 137 | for worker in workers: 138 | if worker in active_children(): 139 | worker.terminate() 140 | sys.exit(0) 141 | raise KeyboardInterrupt() 142 | 143 | 144 | @Register('gearman', 'worker') 145 | def _gearman_worker_process(doneq, connection, tasks=None, client_id=None, verbose=False, **kwargs): 146 | """多进程处理器 147 | """ 148 | try: 149 | worker_class = GearmanWorker 150 | try: 151 | # 连接 152 | host_list = connection.split(',') 153 | gm_worker = worker_class(host_list=host_list) 154 | if client_id: 155 | gm_worker.set_client_id(client_id) 156 | # 加载任务 157 | for task in tasks: 158 | taskname = callback = None 159 | if isinstance(task, dict): 160 | taskname = task['task'] 161 | callback = task['callback'] 162 | elif isinstance(task, (list, tuple)): 163 | taskname, callback = task 164 | else: 165 | taskname = task.task 166 | callback = task 167 | if verbose: 168 | logging.info("Registering %s task %s", client_id, taskname) 169 | # 注册任务和对应的回调函数 170 | gm_worker.register_task(taskname, callback) 171 | gm_worker.work() 172 | except gearman.errors.ServerUnavailable, e: 173 | # gearman服务不可用 174 | doneq.put(e) 175 | return 176 | doneq.put(True) 177 | except KeyboardInterrupt: 178 | # 捕获中断信号 179 | pass 180 | 181 | 182 | @Register('redis', 'worker') 183 | def _redis_worker_process(doneq, connection, client_id=None, verbose=False, **kwargs): 184 | '''多进程处理器, 启动RQ Worker 185 | ''' 186 | try: 187 | worker_class = RqWorker 188 | # 连接 189 | redis_conn = Redis.from_url(connection) 190 | with Connection(redis_conn): 191 | worker = worker_class(map(RqQueue, listen)) 192 | worker.work() 193 | doneq.put(True) 194 | except KeyboardInterrupt: 195 | # 捕获中断信号 196 | pass 197 | 198 | 199 | @Register('rabbitmq', 'worker') 200 | def _rabbitmq_worker_process(doneq, connection, tasks=None, client_id=None, verbose=False, **kwargs): 201 | """启动rabbitmq worker 202 | """ 203 | connection = pika.BlockingConnection(pika.ConnectionParameters(connection)) 204 | channel = connection.channel() 205 | for task in tasks: 206 | # taskname = task.task 207 | callback = task 208 | 209 | rabbitmq_kwargs = task.rabbitmq_kwargs or {} 210 | exchange = rabbitmq_kwargs.get('exchange') 211 | exchange_type = rabbitmq_kwargs.get('exchange_type') 212 | queue = rabbitmq_kwargs.get('queue') 213 | routing_keys = rabbitmq_kwargs.get('routing_keys') 214 | 215 | def rmq_cb(ch, method, properties, body): 216 | """rabbitmq callback函数, 收到服务端分发的消息后调用 217 | """ 218 | logging.info('Received message # %s', body) 219 | body = json.loads(body) 220 | job_data = Storage({'data': body}) 221 | callback('placeholder', job_data) # 存在bug, 第一个参数需实现shutdown方法 222 | # ch.basic_ack(delivery_tag = method.delivery_tag) 223 | 224 | # 定义交换机 225 | channel.exchange_declare(exchange=exchange, type=exchange_type) 226 | if queue: 227 | queue_name = queue 228 | channel.queue_declare(queue=queue_name) 229 | else: 230 | # 生成临时队列 231 | # exclusive参数: 接收端退出时，销毁临时产生的队列 232 | result = channel.queue_declare(exclusive=True) 233 | queue_name = result.method.queue 234 | # 绑定队列到交换机上 235 | for routing_key in routing_keys.split(','): 236 | channel.queue_bind(exchange=exchange, 237 | queue=queue_name, 238 | routing_key=routing_key) 239 | 240 | # channel.basic_qos(prefetch_count=1) # Worker公平调度 241 | channel.basic_consume(rmq_cb, queue=queue, no_ack=True) 242 | 243 | channel.start_consuming() 244 | --------------------------------------------------------------------------------