├── .coveragerc ├── .gitignore ├── .travis.yml ├── README.rst ├── autoworker └── __init__.py ├── requirements-dev.txt ├── requirements.txt ├── setup.py └── spec ├── __init__.py ├── autoworker_spec.py ├── enq.py └── test.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = ./autoworker 3 | concurrency = multiprocessing 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | services: 3 | - redis-server 4 | python: 5 | - '2.7' 6 | - '3.5' 7 | install: 8 | - pip install . 9 | - pip install coveralls 10 | - pip install -r requirements-dev.txt 11 | script: 12 | - mamba --enable-coverage 13 | after_success: 14 | - coverage combine 15 | - coverage report 16 | - coveralls 17 | deploy: 18 | provider: pypi 19 | user: gisce 20 | password: 21 | secure: hy0hN0yvQ8KEAA9yxWpCHRxWcEir8BDuYKTUJ1vD+1OVF7Bz+Xzs5cnMWxezULV/dml4seCP/jNrwcythZoCsfxXC8xyLZHib91vs2VYtXTdlSNjbUkAVnGWIxCw/3BM3GQSjUpnjz7A9nWU22ATTBQTUzoCTSFHfZYprY0qKOj0pJFotF7eV9Z/nXltkHcZUS4wJwNO82G2pXLlafRp6QgN3qAkMYjZHN7/13ks8Fi/PmD8+wGeWaqrDzRasdWXbcgh4rFFyl0zR0tuJMnMwH653vwNBG+RaxYp41snHQeOTwEUql1SoG3kAXOZKDB0OcmTscZ0Fpbgy9pnQ/ADg9glpquP7gFJj/0jT0zCNEjcdg5CaC7Bgp3rewh9qO+LeAa1MOLMWvzTn7mbA7Cp6dGqZfkMD2fY0qZsPe9PYU8Afs4wIv5pGbQg/GixXxtwvd3PYWZ+b4TNHfsefk/nbsmsZvn3V8gTI3KhHT0T8LSKsr4gkot1kNXHYwvIqFkF226miB+ieQx19XSn1MO28zp0IcE62BGo4hJ1NmnFv+D4McYGAY96GLI44Xfg/GMEegRbFfs4Qlr4bGC+5CaZ0ZYNqbRaXocBBSQ+8iBWi7WyQwefC1XxzEsCGby+SwyvuQ2PEXs5QbqP8ffB5Dt/Ru8NCXUpAwyhyBSX4RfPJec= 22 | on: 23 | tags: true 24 | repo: gisce/autoworker 25 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | AutoWorker 2 | ========== 3 | 4 | .. image:: https://travis-ci.org/gisce/autoworker.svg?branch=master 5 | :target: https://travis-ci.org/gisce/autoworker 6 | 7 | Spawn RQ Workers automatically 8 | 9 | .. code-block:: python 10 | 11 | from autoworker import AutoWorker 12 | aw = AutoWorker(queue='high', max_procs=6) 13 | aw.work() 14 | 15 | **Note**: From **v0.4.0** failed jobs doesn't go to the failed queue. If you want to enqueue to failed queue you should call `AutoWorker` as following 16 | 17 | .. code-block:: python 18 | 19 | aw = AutoWorker(queue='high', max_procs=6, skip_failed=False) 20 | -------------------------------------------------------------------------------- /autoworker/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | import multiprocessing as mp 5 | from uuid import uuid4 6 | import subprocess 7 | import sys 8 | 9 | from redis import Redis 10 | from rq.defaults import DEFAULT_RESULT_TTL 11 | from rq.contrib.legacy import cleanup_ghosts 12 | from rq.queue import Queue 13 | from rq.worker import Worker, WorkerStatus 14 | from rq.utils import import_attribute 15 | from osconf import config_from_environment 16 | 17 | 18 | MAX_PROCS = int(max(mp.cpu_count() - os.getloadavg()[0], 0) + 1) 19 | """Number of maximum procs we can run 20 | """ 21 | 22 | 23 | class AutoWorkerQueue(Queue): 24 | 25 | def __init__(self, name='default', default_timeout=None, connection=None, 26 | is_async=True, job_class=None, max_workers=None): 27 | super(AutoWorkerQueue, self).__init__( 28 | name=name, default_timeout=default_timeout, connection=connection, 29 | is_async=is_async, job_class=job_class 30 | ) 31 | if max_workers is None: 32 | max_workers = MAX_PROCS 33 | self.max_workers = max_workers 34 | 35 | def enqueue(self, f, *args, **kwargs): 36 | res = super(AutoWorkerQueue, self).enqueue(f, *args, **kwargs) 37 | self.run_autowker() 38 | return res 39 | 40 | def enqueue_job(self, job, pipeline=None, at_front=False): 41 | res = super(AutoWorkerQueue, self).enqueue_job(job, pipeline, at_front) 42 | self.run_autowker() 43 | return res 44 | 45 | def run_autowker(self): 46 | if Worker.count(queue=self) <= self.max_workers: 47 | aw = AutoWorker(self.name, max_procs=1) 48 | aw.work() 49 | 50 | def run_job(self, job): 51 | return super(AutoWorkerQueue, self).run_job(job) 52 | 53 | 54 | class AutoWorker(object): 55 | """AutoWorker allows to spawn multiple RQ Workers using multiprocessing. 56 | :param queue: Queue to listen 57 | :param max_procs: Number of max_procs to spawn 58 | """ 59 | def __init__(self, queue=None, max_procs=None, skip_failed=True, 60 | default_result_ttl=DEFAULT_RESULT_TTL): 61 | if queue is None: 62 | queue = 'default' 63 | if max_procs is None: 64 | self.max_procs = MAX_PROCS 65 | elif 1 <= max_procs < MAX_PROCS + 1: 66 | self.max_procs = max_procs 67 | else: 68 | raise ValueError('Max procs {} not supported'.format(max_procs)) 69 | self.processes = [] 70 | self.config = config_from_environment( 71 | 'AUTOWORKER', 72 | ['redis_url'], 73 | queue_class='rq.Queue', 74 | worker_class='rq.Worker', 75 | job_class='rq.Job', 76 | ) 77 | self.skip_failed = skip_failed 78 | self.default_result_ttl = default_result_ttl 79 | self.connection = Redis.from_url(self.config['redis_url']) 80 | queue_class = import_attribute(self.config['queue_class']) 81 | self.queue = queue_class(queue, connection=self.connection) 82 | 83 | def num_connected_workers(self): 84 | return len([ 85 | w for w in Worker.all(queue=self.queue) if w.state in ( 86 | WorkerStatus.STARTED, WorkerStatus.SUSPENDED, WorkerStatus.BUSY, 87 | WorkerStatus.IDLE 88 | ) 89 | ]) 90 | 91 | @property 92 | def worker_command(self): 93 | """Internal target to use in multiprocessing 94 | """ 95 | 96 | rq_params = { 97 | '-b': '', 98 | '-w': self.config['worker_class'], 99 | '-n': '{}-auto'.format(uuid4().hex), 100 | '-u': self.config['redis_url'], 101 | '--results-ttl': self.default_result_ttl 102 | 103 | } 104 | if self.skip_failed: 105 | rq_params['--disable-default-exception-handler'] = '' 106 | 107 | command = ['rq', 'worker'] 108 | for k, v in rq_params.items(): 109 | command.append(str(k)) 110 | if v: 111 | command.append(str(v)) 112 | command.append(self.queue.name) 113 | return command 114 | 115 | def work(self): 116 | """Spawn the multiple workers using multiprocessing and `self.worker`_ 117 | targget 118 | """ 119 | max_procs = self.max_procs - self.num_connected_workers() 120 | env = os.environ.copy() 121 | env['PYTHONPATH'] = ':'.join(sys.path) 122 | for _ in range(0, max_procs): 123 | subprocess.Popen(self.worker_command, close_fds=True, env=env) 124 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | mamba 2 | expects 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | osconf 2 | rq>=0.10.0 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | INSTALL_REQUIRES = ['rq>=0.10.0', 'osconf'] 4 | 5 | setup( 6 | name='autoworker', 7 | version='0.10.2', 8 | packages=find_packages(exclude=['spec']), 9 | url='https://github.com/gisce/autoworker', 10 | license='MIT', 11 | author='GISCE-TI, S.L.', 12 | author_email='devel@gisce.net', 13 | install_requires=INSTALL_REQUIRES, 14 | description='Start Python RQ Workers automatically', 15 | classifiers=[ 16 | 'Programming Language :: Python', 17 | 'Programming Language :: Python :: 2.7', 18 | 'Programming Language :: Python :: 3.5' 19 | ] 20 | ) 21 | -------------------------------------------------------------------------------- /spec/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gisce/autoworker/a91d8ce173825d80013f58feb59519948aaef0d3/spec/__init__.py -------------------------------------------------------------------------------- /spec/autoworker_spec.py: -------------------------------------------------------------------------------- 1 | import os 2 | from autoworker import AutoWorker 3 | from rq.queue import Queue 4 | 5 | from expects import * 6 | from mamba import * 7 | 8 | # Setup environment variable 9 | os.environ['AUTOWORKER_REDIS_URL'] = 'redis://localhost:6379/0' 10 | 11 | 12 | with description('The autoworker class'): 13 | with context('if not max_procs is defined'): 14 | with it('must be the same as number of cpus + 1'): 15 | import multiprocessing as mp 16 | 17 | a = AutoWorker() 18 | expect(a.max_procs).to(equal(mp.cpu_count() + 1)) 19 | 20 | with context('if max_procs is passed to __init__'): 21 | with it('must be the the same value'): 22 | a = AutoWorker(max_procs=3) 23 | expect(a.max_procs).to(equal(3)) 24 | 25 | with it('must raise an error if is 0 < max_procs < number of cpus + 1'): 26 | def callback(): 27 | import multiprocessing as mp 28 | a = AutoWorker(max_procs=mp.cpu_count() + 2) 29 | 30 | expect(callback).to(raise_error(ValueError)) 31 | 32 | with context('if no queue is defined'): 33 | with it('must be "default" queue'): 34 | a = AutoWorker() 35 | q = Queue('default', connection=a.connection) 36 | expect(a.queue).to(equal(q)) 37 | with context('if a queue is defined'): 38 | with it('have to be the same value'): 39 | a = AutoWorker('low') 40 | q = Queue('low', connection=a.connection) 41 | expect(a.queue).to(equal(q)) 42 | 43 | 44 | with description('An instance of a AutoWorker'): 45 | with before.each: 46 | self.aw = AutoWorker() 47 | 48 | with it('must have a "work" method to spawn max_procs workers'): 49 | self.aw.work() 50 | -------------------------------------------------------------------------------- /spec/enq.py: -------------------------------------------------------------------------------- 1 | from spec.test import print_foo 2 | 3 | from redis import Redis 4 | from rq import Queue 5 | 6 | q = Queue(connection=Redis()) 7 | 8 | for x in range(10): 9 | q.enqueue(print_foo) 10 | -------------------------------------------------------------------------------- /spec/test.py: -------------------------------------------------------------------------------- 1 | def print_foo(): 2 | return "Foooooo" 3 | --------------------------------------------------------------------------------