├── .bumpversion.cfg ├── .coveragerc ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── CHANGELOG.rst ├── LICENSE ├── Makefile ├── README.rst ├── examples └── simple │ ├── README.md │ ├── __init__.py │ └── app.py ├── pytest.ini ├── requirements.txt ├── requirements_dev.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conftest.py ├── hello.py ├── test_app.py ├── test_cli_worker.py ├── test_job.py ├── test_task.py └── test_worker.py ├── tinyq ├── __init__.py ├── __main__.py ├── app.py ├── constants.py ├── exceptions.py ├── job.py ├── queue.py ├── runner.py ├── task.py ├── utils.py └── worker.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | commit = True 3 | tag = True 4 | current_version = 0.3.0 5 | 6 | [bumpversion:file:tinyq/__init__.py] 7 | 8 | [bumpversion:file:setup.py] 9 | 10 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | tinyq/__main__.py 4 | 5 | [report] 6 | exclude_lines = 7 | pragma: no cover 8 | if __name__ == .__main__.: 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: https://github.com/pre-commit/pre-commit-hooks.git 2 | sha: v0.7.1 3 | hooks: 4 | - id: check-merge-conflict 5 | - id: debug-statements 6 | - id: double-quote-string-fixer 7 | - id: end-of-file-fixer 8 | - id: requirements-txt-fixer 9 | - id: trailing-whitespace 10 | - id: flake8 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 3.6 4 | 5 | sudo: false 6 | cache: pip 7 | 8 | services: 9 | - redis-server 10 | 11 | env: 12 | global: 13 | - TINYQ_TESTING_REDIS_URI=redis:// 14 | matrix: 15 | - TOX_ENV=py26 16 | - TOX_ENV=py27 17 | - TOX_ENV=py33 18 | - TOX_ENV=py34 19 | - TOX_ENV=py36 20 | - TOX_ENV=pypy 21 | - TOX_ENV=pypy3 22 | 23 | install: 24 | - pip install coveralls 25 | - pip install tox 26 | - pip install -r requirements.txt 27 | - pip install -r requirements_dev.txt 28 | 29 | script: 30 | - pre-commit run --all-files 31 | - tox -e $TOX_ENV 32 | 33 | after_script: 34 | - coveralls 35 | 36 | matrix: 37 | include: 38 | - python: 3.5 39 | env: TOX_ENV=py35 40 | allow_failures: 41 | - env: TOX_ENV=py26 42 | - env: TOX_ENV=py27 43 | - env: TOX_ENV=py33 44 | - env: TOX_ENV=py34 45 | - env: TOX_ENV=pypy 46 | - env: TOX_ENV=pypy3 47 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ----------- 3 | 4 | 5 | 0.2.0 (2017-04-04) 6 | ==================== 7 | 8 | * Added testing 9 | * Added ``--log-level`` to setup logging level 10 | * Added ``-a`` for alias ``--app`` 11 | * Added ``-l`` for alias ``--log-level`` 12 | * Fxied forget call ``setup_signal_handlers`` 13 | * Check whether all processes is stopped before exit main process 14 | 15 | 16 | 0.1.0 (2017-02-19) 17 | ==================== 18 | 19 | * Initial Release 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 mozillazg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | help: 2 | @echo "test run test" 3 | @echo "publish publish to PyPI" 4 | @echo "publish_test publish to TestPyPI" 5 | @echo "docs_html make html docs" 6 | 7 | .PHONY: test 8 | test: 9 | @echo "run test" 10 | py.test --cov tinyq tests/ 11 | 12 | .PHONY: publish 13 | publish: 14 | @echo "publish to pypi" 15 | python setup.py register 16 | python setup.py publish 17 | 18 | .PHONY: publish_test 19 | publish_test: 20 | python setup.py register -r test 21 | python setup.py sdist upload -r test 22 | python setup.py bdist_wheel upload -r test 23 | 24 | .PHONY: docs_html 25 | docs_html: 26 | cd docs && make html 27 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | tinyq 2 | ===== 3 | 4 | |Build| |Coverage| |Pypi version| 5 | 6 | A tiny job queue framework. 7 | 8 | 9 | Install 10 | ---------- 11 | 12 | :: 13 | 14 | pip install tinyq 15 | 16 | 17 | Usage 18 | ------- 19 | 20 | start redis server :: 21 | 22 | $ redis-server 23 | 24 | 25 | app.py :: 26 | 27 | 28 | from tinyq import Application 29 | 30 | app = Application() 31 | 32 | 33 | @app.task() 34 | def add(m, n): 35 | return m + n 36 | 37 | 38 | add jobs :: 39 | 40 | for m in range(10): 41 | for n in range(3): 42 | add.delay(m, n) 43 | 44 | start worker :: 45 | 46 | $ tinyq -l info 47 | 2017-03-12 21:27:12,322 - WARNING - tinyq.runner[line:73 thread:MainThread(140736379601856) process:MainProcess(15388)] - Starting TinyQ worker, version 0.1.0... 48 | 2017-03-12 21:27:12,446 - INFO - tinyq.worker[line:65 thread:Worker-2(123145554059264) process:MainProcess(15388)] - Got a job: 49 | 2017-03-12 21:27:12,447 - INFO - tinyq.worker[line:67 thread:Worker-2(123145554059264) process:MainProcess(15388)] - Finish run job 50 | 2017-03-12 21:27:12,500 - INFO - tinyq.worker[line:65 thread:Worker-5(123145569824768) process:MainProcess(15388)] - Got a job: 51 | 2017-03-12 21:27:12,501 - INFO - tinyq.worker[line:67 thread:Worker-5(123145569824768) process:MainProcess(15388)] - Finish run job 52 | 2017-03-12 21:27:12,610 - INFO - tinyq.worker[line:65 thread:Worker-1(123145548804096) process:MainProcess(15388)] - Got a job: 53 | 2017-03-12 21:27:12,610 - INFO - tinyq.worker[line:67 thread:Worker-1(123145548804096) process:MainProcess(15388)] - Finish run job 54 | ^C2017-03-12 21:27:13,863 - WARNING - tinyq.runner[line:144 thread:MainThread(140736379601856) process:MainProcess(15388)] - Received stop signal, warm shutdown... 55 | 2017-03-12 21:27:13,886 - WARNING - tinyq.runner[line:135 thread:Worker-2(123145554059264) process:MainProcess(15388)] - Exit worker Worker-2. 56 | 2017-03-12 21:27:13,896 - WARNING - tinyq.runner[line:135 thread:Worker-7(123145580335104) process:MainProcess(15388)] - Exit worker Worker-7. 57 | 2017-03-12 21:27:13,906 - WARNING - tinyq.runner[line:135 thread:Scheduler(123145538293760) process:MainProcess(15388)] - Exit worker Scheduler. 58 | 2017-03-12 21:27:13,924 - WARNING - tinyq.runner[line:135 thread:Worker-5(123145569824768) process:MainProcess(15388)] - Exit worker Worker-5. 59 | 2017-03-12 21:27:13,936 - WARNING - tinyq.runner[line:135 thread:Worker-0(123145543548928) process:MainProcess(15388)] - Exit worker Worker-0. 60 | 2017-03-12 21:27:13,956 - WARNING - tinyq.runner[line:135 thread:Worker-4(123145564569600) process:MainProcess(15388)] - Exit worker Worker-4. 61 | 2017-03-12 21:27:13,978 - WARNING - tinyq.runner[line:135 thread:Worker-6(123145575079936) process:MainProcess(15388)] - Exit worker Worker-6. 62 | 2017-03-12 21:27:14,017 - WARNING - tinyq.runner[line:135 thread:Worker-1(123145548804096) process:MainProcess(15388)] - Exit worker Worker-1. 63 | 2017-03-12 21:27:14,068 - WARNING - tinyq.runner[line:135 thread:Worker-3(123145559314432) process:MainProcess(15388)] - Exit worker Worker-3. 64 | 2017-03-12 21:27:14,068 - WARNING - tinyq.runner[line:101 thread:MainThread(140736379601856) process:MainProcess(15388)] - Exit workers. 65 | $ 66 | 67 | .. |Build| image:: https://img.shields.io/travis/mozillazg/tinyq/master.svg 68 | :target: https://travis-ci.org/mozillazg/tinyq 69 | .. |Coverage| image:: https://img.shields.io/coveralls/mozillazg/tinyq/master.svg 70 | :target: https://coveralls.io/r/mozillazg/tinyq 71 | .. |PyPI version| image:: https://img.shields.io/pypi/v/tinyq.svg 72 | :target: https://pypi.python.org/pypi/tinyq 73 | -------------------------------------------------------------------------------- /examples/simple/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1. 启动一个 redis 服务 4 | 5 | $ redis-server 6 | 7 | 2. 添加一些 job 8 | 9 | $ python app.py 10 | 11 | 3. 运行一个 worker: 12 | 13 | $ tinyq 14 | -------------------------------------------------------------------------------- /examples/simple/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /examples/simple/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from tinyq import Application 3 | 4 | app = Application() 5 | 6 | 7 | @app.task() 8 | def add(m, n): 9 | return m + n 10 | 11 | 12 | if __name__ == '__main__': 13 | for m in range(3): 14 | print(repr(add.delay(m, 2))) 15 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -slv --tb=short 3 | norecursedirs = .git __pycache__ 4 | cov-report= term-missing 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | redis 2 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | pre-commit 4 | pytest 5 | pytest-cov 6 | pytest-random 7 | tox 8 | wheel>=0.21 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | import os 4 | 5 | try: 6 | from setuptools import setup 7 | except ImportError: 8 | from distutils.core import setup 9 | 10 | if sys.argv[-1] == 'publish': 11 | os.system('python setup.py sdist upload') 12 | os.system('python setup.py bdist_wheel upload') 13 | sys.exit() 14 | 15 | 16 | def long_description(): 17 | with open('README.rst', encoding='utf8') as f: 18 | return f.read() 19 | 20 | 21 | requirements = [ 22 | 'redis', 23 | ] 24 | 25 | setup( 26 | name='tinyq', 27 | version='0.3.0', 28 | description='A tiny job queue framework.', 29 | long_description=long_description(), 30 | url='https://github.com/mozillazg/tinyq', 31 | author='mozillazg', 32 | author_email='mozillazg101@gmail.com', 33 | license='MIT', 34 | package_data={'': ['LICENSE']}, 35 | packages=['tinyq'], 36 | package_dir={'tinyq': 'tinyq'}, 37 | include_package_data=True, 38 | install_requires=requirements, 39 | zip_safe=False, 40 | entry_points={ 41 | 'console_scripts': [ 42 | 'tinyq = tinyq.__main__:main', 43 | ], 44 | }, 45 | classifiers=[ 46 | 'Development Status :: 3 - Alpha', 47 | 'Intended Audience :: Developers', 48 | 'License :: OSI Approved :: MIT License', 49 | 'Operating System :: OS Independent', 50 | 'Programming Language :: Python', 51 | 'Programming Language :: Python :: 3', 52 | 'Programming Language :: Python :: 3.3', 53 | 'Programming Language :: Python :: 3.4', 54 | 'Programming Language :: Python :: 3.5', 55 | 'Programming Language :: Python :: 3.6', 56 | 'Programming Language :: Python :: Implementation :: CPython', 57 | 'Programming Language :: Python :: Implementation :: PyPy', 58 | 'Topic :: Utilities', 59 | ], 60 | keywords='queue, 队列', 61 | ) 62 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozillazg/tinyq/fd9ecc593931c9b315c4aeb9150389b3e4ae670e/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | import pytest 5 | import redis 6 | 7 | from tinyq.app import Application 8 | from tinyq import task 9 | 10 | redis_uri = os.environ['TINYQ_TESTING_REDIS_URI'] 11 | 12 | 13 | @pytest.fixture() 14 | def app(): 15 | instance = redis.StrictRedis.from_url(redis_uri) 16 | app = Application(instance) 17 | instance.flushdb() 18 | task._task_names.clear() 19 | yield app 20 | instance.flushdb() 21 | task._task_names.clear() 22 | 23 | 24 | @pytest.fixture() 25 | def redis_instance(): 26 | instance = redis.StrictRedis.from_url(redis_uri) 27 | instance.flushdb() 28 | yield instance 29 | instance.flushdb() 30 | -------------------------------------------------------------------------------- /tests/hello.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from .conftest import redis_uri 3 | from tinyq.app import Application 4 | 5 | app = Application(redis_uri) 6 | 7 | 8 | @app.task() 9 | def hello(x, y): 10 | return x + y 11 | -------------------------------------------------------------------------------- /tests/test_app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | def test_app_delay(app): 5 | 6 | @app.task() 7 | def count(x, y): 8 | return x + y 9 | 10 | count.delay(1, 2) 11 | assert len(app.schedule_queue.connection.keys('*')) == 1 12 | assert app.schedule_queue.dequeue() 13 | 14 | 15 | def test_app_call(app): 16 | 17 | @app.task() 18 | def count(x, y): 19 | return x + y 20 | 21 | assert count(1, 2) == 3 22 | assert len(app.schedule_queue.connection.keys('*')) == 0 23 | -------------------------------------------------------------------------------- /tests/test_cli_worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from multiprocessing import Process 3 | import os 4 | import signal 5 | import time 6 | 7 | import pytest 8 | 9 | from .hello import redis_uri 10 | from tinyq.runner import main 11 | 12 | args = ['--uri', redis_uri, '--app', 'tests.hello.app'] 13 | 14 | 15 | def test_worker_import_error(): 16 | with pytest.raises(ImportError): 17 | main(args=[], start_now=False) 18 | 19 | 20 | def test_worker(): 21 | worker = main(args, start_now=False) 22 | worker.start_works() 23 | worker.setup_signal_handlers() 24 | assert len([p for p in worker.process_list if p.is_alive()]) == \ 25 | worker.worker_number + 2 26 | worker.stop() 27 | time.sleep(1) 28 | 29 | assert len([p for p in worker.process_list if p.is_alive()]) == 1 30 | 31 | 32 | def test_worker_sigint(): 33 | worker = main(args, start_now=False) 34 | process = Process(target=worker.start) 35 | process.start() 36 | time.sleep(1) 37 | os.kill(process.pid, signal.SIGINT) 38 | 39 | 40 | def test_worker_sigterm(): 41 | worker = main(args, start_now=False) 42 | process = Process(target=worker.start) 43 | process.start() 44 | time.sleep(1) 45 | os.kill(process.pid, signal.SIGTERM) 46 | -------------------------------------------------------------------------------- /tests/test_job.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pytest 3 | 4 | from tinyq.exceptions import ( 5 | SerializeError, DeserializeError, JobFailedError, TinyQError 6 | ) 7 | from tinyq.job import Job 8 | 9 | 10 | def test_delay_new_job(app): 11 | @app.task() 12 | def count(x, y): 13 | return x + y 14 | 15 | job = count.delay(2, 3) 16 | 17 | assert job.id 18 | assert job.task_name == 'count' 19 | assert job.run() == 5 20 | 21 | new_job = job.loads(job.dumps()) 22 | assert new_job._func is job._func 23 | assert new_job.run() == job.run() 24 | 25 | 26 | def test_job_dumps_error(app): 27 | @app.task() 28 | def test(x): 29 | return x 30 | 31 | job = Job(lambda: 'test', lambda: 'hello', {}) 32 | 33 | with pytest.raises(SerializeError): 34 | job.dumps() 35 | 36 | 37 | def test_job_loads_error(): 38 | 39 | with pytest.raises(DeserializeError): 40 | Job.loads('hello') 41 | 42 | 43 | def test_run_job_failed(): 44 | job = Job(lambda x: x + 1, lambda: 'hello', {}) 45 | 46 | with pytest.raises(JobFailedError): 47 | job.run() 48 | 49 | with pytest.raises(TinyQError): 50 | job.run() 51 | -------------------------------------------------------------------------------- /tests/test_task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from tinyq import task 6 | from tinyq.exceptions import TaskNameConflictError 7 | 8 | 9 | def test_app_task_name_default(app): 10 | @app.task() 11 | def count(x, y): 12 | return x + y 13 | 14 | assert 'count' in task._task_names 15 | assert task.get_func_via_task_name('count') 16 | 17 | 18 | def test_app_task_name_custom(app): 19 | @app.task(name='test') 20 | def count(x, y): 21 | return x + y 22 | 23 | assert 'test' in task._task_names 24 | assert task.get_func_via_task_name('test') 25 | 26 | 27 | def test_app_task_name_error(app): 28 | @app.task() 29 | def count(): pass 30 | 31 | with pytest.raises(TaskNameConflictError): 32 | @app.task() # noqa 33 | def count(): pass 34 | -------------------------------------------------------------------------------- /tests/test_worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from tinyq.worker import SchedulerWorker, JobWorker 3 | 4 | 5 | def test_scheduler_worker(app): 6 | schedule_queue = app.schedule_queue 7 | job_queue = app.job_queue 8 | worker = SchedulerWorker(schedule_queue, job_queue) 9 | 10 | @app.task() 11 | def count(x, y): 12 | return x + y 13 | 14 | count.delay(1, 3) 15 | assert len(schedule_queue.connection.keys(schedule_queue.key)) == 1 16 | assert len(job_queue.connection.keys(job_queue.key)) == 0 17 | assert worker.run_once() 18 | assert len(schedule_queue.connection.keys(schedule_queue.key)) == 0 19 | assert len(job_queue.connection.keys(job_queue.key)) == 1 20 | 21 | 22 | def test_scheduler_worker_no_job(app, redis_instance): 23 | schedule_queue = app.schedule_queue 24 | job_queue = app.job_queue 25 | worker = SchedulerWorker(schedule_queue, job_queue) 26 | 27 | assert len(schedule_queue.connection.keys(schedule_queue.key)) == 0 28 | assert len(job_queue.connection.keys(job_queue.key)) == 0 29 | assert worker.run_once() is None 30 | assert len(schedule_queue.connection.keys(schedule_queue.key)) == 0 31 | assert len(job_queue.connection.keys(job_queue.key)) == 0 32 | 33 | 34 | def test_job_worker(app): 35 | schedule_queue = app.schedule_queue 36 | job_queue = app.job_queue 37 | scheduler_worker = SchedulerWorker(schedule_queue, job_queue) 38 | job_worker = JobWorker(job_queue) 39 | 40 | @app.task() 41 | def count(x, y): 42 | return x + y 43 | 44 | count.delay(1, 3) 45 | assert len(job_queue.connection.keys(job_queue.key)) == 0 46 | assert scheduler_worker.run_once() 47 | assert len(job_queue.connection.keys(job_queue.key)) == 1 48 | assert job_worker.run_once() == 4 49 | assert len(job_queue.connection.keys(job_queue.key)) == 0 50 | 51 | 52 | def test_job_worker_no_job(app): 53 | schedule_queue = app.schedule_queue 54 | job_queue = app.job_queue 55 | scheduler_worker = SchedulerWorker(schedule_queue, job_queue) 56 | job_worker = JobWorker(job_queue) 57 | assert len(job_queue.connection.keys(job_queue.key)) == 0 58 | scheduler_worker.run_once() 59 | assert len(job_queue.connection.keys(job_queue.key)) == 0 60 | assert job_worker.run_once() is None 61 | assert len(job_queue.connection.keys(job_queue.key)) == 0 62 | -------------------------------------------------------------------------------- /tinyq/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from tinyq.app import Application # noqa 3 | 4 | __version__ = '0.3.0' 5 | __author__ = 'mozillazg' 6 | __license__ = 'MIT' 7 | __copyright__ = 'Copyright (c) 2017 mozillazg' 8 | -------------------------------------------------------------------------------- /tinyq/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | 4 | from tinyq import runner 5 | 6 | 7 | def main(): 8 | sys.exit(runner.main()) 9 | -------------------------------------------------------------------------------- /tinyq/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from tinyq.constants import DEFAULT_SCHEDULE_QUEUE_KEY, DEFAULT_JOB_QUEUE_KEY 3 | from tinyq.queue import RedisQueue 4 | from tinyq.task import Task 5 | 6 | 7 | class Application: 8 | """ 9 | : 10 | 11 | app = Application('redis://') 12 | 13 | @app.task() 14 | def add(m, n): 15 | return m + n 16 | 17 | add.delay(2, 3) 18 | 19 | """ 20 | def __init__(self, uri_or_instance='redis://', 21 | schedule_queue_key=DEFAULT_SCHEDULE_QUEUE_KEY, 22 | job_queue_key=DEFAULT_JOB_QUEUE_KEY): 23 | self.schedule_queue = RedisQueue(uri_or_instance, schedule_queue_key) 24 | self.job_queue = RedisQueue(uri_or_instance, job_queue_key) 25 | 26 | def task(self, name=None): 27 | """ 28 | 29 | :param name: task name 30 | :return: 31 | """ 32 | task = Task(self.schedule_queue) 33 | return task(name=name) 34 | -------------------------------------------------------------------------------- /tinyq/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | DEFAULT_SCHEDULE_QUEUE_KEY = 'schedules' 4 | DEFAULT_JOB_QUEUE_KEY = 'jobs' 5 | -------------------------------------------------------------------------------- /tinyq/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class TinyQError(Exception): 5 | pass 6 | 7 | 8 | class TaskNameConflictError(TinyQError): 9 | def __init__(self, name): 10 | super().__init__(name) 11 | 12 | self.name = name 13 | 14 | def __str__(self): 15 | return '<{cls_name}: Same name {task_name} already registered!'.format( 16 | cls_name=self.__class__.__name__, task_name=self.name 17 | ) 18 | 19 | 20 | class JobFailedError(TinyQError): 21 | def __init__(self, job): 22 | super().__init__(job) 23 | 24 | self.job = job 25 | 26 | def __str__(self): 27 | return '<{cls_name}: Run {job} failed!'.format( 28 | cls_name=self.__class__.__name__, job=self.job 29 | ) 30 | 31 | 32 | class SerializeError(TinyQError): 33 | def __init__(self, obj): 34 | super().__init__(obj) 35 | 36 | self.obj = obj 37 | 38 | def __str__(self): 39 | return '<{cls_name}: Can not serialize object: {obj}!'.format( 40 | cls_name=self.__class__.__name__, obj=repr(self.obj) 41 | ) 42 | 43 | 44 | class DeserializeError(TinyQError): 45 | def __init__(self, data): 46 | super().__init__(data) 47 | 48 | self.data = data 49 | 50 | def __str__(self): 51 | return '<{cls_name}: Can not deserialize data: {data}!'.format( 52 | cls_name=self.__class__.__name__, data=repr(self.data) 53 | ) 54 | -------------------------------------------------------------------------------- /tinyq/job.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import copy 3 | import logging 4 | import pickle 5 | import uuid 6 | 7 | from tinyq.exceptions import ( 8 | SerializeError, DeserializeError, 9 | JobFailedError 10 | ) 11 | from tinyq.utils import gen_task_name_via_func 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class Job: 17 | def __init__(self, func, func_args, func_kwargs, id=None): 18 | self._func = func 19 | self._func_args = func_args 20 | self._func_kwargs = func_kwargs 21 | self._task_name = gen_task_name_via_func(func) 22 | self._id = id or self._gen_id() 23 | 24 | @property 25 | def id(self): 26 | return self._id 27 | 28 | @property 29 | def task_name(self): 30 | return self._task_name 31 | 32 | def run(self): 33 | try: 34 | return self._func(*self._func_args, **self._func_kwargs) 35 | except Exception as e: 36 | raise JobFailedError(self) from e 37 | 38 | def dumps(self): 39 | obj = copy.deepcopy(self) 40 | obj._func = None 41 | try: 42 | return pickle.dumps(obj) 43 | except Exception as e: 44 | raise SerializeError(self) from e 45 | 46 | @staticmethod 47 | def loads(data): 48 | from tinyq.task import get_func_via_task_name 49 | try: 50 | obj = pickle.loads(data) 51 | except Exception as e: 52 | raise DeserializeError(data) from e 53 | 54 | obj._func = get_func_via_task_name(obj._task_name) 55 | return obj 56 | 57 | @staticmethod 58 | def _gen_id(): 59 | return str(uuid.uuid4()) 60 | 61 | def __str__(self): 62 | return ''.format( 63 | id=self.id, task_name=self.task_name 64 | ) 65 | 66 | def __repr__(self): 67 | return ( 68 | ''.format( 70 | id=self.id, task_name=self.task_name, args=self._func_args, 71 | kwargs=self._func_kwargs) 72 | ) 73 | -------------------------------------------------------------------------------- /tinyq/queue.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from redis import StrictRedis 3 | 4 | 5 | class RedisQueue: 6 | key_prefix = 'tinyq' 7 | 8 | def __init__(self, uri_or_instance, key): 9 | if isinstance(uri_or_instance, StrictRedis): 10 | self.connection = uri_or_instance 11 | else: 12 | self.connection = StrictRedis.from_url(uri_or_instance) 13 | self.key = '{prefix}:{key}'.format(prefix=self.key_prefix, key=key) 14 | 15 | def enqueue(self, data): 16 | """入队""" 17 | return self.connection.rpush(self.key, data) 18 | 19 | def dequeue(self): 20 | """出队""" 21 | return self.connection.lpop(self.key) 22 | -------------------------------------------------------------------------------- /tinyq/runner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import argparse 3 | from multiprocessing import cpu_count, current_process 4 | import logging 5 | import os 6 | import signal 7 | import sys 8 | 9 | from tinyq import __version__ 10 | from tinyq.worker import ( 11 | SchedulerWorker, JobWorker, ThreadWorkerCreator 12 | ) 13 | from tinyq.task import _task_names 14 | from tinyq.utils import import_object_from_path 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | def setup_logging(args_obj): 20 | if args_obj.verbose: 21 | level = logging.DEBUG 22 | else: 23 | level = logging.getLevelName(args_obj.log_level.upper()) 24 | formatter = logging.Formatter( 25 | fmt='%(asctime)-15s - %(levelname)s - %(name)s' 26 | '[line:%(lineno)d thread:%(threadName)s(%(thread)d) ' 27 | 'process:%(processName)s(%(process)d)]' 28 | ' - %(message)s' 29 | ) 30 | 31 | logger = logging.getLogger() 32 | logger.setLevel(level) 33 | handler = logging.StreamHandler() 34 | handler.setFormatter(formatter) 35 | logger.addHandler(handler) 36 | 37 | 38 | def parse_args(args): 39 | parser = argparse.ArgumentParser(description='Starts a TinyQ worker.') 40 | parser.add_argument('-V', '--version', action='version', 41 | version=__version__) 42 | parser.add_argument('-u', '--uri', default='redis://', 43 | help='The Redis URI (default: redis://)') 44 | parser.add_argument('-v', '--verbose', action='store_true', default=False, 45 | help='Show more output') 46 | parser.add_argument('-w', '--worker-number', type=int, default=cpu_count(), 47 | help='Worker number (default: {0})'.format(cpu_count()) 48 | ) 49 | parser.add_argument('-a', '--app', default='app.app', 50 | help='Application path (default: app.app)') 51 | parser.add_argument('-l', '--log-level', default='warn', 52 | choices=('debug', 'info', 'warn', 'error', 'critical'), 53 | help='Logging level (default: warn)') 54 | return parser.parse_args(args) 55 | 56 | 57 | class Worker: 58 | def __init__(self, schedule_queue, job_queue, worker_creator, 59 | worker_number=1, schedule_sleep_interval=0.1, 60 | worker_sleep_interval=0.1, 61 | main_stop_flag_timeout=0.1): 62 | self.schedule_queue = schedule_queue 63 | self.job_queue = job_queue 64 | self.scheduler_sleep_interval = schedule_sleep_interval 65 | self.worker_sleep_interval = worker_sleep_interval 66 | self.worker_creator = worker_creator() 67 | self.worker_number = worker_number 68 | self.received_stop = False 69 | self.main_stop_flag_timeout = main_stop_flag_timeout 70 | self.process_list = [current_process()] 71 | 72 | def start(self): # pragma: nocover 73 | logger.warn('Starting TinyQ worker, version {0}...'.format(__version__) 74 | ) 75 | logger.debug('Task names:\n{0}'.format( 76 | '\n'.join('* ' + name for name in _task_names) 77 | )) 78 | self.start_works() 79 | self.setup_signal_handlers() 80 | 81 | while True: 82 | try: 83 | self.worker_creator.stop_flag.wait(self.main_stop_flag_timeout) 84 | except KeyboardInterrupt: 85 | logger.warn('Received Ctrl + C, warm shutdown...') 86 | self.stop() 87 | except: 88 | logger.exception('Got exception, will exit worker') 89 | self.stop() 90 | else: 91 | if self.received_stop: 92 | self.stop() 93 | 94 | if self.worker_creator.is_stopped(): 95 | for process in self.process_list[1:]: 96 | if self.worker_creator.is_alive(process): 97 | break 98 | else: 99 | break 100 | 101 | logger.warn('Exit workers.') 102 | 103 | def start_works(self): 104 | logger.debug('Create scheduler worker.') 105 | scheduler = SchedulerWorker( 106 | self.schedule_queue, self.job_queue, 107 | sleep_interval=self.scheduler_sleep_interval 108 | ) 109 | scheduler_process = self.create_process(scheduler, name='Scheduler') 110 | self.process_list.append(scheduler_process) 111 | 112 | logger.debug('Create job workers.') 113 | job_worker_process_list = [] 114 | job_worker = JobWorker(self.job_queue, 115 | sleep_interval=self.worker_sleep_interval) 116 | for n in range(self.worker_number): 117 | process = self.create_process(job_worker, 118 | name='Worker-{0}'.format(n)) 119 | job_worker_process_list.append(process) 120 | self.process_list.append(process) 121 | 122 | logger.debug('Start scheduler worker...') 123 | scheduler_process.start() 124 | logger.debug('Start job workers...') 125 | for worker in job_worker_process_list: 126 | worker.start() 127 | 128 | def create_process(self, worker, name): 129 | def func(): 130 | try: 131 | logger.debug('Started worker: {0}'.format(name)) 132 | while not self.worker_creator.is_stopped(): 133 | worker.run_once() 134 | worker.sleep() 135 | logger.warn('Exit worker {0}.'.format(name)) 136 | except KeyboardInterrupt: 137 | pass 138 | except: 139 | logger.exception('Worker {0} dead!'.format(name)) 140 | return self.worker_creator.create(func, name) 141 | 142 | def setup_signal_handlers(self): 143 | def stop_process(signum, frame): 144 | logger.warn('Received stop signal, warm shutdown...') 145 | self.received_stop = True 146 | 147 | signal.signal(signal.SIGINT, stop_process) 148 | signal.signal(signal.SIGTERM, stop_process) 149 | 150 | def stop(self): 151 | self.worker_creator.set_stop() 152 | 153 | 154 | def get_app(app_path): 155 | try: 156 | app = import_object_from_path(app_path) 157 | return app 158 | except (ImportError, AttributeError): 159 | current_dir = os.getcwd() 160 | if current_dir not in sys.path: 161 | sys.path.insert(0, current_dir) 162 | return get_app(app_path) 163 | else: 164 | raise ImportError( 165 | 'Import App object from path "{0}" failed!'.format(app_path) 166 | ) 167 | 168 | 169 | def main(args=None, start_now=True): 170 | args_obj = parse_args(args) 171 | app_path = args_obj.app 172 | app = get_app(app_path) 173 | setup_logging(args_obj) 174 | 175 | worker = Worker(app.schedule_queue, app.job_queue, 176 | worker_creator=ThreadWorkerCreator, 177 | worker_number=args_obj.worker_number) 178 | if start_now: 179 | worker.start() 180 | return 181 | 182 | return worker 183 | 184 | 185 | if __name__ == '__main__': 186 | main() 187 | -------------------------------------------------------------------------------- /tinyq/task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import functools 3 | import logging 4 | 5 | from tinyq.exceptions import TaskNameConflictError 6 | from tinyq.job import Job 7 | from tinyq.utils import gen_task_name_via_func 8 | 9 | logger = logging.getLogger(__name__) 10 | _task_names = {} 11 | 12 | 13 | class Task: 14 | def __init__(self, schedule_queue): 15 | self.queue = schedule_queue 16 | 17 | def __call__(self, name=None): 18 | def wrapper(func, name=name): 19 | # 注册 task name 20 | if not name: 21 | name = gen_task_name_via_func(func) 22 | if name in _task_names: 23 | raise TaskNameConflictError(name) 24 | _task_names[name] = func 25 | 26 | delay_wrapper = DelayWrapper(self.queue, func) 27 | return functools.wraps(func)(delay_wrapper) 28 | 29 | return wrapper 30 | 31 | 32 | class DelayWrapper: 33 | """装饰函数，增加 delay 方法""" 34 | def __init__(self, schedule_queue, func): 35 | self.queue = schedule_queue 36 | self.func = func 37 | 38 | def delay(self, *args, **kwargs): 39 | logger.debug( 40 | 'Delay func({func!r}) with: args({args!r}), ' 41 | 'kwargs({kwargs!r})'.format( 42 | func=self.func, args=args, kwargs=kwargs 43 | ) 44 | ) 45 | job = Job(func=self.func, func_args=args, func_kwargs=kwargs) 46 | job_data = job.dumps() 47 | self.queue.enqueue(job_data) 48 | return job 49 | 50 | def __call__(self, *args, **kwargs): 51 | return self.func(*args, **kwargs) 52 | 53 | 54 | def get_func_via_task_name(name): 55 | return _task_names[name] 56 | -------------------------------------------------------------------------------- /tinyq/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import importlib 3 | 4 | 5 | def gen_task_name_via_func(func): 6 | """生成函数对象对应的 task name""" 7 | return '{name}'.format(name=func.__name__) 8 | 9 | 10 | def import_object_from_path(path, default_obj_name='app'): 11 | """从定义的字符串信息中导入对象 12 | 13 | :param path: ``task.app`` 14 | """ 15 | module_name, obj_name = path.rsplit('.', 1) 16 | if not obj_name: 17 | obj_name = default_obj_name 18 | 19 | module = importlib.import_module(module_name) 20 | return getattr(module, obj_name) 21 | -------------------------------------------------------------------------------- /tinyq/worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import abc 3 | import multiprocessing 4 | import logging 5 | import random 6 | import threading 7 | import time 8 | 9 | from tinyq.exceptions import JobFailedError 10 | from tinyq.job import Job 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class BaseWorker(metaclass=abc.ABCMeta): 16 | 17 | @abc.abstractmethod 18 | def run_once(self): 19 | pass 20 | 21 | @abc.abstractmethod 22 | def sleep(self): 23 | pass 24 | 25 | 26 | class SchedulerWorker(BaseWorker): 27 | def __init__(self, schedule_queue, job_queue, sleep_interval=1): 28 | self.queue = schedule_queue 29 | self.job_queue = job_queue 30 | self.sleep_interval = sleep_interval 31 | 32 | def run_once(self): 33 | try: 34 | job = self._get_job() 35 | if job is not None: 36 | logger.debug('Schedule new job: {job}.'.format(job=job)) 37 | return self._schedule_job(job) 38 | except: 39 | logger.exception('Raise an exception when schedule job!') 40 | 41 | def sleep(self): 42 | time.sleep(self.sleep_interval * (1 + random.SystemRandom().random())) 43 | 44 | def _get_job(self): 45 | data = self.queue.dequeue() 46 | if data is None: 47 | return 48 | 49 | return Job.loads(data) 50 | 51 | def _schedule_job(self, job): 52 | logger.debug('Put a new job({job}) into job queue.'.format(job=job)) 53 | return self.job_queue.enqueue(job.dumps()) 54 | 55 | 56 | class JobWorker(BaseWorker): 57 | def __init__(self, job_queue, sleep_interval=1): 58 | self.queue = job_queue 59 | self.sleep_interval = sleep_interval 60 | 61 | def run_once(self): 62 | try: 63 | job = self._get_job() 64 | if job is not None: 65 | logger.info('Got a job: {job}'.format(job=job)) 66 | result = self.run_job(job) 67 | logger.info('Finish run job {job}'.format(job=job)) 68 | return result 69 | except: 70 | logger.exception('Raise an exception when run job!') 71 | 72 | def sleep(self): 73 | time.sleep(self.sleep_interval * (1 + random.SystemRandom().random())) 74 | 75 | def _get_job(self): 76 | data = self.queue.dequeue() 77 | if data is None: 78 | return 79 | 80 | return Job.loads(data) 81 | 82 | def run_job(self, job): 83 | logger.debug('Start run a job: {job}'.format(job=job)) 84 | try: 85 | result = job.run() 86 | logger.debug('Run job({job!r}) success. Result: {result!r}'.format( 87 | job=job, result=result)) 88 | return result 89 | except JobFailedError as e: 90 | logger.exception('Run job {job} failed!'.format(job=job)) 91 | 92 | 93 | class BaseWorkerCreator(metaclass=abc.ABCMeta): 94 | 95 | @abc.abstractmethod 96 | def create(self, runnable, name): 97 | pass 98 | 99 | @abc.abstractmethod 100 | def set_stop(self): 101 | pass 102 | 103 | @abc.abstractmethod 104 | def is_stopped(self): 105 | pass 106 | 107 | @abc.abstractmethod 108 | def is_alive(self, process): 109 | pass 110 | 111 | 112 | class ThreadWorkerCreator(BaseWorkerCreator): 113 | def __init__(self): 114 | self.stop_flag = threading.Event() 115 | 116 | def create(self, runnable, name): 117 | thread = threading.Thread(target=runnable, name=name) 118 | return thread 119 | 120 | def set_stop(self): 121 | self.stop_flag.set() 122 | 123 | def is_stopped(self): 124 | return self.stop_flag.is_set() 125 | 126 | def is_alive(self, process): 127 | return process.is_alive() 128 | 129 | 130 | class ProcessWorkerCreator(BaseWorkerCreator): 131 | def __init__(self): 132 | self.stop_flag = multiprocessing.Event() 133 | 134 | def create(self, runnable, name): 135 | process = multiprocessing.Process(target=runnable, name=name) 136 | return process 137 | 138 | def set_stop(self): 139 | self.stop_flag.set() 140 | 141 | def is_stopped(self): 142 | return self.stop_flag.is_set() 143 | 144 | def is_alive(self, process): 145 | return process.is_alive() 146 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py26, py27, py33, py34, py35, py36, pypy, pypy3 8 | 9 | [base] 10 | deps = 11 | -r{toxinidir}/requirements_dev.txt 12 | 13 | [testenv] 14 | deps = {[base]deps} 15 | commands = py.test --cov tinyq tests 16 | passenv = 17 | TINYQ_TESTING_REDIS_URI 18 | --------------------------------------------------------------------------------