├── tests ├── __init__.py ├── test_class.py ├── conftest.py ├── test_timeout.py ├── test_collision.py └── instances.py ├── LICENSE ├── tox.ini ├── .gitignore ├── CONTRIBUTING.md ├── .travis.yml ├── appveyor.yml ├── setup.py ├── README.rst └── flask_celery.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Allows importing.""" 2 | -------------------------------------------------------------------------------- /tests/test_class.py: -------------------------------------------------------------------------------- 1 | """Test the Celery class.""" 2 | 3 | import pytest 4 | 5 | from flask_celery import Celery 6 | from tests.instances import app 7 | 8 | 9 | class FakeApp(object): 10 | """Mock Flask application.""" 11 | 12 | config = dict(CELERY_BROKER_URL='redis://localhost', CELERY_RESULT_BACKEND='redis://localhost') 13 | static_url_path = '' 14 | import_name = '' 15 | 16 | def register_blueprint(self, _): 17 | """Mock register_blueprint method.""" 18 | pass 19 | 20 | 21 | def test_multiple(): 22 | """Test attempted re-initialization of extension.""" 23 | assert 'celery' in app.extensions 24 | 25 | with pytest.raises(ValueError): 26 | Celery(app) 27 | 28 | 29 | def test_one_dumb_line(): 30 | """For test coverage.""" 31 | flask_app = FakeApp() 32 | Celery(flask_app) 33 | assert 'celery' in flask_app.extensions 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Robpol86 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Configure tests.""" 2 | 3 | import threading 4 | import time 5 | 6 | import pytest 7 | from celery.signals import worker_ready 8 | 9 | from tests.instances import app, celery 10 | 11 | WORKER_READY = list() 12 | 13 | 14 | class Worker(threading.Thread): 15 | """Run the Celery worker in a background thread.""" 16 | 17 | def run(self): 18 | """Run the thread.""" 19 | celery_args = ['-C', '-q', '-c', '1', '-P', 'solo', '--without-gossip'] 20 | with app.app_context(): 21 | celery.worker_main(celery_args) 22 | 23 | 24 | @worker_ready.connect 25 | def on_worker_ready(**_): 26 | """Called when the Celery worker thread is ready to do work. 27 | 28 | This is to avoid race conditions since everything is in one python process. 29 | """ 30 | WORKER_READY.append(True) 31 | 32 | 33 | @pytest.fixture(autouse=True, scope='session') 34 | def celery_worker(): 35 | """Start the Celery worker in a background thread.""" 36 | thread = Worker() 37 | thread.daemon = True 38 | thread.start() 39 | for i in range(10): # Wait for worker to finish initializing to avoid a race condition I've been experiencing. 40 | if WORKER_READY: 41 | break 42 | time.sleep(1) 43 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [general] 2 | install_requires = 3 | flask==0.12 4 | celery==3.1.11 5 | name = flask_celery 6 | 7 | [tox] 8 | envlist = lint,py{34,27} 9 | 10 | [testenv] 11 | commands = 12 | py.test --cov-report term-missing --cov-report xml --cov {[general]name} --cov-config tox.ini {posargs:tests} 13 | deps = 14 | {[general]install_requires} 15 | Flask-Redis-Helper==1.0.0 16 | Flask-SQLAlchemy==2.1 17 | pg8000==1.10.6 18 | PyMySQL==0.7.9 19 | pytest-cov==2.4.0 20 | passenv = 21 | BROKER 22 | usedevelop = True 23 | 24 | [testenv:lint] 25 | commands = 26 | python setup.py check --strict 27 | python setup.py check --strict -m 28 | python setup.py check --strict -s 29 | python setup.py check_version 30 | flake8 --application-import-names={[general]name},tests 31 | pylint --rcfile=tox.ini setup.py {[general]name} 32 | deps = 33 | {[general]install_requires} 34 | flake8-docstrings==1.0.3 35 | flake8-import-order==0.11 36 | flake8==3.2.1 37 | pep8-naming==0.4.1 38 | pylint==1.6.5 39 | 40 | [flake8] 41 | exclude = .tox/*,build/*,docs/*,env/*,get-pip.py 42 | import-order-style = smarkets 43 | max-line-length = 120 44 | statistics = True 45 | 46 | [pylint] 47 | disable = 48 | locally-disabled, 49 | missing-docstring, 50 | protected-access, 51 | too-few-public-methods, 52 | ignore = .tox/*,build/*,docs/*,env/*,get-pip.py 53 | max-args = 7 54 | max-line-length = 120 55 | reports = no 56 | 57 | [run] 58 | branch = True 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # Robpol86 92 | *.rpm 93 | .idea/ 94 | requirements*.txt 95 | .DS_Store 96 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Everyone that wants to contribute to the project should read this document. 4 | 5 | ## Getting Started 6 | 7 | You may follow these steps if you wish to create a pull request. Fork the repo and clone it on your local machine. Then 8 | in the project's directory: 9 | 10 | ```bash 11 | virtualenv env # Create a virtualenv for the project's dependencies. 12 | source env/bin/activate # Activate the virtualenv. 13 | pip install tox # Install tox, which runs linting and tests. 14 | tox # This runs all tests on your local machine. Make sure they pass. 15 | ``` 16 | 17 | If you don't have Python 2.7 or 3.4 installed you can manually run tests on one specific version by running 18 | `tox -e lint,py35` (for Python 3.5) instead. 19 | 20 | ## Consistency and Style 21 | 22 | Keep code style consistent with the rest of the project. Some suggestions: 23 | 24 | 1. **Write tests for your new features.** `if new_feature else` **Write tests for bug-causing scenarios.** 25 | 2. Write docstrings for all classes, functions, methods, modules, etc. 26 | 3. Document all function/method arguments and return values. 27 | 4. Document all class variables instance variables. 28 | 5. Documentation guidelines also apply to tests, though not as strict. 29 | 6. Keep code style consistent, such as the kind of quotes to use and spacing. 30 | 7. Don't use `except:` or `except Exception:` unless you have a `raise` in the block. Be specific about error handling. 31 | 8. Don't use `isinstance()` (it breaks [duck typing](https://en.wikipedia.org/wiki/Duck_typing#In_Python)). 32 | 33 | ## Thanks 34 | 35 | Thanks for fixing bugs or adding features to the project! 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Configure. 2 | language: python 3 | python: 4 | - 3.4 5 | - 3.3 6 | - pypy 7 | - 2.7 8 | - 2.6 9 | services: [redis-server] 10 | sudo: false 11 | 12 | # Environment and matrix. 13 | env: 14 | - BROKER: sqlite 15 | - BROKER: mysql 16 | - BROKER: postgres 17 | - BROKER: redis 18 | - BROKER: redis_sock,/tmp/redis.sock 19 | matrix: 20 | include: 21 | - python: 3.4 22 | services: [] 23 | env: TOX_ENV=lint 24 | before_script: [] 25 | after_success: [] 26 | 27 | # Run. 28 | install: pip install tox 29 | before_script: 30 | - if [[ $BROKER == redis_sock* ]]; then echo -e "daemonize yes\nunixsocket /tmp/redis.sock\nport 0" |redis-server -; fi 31 | - if [ $BROKER == mysql ]; then mysql -u root -e 'CREATE DATABASE flask_celery_helper_test;'; fi 32 | - if [ $BROKER == mysql ]; then mysql -u root -e 'GRANT ALL PRIVILEGES ON flask_celery_helper_test.* TO "user"@"localhost" IDENTIFIED BY "pass";'; fi 33 | - if [ $BROKER == postgres ]; then psql -U postgres -c 'CREATE DATABASE flask_celery_helper_test;'; fi 34 | - if [ $BROKER == postgres ]; then psql -U postgres -c "CREATE USER user1 WITH PASSWORD 'pass';"; fi 35 | - if [ $BROKER == postgres ]; then psql -U postgres -c 'GRANT ALL PRIVILEGES ON DATABASE flask_celery_helper_test TO user1;'; fi 36 | script: tox -e ${TOX_ENV:-py} 37 | after_success: 38 | - bash <(curl -s https://codecov.io/bash) 39 | 40 | # Deploy. 41 | deploy: 42 | provider: pypi 43 | user: Robpol86 44 | password: 45 | secure: 46 | "liwn5bHqjAtW+gRX6r4VgWuc44OUwfGSne4fTxb6G2pnPNW/IneVspQ2bFXeuQDdXzyLoOe\ 47 | bKa8bxjRurUEHedjV9UG9fVZwVsWU981aWOxeEl+6kLkpJ2fE9UVeK7T1O+RzzhkWhHq2/YL\ 48 | 4BjBqzOLuBSAGnXZAnwH55Z6HY2g=" 49 | on: 50 | condition: $TRAVIS_PYTHON_VERSION = 3.4 51 | tags: true 52 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # Configure. 2 | services: 3 | - mysql 4 | - postgresql 5 | 6 | # Environment and matrix. 7 | environment: 8 | PATH: C:\%PYTHON%;C:\%PYTHON%\Scripts;C:\Program Files\MySQL\MySQL Server 5.7\bin;C:\Program Files\PostgreSQL\9.5\bin;%PATH% 9 | PGPASSWORD: Password12! 10 | PYTHON: Python34 11 | matrix: 12 | - TOX_ENV: lint 13 | BROKER: sqlite 14 | - TOX_ENV: py34 15 | BROKER: sqlite 16 | - TOX_ENV: py33 17 | BROKER: sqlite 18 | - TOX_ENV: py27 19 | BROKER: sqlite 20 | - TOX_ENV: py 21 | PYTHON: Python34-x64 22 | BROKER: sqlite 23 | - TOX_ENV: py 24 | PYTHON: Python33-x64 25 | BROKER: sqlite 26 | - TOX_ENV: py 27 | PYTHON: Python27-x64 28 | BROKER: sqlite 29 | 30 | - TOX_ENV: lint 31 | BROKER: mysql 32 | - TOX_ENV: py34 33 | BROKER: mysql 34 | - TOX_ENV: py33 35 | BROKER: mysql 36 | - TOX_ENV: py27 37 | BROKER: mysql 38 | - TOX_ENV: py 39 | PYTHON: Python34-x64 40 | BROKER: mysql 41 | - TOX_ENV: py 42 | PYTHON: Python33-x64 43 | BROKER: mysql 44 | - TOX_ENV: py 45 | PYTHON: Python27-x64 46 | BROKER: mysql 47 | 48 | - TOX_ENV: lint 49 | BROKER: postgres 50 | - TOX_ENV: py34 51 | BROKER: postgres 52 | - TOX_ENV: py33 53 | BROKER: postgres 54 | - TOX_ENV: py27 55 | BROKER: postgres 56 | - TOX_ENV: py 57 | PYTHON: Python34-x64 58 | BROKER: postgres 59 | - TOX_ENV: py 60 | PYTHON: Python33-x64 61 | BROKER: postgres 62 | - TOX_ENV: py 63 | PYTHON: Python27-x64 64 | BROKER: postgres 65 | 66 | - TOX_ENV: lint 67 | BROKER: redis 68 | - TOX_ENV: py34 69 | BROKER: redis 70 | - TOX_ENV: py33 71 | BROKER: redis 72 | - TOX_ENV: py27 73 | BROKER: redis 74 | - TOX_ENV: py 75 | PYTHON: Python34-x64 76 | BROKER: redis 77 | - TOX_ENV: py 78 | PYTHON: Python33-x64 79 | BROKER: redis 80 | - TOX_ENV: py 81 | PYTHON: Python27-x64 82 | BROKER: redis 83 | 84 | # Run. 85 | build_script: pip install tox 86 | after_build: 87 | - IF %BROKER% EQU redis cinst redis-64 88 | - IF %BROKER% EQU redis redis-server --service-install 89 | - IF %BROKER% EQU redis redis-server --service-start 90 | - IF %BROKER% EQU mysql mysql -u root -p"Password12!" -e "CREATE DATABASE flask_celery_helper_test;" 91 | - IF %BROKER% EQU mysql mysql -u root -p"Password12!" -e "GRANT ALL PRIVILEGES ON flask_celery_helper_test.* TO 'user'@'localhost' IDENTIFIED BY 'pass';" 92 | - IF %BROKER% EQU postgres psql -U postgres -c "CREATE DATABASE flask_celery_helper_test;" 93 | - IF %BROKER% EQU postgres psql -U postgres -c "CREATE USER user1 WITH PASSWORD 'pass';" 94 | - IF %BROKER% EQU postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE flask_celery_helper_test TO user1;" 95 | test_script: tox -e %TOX_ENV% 96 | on_success: IF %TOX_ENV% NEQ lint pip install codecov & codecov 97 | -------------------------------------------------------------------------------- /tests/test_timeout.py: -------------------------------------------------------------------------------- 1 | """Test single-instance lock timeout.""" 2 | 3 | import time 4 | 5 | import pytest 6 | 7 | from flask_celery import _select_manager, OtherInstanceError 8 | from tests.instances import celery 9 | 10 | 11 | @pytest.mark.parametrize('task_name,timeout', [ 12 | ('tests.instances.mul', 20), ('tests.instances.add', 300), ('tests.instances.add2', 70), 13 | ('tests.instances.add3', 80) 14 | ]) 15 | def test_instances(task_name, timeout): 16 | """Test task instances.""" 17 | manager_class = _select_manager(celery.backend.__class__.__name__) 18 | manager_instance = list() 19 | task = celery.tasks[task_name] 20 | original_exit = manager_class.__exit__ 21 | 22 | def new_exit(self, *_): 23 | manager_instance.append(self) 24 | return original_exit(self, *_) 25 | setattr(manager_class, '__exit__', new_exit) 26 | task.apply_async(args=(4, 4)).get() 27 | setattr(manager_class, '__exit__', original_exit) 28 | assert timeout == manager_instance[0].timeout 29 | 30 | 31 | @pytest.mark.parametrize('key,value', [('CELERYD_TASK_TIME_LIMIT', 200), ('CELERYD_TASK_SOFT_TIME_LIMIT', 100)]) 32 | def test_settings(key, value): 33 | """Test different Celery time limit settings.""" 34 | celery.conf.update({key: value}) 35 | manager_class = _select_manager(celery.backend.__class__.__name__) 36 | manager_instance = list() 37 | original_exit = manager_class.__exit__ 38 | 39 | def new_exit(self, *_): 40 | manager_instance.append(self) 41 | return original_exit(self, *_) 42 | setattr(manager_class, '__exit__', new_exit) 43 | tasks = [ 44 | ('tests.instances.mul', 20), ('tests.instances.add', value), ('tests.instances.add2', 70), 45 | ('tests.instances.add3', 80) 46 | ] 47 | 48 | for task_name, timeout in tasks: 49 | task = celery.tasks[task_name] 50 | task.apply_async(args=(4, 4)).get() 51 | assert timeout == manager_instance.pop().timeout 52 | setattr(manager_class, '__exit__', original_exit) 53 | 54 | celery.conf.update({key: None}) 55 | 56 | 57 | def test_expired(): 58 | """Test timeout expired task instances.""" 59 | celery.conf.update({'CELERYD_TASK_TIME_LIMIT': 5}) 60 | manager_class = _select_manager(celery.backend.__class__.__name__) 61 | manager_instance = list() 62 | task = celery.tasks['tests.instances.add'] 63 | original_exit = manager_class.__exit__ 64 | 65 | def new_exit(self, *_): 66 | manager_instance.append(self) 67 | return None 68 | setattr(manager_class, '__exit__', new_exit) 69 | 70 | # Run the task and don't remove the lock after a successful run. 71 | assert 8 == task.apply_async(args=(4, 4)).get() 72 | setattr(manager_class, '__exit__', original_exit) 73 | 74 | # Run again, lock is still active so this should fail. 75 | with pytest.raises(OtherInstanceError): 76 | task.apply_async(args=(4, 4)).get() 77 | 78 | # Wait 5 seconds (per CELERYD_TASK_TIME_LIMIT), then re-run, should work. 79 | time.sleep(5) 80 | assert 8 == task.apply_async(args=(4, 4)).get() 81 | celery.conf.update({'CELERYD_TASK_TIME_LIMIT': None}) 82 | -------------------------------------------------------------------------------- /tests/test_collision.py: -------------------------------------------------------------------------------- 1 | """Test single-instance collision.""" 2 | 3 | import pytest 4 | 5 | from flask_celery import _select_manager, OtherInstanceError 6 | from tests.instances import celery 7 | 8 | PARAMS = [('tests.instances.add', 8), ('tests.instances.mul', 16), ('tests.instances.sub', 0)] 9 | 10 | 11 | @pytest.mark.parametrize('task_name,expected', PARAMS) 12 | def test_basic(task_name, expected): 13 | """Test no collision.""" 14 | task = celery.tasks[task_name] 15 | assert expected == task.apply_async(args=(4, 4)).get() 16 | 17 | 18 | @pytest.mark.parametrize('task_name,expected', PARAMS) 19 | def test_collision(task_name, expected): 20 | """Test single-instance collision.""" 21 | manager_class = _select_manager(celery.backend.__class__.__name__) 22 | manager_instance = list() 23 | task = celery.tasks[task_name] 24 | 25 | # First run the task and prevent it from removing the lock. 26 | def new_exit(self, *_): 27 | manager_instance.append(self) 28 | return None 29 | original_exit = manager_class.__exit__ 30 | setattr(manager_class, '__exit__', new_exit) 31 | assert expected == task.apply_async(args=(4, 4)).get() 32 | setattr(manager_class, '__exit__', original_exit) 33 | assert manager_instance[0].is_already_running is True 34 | 35 | # Now run it again. 36 | with pytest.raises(OtherInstanceError) as e: 37 | task.apply_async(args=(4, 4)).get() 38 | if manager_instance[0].include_args: 39 | assert str(e.value).startswith('Failed to acquire lock, {0}.args.'.format(task_name)) 40 | else: 41 | assert 'Failed to acquire lock, {0} already running.'.format(task_name) == str(e.value) 42 | assert manager_instance[0].is_already_running is True 43 | 44 | # Clean up. 45 | manager_instance[0].reset_lock() 46 | assert manager_instance[0].is_already_running is False 47 | 48 | # Once more. 49 | assert expected == task.apply_async(args=(4, 4)).get() 50 | 51 | 52 | def test_include_args(): 53 | """Test single-instance collision with task arguments taken into account.""" 54 | manager_class = _select_manager(celery.backend.__class__.__name__) 55 | manager_instance = list() 56 | task = celery.tasks['tests.instances.mul'] 57 | 58 | # First run the tasks and prevent them from removing the locks. 59 | def new_exit(self, *_): 60 | """Expected to be run twice.""" 61 | manager_instance.append(self) 62 | return None 63 | original_exit = manager_class.__exit__ 64 | setattr(manager_class, '__exit__', new_exit) 65 | assert 16 == task.apply_async(args=(4, 4)).get() 66 | assert 20 == task.apply_async(args=(5, 4)).get() 67 | setattr(manager_class, '__exit__', original_exit) 68 | assert manager_instance[0].is_already_running is True 69 | assert manager_instance[1].is_already_running is True 70 | 71 | # Now run them again. 72 | with pytest.raises(OtherInstanceError) as e: 73 | task.apply_async(args=(4, 4)).get() 74 | assert str(e.value).startswith('Failed to acquire lock, tests.instances.mul.args.') 75 | assert manager_instance[0].is_already_running is True 76 | with pytest.raises(OtherInstanceError) as e: 77 | task.apply_async(args=(5, 4)).get() 78 | assert str(e.value).startswith('Failed to acquire lock, tests.instances.mul.args.') 79 | assert manager_instance[1].is_already_running is True 80 | 81 | # Clean up. 82 | manager_instance[0].reset_lock() 83 | assert manager_instance[0].is_already_running is False 84 | manager_instance[1].reset_lock() 85 | assert manager_instance[1].is_already_running is False 86 | 87 | # Once more. 88 | assert 16 == task.apply_async(args=(4, 4)).get() 89 | assert 20 == task.apply_async(args=(5, 4)).get() 90 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Setup script for the project.""" 3 | 4 | import codecs 5 | import os 6 | import re 7 | 8 | from setuptools import Command, setup 9 | 10 | IMPORT = 'flask_celery' 11 | INSTALL_REQUIRES = ['flask', 'celery'] 12 | LICENSE = 'MIT' 13 | NAME = 'Flask-Celery-Helper' 14 | VERSION = '1.1.0' 15 | 16 | 17 | def readme(path='README.rst'): 18 | """Try to read README.rst or return empty string if failed. 19 | 20 | :param str path: Path to README file. 21 | 22 | :return: File contents. 23 | :rtype: str 24 | """ 25 | path = os.path.realpath(os.path.join(os.path.dirname(__file__), path)) 26 | handle = None 27 | url_prefix = 'https://raw.githubusercontent.com/Robpol86/{name}/v{version}/'.format(name=NAME, version=VERSION) 28 | try: 29 | handle = codecs.open(path, encoding='utf-8') 30 | return handle.read(131072).replace('.. image:: docs', '.. image:: {0}docs'.format(url_prefix)) 31 | except IOError: 32 | return '' 33 | finally: 34 | getattr(handle, 'close', lambda: None)() 35 | 36 | 37 | class CheckVersion(Command): 38 | """Make sure version strings and other metadata match here, in module/package, tox, and other places.""" 39 | 40 | description = 'verify consistent version/etc strings in project' 41 | user_options = [] 42 | 43 | @classmethod 44 | def initialize_options(cls): 45 | """Required by distutils.""" 46 | pass 47 | 48 | @classmethod 49 | def finalize_options(cls): 50 | """Required by distutils.""" 51 | pass 52 | 53 | @classmethod 54 | def run(cls): 55 | """Check variables.""" 56 | project = __import__(IMPORT, fromlist=['']) 57 | for expected, var in [('@Robpol86', '__author__'), (LICENSE, '__license__'), (VERSION, '__version__')]: 58 | if getattr(project, var) != expected: 59 | raise SystemExit('Mismatch: {0}'.format(var)) 60 | # Check changelog. 61 | if not re.compile(r'^%s - \d{4}-\d{2}-\d{2}[\r\n]' % VERSION, re.MULTILINE).search(readme()): 62 | raise SystemExit('Version not found in readme/changelog file.') 63 | # Check tox. 64 | if INSTALL_REQUIRES: 65 | contents = readme('tox.ini') 66 | section = re.compile(r'[\r\n]+install_requires =[\r\n]+(.+?)[\r\n]+\w', re.DOTALL).findall(contents) 67 | if not section: 68 | raise SystemExit('Missing install_requires section in tox.ini.') 69 | in_tox = re.findall(r' ([^=]+)==[\w\d.-]+', section[0]) 70 | if INSTALL_REQUIRES != in_tox: 71 | raise SystemExit('Missing/unordered pinned dependencies in tox.ini.') 72 | 73 | 74 | if __name__ == '__main__': 75 | setup( 76 | author='@Robpol86', 77 | author_email='robpol86@gmail.com', 78 | classifiers=[ 79 | 'Development Status :: 5 - Production/Stable', 80 | 'Environment :: Web Environment', 81 | 'Environment :: MacOS X', 82 | 'Environment :: Win32 (MS Windows)', 83 | 'Framework :: Flask', 84 | 'Intended Audience :: Developers', 85 | 'License :: OSI Approved :: MIT License', 86 | 'Operating System :: MacOS :: MacOS X', 87 | 'Operating System :: Microsoft :: Windows', 88 | 'Operating System :: POSIX', 89 | 'Operating System :: POSIX :: Linux', 90 | 'Programming Language :: Python :: 2.6', 91 | 'Programming Language :: Python :: 2.7', 92 | 'Programming Language :: Python :: 3.3', 93 | 'Programming Language :: Python :: 3.4', 94 | 'Programming Language :: Python :: Implementation :: PyPy', 95 | 'Topic :: Software Development :: Libraries', 96 | ], 97 | cmdclass=dict(check_version=CheckVersion), 98 | description='Celery support for Flask without breaking PyCharm inspections.', 99 | install_requires=INSTALL_REQUIRES, 100 | keywords='flask celery redis', 101 | license=LICENSE, 102 | long_description=readme(), 103 | name=NAME, 104 | py_modules=[IMPORT], 105 | url='https://github.com/Robpol86/' + NAME, 106 | version=VERSION, 107 | zip_safe=False, 108 | ) 109 | -------------------------------------------------------------------------------- /tests/instances.py: -------------------------------------------------------------------------------- 1 | """Handle Flask and Celery application global-instances.""" 2 | 3 | import os 4 | 5 | from flask import Flask 6 | from flask_redis import Redis 7 | from flask_sqlalchemy import SQLAlchemy 8 | 9 | from flask_celery import Celery, single_instance 10 | 11 | 12 | def generate_config(): 13 | """Generate a Flask config dict with settings for a specific broker based on an environment variable. 14 | 15 | To be merged into app.config. 16 | 17 | :return: Flask config to be fed into app.config.update(). 18 | :rtype: dict 19 | """ 20 | config = dict() 21 | 22 | if os.environ.get('BROKER') == 'rabbit': 23 | config['CELERY_BROKER_URL'] = 'amqp://user:pass@localhost//' 24 | elif os.environ.get('BROKER') == 'redis': 25 | config['REDIS_URL'] = 'redis://localhost/1' 26 | config['CELERY_BROKER_URL'] = config['REDIS_URL'] 27 | elif os.environ.get('BROKER', '').startswith('redis_sock,'): 28 | config['REDIS_URL'] = 'redis+socket://' + os.environ['BROKER'].split(',', 1)[1] 29 | config['CELERY_BROKER_URL'] = config['REDIS_URL'] 30 | elif os.environ.get('BROKER') == 'mongo': 31 | config['CELERY_BROKER_URL'] = 'mongodb://user:pass@localhost/test' 32 | elif os.environ.get('BROKER') == 'couch': 33 | config['CELERY_BROKER_URL'] = 'couchdb://user:pass@localhost/test' 34 | elif os.environ.get('BROKER') == 'beanstalk': 35 | config['CELERY_BROKER_URL'] = 'beanstalk://user:pass@localhost/test' 36 | elif os.environ.get('BROKER') == 'iron': 37 | config['CELERY_BROKER_URL'] = 'ironmq://project:token@/test' 38 | else: 39 | if os.environ.get('BROKER') == 'mysql': 40 | config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://user:pass@localhost/flask_celery_helper_test' 41 | elif os.environ.get('BROKER') == 'postgres': 42 | config['SQLALCHEMY_DATABASE_URI'] = 'postgresql+pg8000://user1:pass@localhost/flask_celery_helper_test' 43 | else: 44 | file_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test_database.sqlite') 45 | config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + file_path 46 | config['CELERY_BROKER_URL'] = 'sqla+' + config['SQLALCHEMY_DATABASE_URI'] 47 | config['CELERY_RESULT_BACKEND'] = 'db+' + config['SQLALCHEMY_DATABASE_URI'] 48 | 49 | if 'CELERY_BROKER_URL' in config and 'CELERY_RESULT_BACKEND' not in config: 50 | config['CELERY_RESULT_BACKEND'] = config['CELERY_BROKER_URL'] 51 | 52 | return config 53 | 54 | 55 | def generate_context(config): 56 | """Create the Flask app context and initializes any extensions such as Celery, Redis, SQLAlchemy, etc. 57 | 58 | :param dict config: Partial Flask config dict from generate_config(). 59 | 60 | :return: The Flask app instance. 61 | """ 62 | flask_app = Flask(__name__) 63 | flask_app.config.update(config) 64 | flask_app.config['TESTING'] = True 65 | flask_app.config['CELERY_ACCEPT_CONTENT'] = ['pickle'] 66 | 67 | if 'SQLALCHEMY_DATABASE_URI' in flask_app.config: 68 | db = SQLAlchemy(flask_app) 69 | db.engine.execute('DROP TABLE IF EXISTS celery_tasksetmeta;') 70 | elif 'REDIS_URL' in flask_app.config: 71 | redis = Redis(flask_app) 72 | redis.flushdb() 73 | 74 | Celery(flask_app) 75 | return flask_app 76 | 77 | 78 | def get_flask_celery_apps(): 79 | """Call generate_context() and generate_config(). 80 | 81 | :return: First item is the Flask app instance, second is the Celery app instance. 82 | :rtype: tuple 83 | """ 84 | config = generate_config() 85 | flask_app = generate_context(config=config) 86 | celery_app = flask_app.extensions['celery'].celery 87 | return flask_app, celery_app 88 | 89 | 90 | app, celery = get_flask_celery_apps() 91 | 92 | 93 | @celery.task(bind=True) 94 | @single_instance 95 | def add(x, y): 96 | """Celery task: add numbers.""" 97 | return x + y 98 | 99 | 100 | @celery.task(bind=True) 101 | @single_instance(include_args=True, lock_timeout=20) 102 | def mul(x, y): 103 | """Celery task: multiply numbers.""" 104 | return x * y 105 | 106 | 107 | @celery.task(bind=True) 108 | @single_instance() 109 | def sub(x, y): 110 | """Celery task: subtract numbers.""" 111 | return x - y 112 | 113 | 114 | @celery.task(bind=True, time_limit=70) 115 | @single_instance 116 | def add2(x, y): 117 | """Celery task: add numbers.""" 118 | return x + y 119 | 120 | 121 | @celery.task(bind=True, soft_time_limit=80) 122 | @single_instance 123 | def add3(x, y): 124 | """Celery task: add numbers.""" 125 | return x + y 126 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Flask-Celery-Helper 3 | =================== 4 | 5 | Even though the `Flask documentation `_ says Celery extensions are 6 | unnecessary now, I found that I still need an extension to properly use Celery in large Flask applications. Specifically 7 | I need an init_app() method to initialize Celery after I instantiate it. 8 | 9 | This extension also comes with a ``single_instance`` method. 10 | 11 | * Python 2.6, 2.7, PyPy, 3.3, and 3.4 supported on Linux and OS X. 12 | * Python 2.7, 3.3, and 3.4 supported on Windows (both 32 and 64 bit versions of Python). 13 | 14 | .. image:: https://img.shields.io/appveyor/ci/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square&label=AppVeyor%20CI 15 | :target: https://ci.appveyor.com/project/Robpol86/Flask-Celery-Helper 16 | :alt: Build Status Windows 17 | 18 | .. image:: https://img.shields.io/travis/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square&label=Travis%20CI 19 | :target: https://travis-ci.org/Robpol86/Flask-Celery-Helper 20 | :alt: Build Status 21 | 22 | .. image:: https://img.shields.io/codecov/c/github/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square&label=Codecov 23 | :target: https://codecov.io/gh/Robpol86/Flask-Celery-Helper 24 | :alt: Coverage Status 25 | 26 | .. image:: https://img.shields.io/pypi/v/Flask-Celery-Helper.svg?style=flat-square&label=Latest 27 | :target: https://pypi.python.org/pypi/Flask-Celery-Helper 28 | :alt: Latest Version 29 | 30 | Attribution 31 | =========== 32 | 33 | Single instance decorator inspired by 34 | `Ryan Roemer `_. 35 | 36 | Supported Libraries 37 | =================== 38 | 39 | * `Flask `_ 0.12 40 | * `Redis `_ 3.2.6 41 | * `Celery `_ 3.1.11 42 | 43 | Quickstart 44 | ========== 45 | 46 | Install: 47 | 48 | .. code:: bash 49 | 50 | pip install Flask-Celery-Helper 51 | 52 | Examples 53 | ======== 54 | 55 | Basic Example 56 | ------------- 57 | 58 | .. code:: python 59 | 60 | # example.py 61 | from flask import Flask 62 | from flask_celery import Celery 63 | 64 | app = Flask('example') 65 | app.config['CELERY_BROKER_URL'] = 'redis://localhost' 66 | app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost' 67 | celery = Celery(app) 68 | 69 | @celery.task() 70 | def add_together(a, b): 71 | return a + b 72 | 73 | if __name__ == '__main__': 74 | result = add_together.delay(23, 42) 75 | print(result.get()) 76 | 77 | Run these two commands in separate terminals: 78 | 79 | .. code:: bash 80 | 81 | celery -A example.celery worker 82 | python example.py 83 | 84 | Factory Example 85 | --------------- 86 | 87 | .. code:: python 88 | 89 | # extensions.py 90 | from flask_celery import Celery 91 | 92 | celery = Celery() 93 | 94 | .. code:: python 95 | 96 | # application.py 97 | from flask import Flask 98 | from extensions import celery 99 | 100 | def create_app(): 101 | app = Flask(__name__) 102 | app.config['CELERY_IMPORTS'] = ('tasks.add_together', ) 103 | app.config['CELERY_BROKER_URL'] = 'redis://localhost' 104 | app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost' 105 | celery.init_app(app) 106 | return app 107 | 108 | .. code:: python 109 | 110 | # tasks.py 111 | from extensions import celery 112 | 113 | @celery.task() 114 | def add_together(a, b): 115 | return a + b 116 | 117 | .. code:: python 118 | 119 | # manage.py 120 | from application import create_app 121 | 122 | app = create_app() 123 | app.run() 124 | 125 | Single Instance Example 126 | ----------------------- 127 | 128 | .. code:: python 129 | 130 | # example.py 131 | import time 132 | from flask import Flask 133 | from flask_celery import Celery, single_instance 134 | from flask_redis import Redis 135 | 136 | app = Flask('example') 137 | app.config['REDIS_URL'] = 'redis://localhost' 138 | app.config['CELERY_BROKER_URL'] = 'redis://localhost' 139 | app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost' 140 | celery = Celery(app) 141 | Redis(app) 142 | 143 | @celery.task(bind=True) 144 | @single_instance 145 | def sleep_one_second(a, b): 146 | time.sleep(1) 147 | return a + b 148 | 149 | if __name__ == '__main__': 150 | task1 = sleep_one_second.delay(23, 42) 151 | time.sleep(0.1) 152 | task2 = sleep_one_second.delay(20, 40) 153 | results1 = task1.get(propagate=False) 154 | results2 = task2.get(propagate=False) 155 | print(results1) # 65 156 | if isinstance(results2, Exception) and str(results2) == 'Failed to acquire lock.': 157 | print('Another instance is already running.') 158 | else: 159 | print(results2) # Should not happen. 160 | 161 | .. changelog-section-start 162 | 163 | Changelog 164 | ========= 165 | 166 | This project adheres to `Semantic Versioning `_. 167 | 168 | Unreleased 169 | ---------- 170 | 171 | Changed 172 | * Supporting Flask 0.12, switching from ``flask.ext.celery`` to ``flask_celery`` import recommendation. 173 | 174 | 1.1.0 - 2014-12-28 175 | ------------------ 176 | 177 | Added 178 | * Windows support. 179 | * ``single_instance`` supported on SQLite/MySQL/PostgreSQL in addition to Redis. 180 | 181 | Changed 182 | * ``CELERY_RESULT_BACKEND`` no longer mandatory. 183 | * Breaking changes: ``flask.ext.celery.CELERY_LOCK`` moved to ``flask.ext.celery._LockManagerRedis.CELERY_LOCK``. 184 | 185 | 1.0.0 - 2014-11-01 186 | ------------------ 187 | 188 | Added 189 | * Support for non-Redis backends. 190 | 191 | 0.2.2 - 2014-08-11 192 | ------------------ 193 | 194 | Added 195 | * Python 2.6 and 3.x support. 196 | 197 | 0.2.1 - 2014-06-18 198 | ------------------ 199 | 200 | Fixed 201 | * ``single_instance`` arguments with functools. 202 | 203 | 0.2.0 - 2014-06-18 204 | ------------------ 205 | 206 | Added 207 | * ``include_args`` argument to ``single_instance``. 208 | 209 | 0.1.0 - 2014-06-01 210 | ------------------ 211 | 212 | * Initial release. 213 | 214 | .. changelog-section-end 215 | -------------------------------------------------------------------------------- /flask_celery.py: -------------------------------------------------------------------------------- 1 | """Celery support for Flask without breaking PyCharm inspections. 2 | 3 | https://github.com/Robpol86/Flask-Celery-Helper 4 | https://pypi.python.org/pypi/Flask-Celery-Helper 5 | """ 6 | 7 | import hashlib 8 | from datetime import datetime, timedelta 9 | from functools import partial, wraps 10 | from logging import getLogger 11 | 12 | from celery import _state, Celery as CeleryClass 13 | 14 | __author__ = '@Robpol86' 15 | __license__ = 'MIT' 16 | __version__ = '1.1.0' 17 | 18 | 19 | class OtherInstanceError(Exception): 20 | """Raised when Celery task is already running, when lock exists and has not timed out.""" 21 | 22 | pass 23 | 24 | 25 | class _LockManager(object): 26 | """Base class for other lock managers.""" 27 | 28 | def __init__(self, celery_self, timeout, include_args, args, kwargs): 29 | """May raise NotImplementedError if the Celery backend is not supported. 30 | 31 | :param celery_self: From wrapped() within single_instance(). It is the `self` object specified in a binded 32 | Celery task definition (implicit first argument of the Celery task when @celery.task(bind=True) is used). 33 | :param int timeout: Lock's timeout value in seconds. 34 | :param bool include_args: If single instance should take arguments into account. 35 | :param iter args: The task instance's args. 36 | :param dict kwargs: The task instance's kwargs. 37 | """ 38 | self.celery_self = celery_self 39 | self.timeout = timeout 40 | self.include_args = include_args 41 | self.args = args 42 | self.kwargs = kwargs 43 | self.log = getLogger('{0}:{1}'.format(self.__class__.__name__, self.task_identifier)) 44 | 45 | @property 46 | def task_identifier(self): 47 | """Return the unique identifier (string) of a task instance.""" 48 | task_id = self.celery_self.name 49 | if self.include_args: 50 | merged_args = str(self.args) + str([(k, self.kwargs[k]) for k in sorted(self.kwargs)]) 51 | task_id += '.args.{0}'.format(hashlib.md5(merged_args.encode('utf-8')).hexdigest()) 52 | return task_id 53 | 54 | 55 | class _LockManagerRedis(_LockManager): 56 | """Handle locking/unlocking for Redis backends.""" 57 | 58 | CELERY_LOCK = '_celery.single_instance.{task_id}' 59 | 60 | def __init__(self, celery_self, timeout, include_args, args, kwargs): 61 | super(_LockManagerRedis, self).__init__(celery_self, timeout, include_args, args, kwargs) 62 | self.lock = None 63 | 64 | def __enter__(self): 65 | redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier) 66 | self.lock = self.celery_self.backend.client.lock(redis_key, timeout=self.timeout) 67 | self.log.debug('Timeout %ds | Redis key %s', self.timeout, redis_key) 68 | if not self.lock.acquire(blocking=False): 69 | self.log.debug('Another instance is running.') 70 | raise OtherInstanceError('Failed to acquire lock, {0} already running.'.format(self.task_identifier)) 71 | else: 72 | self.log.debug('Got lock, running.') 73 | 74 | def __exit__(self, exc_type, *_): 75 | if exc_type == OtherInstanceError: 76 | # Failed to get lock last time, not releasing. 77 | return 78 | self.log.debug('Releasing lock.') 79 | self.lock.release() 80 | 81 | @property 82 | def is_already_running(self): 83 | """Return True if lock exists and has not timed out.""" 84 | redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier) 85 | return self.celery_self.backend.client.exists(redis_key) 86 | 87 | def reset_lock(self): 88 | """Removed the lock regardless of timeout.""" 89 | redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier) 90 | self.celery_self.backend.client.delete(redis_key) 91 | 92 | 93 | class _LockManagerDB(_LockManager): 94 | """Handle locking/unlocking for SQLite/MySQL/PostgreSQL/etc backends.""" 95 | 96 | def __init__(self, celery_self, timeout, include_args, args, kwargs): 97 | super(_LockManagerDB, self).__init__(celery_self, timeout, include_args, args, kwargs) 98 | self.save_group = getattr(self.celery_self.backend, '_save_group') 99 | self.restore_group = getattr(self.celery_self.backend, '_restore_group') 100 | self.delete_group = getattr(self.celery_self.backend, '_delete_group') 101 | 102 | def __enter__(self): 103 | self.log.debug('Timeout %ds', self.timeout) 104 | try: 105 | self.save_group(self.task_identifier, None) 106 | except Exception as exc: # pylint: disable=broad-except 107 | if 'IntegrityError' not in str(exc) and 'ProgrammingError' not in str(exc): 108 | raise 109 | difference = datetime.utcnow() - self.restore_group(self.task_identifier)['date_done'] 110 | if difference < timedelta(seconds=self.timeout): 111 | self.log.debug('Another instance is running.') 112 | raise OtherInstanceError('Failed to acquire lock, {0} already running.'.format(self.task_identifier)) 113 | self.log.debug('Timeout expired, stale lock found, releasing lock.') 114 | self.delete_group(self.task_identifier) 115 | self.save_group(self.task_identifier, None) 116 | self.log.debug('Got lock, running.') 117 | 118 | def __exit__(self, exc_type, *_): 119 | if exc_type == OtherInstanceError: 120 | # Failed to get lock last time, not releasing. 121 | return 122 | self.log.debug('Releasing lock.') 123 | self.delete_group(self.task_identifier) 124 | 125 | @property 126 | def is_already_running(self): 127 | """Return True if lock exists and has not timed out.""" 128 | date_done = (self.restore_group(self.task_identifier) or dict()).get('date_done') 129 | if not date_done: 130 | return False 131 | difference = datetime.utcnow() - date_done 132 | return difference < timedelta(seconds=self.timeout) 133 | 134 | def reset_lock(self): 135 | """Removed the lock regardless of timeout.""" 136 | self.delete_group(self.task_identifier) 137 | 138 | 139 | def _select_manager(backend_name): 140 | """Select the proper LockManager based on the current backend used by Celery. 141 | 142 | :raise NotImplementedError: If Celery is using an unsupported backend. 143 | 144 | :param str backend_name: Class name of the current Celery backend. Usually value of 145 | current_app.extensions['celery'].celery.backend.__class__.__name__. 146 | 147 | :return: Class definition object (not instance). One of the _LockManager* classes. 148 | """ 149 | if backend_name == 'RedisBackend': 150 | lock_manager = _LockManagerRedis 151 | elif backend_name == 'DatabaseBackend': 152 | lock_manager = _LockManagerDB 153 | else: 154 | raise NotImplementedError 155 | return lock_manager 156 | 157 | 158 | class _CeleryState(object): 159 | """Remember the configuration for the (celery, app) tuple. Modeled from SQLAlchemy.""" 160 | 161 | def __init__(self, celery, app): 162 | self.celery = celery 163 | self.app = app 164 | 165 | 166 | # noinspection PyProtectedMember 167 | class Celery(CeleryClass): 168 | """Celery extension for Flask applications. 169 | 170 | Involves a hack to allow views and tests importing the celery instance from extensions.py to access the regular 171 | Celery instance methods. This is done by subclassing celery.Celery and overwriting celery._state._register_app() 172 | with a lambda/function that does nothing at all. 173 | 174 | That way, on the first super() in this class' __init__(), all of the required instance objects are initialized, but 175 | the Celery application is not registered. This class will be initialized in extensions.py but at that moment the 176 | Flask application is not yet available. 177 | 178 | Then, once the Flask application is available, this class' init_app() method will be called, with the Flask 179 | application as an argument. init_app() will again call celery.Celery.__init__() but this time with the 180 | celery._state._register_app() restored to its original functionality. in init_app() the actual Celery application is 181 | initialized like normal. 182 | """ 183 | 184 | def __init__(self, app=None): 185 | """If app argument provided then initialize celery using application config values. 186 | 187 | If no app argument provided you should do initialization later with init_app method. 188 | 189 | :param app: Flask application instance. 190 | """ 191 | self.original_register_app = _state._register_app # Backup Celery app registration function. 192 | _state._register_app = lambda _: None # Upon Celery app registration attempt, do nothing. 193 | super(Celery, self).__init__() 194 | if app is not None: 195 | self.init_app(app) 196 | 197 | def init_app(self, app): 198 | """Actual method to read celery settings from app configuration and initialize the celery instance. 199 | 200 | :param app: Flask application instance. 201 | """ 202 | _state._register_app = self.original_register_app # Restore Celery app registration function. 203 | if not hasattr(app, 'extensions'): 204 | app.extensions = dict() 205 | if 'celery' in app.extensions: 206 | raise ValueError('Already registered extension CELERY.') 207 | app.extensions['celery'] = _CeleryState(self, app) 208 | 209 | # Instantiate celery and read config. 210 | super(Celery, self).__init__(app.import_name, broker=app.config['CELERY_BROKER_URL']) 211 | 212 | # Set result backend default. 213 | if 'CELERY_RESULT_BACKEND' in app.config: 214 | self._preconf['CELERY_RESULT_BACKEND'] = app.config['CELERY_RESULT_BACKEND'] 215 | 216 | self.conf.update(app.config) 217 | task_base = self.Task 218 | 219 | # Add Flask app context to celery instance. 220 | class ContextTask(task_base): 221 | def __call__(self, *_args, **_kwargs): 222 | with app.app_context(): 223 | return task_base.__call__(self, *_args, **_kwargs) 224 | setattr(ContextTask, 'abstract', True) 225 | setattr(self, 'Task', ContextTask) 226 | 227 | 228 | def single_instance(func=None, lock_timeout=None, include_args=False): 229 | """Celery task decorator. Forces the task to have only one running instance at a time. 230 | 231 | Use with binded tasks (@celery.task(bind=True)). 232 | 233 | Modeled after: 234 | http://loose-bits.com/2010/10/distributed-task-locking-in-celery.html 235 | http://blogs.it.ox.ac.uk/inapickle/2012/01/05/python-decorators-with-optional-arguments/ 236 | 237 | Written by @Robpol86. 238 | 239 | :raise OtherInstanceError: If another instance is already running. 240 | 241 | :param function func: The function to decorate, must be also decorated by @celery.task. 242 | :param int lock_timeout: Lock timeout in seconds plus five more seconds, in-case the task crashes and fails to 243 | release the lock. If not specified, the values of the task's soft/hard limits are used. If all else fails, 244 | timeout will be 5 minutes. 245 | :param bool include_args: Include the md5 checksum of the arguments passed to the task in the Redis key. This allows 246 | the same task to run with different arguments, only stopping a task from running if another instance of it is 247 | running with the same arguments. 248 | """ 249 | if func is None: 250 | return partial(single_instance, lock_timeout=lock_timeout, include_args=include_args) 251 | 252 | @wraps(func) 253 | def wrapped(celery_self, *args, **kwargs): 254 | """Wrapped Celery task, for single_instance().""" 255 | # Select the manager and get timeout. 256 | timeout = ( 257 | lock_timeout or celery_self.soft_time_limit or celery_self.time_limit 258 | or celery_self.app.conf.get('CELERYD_TASK_SOFT_TIME_LIMIT') 259 | or celery_self.app.conf.get('CELERYD_TASK_TIME_LIMIT') 260 | or (60 * 5) 261 | ) 262 | manager_class = _select_manager(celery_self.backend.__class__.__name__) 263 | lock_manager = manager_class(celery_self, timeout, include_args, args, kwargs) 264 | 265 | # Lock and execute. 266 | with lock_manager: 267 | ret_value = func(*args, **kwargs) 268 | return ret_value 269 | return wrapped 270 | --------------------------------------------------------------------------------