├── .coveragerc ├── .flake8 ├── .github └── workflows │ └── python-package.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENCE ├── README.md ├── django_huey ├── __init__.py ├── apps.py ├── config.py ├── exceptions.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── djangohuey.py ├── models.py └── utils.py ├── setup.py └── tests ├── __init__.py ├── queuesinvalid.py ├── queuesvalid.py ├── requirements ├── base.txt └── monitor.txt ├── settings ├── base.py └── monitor.py ├── tests_apps.py ├── tests_config.py ├── tests_decorators.py └── tests_multiqueue.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | env/* 4 | tests/* -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403 3 | max-line-length = 100 4 | max-complexity = 10 5 | select = B,C,E,F,W,T4,B9 -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | branches: [ main ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: 20 | - "3.8" 21 | - "3.9" 22 | - "3.10" 23 | - "3.11" 24 | - "3.12" 25 | 26 | steps: 27 | - uses: actions/checkout@v4 28 | - name: Set up Python ${{ matrix.python-version }} 29 | uses: actions/setup-python@v4 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | python -m pip install flake8 36 | pip install -r tests/requirements/base.txt 37 | - name: Lint with flake8 38 | run: | 39 | # stop the build if there are Python syntax errors or undefined names 40 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 41 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 42 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 43 | - name: Test with unittest 44 | env: 45 | DJANGO_SETTINGS_MODULE: tests.settings.base 46 | run: | 47 | python -W all -m unittest 48 | - name: Install huey_monitor 49 | run: | 50 | python -m pip install --upgrade pip 51 | python -m pip install flake8 52 | pip install -r tests/requirements/monitor.txt 53 | - name: Test with monitor installed 54 | env: 55 | DJANGO_SETTINGS_MODULE: tests.settings.monitor 56 | run: | 57 | python -W all -m unittest 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | env/ 2 | *.pyc 3 | /dist/ 4 | /*.egg-info 5 | build/ 6 | TODOs.md 7 | htmlcov/ 8 | .coverage 9 | .local.env -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: 23.10.1 4 | hooks: 5 | - id: black 6 | language_version: python3.10 7 | - repo: https://github.com/pycqa/flake8 8 | rev: 3.9.2 9 | hooks: 10 | - id: flake8 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.3.0 - 2025-06-01 4 | ### Added 5 | - Support for huey context_task decorator (@tcely) 6 | 7 | ## 1.2.1 - 2024-06-16 8 | ### Fixed 9 | - Import error on automatic config of `huey-monitor` 10 | 11 | ## 1.2.0 - 2024-04-14 12 | ### Added 13 | - Automatic configuration of queues with `huey-monitor` 14 | 15 | ## 1.1.2 - 2023-11-01 16 | ### Fixed 17 | - [#17](https://github.com/gaiacoop/django-huey/issues/17) - Support for python 3.12 and Django 5.0 18 | 19 | ## 1.1.1 - 2022-02-07 20 | ### Fixed 21 | - [#8](https://github.com/gaiacoop/django-huey/issues/8) - Redis was required when using SqliteHuey 22 | 23 | ## 1.1.0 - 2022-01-18 24 | ### Added 25 | - Allow definition of specific queues file path. 26 | - Configuration error is raised if two queues have the same name. 27 | 28 | ## 1.0.1 - 2022-01-14 29 | ### Added 30 | - Close db connections before task body. https://github.com/coleifer/huey/commit/e77acf307bfdade914ab7f91c65dbbc183af5d8f 31 | 32 | ## 1.0.0 - 2021-05-19 33 | **Note:** This release contains breaking changes, see them below with the migration instructions. 34 | 35 | ### Added 36 | - Allow definition of a default queue. 37 | 38 | ### Changed 39 | - HUEYS django setting renamed to DJANGO_HUEY. 40 | - Change command run_djangohuey to djangohuey. 41 | 42 | --- 43 | 44 | ## 0.2.0 - 2021-04-18 45 | 46 | ### Added 47 | *Nothing added this release* 48 | 49 | ### Changed 50 | *Nothing changed this release* 51 | 52 | ### Fixed 53 | - When a huey name was not provided, default django db name was used. Now it's defaulted to queue name. 54 | 55 | ### Removed 56 | - Removed incompatibility with HUEY setting used by [huey](https://github.com/coleifer/huey) project. 57 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright 2021 GAIA - Cooperativa de desarrollo de software 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![Version](https://img.shields.io/badge/version-1.3.0-informational.svg) 2 | 3 | # django-huey 4 | 5 | *** 6 | 7 | This package is an extension of [huey](https://github.com/coleifer/huey) contrib djhuey package that allows users to manage multiple queues. 8 | 9 | ## Compatible versions 10 | | Package | Version | 11 | | ----------- | ----------- | 12 | | Django | 5.0 | 13 | | Django | 4.2 | 14 | | Django | 3.2 | 15 | | huey | 2.5 | 16 | | huey | 2.4 | 17 | 18 | ## Installation 19 | 20 | Using pip package manager run: 21 | ``` 22 | pip install django-huey 23 | ``` 24 | 25 | Then, in your **settings.py** file add django_huey to the INSTALLED_APPS: 26 | ```python 27 | INSTALLED_APPS = [ 28 | ... 29 | 'django_huey', 30 | ] 31 | ``` 32 | 33 | ## Configuration 34 | In **settings.py** you must add the DJANGO_HUEY setting: 35 | ```python 36 | DJANGO_HUEY = { 37 | 'default': 'first', #this name must match with any of the queues defined below. 38 | 'queues': { 39 | 'first': {#this name will be used in decorators below 40 | 'huey_class': 'huey.RedisHuey', 41 | 'name': 'first_tasks', 42 | 'consumer': { 43 | 'workers': 2, 44 | 'worker_type': 'thread', 45 | }, 46 | }, 47 | 'emails': {#this name will be used in decorators below 48 | 'huey_class': 'huey.RedisHuey', 49 | 'name': 'emails_tasks', 50 | 'consumer': { 51 | 'workers': 5, 52 | 'worker_type': 'thread', 53 | }, 54 | } 55 | } 56 | } 57 | ``` 58 | 59 | ### Including queues from files 60 | *new in 1.1.0* 61 | 62 | You can also include a queue configuration from another file, located in one of your apps. 63 | Use django_huey.utils.include to do so: 64 | 65 | In **settings.py** you may have: 66 | ```python 67 | DJANGO_HUEY = { 68 | 'default': 'first', #this name must match with any of the queues defined below. 69 | 'queues': { 70 | # Your current queues definitions 71 | } 72 | } 73 | 74 | # This is new 75 | from django_huey.utils import include 76 | DJANGO_HUEY["queues"].update(include("example_app.queues")) 77 | ``` 78 | 79 | And in your `example_app.queues`: 80 | ```python 81 | queues = { 82 | "test": { 83 | "huey_class": "huey.MemoryHuey", 84 | "results": True, 85 | "store_none": False, 86 | "immediate": False, 87 | "utc": True, 88 | "blocking": True, 89 | "consumer": { 90 | "workers": 1, 91 | "worker_type": "thread", 92 | "initial_delay": 0.1, 93 | "backoff": 1.15, 94 | "max_delay": 10.0, 95 | "scheduler_interval": 60, 96 | "periodic": True, 97 | "check_worker_health": True, 98 | "health_check_interval": 300, 99 | }, 100 | }, 101 | } 102 | ``` 103 | Note: in your queues file, you should declare a variable called `queues`, so they can be included. If the variable doesn't exist, an `AttributeError` will be raised. 104 | 105 | ### Usage 106 | Now you will be able to run multiple queues using: 107 | ``` 108 | python manage.py djangohuey --queue first 109 | python manage.py djangohuey --queue emails 110 | ``` 111 | Each queue must be run in a different terminal. 112 | 113 | If you defined a default queue, you can just run: 114 | ``` 115 | python manage.py djangohuey 116 | ``` 117 | And the default queue will be used. 118 | 119 | 120 | ### Configuring tasks 121 | You can use usual *huey* decorators to register tasks, but they must be imported from django_huey as shown below: 122 | 123 | ```python 124 | from django_huey import db_task, task 125 | 126 | @task() #Use the default queue 'first' 127 | def some_func_that_uses_default_queue(): 128 | # perform some db task 129 | pass 130 | 131 | @db_task(queue='first') 132 | def some_func(): 133 | # perform some db task 134 | pass 135 | 136 | @task(queue='emails') 137 | def send_mails(): 138 | # send some emails 139 | pass 140 | ``` 141 | 142 | All the args and kwargs defined in huey decorators should work in the same way, if not, let us know. 143 | 144 | ### Importing a huey instance 145 | Sometimes you'll need to import a huey instance in order to do some advanced configuration, for example, when using huey pipelines. 146 | 147 | You can do that by using the get_queue function from django_huey: 148 | ```python 149 | from django_huey import get_queue 150 | 151 | first_q = get_queue('first') 152 | 153 | @first_q.task() 154 | def some_func(): 155 | pass 156 | ``` 157 | 158 | ### Integration with huey monitor 159 | You can use django-huey with [huey monitor](https://github.com/boxine/django-huey-monitor). 160 | -------------------------------------------------------------------------------- /django_huey/__init__.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from django.conf import settings 3 | from django.db import close_old_connections 4 | 5 | from django_huey.config import DjangoHueySettingsReader 6 | 7 | DJANGO_HUEY = getattr(settings, "DJANGO_HUEY", None) 8 | 9 | config = DjangoHueySettingsReader(DJANGO_HUEY) 10 | config.configure() 11 | 12 | 13 | def get_close_db_for_queue(queue): 14 | def close_db(fn): 15 | """Decorator to be used with tasks that may operate on the database.""" 16 | 17 | @wraps(fn) 18 | def inner(*args, **kwargs): 19 | instance = get_queue(queue) 20 | if not instance.immediate: 21 | close_old_connections() 22 | try: 23 | return fn(*args, **kwargs) 24 | finally: 25 | if not instance.immediate: 26 | close_old_connections() 27 | 28 | return inner 29 | 30 | return close_db 31 | 32 | 33 | def get_queue(queue): 34 | return config.get_queue(queue) 35 | 36 | 37 | def get_queue_name(queue): 38 | return config.get_queue_name(queue) 39 | 40 | 41 | def task(*args, queue=None, **kwargs): 42 | return get_queue(queue).task(*args, **kwargs) 43 | 44 | 45 | def context_task(*args, queue=None, **kwargs): 46 | return get_queue(queue).context_task(*args, **kwargs) 47 | 48 | 49 | def periodic_task(*args, queue=None, **kwargs): 50 | return get_queue(queue).periodic_task(*args, **kwargs) 51 | 52 | 53 | def lock_task(*args, queue=None, **kwargs): 54 | return get_queue(queue).lock_task(*args, **kwargs) 55 | 56 | 57 | # Task management. 58 | 59 | 60 | def enqueue(*args, queue=None, **kwargs): 61 | return get_queue(queue).enqueue(*args, **kwargs) 62 | 63 | 64 | def restore(*args, queue=None, **kwargs): 65 | return get_queue(queue).restore(*args, **kwargs) 66 | 67 | 68 | def restore_all(*args, queue=None, **kwargs): 69 | return get_queue(queue).restore_all(*args, **kwargs) 70 | 71 | 72 | def restore_by_id(*args, queue=None, **kwargs): 73 | return get_queue(queue).restore_by_id(*args, **kwargs) 74 | 75 | 76 | def revoke(*args, queue=None, **kwargs): 77 | return get_queue(queue).revoke(*args, **kwargs) 78 | 79 | 80 | def revoke_all(*args, queue=None, **kwargs): 81 | return get_queue(queue).revoke_all(*args, **kwargs) 82 | 83 | 84 | def revoke_by_id(*args, queue=None, **kwargs): 85 | return get_queue(queue).revoke_by_id(*args, **kwargs) 86 | 87 | 88 | def is_revoked(*args, queue=None, **kwargs): 89 | return get_queue(queue).is_revoked(*args, **kwargs) 90 | 91 | 92 | def result(*args, queue=None, **kwargs): 93 | return get_queue(queue).result(*args, **kwargs) 94 | 95 | 96 | def scheduled(*args, queue=None, **kwargs): 97 | return get_queue(queue).scheduled(*args, **kwargs) 98 | 99 | 100 | # Hooks. 101 | def on_startup(*args, queue=None, **kwargs): 102 | return get_queue(queue).on_startup(*args, **kwargs) 103 | 104 | 105 | def on_shutdown(*args, queue=None, **kwargs): 106 | return get_queue(queue).on_shutdown(*args, **kwargs) 107 | 108 | 109 | def pre_execute(*args, queue=None, **kwargs): 110 | return get_queue(queue).pre_execute(*args, **kwargs) 111 | 112 | 113 | def post_execute(*args, queue=None, **kwargs): 114 | return get_queue(queue).post_execute(*args, **kwargs) 115 | 116 | 117 | def signal(*args, queue=None, **kwargs): 118 | return get_queue(queue).signal(*args, **kwargs) 119 | 120 | 121 | def disconnect_signal(*args, queue=None, **kwargs): 122 | return get_queue(queue).disconnect_signal(*args, **kwargs) 123 | 124 | 125 | def db_task(*args, **kwargs): 126 | queue = kwargs.get("queue") 127 | 128 | def decorator(fn): 129 | ret = task(*args, **kwargs)(get_close_db_for_queue(queue)(fn)) 130 | ret.call_local = fn 131 | return ret 132 | 133 | return decorator 134 | 135 | 136 | def db_periodic_task(*args, **kwargs): 137 | queue = kwargs.get("queue") 138 | 139 | def decorator(fn): 140 | ret = periodic_task(*args, **kwargs)(get_close_db_for_queue(queue)(fn)) 141 | ret.call_local = fn 142 | return ret 143 | 144 | return decorator 145 | -------------------------------------------------------------------------------- /django_huey/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DjangoHueyConfig(AppConfig): 5 | name = "django_huey" 6 | 7 | def ready(self): 8 | try: 9 | monitor_installed = True 10 | from huey_monitor.tasks import startup_handler, store_signals 11 | except Exception: 12 | monitor_installed = False 13 | 14 | if monitor_installed: 15 | from django_huey import config, signal, on_startup 16 | 17 | for queuename in config.hueys_setting.keys(): 18 | signal(queue=queuename)(store_signals) 19 | on_startup(queue=queuename)(startup_handler) 20 | -------------------------------------------------------------------------------- /django_huey/config.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from django.conf import settings 3 | from django_huey.exceptions import ConfigurationError 4 | 5 | 6 | default_backend_path = "huey.RedisHuey" 7 | 8 | 9 | def get_backend(import_path=default_backend_path): 10 | module_path, class_name = import_path.rsplit(".", 1) 11 | module = import_module(module_path) 12 | return getattr(module, class_name) 13 | 14 | 15 | class DjangoHueySettingsReader: 16 | def __init__(self, hueys_setting): 17 | if not isinstance(hueys_setting, dict): 18 | raise ConfigurationError("Error: DJANGO_HUEY must be a dictionary") 19 | self.hueys_setting = hueys_setting.get("queues") 20 | self.default_queue = hueys_setting.get("default") 21 | 22 | if ( 23 | self.default_queue is not None 24 | and self.default_queue not in self.hueys_setting 25 | ): 26 | raise ConfigurationError( 27 | f"Queue defined as default: {self.default_queue}, is not configured in DJANGO_HUEY." 28 | ) 29 | self.hueys = {} 30 | 31 | def configure(self): 32 | new_hueys = dict() 33 | queue_names = set() 34 | for queue_name, config in self.hueys_setting.items(): 35 | huey_config = config.copy() 36 | 37 | name = huey_config.pop("name", queue_name) 38 | if name in queue_names: 39 | raise ConfigurationError( 40 | f"There are more than one queue with the name '{name}'. Check DJANGO_HUEY in your settings file." 41 | ) 42 | queue_names.add(name) 43 | new_hueys[queue_name] = self._configure_instance(huey_config, name) 44 | 45 | self.hueys_setting = new_hueys 46 | 47 | def include(self, queues): 48 | new_hueys = dict() 49 | queue_names = set() 50 | for queue_name, config in queues.items(): 51 | huey_config = config.copy() 52 | 53 | name = huey_config.pop("name", queue_name) 54 | if name in queue_names or name in self.hueys_setting: 55 | raise ConfigurationError( 56 | f"There are more than one queue with the name '{name}'. Check DJANGO_HUEY in your settings file." 57 | ) 58 | queue_names.add(name) 59 | new_hueys[queue_name] = self._configure_instance(huey_config, name) 60 | 61 | self.hueys_setting.update(new_hueys) 62 | 63 | def get_queue(self, queue): 64 | return self.hueys_setting[self.get_queue_name(queue)] 65 | 66 | def get_queue_name(self, queue_name): 67 | if queue_name is None: 68 | if self.default_queue is None: 69 | self._raise_queue_config_error() 70 | 71 | queue_name = self.default_queue 72 | return queue_name 73 | 74 | def _raise_queue_config_error(self): 75 | raise ConfigurationError( 76 | """ 77 | Command djangohuey must receive a --queue parameter or define a default queue in DJANGO_HUEY setting. 78 | i.e.: 79 | python manage.py djangohuey --queue first 80 | 81 | or in settings file: 82 | 83 | DJANGO_HUEY = { 84 | 'default': 'your-default-queue-name', 85 | 'queues': { 86 | #Your queues here 87 | } 88 | } 89 | """ 90 | ) 91 | 92 | def _configure_instance(self, huey_config, name): 93 | if "backend_class" in huey_config: 94 | huey_config["huey_class"] = huey_config.pop("backend_class") 95 | backend_path = huey_config.pop("huey_class", default_backend_path) 96 | conn_kwargs = huey_config.pop("connection", {}) 97 | try: 98 | del huey_config["consumer"] # Don't need consumer opts here. 99 | except KeyError: 100 | pass 101 | if "immediate" not in huey_config: 102 | huey_config["immediate"] = settings.DEBUG 103 | huey_config.update(conn_kwargs) 104 | 105 | try: 106 | backend_cls = get_backend(backend_path) 107 | except (ValueError, ImportError, AttributeError): 108 | raise ConfigurationError( 109 | f"Error: could not import Huey backend: {backend_path}" 110 | ) 111 | 112 | return backend_cls(name, **huey_config) 113 | -------------------------------------------------------------------------------- /django_huey/exceptions.py: -------------------------------------------------------------------------------- 1 | class ConfigurationError(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /django_huey/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaiacoop/django-huey/9e36eb649beaeba7843b7b9e855af62f091dd41d/django_huey/management/__init__.py -------------------------------------------------------------------------------- /django_huey/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaiacoop/django-huey/9e36eb649beaeba7843b7b9e855af62f091dd41d/django_huey/management/commands/__init__.py -------------------------------------------------------------------------------- /django_huey/management/commands/djangohuey.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | from django.conf import settings 5 | from django.core.management.base import BaseCommand 6 | from django.utils.module_loading import autodiscover_modules 7 | 8 | from huey.consumer_options import ConsumerConfig 9 | from huey.consumer_options import OptionParserHandler 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class Command(BaseCommand): 15 | """ 16 | Queue consumer. Example usage:: 17 | 18 | To start the consumer (note you must export the settings module): 19 | 20 | django-admin.py djangohuey queuename 21 | """ 22 | 23 | help = "Run the queue consumer" 24 | _type_map = {"int": int, "float": float} 25 | 26 | def add_arguments(self, parser): 27 | option_handler = OptionParserHandler() 28 | groups = ( 29 | option_handler.get_logging_options(), 30 | option_handler.get_worker_options(), 31 | option_handler.get_scheduler_options(), 32 | ) 33 | for option_list in groups: 34 | for short, full, kwargs in option_list: 35 | if short == "-v": 36 | full = "--huey-verbose" 37 | short = "-V" 38 | if "type" in kwargs: 39 | kwargs["type"] = self._type_map[kwargs["type"]] 40 | kwargs.setdefault("default", None) 41 | parser.add_argument(full, short, **kwargs) 42 | 43 | parser.add_argument( 44 | "-A", 45 | "--disable-autoload", 46 | action="store_true", 47 | dest="disable_autoload", 48 | help='Do not autoload "tasks.py"', 49 | ) 50 | parser.add_argument( 51 | "--queue", 52 | action="store", 53 | dest="queue", 54 | help="Name of the queue consumer to run", 55 | ) 56 | 57 | def handle(self, *args, **options): 58 | from django_huey import get_queue, get_queue_name 59 | 60 | # Python 3.8+ on MacOS uses an incompatible multiprocess model. In this 61 | # case we must explicitly configure mp to use fork(). 62 | if sys.version_info >= (3, 8) and sys.platform == "darwin": 63 | # Apparently this was causing a "context has already been set" 64 | # error for some user. We'll just pass and hope for the best. 65 | # They're apple users so presumably nothing important will be lost. 66 | import multiprocessing 67 | 68 | try: 69 | multiprocessing.set_start_method("fork") 70 | except RuntimeError: 71 | pass 72 | 73 | queue_name = options.get("queue") 74 | queue = get_queue(queue_name) 75 | queue_name = get_queue_name(queue_name) 76 | 77 | consumer_options = {} 78 | consumer_options.update(self.default_queue_settings(queue_name)) 79 | 80 | for key, value in options.items(): 81 | if value is not None: 82 | consumer_options[key] = value 83 | 84 | consumer_options.setdefault( 85 | "verbose", consumer_options.pop("huey_verbose", None) 86 | ) 87 | 88 | if not options.get("disable_autoload"): 89 | autodiscover_modules("tasks") 90 | 91 | logger = logging.getLogger("huey") 92 | 93 | config = ConsumerConfig(**consumer_options) 94 | config.validate() 95 | 96 | # Only configure the "huey" logger if it has no handlers. For example, 97 | # some users may configure the huey logger via the Django global 98 | # logging config. This prevents duplicating log messages: 99 | if not logger.handlers: 100 | config.setup_logger(logger) 101 | 102 | consumer = queue.create_consumer(**config.values) 103 | consumer.run() 104 | 105 | def default_queue_settings(self, queue): 106 | try: 107 | return settings.DJANGO_HUEY["queues"][queue].get("consumer", {}) 108 | except AttributeError: 109 | pass 110 | return {} 111 | -------------------------------------------------------------------------------- /django_huey/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaiacoop/django-huey/9e36eb649beaeba7843b7b9e855af62f091dd41d/django_huey/models.py -------------------------------------------------------------------------------- /django_huey/utils.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from django_huey.exceptions import ConfigurationError 3 | from django_huey import config 4 | 5 | 6 | def include(path): 7 | if isinstance(path, str): 8 | try: 9 | queue_module = import_module(path) 10 | except ModuleNotFoundError: 11 | raise ConfigurationError( 12 | f"No module named '{path}'. Review included queues modules in DJANGO_HUEY." 13 | ) 14 | queues = getattr(queue_module, "queues") 15 | config.include(queues) 16 | return queues 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setup( 7 | name="django-huey", 8 | version="1.3.0", 9 | scripts=[], 10 | author="GAIA - Cooperativa de desarrollo de software", 11 | author_email="contacto@gaiacoop.tech", 12 | description="An extension for django and huey that supports multi queue management", 13 | long_description=long_description, 14 | install_requires=[ 15 | "django>=3.2", 16 | "huey>=2.0", 17 | ], 18 | long_description_content_type="text/markdown", 19 | url="https://github.com/gaiacoop/django-huey", 20 | packages=find_packages(), 21 | include_package_data=True, 22 | classifiers=[ 23 | "Development Status :: 5 - Production/Stable", 24 | "Programming Language :: Python :: 3", 25 | "Framework :: Django :: 2.2", 26 | "Framework :: Django :: 3.1", 27 | "Framework :: Django :: 3.2", 28 | "Framework :: Django :: 4.0", 29 | "Framework :: Django :: 4.1", 30 | "Framework :: Django :: 4.2", 31 | "Framework :: Django :: 5.0", 32 | "Intended Audience :: Developers", 33 | "License :: OSI Approved :: MIT License", 34 | "Operating System :: OS Independent", 35 | ], 36 | python_requires=">=3.8", 37 | ) 38 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") 4 | -------------------------------------------------------------------------------- /tests/queuesinvalid.py: -------------------------------------------------------------------------------- 1 | q = { 2 | "test": { 3 | "huey_class": "huey.MemoryHuey", 4 | "results": True, 5 | "store_none": False, 6 | "immediate": False, 7 | "utc": True, 8 | "blocking": True, 9 | "consumer": { 10 | "workers": 1, 11 | "worker_type": "thread", 12 | "initial_delay": 0.1, 13 | "backoff": 1.15, 14 | "max_delay": 10.0, 15 | "scheduler_interval": 60, 16 | "periodic": True, 17 | "check_worker_health": True, 18 | "health_check_interval": 300, 19 | }, 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /tests/queuesvalid.py: -------------------------------------------------------------------------------- 1 | queues = { 2 | "test": { 3 | "huey_class": "huey.MemoryHuey", 4 | "results": True, 5 | "store_none": False, 6 | "immediate": False, 7 | "utc": True, 8 | "blocking": True, 9 | "consumer": { 10 | "workers": 1, 11 | "worker_type": "thread", 12 | "initial_delay": 0.1, 13 | "backoff": 1.15, 14 | "max_delay": 10.0, 15 | "scheduler_interval": 60, 16 | "periodic": True, 17 | "check_worker_health": True, 18 | "health_check_interval": 300, 19 | }, 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /tests/requirements/base.txt: -------------------------------------------------------------------------------- 1 | Django>=4.2a1,<5.0 ; python_version < "3.10" 2 | Django>=5.0a1,<5.1 ; python_version >= "3.10" 3 | huey==2.5.0 4 | -------------------------------------------------------------------------------- /tests/requirements/monitor.txt: -------------------------------------------------------------------------------- 1 | -r base.txt 2 | django-huey-monitor<0.5.0 ; python_version < "3.9" 3 | django-huey-monitor>=0.8.1 ; python_version >= "3.9" -------------------------------------------------------------------------------- /tests/settings/base.py: -------------------------------------------------------------------------------- 1 | DEBUG = True 2 | 3 | DATABASES = { 4 | "default": {"NAME": "testdatabase", "ENGINE": "django.db.backends.sqlite3"} 5 | } 6 | 7 | 8 | SECRET_KEY = "django_tests_secret_key" 9 | TIME_ZONE = "America/Chicago" 10 | LANGUAGE_CODE = "en-us" 11 | ADMIN_MEDIA_PREFIX = "/static/admin/" 12 | STATICFILES_DIRS = () 13 | 14 | MIDDLEWARE_CLASSES = [] 15 | 16 | INSTALLED_APPS = ["django_huey"] 17 | 18 | USE_TZ = True 19 | 20 | DJANGO_HUEY = { 21 | "default": "multi-2", 22 | "queues": { 23 | "multi-1": { 24 | "huey_class": "huey.MemoryHuey", # Huey implementation to use. 25 | "results": True, # Store return values of tasks. 26 | "store_none": False, # If a task returns None, do not save to results. 27 | "immediate": False, # If DEBUG=True, run synchronously. 28 | "utc": True, # Use UTC for all times internally. 29 | "blocking": True, # Perform blocking pop rather than poll Redis. 30 | "consumer": { 31 | "workers": 1, 32 | "worker_type": "thread", 33 | "initial_delay": 0.1, # Smallest polling interval, same as -d. 34 | "backoff": 1.15, # Exponential backoff using this rate, -b. 35 | "max_delay": 10.0, # Max possible polling interval, -m. 36 | "scheduler_interval": 60, # Check schedule every second, -s. 37 | "periodic": True, # Enable crontab feature. 38 | "check_worker_health": True, # Enable worker health checks. 39 | "health_check_interval": 300, # Check worker health every second. 40 | }, 41 | }, 42 | "multi-2": { 43 | "huey_class": "huey.MemoryHuey", # Huey implementation to use. 44 | "results": True, # Store return values of tasks. 45 | "store_none": False, # If a task returns None, do not save to results. 46 | "immediate": False, # If DEBUG=True, run synchronously. 47 | "utc": True, # Use UTC for all times internally. 48 | "blocking": True, # Perform blocking pop rather than poll Redis. 49 | "consumer": { 50 | "workers": 1, 51 | "worker_type": "thread", 52 | "initial_delay": 0.1, # Smallest polling interval, same as -d. 53 | "backoff": 1.15, # Exponential backoff using this rate, -b. 54 | "max_delay": 10.0, # Max possible polling interval, -m. 55 | "scheduler_interval": 60, # Check schedule every second, -s. 56 | "periodic": True, # Enable crontab feature. 57 | "check_worker_health": True, # Enable worker health checks. 58 | "health_check_interval": 300, # Check worker health every second. 59 | }, 60 | }, 61 | }, 62 | } 63 | HUEY = { 64 | "name": "test", 65 | "immediate": True, 66 | } 67 | -------------------------------------------------------------------------------- /tests/settings/monitor.py: -------------------------------------------------------------------------------- 1 | from .base import * # noqa 2 | 3 | INSTALLED_APPS = ["django_huey", "huey_monitor"] 4 | -------------------------------------------------------------------------------- /tests/tests_apps.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import django 3 | 4 | 5 | class DjangoHueyConfigTests(unittest.TestCase): 6 | def test_djangohuey_config_with_monitor_installed(self): 7 | django.setup() 8 | -------------------------------------------------------------------------------- /tests/tests_config.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from huey import RedisHuey, MemoryHuey 3 | from django_huey.exceptions import ConfigurationError 4 | from django_huey.utils import include 5 | from django_huey.config import DjangoHueySettingsReader 6 | 7 | 8 | class DjangoHueyConfigTests(unittest.TestCase): 9 | def test_djangohuey_config_with_no_settings(self): 10 | 11 | with self.assertRaises(ConfigurationError) as cm: 12 | DjangoHueySettingsReader(None) 13 | 14 | self.assertEqual("Error: DJANGO_HUEY must be a dictionary", str(cm.exception)) 15 | 16 | def test_djangohuey_configure_does_not_raise_error_when_both_settings_are_defined( 17 | self, 18 | ): 19 | DJANGO_HUEY = { 20 | "queues": { 21 | "queuename": { 22 | "name": "test", 23 | "immediate": True, 24 | } 25 | } 26 | } 27 | config = DjangoHueySettingsReader(DJANGO_HUEY) 28 | 29 | config.configure() 30 | 31 | def test_djangohuey_get_queue_when_queue_is_none(self): 32 | DJANGO_HUEY = { 33 | "queues": { 34 | "queuename": { 35 | "name": "test", 36 | "immediate": True, 37 | } 38 | } 39 | } 40 | config = DjangoHueySettingsReader(DJANGO_HUEY) 41 | 42 | config.configure() 43 | 44 | with self.assertRaises(ConfigurationError) as cm: 45 | config.get_queue(None) 46 | 47 | self.assertEqual( 48 | """ 49 | Command djangohuey must receive a --queue parameter or define a default queue in DJANGO_HUEY setting. 50 | i.e.: 51 | python manage.py djangohuey --queue first 52 | 53 | or in settings file: 54 | 55 | DJANGO_HUEY = { 56 | 'default': 'your-default-queue-name', 57 | 'queues': { 58 | #Your queues here 59 | } 60 | } 61 | """, 62 | str(cm.exception), 63 | ) 64 | 65 | def test_djangohuey_get_queue_when_queue_is_none_and_default_queue_is_defined(self): 66 | DJANGO_HUEY = { 67 | "default": "queuename", 68 | "queues": { 69 | "queuename": { 70 | "name": "queue-default", 71 | "immediate": True, 72 | } 73 | }, 74 | } 75 | config = DjangoHueySettingsReader(DJANGO_HUEY) 76 | 77 | config.configure() 78 | 79 | queue = config.get_queue(None) 80 | 81 | self.assertEqual(queue.name, "queue-default") 82 | 83 | def test_djangohuey_invalid_default_queue(self): 84 | DJANGO_HUEY = { 85 | "default": "invalid-queue", 86 | "queues": { 87 | "queuename": { 88 | "name": "queue-default", 89 | "immediate": True, 90 | } 91 | }, 92 | } 93 | with self.assertRaises(ConfigurationError) as cm: 94 | DjangoHueySettingsReader(DJANGO_HUEY) 95 | 96 | self.assertEqual( 97 | "Queue defined as default: invalid-queue, is not configured in DJANGO_HUEY.", 98 | str(cm.exception), 99 | ) 100 | 101 | def test_djangohuey_configure_when_django_huey_setting_is_defined(self, *args): 102 | DJANGO_HUEY = { 103 | "queues": { 104 | "first": { 105 | "huey_class": "huey.RedisHuey", # Huey implementation to use. 106 | "name": "testname", # Use db name for huey. 107 | }, 108 | "mails": { 109 | "huey_class": "huey.MemoryHuey", # Huey implementation to use. 110 | "name": "testnamememory", # Use db name for huey. 111 | }, 112 | } 113 | } 114 | 115 | config = DjangoHueySettingsReader(DJANGO_HUEY) 116 | 117 | config.configure() 118 | 119 | self.assertTrue(isinstance(config.get_queue("first"), RedisHuey)) 120 | self.assertEqual(config.get_queue("first").name, "testname") 121 | self.assertTrue(isinstance(config.get_queue("mails"), MemoryHuey)) 122 | self.assertEqual(config.get_queue("mails").name, "testnamememory") 123 | 124 | def test_djangohuey_configure_when_django_huey_setting_is_an_object_raises_error( 125 | self, *args 126 | ): 127 | DJANGO_HUEY = object() 128 | 129 | with self.assertRaises(ConfigurationError) as cm: 130 | DjangoHueySettingsReader(DJANGO_HUEY) 131 | 132 | self.assertEqual("Error: DJANGO_HUEY must be a dictionary", str(cm.exception)) 133 | 134 | def test_djangohuey_if_name_is_not_defined_queue_name_is_default(self, *args): 135 | DJANGO_HUEY = { 136 | "queues": { 137 | "first": { 138 | "huey_class": "huey.RedisHuey", # Huey implementation to use. 139 | }, 140 | "mails": { 141 | "huey_class": "huey.MemoryHuey", # Huey implementation to use. 142 | }, 143 | } 144 | } 145 | 146 | config = DjangoHueySettingsReader(DJANGO_HUEY) 147 | 148 | config.configure() 149 | self.assertEqual(config.get_queue("first").name, "first") 150 | 151 | self.assertEqual(config.get_queue("mails").name, "mails") 152 | 153 | def test_djangohuey_with_backend_class(self, *args): 154 | DJANGO_HUEY = { 155 | "queues": { 156 | "first": { 157 | "backend_class": "huey.RedisHuey", 158 | } 159 | } 160 | } 161 | 162 | config = DjangoHueySettingsReader(DJANGO_HUEY) 163 | 164 | config.configure() 165 | self.assertTrue(isinstance(config.get_queue("first"), RedisHuey)) 166 | 167 | def test_djangohuey_invalid_backend_class(self, *args): 168 | DJANGO_HUEY = { 169 | "queues": { 170 | "first": { 171 | "backend_class": "huey.RedisHuey2", 172 | } 173 | } 174 | } 175 | 176 | with self.assertRaises(ConfigurationError) as cm: 177 | config = DjangoHueySettingsReader(DJANGO_HUEY) 178 | config.configure() 179 | 180 | self.assertEqual( 181 | "Error: could not import Huey backend: huey.RedisHuey2", str(cm.exception) 182 | ) 183 | 184 | def test_djangohuey_configure_when_there_are_two_queues_with_the_same_name( 185 | self, *args 186 | ): 187 | DJANGO_HUEY = { 188 | "queues": { 189 | "first": { 190 | "huey_class": "huey.RedisHuey", 191 | "name": "first", 192 | }, 193 | "mails": { 194 | "huey_class": "huey.MemoryHuey", 195 | "name": "first", 196 | }, 197 | } 198 | } 199 | 200 | config = DjangoHueySettingsReader(DJANGO_HUEY) 201 | 202 | with self.assertRaises(ConfigurationError) as cm: 203 | config.configure() 204 | 205 | self.assertEqual( 206 | "There are more than one queue with the name 'first'. Check DJANGO_HUEY in your settings file.", 207 | str(cm.exception), 208 | ) 209 | 210 | def test_djangohuey_get_queue_when_queue_is_included_from_module(self): 211 | DJANGO_HUEY = { 212 | "queues": { 213 | **include("tests.queuesvalid"), 214 | "mails": { 215 | "huey_class": "huey.MemoryHuey", 216 | "name": "first", 217 | }, 218 | }, 219 | } 220 | config = DjangoHueySettingsReader(DJANGO_HUEY) 221 | 222 | config.configure() 223 | queue = config.get_queue("test") 224 | 225 | self.assertEqual(queue.name, "test") 226 | 227 | def test_djangohuey_include_module_without_queue_variable_raises_attribute_error( 228 | self, 229 | ): 230 | with self.assertRaises(AttributeError): 231 | { 232 | "queues": {**include("tests.queuesinvalid")}, 233 | } 234 | 235 | def test_djangohuey_include_non_existing_module(self): 236 | with self.assertRaises(ConfigurationError) as cm: 237 | { 238 | "queues": {**include("tests.nonexisting")}, 239 | } 240 | self.assertEqual( 241 | "No module named 'tests.nonexisting'. Review included queues modules in DJANGO_HUEY.", 242 | str(cm.exception), 243 | ) 244 | -------------------------------------------------------------------------------- /tests/tests_decorators.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from unittest import mock 4 | from django_huey import ( 5 | task, 6 | periodic_task, 7 | lock_task, 8 | enqueue, 9 | restore, 10 | restore_all, 11 | restore_by_id, 12 | revoke, 13 | revoke_all, 14 | revoke_by_id, 15 | is_revoked, 16 | result, 17 | scheduled, 18 | on_startup, 19 | on_shutdown, 20 | pre_execute, 21 | post_execute, 22 | signal, 23 | disconnect_signal, 24 | ) 25 | 26 | DECORATORS = [ 27 | task, 28 | periodic_task, 29 | lock_task, 30 | enqueue, 31 | restore, 32 | restore_all, 33 | restore_by_id, 34 | revoke, 35 | revoke_all, 36 | revoke_by_id, 37 | is_revoked, 38 | result, 39 | scheduled, 40 | on_startup, 41 | on_shutdown, 42 | pre_execute, 43 | post_execute, 44 | signal, 45 | disconnect_signal, 46 | ] 47 | 48 | 49 | class DecoratorsTest(unittest.TestCase): 50 | def make_test_decorator(self, decorator): 51 | with mock.patch("django_huey.get_queue") as obj: 52 | 53 | @decorator(queue="test_queue") 54 | def test_func(): 55 | pass 56 | 57 | obj.assert_called_once_with("test_queue") 58 | 59 | def test_decorators(self): 60 | for decorator in DECORATORS: 61 | self.make_test_decorator(decorator) 62 | -------------------------------------------------------------------------------- /tests/tests_multiqueue.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from django_huey import task, get_queue 4 | 5 | 6 | class MultiQueueTest(unittest.TestCase): 7 | def setUp(self): 8 | 9 | get_queue("multi-1").flush() 10 | get_queue("multi-2").flush() 11 | 12 | def test_task_uses_specified_queue(self): 13 | @task(queue="multi-1") 14 | def some_fun(): 15 | pass 16 | 17 | some_fun() 18 | 19 | queue1 = get_queue("multi-1") 20 | queue2 = get_queue("multi-2") 21 | self.assertEqual(len(queue1), 1) 22 | self.assertEqual(len(queue2), 0) 23 | 24 | @task(queue="multi-2") 25 | def some_other_fun(): 26 | pass 27 | 28 | some_other_fun() 29 | 30 | queue1 = get_queue("multi-1") 31 | queue2 = get_queue("multi-2") 32 | self.assertEqual(len(queue1), 1) 33 | self.assertEqual(len(queue2), 1) 34 | 35 | def test_task_uses_default_queue_if_not_specified(self): 36 | @task() 37 | def some_default_func(): 38 | pass 39 | 40 | some_default_func() 41 | 42 | queue1 = get_queue("multi-1") 43 | queue2 = get_queue("multi-2") 44 | self.assertEqual(len(queue1), 0) 45 | self.assertEqual(len(queue2), 1) 46 | --------------------------------------------------------------------------------