├── django_toosimple_q ├── contrib │ ├── __init__.py │ └── mail │ │ ├── __init__.py │ │ ├── backend.py │ │ ├── tasks.py │ │ └── tests.py ├── tests │ ├── __init__.py │ ├── demo │ │ ├── __init__.py │ │ └── tasks.py │ ├── concurrency │ │ ├── __init__.py │ │ └── tasks.py │ ├── urls.py │ ├── settings_bg.py │ ├── utils.py │ ├── settings_bg_lag.py │ ├── tests_integration.py │ ├── settings.py │ ├── tests_regression.py │ ├── tests_concurrency.py │ ├── tests_worker.py │ ├── tests_admin.py │ ├── base.py │ ├── tests_schedules.py │ └── tests_tasks.py ├── management │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── worker.py │ └── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ └── __init__.cpython-37.pyc ├── migrations │ ├── __init__.py │ ├── 0003_task_queue.py │ ├── 0004_auto_20200507_1339.py │ ├── 0007_schedule_datetime_kwarg.py │ ├── 0014_alter_workerstatus_exit_code.py │ ├── 0006_task_replacement.py │ ├── 0009_auto_20210902_2245.py │ ├── 0012_rename_last_run_scheduleexec_last_task.py │ ├── 0013_workerstatus_exit_code_workerstatus_exit_log.py │ ├── 0002_auto_20191101_1838.py │ ├── 0015_taskexec_result_preview.py │ ├── 0016_alter_scheduleexec_options_alter_taskexec_options.py │ ├── 0008_auto_20210902_2111.py │ ├── 0005_auto_20210302_1748.py │ ├── 0011_workerstatus.py │ ├── 0010_auto_20220324_0419.py │ └── 0001_initial.py ├── templates │ └── toosimpleq │ │ ├── task.html │ │ └── schedule.html ├── __init__.py ├── registry.py ├── apps.py ├── logging.py ├── schedule.py ├── decorators.py ├── task.py ├── models.py └── admin.py ├── requirements.txt ├── .gitignore ├── MANIFEST.in ├── requirements-dev.txt ├── .editorconfig ├── scripts ├── ci_set_version_from_git.py └── ci_assert_version_from_git.py ├── manage.py ├── Dockerfile ├── .github └── workflows │ ├── pypi.yml │ └── test.yml ├── .pre-commit-config.yaml ├── LICENSE ├── setup.py ├── docker-compose.yml └── README.md /django_toosimple_q/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/contrib/mail/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/demo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/concurrency/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /django_toosimple_q/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django>=3.2 2 | django-picklefield>=3.0 3 | croniter>=1.3 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .vscode 3 | *.egg-info 4 | *.sqlite3 5 | build 6 | dist 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include django_toosimple_q * 2 | recursive-exclude django_toosimple_q *.pyc 3 | -------------------------------------------------------------------------------- /django_toosimple_q/templates/toosimpleq/task.html: -------------------------------------------------------------------------------- 1 | 2 | queue: {{task.queue}}
3 | priority: {{task.priority}} 4 | 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # tests 2 | freezegun==1.5.1 3 | psycopg2==2.* 4 | 5 | # devenv 6 | pre-commit 7 | mypy 8 | 9 | # django-toosimple-q 10 | -e . 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | -------------------------------------------------------------------------------- /django_toosimple_q/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "dev" # DO NOT CHANGE THIS LINE - it will be replaced by CI workflow 2 | default_app_config = "django_toosimple_q.apps.DjangoToosimpleQConfig" 3 | -------------------------------------------------------------------------------- /django_toosimple_q/management/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/olivierdalang/django-toosimple-q/HEAD/django_toosimple_q/management/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /django_toosimple_q/management/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/olivierdalang/django-toosimple-q/HEAD/django_toosimple_q/management/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /django_toosimple_q/templates/toosimpleq/schedule.html: -------------------------------------------------------------------------------- 1 | 2 | cron: {{schedule.cron}}
3 | args: {{schedule.args}}
4 | kwargs: {{schedule.kwargs}}
5 | queue: {{schedule.queue}} 6 | 7 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from django.contrib import admin 4 | from django.urls import path 5 | 6 | urlpatterns = [ 7 | path("admin/", admin.site.urls), 8 | ] 9 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/settings_bg.py: -------------------------------------------------------------------------------- 1 | """Settings overrides for testing background workers""" 2 | 3 | from .settings import * 4 | 5 | # The background workers must also use the test database 6 | DATABASES["default"].update(DATABASES["default"]["TEST"]) 7 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def is_postgres(): 5 | return os.getenv("TOOSIMPLEQ_TEST_DB", None) == "postgres" 6 | 7 | 8 | class FakeException(Exception): 9 | """An artification exception to simulate an unexpected error""" 10 | -------------------------------------------------------------------------------- /django_toosimple_q/contrib/mail/backend.py: -------------------------------------------------------------------------------- 1 | from django.core.mail.backends.base import BaseEmailBackend 2 | 3 | from .tasks import send_email 4 | 5 | 6 | class QueueBackend(BaseEmailBackend): 7 | def __init__(self, **kwargs): 8 | super().__init__(kwargs) 9 | 10 | def send_messages(self, email_messages): 11 | send_email.queue(email_messages) 12 | -------------------------------------------------------------------------------- /scripts/ci_set_version_from_git.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | basedir = os.path.dirname(os.path.dirname(__file__)) 4 | 5 | name = os.getenv("GITHUB_TAG").split("/")[2] 6 | path = os.path.join(basedir, "django_toosimple_q", "__init__.py") 7 | 8 | # read file 9 | contents = open(path, "r").read() 10 | 11 | # replace contents 12 | open(path, "w").write(contents.replace("dev", name, 1)) 13 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import absolute_import, unicode_literals 4 | 5 | import os 6 | import sys 7 | 8 | if __name__ == "__main__": 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_toosimple_q.tests.settings") 10 | from django.core.management import execute_from_command_line 11 | 12 | execute_from_command_line(sys.argv) 13 | -------------------------------------------------------------------------------- /scripts/ci_assert_version_from_git.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | basedir = os.path.dirname(os.path.dirname(__file__)) 5 | sys.path.append(basedir) 6 | 7 | import django_toosimple_q # noqa 8 | 9 | version = django_toosimple_q.__version__ 10 | tag = os.getenv("GITHUB_TAG") 11 | 12 | assert ( 13 | f"refs/heads/{version}" == tag or f"refs/tags/{version}" == tag 14 | ), f"Version mismatch : {version} != {tag}" 15 | -------------------------------------------------------------------------------- /django_toosimple_q/registry.py: -------------------------------------------------------------------------------- 1 | class Registry(dict): 2 | def for_queue(self, queues=None, excluded_queues=None): 3 | for item in self.values(): 4 | if queues and item.queue not in queues: 5 | continue 6 | if excluded_queues and item.queue in excluded_queues: 7 | continue 8 | yield item 9 | 10 | 11 | schedules_registry = Registry() 12 | tasks_registry = Registry() 13 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/settings_bg_lag.py: -------------------------------------------------------------------------------- 1 | """Settings overrides for testing concurrent workers by simulating latency with toxiproxy which""" 2 | 3 | from .settings_bg import * 4 | 5 | # On postgres, background workers connect through toxiproxy to simluate latency 6 | if is_postgres(): 7 | DATABASES["default"]["HOST"] = os.environ.get("POSTGRES_HOST_WORKER", "127.0.0.1") 8 | DATABASES["default"]["PORT"] = os.environ.get("POSTGRES_PORT_WORKER", "5433") 9 | -------------------------------------------------------------------------------- /django_toosimple_q/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils.module_loading import autodiscover_modules 3 | 4 | from .logging import logger, show_registry 5 | 6 | 7 | class DjangoToosimpleQConfig(AppConfig): 8 | name = "django_toosimple_q" 9 | label = "toosimpleq" 10 | 11 | def ready(self): 12 | # Autodicover tasks.py modules 13 | 14 | logger.info("Autodiscovering tasks.py...") 15 | autodiscover_modules("tasks") 16 | 17 | show_registry() 18 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0003_task_queue.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.6 on 2019-12-04 13:32 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0002_auto_20191101_1838"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="task", 14 | name="queue", 15 | field=models.CharField(default="default", max_length=32), 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0004_auto_20200507_1339.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-05-07 11:39 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0003_task_queue"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="schedule", 14 | name="name", 15 | field=models.CharField(max_length=1024, unique=True), 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0007_schedule_datetime_kwarg.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2021-06-22 17:08 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0006_task_replacement"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="schedule", 14 | name="datetime_kwarg", 15 | field=models.CharField(blank=True, max_length=1024, null=True), 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /django_toosimple_q/contrib/mail/tasks.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.mail import get_connection 3 | 4 | from django_toosimple_q.decorators import register_task 5 | 6 | 7 | @register_task(unique=True, retries=10, retry_delay=3) 8 | def send_email(emails): 9 | backend = getattr( 10 | settings, 11 | "TOOSIMPLEQ_EMAIL_BACKEND", 12 | "django.core.mail.backends.smtp.EmailBackend", 13 | ) 14 | 15 | conn = get_connection(backend=backend) 16 | conn.open() 17 | conn.send_messages(emails) 18 | conn.close() 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG TOOSIMPLEQ_PY_VERSION 2 | 3 | FROM python:$TOOSIMPLEQ_PY_VERSION 4 | 5 | WORKDIR /app 6 | 7 | # Install app in editable mode 8 | ADD ./requirements.txt /app/requirements.txt 9 | ADD ./requirements-dev.txt /app/requirements-dev.txt 10 | RUN touch /app/README.md 11 | ADD ./django_toosimple_q/__init__.py /app/django_toosimple_q/__init__.py 12 | ADD ./setup.py /app/setup.py 13 | RUN pip install -r requirements-dev.txt 14 | 15 | # Override django version 16 | ARG TOOSIMPLEQ_DJ_VERSION 17 | RUN pip install Django==$TOOSIMPLEQ_DJ_VERSION 18 | 19 | # Add source files 20 | ADD . /app 21 | 22 | # Default command runs tests 23 | ENTRYPOINT ["python", "manage.py"] 24 | CMD ["--help"] 25 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0014_alter_workerstatus_exit_code.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.5 on 2023-02-23 23:54 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0013_workerstatus_exit_code_workerstatus_exit_log"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="workerstatus", 14 | name="exit_code", 15 | field=models.IntegerField( 16 | blank=True, 17 | choices=[(0, "Stopped"), (77, "Terminated"), (99, "Crashed")], 18 | null=True, 19 | ), 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0006_task_replacement.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2021-06-14 19:54 2 | 3 | import django.db.models.deletion 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("toosimpleq", "0005_auto_20210302_1748"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="task", 15 | name="replaced_by", 16 | field=models.ForeignKey( 17 | blank=True, 18 | null=True, 19 | on_delete=django.db.models.deletion.SET_NULL, 20 | to="toosimpleq.Task", 21 | ), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0009_auto_20210902_2245.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-09-02 20:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0008_auto_20210902_2111"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="schedule", 14 | name="id", 15 | field=models.BigAutoField(primary_key=True, serialize=False), 16 | ), 17 | migrations.AlterField( 18 | model_name="task", 19 | name="id", 20 | field=models.BigAutoField(primary_key=True, serialize=False), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0012_rename_last_run_scheduleexec_last_task.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.5 on 2023-01-12 21:14 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0011_workerstatus"), 9 | ] 10 | 11 | operations = [ 12 | migrations.RenameField( 13 | model_name="scheduleexec", 14 | old_name="last_run", 15 | new_name="last_task", 16 | ), 17 | migrations.AddField( 18 | model_name="scheduleexec", 19 | name="last_due", 20 | field=models.DateTimeField(blank=True, null=True), 21 | ), 22 | migrations.RemoveField( 23 | model_name="scheduleexec", 24 | name="last_tick", 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0013_workerstatus_exit_code_workerstatus_exit_log.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.5 on 2023-01-11 23:24 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0012_rename_last_run_scheduleexec_last_task"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="workerstatus", 14 | name="exit_log", 15 | field=models.TextField(blank=True, null=True), 16 | ), 17 | migrations.AddField( 18 | model_name="workerstatus", 19 | name="exit_code", 20 | field=models.IntegerField( 21 | blank=True, 22 | choices=[(0, "Stopped"), (1, "Terminated"), (2, "Crashed")], 23 | null=True, 24 | ), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: pypi 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Install dependencies 14 | run: | 15 | python -m pip install --upgrade pip 16 | pip install setuptools wheel twine==6.0.1 17 | - name: Set version 18 | env: 19 | GITHUB_TAG: ${{github.ref}} 20 | run: python scripts/ci_set_version_from_git.py 21 | - name: Build 22 | run: python setup.py sdist bdist_wheel 23 | - name: Assert version 24 | env: 25 | GITHUB_TAG: ${{github.ref}} 26 | run: python scripts/ci_assert_version_from_git.py 27 | - name: Publish 28 | env: 29 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 30 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 31 | run: twine upload dist/* 32 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0002_auto_20191101_1838.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.6 on 2019-11-01 17:38 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0001_initial"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="task", 14 | name="state", 15 | field=models.CharField( 16 | choices=[ 17 | ("QUEUED", "QUEUED"), 18 | ("PROCESSING", "PROCESSING"), 19 | ("FAILED", "FAILED"), 20 | ("SUCCEEDED", "SUCCEEDED"), 21 | ("INVALID", "INVALID"), 22 | ("INTERRUPTED", "INTERRUPTED"), 23 | ], 24 | default="QUEUED", 25 | max_length=32, 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0015_taskexec_result_preview.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1 on 2024-03-04 15:28 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | def populate_result_preview(apps, schema_editor): 7 | # Populate the new result preview field 8 | TaskExec = apps.get_model("toosimpleq", "TaskExec") 9 | TaskExec.objects.filter(result__isnull=False).update( 10 | result_preview="*preview not generated*" 11 | ) 12 | 13 | 14 | class Migration(migrations.Migration): 15 | dependencies = [ 16 | ("toosimpleq", "0014_alter_workerstatus_exit_code"), 17 | ] 18 | 19 | operations = [ 20 | migrations.AddField( 21 | model_name="taskexec", 22 | name="result_preview", 23 | field=models.CharField( 24 | blank=True, editable=False, max_length=255, null=True 25 | ), 26 | ), 27 | migrations.RunPython(populate_result_preview), 28 | ] 29 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0016_alter_scheduleexec_options_alter_taskexec_options.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1 on 2025-04-17 08:30 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("toosimpleq", "0015_taskexec_result_preview"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterModelOptions( 13 | name="scheduleexec", 14 | options={ 15 | "permissions": [ 16 | ("force_run_scheduleexec", "Can force execution of schedules") 17 | ], 18 | "verbose_name": "Schedule Execution", 19 | }, 20 | ), 21 | migrations.AlterModelOptions( 22 | name="taskexec", 23 | options={ 24 | "permissions": [("requeue_taskexec", "Can requeue tasks")], 25 | "verbose_name": "Task Execution", 26 | }, 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0008_auto_20210902_2111.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2021-09-02 19:11 2 | 3 | import django.db.models.deletion 4 | import django.utils.timezone 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | dependencies = [ 10 | ("toosimpleq", "0007_schedule_datetime_kwarg"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="schedule", 16 | name="last_check", 17 | field=models.DateTimeField( 18 | blank=True, default=django.utils.timezone.now, null=True 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="schedule", 23 | name="last_run", 24 | field=models.ForeignKey( 25 | blank=True, 26 | null=True, 27 | on_delete=django.db.models.deletion.SET_NULL, 28 | to="toosimpleq.task", 29 | ), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # files: '^.*\.py|.*\.tpl|.*\.sql|.*\.yaml|.*\.txt|.*\.md$' 2 | # exclude: '^.*\.py|.*\.tpl|.*\.sql|.*\.yaml|.*\.txt|.*\.md$' 3 | 4 | repos: 5 | # Fix end of files 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v4.4.0 8 | hooks: 9 | - id: trailing-whitespace 10 | - id: end-of-file-fixer 11 | - id: mixed-line-ending 12 | args: 13 | - '--fix=lf' 14 | 15 | # Remove unused imports/variables 16 | - repo: https://github.com/myint/autoflake 17 | rev: v2.0.1 18 | hooks: 19 | - id: autoflake 20 | args: 21 | - "--in-place" 22 | - "--remove-all-unused-imports" 23 | - "--remove-unused-variable" 24 | 25 | # Sort imports 26 | - repo: https://github.com/pycqa/isort 27 | rev: "5.12.0" 28 | hooks: 29 | - id: isort 30 | args: 31 | - --profile 32 | - black 33 | 34 | # Black formatting 35 | - repo: https://github.com/psf/black 36 | rev: "23.1.0" 37 | hooks: 38 | - id: black 39 | -------------------------------------------------------------------------------- /django_toosimple_q/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .registry import schedules_registry, tasks_registry 4 | 5 | formatter = logging.Formatter( 6 | "[%(asctime)s][%(levelname)s][toosimpleq] %(message)s", "%Y-%m-%d %H:%M:%S" 7 | ) 8 | 9 | handler = logging.StreamHandler() 10 | handler.setFormatter(formatter) 11 | 12 | logger = logging.getLogger("toosimpleq") 13 | logger.setLevel(logging.INFO) 14 | logger.addHandler(handler) 15 | 16 | 17 | def show_registry(): 18 | """Helper functions that shows the registry contents""" 19 | 20 | if len(schedules_registry): 21 | schedules_names = ", ".join(schedules_registry.keys()) 22 | logger.info( 23 | f"{len(schedules_registry)} schedules registered: {schedules_names}" 24 | ) 25 | else: 26 | logger.info("No schedules registered") 27 | 28 | if len(tasks_registry): 29 | tasks_names = ", ".join(tasks_registry.keys()) 30 | logger.info(f"{len(tasks_registry)} tasks registered: {tasks_names}") 31 | else: 32 | logger.info("No tasks registered") 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Olivier Dalang 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/concurrency/tasks.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.contrib.auth.models import User 4 | from django.db import IntegrityError 5 | 6 | from ...decorators import register_task, schedule_task 7 | 8 | 9 | @schedule_task(cron="* * * * *", queue="schedules", run_on_creation=True) 10 | @register_task(name="create_user", queue="tasks") 11 | def create_user(): 12 | time.sleep(0.5) 13 | retry = 0 14 | while True: 15 | suffix = f"-copy{retry}" if retry > 0 else "" 16 | try: 17 | User.objects.create(username=f"user{suffix}") 18 | break 19 | except IntegrityError: 20 | retry += 1 21 | 22 | if retry > 0: 23 | raise Exception("Failed: had to rename the user") 24 | return 0 25 | 26 | 27 | @register_task(name="sleep_task", queue="tasks") 28 | def sleep_task(duration): 29 | t1 = time.time() 30 | while time.time() - t1 < duration: 31 | pass 32 | return True 33 | 34 | 35 | @register_task(name="output_string_task", queue="tasks") 36 | def output_string_task(): 37 | return "***OUTPUT_A***" 38 | 39 | 40 | @schedule_task(cron="* * * * * *", queue="regr_schedule_short") 41 | @register_task(name="regr_schedule_short", queue="_") 42 | def regr_schedule_short(): 43 | return True 44 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_integration.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.core.management import call_command 4 | from django.test import TestCase 5 | 6 | 7 | class TestIntegration(TestCase): 8 | def test_readme(self): 9 | # Test code in the readme 10 | 11 | readme = open("README.md", "r").read() 12 | 13 | # This finds all ```python``` blocks 14 | python_blocks = re.findall(r"```python\n([\s\S]*?)```", readme, re.MULTILINE) 15 | 16 | # Run each block 17 | full_python_code = "" 18 | for python_block in python_blocks: 19 | # We concatenate all blocks with the previous ones (so we keep imports) 20 | full_python_code += python_block 21 | try: 22 | exec(full_python_code) 23 | except Exception as e: 24 | hr = "~" * 80 25 | raise Exception( 26 | f"Invalid readme block:\n{hr}\n{python_block}{hr}" 27 | ) from e 28 | 29 | def test_makemigrations(self): 30 | # Ensure migrations are up to date with model changes 31 | try: 32 | call_command("makemigrations", "--check", "--dry-run") 33 | except SystemExit: 34 | raise AssertionError( 35 | "Migrations are not up to date. You need to run `makemigrations`." 36 | ) 37 | -------------------------------------------------------------------------------- /django_toosimple_q/schedule.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Dict, List, Optional 3 | 4 | from .logging import logger 5 | from .task import Task 6 | 7 | 8 | class Schedule: 9 | """A configuration for repeated execution of tasks. These are typically configured in `tasks.py`""" 10 | 11 | def __init__( 12 | self, 13 | name: str, 14 | task: Task, 15 | cron: str, 16 | queue: str = "default", 17 | args: List = [], 18 | kwargs: Dict = {}, 19 | datetime_kwarg: str = None, 20 | catch_up: bool = False, 21 | run_on_creation: bool = False, 22 | ): 23 | self.name = name 24 | self.task = task 25 | self.cron = cron 26 | self.queue = queue 27 | self.args = args 28 | self.kwargs = kwargs 29 | self.datetime_kwarg = datetime_kwarg 30 | self.catch_up = catch_up 31 | self.run_on_creation = run_on_creation 32 | 33 | def execute(self, dues: List[Optional[datetime]]): 34 | """Enqueues the related tasks at the given due dates""" 35 | 36 | # We enqueue the due tasks 37 | for due in dues: 38 | logger.debug(f"{self} is due at {due}") 39 | 40 | dt_kwarg = {} 41 | if self.datetime_kwarg: 42 | dt_kwarg = {self.datetime_kwarg: due} 43 | 44 | self.task.enqueue(*self.args, due=due, **dt_kwarg, **self.kwargs) 45 | 46 | def __str__(self): 47 | return f"Schedule {self.name}" 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from setuptools import find_packages, setup 5 | 6 | 7 | def get_version(*file_paths): 8 | """Retrieves the version from django_toosimple_q/__init__.py""" 9 | filename = os.path.join(os.path.dirname(__file__), *file_paths) 10 | version_file = open(filename).read() 11 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) 12 | if version_match: 13 | version_str = version_match.group(1) 14 | return "0.0.0-dev" if version_str == "dev" else version_str 15 | raise RuntimeError("Unable to find version string.") 16 | 17 | 18 | setup( 19 | name="django-toosimple-q", 20 | version=get_version("django_toosimple_q", "__init__.py"), 21 | description="""A simplistic task queue and cron-like scheduler for Django""", 22 | long_description=open("README.md").read(), 23 | long_description_content_type="text/markdown", 24 | author="Olivier Dalang", 25 | author_email="olivier.dalang@gmail.com", 26 | url="https://github.com/olivierdalang/django-toosimple-q", 27 | packages=find_packages(include=["django_toosimple_q", "django_toosimple_q.*"]), 28 | include_package_data=True, 29 | install_requires=open("requirements.txt").readlines(), 30 | license="MIT", 31 | zip_safe=False, 32 | keywords="django-toosimple-q", 33 | classifiers=[ 34 | "Development Status :: 3 - Alpha", 35 | "Framework :: Django :: 2.2", 36 | "Intended Audience :: Developers", 37 | "License :: OSI Approved :: BSD License", 38 | "Natural Language :: English", 39 | "Programming Language :: Python :: 3", 40 | "Programming Language :: Python :: 3.7", 41 | ], 42 | ) 43 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | name: Tests dj${{ matrix.dj }} / ${{ matrix.db }} / py${{ matrix.py }} 12 | 13 | runs-on: ubuntu-latest 14 | strategy: 15 | # all supported django version on lowest/highest supported python 16 | # see https://www.djangoproject.com/download/#supported-versions 17 | # and https://docs.djangoproject.com/en/4.2/faq/install/#what-python-version-can-i-use-with-django 18 | matrix: 19 | include: 20 | - { dj: "4.2", db: "sqlite", py: "3.8" } 21 | - { dj: "4.2", db: "sqlite", py: "3.12" } 22 | - { dj: "4.2", db: "postgres", py: "3.8" } 23 | - { dj: "4.2", db: "postgres", py: "3.12" } 24 | - { dj: "5.1", db: "sqlite", py: "3.10" } 25 | - { dj: "5.1", db: "sqlite", py: "3.12" } 26 | - { dj: "5.1", db: "postgres", py: "3.10" } 27 | - { dj: "5.1", db: "postgres", py: "3.12" } 28 | - { dj: "5.2", db: "sqlite", py: "3.10" } 29 | - { dj: "5.2", db: "sqlite", py: "3.13" } 30 | - { dj: "5.2", db: "postgres", py: "3.10" } 31 | - { dj: "5.2", db: "postgres", py: "3.13" } 32 | fail-fast: false 33 | 34 | env: 35 | TOOSIMPLEQ_DJ_VERSION: ${{ matrix.dj }} 36 | TOOSIMPLEQ_PY_VERSION: ${{ matrix.py }} 37 | TOOSIMPLEQ_TEST_DB: ${{ matrix.db }} 38 | 39 | steps: 40 | - uses: actions/checkout@v1 41 | 42 | - name: Lint with pre-commit 43 | uses: pre-commit/action@v3.0.1 44 | 45 | - name: Docker build 46 | run: docker compose build 47 | 48 | - name: Run tests 49 | run: docker compose run django test 50 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0005_auto_20210302_1748.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2021-03-02 16:48 2 | 3 | import django.utils.timezone 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("toosimpleq", "0004_auto_20200507_1339"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="task", 15 | name="due", 16 | field=models.DateTimeField(default=django.utils.timezone.now), 17 | ), 18 | migrations.AddField( 19 | model_name="task", 20 | name="retries", 21 | field=models.IntegerField( 22 | default=0, help_text="retries left, -1 means infinite" 23 | ), 24 | ), 25 | migrations.AddField( 26 | model_name="task", 27 | name="retry_delay", 28 | field=models.IntegerField( 29 | default=0, 30 | help_text="Delay before next retry in seconds. Will double after each failure.", 31 | ), 32 | ), 33 | migrations.AlterField( 34 | model_name="task", 35 | name="state", 36 | field=models.CharField( 37 | choices=[ 38 | ("QUEUED", "QUEUED"), 39 | ("SLEEPING", "SLEEPING"), 40 | ("PROCESSING", "PROCESSING"), 41 | ("FAILED", "FAILED"), 42 | ("SUCCEEDED", "SUCCEEDED"), 43 | ("INVALID", "INVALID"), 44 | ("INTERRUPTED", "INTERRUPTED"), 45 | ], 46 | default="QUEUED", 47 | max_length=32, 48 | ), 49 | ), 50 | ] 51 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0011_workerstatus.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.12 on 2022-03-27 20:22 2 | 3 | import datetime 4 | 5 | import django.db.models.deletion 6 | import django.utils.timezone 7 | from django.db import migrations, models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | ("toosimpleq", "0010_auto_20220324_0419"), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name="WorkerStatus", 18 | fields=[ 19 | ("id", models.BigAutoField(primary_key=True, serialize=False)), 20 | ("label", models.CharField(max_length=1024, unique=True)), 21 | ("included_queues", models.JSONField(default=list)), 22 | ("excluded_queues", models.JSONField(default=list)), 23 | ( 24 | "timeout", 25 | models.DurationField(default=datetime.timedelta(seconds=3600)), 26 | ), 27 | ("last_tick", models.DateTimeField(default=django.utils.timezone.now)), 28 | ("started", models.DateTimeField(default=django.utils.timezone.now)), 29 | ("stopped", models.DateTimeField(blank=True, null=True)), 30 | ], 31 | options={ 32 | "verbose_name": "Worker Status", 33 | "verbose_name_plural": "Workers Statuses", 34 | }, 35 | ), 36 | migrations.AddField( 37 | model_name="taskexec", 38 | name="worker", 39 | field=models.ForeignKey( 40 | blank=True, 41 | null=True, 42 | on_delete=django.db.models.deletion.SET_NULL, 43 | to="toosimpleq.workerstatus", 44 | ), 45 | ), 46 | ] 47 | -------------------------------------------------------------------------------- /django_toosimple_q/decorators.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ImproperlyConfigured 2 | 3 | from .registry import schedules_registry, tasks_registry 4 | from .schedule import Schedule 5 | from .task import Task 6 | 7 | 8 | def register_task(**kwargs): 9 | """Attaches ._task attribute, the .queue() method and adds the callable to the tasks registry""" 10 | 11 | def inner(func): 12 | # Default name is the qualified function name 13 | if "name" not in kwargs: 14 | kwargs["name"] = func.__globals__["__name__"] + "." + func.__qualname__ 15 | 16 | # Create the task instance 17 | kwargs["callable"] = func 18 | task = Task(**kwargs) 19 | 20 | # Attach that instance to the callable 21 | func._task = task 22 | 23 | # Include the `queue` callable 24 | func.queue = task.enqueue 25 | 26 | # Add to the registry 27 | tasks_registry[task.name] = task 28 | 29 | # Decorator returns the function itself 30 | return func 31 | 32 | return inner 33 | 34 | 35 | def schedule_task(**kwargs): 36 | """Adds the task to the schedules registry""" 37 | 38 | def inner(func): 39 | if not hasattr(func, "_task"): 40 | raise ImproperlyConfigured( 41 | "Only registered tasks can be scheduled." 42 | " Are you sure you registered your callable with the @register_task() decorator ?" 43 | ) 44 | 45 | # Default name is the name of the task 46 | if "name" not in kwargs: 47 | kwargs["name"] = func._task.name 48 | 49 | # Create the schedule instance 50 | kwargs["task"] = func._task 51 | schedule = Schedule(**kwargs) 52 | 53 | # Add to the registry 54 | schedules_registry[kwargs["name"]] = schedule 55 | 56 | # Decorator returns the function itself 57 | return func 58 | 59 | return inner 60 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Docker-compose configuration to run tests 2 | 3 | version: "3" 4 | 5 | x-default-django: 6 | &default-django 7 | build: 8 | context: . 9 | args: 10 | TOOSIMPLEQ_DJ_VERSION: ${TOOSIMPLEQ_DJ_VERSION:-5.0} 11 | TOOSIMPLEQ_PY_VERSION: ${TOOSIMPLEQ_PY_VERSION:-3.11} 12 | depends_on: 13 | postgres: 14 | condition: service_healthy 15 | toxiproxy-config: 16 | condition: service_started 17 | environment: 18 | TOOSIMPLEQ_TEST_DB: ${TOOSIMPLEQ_TEST_DB:-postgres} 19 | POSTGRES_HOST: postgres 20 | POSTGRES_PORT: 5432 21 | POSTGRES_HOST_WORKER: postgres-laggy 22 | POSTGRES_PORT_WORKER: 5433 23 | volumes: 24 | - .:/app 25 | 26 | services: 27 | 28 | django: 29 | <<: *default-django 30 | command: runserver 0.0.0.0:8000 31 | ports: 32 | - 8000:8000 33 | 34 | worker: 35 | <<: *default-django 36 | command: worker --queue demo --verbosity 3 37 | 38 | postgres: 39 | image: postgres 40 | environment: 41 | POSTGRES_PASSWORD: postgres 42 | healthcheck: 43 | test: ["CMD", "bash", "-c", "pg_isready -U postgres"] 44 | interval: 5s 45 | retries: 6 46 | start_period: 1s 47 | ports: 48 | - 5432:5432 49 | 50 | postgres-laggy: 51 | image: ghcr.io/shopify/toxiproxy 52 | depends_on: 53 | postgres: 54 | condition: service_healthy 55 | environment: 56 | POSTGRES_PASSWORD: postgres 57 | ports: 58 | - 5433:5433 59 | 60 | toxiproxy-config: 61 | image: docker:cli 62 | depends_on: 63 | postgres-laggy: 64 | condition: service_started 65 | volumes: 66 | - /var/run/docker.sock:/var/run/docker.sock 67 | command: | 68 | sh -c ' 69 | docker exec django-toosimple-q-postgres-laggy-1 /toxiproxy-cli create -l 0.0.0.0:5433 -u postgres:5432 postgres 70 | docker exec django-toosimple-q-postgres-laggy-1 /toxiproxy-cli toxic add -t latency -n my_lag -a latency=100 -a jitter=5 postgres 71 | ' 72 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/demo/tasks.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import random 3 | import sys 4 | import time 5 | 6 | from django.utils import timezone 7 | from django.utils.formats import time_format 8 | 9 | from ...decorators import register_task, schedule_task 10 | from ...models import TaskExec 11 | 12 | 13 | @schedule_task( 14 | cron="* * * * * */30", datetime_kwarg="scheduled_time", queue="demo", catch_up=True 15 | ) 16 | @register_task(name="say_hi", queue="demo") 17 | def say_hi(scheduled_time): 18 | if scheduled_time is None: 19 | return "Hi ! This was not scheduled..." 20 | return f"Hi at {time_format(scheduled_time)}" 21 | 22 | 23 | @schedule_task(cron="0 * * * * *", queue="demo") 24 | @register_task(name="flaky", retries=3, retry_delay=2, queue="demo") 25 | def flaky(): 26 | if random.random() < 0.5: 27 | raise Exception("This failed, we'll retry !") 28 | else: 29 | return "This succeeded" 30 | 31 | 32 | @schedule_task(cron="0 * * * * *", queue="demo") 33 | @register_task(name="logging", queue="demo") 34 | def logging(): 35 | sys.stdout.write("This should go to standard output") 36 | sys.stderr.write("This should go to error output") 37 | return "This is the result" 38 | 39 | 40 | @schedule_task(cron="0 */5 * * * *", queue="demo") 41 | @register_task(name="long_running", queue="demo") 42 | def long_running(): 43 | text = f"started at {timezone.now()}\n" 44 | time.sleep(15) 45 | text += f"continue at {timezone.now()}\n" 46 | time.sleep(15) 47 | text += f"continue at {timezone.now()}\n" 48 | time.sleep(15) 49 | text += f"continue at {timezone.now()}\n" 50 | time.sleep(15) 51 | text += f"finishing at {timezone.now()}\n" 52 | return text 53 | 54 | 55 | @schedule_task(cron="manual", queue="demo") 56 | @register_task(name="cleanup", queue="demo", priority=-5) 57 | def cleanup(): 58 | old_tasks_execs = TaskExec.objects.filter( 59 | created__lte=timezone.now() - datetime.timedelta(minutes=10) 60 | ) 61 | deletes = old_tasks_execs.delete() 62 | print(f"Deleted {deletes}") 63 | return True 64 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/settings.py: -------------------------------------------------------------------------------- 1 | """Settings for running tests""" 2 | 3 | import os 4 | 5 | from .utils import is_postgres 6 | 7 | DEBUG = True 8 | USE_TZ = True 9 | TIME_ZONE = "UTC" 10 | SECRET_KEY = "secret_key" 11 | 12 | if is_postgres(): 13 | DATABASES = { 14 | "default": { 15 | "ENGINE": "django.db.backends.postgresql", 16 | "HOST": os.environ.get("POSTGRES_HOST", "127.0.0.1"), 17 | "PORT": os.environ.get("POSTGRES_PORT", "5432"), 18 | "NAME": "postgres", 19 | "USER": "postgres", 20 | "PASSWORD": "postgres", 21 | "TEST": { 22 | "NAME": "test_postgres", 23 | }, 24 | } 25 | } 26 | else: 27 | DATABASES = { 28 | "default": { 29 | "ENGINE": "django.db.backends.sqlite3", 30 | "NAME": "db.sqlite3", 31 | "OPTIONS": {"timeout": 50}, 32 | "TEST": { 33 | "NAME": "db-test.sqlite3", 34 | }, 35 | } 36 | } 37 | 38 | INSTALLED_APPS = [ 39 | "django_toosimple_q.tests.concurrency", 40 | "django_toosimple_q.tests.demo", 41 | "django_toosimple_q", 42 | "django.contrib.admin", 43 | "django.contrib.auth", 44 | "django.contrib.contenttypes", 45 | "django.contrib.messages", 46 | "django.contrib.sessions", 47 | "django.contrib.staticfiles", 48 | ] 49 | 50 | ROOT_URLCONF = "django_toosimple_q.tests.urls" 51 | 52 | MIDDLEWARE = ( 53 | "django.contrib.sessions.middleware.SessionMiddleware", 54 | "django.contrib.auth.middleware.AuthenticationMiddleware", 55 | "django.contrib.messages.middleware.MessageMiddleware", 56 | ) 57 | 58 | TEMPLATES = [ 59 | { 60 | "BACKEND": "django.template.backends.django.DjangoTemplates", 61 | "APP_DIRS": True, 62 | "OPTIONS": { 63 | "context_processors": [ 64 | "django.contrib.auth.context_processors.auth", 65 | "django.contrib.messages.context_processors.messages", 66 | "django.template.context_processors.request", 67 | ] 68 | }, 69 | } 70 | ] 71 | 72 | STATIC_URL = "/static/" 73 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_regression.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.core import management 4 | 5 | from django_toosimple_q.decorators import register_task, schedule_task 6 | from django_toosimple_q.models import TaskExec 7 | from django_toosimple_q.registry import schedules_registry, tasks_registry 8 | 9 | from .base import TooSimpleQBackgroundTestCase, TooSimpleQRegularTestCase 10 | 11 | 12 | class TestRegressionBackground(TooSimpleQBackgroundTestCase): 13 | def test_regr_schedule_short(self): 14 | # Regression test for an issue where a schedule with smaller periods was not always processed 15 | 16 | # A worker that ticks every second should trigger a schedule due every second 17 | self.start_worker_in_background( 18 | queue="regr_schedule_short", tick=1, until_done=False, verbosity=3 19 | ) 20 | time.sleep(20) 21 | 22 | # It should do almost 20 tasks 23 | self.assertGreaterEqual(TaskExec.objects.all().count(), 18) 24 | 25 | 26 | class TestRegressionRegular(TooSimpleQRegularTestCase): 27 | def test_deleting_schedule(self): 28 | # Regression test for an issue where deleting a schedule in code would crash the admin view 29 | 30 | @schedule_task(cron="0 12 * * *", datetime_kwarg="scheduled_on") 31 | @register_task(name="normal") 32 | def a(scheduled_on): 33 | return f"{scheduled_on:%Y-%m-%d %H:%M}" 34 | 35 | management.call_command("worker", "--until_done") 36 | 37 | schedules_registry.clear() 38 | tasks_registry.clear() 39 | 40 | # the admin view still works even for deleted schedules 41 | response = self.client.get("/admin/toosimpleq/scheduleexec/") 42 | self.assertEqual(response.status_code, 200) 43 | 44 | def test_different_schedule_and_task(self): 45 | # Regression test for an issue where schedule with a different name than the task would fail 46 | 47 | @schedule_task(cron="0 12 * * *", name="name_a", run_on_creation=True) 48 | @register_task(name="name_b") 49 | def a(): 50 | return True 51 | 52 | management.call_command("worker", "--until_done") 53 | -------------------------------------------------------------------------------- /django_toosimple_q/task.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | from django.utils import timezone 4 | 5 | from .logging import logger 6 | 7 | 8 | class Task: 9 | """Represents an asnychronous task. 10 | 11 | This class is responsible of queuing and executing the tasks, by managing 12 | TaskExec instances.""" 13 | 14 | def __init__( 15 | self, 16 | name: str, 17 | callable: Callable, 18 | queue: str = "default", 19 | priority: int = 0, 20 | unique: bool = False, 21 | retries: int = 0, 22 | retry_delay: int = 0, 23 | ): 24 | self.name = name 25 | self.callable = callable 26 | self.queue = queue 27 | self.priority = priority 28 | self.unique = unique 29 | self.retries = retries 30 | self.retry_delay = retry_delay 31 | 32 | def enqueue(self, *args_, due=None, **kwargs_): 33 | """Creates a TaskExec instance, effectively queuing execution of this task. 34 | 35 | Returns the created TaskExec, or False if no task was created (which can happen 36 | for tasks set as unique, if that task already exists).""" 37 | 38 | from .models import TaskExec 39 | 40 | logger.debug(f"Enqueuing task '{self.name}'") 41 | 42 | due_datetime = due or timezone.now() 43 | 44 | if self.unique: 45 | existing_tasks = TaskExec.objects.filter( 46 | task_name=self.name, args=args_, kwargs=kwargs_ 47 | ) 48 | # If already queued, we don't do anything 49 | queued_task = existing_tasks.filter(state=TaskExec.States.QUEUED).first() 50 | if queued_task is not None: 51 | return False 52 | # If there's already a same task that's sleeping 53 | sleeping_task = existing_tasks.filter( 54 | state=TaskExec.States.SLEEPING 55 | ).first() 56 | if sleeping_task is not None: 57 | if due is None: 58 | # If the queuing is not delayed, we enqueue it now 59 | sleeping_task.due = due_datetime 60 | sleeping_task.state = TaskExec.States.QUEUED 61 | sleeping_task.save() 62 | elif sleeping_task.due > due_datetime: 63 | # If it's delayed to less than the current due date of the task 64 | sleeping_task.due = min(sleeping_task.due, due_datetime) 65 | sleeping_task.save() 66 | return False 67 | 68 | return TaskExec.objects.create( 69 | task_name=self.name, 70 | args=args_, 71 | kwargs=kwargs_, 72 | state=TaskExec.States.SLEEPING if due else TaskExec.States.QUEUED, 73 | due=due_datetime, 74 | retries=self.retries, 75 | retry_delay=self.retry_delay, 76 | ) 77 | 78 | def __str__(self): 79 | return f"Task {self.name}" 80 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0010_auto_20220324_0419.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.12 on 2022-03-23 20:40 2 | 3 | import django 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("toosimpleq", "0009_auto_20210902_2245"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RenameModel( 14 | old_name="Schedule", 15 | new_name="ScheduleExec", 16 | ), 17 | migrations.RenameModel( 18 | old_name="Task", 19 | new_name="TaskExec", 20 | ), 21 | migrations.AlterModelOptions( 22 | name="scheduleexec", 23 | options={"verbose_name": "Schedule Execution"}, 24 | ), 25 | migrations.AlterModelOptions( 26 | name="taskexec", 27 | options={"verbose_name": "Task Execution"}, 28 | ), 29 | migrations.RenameField( 30 | model_name="taskexec", 31 | old_name="function", 32 | new_name="task_name", 33 | ), 34 | migrations.RenameField( 35 | model_name="scheduleexec", 36 | old_name="last_check", 37 | new_name="last_tick", 38 | ), 39 | migrations.AlterField( 40 | model_name="scheduleexec", 41 | name="last_tick", 42 | field=models.DateTimeField(default=django.utils.timezone.now), 43 | ), 44 | migrations.RemoveField( 45 | model_name="scheduleexec", 46 | name="args", 47 | ), 48 | migrations.RemoveField( 49 | model_name="scheduleexec", 50 | name="catch_up", 51 | ), 52 | migrations.RemoveField( 53 | model_name="scheduleexec", 54 | name="cron", 55 | ), 56 | migrations.RemoveField( 57 | model_name="scheduleexec", 58 | name="datetime_kwarg", 59 | ), 60 | migrations.RemoveField( 61 | model_name="scheduleexec", 62 | name="function", 63 | ), 64 | migrations.RemoveField( 65 | model_name="scheduleexec", 66 | name="kwargs", 67 | ), 68 | migrations.AddField( 69 | model_name="scheduleexec", 70 | name="state", 71 | field=models.CharField( 72 | choices=[("ACTIVE", "Active"), ("INVALID", "Invalid")], 73 | default="ACTIVE", 74 | max_length=32, 75 | ), 76 | ), 77 | migrations.AddField( 78 | model_name="taskexec", 79 | name="error", 80 | field=models.TextField(blank=True, null=True), 81 | ), 82 | migrations.AlterField( 83 | model_name="taskexec", 84 | name="state", 85 | field=models.CharField( 86 | choices=[ 87 | ("SLEEPING", "Sleeping"), 88 | ("QUEUED", "Queued"), 89 | ("PROCESSING", "Processing"), 90 | ("SUCCEEDED", "Succeeded"), 91 | ("INTERRUPTED", "Interrupted"), 92 | ("FAILED", "Failed"), 93 | ("INVALID", "Invalid"), 94 | ], 95 | default="QUEUED", 96 | max_length=32, 97 | ), 98 | ), 99 | migrations.RemoveField( 100 | model_name="taskexec", 101 | name="priority", 102 | ), 103 | migrations.RemoveField( 104 | model_name="taskexec", 105 | name="queue", 106 | ), 107 | ] 108 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_concurrency.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import unittest 4 | 5 | from django.contrib.auth.models import User 6 | 7 | from django_toosimple_q.models import TaskExec, WorkerStatus 8 | 9 | from .base import TooSimpleQBackgroundTestCase 10 | from .concurrency.tasks import create_user, sleep_task 11 | from .utils import is_postgres 12 | 13 | COUNT = 32 14 | 15 | 16 | # FIXME: not sure if we really can't have this working on SQLITE ? 17 | @unittest.skipIf(not is_postgres(), "requires postgres backend") 18 | class ConcurrencyTest(TooSimpleQBackgroundTestCase): 19 | """This runs some concurrency tests. It sets up a database with simulated lag to 20 | increase race conditions likelyhood, thus requires a running docker daemon.""" 21 | 22 | postgres_lag_for_background_worker = True 23 | 24 | def test_schedules(self): 25 | # We create COUNT workers with different labels 26 | for i in range(COUNT): 27 | self.start_worker_in_background( 28 | queue="schedules", 29 | label=f"w-{i}", 30 | verbosity=3, 31 | once=True, 32 | until_done=False, 33 | ) 34 | self.workers_get_stdout() 35 | 36 | # Ensure they were all created 37 | self.assertEqual(WorkerStatus.objects.count(), COUNT) 38 | 39 | # The schedule should have run just once and thus the task only queued once despite run_on_creation 40 | self.assertEqual(TaskExec.objects.count(), 1) 41 | 42 | def test_tasks(self): 43 | # Create a task 44 | create_user.queue() 45 | 46 | self.assertEqual(User.objects.count(), 0) 47 | 48 | # We create COUNT workers with different labels 49 | for i in range(COUNT): 50 | self.start_worker_in_background( 51 | queue="tasks", label=f"w-{i}", verbosity=3, once=True, until_done=False 52 | ) 53 | self.workers_get_stdout() 54 | 55 | # Ensure they were all created 56 | self.assertEqual(WorkerStatus.objects.count(), COUNT) 57 | 58 | # The task shouldn't have run concurrently and thus have run only once 59 | self.assertEqual(User.objects.count(), 1) 60 | 61 | def test_task_processing_state(self): 62 | t = sleep_task.queue(duration=10) 63 | 64 | # Check that the task correctly queued 65 | t.refresh_from_db() 66 | self.assertEqual(t.state, TaskExec.States.QUEUED) 67 | 68 | # Start the task in a background process 69 | self.start_worker_in_background(queue="tasks") 70 | 71 | # Check that it is now processing 72 | time.sleep(5) 73 | t.refresh_from_db() 74 | self.assertEqual(t.state, TaskExec.States.PROCESSING) 75 | 76 | # Wait for the background process to finish 77 | self.workers_get_stdout() 78 | 79 | # Check that it correctly succeeds 80 | t.refresh_from_db() 81 | self.assertEqual(t.state, TaskExec.States.SUCCEEDED) 82 | 83 | @unittest.skipIf( 84 | os.name == "nt", "didn't find a way to gracefully stop subprocess on windows" 85 | ) 86 | def test_task_graceful_stop(self): 87 | """Ensure that on graceful stop, running tasks status is set to interrupted and a replacement task is created""" 88 | 89 | t = sleep_task.queue(duration=10) 90 | 91 | # Check that the task correctly queued 92 | t.refresh_from_db() 93 | self.assertEqual(t.state, TaskExec.States.QUEUED) 94 | 95 | # Start the task in a background process 96 | self.start_worker_in_background(queue="tasks") 97 | 98 | # Check that it is now processing 99 | time.sleep(5) 100 | t.refresh_from_db() 101 | self.assertEqual(t.state, TaskExec.States.PROCESSING) 102 | 103 | # Gracefully stop the background process 104 | self.workers_gracefully_stop() 105 | 106 | # Wait for the background process to finish 107 | self.processes[0].wait(timeout=5) 108 | 109 | # Check that the state is correctly set to interrupted and that a replacing task was added 110 | t.refresh_from_db() 111 | self.assertEqual(t.state, TaskExec.States.INTERRUPTED) 112 | self.assertEqual(t.replaced_by.state, TaskExec.States.SLEEPING) 113 | -------------------------------------------------------------------------------- /django_toosimple_q/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.6 on 2019-11-01 12:50 2 | 3 | import django.db.models.deletion 4 | import django.utils.timezone 5 | import picklefield.fields 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | initial = True 11 | 12 | dependencies = [] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name="Task", 17 | fields=[ 18 | ( 19 | "id", 20 | models.AutoField( 21 | auto_created=True, 22 | primary_key=True, 23 | serialize=False, 24 | verbose_name="ID", 25 | ), 26 | ), 27 | ("function", models.CharField(max_length=1024)), 28 | ( 29 | "args", 30 | picklefield.fields.PickledObjectField( 31 | blank=True, default=list, editable=False 32 | ), 33 | ), 34 | ( 35 | "kwargs", 36 | picklefield.fields.PickledObjectField( 37 | blank=True, default=dict, editable=False 38 | ), 39 | ), 40 | ("priority", models.IntegerField(default=0)), 41 | ("created", models.DateTimeField(default=django.utils.timezone.now)), 42 | ("started", models.DateTimeField(blank=True, null=True)), 43 | ("finished", models.DateTimeField(blank=True, null=True)), 44 | ( 45 | "state", 46 | models.CharField( 47 | choices=[ 48 | ("QUEUED", "QUEUED"), 49 | ("PROCESSING", "PROCESSING"), 50 | ("FAILED", "FAILED"), 51 | ("SUCCEEDED", "SUCCEEDED"), 52 | ("INVALID", "INVALID"), 53 | ], 54 | default="QUEUED", 55 | max_length=32, 56 | ), 57 | ), # NOQA 58 | ( 59 | "result", 60 | picklefield.fields.PickledObjectField( 61 | blank=True, editable=False, null=True 62 | ), 63 | ), 64 | ("stdout", models.TextField(blank=True, default="")), 65 | ("stderr", models.TextField(blank=True, default="")), 66 | ], 67 | ), 68 | migrations.CreateModel( 69 | name="Schedule", 70 | fields=[ 71 | ( 72 | "id", 73 | models.AutoField( 74 | auto_created=True, 75 | primary_key=True, 76 | serialize=False, 77 | verbose_name="ID", 78 | ), 79 | ), 80 | ("name", models.CharField(max_length=1024)), 81 | ("function", models.CharField(max_length=1024)), 82 | ( 83 | "args", 84 | picklefield.fields.PickledObjectField( 85 | blank=True, default=list, editable=False 86 | ), 87 | ), 88 | ( 89 | "kwargs", 90 | picklefield.fields.PickledObjectField( 91 | blank=True, default=dict, editable=False 92 | ), 93 | ), 94 | ( 95 | "last_check", 96 | models.DateTimeField(default=django.utils.timezone.now, null=True), 97 | ), 98 | ("catch_up", models.BooleanField(default=False)), 99 | ("cron", models.CharField(max_length=1024)), 100 | ( 101 | "last_run", 102 | models.ForeignKey( 103 | null=True, 104 | on_delete=django.db.models.deletion.SET_NULL, 105 | to="toosimpleq.Task", 106 | ), 107 | ), 108 | ], 109 | ), 110 | ] 111 | -------------------------------------------------------------------------------- /django_toosimple_q/contrib/mail/tests.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | from django.core import mail, management 4 | from django.core.mail import send_mail, send_mass_mail 5 | from django.test.utils import override_settings 6 | 7 | from ...models import TaskExec 8 | from ...tests.base import TooSimpleQRegularTestCase 9 | from . import tasks as mail_tasks 10 | 11 | 12 | class TestMail(TooSimpleQRegularTestCase): 13 | def setUp(self): 14 | super().setUp() 15 | # Reload the tasks modules to repopulate the registries (emulates auto-discovery) 16 | importlib.reload(mail_tasks) 17 | 18 | @override_settings( 19 | EMAIL_BACKEND="django_toosimple_q.contrib.mail.backend.QueueBackend", 20 | TOOSIMPLEQ_EMAIL_BACKEND="django.core.mail.backends.locmem.EmailBackend", 21 | ) 22 | def test_queue_mail(self): 23 | self.assertQueue(0) 24 | 25 | send_mail( 26 | "Subject here", 27 | "Here is the message.", 28 | "from@example.com", 29 | ["to@example.com"], 30 | ) 31 | 32 | self.assertQueue(1, state=TaskExec.States.QUEUED) 33 | self.assertQueue(1) 34 | self.assertEqual(len(mail.outbox), 0) 35 | 36 | management.call_command("worker", "--until_done") 37 | 38 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 39 | self.assertQueue(1) 40 | self.assertEqual(len(mail.outbox), 1) 41 | 42 | @override_settings( 43 | EMAIL_BACKEND="django_toosimple_q.contrib.mail.backend.QueueBackend", 44 | TOOSIMPLEQ_EMAIL_BACKEND="django.core.mail.backends.locmem.EmailBackend", 45 | ) 46 | def test_queue_mail_two(self): 47 | self.assertQueue(0) 48 | 49 | send_mail( 50 | "Subject here", 51 | "Here is the message.", 52 | "from@example.com", 53 | ["to@example.com"], 54 | ) 55 | send_mail( 56 | "Other subject here", 57 | "Here is the message.", 58 | "from@example.com", 59 | ["to@example.com"], 60 | ) 61 | 62 | self.assertQueue(2, state=TaskExec.States.QUEUED) 63 | self.assertQueue(2) 64 | self.assertEqual(len(mail.outbox), 0) 65 | 66 | management.call_command("worker", "--until_done") 67 | 68 | self.assertQueue(2, state=TaskExec.States.SUCCEEDED) 69 | self.assertQueue(2) 70 | self.assertEqual(len(mail.outbox), 2) 71 | 72 | @override_settings( 73 | EMAIL_BACKEND="django_toosimple_q.contrib.mail.backend.QueueBackend", 74 | TOOSIMPLEQ_EMAIL_BACKEND="django.core.mail.backends.locmem.EmailBackend", 75 | ) 76 | def test_queue_mail_duplicate(self): 77 | self.assertQueue(0) 78 | 79 | send_mail( 80 | "Subject here", 81 | "Here is the message.", 82 | "from@example.com", 83 | ["to@example.com"], 84 | ) 85 | send_mail( 86 | "Subject here", 87 | "Here is the message.", 88 | "from@example.com", 89 | ["to@example.com"], 90 | ) 91 | 92 | self.assertQueue(1, state=TaskExec.States.QUEUED) 93 | self.assertQueue(1) 94 | self.assertEqual(len(mail.outbox), 0) 95 | 96 | management.call_command("worker", "--until_done") 97 | 98 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 99 | self.assertQueue(1) 100 | self.assertEqual(len(mail.outbox), 1) 101 | 102 | @override_settings( 103 | EMAIL_BACKEND="django_toosimple_q.contrib.mail.backend.QueueBackend", 104 | TOOSIMPLEQ_EMAIL_BACKEND="django.core.mail.backends.locmem.EmailBackend", 105 | ) 106 | def test_queue_mass_mail(self): 107 | self.assertQueue(0) 108 | 109 | send_mass_mail( 110 | [ 111 | ("Subject A", "Message.", "from@example.com", ["to@example.com"]), 112 | ("Subject B", "Message.", "from@example.com", ["to@example.com"]), 113 | ("Subject C", "Message.", "from@example.com", ["to@example.com"]), 114 | ] 115 | ) 116 | 117 | self.assertQueue(1, state=TaskExec.States.QUEUED) 118 | self.assertQueue(1) 119 | self.assertEqual(len(mail.outbox), 0) 120 | 121 | management.call_command("worker", "--until_done") 122 | 123 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 124 | self.assertQueue(1) 125 | self.assertEqual(len(mail.outbox), 3) 126 | 127 | @override_settings( 128 | EMAIL_BACKEND="django_toosimple_q.contrib.mail.backend.QueueBackend", 129 | TOOSIMPLEQ_EMAIL_BACKEND="failing_backend", 130 | ) 131 | def test_queue_mail_failing_backend(self): 132 | self.assertQueue(0) 133 | 134 | send_mail( 135 | "Subject here", 136 | "Here is the message.", 137 | "from@example.com", 138 | ["to@example.com"], 139 | ) 140 | 141 | self.assertQueue(1, state=TaskExec.States.QUEUED) 142 | self.assertQueue(1) 143 | self.assertEqual(len(mail.outbox), 0) 144 | 145 | management.call_command("worker", "--until_done") 146 | 147 | self.assertQueue(1, state=TaskExec.States.FAILED, replaced=True) 148 | self.assertQueue(1, state=TaskExec.States.SLEEPING) 149 | self.assertQueue(2) 150 | self.assertEqual(len(mail.outbox), 0) 151 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_worker.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import time 3 | 4 | from django.core import management 5 | from freezegun import freeze_time 6 | 7 | from django_toosimple_q.decorators import register_task 8 | from django_toosimple_q.models import TaskExec, WorkerStatus 9 | 10 | from ..models import WorkerStatus 11 | from .base import TooSimpleQBackgroundTestCase, TooSimpleQRegularTestCase 12 | from .concurrency.tasks import sleep_task 13 | 14 | 15 | class TestWorker(TooSimpleQRegularTestCase): 16 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 17 | def test_worker(self, frozen_datetime): 18 | """Checking that worker status are correctly created""" 19 | 20 | # Syntaxic sugar 21 | S = WorkerStatus.States 22 | 23 | # Workers status is correctly created 24 | management.call_command("worker", "--once", "--label", "w1") 25 | ws = WorkerStatus.objects.all() 26 | self.assertEqual(ws.count(), 1) 27 | self.assertCountEqual([w.label for w in ws], ["w1"]) 28 | self.assertCountEqual([w.state for w in ws], [S.STOPPED]) 29 | 30 | # A second call doesn't change it 31 | ws = WorkerStatus.objects.all() 32 | self.assertEqual(ws.count(), 1) 33 | self.assertCountEqual([w.label for w in ws], ["w1"]) 34 | self.assertCountEqual([w.state for w in ws], [S.STOPPED]) 35 | 36 | # Another worker adds a new status 37 | management.call_command("worker", "--once", "--label", "w2") 38 | ws = WorkerStatus.objects.all() 39 | self.assertEqual(ws.count(), 2) 40 | self.assertCountEqual([w.label for w in ws], ["w1", "w2"]) 41 | self.assertCountEqual([w.state for w in ws], [S.STOPPED, S.STOPPED]) 42 | 43 | # Worker are properly populating attached to task 44 | @register_task(name="a") 45 | def a(): 46 | return True 47 | 48 | t = a.queue() 49 | self.assertEqual(t.worker, None) 50 | management.call_command("worker", "--once", "--label", "w2") 51 | t.refresh_from_db() 52 | self.assertEqual(t.worker.label, "w2") 53 | 54 | # TODO: test for worker timeout status 55 | # TODO: test for no label/pid clashes with multiple workers 56 | 57 | 58 | class TestWorkerExit(TooSimpleQBackgroundTestCase): 59 | """This tests all types of worker termination (kill, terminate, quit, crash) and their effect on tasks and output states""" 60 | 61 | def _start_worker_with_task(self, duration=10): 62 | """Helper to create a worker that picked up a 10s task""" 63 | 64 | # Add a task that takes some time 65 | sleep_task.queue(duration=duration) 66 | 67 | # Start a worker 68 | self.start_worker_in_background( 69 | queue="tasks", tick=1, until_done=False, verbosity=3, timeout=3 70 | ) 71 | 72 | # Wait for the task to be picked up by the worker 73 | self.wait_for_qs(TaskExec.objects.filter(state=TaskExec.States.PROCESSING)) 74 | 75 | # Keep id of the first task for further reference 76 | self.__first_task_pk = TaskExec.objects.first().pk 77 | 78 | # Let the worker actually start processing the task 79 | # FIXME: this should not be needed 80 | time.sleep(1) 81 | 82 | @property 83 | def workerstatus(self): 84 | """The workerstatus object corresponding to the worker""" 85 | return WorkerStatus.objects.first() 86 | 87 | @property 88 | def taskexec(self): 89 | """The long running task that should have been picked up by the worker""" 90 | return TaskExec.objects.get(pk=self.__first_task_pk) 91 | 92 | @property 93 | def process(self): 94 | """The subprocess""" 95 | return self.processes[0] 96 | 97 | def test_kill(self): 98 | self._start_worker_with_task() 99 | 100 | # Hard kill, the worker can't cleanly quit 101 | self.process.kill() 102 | 103 | # -9 is exit code for SIGKILL 104 | exit_code = self.process.wait(timeout=5) 105 | self.assertEqual(exit_code, -9) 106 | 107 | # Initially the worker still looks online 108 | self.assertEqual(self.workerstatus.state, WorkerStatus.States.ONLINE) 109 | 110 | # The task also will look like it's still processing 111 | self.assertEqual(self.taskexec.state, TaskExec.States.PROCESSING) 112 | 113 | # After a while though, it should timeout 114 | time.sleep(5) 115 | self.assertEqual(self.workerstatus.state, WorkerStatus.States.TIMEDOUT) 116 | self.assertIsNone(self.workerstatus.exit_log) 117 | 118 | # Same for the task 119 | # FIXME: we have no timeout status for tasks, so it stays processing indefinitely. We should 120 | # add a timeout status and assign it 121 | self.assertEqual(self.taskexec.state, TaskExec.States.PROCESSING) 122 | 123 | def test_terminate(self): 124 | self._start_worker_with_task() 125 | 126 | # Soft kill, the worker should interrupt the task and quit as soon as possible 127 | self.process.send_signal(signal.SIGTERM) 128 | 129 | # We should have our custom exit code 130 | exit_code = self.process.wait(timeout=5) 131 | self.assertEqual(exit_code, WorkerStatus.ExitCodes.TERMINATED.value) 132 | 133 | # The worker should correctly set its state 134 | self.assertEqual(self.workerstatus.state, WorkerStatus.States.TERMINATED) 135 | self.assertIn("KeyboardInterrupt", self.workerstatus.exit_log) 136 | 137 | # The task should be noted as interrupted, and replaced by another task 138 | self.assertEqual(self.taskexec.state, TaskExec.States.INTERRUPTED) 139 | self.assertIsNone(self.taskexec.result) 140 | self.assertIsNotNone(self.taskexec.replaced_by) 141 | self.assertEqual(self.taskexec.replaced_by.state, TaskExec.States.SLEEPING) 142 | 143 | def test_worker_crash(self): 144 | self._start_worker_with_task() 145 | 146 | # Simluate a crash by deleting the workerexec instance 147 | self.process.send_signal(signal.SIGUSR1) 148 | 149 | # The exit code should be 0, it's a graceful quit 150 | exit_code = self.process.wait(timeout=15) 151 | self.assertEqual(exit_code, WorkerStatus.ExitCodes.CRASHED.value) 152 | 153 | # The worker should correctly set its state 154 | self.assertEqual(self.workerstatus.state, WorkerStatus.States.CRASHED) 155 | self.assertIn("FakeException", self.workerstatus.exit_log) 156 | 157 | # The failure is not linked to the task 158 | 159 | def test_quit(self): 160 | self._start_worker_with_task() 161 | 162 | # Regular quit, the process should let the task finish, then exit 163 | self.process.send_signal(signal.SIGINT) 164 | 165 | # The exit code should be 0, it's a graceful quit 166 | exit_code = self.process.wait(timeout=15) 167 | self.assertEqual(exit_code, WorkerStatus.ExitCodes.STOPPED.value) 168 | 169 | # The worker should correctly set its state 170 | self.assertEqual(self.workerstatus.state, WorkerStatus.States.STOPPED) 171 | self.assertEqual("", self.workerstatus.exit_log) 172 | 173 | # The task should have been left enough time to finish 174 | self.assertEqual(self.taskexec.state, TaskExec.States.SUCCEEDED) 175 | self.assertTrue(self.taskexec.result) 176 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.admin.models import CHANGE, LogEntry 3 | from django.contrib.auth.models import Permission, User 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.core import management 6 | from django.test import RequestFactory 7 | 8 | from django_toosimple_q.decorators import register_task, schedule_task 9 | from django_toosimple_q.models import ScheduleExec, TaskExec 10 | 11 | from .base import TooSimpleQRegularTestCase 12 | 13 | 14 | class TestAdmin(TooSimpleQRegularTestCase): 15 | def test_task_admin(self): 16 | """Check if task admin pages work""" 17 | 18 | @register_task(name="a") 19 | def a(): 20 | return 2 21 | 22 | task_exec = a.queue() 23 | 24 | management.call_command("worker", "--until_done") 25 | 26 | response = self.client.get("/admin/toosimpleq/taskexec/") 27 | self.assertEqual(response.status_code, 200) 28 | 29 | response = self.client.get(f"/admin/toosimpleq/taskexec/{task_exec.pk}/change/") 30 | self.assertEqual(response.status_code, 200) 31 | 32 | def test_schedule_admin(self): 33 | """Check if schedule admin pages work""" 34 | 35 | @schedule_task(cron="* * * * *") 36 | @register_task(name="a") 37 | def a(): 38 | return 2 39 | 40 | management.call_command("worker", "--until_done") 41 | 42 | response = self.client.get("/admin/toosimpleq/scheduleexec/") 43 | self.assertEqual(response.status_code, 200) 44 | 45 | scheduleexec = ScheduleExec.objects.first() 46 | response = self.client.get( 47 | f"/admin/toosimpleq/scheduleexec/{scheduleexec.pk}/change/" 48 | ) 49 | self.assertEqual(response.status_code, 200) 50 | 51 | def test_manual_schedule_admin(self): 52 | """Check that manual schedule admin action work""" 53 | 54 | @schedule_task(cron="manual") 55 | @register_task(name="a") 56 | def a(): 57 | return 2 58 | 59 | self.assertSchedule("a", None) 60 | management.call_command("worker", "--until_done") 61 | self.assertQueue(0) 62 | 63 | data = { 64 | "action": "action_force_run", 65 | "_selected_action": ScheduleExec.objects.get(name="a").pk, 66 | } 67 | response = self.client.post( 68 | "/admin/toosimpleq/scheduleexec/", data, follow=True 69 | ) 70 | self.assertEqual(response.status_code, 200) 71 | 72 | self.assertQueue(1, state=TaskExec.States.QUEUED) 73 | 74 | management.call_command("worker", "--until_done") 75 | 76 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 77 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 78 | 79 | def test_schedule_admin_force_action(self): 80 | """Check if he force execute schedule action works""" 81 | 82 | @schedule_task(cron="13 0 1 1 *") 83 | @register_task(name="a") 84 | def a(): 85 | return 2 86 | 87 | self.assertSchedule("a", None) 88 | self.assertQueue(0, state=TaskExec.States.SUCCEEDED) 89 | self.assertQueue(0, state=TaskExec.States.QUEUED) 90 | 91 | management.call_command("worker", "--until_done") 92 | 93 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 94 | self.assertQueue(0, state=TaskExec.States.SUCCEEDED) 95 | self.assertQueue(0, state=TaskExec.States.QUEUED) 96 | 97 | data = { 98 | "action": "action_force_run", 99 | "_selected_action": ScheduleExec.objects.get(name="a").pk, 100 | } 101 | response = self.client.post( 102 | "/admin/toosimpleq/scheduleexec/", data, follow=True 103 | ) 104 | self.assertEqual(response.status_code, 200) 105 | 106 | management.call_command("worker", "--until_done") 107 | 108 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 109 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 110 | self.assertQueue(0, state=TaskExec.States.QUEUED) 111 | 112 | # ensure the action gets logged 113 | self.assertEqual( 114 | LogEntry.objects.filter( 115 | content_type_id=ContentType.objects.get_for_model(ScheduleExec).pk, 116 | action_flag=CHANGE, 117 | ).count(), 118 | 1, 119 | ) 120 | 121 | def test_task_admin_requeue_action(self): 122 | """Check if the requeue action works""" 123 | 124 | @register_task(name="a") 125 | def a(): 126 | return 2 127 | 128 | task_exec = a.queue() 129 | 130 | self.assertQueue(0, state=TaskExec.States.SUCCEEDED) 131 | self.assertQueue(1, state=TaskExec.States.QUEUED) 132 | 133 | management.call_command("worker", "--until_done") 134 | 135 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 136 | self.assertQueue(0, state=TaskExec.States.QUEUED) 137 | 138 | data = { 139 | "action": "action_requeue", 140 | "_selected_action": task_exec.pk, 141 | } 142 | response = self.client.post("/admin/toosimpleq/taskexec/", data, follow=True) 143 | self.assertEqual(response.status_code, 200) 144 | 145 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 146 | self.assertQueue(1, state=TaskExec.States.QUEUED) 147 | 148 | management.call_command("worker", "--until_done") 149 | 150 | self.assertQueue(2, state=TaskExec.States.SUCCEEDED) 151 | self.assertQueue(0, state=TaskExec.States.QUEUED) 152 | 153 | # ensure the action gets logged 154 | self.assertEqual( 155 | LogEntry.objects.filter( 156 | content_type_id=ContentType.objects.get_for_model(TaskExec).pk, 157 | action_flag=CHANGE, 158 | ).count(), 159 | 1, 160 | ) 161 | 162 | def test_task_admin_result_preview(self): 163 | """Check the the task results correctly displays, including if long""" 164 | 165 | @register_task() 166 | def a(length): 167 | return "o" * length 168 | 169 | # a short result appears as is 170 | a.queue(length=10) 171 | management.call_command("worker", "--until_done") 172 | response = self.client.get("/admin/toosimpleq/taskexec/", follow=True) 173 | self.assertContains(response, "o" * 10) 174 | 175 | # a long results gets trimmed 176 | a.queue(length=300) 177 | management.call_command("worker", "--until_done") 178 | response = self.client.get("/admin/toosimpleq/taskexec/", follow=True) 179 | self.assertContains(response, "o" * 254 + "…") 180 | 181 | def test_admin_actions_permissions(self): 182 | """Check that admin actions are only available with the correct permissions""" 183 | 184 | perms = Permission.objects.filter( 185 | codename__in=["force_run_scheduleexec", "requeue_taskexec"] 186 | ) 187 | 188 | request_with = RequestFactory().get("/some-url/") 189 | request_with.user = User.objects.create(username="mike") 190 | request_with.user.user_permissions.set(perms) 191 | 192 | request_without = RequestFactory().get("/some-url/") 193 | request_without.user = User.objects.create(username="peter") 194 | 195 | # prefer admin.site.get_model_admin(TaskExec) once we drop support for 4.2 196 | task_model_admin = admin.site._registry[TaskExec] 197 | schedule_model_admin = admin.site._registry[ScheduleExec] 198 | 199 | self.assertCountEqual( 200 | task_model_admin.get_actions(request_with).keys(), ["action_requeue"] 201 | ) 202 | self.assertCountEqual( 203 | schedule_model_admin.get_actions(request_with).keys(), ["action_force_run"] 204 | ) 205 | self.assertCountEqual(task_model_admin.get_actions(request_without).keys(), []) 206 | self.assertCountEqual( 207 | schedule_model_admin.get_actions(request_without).keys(), [] 208 | ) 209 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | import signal 3 | import subprocess 4 | import time 5 | from typing import List 6 | 7 | from django.contrib.auth.models import User 8 | from django.core import mail 9 | from django.db.models import Count 10 | from django.test import Client, TestCase, TransactionTestCase 11 | 12 | from django_toosimple_q.models import ScheduleExec, TaskExec 13 | from django_toosimple_q.registry import schedules_registry, tasks_registry 14 | 15 | from ..logging import logger 16 | 17 | 18 | class TooSimpleQTestCaseMixin: 19 | """ 20 | Base TestCase for TooSimpleQ. 21 | 22 | - Clears the schedules and task registries (reverting the autodiscovery) 23 | - Creates a superuser 24 | - Clears the mailbox 25 | - Adds assertQueue and assertTask helpers 26 | 27 | Use this if you want to keep autoloaded task (from contrib.mail) from polluting the tests. 28 | """ 29 | 30 | def setUp(self): 31 | # Clean the registry 32 | schedules_registry.clear() 33 | tasks_registry.clear() 34 | 35 | # Create a superuser 36 | user = User.objects.create_superuser("admin", "test@example.com", "pass") 37 | self.client = Client() 38 | self.client.force_login(user) 39 | 40 | # Clear the mailbox 41 | mail.outbox.clear() 42 | 43 | def assertQueue( 44 | self, expected_count, task_name=None, state=None, replaced=None, due=None 45 | ): 46 | tasks = TaskExec.objects.all() 47 | if task_name: 48 | tasks = tasks.filter(task_name=task_name) 49 | if state: 50 | tasks = tasks.filter(state=state) 51 | if replaced is not None: 52 | tasks = tasks.filter(replaced_by__isnull=not replaced) 53 | if due is not None: 54 | tasks = tasks.filter(due=due) 55 | actual_count = tasks.count() 56 | if actual_count != expected_count: 57 | vals = ( 58 | TaskExec.objects.values("task_name", "state") 59 | .annotate(count=Count("*")) 60 | .order_by("task_name", "state") 61 | ) 62 | debug = "\n".join( 63 | f"{v['task_name']}/{v['state']} : {v['count']}" for v in vals 64 | ) 65 | raise AssertionError( 66 | f"Expected {expected_count} tasks, got {actual_count} tasks.\n{debug}" 67 | ) 68 | 69 | def assertResults(self, expected=[], task_name=None): 70 | tasks_execs = TaskExec.objects.order_by("created", "result") 71 | if task_name: 72 | tasks_execs = tasks_execs.filter(task_name=task_name) 73 | results = list(tasks_execs.values_list("result", flat=True)) 74 | 75 | self.assertEqual(results, expected) 76 | 77 | def assertTask(self, task, expected_state): 78 | actual_state = TaskExec.objects.get(pk=task.pk).state 79 | if actual_state != expected_state: 80 | raise AssertionError( 81 | f"Expected {expected_state}, got {actual_state} [{task}]" 82 | ) 83 | 84 | def assertSchedule(self, name, expected_state): 85 | try: 86 | state = ScheduleExec.objects.get(name=name).state 87 | except ScheduleExec.DoesNotExist: 88 | state = None 89 | 90 | if state != expected_state: 91 | raise AssertionError(f"Expected {expected_state}, got {state} [{name}]") 92 | 93 | 94 | class TooSimpleQRegularTestCase(TooSimpleQTestCaseMixin, TestCase): 95 | """ 96 | Base TestCase for TooSimpleQ. 97 | 98 | See TooSimpleQTestCaseMixin 99 | """ 100 | 101 | 102 | class TooSimpleQBackgroundTestCase(TransactionTestCase): 103 | """ 104 | Base TransactionTestCase for TooSimpleQ. 105 | 106 | - Ensures the database is accessible from background workers (transactiontestcase do no wrap tests in transactions) 107 | - Adds some helpers methods to manage the workers 108 | 109 | See TooSimpleQTestCaseMixin. 110 | """ 111 | 112 | postgres_lag_for_background_worker = False 113 | 114 | def setUp(self): 115 | super().setUp() 116 | self.processes: List[subprocess.Popen] = [] 117 | 118 | def tearDown(self): 119 | super().tearDown() 120 | 121 | self.workers_gracefully_stop() 122 | self.workers_kill() 123 | 124 | def start_worker_in_background( 125 | self, 126 | queue=None, 127 | tick=None, 128 | until_done=True, 129 | once=False, 130 | skip_checks=True, 131 | verbosity=None, 132 | label=None, 133 | timeout=None, 134 | ): 135 | """Starts N workers in the background on the specified queue.""" 136 | 137 | if self.postgres_lag_for_background_worker: 138 | settings = "django_toosimple_q.tests.settings_bg_lag" 139 | else: 140 | settings = "django_toosimple_q.tests.settings_bg" 141 | 142 | command = ["python", "manage.py", "worker"] 143 | 144 | if label: 145 | command.extend(["--label", str(label)]) 146 | if tick: 147 | command.extend(["--tick", str(tick)]) 148 | if queue: 149 | command.extend(["--queue", str(queue)]) 150 | if until_done: 151 | command.extend(["--until_done"]) 152 | if once: 153 | command.extend(["--once"]) 154 | if skip_checks: 155 | command.extend(["--skip-checks"]) 156 | if verbosity: 157 | command.extend(["--verbosity", str(verbosity)]) 158 | if timeout: 159 | command.extend(["--timeout", str(timeout)]) 160 | 161 | logger.debug(f"Starting workers: {' '.join(command)}") 162 | self.processes.append( 163 | subprocess.Popen( 164 | command, 165 | encoding="utf-8", 166 | env={**os.environ, "DJANGO_SETTINGS_MODULE": settings}, 167 | stdout=subprocess.PIPE, 168 | stderr=subprocess.STDOUT, 169 | ) 170 | ) 171 | 172 | def wait_for_qs(self, queryset, exists=True, timeout=15): 173 | """Waits until the queryset exists (or does not exist)""" 174 | start_time = time.time() 175 | while queryset.exists() == (not exists): 176 | if (time.time() - start_time) > timeout: 177 | raise AssertionError( 178 | f"Expected queryset was not present after {timeout} seconds" 179 | if exists 180 | else f"Unexpected queryset was still present after {timeout} seconds" 181 | ) 182 | 183 | def wait_for_tasks(self, timeout=15): 184 | """Waits untill all tasks are marked as done in the database""" 185 | return self.wait_for_qs( 186 | TaskExec.objects.filter(state__in=TaskExec.States.todo()), 187 | exists=False, 188 | timeout=timeout, 189 | ) 190 | 191 | def workers_get_stdout(self): 192 | """Stops the workers if needed and returns the stdout of the last worker, or raises an exception on error. 193 | 194 | Can be used to check output or to assert success""" 195 | 196 | outputs = [] 197 | for process in self.processes: 198 | try: 199 | stdout, stderr = process.communicate(timeout=15) 200 | except subprocess.TimeoutExpired: 201 | process.kill() 202 | stdout, stderr = process.communicate() 203 | outputs.append((process.returncode, stdout, stderr)) 204 | 205 | # Outputs that errored 206 | error_outputs = [o for o in outputs if o[0] != 0] 207 | 208 | # Last output is last one 209 | last_output = error_outputs[-1] if error_outputs else outputs[-1] 210 | 211 | last_retcod, last_stdout, last_stderr = last_output 212 | 213 | # Raise exception if error 214 | if last_retcod != 0: 215 | all_retcodes = ", ".join([f"{r[0]}" for r in outputs]) 216 | logger.warn(f"Some workers errored. All return codes: {all_retcodes}\n") 217 | logger.warn(f"Last error retcod:\n{last_retcod}\n") 218 | logger.warn(f"Last error stdout:\n{last_stdout}\n") 219 | logger.warn(f"Last error stderr:\n{last_stderr}") 220 | raise AssertionError(f"Some workers errored.") 221 | 222 | return last_stdout 223 | 224 | def workers_gracefully_stop(self): 225 | """Gracefully stops all workers (note that you must still wait for them to finish using wait_for_success).""" 226 | 227 | for process in self.processes: 228 | if os.name == "nt": 229 | # FIXME: This is buggy. When running, test passes, but then the test stops, and further 230 | # tests are not run. Not sure if sending CTRL_C to the child process also affects the current 231 | # process for some reason ? 232 | process.send_signal(signal.CTRL_C_EVENT) 233 | else: 234 | process.send_signal(signal.SIGTERM) 235 | 236 | def workers_kill(self): 237 | """Forces kill all processes (use this for cleanup)""" 238 | 239 | open_processes = [p for p in self.processes if p.poll() is None] 240 | if not open_processes: 241 | return 242 | 243 | logger.warn(f"Killing {len(open_processes)} dangling worker processes...") 244 | for process in open_processes: 245 | process.kill() 246 | -------------------------------------------------------------------------------- /django_toosimple_q/models.py: -------------------------------------------------------------------------------- 1 | import io 2 | import traceback 3 | from contextlib import redirect_stderr, redirect_stdout 4 | from datetime import datetime, timedelta 5 | from typing import List 6 | 7 | from croniter import croniter, croniter_range 8 | from django.db import models 9 | from django.template.defaultfilters import truncatechars 10 | from django.utils import timezone 11 | from django.utils.functional import cached_property 12 | from django.utils.timezone import now 13 | from django.utils.translation import gettext_lazy as _ 14 | from picklefield.fields import PickledObjectField 15 | 16 | from .logging import logger 17 | from .registry import schedules_registry, tasks_registry 18 | 19 | 20 | class TaskExec(models.Model): 21 | """TaskExecution represent a specific planned or past call of a task, including inputs (arguments) and outputs. 22 | 23 | This is a model, whose instanced are typically created using `mycallable.queue()` or from schedules. 24 | """ 25 | 26 | class Meta: 27 | verbose_name = "Task Execution" 28 | permissions = [ 29 | ("requeue_taskexec", "Can requeue tasks"), 30 | ] 31 | 32 | class States(models.TextChoices): 33 | SLEEPING = "SLEEPING", _("Sleeping") 34 | QUEUED = "QUEUED", _("Queued") 35 | PROCESSING = "PROCESSING", _("Processing") 36 | SUCCEEDED = "SUCCEEDED", _("Succeeded") 37 | INTERRUPTED = "INTERRUPTED", _("Interrupted") 38 | FAILED = "FAILED", _("Failed") 39 | INVALID = "INVALID", _("Invalid") 40 | 41 | @classmethod 42 | def icon(cls, state): 43 | if state == cls.SLEEPING: 44 | return "💤" 45 | elif state == cls.QUEUED: 46 | return "⌚" 47 | elif state == cls.PROCESSING: 48 | return "🚧" 49 | elif state == cls.SUCCEEDED: 50 | return "✔️" 51 | elif state == cls.FAILED: 52 | return "❌" 53 | elif state == cls.INTERRUPTED: 54 | return "🛑" 55 | elif state == cls.INVALID: 56 | return "⚠️" 57 | raise NotImplementedError(f"Unknown state: {state}") 58 | 59 | @classmethod 60 | def todo(cls) -> List[str]: 61 | """A list of values that are not done (opposite of done)""" 62 | return [ 63 | cls.SLEEPING.value, 64 | cls.QUEUED.value, 65 | cls.PROCESSING.value, 66 | ] 67 | 68 | @classmethod 69 | def done(cls) -> List[str]: 70 | """A list of values that are done (opposite of todo)""" 71 | return [v for v in cls.values if v not in cls.todo()] 72 | 73 | id = models.BigAutoField(primary_key=True) 74 | task_name = models.CharField(max_length=1024) 75 | args = PickledObjectField(blank=True, default=list) 76 | kwargs = PickledObjectField(blank=True, default=dict) 77 | retries = models.IntegerField( 78 | default=0, help_text="retries left, -1 means infinite" 79 | ) 80 | retry_delay = models.IntegerField( 81 | default=0, 82 | help_text="Delay before next retry in seconds. Will double after each failure.", 83 | ) 84 | 85 | due = models.DateTimeField(default=now) 86 | created = models.DateTimeField(default=now) 87 | started = models.DateTimeField(blank=True, null=True) 88 | finished = models.DateTimeField(blank=True, null=True) 89 | state = models.CharField( 90 | max_length=32, choices=States.choices, default=States.QUEUED 91 | ) 92 | result = PickledObjectField(blank=True, null=True) 93 | result_preview = models.CharField( 94 | max_length=255, blank=True, null=True, editable=False 95 | ) 96 | error = models.TextField(blank=True, null=True) 97 | replaced_by = models.ForeignKey( 98 | "self", null=True, blank=True, on_delete=models.SET_NULL 99 | ) 100 | worker = models.ForeignKey( 101 | "WorkerStatus", null=True, blank=True, on_delete=models.SET_NULL 102 | ) 103 | 104 | stdout = models.TextField(blank=True, default="") 105 | stderr = models.TextField(blank=True, default="") 106 | 107 | def __str__(self): 108 | return f"Task '{self.task_name}' {self.icon} [{self.id}]" 109 | 110 | @property 111 | def task(self): 112 | """The corresponding task instance, or None if it's not in the registry""" 113 | try: 114 | return tasks_registry[self.task_name] 115 | except KeyError: 116 | return None 117 | 118 | @property 119 | def icon(self): 120 | return TaskExec.States.icon(self.state) 121 | 122 | def execute(self): 123 | logger.info(f"{self} started") 124 | try: 125 | # Get the task from the registry 126 | task = tasks_registry[self.task_name] 127 | 128 | # Run the task 129 | stdout, stderr = io.StringIO(), io.StringIO() 130 | with redirect_stderr(stderr), redirect_stdout(stdout): 131 | self.result = task.callable(*self.args, **self.kwargs) 132 | self.result_preview = truncatechars(str(self.result), 255) 133 | logger.info(f"{self} succeeded") 134 | self.state = TaskExec.States.SUCCEEDED 135 | except Exception: 136 | logger.warning(f"{self} failed !") 137 | self.state = TaskExec.States.FAILED 138 | self.error = traceback.format_exc() 139 | if self.retries != 0: 140 | self.create_replacement(is_retry=True) 141 | finally: 142 | self.finished = now() 143 | self.stdout = stdout.getvalue() 144 | self.stderr = stderr.getvalue() 145 | self.save() 146 | 147 | def create_replacement(self, is_retry): 148 | logger.info(f"Creating a replacement task for {self}") 149 | 150 | if is_retry: 151 | # If it's a retry (failed task), we increment the retry count 152 | retries = self.retries - 1 if self.retries > 0 else -1 153 | delay = self.retry_delay * 2 154 | else: 155 | # If it's a replacement, we don't 156 | retries = self.retries 157 | delay = self.retry_delay 158 | 159 | replaced_by = TaskExec.objects.create( 160 | task_name=self.task_name, 161 | args=self.args, 162 | kwargs=self.kwargs, 163 | retries=retries, 164 | retry_delay=delay, 165 | state=TaskExec.States.SLEEPING, 166 | due=now() + timedelta(seconds=self.retry_delay), 167 | ) 168 | self.replaced_by = replaced_by 169 | self.save() 170 | 171 | 172 | class ScheduleExec(models.Model): 173 | class Meta: 174 | verbose_name = "Schedule Execution" 175 | permissions = [ 176 | ("force_run_scheduleexec", "Can force execution of schedules"), 177 | ] 178 | 179 | class States(models.TextChoices): 180 | ACTIVE = "ACTIVE", _("Active") 181 | INVALID = "INVALID", _("Invalid") 182 | 183 | @classmethod 184 | def icon(cls, state): 185 | if state == cls.ACTIVE: 186 | return "🟢" 187 | elif state == cls.INVALID: 188 | return "⚠️" 189 | raise NotImplementedError(f"Unknown state: {state}") 190 | 191 | id = models.BigAutoField(primary_key=True) 192 | name = models.CharField(max_length=1024, unique=True) 193 | last_due = models.DateTimeField(null=True, blank=True) 194 | last_task = models.ForeignKey( 195 | TaskExec, null=True, blank=True, on_delete=models.SET_NULL 196 | ) 197 | state = models.CharField( 198 | max_length=32, choices=States.choices, default=States.ACTIVE 199 | ) 200 | 201 | def __str__(self): 202 | return f"Schedule '{self.name}' {self.icon}" 203 | 204 | @property 205 | def schedule(self): 206 | """The corresponding schedule instance, or None if it's not in the registry""" 207 | try: 208 | return schedules_registry[self.name] 209 | except KeyError: 210 | return None 211 | 212 | @property 213 | def icon(self): 214 | return ScheduleExec.States.icon(self.state) 215 | 216 | @cached_property 217 | def past_dues(self): 218 | if self.schedule is None: 219 | # Deal with invalid schedule (e.g. deleted from the code but still in the DB) 220 | return [] 221 | 222 | if self.schedule.cron == "manual": 223 | # A manual schedule is never due 224 | return [] 225 | 226 | if self.last_due is None: 227 | # If the schedule has no last due date (probaby create with run_on_creation), we run it 228 | return [croniter(self.schedule.cron, now()).get_prev(datetime)] 229 | 230 | # Otherwise, we find all execution times since last check 231 | dues = list( 232 | croniter_range(self.last_due, now(), self.schedule.cron, exclude_ends=True) 233 | ) 234 | # We keep only the last one if catchup wasn't specified 235 | if not self.schedule.catch_up: 236 | return dues[-1:] 237 | 238 | return dues 239 | 240 | @cached_property 241 | def upcomming_due(self): 242 | if self.schedule.cron == "manual": 243 | # A manual schedule is never due 244 | return None 245 | 246 | return croniter(self.schedule.cron, timezone.now()).get_next(datetime) 247 | 248 | def execute(self): 249 | did_something = False 250 | 251 | if self.past_dues: 252 | logger.info(f"{self} is due ({len(self.past_dues)} occurences)") 253 | self.schedule.execute(self.past_dues) 254 | did_something = True 255 | self.last_due = self.past_dues[-1] 256 | 257 | self.state = ScheduleExec.States.ACTIVE 258 | self.save() 259 | 260 | return did_something 261 | 262 | 263 | class WorkerStatus(models.Model): 264 | """Represents the status of a worker. At each tick, the worker will update it's status. 265 | After a certain tim""" 266 | 267 | class Meta: 268 | verbose_name = "Worker Status" 269 | verbose_name_plural = "Workers Statuses" 270 | 271 | class ExitCodes(models.IntegerChoices): 272 | STOPPED = 0, _("Stopped") 273 | TERMINATED = 77, _("Terminated") 274 | CRASHED = 99, _("Crashed") 275 | 276 | class States(models.TextChoices): 277 | ONLINE = "ONLINE", _("Online") 278 | STOPPED = "STOPPED", _("Stopped") 279 | TERMINATED = "TERMINATED", _("Terminated") 280 | CRASHED = "CRASHED", _("Crashed") 281 | TIMEDOUT = "TIMEDOUT", _("Timedout") 282 | 283 | @classmethod 284 | def icon(cls, state): 285 | if state == cls.ONLINE: 286 | return "🟢" 287 | elif state == cls.STOPPED: 288 | return "⚪" 289 | elif state == cls.TERMINATED: 290 | return "🟧" 291 | elif state == cls.CRASHED: 292 | return "🟥" 293 | elif state == cls.TIMEDOUT: 294 | return "❓" 295 | raise NotImplementedError(f"Unknown state: {state}") 296 | 297 | id = models.BigAutoField(primary_key=True) 298 | label = models.CharField(max_length=1024, unique=True) 299 | included_queues = models.JSONField(default=list) 300 | excluded_queues = models.JSONField(default=list) 301 | timeout = models.DurationField(default=timedelta(hours=1)) 302 | last_tick = models.DateTimeField(default=now) 303 | started = models.DateTimeField(default=now) 304 | stopped = models.DateTimeField(null=True, blank=True) 305 | exit_code = models.IntegerField(choices=ExitCodes.choices, null=True, blank=True) 306 | exit_log = models.TextField(null=True, blank=True) 307 | 308 | @property 309 | def state(self): 310 | if self.stopped: 311 | if self.exit_code == WorkerStatus.ExitCodes.STOPPED: 312 | return WorkerStatus.States.STOPPED 313 | elif self.exit_code == WorkerStatus.ExitCodes.TERMINATED: 314 | return WorkerStatus.States.TERMINATED 315 | else: 316 | return WorkerStatus.States.CRASHED 317 | elif self.last_tick < now() - self.timeout: 318 | return WorkerStatus.States.TIMEDOUT 319 | else: 320 | return WorkerStatus.States.ONLINE 321 | 322 | def __str__(self): 323 | return f"Worker '{self.label}' {self.icon}" 324 | 325 | @property 326 | def icon(self): 327 | return WorkerStatus.States.icon(self.state) 328 | -------------------------------------------------------------------------------- /django_toosimple_q/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.admin.models import CHANGE, LogEntry 3 | from django.contrib.contenttypes.models import ContentType 4 | from django.contrib.messages.constants import SUCCESS 5 | from django.db.models import F 6 | from django.db.models.functions import Coalesce 7 | from django.template.defaultfilters import truncatechars 8 | from django.template.loader import render_to_string 9 | from django.urls import reverse 10 | from django.utils import timezone 11 | from django.utils.formats import date_format 12 | from django.utils.html import escape, format_html 13 | from django.utils.safestring import mark_safe 14 | from django.utils.translation import gettext_lazy as _ 15 | 16 | from .models import ScheduleExec, TaskExec, WorkerStatus 17 | from .registry import schedules_registry, tasks_registry 18 | 19 | 20 | class AbstractQueueListFilter(admin.SimpleListFilter): 21 | title = _("queue") 22 | parameter_name = "queue" 23 | registry = None 24 | name_field = None 25 | 26 | def lookups(self, request, model_admin): 27 | queues = set(item.queue for item in self.registry.values()) 28 | return [(q, q) for q in sorted(list(queues))] 29 | 30 | def queryset(self, request, queryset): 31 | queue = self.value() 32 | if queue: 33 | names = [ 34 | item.name for item in self.registry.values() if item.queue == queue 35 | ] 36 | return queryset.filter(**{f"{self.name_field}__in": names}) 37 | 38 | 39 | class TaskQueueListFilter(AbstractQueueListFilter): 40 | registry = tasks_registry 41 | name_field = "task_name" 42 | 43 | 44 | class ScheduleQueueListFilter(AbstractQueueListFilter): 45 | registry = schedules_registry 46 | name_field = "name" 47 | 48 | 49 | class ReadOnlyAdmin(admin.ModelAdmin): 50 | def has_change_permission(self, request, obj=None): 51 | return False 52 | 53 | def has_add_permission(self, request): 54 | return False 55 | 56 | 57 | @admin.register(TaskExec) 58 | class TaskExecAdmin(ReadOnlyAdmin): 59 | list_display = [ 60 | "icon", 61 | "task_name", 62 | "arguments_", 63 | "timestamp_", 64 | "execution_time_", 65 | "replaced_by_", 66 | "result_preview", 67 | "task_", 68 | ] 69 | list_display_links = ["task_name"] 70 | list_filter = ["task_name", TaskQueueListFilter, "state"] 71 | actions = ["action_requeue"] 72 | ordering = ["-created"] 73 | readonly_fields = ["task_", "result"] 74 | date_hierarchy = "created" 75 | fieldsets = [ 76 | ( 77 | None, 78 | {"fields": ["icon", "task_name", "state", "task_"]}, 79 | ), 80 | ( 81 | "Arguments", 82 | {"fields": ["args", "kwargs"]}, 83 | ), 84 | ( 85 | "Time", 86 | {"fields": ["due_", "created_", "started_", "finished_"]}, 87 | ), 88 | ( 89 | "Retries", 90 | {"fields": ["retries", "retry_delay", "replaced_by"]}, 91 | ), 92 | ( 93 | "Execution", 94 | {"fields": ["worker", "error"]}, 95 | ), 96 | ( 97 | "Output", 98 | {"fields": ["stdout", "stderr", "result"]}, 99 | ), 100 | ] 101 | 102 | def get_queryset(self, request): 103 | # defer stdout, stderr and results which may host large values 104 | qs = super().get_queryset(request) 105 | qs = qs.defer("stdout", "stderr", "result") 106 | # aggregate time for an unique field 107 | qs = qs.annotate( 108 | sortable_time=Coalesce("finished", "started", "due", "created"), 109 | execution_time=F("finished") - F("started"), 110 | ) 111 | return qs 112 | 113 | def arguments_(self, obj): 114 | return format_html( 115 | "{}
{}", 116 | truncatechars(str(obj.args), 32), 117 | truncatechars(str(obj.kwargs), 32), 118 | ) 119 | 120 | @admin.display(ordering="due") 121 | def due_(self, obj): 122 | return short_naturaltime(obj.due) 123 | 124 | @admin.display(ordering="created") 125 | def created_(self, obj): 126 | return short_naturaltime(obj.created) 127 | 128 | @admin.display(ordering="started") 129 | def started_(self, obj): 130 | return short_naturaltime(obj.started) 131 | 132 | @admin.display(ordering="finished") 133 | def finished_(self, obj): 134 | return short_naturaltime(obj.finished) 135 | 136 | @admin.display(ordering="sortable_time") 137 | def timestamp_(self, obj): 138 | if obj.finished: 139 | label = "finished" 140 | elif obj.started: 141 | label = "started" 142 | elif obj.due: 143 | label = "due" 144 | else: 145 | label = "created" 146 | return mark_safe(f"{short_naturaltime(obj.sortable_time)} [{label}]") 147 | 148 | @admin.display(ordering="execution_time") 149 | def execution_time_(self, obj): 150 | if not obj.execution_time: 151 | return None 152 | return short_seconds(obj.execution_time.seconds, additional_details=1) 153 | 154 | def replaced_by_(self, obj): 155 | if obj.replaced_by: 156 | return f"{obj.replaced_by.icon} [{obj.replaced_by.pk}]" 157 | 158 | def task_(self, obj): 159 | if not obj.task: 160 | return None 161 | return render_to_string("toosimpleq/task.html", {"task": obj.task}) 162 | 163 | def get_actions(self, request): 164 | actions = super().get_actions(request) 165 | if not request.user.has_perm("toosimpleq.requeue_taskexec"): 166 | actions.pop("action_requeue", None) 167 | return actions 168 | 169 | @admin.display(description="Requeue task") 170 | def action_requeue(self, request, queryset): 171 | for task in queryset: 172 | new_task = tasks_registry[task.task_name].enqueue(*task.args, **task.kwargs) 173 | LogEntry.objects.log_action( 174 | user_id=request.user.id, 175 | content_type_id=ContentType.objects.get_for_model(task).pk, 176 | object_id=task.pk, 177 | object_repr=str(task), 178 | action_flag=CHANGE, 179 | change_message=( 180 | f"Requeued task through admin action (new task id: {new_task.pk})" 181 | ), 182 | ) 183 | self.message_user( 184 | request, f"{queryset.count()} tasks successfully requeued", level=SUCCESS 185 | ) 186 | 187 | 188 | @admin.register(ScheduleExec) 189 | class ScheduleExecAdmin(ReadOnlyAdmin): 190 | list_display = [ 191 | "icon", 192 | "name", 193 | "last_due_", 194 | "next_due_", 195 | "last_task_", 196 | "schedule_", 197 | ] 198 | list_display_links = ["name"] 199 | ordering = ["-last_due"] 200 | list_filter = ["name", ScheduleQueueListFilter, "state"] 201 | actions = ["action_force_run"] 202 | readonly_fields = ["schedule_"] 203 | fieldsets = [ 204 | ( 205 | None, 206 | {"fields": ["icon", "name", "state", "schedule_"]}, 207 | ), 208 | ( 209 | "Time", 210 | {"fields": ["last_due_", "next_due_"]}, 211 | ), 212 | ( 213 | "Execution", 214 | {"fields": ["last_task_"]}, 215 | ), 216 | ] 217 | 218 | def schedule_(self, obj): 219 | if not obj.schedule: 220 | return None 221 | return render_to_string("toosimpleq/schedule.html", {"schedule": obj.schedule}) 222 | 223 | def last_task_(self, obj): 224 | if obj.last_task: 225 | app, model = obj.last_task._meta.app_label, obj.last_task._meta.model_name 226 | edit_link = reverse(f"admin:{app}_{model}_change", args=(obj.last_task_id,)) 227 | return format_html('{}', edit_link, obj.last_task) 228 | return "-" 229 | 230 | @admin.display(ordering="last_due") 231 | def last_due_(self, obj): 232 | return short_naturaltime(obj.last_due) 233 | 234 | @admin.display() 235 | def next_due_(self, obj): 236 | # for schedule not in the code anymore 237 | if not obj.schedule: 238 | return "invalid" 239 | 240 | if len(obj.past_dues) >= 1: 241 | next_due = obj.past_dues[0] 242 | else: 243 | next_due = obj.upcomming_due 244 | 245 | if next_due is None: 246 | return "never" 247 | 248 | formatted_next_due = short_naturaltime(next_due) 249 | if len(obj.past_dues) > 1: 250 | formatted_next_due += mark_safe(f" [×{len(obj.past_dues)}]") 251 | if next_due < timezone.now(): 252 | return mark_safe(f"{formatted_next_due}") 253 | return formatted_next_due 254 | 255 | def get_actions(self, request): 256 | actions = super().get_actions(request) 257 | if not request.user.has_perm("toosimpleq.force_run_scheduleexec"): 258 | actions.pop("action_force_run", None) 259 | return actions 260 | 261 | @admin.display(description="Force run schedule") 262 | def action_force_run(self, request, queryset): 263 | for schedule_exec in queryset: 264 | LogEntry.objects.log_action( 265 | user_id=request.user.id, 266 | content_type_id=ContentType.objects.get_for_model(schedule_exec).pk, 267 | object_id=schedule_exec.pk, 268 | object_repr=str(schedule_exec), 269 | action_flag=CHANGE, 270 | change_message=("Forced schedule execution through admin action"), 271 | ) 272 | schedule_exec.schedule.execute(dues=[None]) 273 | self.message_user( 274 | request, 275 | f"{queryset.count()} schedules successfully executed", 276 | level=SUCCESS, 277 | ) 278 | 279 | 280 | @admin.register(WorkerStatus) 281 | class WorkerStatusAdmin(ReadOnlyAdmin): 282 | list_display = [ 283 | "icon", 284 | "label", 285 | "last_tick_", 286 | "started_", 287 | "stopped_", 288 | "included_queues", 289 | "excluded_queues", 290 | ] 291 | list_display_links = ["label"] 292 | ordering = ["-started", "label"] 293 | readonly_fields = ["state"] 294 | fieldsets = [ 295 | ( 296 | None, 297 | {"fields": ["icon", "label"]}, 298 | ), 299 | ( 300 | "Queues", 301 | {"fields": ["included_queues", "excluded_queues"]}, 302 | ), 303 | ( 304 | "Time", 305 | {"fields": ["timeout", "last_tick_", "started_", "stopped_"]}, 306 | ), 307 | ( 308 | "Exit state", 309 | {"fields": ["exit_code", "exit_log"]}, 310 | ), 311 | ] 312 | 313 | @admin.display(ordering="last_tick") 314 | def last_tick_(self, obj): 315 | return short_naturaltime(obj.last_tick) 316 | 317 | @admin.display(ordering="started") 318 | def started_(self, obj): 319 | return short_naturaltime(obj.started) 320 | 321 | @admin.display(ordering="stopped") 322 | def stopped_(self, obj): 323 | return short_naturaltime(obj.stopped) 324 | 325 | 326 | def short_seconds(seconds, additional_details=0): 327 | if seconds is None: 328 | return None 329 | disps = [ 330 | (60, "second"), 331 | (60 * 60, "minute"), 332 | (60 * 60 * 24, "hour"), 333 | (60 * 60 * 24 * 7, "day"), 334 | (60 * 60 * 24 * 30, "week"), 335 | (60 * 60 * 24 * 365, "month"), 336 | (float("inf"), "year"), 337 | ] 338 | last_v = 1 339 | for threshold, abbr in disps: 340 | if abs(seconds) < threshold: 341 | count = int(abs(seconds) // last_v) 342 | plural = "s" if count > 1 else "" 343 | text = f"{count} {abbr}{plural}" 344 | if additional_details: 345 | remainder = seconds - count * last_v 346 | if remainder > 0: 347 | text += " " + short_seconds(remainder, additional_details - 1) 348 | return text 349 | last_v = threshold 350 | 351 | 352 | def short_naturaltime(datetime): 353 | if datetime is None: 354 | return None 355 | seconds = (timezone.now() - datetime).total_seconds() 356 | text = short_seconds(seconds) 357 | shorttime = f"in {text}" if seconds < 0 else f"{text} ago" 358 | longtime = date_format(datetime, format="DATETIME_FORMAT", use_l10n=True) 359 | return mark_safe(f'{shorttime}') 360 | -------------------------------------------------------------------------------- /django_toosimple_q/management/commands/worker.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import os 4 | import signal 5 | from time import sleep 6 | from traceback import format_exc 7 | 8 | from django.core.management.base import BaseCommand, CommandError 9 | from django.db import IntegrityError, transaction 10 | from django.db.models import Case, Value, When 11 | from django.utils.timezone import now 12 | 13 | from ...logging import logger 14 | from ...models import ScheduleExec, TaskExec, WorkerStatus 15 | from ...registry import schedules_registry, tasks_registry 16 | from ...tests.utils import FakeException 17 | 18 | 19 | class Command(BaseCommand): 20 | help = "Run tasks and schedules" 21 | 22 | def add_arguments(self, parser): 23 | queue = parser.add_mutually_exclusive_group() 24 | queue.add_argument( 25 | "--queue", 26 | action="append", 27 | help="which queue to run (can be used several times, all queues are run if not provided)", 28 | ) 29 | queue.add_argument( 30 | "--exclude_queue", 31 | action="append", 32 | help="which queue not to run (can be used several times, all queues are run if not provided)", 33 | ) 34 | 35 | mode = parser.add_mutually_exclusive_group() 36 | mode.add_argument( 37 | "--once", 38 | action="store_true", 39 | help="run once then exit (useful for debugging)", 40 | ) 41 | mode.add_argument( 42 | "--until_done", 43 | action="store_true", 44 | help="run until no tasks are available then exit (useful for debugging)", 45 | ) 46 | 47 | parser.add_argument( 48 | "--no_schedules", 49 | action="store_true", 50 | help="do not process schedules", 51 | ) 52 | 53 | parser.add_argument( 54 | "--tick", 55 | default=10.0, 56 | type=float, 57 | help="frequency in seconds at which the database is checked for new tasks/schedules", 58 | ) 59 | 60 | parser.add_argument( 61 | "--label", 62 | default=r"worker-{pid}", 63 | help=r"the name of the worker to help identifying it ('{pid}' will be replaced by the process id)", 64 | ) 65 | parser.add_argument( 66 | "--timeout", 67 | default=60 * 5, 68 | type=float, 69 | help="the time in seconds after which this worker will be considered offline (set this to a value higher than the longest tasks this worker will execute)", 70 | ) 71 | 72 | def handle(self, *args, **options): 73 | try: 74 | self._handle(*args, **options) 75 | except Exception as e: 76 | logger.exception(e) 77 | raise e 78 | 79 | def _handle(self, *args, **options): 80 | # Handle interuption signals 81 | signal.signal(signal.SIGINT, self.handle_signal) 82 | signal.signal(signal.SIGTERM, self.handle_signal) 83 | # Custom signal to provoke an artifical exception, used for testing only 84 | signal.signal(signal.SIGUSR1, self.handle_signal) 85 | 86 | # TODO: replace by simple-parsing 87 | self.queues = options["queue"] or [] 88 | self.excluded_queues = options["exclude_queue"] or [] 89 | self.tick_duration = options["tick"] 90 | self.once = options["once"] 91 | self.until_done = options["until_done"] 92 | self.no_schedules = options["no_schedules"] 93 | self.timeout = options["timeout"] 94 | self.label = options["label"].replace(r"{pid}", f"{os.getpid()}") 95 | 96 | self.exit_requested = False 97 | self.simulate_exception = False 98 | self.cur_task_exec = None 99 | 100 | logger.info(f"Starting worker '{self.label}'...") 101 | if self.queues: 102 | logger.info(f"Included queues: {self.queues}") 103 | elif self.excluded_queues: 104 | logger.info(f"Excluded queues: {self.excluded_queues}") 105 | 106 | self.verbosity = int(options["verbosity"]) 107 | if self.verbosity == 0: 108 | logger.setLevel(logging.WARNING) 109 | elif self.verbosity == 1: 110 | logger.setLevel(logging.INFO) 111 | else: 112 | logger.setLevel(logging.DEBUG) 113 | 114 | # On startup, we report the worker went online 115 | logger.debug(f"Get or create worker instance") 116 | self.worker_status, _ = WorkerStatus.objects.update_or_create( 117 | label=self.label, 118 | defaults={ 119 | "started": now(), 120 | "stopped": None, 121 | "exit_code": None, 122 | "exit_log": None, 123 | "included_queues": self.queues, 124 | "excluded_queues": self.excluded_queues, 125 | "timeout": datetime.timedelta(seconds=self.timeout), 126 | }, 127 | ) 128 | 129 | exc = None 130 | 131 | try: 132 | # Run the loop 133 | while self.do_loop(): 134 | pass 135 | 136 | self.worker_status.exit_code = WorkerStatus.ExitCodes.STOPPED.value 137 | self.worker_status.exit_log = "" 138 | 139 | except (KeyboardInterrupt, SystemExit) as e: 140 | exc = e 141 | logger.critical(f"Terminated by user request") 142 | if self.cur_task_exec: 143 | logger.critical(f"{self.cur_task_exec} got terminated !") 144 | self.cur_task_exec.state = TaskExec.States.INTERRUPTED 145 | self.cur_task_exec.save() 146 | self.cur_task_exec.create_replacement(is_retry=False) 147 | self.cur_task_exec = None 148 | self.worker_status.exit_code = WorkerStatus.ExitCodes.TERMINATED.value 149 | self.worker_status.exit_log = format_exc() 150 | 151 | except Exception as e: 152 | exc = e 153 | logger.exception(e) 154 | self.worker_status.exit_code = WorkerStatus.ExitCodes.CRASHED.value 155 | self.worker_status.exit_log = format_exc() 156 | 157 | finally: 158 | self.worker_status.stopped = now() 159 | self.worker_status.save() 160 | 161 | if self.worker_status.exit_code: 162 | raise CommandError(returncode=self.worker_status.exit_code) from exc 163 | 164 | def do_loop(self) -> bool: 165 | """Runs one tick, returns True if it should continue looping""" 166 | 167 | logger.debug(f"Tick !") 168 | 169 | last_run = now() 170 | 171 | did_something = False 172 | 173 | logger.debug(f"1. Update status...") 174 | self.worker_status.last_tick = now() 175 | self.worker_status.save() 176 | 177 | logger.debug(f"2. Disabling orphaned schedules") 178 | with transaction.atomic(): 179 | if ( 180 | ScheduleExec.objects.exclude(state=ScheduleExec.States.INVALID) 181 | .exclude(name__in=schedules_registry.keys()) 182 | .update(state=ScheduleExec.States.INVALID) 183 | ): 184 | logger.warning(f"Found invalid schedules") 185 | 186 | logger.debug(f"3. Disabling orphaned tasks") 187 | with transaction.atomic(): 188 | if ( 189 | TaskExec.objects.exclude(state=TaskExec.States.INVALID) 190 | .exclude(task_name__in=tasks_registry.keys()) 191 | .update(state=TaskExec.States.INVALID) 192 | ): 193 | logger.warning(f"Found invalid tasks") 194 | 195 | if not self.no_schedules: 196 | logger.debug(f"4. Create missing schedules") 197 | existing_schedules_names = ScheduleExec.objects.values_list( 198 | "name", flat=True 199 | ) 200 | for schedule in self._relevant_schedules: 201 | # Create the schedule exec if it does not exist 202 | if schedule.name in existing_schedules_names: 203 | continue 204 | try: 205 | last_due = None if schedule.run_on_creation else now() 206 | ScheduleExec.objects.create(name=schedule.name, last_due=last_due) 207 | logger.debug(f"Created schedule {schedule.name}") 208 | except IntegrityError: 209 | # This could happen with concurrent workers, and can be ignored 210 | logger.debug( 211 | f"Schedule {schedule.name} already exists, probably created concurrently" 212 | ) 213 | 214 | logger.debug(f"5. Execute schedules") 215 | with transaction.atomic(): 216 | for schedule_exec in self._build_schedules_list_qs(): 217 | did_something |= schedule_exec.execute() 218 | 219 | logger.debug(f"6. Waking up tasks") 220 | TaskExec.objects.filter(state=TaskExec.States.SLEEPING).filter( 221 | due__lte=now() 222 | ).update(state=TaskExec.States.QUEUED) 223 | 224 | logger.debug(f"7. Locking task") 225 | with transaction.atomic(): 226 | self.cur_task_exec = self._build_due_tasks_qs().first() 227 | if self.cur_task_exec: 228 | logger.debug(f"Picking up for execution : {self.cur_task_exec}") 229 | self.cur_task_exec.started = now() 230 | self.cur_task_exec.state = TaskExec.States.PROCESSING 231 | self.cur_task_exec.worker = self.worker_status 232 | self.cur_task_exec.save() 233 | 234 | logger.debug(f"8. Running task") 235 | if self.cur_task_exec: 236 | logger.debug(f"Executing : {self.cur_task_exec}") 237 | did_something = True 238 | self.cur_task_exec.execute() 239 | self.cur_task_exec = None 240 | 241 | if self.once: 242 | logger.info("Exiting loop because --once was passed") 243 | return False 244 | 245 | if self.until_done and not did_something: 246 | logger.info("Exiting loop because --until_done was passed") 247 | return False 248 | 249 | if self.exit_requested: 250 | logger.critical("Exiting gracefully on user request") 251 | return False 252 | 253 | if self.simulate_exception: 254 | # for testing 255 | raise FakeException() 256 | 257 | if not did_something: 258 | logger.debug(f"Waiting for next tick...") 259 | next_run = last_run + datetime.timedelta(seconds=self.tick_duration) 260 | while not self.exit_requested and now() < next_run: 261 | sleep(1) 262 | 263 | return True 264 | 265 | def handle_signal(self, sig, stackframe): 266 | # For testing, simulates a unexpected crash of the worker 267 | if sig == signal.SIGUSR1: 268 | self.simulate_exception = True 269 | return 270 | 271 | # A termination signal or a second interruption signal should force exit 272 | if sig == signal.SIGTERM or (sig == signal.SIGINT and self.exit_requested): 273 | logger.critical(f"User requested termination...") 274 | raise KeyboardInterrupt() 275 | 276 | # A first interruption signal is graceful exit 277 | if sig == signal.SIGINT: 278 | logger.critical(f"User requested graceful exit...") 279 | if self.cur_task_exec is not None: 280 | logger.critical(f"Waiting for `{self.cur_task_exec}` to finish...") 281 | self.exit_requested = True 282 | 283 | @property 284 | def _relevant_schedules(self): 285 | """Get a list of schedules for this worker""" 286 | return schedules_registry.for_queue(self.queues, self.excluded_queues) 287 | 288 | def _build_schedules_list_qs(self): 289 | """The queryset to select the list of schedules for update""" 290 | 291 | return ScheduleExec.objects.filter( 292 | name__in=[s.name for s in self._relevant_schedules] 293 | ).select_for_update(skip_locked=True) 294 | 295 | @property 296 | def _relevant_tasks(self): 297 | """Get a list of tasks for this worker""" 298 | return tasks_registry.for_queue(self.queues, self.excluded_queues) 299 | 300 | def _build_due_tasks_qs(self): 301 | """The queryset to select the task due by this worker for update""" 302 | 303 | # Build a order_by clause using the task priorities 304 | whens = [ 305 | When(task_name=t.name, then=Value(-t.priority)) 306 | for t in self._relevant_tasks 307 | ] 308 | order_by_priority = Case(*whens, default=Value(0)) 309 | 310 | # Build the queryset 311 | return ( 312 | TaskExec.objects.filter(state=TaskExec.States.QUEUED) 313 | .filter(task_name__in=[t.name for t in self._relevant_tasks]) 314 | .order_by(order_by_priority, "due", "created") 315 | .select_for_update(skip_locked=True) 316 | ) 317 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_schedules.py: -------------------------------------------------------------------------------- 1 | from django.core import management 2 | from freezegun import freeze_time 3 | 4 | from django_toosimple_q.decorators import register_task, schedule_task 5 | from django_toosimple_q.models import ScheduleExec, TaskExec 6 | from django_toosimple_q.registry import schedules_registry 7 | 8 | from .base import TooSimpleQRegularTestCase 9 | 10 | 11 | class TestSchedules(TooSimpleQRegularTestCase): 12 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 13 | def test_schedule(self, frozen_datetime): 14 | """Testing schedules""" 15 | 16 | @schedule_task(cron="0 12 * * *", datetime_kwarg="scheduled_on") 17 | @register_task(name="normal") 18 | def a(scheduled_on): 19 | return f"{scheduled_on:%Y-%m-%d %H:%M}" 20 | 21 | @schedule_task( 22 | cron="0 12 * * *", run_on_creation=True, datetime_kwarg="scheduled_on" 23 | ) 24 | @register_task(name="autostart") 25 | def b(scheduled_on): 26 | return f"{scheduled_on:%Y-%m-%d %H:%M}" 27 | 28 | @schedule_task(cron="0 12 * * *", catch_up=True, datetime_kwarg="scheduled_on") 29 | @register_task(name="catchup") 30 | def c(scheduled_on): 31 | return f"{scheduled_on:%Y-%m-%d %H:%M}" 32 | 33 | @schedule_task( 34 | cron="0 12 * * *", 35 | run_on_creation=True, 36 | catch_up=True, 37 | datetime_kwarg="scheduled_on", 38 | ) 39 | @register_task(name="autostartcatchup") 40 | def d(scheduled_on): 41 | return f"{scheduled_on:%Y-%m-%d %H:%M}" 42 | 43 | self.assertEqual(len(schedules_registry), 4) 44 | self.assertEqual(ScheduleExec.objects.count(), 0) 45 | self.assertQueue(0) 46 | 47 | management.call_command("worker", "--until_done") 48 | 49 | # first run, only tasks with run_on_creation=True should run as no time passed 50 | self.assertQueue(0, task_name="normal") 51 | self.assertQueue(1, task_name="autostart") 52 | self.assertQueue(0, task_name="catchup") 53 | self.assertQueue(1, task_name="autostartcatchup") 54 | self.assertQueue(2) 55 | 56 | management.call_command("worker", "--until_done") 57 | 58 | # second run, no time passed so no change 59 | self.assertQueue(0, task_name="normal") 60 | self.assertQueue(1, task_name="autostart") 61 | self.assertQueue(0, task_name="catchup") 62 | self.assertQueue(1, task_name="autostartcatchup") 63 | self.assertQueue(2) 64 | 65 | frozen_datetime.move_to("2020-01-02") 66 | management.call_command("worker", "--until_done") 67 | 68 | # one day passed, all tasks should have run once 69 | self.assertQueue(1, task_name="normal") 70 | self.assertQueue(2, task_name="autostart") 71 | self.assertQueue(1, task_name="catchup") 72 | self.assertQueue(2, task_name="autostartcatchup") 73 | self.assertQueue(6) 74 | 75 | frozen_datetime.move_to("2020-01-05") 76 | management.call_command("worker", "--until_done") 77 | 78 | # three day passed, catch_up should have run thrice and other once 79 | self.assertQueue(2, task_name="normal") 80 | self.assertQueue(3, task_name="autostart") 81 | self.assertQueue(4, task_name="catchup") 82 | self.assertQueue(5, task_name="autostartcatchup") 83 | self.assertQueue(14) 84 | 85 | # make sure all tasks succeeded 86 | self.assertQueue(14, state=TaskExec.States.SUCCEEDED) 87 | 88 | # make sure we got correct dates 89 | self.assertResults( 90 | task_name="normal", 91 | expected=[ 92 | "2020-01-01 12:00", 93 | "2020-01-04 12:00", 94 | ], 95 | ) 96 | self.assertResults( 97 | task_name="autostart", 98 | expected=[ 99 | "2019-12-31 12:00", 100 | "2020-01-01 12:00", 101 | "2020-01-04 12:00", 102 | ], 103 | ) 104 | self.assertResults( 105 | task_name="catchup", 106 | expected=[ 107 | "2020-01-01 12:00", 108 | "2020-01-02 12:00", 109 | "2020-01-03 12:00", 110 | "2020-01-04 12:00", 111 | ], 112 | ) 113 | self.assertResults( 114 | task_name="autostartcatchup", 115 | expected=[ 116 | "2019-12-31 12:00", 117 | "2020-01-01 12:00", 118 | "2020-01-02 12:00", 119 | "2020-01-03 12:00", 120 | "2020-01-04 12:00", 121 | ], 122 | ) 123 | 124 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 125 | def test_manual_schedule(self, frozen_datetime): 126 | """Testing manual schedules""" 127 | 128 | @schedule_task(cron="manual", datetime_kwarg="scheduled_on") 129 | @register_task(name="normal") 130 | def a(scheduled_on): 131 | return f"{scheduled_on:%Y-%m-%d %H:%M}" 132 | 133 | self.assertEqual(len(schedules_registry), 1) 134 | self.assertEqual(ScheduleExec.objects.count(), 0) 135 | self.assertQueue(0) 136 | 137 | # a "manual" schedule never runs 138 | management.call_command("worker", "--until_done") 139 | frozen_datetime.move_to("2050-01-01") 140 | management.call_command("worker", "--until_done") 141 | self.assertQueue(0) 142 | 143 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 144 | def test_invalid_schedule(self, frozen_datetime): 145 | """Testing invalid schedules""" 146 | 147 | @schedule_task(cron="0 * * * *") 148 | @register_task(name="valid") 149 | def a(): 150 | return f"Valid task" 151 | 152 | @schedule_task(cron="0 * * * *") 153 | @register_task(name="invalid") 154 | def a(): 155 | return f"Invalid task" 156 | 157 | all_schedules = ScheduleExec.objects.all() 158 | 159 | management.call_command("worker", "--until_done") 160 | 161 | self.assertEqual( 162 | all_schedules.filter(state=ScheduleExec.States.ACTIVE).count(), 2 163 | ) 164 | self.assertEqual( 165 | all_schedules.filter(state=ScheduleExec.States.INVALID).count(), 0 166 | ) 167 | self.assertEqual(all_schedules.count(), 2) 168 | 169 | del schedules_registry["invalid"] 170 | 171 | management.call_command("worker", "--until_done") 172 | 173 | self.assertEqual( 174 | all_schedules.filter(state=ScheduleExec.States.ACTIVE).count(), 1 175 | ) 176 | self.assertEqual( 177 | all_schedules.filter(state=ScheduleExec.States.INVALID).count(), 1 178 | ) 179 | self.assertEqual(all_schedules.count(), 2) 180 | 181 | def test_named_queues(self): 182 | """Checking named queues""" 183 | 184 | @schedule_task(cron="* * * * *") # queue="default" 185 | @register_task(name="a") 186 | def a(x): 187 | return x * 2 188 | 189 | @schedule_task(cron="* * * * *", queue="queue_b") 190 | @register_task(name="b") 191 | def b(x): 192 | return x * 2 193 | 194 | @schedule_task(cron="* * * * *", queue="queue_c") 195 | @register_task(name="c") 196 | def c(x): 197 | return x * 2 198 | 199 | @schedule_task(cron="* * * * *", queue="queue_d") 200 | @register_task(name="d") 201 | def d(x): 202 | return x * 2 203 | 204 | @schedule_task(cron="* * * * *", queue="queue_e") 205 | @register_task(name="e") 206 | def e(x): 207 | return x * 2 208 | 209 | @schedule_task(cron="* * * * *", queue="queue_f") 210 | @register_task(name="f") 211 | def f(x): 212 | return x * 2 213 | 214 | @schedule_task(cron="* * * * *", queue="queue_g") 215 | @register_task(name="g") 216 | def g(x): 217 | return x * 2 218 | 219 | @schedule_task(cron="* * * * *", queue="queue_h") 220 | @register_task(name="h") 221 | def h(x): 222 | return x * 2 223 | 224 | self.assertSchedule("a", None) 225 | self.assertSchedule("b", None) 226 | self.assertSchedule("c", None) 227 | self.assertSchedule("d", None) 228 | self.assertSchedule("e", None) 229 | self.assertSchedule("f", None) 230 | self.assertSchedule("g", None) 231 | self.assertSchedule("h", None) 232 | 233 | # make sure schedules get assigned to default queue by default 234 | management.call_command("worker", "--until_done", "--queue", "default") 235 | 236 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 237 | self.assertSchedule("b", None) 238 | self.assertSchedule("c", None) 239 | self.assertSchedule("d", None) 240 | self.assertSchedule("e", None) 241 | self.assertSchedule("f", None) 242 | self.assertSchedule("g", None) 243 | self.assertSchedule("h", None) 244 | 245 | # make sure worker only runs their queue 246 | management.call_command("worker", "--until_done", "--queue", "queue_c") 247 | 248 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 249 | self.assertSchedule("b", None) 250 | self.assertSchedule("c", ScheduleExec.States.ACTIVE) 251 | self.assertSchedule("d", None) 252 | self.assertSchedule("e", None) 253 | self.assertSchedule("f", None) 254 | self.assertSchedule("g", None) 255 | self.assertSchedule("h", None) 256 | 257 | # make sure worker can run multiple queues 258 | management.call_command( 259 | "worker", "--until_done", "--queue", "queue_b", "--queue", "queue_d" 260 | ) 261 | 262 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 263 | self.assertSchedule("b", ScheduleExec.States.ACTIVE) 264 | self.assertSchedule("c", ScheduleExec.States.ACTIVE) 265 | self.assertSchedule("d", ScheduleExec.States.ACTIVE) 266 | self.assertSchedule("e", None) 267 | self.assertSchedule("f", None) 268 | self.assertSchedule("g", None) 269 | self.assertSchedule("h", None) 270 | 271 | # make sure worker exclude queue works 272 | management.call_command( 273 | "worker", 274 | "--until_done", 275 | "--exclude_queue", 276 | "queue_g", 277 | "--exclude_queue", 278 | "queue_h", 279 | ) 280 | 281 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 282 | self.assertSchedule("b", ScheduleExec.States.ACTIVE) 283 | self.assertSchedule("c", ScheduleExec.States.ACTIVE) 284 | self.assertSchedule("d", ScheduleExec.States.ACTIVE) 285 | self.assertSchedule("e", ScheduleExec.States.ACTIVE) 286 | self.assertSchedule("f", ScheduleExec.States.ACTIVE) 287 | self.assertSchedule("g", None) 288 | self.assertSchedule("h", None) 289 | 290 | # make sure worker run all queues by default 291 | management.call_command("worker", "--until_done") 292 | 293 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 294 | self.assertSchedule("b", ScheduleExec.States.ACTIVE) 295 | self.assertSchedule("c", ScheduleExec.States.ACTIVE) 296 | self.assertSchedule("d", ScheduleExec.States.ACTIVE) 297 | self.assertSchedule("e", ScheduleExec.States.ACTIVE) 298 | self.assertSchedule("f", ScheduleExec.States.ACTIVE) 299 | self.assertSchedule("g", ScheduleExec.States.ACTIVE) 300 | self.assertSchedule("h", ScheduleExec.States.ACTIVE) 301 | 302 | def test_no_schedules(self): 303 | """Checking named queues""" 304 | 305 | @schedule_task(cron="0 * * * *", run_on_creation=True) 306 | @register_task(name="a") 307 | def a(): 308 | return True 309 | 310 | @schedule_task(cron="0 * * * *", run_on_creation=True) 311 | @register_task(name="b") 312 | def b(): 313 | return True 314 | 315 | a.queue() 316 | 317 | self.assertSchedule("a", None) 318 | self.assertSchedule("b", None) 319 | self.assertQueue(1, "a", TaskExec.States.QUEUED) 320 | self.assertQueue(0, "b") 321 | 322 | # runninng with --no_schedules doesn't populate schedules but runs tasks 323 | management.call_command("worker", "--until_done", "--no_schedules") 324 | 325 | print(TaskExec.objects.all().last().error) 326 | 327 | self.assertSchedule("a", None) 328 | self.assertSchedule("b", None) 329 | self.assertQueue(1, "a", TaskExec.States.SUCCEEDED) 330 | self.assertQueue(0, "b") 331 | 332 | # runninng with --no_schedules doesn't pickup existing schedules either 333 | ScheduleExec.objects.create(name="a", state=ScheduleExec.States.INVALID) 334 | management.call_command("worker", "--until_done", "--no_schedules") 335 | 336 | self.assertSchedule("a", ScheduleExec.States.INVALID) 337 | self.assertSchedule("b", None) 338 | self.assertQueue(1, "a", TaskExec.States.SUCCEEDED) 339 | self.assertQueue(0, "b") 340 | 341 | # just ensure schedules work as expected without --no_schedules 342 | management.call_command("worker", "--until_done") 343 | 344 | self.assertSchedule("a", ScheduleExec.States.ACTIVE) 345 | self.assertSchedule("b", ScheduleExec.States.ACTIVE) 346 | self.assertQueue(2, "a", TaskExec.States.SUCCEEDED) 347 | self.assertQueue(1, "b", TaskExec.States.SUCCEEDED) 348 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django Too Simple Queue 2 | 3 | [![PyPI version](https://badge.fury.io/py/django-toosimple-q.svg)](https://pypi.org/project/django-toosimple-q/) 4 | [![test](https://github.com/olivierdalang/django-toosimple-q/actions/workflows/test.yml/badge.svg)](https://github.com/olivierdalang/django-toosimple-q/actions/workflows/test.yml) 5 | 6 | This packages provides a simplistic task queue and scheduler for Django. 7 | 8 | It is geared towards basic apps, where simplicity primes. The package offers simple decorator syntax, including cron-like schedules. 9 | 10 | Features : 11 | 12 | - no celery/redis/rabbitmq/whatever... just Django ! 13 | - clean decorator syntax to register tasks and schedules 14 | - simple queuing syntax 15 | - cron-like scheduling 16 | - tasks.py autodiscovery 17 | - django admin integration 18 | - tasks results stored using the Django ORM 19 | - replacement tasks on interruption 20 | 21 | Limitations : 22 | 23 | - no multithreading yet (but running multiple workers should work) 24 | - not well suited for projects spawning a high volume of tasks 25 | 26 | Compatibility: 27 | 28 | - Django 3.2 and 4.0 29 | - Python 3.8, 3.9, 3.10 30 | 31 | ## Installation 32 | 33 | Install the library : 34 | ```shell 35 | pip install django-toosimple-q 36 | ``` 37 | 38 | Enable the app in `settings.py` : 39 | ```python 40 | INSTALLED_APPS = [ 41 | # ... 42 | 'django_toosimple_q', 43 | # ... 44 | ] 45 | ``` 46 | 47 | ## Quickstart 48 | 49 | Tasks need to be registered using the `@register_task()` decorator. Once registered, they can be added to the queue by calling the `.queue()` function. 50 | 51 | ```python 52 | from django_toosimple_q.decorators import register_task 53 | 54 | # Register a task 55 | @register_task() 56 | def my_task(name): 57 | return f"Hello {name} !" 58 | 59 | # Enqueue tasks 60 | my_task.queue("John") 61 | my_task.queue("Peter") 62 | ``` 63 | 64 | Registered tasks can be scheduled from code using this cron-like syntax : 65 | ```python 66 | from django_toosimple_q.decorators import register_task, schedule_task 67 | 68 | # Register and schedule tasks (each morning at 8:30) 69 | @schedule_task(cron="30 8 * * *", args=['John']) 70 | @register_task() 71 | def morning_routine(name): 72 | return f"Good morning {name} !" 73 | ``` 74 | 75 | To consume the tasks, you need to run at least one worker : 76 | ```shell 77 | python manage.py worker 78 | ``` 79 | The workers will take care of adding scheduled tasks to the queue when needed, and will execute the tasks. 80 | 81 | The package autoloads `tasks.py` from all installed apps. While this is the recommended place to define your tasks, you can do so from anywhere in your code. 82 | 83 | ## Advanced usage 84 | 85 | ### Tasks 86 | 87 | You can optionnaly give a custom name to your tasks. This is required when your task is defined in a local scope. 88 | ```python 89 | @register_task(name="my_favourite_task") 90 | def my_task(): 91 | ... 92 | ``` 93 | 94 | You can set task priorities. 95 | ```python 96 | @register_task(priority=0) 97 | def my_favourite_task(): 98 | ... 99 | 100 | @register_task(priority=1) 101 | def my_other_task(): 102 | ... 103 | 104 | # Enqueue tasks 105 | my_other_task.queue() 106 | my_favourite_task.queue() # will be executed before the other one 107 | ``` 108 | 109 | You can define `retries=N` and `retry_delay=S` to retry the task in case of failure. The delay (in second) will double on each failure. 110 | 111 | ```python 112 | @register_task(retries=10, retry_delay=60) 113 | def download_data(): 114 | ... 115 | ``` 116 | 117 | You can mark a task as `unique=True` if the task shouldn't be queued again if already queued with the same arguments. This is usefull for tasks such as cleaning or refreshing. 118 | 119 | ```python 120 | @register_task(unique=True) 121 | def cleanup(): 122 | ... 123 | 124 | cleanup.queue() 125 | cleanup.queue() # this will be ignored as long as the first one is still queued 126 | ``` 127 | 128 | You can assign tasks to specific queues, and then have your worker only consume tasks from specific queues using `--queue myqueue` or `--exclude_queue myqueue`. By default, workers consume all tasks. 129 | 130 | ```python 131 | @register_task(queue='long_running') 132 | def long_task(): 133 | ... 134 | 135 | @register_task() 136 | def short_task(): 137 | ... 138 | 139 | # Then run those with these workers, so that long 140 | # running tasks don't prevent short running tasks 141 | # from being run : 142 | # manage.py worker --exclude_queue long_running 143 | # manage.py worker 144 | ``` 145 | 146 | You can enqueue tasks with a specific due date. 147 | ```python 148 | @register_task() 149 | def my_task(): 150 | ... 151 | 152 | # Enqueue tasks 153 | from datetime import datetime, timedelta 154 | my_task.queue("John", due=datetime.now() + timedelta(hours=1)) 155 | ``` 156 | 157 | The `queue()` function returns a `TaskExec` model instance, which holds information about the task execution, including the task result. 158 | 159 | ```python 160 | from django.core.management import call_command 161 | from django_toosimple_q.models import TaskExec 162 | 163 | @register_task() 164 | def multiply(a, b): 165 | return a * b 166 | 167 | t = multiply.queue(3, 4) 168 | 169 | assert t.state == TaskExec.States.QUEUED 170 | assert t.result == None 171 | 172 | call_command("worker", "--until_done") # equivalent to `python manage.py worker --until_done` 173 | 174 | t.refresh_from_db() 175 | assert t.state == TaskExec.States.SUCCEEDED 176 | assert t.result == 12 177 | ``` 178 | 179 | ### Schedules 180 | 181 | You may define multiple schedules for the same task. In this case, it is mandatory to specify a unique name : 182 | 183 | ```python 184 | @schedule_task(name="afternoon_routine", cron="30 16 * * *", args=['afternoon']) 185 | @schedule_task(name="morning_routine", cron="30 8 * * *", args=['morning']) 186 | @register_task() 187 | def my_task(time_of_day): 188 | return f"Good {time_of_day} John !" 189 | ``` 190 | 191 | By default, `last_run` is set to `now()` on schedule creation. This means they will only run on next cron occurence. If you need your schedules to be run as soon as possible after initialisation, you can specify `run_on_creation=True`. 192 | 193 | ```python 194 | @schedule_task(cron="30 8 * * *", run_on_creation=True) 195 | @register_task() 196 | def my_task(): 197 | ... 198 | ``` 199 | 200 | By default, if some crons where missed (e.g. after a server shutdown or if the workers can't keep up with all tasks), the missed tasks will be lost. If you need the tasks to catch up, set `catch_up=True`. 201 | 202 | ```python 203 | @schedule_task(cron="30 8 * * *", catch_up=True) 204 | @register_task() 205 | def my_task(): 206 | ... 207 | ``` 208 | 209 | You may get the schedule's cron datetime provided as a keyword argument to the task using the `datetime_kwarg` argument. This is often useful in combination with catch_up, for things like report generation. Remember to treat the case where the argument is `None` (which happens when the task is run outside of the schedule). 210 | 211 | ```python 212 | @schedule_task(cron="30 8 * * *", datetime_kwarg="scheduled_on", catch_up=True) 213 | @register_task() 214 | def my_task(scheduled_on): 215 | if scheduled_on: 216 | return f"This was scheduled for {scheduled_on.isoformat()}." 217 | else: 218 | return "This was not scheduled." 219 | ``` 220 | 221 | Similarly to tasks, you can assign schedules to specific queues, and then have your worker only consume tasks from specific queues using `--queue myqueue` or `--exclude_queue myqueue`. 222 | 223 | ```python 224 | @schedule_task(cron="30 8 * * *", queue='scheduler') 225 | @register_task(queue='worker') 226 | def task(): 227 | ... 228 | 229 | # Then run those with these workers 230 | # manage.py worker --queue scheduler 231 | # manage.py worker --queue worker 232 | ``` 233 | 234 | Schedule's cron support a non-standard sixth argument for seconds : 235 | ```python 236 | from django_toosimple_q.decorators import register_task, schedule_task 237 | 238 | # A schedule running every 15 seconds 239 | @schedule_task(cron="* * * * * */15") 240 | @register_task() 241 | def morning_routine(): 242 | return f"15 seconds passed !" 243 | ``` 244 | 245 | Schedule's cron can also be set to `manual` in which case it never runs, but can only be triggered manually from the admin : 246 | ```python 247 | from django_toosimple_q.decorators import register_task, schedule_task 248 | 249 | # A schedule that only runs when manually triggered 250 | @schedule_task(cron="manual") 251 | @register_task() 252 | def for_special_occasions(): 253 | return f"this was triggered manually !" 254 | ``` 255 | 256 | ### Management comment 257 | 258 | Besides standard django management commands arguments, the management command supports following arguments. 259 | 260 | ``` 261 | usage: manage.py worker [--queue QUEUE | --exclude_queue EXCLUDE_QUEUE] 262 | [--tick TICK] 263 | [--once | --until_done] 264 | [--no_schedules] 265 | [--label LABEL] 266 | [--timeout TIMEOUT] 267 | 268 | optional arguments: 269 | --queue QUEUE which queue to run (can be used several times, all 270 | queues are run if not provided) 271 | --exclude_queue EXCLUDE_QUEUE 272 | which queue not to run (can be used several times, all 273 | queues are run if not provided) 274 | --tick TICK frequency in seconds at which the database is checked 275 | for new tasks/schedules 276 | --once run once then exit (useful for debugging) 277 | --until_done run until no tasks are available then exit (useful for 278 | debugging) 279 | --no_schedules do not process schedules 280 | --label LABEL the name of the worker to help identifying it ('{pid}' 281 | will be replaced by the process id) 282 | --timeout TIMEOUT the time in seconds after which this worker will be considered 283 | offline (set this to a value higher than the longest tasks this 284 | worker will execute) 285 | ``` 286 | 287 | ## Contrib apps 288 | 289 | ### django_toosimple_q.contrib.mail 290 | 291 | A queued email backend to send emails asynchronously, preventing your website from failing completely in case the upstream backend is down. 292 | 293 | Enable and configure the app in `settings.py` : 294 | ```python 295 | INSTALLED_APPS = [ 296 | # ... 297 | 'django_toosimple_q.contrib.mail', 298 | # ... 299 | ] 300 | 301 | EMAIL_BACKEND = 'django_toosimple_q.contrib.mail.backends.QueueBackend' 302 | 303 | # Actual Django email backend used, defaults to django.core.mail.backends.smtp.EmailBackend, see https://docs.djangoproject.com/en/3.2/ref/settings/#email-backend 304 | TOOSIMPLEQ_EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' 305 | ``` 306 | 307 | Head to the [Django documentation](https://docs.djangoproject.com/en/4.0/topics/email/) for usage. 308 | 309 | ## Dev 310 | 311 | ### Automated tests 312 | 313 | To run tests, we recommend using Docker : 314 | ```shell 315 | docker compose build 316 | # run all tests 317 | docker compose run django test 318 | # or to run just a specific test 319 | docker compose run django test django_toosimple_q.tests.tests_worker.TestWorker 320 | ``` 321 | 322 | Tests are run automatically on github. 323 | 324 | ### Manual testing 325 | 326 | Manual testing can be done like this: 327 | 328 | ```shell 329 | # start a dev server and a worker 330 | docker compose build 331 | docker compose run django migrate 332 | docker compose run django createsuperuser 333 | docker compose up 334 | ``` 335 | 336 | Then connect on 127.0.0.1:8000/admin/ 337 | 338 | ### Without docker 339 | 340 | To run tests locally without Docker (by default, tests runs against an in-memory sqlite database): 341 | 342 | ```shell 343 | pip install -r requirements-dev.txt 344 | python manage.py test 345 | ``` 346 | 347 | ### Contribute 348 | 349 | Code style is done with pre-commit : 350 | ```shell 351 | pip install -r requirements-dev.txt 352 | pre-commit install 353 | ``` 354 | 355 | ## Internals 356 | 357 | ### Terms 358 | 359 | **Task**: a callable with a known name in the *registry*. These are typically registered in `tasks.py`. 360 | 361 | **TaskExecution**: a specific planned or past call of a *task*, including inputs (arguments) and outputs. This is a model, whose instanced are typically created using `mycallable.queue()` or from schedules. 362 | 363 | **Schedule**: a configuration for repeated execution of *tasks*. These are typically configured in `tasks.py`. 364 | 365 | **ScheduleExecution**: the last execution of a *schedule* (e.g. keeps track of the last time a schedule actually lead to generate a task execution). This is a model, whose instances are created by the worker. 366 | 367 | **Registry**: a dictionary keeping all registered schedules and tasks. 368 | 369 | **Worker**: a management command that executes schedules and tasks on a regular basis. 370 | 371 | 372 | ## Changelog 373 | 374 | - 2025-09-01 : v1.0.0 **⚠ BACKWARDS INCOMPATIBLE RELEASE ⚠** 375 | - feature: added workerstatus to the admin, allowing to monitor workers 376 | - feature: queue tasks for later (`mytask.queue(due=now()+timedelta(hours=2))`) 377 | - feature: assign queues to schedules (`@schedule_task(queue="schedules")`) 378 | - feature: allow manual schedules that are only run manually through the admin (`@schedule_task(cron="manual")`) 379 | - feature: custom permissions to force run schedules and to requeue tasks 380 | - feature: log admin actions 381 | - refactor: removed non-execution related data from the database (clarifying the fact tha the source of truth is the registry) 382 | - refactor: better support for concurrent workers 383 | - refactor: better names for models and decorators 384 | - refactor: optimise task exec admin listing when `results`, `stdout` or `stderr` holds large data 385 | - infra: included a demo project 386 | - infra: improved testing, including for concurrency behaviour 387 | - infra: updated compatibility to Django 4.2/5.1/5.2 and Python 3.8-3.13 388 | - quick migration guide: 389 | - rename `@schedule` -> `@schedule_task` 390 | - task name must now be provided as a kwarg: `@register_task("mytask")` -> `@register_task(name="mytask")`) 391 | - replace `@schedule_task(..., last_check=None)` -> `@schedule_task(..., run_on_creation=True)` 392 | - models: `Schedule` -> `ScheduleExec` and `Task` -> `TaskExec` 393 | - renamed `ScheduleExec.last_run` to `ScheduleExec.last_task` 394 | 395 | - 2022-03-24 : v0.4.0 396 | - made `last_check` and `last_run` optional in the admin 397 | - defined `id` fields 398 | 399 | - 2021-07-15 : v0.3.0 400 | - added `contrib.mail` 401 | - task replacement now tracked with a FK instead of a state 402 | - also run tests on postgres 403 | - added `datetime_kwarg` argument to schedules 404 | 405 | - 2021-06-11 : v0.2.0 406 | - added `retries`, `retry_delay` options for tasks 407 | - improve logging 408 | 409 | - 2020-11-12 : v0.1.0 410 | - fixed bug where updating schedule failed 411 | - fixed worker not doing all available tasks for each tick 412 | - added --tick argument 413 | - enforce uniqueness of schedule 414 | -------------------------------------------------------------------------------- /django_toosimple_q/tests/tests_tasks.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from django.core import management 4 | from django.utils import timezone 5 | from freezegun import freeze_time 6 | 7 | from django_toosimple_q.decorators import register_task 8 | from django_toosimple_q.models import TaskExec 9 | 10 | from .base import TooSimpleQRegularTestCase 11 | 12 | 13 | class TestTasks(TooSimpleQRegularTestCase): 14 | def test_task_states(self): 15 | """Checking correctness of task states""" 16 | 17 | @register_task(name="a") 18 | def a(x): 19 | return x * 2 20 | 21 | @register_task(name="b") 22 | def b(x): 23 | return x / 0 24 | 25 | # Succeeding task 26 | t = a.queue(2) 27 | self.assertEqual(t.state, TaskExec.States.QUEUED) 28 | management.call_command("worker", "--once") 29 | t.refresh_from_db() 30 | self.assertEqual(t.state, TaskExec.States.SUCCEEDED) 31 | self.assertEqual(t.result, 4) 32 | self.assertEqual(t.error, None) 33 | 34 | # Failing task 35 | t = b.queue(2) 36 | self.assertEqual(t.state, TaskExec.States.QUEUED) 37 | management.call_command("worker", "--once") 38 | t.refresh_from_db() 39 | self.assertEqual(t.state, TaskExec.States.FAILED) 40 | self.assertEqual(t.result, None) 41 | self.assertNotEqual(t.error, None) 42 | 43 | # Task with due date 44 | t = a.queue(2, due=timezone.now() - datetime.timedelta(hours=1)) 45 | self.assertEqual(t.state, TaskExec.States.SLEEPING) 46 | management.call_command("worker", "--once") 47 | t.refresh_from_db() 48 | self.assertEqual(t.state, TaskExec.States.SUCCEEDED) 49 | self.assertEqual(t.result, 4) 50 | self.assertEqual(t.error, None) 51 | 52 | # Invalid task 53 | t = TaskExec.objects.create(task_name="invalid") 54 | self.assertEqual(t.state, TaskExec.States.QUEUED) 55 | management.call_command("worker", "--once") 56 | t.refresh_from_db() 57 | self.assertEqual(t.state, TaskExec.States.INVALID) 58 | self.assertEqual(t.result, None) 59 | self.assertEqual(t.error, None) 60 | 61 | def test_task_registration(self): 62 | """Checking task registration""" 63 | 64 | # We cannot run arbitrary functions 65 | self.assertQueue(0) 66 | t = TaskExec.objects.create(task_name="print", args=["test"]) 67 | self.assertQueue(1, state=TaskExec.States.QUEUED) 68 | management.call_command("worker", "--once") 69 | self.assertQueue(1, state=TaskExec.States.INVALID) 70 | 71 | # We can run registered functions 72 | @register_task(name="a") 73 | def a(x): 74 | pass 75 | 76 | TaskExec.objects.create(task_name="a", args=["test"]) 77 | self.assertQueue(1, state=TaskExec.States.QUEUED) 78 | management.call_command("worker", "--once") 79 | self.assertQueue(1, state=TaskExec.States.SUCCEEDED) 80 | 81 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 82 | def test_task_queuing(self, frozen_datetime): 83 | """Checking task queuing""" 84 | 85 | @register_task(name="a") 86 | def a(x): 87 | return x * 2 88 | 89 | t1 = a.queue(1) 90 | t2 = a.queue(2) 91 | t3 = a.queue(3, due=timezone.make_aware(datetime.datetime(2020, 1, 1, 2))) 92 | t4 = a.queue(4, due=timezone.make_aware(datetime.datetime(2020, 1, 1, 1))) 93 | self.assertEqual(t1.state, TaskExec.States.QUEUED) 94 | self.assertEqual(t2.state, TaskExec.States.QUEUED) 95 | self.assertEqual(t3.state, TaskExec.States.SLEEPING) 96 | self.assertEqual(t4.state, TaskExec.States.SLEEPING) 97 | self.assertQueue(4) 98 | 99 | # Run a due task 100 | management.call_command("worker", "--once") 101 | t1.refresh_from_db() 102 | t2.refresh_from_db() 103 | t3.refresh_from_db() 104 | t4.refresh_from_db() 105 | self.assertEqual(t1.state, TaskExec.States.SUCCEEDED) 106 | self.assertEqual(t2.state, TaskExec.States.QUEUED) 107 | self.assertEqual(t3.state, TaskExec.States.SLEEPING) 108 | self.assertEqual(t4.state, TaskExec.States.SLEEPING) 109 | 110 | # Run a due task 111 | management.call_command("worker", "--once") 112 | t1.refresh_from_db() 113 | t2.refresh_from_db() 114 | t3.refresh_from_db() 115 | t4.refresh_from_db() 116 | self.assertEqual(t1.state, TaskExec.States.SUCCEEDED) 117 | self.assertEqual(t2.state, TaskExec.States.SUCCEEDED) 118 | self.assertEqual(t3.state, TaskExec.States.SLEEPING) 119 | self.assertEqual(t4.state, TaskExec.States.SLEEPING) 120 | 121 | # All currently due tasks have been run, nothing happens 122 | management.call_command("worker", "--once") 123 | t1.refresh_from_db() 124 | t2.refresh_from_db() 125 | t3.refresh_from_db() 126 | t4.refresh_from_db() 127 | self.assertEqual(t1.state, TaskExec.States.SUCCEEDED) 128 | self.assertEqual(t2.state, TaskExec.States.SUCCEEDED) 129 | self.assertEqual(t3.state, TaskExec.States.SLEEPING) 130 | self.assertEqual(t4.state, TaskExec.States.SLEEPING) 131 | 132 | # We move to the future, due tasks are now queued, and the first due one is run 133 | frozen_datetime.move_to(datetime.datetime(2020, 1, 1, 5)) 134 | management.call_command("worker", "--once") 135 | t1.refresh_from_db() 136 | t2.refresh_from_db() 137 | t3.refresh_from_db() 138 | t4.refresh_from_db() 139 | self.assertEqual(t1.state, TaskExec.States.SUCCEEDED) 140 | self.assertEqual(t2.state, TaskExec.States.SUCCEEDED) 141 | self.assertEqual(t3.state, TaskExec.States.QUEUED) 142 | self.assertEqual(t4.state, TaskExec.States.SUCCEEDED) 143 | 144 | # Now the last one is run too 145 | management.call_command("worker", "--once") 146 | t1.refresh_from_db() 147 | t2.refresh_from_db() 148 | t3.refresh_from_db() 149 | t4.refresh_from_db() 150 | self.assertEqual(t1.state, TaskExec.States.SUCCEEDED) 151 | self.assertEqual(t2.state, TaskExec.States.SUCCEEDED) 152 | self.assertEqual(t3.state, TaskExec.States.SUCCEEDED) 153 | self.assertEqual(t4.state, TaskExec.States.SUCCEEDED) 154 | 155 | def test_task_queuing_with_priorities(self): 156 | """Checking task queuing with priorities""" 157 | 158 | @register_task(name="p2", priority=2) 159 | def p2(x): 160 | return x * 2 161 | 162 | @register_task(name="p1a", priority=1) 163 | def p1a(x): 164 | return x * 2 165 | 166 | @register_task(name="p1b", priority=1) 167 | def p1b(x): 168 | return x * 2 169 | 170 | p1a.queue(1) 171 | p1b.queue(1) 172 | p2.queue(1) 173 | 174 | self.assertQueue(1, task_name="p2", state=TaskExec.States.QUEUED) 175 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.QUEUED) 176 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.QUEUED) 177 | self.assertQueue(0, task_name="p2", state=TaskExec.States.SUCCEEDED) 178 | self.assertQueue(0, task_name="p1a", state=TaskExec.States.SUCCEEDED) 179 | self.assertQueue(0, task_name="p1b", state=TaskExec.States.SUCCEEDED) 180 | self.assertQueue(3) 181 | 182 | management.call_command("worker", "--once") 183 | 184 | self.assertQueue(0, task_name="p2", state=TaskExec.States.QUEUED) 185 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.QUEUED) 186 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.QUEUED) 187 | self.assertQueue(1, task_name="p2", state=TaskExec.States.SUCCEEDED) 188 | self.assertQueue(0, task_name="p1a", state=TaskExec.States.SUCCEEDED) 189 | self.assertQueue(0, task_name="p1b", state=TaskExec.States.SUCCEEDED) 190 | self.assertQueue(3) 191 | 192 | management.call_command("worker", "--once") 193 | 194 | self.assertQueue(0, task_name="p2", state=TaskExec.States.QUEUED) 195 | self.assertQueue(0, task_name="p1a", state=TaskExec.States.QUEUED) 196 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.QUEUED) 197 | self.assertQueue(1, task_name="p2", state=TaskExec.States.SUCCEEDED) 198 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.SUCCEEDED) 199 | self.assertQueue(0, task_name="p1b", state=TaskExec.States.SUCCEEDED) 200 | self.assertQueue(3) 201 | 202 | management.call_command("worker", "--once") 203 | 204 | self.assertQueue(0, task_name="p2", state=TaskExec.States.QUEUED) 205 | self.assertQueue(0, task_name="p1a", state=TaskExec.States.QUEUED) 206 | self.assertQueue(0, task_name="p1b", state=TaskExec.States.QUEUED) 207 | self.assertQueue(1, task_name="p2", state=TaskExec.States.SUCCEEDED) 208 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.SUCCEEDED) 209 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.SUCCEEDED) 210 | self.assertQueue(3) 211 | 212 | p2.queue(1) 213 | p1b.queue(1) 214 | p1a.queue(1) 215 | 216 | self.assertQueue(1, task_name="p2", state=TaskExec.States.QUEUED) 217 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.QUEUED) 218 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.QUEUED) 219 | self.assertQueue(1, task_name="p2", state=TaskExec.States.SUCCEEDED) 220 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.SUCCEEDED) 221 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.SUCCEEDED) 222 | self.assertQueue(6) 223 | 224 | management.call_command("worker", "--once") 225 | 226 | self.assertQueue(0, task_name="p2", state=TaskExec.States.QUEUED) 227 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.QUEUED) 228 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.QUEUED) 229 | self.assertQueue(2, task_name="p2", state=TaskExec.States.SUCCEEDED) 230 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.SUCCEEDED) 231 | self.assertQueue(1, task_name="p1b", state=TaskExec.States.SUCCEEDED) 232 | self.assertQueue(6) 233 | 234 | management.call_command("worker", "--once") 235 | 236 | self.assertQueue(0, task_name="p2", state=TaskExec.States.QUEUED) 237 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.QUEUED) 238 | self.assertQueue(0, task_name="p1b", state=TaskExec.States.QUEUED) 239 | self.assertQueue(2, task_name="p2", state=TaskExec.States.SUCCEEDED) 240 | self.assertQueue(1, task_name="p1a", state=TaskExec.States.SUCCEEDED) 241 | self.assertQueue(2, task_name="p1b", state=TaskExec.States.SUCCEEDED) 242 | self.assertQueue(6) 243 | 244 | management.call_command("worker", "--once") 245 | 246 | self.assertQueue(0, task_name="p2", state=TaskExec.States.QUEUED) 247 | self.assertQueue(0, task_name="p1a", state=TaskExec.States.QUEUED) 248 | self.assertQueue(0, task_name="p1b", state=TaskExec.States.QUEUED) 249 | self.assertQueue(2, task_name="p2", state=TaskExec.States.SUCCEEDED) 250 | self.assertQueue(2, task_name="p1a", state=TaskExec.States.SUCCEEDED) 251 | self.assertQueue(2, task_name="p1b", state=TaskExec.States.SUCCEEDED) 252 | self.assertQueue(6) 253 | 254 | def test_task_queuing_with_unique(self): 255 | """Checking task queuing with unique""" 256 | 257 | @register_task(name="normal") 258 | def normal(x): 259 | return x * 2 260 | 261 | @register_task(name="unique", unique=True) 262 | def unique(x): 263 | return x * 2 264 | 265 | self.assertQueue(0) 266 | 267 | normal.queue(1) 268 | normal.queue(1) 269 | unique.queue(1) 270 | unique.queue(1) 271 | 272 | self.assertQueue(1, task_name="unique", state=TaskExec.States.QUEUED) 273 | self.assertQueue(2, task_name="normal", state=TaskExec.States.QUEUED) 274 | self.assertQueue(0, task_name="unique", state=TaskExec.States.SUCCEEDED) 275 | self.assertQueue(0, task_name="normal", state=TaskExec.States.SUCCEEDED) 276 | self.assertQueue(3) 277 | 278 | management.call_command("worker", "--until_done") 279 | 280 | self.assertQueue(0, task_name="unique", state=TaskExec.States.QUEUED) 281 | self.assertQueue(0, task_name="normal", state=TaskExec.States.QUEUED) 282 | self.assertQueue(1, task_name="unique", state=TaskExec.States.SUCCEEDED) 283 | self.assertQueue(2, task_name="normal", state=TaskExec.States.SUCCEEDED) 284 | self.assertQueue(3) 285 | 286 | normal.queue(1) 287 | normal.queue(1) 288 | unique.queue(1) 289 | unique.queue(1) 290 | 291 | self.assertQueue(1, task_name="unique", state=TaskExec.States.QUEUED) 292 | self.assertQueue(2, task_name="normal", state=TaskExec.States.QUEUED) 293 | self.assertQueue(1, task_name="unique", state=TaskExec.States.SUCCEEDED) 294 | self.assertQueue(2, task_name="normal", state=TaskExec.States.SUCCEEDED) 295 | self.assertQueue(6) 296 | 297 | def test_task_retries(self): 298 | """Checking task retries""" 299 | 300 | @register_task(name="div_zero", retries=10) 301 | def div_zero(x): 302 | return x / 0 303 | 304 | self.assertQueue(0) 305 | 306 | div_zero.queue(1) 307 | 308 | management.call_command("worker", "--until_done") 309 | 310 | self.assertQueue(0, task_name="div_zero", state=TaskExec.States.QUEUED) 311 | self.assertQueue(0, task_name="div_zero", state=TaskExec.States.SLEEPING) 312 | self.assertQueue( 313 | 1, task_name="div_zero", state=TaskExec.States.FAILED, replaced=False 314 | ) 315 | self.assertQueue( 316 | 10, task_name="div_zero", state=TaskExec.States.FAILED, replaced=True 317 | ) 318 | self.assertQueue(11) 319 | 320 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 321 | def test_task_retries_delay(self, frozen_datetime): 322 | """Checking task retries with delay""" 323 | 324 | @register_task(name="div_zero", retries=3, retry_delay=60) 325 | def div_zero(x): 326 | return x / 0 327 | 328 | self.assertQueue(0) 329 | 330 | # Create the task 331 | div_zero.queue(1) 332 | self.assertQueue( 333 | 1, 334 | task_name="div_zero", 335 | state=TaskExec.States.QUEUED, 336 | replaced=False, 337 | due=datetime.datetime(2020, 1, 1, 0, 0), 338 | ) 339 | self.assertQueue(1) 340 | 341 | # the task failed, it should be replaced with a task due in the future (+1 min) 342 | management.call_command("worker", "--until_done") 343 | self.assertQueue( 344 | 1, 345 | task_name="div_zero", 346 | state=TaskExec.States.FAILED, 347 | replaced=True, 348 | due=datetime.datetime(2020, 1, 1, 0, 0), 349 | ) 350 | self.assertQueue( 351 | 1, 352 | task_name="div_zero", 353 | state=TaskExec.States.SLEEPING, 354 | replaced=False, 355 | due=datetime.datetime(2020, 1, 1, 0, 1), 356 | ) 357 | self.assertQueue(2) 358 | 359 | # if we don't wait, no further task will be processed 360 | management.call_command("worker", "--until_done") 361 | self.assertQueue( 362 | 1, 363 | task_name="div_zero", 364 | state=TaskExec.States.FAILED, 365 | replaced=True, 366 | due=datetime.datetime(2020, 1, 1, 0, 0), 367 | ) 368 | self.assertQueue( 369 | 1, 370 | task_name="div_zero", 371 | state=TaskExec.States.SLEEPING, 372 | replaced=False, 373 | due=datetime.datetime(2020, 1, 1, 0, 1), 374 | ) 375 | self.assertQueue(2) 376 | 377 | # if we wait, one retry will be done ( +1 + 2*+1 = +3min) 378 | frozen_datetime.move_to(datetime.datetime(2020, 1, 1, 0, 1)) 379 | management.call_command("worker", "--until_done") 380 | self.assertQueue( 381 | 1, 382 | task_name="div_zero", 383 | state=TaskExec.States.FAILED, 384 | replaced=True, 385 | due=datetime.datetime(2020, 1, 1, 0, 0), 386 | ) 387 | self.assertQueue( 388 | 1, 389 | task_name="div_zero", 390 | state=TaskExec.States.FAILED, 391 | replaced=True, 392 | due=datetime.datetime(2020, 1, 1, 0, 1), 393 | ) 394 | self.assertQueue( 395 | 1, 396 | task_name="div_zero", 397 | state=TaskExec.States.SLEEPING, 398 | replaced=False, 399 | due=datetime.datetime(2020, 1, 1, 0, 3), 400 | ) 401 | self.assertQueue(3) 402 | 403 | # if we wait more, delay continues to increase ( +1 + 2*+1 + 2*2*+1 = +7min) 404 | frozen_datetime.move_to(datetime.datetime(2020, 1, 1, 0, 3)) 405 | management.call_command("worker", "--until_done") 406 | self.assertQueue( 407 | 1, 408 | task_name="div_zero", 409 | state=TaskExec.States.FAILED, 410 | replaced=True, 411 | due=datetime.datetime(2020, 1, 1, 0, 0), 412 | ) 413 | self.assertQueue( 414 | 1, 415 | task_name="div_zero", 416 | state=TaskExec.States.FAILED, 417 | replaced=True, 418 | due=datetime.datetime(2020, 1, 1, 0, 1), 419 | ) 420 | self.assertQueue( 421 | 1, 422 | task_name="div_zero", 423 | state=TaskExec.States.FAILED, 424 | replaced=True, 425 | due=datetime.datetime(2020, 1, 1, 0, 3), 426 | ) 427 | self.assertQueue( 428 | 1, 429 | task_name="div_zero", 430 | state=TaskExec.States.SLEEPING, 431 | replaced=False, 432 | due=datetime.datetime(2020, 1, 1, 0, 7), 433 | ) 434 | self.assertQueue(4) 435 | 436 | # if we wait more, last task runs, but we're out of retries, so no new task 437 | frozen_datetime.move_to(datetime.datetime(2020, 1, 1, 0, 7)) 438 | management.call_command("worker", "--until_done") 439 | self.assertQueue( 440 | 1, 441 | task_name="div_zero", 442 | state=TaskExec.States.FAILED, 443 | replaced=True, 444 | due=datetime.datetime(2020, 1, 1, 0, 0), 445 | ) 446 | self.assertQueue( 447 | 1, 448 | task_name="div_zero", 449 | state=TaskExec.States.FAILED, 450 | replaced=True, 451 | due=datetime.datetime(2020, 1, 1, 0, 1), 452 | ) 453 | self.assertQueue( 454 | 1, 455 | task_name="div_zero", 456 | state=TaskExec.States.FAILED, 457 | replaced=True, 458 | due=datetime.datetime(2020, 1, 1, 0, 3), 459 | ) 460 | self.assertQueue( 461 | 1, 462 | task_name="div_zero", 463 | state=TaskExec.States.FAILED, 464 | replaced=False, 465 | due=datetime.datetime(2020, 1, 1, 0, 7), 466 | ) 467 | self.assertQueue(4) 468 | 469 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 470 | def test_task_retries_delay_unique(self, frozen_datetime): 471 | """Checking unique task retries with delay""" 472 | 473 | @register_task(name="div_zero", retries=10, retry_delay=10, unique=True) 474 | def div_zero(x): 475 | return x / 0 476 | 477 | self.assertQueue(0) 478 | 479 | div_zero.queue(1) 480 | 481 | self.assertQueue(1) 482 | 483 | management.call_command("worker", "--until_done") 484 | 485 | self.assertQueue(0, task_name="div_zero", state=TaskExec.States.QUEUED) 486 | self.assertQueue( 487 | 1, task_name="div_zero", state=TaskExec.States.FAILED, replaced=True 488 | ) 489 | self.assertQueue(1, task_name="div_zero", state=TaskExec.States.SLEEPING) 490 | self.assertQueue(2) 491 | self.assertEqual(TaskExec.objects.last().retries, 9) 492 | self.assertEqual(TaskExec.objects.last().retry_delay, 20) 493 | 494 | # if we don't wait, no further task will be processed 495 | management.call_command("worker", "--until_done") 496 | 497 | self.assertQueue(0, task_name="div_zero", state=TaskExec.States.QUEUED) 498 | self.assertQueue( 499 | 1, task_name="div_zero", state=TaskExec.States.FAILED, replaced=True 500 | ) 501 | self.assertQueue(1, task_name="div_zero", state=TaskExec.States.SLEEPING) 502 | self.assertQueue(2) 503 | self.assertEqual(TaskExec.objects.last().retries, 9) 504 | self.assertEqual(TaskExec.objects.last().retry_delay, 20) 505 | 506 | # if we requeue the task, it will be run immediatly 507 | div_zero.queue(1) 508 | self.assertQueue(2) 509 | 510 | self.assertQueue(1, task_name="div_zero", state=TaskExec.States.QUEUED) 511 | self.assertQueue( 512 | 1, task_name="div_zero", state=TaskExec.States.FAILED, replaced=True 513 | ) 514 | self.assertQueue(0, task_name="div_zero", state=TaskExec.States.SLEEPING) 515 | self.assertQueue(2) 516 | self.assertEqual(TaskExec.objects.last().retries, 9) 517 | self.assertEqual(TaskExec.objects.last().retry_delay, 20) 518 | 519 | management.call_command("worker", "--until_done") 520 | 521 | self.assertQueue(0, task_name="div_zero", state=TaskExec.States.QUEUED) 522 | self.assertQueue( 523 | 2, task_name="div_zero", state=TaskExec.States.FAILED, replaced=True 524 | ) 525 | self.assertQueue(1, task_name="div_zero", state=TaskExec.States.SLEEPING) 526 | self.assertQueue(3) 527 | self.assertEqual(TaskExec.objects.last().retries, 8) 528 | self.assertEqual(TaskExec.objects.last().retry_delay, 40) 529 | 530 | @freeze_time("2020-01-01", as_kwarg="frozen_datetime") 531 | def test_task_due_date(self, frozen_datetime): 532 | """Checking unique task retries with delay""" 533 | 534 | @register_task(name="my_task", unique=True) 535 | def my_task(x): 536 | return x * 2 537 | 538 | def getDues(): 539 | return [ 540 | (t.state, f"{t.due:%Y-%m-%d %H:%M}") 541 | for t in TaskExec.objects.order_by("due") 542 | ] 543 | 544 | # Normal queue is due right now 545 | my_task.queue(1) 546 | self.assertEqual( 547 | getDues(), 548 | [ 549 | (TaskExec.States.QUEUED.value, "2020-01-01 00:00"), 550 | ], 551 | ) 552 | management.call_command("worker", "--until_done") 553 | self.assertEqual( 554 | getDues(), 555 | [ 556 | (TaskExec.States.SUCCEEDED.value, "2020-01-01 00:00"), 557 | ], 558 | ) 559 | 560 | # Delayed queue is due in the future 561 | my_task.queue(1, due=timezone.now() + datetime.timedelta(hours=3)) 562 | self.assertEqual( 563 | getDues(), 564 | [ 565 | (TaskExec.States.SUCCEEDED.value, "2020-01-01 00:00"), 566 | (TaskExec.States.SLEEPING.value, "2020-01-01 03:00"), 567 | ], 568 | ) 569 | 570 | # Delayed closer reduces the due date 571 | my_task.queue(1, due=timezone.now() + datetime.timedelta(hours=2)) 572 | self.assertEqual( 573 | getDues(), 574 | [ 575 | (TaskExec.States.SUCCEEDED.value, "2020-01-01 00:00"), 576 | (TaskExec.States.SLEEPING.value, "2020-01-01 02:00"), 577 | ], 578 | ) 579 | 580 | def test_named_queues(self): 581 | """Checking named queues""" 582 | 583 | @register_task(name="a") # queue="default" 584 | def a(x): 585 | return x * 2 586 | 587 | @register_task(name="b", queue="queue_b") 588 | def b(x): 589 | return x * 2 590 | 591 | @register_task(name="c", queue="queue_c") 592 | def c(x): 593 | return x * 2 594 | 595 | @register_task(name="d", queue="queue_d") 596 | def d(x): 597 | return x * 2 598 | 599 | @register_task(name="e", queue="queue_e") 600 | def e(x): 601 | return x * 2 602 | 603 | @register_task(name="f", queue="queue_f") 604 | def f(x): 605 | return x * 2 606 | 607 | @register_task(name="g", queue="queue_g") 608 | def g(x): 609 | return x * 2 610 | 611 | @register_task(name="h", queue="queue_h") 612 | def h(x): 613 | return x * 2 614 | 615 | task_a = a.queue(1) 616 | task_b = b.queue(1) 617 | task_c = c.queue(1) 618 | task_d = d.queue(1) 619 | task_e = e.queue(1) 620 | task_f = f.queue(1) 621 | task_g = g.queue(1) 622 | task_h = h.queue(1) 623 | 624 | self.assertTask(task_a, TaskExec.States.QUEUED) 625 | self.assertTask(task_b, TaskExec.States.QUEUED) 626 | self.assertTask(task_c, TaskExec.States.QUEUED) 627 | self.assertTask(task_d, TaskExec.States.QUEUED) 628 | self.assertTask(task_e, TaskExec.States.QUEUED) 629 | self.assertTask(task_f, TaskExec.States.QUEUED) 630 | self.assertTask(task_g, TaskExec.States.QUEUED) 631 | self.assertTask(task_h, TaskExec.States.QUEUED) 632 | 633 | # make sure tasks get assigned to default queue by default 634 | management.call_command("worker", "--until_done", "--queue", "default") 635 | 636 | self.assertTask(task_a, TaskExec.States.SUCCEEDED) 637 | self.assertTask(task_b, TaskExec.States.QUEUED) 638 | self.assertTask(task_c, TaskExec.States.QUEUED) 639 | self.assertTask(task_d, TaskExec.States.QUEUED) 640 | self.assertTask(task_e, TaskExec.States.QUEUED) 641 | self.assertTask(task_f, TaskExec.States.QUEUED) 642 | self.assertTask(task_g, TaskExec.States.QUEUED) 643 | self.assertTask(task_h, TaskExec.States.QUEUED) 644 | 645 | # make sure worker only runs their queue 646 | management.call_command("worker", "--until_done", "--queue", "queue_c") 647 | 648 | self.assertTask(task_a, TaskExec.States.SUCCEEDED) 649 | self.assertTask(task_b, TaskExec.States.QUEUED) 650 | self.assertTask(task_c, TaskExec.States.SUCCEEDED) 651 | self.assertTask(task_d, TaskExec.States.QUEUED) 652 | self.assertTask(task_e, TaskExec.States.QUEUED) 653 | self.assertTask(task_f, TaskExec.States.QUEUED) 654 | self.assertTask(task_g, TaskExec.States.QUEUED) 655 | self.assertTask(task_h, TaskExec.States.QUEUED) 656 | 657 | # make sure worker can run multiple queues 658 | management.call_command( 659 | "worker", "--until_done", "--queue", "queue_b", "--queue", "queue_d" 660 | ) 661 | 662 | self.assertTask(task_a, TaskExec.States.SUCCEEDED) 663 | self.assertTask(task_b, TaskExec.States.SUCCEEDED) 664 | self.assertTask(task_c, TaskExec.States.SUCCEEDED) 665 | self.assertTask(task_d, TaskExec.States.SUCCEEDED) 666 | self.assertTask(task_e, TaskExec.States.QUEUED) 667 | self.assertTask(task_f, TaskExec.States.QUEUED) 668 | self.assertTask(task_g, TaskExec.States.QUEUED) 669 | self.assertTask(task_h, TaskExec.States.QUEUED) 670 | 671 | # make sure worker exclude queue works 672 | management.call_command( 673 | "worker", 674 | "--until_done", 675 | "--exclude_queue", 676 | "queue_g", 677 | "--exclude_queue", 678 | "queue_h", 679 | ) 680 | 681 | self.assertTask(task_a, TaskExec.States.SUCCEEDED) 682 | self.assertTask(task_b, TaskExec.States.SUCCEEDED) 683 | self.assertTask(task_c, TaskExec.States.SUCCEEDED) 684 | self.assertTask(task_d, TaskExec.States.SUCCEEDED) 685 | self.assertTask(task_e, TaskExec.States.SUCCEEDED) 686 | self.assertTask(task_f, TaskExec.States.SUCCEEDED) 687 | self.assertTask(task_g, TaskExec.States.QUEUED) 688 | self.assertTask(task_h, TaskExec.States.QUEUED) 689 | 690 | # make sure worker run all queues by default 691 | management.call_command("worker", "--until_done") 692 | 693 | self.assertTask(task_a, TaskExec.States.SUCCEEDED) 694 | self.assertTask(task_b, TaskExec.States.SUCCEEDED) 695 | self.assertTask(task_c, TaskExec.States.SUCCEEDED) 696 | self.assertTask(task_d, TaskExec.States.SUCCEEDED) 697 | self.assertTask(task_e, TaskExec.States.SUCCEEDED) 698 | self.assertTask(task_f, TaskExec.States.SUCCEEDED) 699 | self.assertTask(task_g, TaskExec.States.SUCCEEDED) 700 | self.assertTask(task_h, TaskExec.States.SUCCEEDED) 701 | --------------------------------------------------------------------------------