├── .coveragerc ├── .flake8 ├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── README.md ├── djtriggers ├── __init__.py ├── checks.py ├── exceptions.py ├── locking.py ├── loggers │ ├── __init__.py │ ├── base.py │ ├── database.py │ └── python_logging.py ├── logic.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── clean_triggers.py │ │ └── process_triggers.py ├── managers.py ├── migrations │ ├── 0001_initial.py │ ├── 0001_squashed_0006_auto_20171003_0945.py │ ├── 0002_auto_20151208_1454.py │ ├── 0003_auto_20160512_0847.py │ ├── 0004_auto_20170216_1007.py │ ├── 0005_trigger_successful.py │ ├── 0006_auto_20171003_0945.py │ ├── 0007_auto_20201001_1530.py │ └── __init__.py ├── models.py ├── tasks.py └── tests │ ├── __init__.py │ ├── factories │ ├── __init__.py │ └── triggers.py │ ├── models.py │ ├── test_dummy.py │ ├── test_logic.py │ └── test_models.py ├── manage.py ├── pyproject.toml ├── pytest.ini ├── requirements └── requirements_test.txt └── settings_test.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = locking/ 4 | omit = 5 | # Ignore migrations 6 | */migrations/* 7 | 8 | # Ignore requirement files 9 | requirements/* 10 | 11 | # Ignore manage files 12 | manage* 13 | 14 | # Ignore admin files 15 | djtriggers/*/admin.py 16 | [report] 17 | exclude_lines= 18 | pragma: no cover 19 | def __repr__ 20 | def __iter__ 21 | def __str__ 22 | def __unicode__ 23 | #fail_under = 99 24 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ; We use a line length of 120 instead of pep8's default 79 3 | max-line-length = 120 4 | ; Files not checked: 5 | ; - migrations: most of these are autogenerated and don't need a check 6 | ; - manage: these are autogenerated and don't need a check 7 | exclude = *migrations,manage.py,venv,dist,build 8 | ; Cyclomatic complexity check with mccabe. Should be pushed down further. 9 | max-complexity = 30 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ######## 2 | # Python 3 | ######## 4 | *.py[co] 5 | 6 | # Packages 7 | .cache 8 | *.egg 9 | *.eggs 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | 28 | #Mr Developer 29 | .mr.developer.cfg 30 | 31 | # Django extras 32 | *.log 33 | *.pot 34 | 35 | # backup files 36 | *.bak 37 | .*.sw[a-z] 38 | *.un~ 39 | *.po-backup* 40 | 41 | ######### 42 | # Eclipse 43 | ######### 44 | *.pydevproject 45 | .project 46 | .metadata 47 | bin/** 48 | tmp/** 49 | tmp/**/* 50 | *.tmp 51 | *.bak 52 | *.swp 53 | *~.nib 54 | local.properties 55 | .classpath 56 | .settings/ 57 | .loadpath 58 | 59 | # CDT-specific 60 | .cproject 61 | 62 | # virtualenv 63 | .venv/ 64 | 65 | ######### 66 | # PyCharm 67 | ######### 68 | 69 | .idea/ 70 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: xenial 3 | env: 4 | DJANGO='>=2.2 5 | python: 6 | - "3.8" 7 | install: 8 | - pip install --upgrade pip 9 | - pip install "Django${DJANGO}" 10 | - pip install -e . 11 | - pip install -r requirements/requirements_test.txt 12 | before_script: 13 | flake8 djtriggers/ 14 | script: 15 | pytest -v --capture=sys --cov=djtriggers/ djtriggers/ --cov-report term-missing:skip-covered 16 | after_success: 17 | coveralls 18 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Unleashed NV 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Coverage Status](https://coveralls.io/repos/github/vikingco/django-triggers/badge.svg)](https://coveralls.io/github/vikingco/django-triggers) 2 | [![CI Status](https://travis-ci.org/vikingco/django-triggers.svg?branch=master)](https://travis-ci.org/vikingco/django-triggers) 3 | 4 | About 5 | ----- 6 | 7 | Django Triggers is a light-weight framework for having one part of an 8 | application generate a trigger while another part responds to to it. 9 | Triggers are persistent and can be scheduled to be processed at a later 10 | time. 11 | 12 | Usage 13 | ----- 14 | 15 | Triggers are defined by subclassing the `Trigger` model. `Trigger` defines 16 | common data structures and logic for all child triggers. The only thing a 17 | child should have to do is override the `_process` method and set `typed` to 18 | a unique slug. 19 | 20 | Settings 21 | -------- 22 | 23 | The following settings are used: 24 | - `DJTRIGGERS_TRIES_BEFORE_WARNING`: the number of times a task can be retried before a warning is logged. Defaults to 3. 25 | - `DJTRIGGERS_TRIES_BEFORE_ERROR`: the number of times a task can be retried before an error is raised. Defaults to 5. 26 | - `DJTRIGGERS_ASYNC_HANDLING`: whether processing should be asynchronous (using Celery) or not. Default to False. 27 | - `DJTRIGGERS_CELERY_TASK_MAX_RETRIES`: the number of times the Celery task for a trigger should be retried. Defaults to 0. 28 | - `DJTRIGGERS_TYPE_TO_TABLE`: mapping of trigger types to database tables. Used for the cleanup script. Defaults to `{}`. 29 | - `DJTRIGGERS_REDIS_URL`: the URL of the Redis instance used for locks. 30 | - `DJTRIGGERS_LOGGERS`: separate logging config for django-triggers. Defaults to `()`. 31 | 32 | 33 | Examples 34 | -------- 35 | 36 | General use 37 | =========== 38 | 39 | ```python 40 | 41 | from djtriggers.models import Trigger 42 | 43 | class BreakfastTrigger(Trigger): 44 | class Meta: 45 | # There is no trigger specific data so make a proxy model. 46 | # This ensures no additional db table is created. 47 | proxy = True 48 | typed = 'breakfast' 49 | 50 | def _process(self, dictionary={}): 51 | prepare_toast() 52 | prepare_juice() 53 | eat() 54 | 55 | ``` 56 | 57 | Trigger specific data 58 | ===================== 59 | 60 | ```python 61 | 62 | from djtriggers.models import Trigger 63 | 64 | class PayBill(Trigger): 65 | class Meta: 66 | # We need a regular model as the trigger specific data needs a 67 | # place to live in the db. 68 | proxy = False 69 | 70 | amount = models.IntegerField() 71 | recipient = models.ForeignKey(User) 72 | 73 | def _process(self, dictionary={}): 74 | amount = self.amount 75 | recipient = self.recipient 76 | check_balance() 77 | pay_bill(amount, recipient) 78 | 79 | ``` 80 | 81 | Trigger processing 82 | ================== 83 | 84 | ```python 85 | 86 | from .models import BreakfastTrigger 87 | from .exceptions import ProcessError 88 | 89 | trigger = BreakfastTrigger.objects.get(pk=1) 90 | try: 91 | trigger.process() 92 | except ProcessError as e: 93 | report_error(e) 94 | 95 | ``` 96 | 97 | Delayed processing 98 | ================== 99 | 100 | ```python 101 | 102 | from .models import BreakfastTrigger 103 | 104 | trigger = BreakfastTrigger() 105 | # Process 8 hours later (this can be any datetime) 106 | trigger.process_after = now() + timedelta(hour=8) 107 | 108 | ``` 109 | -------------------------------------------------------------------------------- /djtriggers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vikingco/django-triggers/807c34a7c1c0c3b8df67583ddbdd33e8c2c41399/djtriggers/__init__.py -------------------------------------------------------------------------------- /djtriggers/checks.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | 3 | logger = getLogger(__name__) 4 | 5 | 6 | def run_checks(checks): 7 | """ 8 | Run a number of checks. 9 | 10 | :param tuple checks: a tuple of tuples, with check name and parameters dict. 11 | 12 | :returns: whether all checks succeeded, and the results of each check 13 | :rtype: tuple of (bool, dict) 14 | """ 15 | results = {} 16 | all_succeeded = True 17 | for check, kwargs in checks: 18 | succeeded = check(**kwargs).succeeded 19 | if isinstance(succeeded, bool): 20 | if not succeeded: 21 | all_succeeded = False 22 | results[check.__name__] = succeeded 23 | elif isinstance(succeeded, tuple): 24 | if not succeeded[0]: 25 | all_succeeded = False 26 | results[check.__name__] = succeeded 27 | return (all_succeeded, results) 28 | 29 | 30 | class Check(object): 31 | def run(self): 32 | self._result = self._run() 33 | return self._result 34 | 35 | def _run(self): 36 | raise NotImplementedError() 37 | 38 | @property 39 | def result(self): 40 | if not self.has_run: 41 | return self.run() 42 | return self._result 43 | 44 | @property 45 | def has_run(self): 46 | return hasattr(self, '_result') 47 | 48 | @property 49 | def succeeded(self): 50 | return bool(self.result) 51 | 52 | def __nonzero__(self): 53 | return self.succeeded 54 | -------------------------------------------------------------------------------- /djtriggers/exceptions.py: -------------------------------------------------------------------------------- 1 | class AlreadyProcessedError(Exception): 2 | pass 3 | 4 | 5 | class ProcessError(Exception): 6 | pass 7 | 8 | 9 | class ProcessLaterError(ProcessError): 10 | def __init__(self, process_after, *args): 11 | super(ProcessLaterError, self).__init__(*args) 12 | self.process_after = process_after 13 | -------------------------------------------------------------------------------- /djtriggers/locking.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from django.conf import settings 3 | from redis import Redis 4 | from redis.lock import Lock 5 | from typing import Generator 6 | 7 | 8 | @contextmanager 9 | def redis_lock(name: str, **kwargs) -> Generator: 10 | """ 11 | Acquire a Redis lock. This is a wrapper around redis.lock.Lock(), that also works in tests (there, the lock is 12 | always granted without any checks). 13 | 14 | Relevant kwargs are: 15 | - blocking_timeout: how many seconds to try to acquire the lock. Use 0 for a non-blocking lock. 16 | The default is None, which means we wait forever. 17 | - timeout: how many seconds to keep the lock for. The default is None, which means it remains locked forever. 18 | 19 | Raises redis.exceptions.LockError if the lock couldn't be acquired or released. 20 | """ 21 | if settings.DJTRIGGERS_REDIS_URL.startswith('redis'): # pragma: no cover 22 | with Lock(redis=Redis.from_url(settings.DJTRIGGERS_REDIS_URL), name=name, **kwargs): 23 | yield 24 | else: 25 | yield 26 | -------------------------------------------------------------------------------- /djtriggers/loggers/__init__.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | try: 3 | from django.urls import get_mod_func 4 | except ImportError: 5 | # For Django <2.0 6 | from django.core.urlresolvers import get_mod_func 7 | 8 | REGISTRY = {} 9 | 10 | loggers = getattr(settings, 'DJTRIGGERS_LOGGERS', ()) 11 | 12 | for entry in loggers: 13 | module_name, class_name = get_mod_func(entry) 14 | logger_class = getattr(__import__(module_name, {}, {}, ['']), class_name) 15 | instance = logger_class() 16 | REGISTRY[class_name] = instance 17 | 18 | 19 | def get_logger(slug): 20 | return REGISTRY.get(slug, None) 21 | -------------------------------------------------------------------------------- /djtriggers/loggers/base.py: -------------------------------------------------------------------------------- 1 | class TriggerLogger(object): 2 | """ 3 | A logger for the processing of a trigger. 4 | """ 5 | 6 | def log_result(self, trigger, message, level=None): 7 | """ 8 | Log the result of a trigger. 9 | :param level: 10 | """ 11 | pass 12 | 13 | def log_message(self, trigger, message, level=None): 14 | """ 15 | Log any message during the processing of a trigger. 16 | """ 17 | pass 18 | -------------------------------------------------------------------------------- /djtriggers/loggers/database.py: -------------------------------------------------------------------------------- 1 | from logging import log, info 2 | 3 | from djtriggers.loggers.base import TriggerLogger 4 | 5 | 6 | class DatabaseLogger(TriggerLogger): 7 | def log_result(self, trigger, message, level=None): 8 | from djtriggers.models import TriggerResult 9 | TriggerResult.objects.create(trigger=trigger, result=message) 10 | 11 | def log_message(self, trigger, message, level=None): 12 | if level: 13 | log(level, message) 14 | else: 15 | info(message) 16 | 17 | 18 | def _prettify(results): 19 | try: 20 | return '\n'.join([str(r) for r in results]) 21 | except TypeError: 22 | return str(results) 23 | 24 | 25 | class DatabaseSerializeLogger(DatabaseLogger): 26 | def log_result(self, trigger, results, level=None): 27 | if results is None: 28 | return 29 | 30 | from djtriggers.models import TriggerResult 31 | TriggerResult.objects.create(trigger=trigger, result=_prettify(results)) 32 | -------------------------------------------------------------------------------- /djtriggers/loggers/python_logging.py: -------------------------------------------------------------------------------- 1 | from djtriggers.loggers.base import TriggerLogger 2 | 3 | from logging import log, info 4 | 5 | 6 | class PythonLogger(TriggerLogger): 7 | """ 8 | Logger using the default python logger. 9 | """ 10 | def log_result(self, trigger, message, level=None): 11 | if level: 12 | log(level, message) 13 | else: 14 | info(message) 15 | 16 | def log_message(self, trigger, message, level=None): 17 | if level: 18 | log(level, message) 19 | else: 20 | info(message) 21 | -------------------------------------------------------------------------------- /djtriggers/logic.py: -------------------------------------------------------------------------------- 1 | from inspect import isabstract 2 | from logging import getLogger 3 | 4 | from dateutil.relativedelta import relativedelta 5 | 6 | from django.apps import apps 7 | from django.conf import settings 8 | from django.db import connections 9 | from django.db.models import Q 10 | from django.utils import timezone 11 | 12 | from .models import Trigger 13 | from .exceptions import ProcessError, ProcessLaterError 14 | from .tasks import process_trigger 15 | 16 | 17 | logger = getLogger(__name__) 18 | 19 | 20 | def process_triggers(use_statsd=False, function_logger=None): 21 | """ 22 | Process all triggers that are ready for processing. 23 | 24 | :param bool use_statsd: whether to use_statsd 25 | :return: None 26 | """ 27 | process_async = getattr(settings, 'DJTRIGGERS_ASYNC_HANDLING', False) 28 | 29 | # Get all triggers that need to be processed 30 | for model in apps.get_models(): 31 | # Check whether it's a trigger 32 | if not issubclass(model, Trigger) or getattr(model, 'typed', None) is None or isabstract(model): 33 | continue 34 | 35 | # Get all triggers of this type that need to be processed 36 | triggers = model.objects.filter(Q(process_after__isnull=True) | Q(process_after__lt=timezone.now()), 37 | date_processed__isnull=True) 38 | 39 | # Process each trigger 40 | for trigger in triggers: 41 | try: 42 | # Process the trigger, either synchronously or in a Celery task 43 | if process_async: 44 | process_trigger.apply_async((trigger.id, trigger._meta.app_label, trigger.__class__.__name__), 45 | {'use_statsd': use_statsd}, 46 | max_retries=getattr(settings, 'DJTRIGGERS_CELERY_TASK_MAX_RETRIES', 0)) 47 | else: 48 | trigger.process() 49 | 50 | # Send stats to statsd if necessary 51 | if use_statsd: 52 | from django_statsd.clients import statsd 53 | statsd.incr('triggers.{}.processed'.format(trigger.trigger_type)) 54 | if trigger.date_processed and trigger.process_after: 55 | statsd.timing('triggers.{}.process_delay_seconds'.format(trigger.trigger_type), 56 | (trigger.date_processed - trigger.process_after).total_seconds()) 57 | # The trigger didn't need processing yet 58 | except ProcessLaterError: 59 | pass 60 | # The trigger raised an (expected) error while processing 61 | except ProcessError: 62 | pass 63 | # In case a trigger got removed (manually or some process), deal with it 64 | except Trigger.DoesNotExist as e: 65 | logger.info(e) 66 | 67 | 68 | def clean_triggers(expiration_dt=None, type_to_table=None): 69 | """ 70 | Clean old processed triggers from the database. 71 | 72 | Args: 73 | expiration_dt (optional datetime): triggers processed before this timestamp will be cleaned up. 74 | Defaults to 2 months before the current time. 75 | type_to_table (optional dict): maps trigger type to database table name. 76 | Defaults to DJTRIGGERS_TYPE_TO_TABLE django setting. 77 | 78 | `type_to_table` contains has information about which trigger has information 79 | in which table. This setting is a dict with the trigger types as keys and two 80 | options for values: 81 | 82 | - a string containing the table where the trigger information is stored 83 | (with a trigger_ptr_id to link it) 84 | - a tuple containing elements of two possible types: 85 | - a string containing the table where the trigger information is stored 86 | (with a trigger_ptr_id to link it) 87 | - a tuple containing a tablename and an id field 88 | 89 | Example: 90 | 91 | {'simple_trigger': 'simple_trigger_table', 92 | 'complex_trigger': ('complex_trigger_table1', 'complex_trigger_table2'), 93 | 'complexer_trigger': (('complexer_trigger_table1', 'complexer_trigger_id'), 'complexer_trigger_table2'), 94 | } 95 | """ 96 | if expiration_dt is None: 97 | expiration_dt = timezone.now() - relativedelta(months=2) 98 | 99 | if type_to_table is None: 100 | type_to_table = getattr(settings, 'DJTRIGGERS_TYPE_TO_TABLE', {}) 101 | 102 | cursor = connections['default'].cursor() 103 | sentinel = object() 104 | nr_deleted = 0 105 | 106 | for trigger in Trigger.objects.filter(date_processed__lt=expiration_dt): 107 | # Delete custom trigger information 108 | table = type_to_table.get(trigger.trigger_type, sentinel) 109 | if isinstance(table, tuple): 110 | for t in table: 111 | if isinstance(t, tuple): 112 | cursor.execute('DELETE FROM {} WHERE {} = {}'.format(t[0], t[1], trigger.id)) 113 | else: 114 | cursor.execute('DELETE FROM {} WHERE trigger_ptr_id = {}'.format(t, trigger.id)) 115 | elif table != sentinel: 116 | cursor.execute('DELETE FROM {} WHERE trigger_ptr_id = {}'.format(table, trigger.id)) 117 | 118 | # Delete the trigger from the main table 119 | trigger.delete() 120 | nr_deleted += 1 121 | 122 | return nr_deleted 123 | -------------------------------------------------------------------------------- /djtriggers/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vikingco/django-triggers/807c34a7c1c0c3b8df67583ddbdd33e8c2c41399/djtriggers/management/__init__.py -------------------------------------------------------------------------------- /djtriggers/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vikingco/django-triggers/807c34a7c1c0c3b8df67583ddbdd33e8c2c41399/djtriggers/management/commands/__init__.py -------------------------------------------------------------------------------- /djtriggers/management/commands/clean_triggers.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | 3 | from django.core.management.base import NoArgsCommand 4 | 5 | from djtriggers.logic import clean_triggers 6 | 7 | 8 | logger = getLogger(__name__) 9 | 10 | 11 | class Command(NoArgsCommand): 12 | help = clean_triggers.__doc__.strip() 13 | 14 | def handle_noargs(self, **options): 15 | nr_deleted = clean_triggers() 16 | logger.info('Cleaned up %s expired triggers', nr_deleted) 17 | -------------------------------------------------------------------------------- /djtriggers/management/commands/process_triggers.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import NoArgsCommand 2 | 3 | from optparse import make_option 4 | 5 | from djtriggers import logic 6 | 7 | 8 | class Command(NoArgsCommand): 9 | option_list = NoArgsCommand.option_list + ( 10 | make_option('--use-statsd', dest='use_statsd', action='store_true', default=False, 11 | help='Send stats about processing to Statsd'), 12 | ) 13 | 14 | def handle_noargs(self, **options): 15 | """ 16 | Process all triggers in order of trigger type. This blocks while 17 | processing the triggers. 18 | """ 19 | logic.process_triggers(use_statsd=options['use_statsd']) 20 | -------------------------------------------------------------------------------- /djtriggers/managers.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class TriggerManager(models.Manager): 5 | def __init__(self, trigger_type): 6 | super(TriggerManager, self).__init__() 7 | self.trigger_type = trigger_type 8 | 9 | def get_queryset(self): 10 | qs = super(TriggerManager, self).get_queryset() 11 | if self.trigger_type: 12 | qs = qs.filter(trigger_type=self.trigger_type) 13 | return qs 14 | 15 | def get_unprocessed_triggers(self): 16 | qs = self.get_queryset() 17 | return qs.filter(date_processed__isnull=True) 18 | -------------------------------------------------------------------------------- /djtriggers/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='Trigger', 15 | fields=[ 16 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 17 | ('trigger_type', models.CharField(max_length=50, db_index=True)), 18 | ('source', models.CharField(max_length=250, null=True, blank=True)), 19 | ('date_received', models.DateTimeField()), 20 | ('date_processed', models.DateTimeField(db_index=True, null=True, blank=True)), 21 | ('process_after', models.DateTimeField(db_index=True, null=True, blank=True)), 22 | ], 23 | options={ 24 | }, 25 | bases=(models.Model,), 26 | ), 27 | migrations.CreateModel( 28 | name='TriggerResult', 29 | fields=[ 30 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 31 | ('result', models.TextField()), 32 | ('trigger', models.ForeignKey(to='djtriggers.Trigger', on_delete=models.CASCADE)), 33 | ], 34 | options={ 35 | }, 36 | bases=(models.Model,), 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /djtriggers/migrations/0001_squashed_0006_auto_20171003_0945.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.5 on 2017-10-03 09:51 3 | from __future__ import unicode_literals 4 | 5 | import datetime 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | import django.utils.timezone 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | replaces = [('djtriggers', '0001_initial'), ('djtriggers', '0002_auto_20151208_1454'), ('djtriggers', '0003_auto_20160512_0847'), ('djtriggers', '0004_auto_20170216_1007'), ('djtriggers', '0005_trigger_successful'), ('djtriggers', '0006_auto_20171003_0945')] 14 | 15 | initial = True 16 | 17 | dependencies = [ 18 | ] 19 | 20 | operations = [ 21 | migrations.CreateModel( 22 | name='Trigger', 23 | fields=[ 24 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 25 | ('trigger_type', models.CharField(db_index=True, max_length=50)), 26 | ('source', models.CharField(blank=True, max_length=150, null=True)), 27 | ('date_received', models.DateTimeField()), 28 | ('date_processed', models.DateTimeField(blank=True, db_index=True, null=True)), 29 | ('process_after', models.DateTimeField(blank=True, db_index=True, null=True)), 30 | ], 31 | ), 32 | migrations.CreateModel( 33 | name='TriggerResult', 34 | fields=[ 35 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 36 | ('result', models.TextField()), 37 | ('trigger', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='djtriggers.Trigger')), 38 | ], 39 | ), 40 | migrations.AlterField( 41 | model_name='trigger', 42 | name='date_received', 43 | field=models.DateTimeField(default=datetime.datetime.now), 44 | ), 45 | migrations.AlterField( 46 | model_name='trigger', 47 | name='source', 48 | field=models.CharField(blank=True, db_index=True, max_length=150, null=True), 49 | ), 50 | migrations.AddField( 51 | model_name='trigger', 52 | name='number_of_tries', 53 | field=models.IntegerField(default=0), 54 | ), 55 | migrations.AlterField( 56 | model_name='trigger', 57 | name='date_received', 58 | field=models.DateTimeField(default=django.utils.timezone.now), 59 | ), 60 | migrations.AddField( 61 | model_name='trigger', 62 | name='successful', 63 | field=models.NullBooleanField(default=None), 64 | ), 65 | migrations.AlterField( 66 | model_name='trigger', 67 | name='source', 68 | field=models.CharField(blank=True, db_index=True, max_length=150, null=True), 69 | ), 70 | ] 71 | -------------------------------------------------------------------------------- /djtriggers/migrations/0002_auto_20151208_1454.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import datetime 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('djtriggers', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='trigger', 17 | name='date_received', 18 | field=models.DateTimeField(default=datetime.datetime.now), 19 | preserve_default=True, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /djtriggers/migrations/0003_auto_20160512_0847.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('djtriggers', '0002_auto_20151208_1454'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='trigger', 16 | name='source', 17 | field=models.CharField(db_index=True, max_length=250, null=True, blank=True), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /djtriggers/migrations/0004_auto_20170216_1007.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.7 on 2017-02-16 10:07 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('djtriggers', '0003_auto_20160512_0847'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='trigger', 18 | name='number_of_tries', 19 | field=models.IntegerField(default=0), 20 | ), 21 | migrations.AlterField( 22 | model_name='trigger', 23 | name='date_received', 24 | field=models.DateTimeField(default=django.utils.timezone.now), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /djtriggers/migrations/0005_trigger_successful.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.13 on 2017-08-24 08:48 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('djtriggers', '0004_auto_20170216_1007'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='trigger', 17 | name='successful', 18 | field=models.NullBooleanField(default=None), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /djtriggers/migrations/0006_auto_20171003_0945.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.5 on 2017-10-03 09:45 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('djtriggers', '0005_trigger_successful'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='trigger', 17 | name='source', 18 | field=models.CharField(blank=True, db_index=True, max_length=150, null=True), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /djtriggers/migrations/0007_auto_20201001_1530.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.5 on 2017-10-03 09:45 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('djtriggers', '0006_auto_20171003_0945'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='trigger', 17 | name='successful', 18 | field=models.BooleanField(default=None, null=True), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /djtriggers/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vikingco/django-triggers/807c34a7c1c0c3b8df67583ddbdd33e8c2c41399/djtriggers/migrations/__init__.py -------------------------------------------------------------------------------- /djtriggers/models.py: -------------------------------------------------------------------------------- 1 | from logging import ERROR, WARNING 2 | from redis.exceptions import LockError 3 | 4 | from django.conf import settings 5 | from django.db import models 6 | from django.db.models.base import ModelBase 7 | from django.utils import timezone 8 | 9 | from .managers import TriggerManager 10 | from .exceptions import ProcessLaterError 11 | from .locking import redis_lock 12 | from .loggers import get_logger 13 | from .loggers.base import TriggerLogger 14 | 15 | 16 | class TriggerBase(ModelBase): 17 | """ 18 | A meta class for all Triggers. Adds a default manager that filters 19 | on type. 20 | """ 21 | def __new__(cls, name, bases, attrs): 22 | super_new = super(TriggerBase, cls).__new__ 23 | typed = attrs.pop('typed', None) 24 | if typed is not None: 25 | attrs['objects'] = TriggerManager(typed) 26 | new_class = super_new(cls, name, bases, attrs) 27 | if typed is None: 28 | return new_class 29 | 30 | new_class.typed = typed 31 | return new_class 32 | 33 | 34 | class Trigger(models.Model): 35 | """ 36 | A persistent Trigger that needs processing. 37 | 38 | Triggers are created when a certain state is reached internally 39 | or externally. Triggers are persistent and need processing after 40 | a certain point in time. 41 | 42 | To create a trigger start by subclassing Trigger and setting the 43 | 'typed' attribute. 'typed' should be a unique slug that identifies 44 | the trigger. Subclasses should also implement '_process()'. 45 | 46 | Subclasses can be proxy models when no extra data needs to be 47 | stored. Otherwise use regular subclassing. This will create an 48 | additional table with trigger specific data and a one-to-one 49 | relation to the triggers table. 50 | 51 | 'source' is a free-form field that can be used to uniquely determine 52 | the source of the trigger. 53 | 54 | There is a logging framework included. If passed a logger argument in the 55 | __init__, that logger will be used, otherwise the class' _logger_class 56 | determines the logger (if any). Default is no logging. 57 | """ 58 | __metaclass__ = TriggerBase 59 | 60 | # Set typed in a subclass to make it a typed trigger. 61 | typed = None 62 | 63 | trigger_type = models.CharField(max_length=50, db_index=True) 64 | source = models.CharField(max_length=150, null=True, blank=True, db_index=True) 65 | date_received = models.DateTimeField(default=timezone.now) 66 | date_processed = models.DateTimeField(null=True, blank=True, db_index=True) 67 | process_after = models.DateTimeField(null=True, blank=True, db_index=True) 68 | number_of_tries = models.IntegerField(default=0) 69 | successful = models.BooleanField(default=None, null=True) 70 | 71 | _logger_class = None 72 | 73 | def __init__(self, *args, **kwargs): 74 | super(Trigger, self).__init__(*args, **kwargs) 75 | if hasattr(self, 'typed'): 76 | self.trigger_type = self.typed 77 | 78 | if self._logger_class: 79 | self.logger = get_logger(self._logger_class) 80 | else: 81 | self.logger = TriggerLogger() 82 | 83 | def set_source(self, *args): 84 | args = [str(arg) for arg in args] 85 | self.source = '$'.join(args) 86 | 87 | def get_source(self): 88 | return tuple(x for x in self.source.split('$') if x != '') 89 | 90 | def process(self, force=False, logger=None, dictionary=None, use_statsd=False): 91 | """ 92 | Executes the Trigger 93 | :param bool force: force the execution 94 | :param string logger: slug of preferred logger 95 | :param dict dictionary: dictionary needed by trigger to execute 96 | :param bool use_statsd: whether to use statsd 97 | :return: None 98 | """ 99 | dictionary = {} if dictionary is None else {} 100 | 101 | # The task gets locked because multiple tasks in the queue can process the same trigger. 102 | # The lock assures no two tasks can process a trigger simultaneously. 103 | # The check for date_processed assures a trigger is not executed multiple times. 104 | try: 105 | with redis_lock('djtriggers-' + str(self.id), blocking_timeout=0): 106 | if logger: 107 | self.logger = get_logger(logger) 108 | now = timezone.now() 109 | if not force and self.date_processed is not None: 110 | # trigger has already been processed. So everything is fine 111 | return 112 | if not force and self.process_after and self.process_after >= now: 113 | raise ProcessLaterError(self.process_after) 114 | 115 | try: 116 | # execute trigger 117 | self.logger.log_result(self, self._process(dictionary)) 118 | self._handle_execution_success(use_statsd) 119 | except ProcessLaterError as e: 120 | self.process_after = e.process_after 121 | self.save() 122 | except Exception as e: 123 | self._handle_execution_failure(e, use_statsd) 124 | raise 125 | except LockError: 126 | pass 127 | 128 | def _process(self, dictionary): 129 | raise NotImplementedError() 130 | 131 | def _handle_execution_failure(self, exception, use_statsd=False): 132 | """ 133 | Handle execution failure of the trigger 134 | :param Exception exception: the exception raised during failure 135 | :param bool use_statsd: whether to use statsd 136 | :return: None 137 | """ 138 | self.number_of_tries += 1 139 | # Log message if starts retrying too much 140 | if self.number_of_tries > getattr(settings, 'DJTRIGGERS_TRIES_BEFORE_WARNING', 3): 141 | # Set a limit for retries 142 | if self.number_of_tries >= getattr(settings, 'DJTRIGGERS_TRIES_BEFORE_ERROR', 5): 143 | # Set date_processed so it doesn't retry anymore 144 | self.date_processed = timezone.now() 145 | self.successful = False 146 | level = ERROR 147 | else: 148 | level = WARNING 149 | 150 | message = 'Processing of {trigger_type} {trigger_key} raised a {exception_type} (try nr. {try_count})'.\ 151 | format(trigger_type=self.trigger_type, trigger_key=self.pk, exception_type=type(exception).__name__, 152 | try_count=self.number_of_tries) 153 | self.logger.log_message(self, message, level=level) 154 | 155 | # Send stats to statsd if necessary 156 | if use_statsd: 157 | from django_statsd.clients import statsd 158 | statsd.incr('triggers.{trigger_type}.failed'.format(trigger_type=self.trigger_type)) 159 | 160 | self.save() 161 | 162 | def _handle_execution_success(self, use_statsd=False): 163 | """ 164 | Handle execution success of the trigger 165 | :param bool use_statsd: whether to use statsd 166 | :return: None 167 | """ 168 | if self.date_processed is None: 169 | now = timezone.now() 170 | self.date_processed = now 171 | 172 | # Send stats to statsd if necessary 173 | if use_statsd: 174 | from django_statsd.clients import statsd 175 | statsd.incr('triggers.{trigger_type}.processed'.format(trigger_type=self.trigger_type)) 176 | if self.date_processed and self.process_after: 177 | statsd.timing('triggers.{trigger_type}.process_delay_seconds'.format(trigger_type=self.trigger_type), 178 | (self.date_processed - self.process_after).total_seconds()) 179 | 180 | self.successful = True 181 | self.save() 182 | 183 | def __repr__(self): 184 | return 'Trigger {trigger_id} of type {trigger_type} ({is_processed}processed)'.format( 185 | trigger_id=self.id, trigger_type=self.trigger_type, is_processed='' if self.date_processed else 'not ') 186 | 187 | 188 | class TriggerResult(models.Model): 189 | trigger = models.ForeignKey(Trigger, on_delete=models.CASCADE) 190 | result = models.TextField() 191 | 192 | def __repr__(self): 193 | return self.result 194 | -------------------------------------------------------------------------------- /djtriggers/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import shared_task 2 | from celery.utils.log import get_task_logger 3 | 4 | from django.apps import apps 5 | 6 | from .models import Trigger 7 | 8 | 9 | logger = get_task_logger(__name__) 10 | 11 | 12 | @shared_task 13 | def process_triggers(): 14 | from .logic import process_triggers as process 15 | process() 16 | 17 | 18 | @shared_task 19 | def clean_triggers(): 20 | from .logic import clean_triggers as clean 21 | clean() 22 | 23 | 24 | @shared_task 25 | def process_trigger(trigger_id, trigger_app_label, trigger_class, *args, **kwargs): 26 | try: 27 | apps.get_model(trigger_app_label, trigger_class).objects.get(id=trigger_id).process(*args, **kwargs) 28 | except Trigger.DoesNotExist: 29 | pass 30 | -------------------------------------------------------------------------------- /djtriggers/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vikingco/django-triggers/807c34a7c1c0c3b8df67583ddbdd33e8c2c41399/djtriggers/tests/__init__.py -------------------------------------------------------------------------------- /djtriggers/tests/factories/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vikingco/django-triggers/807c34a7c1c0c3b8df67583ddbdd33e8c2c41399/djtriggers/tests/factories/__init__.py -------------------------------------------------------------------------------- /djtriggers/tests/factories/triggers.py: -------------------------------------------------------------------------------- 1 | from django.utils import timezone 2 | from factory import DjangoModelFactory 3 | 4 | from djtriggers.tests.models import DummyTrigger 5 | 6 | 7 | class DummyTriggerFactory(DjangoModelFactory): 8 | class Meta: 9 | model = DummyTrigger 10 | 11 | trigger_type = 'dummy_trigger_test' 12 | source = 'tests' 13 | date_received = timezone.now() 14 | date_processed = None 15 | process_after = None 16 | number_of_tries = 0 17 | -------------------------------------------------------------------------------- /djtriggers/tests/models.py: -------------------------------------------------------------------------------- 1 | from djtriggers.models import Trigger 2 | 3 | 4 | class DummyTrigger(Trigger): 5 | class Meta: 6 | proxy = True 7 | 8 | typed = 'dummy_trigger' 9 | 10 | def _process(self, dictionary): 11 | pass 12 | -------------------------------------------------------------------------------- /djtriggers/tests/test_dummy.py: -------------------------------------------------------------------------------- 1 | class TestDummy: 2 | def test_dummy(self): 3 | pass 4 | -------------------------------------------------------------------------------- /djtriggers/tests/test_logic.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from mock import patch 3 | 4 | from django.test import override_settings 5 | from django.test.testcases import TestCase 6 | from django.utils import timezone 7 | 8 | from djtriggers.logic import process_triggers 9 | from djtriggers.tests.factories.triggers import DummyTriggerFactory 10 | 11 | 12 | class SynchronousExecutionTest(TestCase): 13 | def setUp(self): 14 | self.now = timezone.now() 15 | 16 | def test_process_after_now(self): 17 | trigger = DummyTriggerFactory() 18 | process_triggers() 19 | trigger.refresh_from_db() 20 | assert trigger.date_processed is not None 21 | 22 | def test_process_after_yesterday(self): 23 | trigger = DummyTriggerFactory(process_after=self.now - timedelta(days=1)) 24 | process_triggers() 25 | trigger.refresh_from_db() 26 | assert trigger.date_processed is not None 27 | 28 | def test_process_after_tomorrow(self): 29 | trigger = DummyTriggerFactory(process_after=self.now + timedelta(days=1)) 30 | process_triggers() 31 | trigger.refresh_from_db() 32 | assert trigger.date_processed is None 33 | 34 | def test_already_processed(self): 35 | trigger = DummyTriggerFactory(date_processed=self.now) 36 | process_triggers() 37 | trigger.refresh_from_db() 38 | assert trigger.date_processed == self.now 39 | 40 | 41 | class AsynchronousExecutionTest(TestCase): 42 | def setUp(self): 43 | self.now = timezone.now() 44 | 45 | @override_settings(DJTRIGGERS_ASYNC_HANDLING=True) 46 | def test_process_after_now(self): 47 | trigger = DummyTriggerFactory() 48 | with patch('djtriggers.logic.process_trigger.apply_async') as process_trigger_patch: 49 | process_triggers() 50 | process_trigger_patch.assert_called_once_with((trigger.id, 'djtriggers', 'DummyTrigger'), 51 | {'use_statsd': False}, max_retries=0) 52 | 53 | @override_settings(DJTRIGGERS_ASYNC_HANDLING=True) 54 | def test_process_after_yesterday(self): 55 | trigger = DummyTriggerFactory(process_after=self.now - timedelta(days=1)) 56 | with patch('djtriggers.logic.process_trigger.apply_async') as process_trigger_patch: 57 | process_triggers() 58 | process_trigger_patch.assert_called_once_with((trigger.id, 'djtriggers', 'DummyTrigger'), 59 | {'use_statsd': False}, max_retries=0) 60 | 61 | @override_settings(DJTRIGGERS_ASYNC_HANDLING=True) 62 | def test_process_after_tomorrow(self): 63 | DummyTriggerFactory(process_after=self.now + timedelta(days=1)) 64 | with patch('djtriggers.logic.process_trigger.apply_async') as process_trigger_patch: 65 | process_triggers() 66 | assert not process_trigger_patch.called 67 | 68 | @override_settings(DJTRIGGERS_ASYNC_HANDLING=True) 69 | def test_already_processed(self): 70 | DummyTriggerFactory(date_processed=self.now) 71 | with patch('djtriggers.logic.process_trigger.apply_async') as process_trigger_patch: 72 | process_triggers() 73 | assert not process_trigger_patch.called 74 | -------------------------------------------------------------------------------- /djtriggers/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from logging import ERROR, WARNING 3 | from redis.exceptions import LockError 4 | 5 | from mock import patch 6 | from pytest import raises 7 | from django.test import override_settings 8 | from django.test.testcases import TestCase 9 | from django.utils import timezone 10 | 11 | from djtriggers.exceptions import ProcessLaterError 12 | from djtriggers.loggers.base import TriggerLogger 13 | from djtriggers.models import Trigger 14 | from djtriggers.tests.factories.triggers import DummyTriggerFactory 15 | 16 | 17 | class TriggerTest(TestCase): 18 | def test_handle_execution_success(self): 19 | trigger = DummyTriggerFactory() 20 | trigger._handle_execution_success() 21 | 22 | assert trigger.date_processed 23 | assert trigger.successful is True 24 | 25 | @patch('django_statsd.clients.statsd') 26 | def test_handle_execution_success_use_statsd(self, mock_statsd): 27 | trigger = DummyTriggerFactory(process_after=timezone.now()) 28 | trigger._handle_execution_success(use_statsd=True) 29 | 30 | assert trigger.date_processed 31 | assert trigger.successful is True 32 | mock_statsd.incr.assert_called_with( 33 | 'triggers.{trigger_type}.processed'.format(trigger_type=trigger.trigger_type)) 34 | mock_statsd.timing.assert_called_with( 35 | 'triggers.{trigger_type}.process_delay_seconds'.format(trigger_type=trigger.trigger_type), 36 | (trigger.date_processed - trigger.process_after).total_seconds()) 37 | 38 | def test_handle_execution_failure(self): 39 | trigger = DummyTriggerFactory() 40 | original_tries = trigger.number_of_tries 41 | trigger._handle_execution_failure(Exception()) 42 | 43 | assert trigger.number_of_tries == original_tries + 1 44 | 45 | @override_settings(DJTRIGGERS_TRIES_BEFORE_WARNING=3) 46 | @patch.object(TriggerLogger, 'log_message') 47 | def test_handle_execution_failure_dont_raise_when_under_max_retries(self, mock_logger): 48 | exception = Exception() 49 | trigger = DummyTriggerFactory(number_of_tries=2) 50 | original_tries = trigger.number_of_tries 51 | trigger._handle_execution_failure(exception) 52 | 53 | assert trigger.number_of_tries == original_tries + 1 54 | assert not mock_logger.called 55 | 56 | @override_settings(DJTRIGGERS_TRIES_BEFORE_WARNING=3) 57 | @patch.object(TriggerLogger, 'log_message') 58 | def test_handle_execution_failure_tries_exceeded(self, mock_logger): 59 | exception = Exception() 60 | trigger = DummyTriggerFactory(number_of_tries=3) 61 | original_tries = trigger.number_of_tries 62 | trigger._handle_execution_failure(exception) 63 | 64 | assert trigger.number_of_tries == original_tries + 1 65 | message = ('Processing of {trigger_type} {trigger_key} ' 66 | 'raised a {exception_type} (try nr. {try_count})').format(trigger_type=trigger.trigger_type, 67 | trigger_key=trigger.pk, 68 | exception_type=type(exception).__name__, 69 | try_count=original_tries+1) 70 | mock_logger.assert_called_with(trigger, message, level=WARNING) 71 | 72 | @override_settings(DJTRIGGERS_TRIES_BEFORE_WARNING=3) 73 | @override_settings(DJTRIGGERS_TRIES_BEFORE_ERROR=5) 74 | def test_handle_execution_failure_tries_limit(self): 75 | """ 76 | When trigger reaches retries limit and ERROR is raised check message level and trigger successful state. 77 | """ 78 | exception = Exception() 79 | exception_name = type(exception).__name__ 80 | trigger = DummyTriggerFactory(number_of_tries=3) 81 | original_tries = trigger.number_of_tries 82 | 83 | with patch.object(TriggerLogger, 'log_message') as mock_logger: 84 | trigger._handle_execution_failure(exception) 85 | exceeded_retries = original_tries + 1 86 | assert trigger.number_of_tries == exceeded_retries 87 | assert trigger.successful is None 88 | message = ('Processing of {trigger_type} {trigger_key} ' 89 | 'raised a {exception_type} (try nr. {try_count})').format(trigger_type=trigger.trigger_type, 90 | trigger_key=trigger.pk, 91 | exception_type=exception_name, 92 | try_count=exceeded_retries) 93 | mock_logger.assert_called_once_with(trigger, message, level=WARNING) 94 | 95 | # do an extra retry 96 | with patch.object(TriggerLogger, 'log_message') as mock_logger: 97 | trigger._handle_execution_failure(exception) 98 | exceeded_retries = original_tries + 2 99 | assert trigger.number_of_tries == exceeded_retries 100 | assert trigger.successful is False 101 | message = ('Processing of {trigger_type} {trigger_key} ' 102 | 'raised a {exception_type} (try nr. {try_count})').format(trigger_type=trigger.trigger_type, 103 | trigger_key=trigger.pk, 104 | exception_type=exception_name, 105 | try_count=exceeded_retries) 106 | mock_logger.assert_called_once_with(trigger, message, level=ERROR) 107 | 108 | def test_handle_execution_failure_above_tries_limit(self): 109 | """ 110 | Check if ERROR is still raised for a trigger above retry limit 111 | """ 112 | exception = Exception() 113 | exception_name = type(exception).__name__ 114 | trigger = DummyTriggerFactory(number_of_tries=5) 115 | original_tries = trigger.number_of_tries 116 | exceeded_retries = original_tries + 1 117 | 118 | with self.settings(DJTRIGGERS_TRIES_BEFORE_WARNING=3) and self.settings(DJTRIGGERS_TRIES_BEFORE_ERROR=5) and \ 119 | patch.object(TriggerLogger, 'log_message') as mock_logger: 120 | trigger._handle_execution_failure(exception) 121 | assert trigger.number_of_tries == exceeded_retries 122 | assert trigger.successful is False 123 | message = ('Processing of {trigger_type} {trigger_key} ' 124 | 'raised a {exception_type} (try nr. {try_count})').format(trigger_type=trigger.trigger_type, 125 | trigger_key=trigger.pk, 126 | exception_type=exception_name, 127 | try_count=exceeded_retries) 128 | mock_logger.assert_called_once_with(trigger, message, level=ERROR) 129 | 130 | @patch('django_statsd.clients.statsd') 131 | def test_handle_execution_failure_use_statsd(self, mock_statsd): 132 | exception = Exception() 133 | trigger = DummyTriggerFactory() 134 | original_tries = trigger.number_of_tries 135 | trigger._handle_execution_failure(exception, use_statsd=True) 136 | 137 | assert trigger.number_of_tries == original_tries + 1 138 | assert trigger.successful is None 139 | mock_statsd.incr.assert_called_with('triggers.{trigger_type}.failed'.format(trigger_type=trigger.trigger_type)) 140 | 141 | @patch.object(TriggerLogger, 'log_result') 142 | def test_process(self, mock_logger): 143 | trigger = DummyTriggerFactory() 144 | trigger.process() 145 | 146 | mock_logger.assert_called_with(trigger, trigger._process({})) 147 | 148 | @patch.object(TriggerLogger, 'log_result') 149 | def test_process_already_processed(self, mock_logger): 150 | """Reprocessing already processed triggers should just do nothing.""" 151 | trigger = DummyTriggerFactory(date_processed=timezone.now()) 152 | assert trigger.date_processed is not None 153 | assert not mock_logger.called 154 | 155 | def test_process_process_later(self): 156 | trigger = DummyTriggerFactory(process_after=timezone.now() + timedelta(minutes=1)) 157 | with raises(ProcessLaterError): 158 | trigger.process() 159 | 160 | @patch.object(Trigger, '_handle_execution_failure') 161 | def test_process_exception_during_execution(self, mock_fail): 162 | trigger = DummyTriggerFactory() 163 | with patch.object(trigger, '_process', side_effect=Exception), raises(Exception): 164 | trigger.process() 165 | assert mock_fail.called 166 | 167 | @patch.object(TriggerLogger, 'log_result') 168 | def test_process_locked(self, mock_logger): 169 | trigger = DummyTriggerFactory() 170 | with patch('djtriggers.models.redis_lock', side_effect=LockError): 171 | trigger.process() 172 | 173 | assert not mock_logger.called 174 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import unicode_literals 4 | from os import environ 5 | from sys import argv 6 | 7 | if __name__ == '__main__': 8 | environ.setdefault('DJANGO_SETTINGS_MODULE', 'djtriggers.tests.settings') 9 | 10 | from django.core.management import execute_from_command_line 11 | execute_from_command_line(argv) 12 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "django-triggers" 3 | version = "2.2.0" 4 | description = "Framework to create and process triggers." 5 | authors = [ 6 | { name = "Unleashed NV", email = "operations@unleashed.be" }, 7 | ] 8 | readme = "README.md" 9 | classifiers=[ 10 | 'Intended Audience :: Developers', 11 | 'Programming Language :: Python', 12 | 'Operating System :: OS Independent', 13 | 'Environment :: Web Environment', 14 | 'Framework :: Django', 15 | ] 16 | dependencies = [ 17 | "Django>=2.1.5", 18 | "celery>=5.0.0", 19 | "python-dateutil", 20 | "redis>=3.0.0", 21 | ] 22 | 23 | [project.urls] 24 | Repository = "https://github.com/vikingco/django-triggers" 25 | 26 | [tool.setuptools] 27 | include-package-data = true 28 | 29 | [tool.setuptools.packages.find] 30 | include = ["djtriggers*"] 31 | exclude = ["djtriggers.tests*"] 32 | namespaces = false 33 | 34 | [build-system] 35 | requires = ["pip", "setuptools"] 36 | build-backend = "setuptools.build_meta" 37 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE=settings_test 3 | python_files = test*.py 4 | norecursedirs = .* *.egg *.egg-info wheel dist build artifacts venv 5 | addopts= 6 | --ignore=venv 7 | --cov=djtriggers 8 | -------------------------------------------------------------------------------- /requirements/requirements_test.txt: -------------------------------------------------------------------------------- 1 | # Mocking, faking, factories and other test helpers 2 | mock==1.0.1 3 | 4 | # Factories 5 | factory_boy==2.9.1 6 | django-factory_boy==1.0.0 7 | Faker==4.4.0 8 | 9 | # pytest 10 | py==1.9.0 11 | pytest==6.1.1 12 | pytest-django==3.10.0 13 | 14 | # test coverage 15 | pytest-cov==2.5.1 16 | coveralls==1.1 17 | coverage==4.3.4 18 | docopt==0.6.2 19 | requests==2.20.0 20 | 21 | 22 | # Flake8 checks for PEP-8 formatting and code quality 23 | flake8==3.8.4 24 | entrypoints==0.3 25 | pycodestyle==2.6.0 26 | enum34==1.1.6 27 | configparser==3.5.0 28 | pyflakes==2.2.0 29 | pep8==1.7.0 30 | # Checks for cyclomatic complexity 31 | mccabe==0.6.1 32 | # Checks for naming conventions 33 | pep8-naming==0.8.2 34 | # Checks for the use of print statements 35 | flake8-print==3.1.0 36 | # Checks for the use of debug statements (pdb, ...) 37 | flake8-debugger==3.1.0 38 | # Checks for things that should be comprehensions 39 | flake8-comprehensions==2.1.0 40 | # Checks for non-existent mock methods 41 | flake8-mock==0.3 42 | # Checks for mutable function arguments 43 | flake8-mutable==1.2.0 44 | # Checks for TODO statements (and similar) 45 | flake8-todo==0.7 46 | # Checks for uses of double quotes (without containing single quotes) 47 | flake8-quotes==1.0.0 48 | # Checks for uses of old-style string formatting (using % instead of .format()) 49 | flake8-pep3101==1.2.1 50 | # Checks for the overriding of builtins 51 | flake8-builtins-unleashed==1.3.1 52 | # Checks for uses of Django-style asserts 53 | flake8-pytest==1.3 54 | # Checks for bad imports of fake_time (should come from libfaketime_tz_wrapper instead of libfaketime) 55 | flake8-libfaketime==1.1 56 | # Checks for module-level imports 57 | flake8-module-imports==1.1 58 | # Checks for usages of django.translation.activate() 59 | flake8-translation-activate==1.0.2 60 | # Checks for imports with an alias of _ 61 | flake8-ugettext-alias==1.1 62 | # Checks for imports of the User model (instead of get_user_model() or settings.AUTH_USER_MODEL) 63 | flake8-user-model==1.1 64 | 65 | # StatsD allows us to push monitoring data to a Graphite server 66 | django-statsd-unleashed==1.0.6 67 | statsd==3.2.1 68 | -------------------------------------------------------------------------------- /settings_test.py: -------------------------------------------------------------------------------- 1 | SECRET_KEY = 'fake-key' 2 | 3 | DJTRIGGERS_REDIS_URL = '' 4 | 5 | INSTALLED_APPS = [ 6 | 'djtriggers', 7 | ] 8 | 9 | DATABASES = { 10 | 'default': { 11 | 'ENGINE': 'django.db.backends.sqlite3', 12 | 'NAME': 'db.sqlite3', 13 | } 14 | } 15 | --------------------------------------------------------------------------------