├── tests └── __init__.py ├── jaiminho ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── event_cleaner.py │ │ └── events_relay.py ├── migrations │ ├── __init__.py │ ├── 0006_alter_event_id.py │ ├── 0004_event_stream.py │ ├── 0005_event_strategy.py │ ├── 0007_alter_event_id_to_bigint.py │ ├── 0002_event_encoder_event_function_signature_event_options.py │ ├── 0001_initial.py │ └── 0003_remove_event_action_remove_event_encoder_and_more.py ├── admin.py ├── views.py ├── tests │ ├── __init__.py │ ├── utils.py │ ├── test_jaiminho.py │ ├── test_signals.py │ ├── factories.py │ ├── test_models.py │ └── test_settings.py ├── constants.py ├── apps.py ├── signals.py ├── settings.py ├── models.py ├── send.py ├── relayer.py └── publish_strategies.py ├── jaiminho_django_test_project ├── __init__.py ├── app │ ├── __init__.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── views.py │ ├── apps.py │ └── signals.py ├── tests │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── test_validate_event_cleaner.py │ │ │ └── test_validate_relay_events.py │ └── test_send.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── validate_event_cleaner.py │ │ └── validate_events_relay.py ├── tests_functional │ ├── __init__.py │ └── test_send_to_outbox.py ├── asgi.py ├── wsgi.py ├── urls.py ├── send.py └── settings.py ├── .github ├── CODEOWNERS ├── dependabot.yml └── PULL_REQUEST_TEMPLATE.md ├── pytest.ini ├── assets └── jaiminho.jpg ├── .editorconfig ├── requirements-dev.txt ├── setup.cfg ├── manage.py ├── Makefile ├── CONTRIBUTING.md ├── LICENSE ├── CHANGELOG.md ├── setup.py ├── .gitignore ├── .circleci └── config.yml └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @loadsmart/backend 2 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/app/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests_functional/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jaiminho/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /jaiminho/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | 3 | # Create your views here. 4 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE=jaiminho_django_test_project.settings 3 | 4 | -------------------------------------------------------------------------------- /jaiminho/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /assets/jaiminho.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/loadsmart/django-jaiminho/HEAD/assets/jaiminho.jpg -------------------------------------------------------------------------------- /jaiminho_django_test_project/app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/app/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | 3 | # Create your views here. 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | end_of_line = lf 3 | insert_final_newline = true 4 | 5 | [*.{yaml,yml}] 6 | indent_style = space 7 | indent_size = 2 8 | -------------------------------------------------------------------------------- /jaiminho/tests/utils.py: -------------------------------------------------------------------------------- 1 | def foo(): 2 | return True 3 | 4 | 5 | class Bar: 6 | def __init__(self): 7 | self.bar = True 8 | -------------------------------------------------------------------------------- /jaiminho/constants.py: -------------------------------------------------------------------------------- 1 | class PublishStrategyType: 2 | PUBLISH_ON_COMMIT = "publish-on-commit" 3 | KEEP_ORDER = "keep-order" 4 | 5 | CHOICES = ((PUBLISH_ON_COMMIT, "Publish on Commit"), (KEEP_ORDER, "Keep Order")) 6 | -------------------------------------------------------------------------------- /jaiminho/tests/test_jaiminho.py: -------------------------------------------------------------------------------- 1 | def test_imports(): 2 | from jaiminho import admin # noqa 3 | from jaiminho import apps # noqa 4 | from jaiminho import views # noqa 5 | from jaiminho import models # noqa 6 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/management/commands/validate_event_cleaner.py: -------------------------------------------------------------------------------- 1 | from jaiminho.management.commands.event_cleaner import Command 2 | 3 | 4 | class Command(Command): 5 | def __init__(self): 6 | super(Command, self).__init__() 7 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/management/commands/validate_events_relay.py: -------------------------------------------------------------------------------- 1 | from jaiminho.management.commands.events_relay import Command 2 | 3 | 4 | class Command(Command): 5 | def __init__(self): 6 | super(Command, self).__init__() 7 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # Development requirements for Python 3.X 2 | # For package requirements, please see setup.py 3 | 4 | pytest==8.3.5 5 | pytest-cov==5.0.0 6 | pytest-django==4.11.1 7 | pytest-mock~=3.14.1 8 | tox==4.25.0 9 | wheel==0.45.1 10 | factory-boy~=3.3.3 11 | freezegun~=1.5.5 12 | ipdb 13 | setuptools==75.3.0 14 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/app/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class AppConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "jaiminho_django_test_project.app" 7 | 8 | def ready(self): 9 | import jaiminho_django_test_project.app.signals # noqa 10 | -------------------------------------------------------------------------------- /jaiminho/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils import version 3 | 4 | 5 | class JaiminhoConfig(AppConfig): 6 | if version.get_version() >= "3.2": 7 | default_auto_field = "django.db.models.BigAutoField" 8 | else: 9 | default_auto_field = "django.db.models.AutoField" 10 | name = "jaiminho" 11 | -------------------------------------------------------------------------------- /jaiminho/signals.py: -------------------------------------------------------------------------------- 1 | from django import dispatch 2 | 3 | event_published = dispatch.Signal() 4 | event_failed_to_publish = dispatch.Signal() 5 | event_published_by_events_relay = dispatch.Signal() 6 | event_failed_to_publish_by_events_relay = dispatch.Signal() 7 | 8 | 9 | def get_event_payload(args): 10 | try: 11 | if isinstance(args, tuple): 12 | return args[0] 13 | return {} 14 | except IndexError: 15 | return {} 16 | -------------------------------------------------------------------------------- /jaiminho/tests/test_signals.py: -------------------------------------------------------------------------------- 1 | from jaiminho.signals import get_event_payload 2 | 3 | 4 | class TestGetEventPayload: 5 | def test_success_when_args_is_tuple(self): 6 | assert get_event_payload(({"a": 1},)) == {"a": 1} 7 | 8 | def test_return_empty_dict_when_args_empty_iterable(self): 9 | assert get_event_payload(()) == {} 10 | 11 | def test_return_empty_dict_when_args_not_iterable(self): 12 | assert get_event_payload(1) == {} 13 | -------------------------------------------------------------------------------- /jaiminho/tests/factories.py: -------------------------------------------------------------------------------- 1 | import factory 2 | import uuid 3 | 4 | from datetime import datetime 5 | from django.core.serializers.json import DjangoJSONEncoder 6 | 7 | from jaiminho.models import Event 8 | 9 | 10 | class EventFactory(factory.django.DjangoModelFactory): 11 | message = factory.LazyAttribute( 12 | lambda _: b"\x80\x04\x95\x0c\x00\x00\x00\x00\x00\x00\x00}\x94\x8c\x01a\x94K\x01s\x85\x94." 13 | ) 14 | 15 | class Meta: 16 | model = Event 17 | -------------------------------------------------------------------------------- /jaiminho/migrations/0006_alter_event_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2023-06-12 13:43 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("jaiminho", "0005_event_strategy"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="event", 15 | name="id", 16 | field=models.BigAutoField(primary_key=True, serialize=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /jaiminho/migrations/0004_event_stream.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.6 on 2022-08-30 11:56 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("jaiminho", "0003_remove_event_action_remove_event_encoder_and_more"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="event", 14 | name="stream", 15 | field=models.CharField(max_length=100, null=True), 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for jaiminho_django_test_project project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jaiminho_django_test_project.settings") 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for jaiminho_django_test_project project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jaiminho_django_test_project.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /jaiminho/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pytest 4 | from dateutil.tz import UTC 5 | from freezegun import freeze_time 6 | 7 | from jaiminho.tests.factories import EventFactory 8 | 9 | 10 | @pytest.mark.django_db 11 | class TestEvent: 12 | def test_mark_as_sent(self): 13 | event = EventFactory() 14 | assert event.sent_at is None 15 | 16 | with freeze_time("2022-01-01"): 17 | event.mark_as_sent() 18 | assert event.sent_at == datetime(2022, 1, 1, tzinfo=UTC) 19 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | ## Motivation and Context 5 | 6 | 7 | ## How has this been tested? 8 | 9 | 10 | ## What is the current behavior? 11 | 12 | 13 | ## What is the new behavior? 14 | 15 | 16 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | 4 | [tox:tox] 5 | envlist = py38,py39,py310,py311,py312 6 | passenv = TOXENV CI CIRCLECI CIRCLE_* 7 | 8 | [testenv] 9 | deps = 10 | -rrequirements-dev.txt 11 | -e . 12 | commands = pytest --capture=no --cov=jaiminho --cov-config setup.cfg --cov-report xml --junitxml=build/testreport/report.xml 13 | 14 | [coverage:run] 15 | source = jaiminho 16 | 17 | [report] 18 | show_missing = true 19 | exclude_lines = 20 | pragma: no cover 21 | def __repr__ 22 | raise NotImplementedError 23 | 24 | [xml] 25 | output = build/coverage.xml 26 | 27 | [metadata] 28 | description-file = README.md 29 | -------------------------------------------------------------------------------- /jaiminho/migrations/0005_event_strategy.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.5 on 2022-10-14 11:10 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("jaiminho", "0004_event_stream"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="event", 14 | name="strategy", 15 | field=models.CharField( 16 | choices=[ 17 | ("publish-on-commit", "Publish on Commit"), 18 | ("keep-order", "Keep Order"), 19 | ], 20 | max_length=100, 21 | null=True, 22 | ), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /jaiminho/migrations/0007_alter_event_id_to_bigint.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.7 on 2024-11-01 14:40 2 | 3 | from django.db import migrations, connection 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("jaiminho", "0006_alter_event_id"), 10 | ] 11 | 12 | operations = [] 13 | 14 | if connection.vendor == "postgresql": 15 | operations.append( 16 | migrations.RunSQL( 17 | "ALTER TABLE jaiminho_event ALTER COLUMN id TYPE BIGINT;", 18 | ) 19 | ) 20 | elif connection.vendor == "mysql": 21 | operations.append( 22 | migrations.RunSQL( 23 | "ALTER TABLE jaiminho_event MODIFY id BIGINT;", 24 | ) 25 | ) 26 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault( 10 | "DJANGO_SETTINGS_MODULE", "jaiminho_django_test_project.settings" 11 | ) 12 | try: 13 | from django.core.management import execute_from_command_line 14 | except ImportError as exc: 15 | raise ImportError( 16 | "Couldn't import Django. Are you sure it's installed and " 17 | "available on your PYTHONPATH environment variable? Did you " 18 | "forget to activate a virtual environment?" 19 | ) from exc 20 | execute_from_command_line(sys.argv) 21 | 22 | 23 | if __name__ == "__main__": 24 | main() 25 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | clean: clean-build clean-pyc clean-test 2 | 3 | clean-build: 4 | rm -fr build/ 5 | rm -fr dist/ 6 | rm -fr *.egg-info 7 | 8 | clean-pyc: 9 | find . -name '*.pyc' -exec rm -f {} + 10 | find . -name '*.pyo' -exec rm -f {} + 11 | find . -name '*~' -exec rm -f {} + 12 | find . -name '__pycache__' -exec rm -fr {} + 13 | 14 | clean-test: 15 | rm -fr .tox/ 16 | rm -f .coverage 17 | rm -fr htmlcov/ 18 | 19 | install: 20 | python -m venv .venv 21 | ./.venv/bin/python -m pip install -r requirements-dev.txt 22 | 23 | fmt: 24 | black . 25 | 26 | test: 27 | pytest . 28 | 29 | test-all: 30 | tox 31 | 32 | dist: clean 33 | python setup.py bdist_wheel --universal 34 | 35 | release: dist 36 | pip install twine 37 | python -m twine upload --non-interactive --username __token__ --password ${PYPI_TOKEN} dist/* 38 | -------------------------------------------------------------------------------- /jaiminho/settings.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | import sentry_sdk 3 | from django.conf import settings 4 | from django.core.serializers.json import DjangoJSONEncoder 5 | 6 | from jaiminho.constants import PublishStrategyType 7 | 8 | try: 9 | jaiminho_settings = getattr(settings, "JAIMINHO_CONFIG") 10 | except AttributeError: 11 | jaiminho_settings = {} 12 | 13 | persist_all_events = jaiminho_settings.get("PERSIST_ALL_EVENTS", False) 14 | time_to_delete = jaiminho_settings.get("TIME_TO_DELETE", timedelta(days=7)) 15 | delete_after_send = jaiminho_settings.get("DELETE_AFTER_SEND", False) 16 | publish_strategy = jaiminho_settings.get( 17 | "PUBLISH_STRATEGY", PublishStrategyType.PUBLISH_ON_COMMIT 18 | ) 19 | 20 | default_capture_exception = jaiminho_settings.get( 21 | "DEFAULT_CAPTURE_EXCEPTION", sentry_sdk.capture_exception 22 | ) 23 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/urls.py: -------------------------------------------------------------------------------- 1 | """jaiminho_django_test_project URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/4.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | 17 | from django.contrib import admin 18 | from django.urls import path 19 | 20 | urlpatterns = [ 21 | path("admin/", admin.site.urls), 22 | ] 23 | -------------------------------------------------------------------------------- /jaiminho/migrations/0002_event_encoder_event_function_signature_event_options.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.4 on 2022-05-23 19:08 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("jaiminho", "0001_initial"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="event", 14 | name="encoder", 15 | field=models.CharField(max_length=255, null=True), 16 | ), 17 | migrations.AddField( 18 | model_name="event", 19 | name="function_signature", 20 | field=models.CharField(max_length=255, null=True), 21 | ), 22 | migrations.AddField( 23 | model_name="event", 24 | name="options", 25 | field=models.TextField(blank=True), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /jaiminho/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.utils import timezone 3 | 4 | from jaiminho.constants import PublishStrategyType 5 | 6 | MAX_BYTES = 65535 7 | 8 | 9 | class Event(models.Model): 10 | id = models.BigAutoField(primary_key=True) 11 | message = models.BinaryField(null=True, max_length=MAX_BYTES) 12 | function = models.BinaryField(null=True, max_length=MAX_BYTES) 13 | kwargs = models.BinaryField(null=True, max_length=MAX_BYTES) 14 | created_at = models.DateTimeField(auto_now_add=True) 15 | sent_at = models.DateTimeField(null=True) 16 | stream = models.CharField(max_length=100, null=True) 17 | strategy = models.CharField( 18 | max_length=100, null=True, choices=PublishStrategyType.CHOICES 19 | ) 20 | 21 | def mark_as_sent(self): 22 | self.sent_at = timezone.now() 23 | self.save() 24 | 25 | def __str__(self): 26 | return f"Event(id={self.id})" 27 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Jaiminho solved an internal problem we had and we hope it will improve life quality for other people as well. 2 | Improvements and new use cases are most welcome! 3 | To get a change accepted quickly, please check the items below. 4 | 5 | **Short Links to Important Resources**: 6 | bugs: Please open issues in our Github project 7 | comms: You can reach out to the core developer team writing to jaiminho@loadsmart.com. 8 | 9 | **Testing**: The project has two types of tests, unit tests in the jaiminho library itself, 10 | and integration tests, where we use a fake Django app to validate some behaviors like commands and DB-related things. 11 | 12 | Bug report: When opening a new bug or issue, please create a Github issue in this repository. For critical security topics you can mail us directly. 13 | 14 | 15 | For code style, we use black in the latest version and adhere to any convention Django has when creating new apps. 16 | 17 | **Code of Conduct**: We do adhere and follow the Loadsmart culture, please check it out (https://github.com/loadsmart/culture). 18 | -------------------------------------------------------------------------------- /jaiminho/management/commands/event_cleaner.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import timedelta 3 | 4 | from django.core.management import BaseCommand 5 | from django.utils import timezone 6 | 7 | from jaiminho import settings 8 | from jaiminho.models import Event 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class Command(BaseCommand): 14 | def __new__(cls, *args, **kwargs): 15 | assert isinstance(settings.time_to_delete, timedelta) 16 | return super().__new__(cls, *args, **kwargs) 17 | 18 | def handle(self, *args, **options): 19 | deletion_threshold_timestamp = timezone.now() - settings.time_to_delete 20 | 21 | events_to_delete = Event.objects.filter(sent_at__isnull=False).filter( 22 | sent_at__lt=deletion_threshold_timestamp 23 | ) 24 | 25 | logger.info("JAIMINHO-EVENT-CLEANER: Start cleaning up events ..") 26 | 27 | count = events_to_delete._raw_delete(events_to_delete.db) 28 | 29 | logger.info( 30 | "JAIMINHO-EVENT-CLEANER: Successfully deleted %s events", 31 | count, 32 | ) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Loadsmart 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/app/signals.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.dispatch import receiver 4 | import jaiminho.signals 5 | 6 | 7 | def log_metric(name, event_payload, **kwargs): 8 | logging.info("%s: %s", name, event_payload) 9 | 10 | 11 | @receiver(jaiminho.signals.event_published) 12 | def on_event_published(signal, sender, event_payload, **kwargs): 13 | log_metric("event-published", event_payload, **kwargs) 14 | 15 | 16 | @receiver(jaiminho.signals.event_failed_to_publish) 17 | def on_event_not_published(signal, sender, event_payload, **kwargs): 18 | log_metric("event-failed-to-publish", event_payload, **kwargs) 19 | 20 | 21 | @receiver(jaiminho.signals.event_published_by_events_relay) 22 | def on_event_published_through_relay_command(signal, sender, event_payload, **kwargs): 23 | log_metric("event-published-through-outbox", event_payload, **kwargs) 24 | 25 | 26 | @receiver(jaiminho.signals.event_failed_to_publish_by_events_relay) 27 | def on_event_not_published_through_relay_command( 28 | signal, sender, event_payload, **kwargs 29 | ): 30 | log_metric("event-failed-to-publish-through-outbox", event_payload, **kwargs) 31 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/send.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from jaiminho.constants import PublishStrategyType 4 | from jaiminho.send import save_to_outbox, save_to_outbox_stream 5 | 6 | 7 | EXAMPLE_STREAM = "my-stream" 8 | 9 | 10 | class InternalDecoder: 11 | def __call__(self, *args, **kwargs): 12 | print("Hello Internal", args, kwargs) 13 | 14 | 15 | def internal_notify(*args, decoder=None, **kwargs): 16 | if decoder: 17 | decoder(args) 18 | print(args, kwargs) 19 | 20 | 21 | @save_to_outbox 22 | def notify(*args, **kwargs): 23 | internal_notify(*args, **kwargs) 24 | 25 | 26 | @save_to_outbox_stream(EXAMPLE_STREAM) 27 | def notify_to_stream(*args, **kwargs): 28 | internal_notify(*args, **kwargs) 29 | 30 | 31 | @save_to_outbox_stream(EXAMPLE_STREAM, PublishStrategyType.KEEP_ORDER) 32 | def notify_to_stream_overwriting_strategy(*args, **kwargs): 33 | internal_notify(*args, **kwargs) 34 | 35 | 36 | @save_to_outbox_stream(EXAMPLE_STREAM, PublishStrategyType.KEEP_ORDER) 37 | def notify_functional_to_stream_overwriting_strategy(*args, **kwargs): 38 | with open(kwargs["filepath"], "w") as write_file: 39 | json.dump(args, write_file, indent=4) 40 | 41 | 42 | def notify_without_decorator(*args, **kwargs): 43 | internal_notify(*args, **kwargs) 44 | 45 | 46 | __all__ = ("notify", "notify_without_decorator") 47 | -------------------------------------------------------------------------------- /jaiminho/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.4 on 2022-05-16 14:54 2 | 3 | from django.db import migrations, models 4 | from django.utils import version 5 | 6 | try: 7 | from django.db.models import JSONField 8 | except ImportError: 9 | from django.contrib.postgres.fields import JSONField 10 | 11 | 12 | class Migration(migrations.Migration): 13 | initial = True 14 | 15 | dependencies = [] 16 | 17 | if version.get_version() >= "3.2": 18 | auto_field_class = models.BigAutoField 19 | else: 20 | auto_field_class = models.AutoField 21 | 22 | operations = [ 23 | migrations.CreateModel( 24 | name="Event", 25 | fields=[ 26 | ( 27 | "id", 28 | auto_field_class( 29 | auto_created=True, 30 | primary_key=True, 31 | serialize=False, 32 | verbose_name="ID", 33 | ), 34 | ), 35 | ("type", models.CharField(max_length=64)), 36 | ("action", models.CharField(max_length=64)), 37 | ("payload", JSONField()), 38 | ("created_at", models.DateTimeField(auto_now_add=True)), 39 | ("sent_at", models.DateTimeField(null=True)), 40 | ], 41 | ), 42 | ] 43 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Calendar Versioning](https://calver.org/) following 7 | the schema `YYYY.MM.DD.N` been `N` the number of the release of the day. 8 | 9 | ## [Unreleased] 10 | 11 | ## [1.3.1] - 2024-11-01 12 | 13 | ## [1.3.0] - 2024-10-31 14 | 15 | ## [1.2.0] - 2023-07-18 16 | ### Changed 17 | - Updating developer dependencies 18 | - Update event_cleaner to consume less memory #99 19 | 20 | ## [1.1.1] - 2023-04-25 21 | ### Added 22 | - feat: Add support for Python 3.10 and 3.11 #65 23 | 24 | ## [1.1.0] - 2023-03-02 25 | ### Fixed 26 | - Package description #60 27 | 28 | ## [1.0.0] - 2023-03-02 29 | ### Changed 30 | - Minor documentation improvements #58 31 | 32 | ## 0.0.0 - 2023-03-01 33 | 34 | [Unreleased]: https://github.com/loadsmart/django-jaiminho/compare/1.3.1...HEAD 35 | [1.3.1]: https://github.com/loadsmart/django-jaiminho/compare/1.3.0...1.3.1 36 | [1.3.0]: https://github.com/loadsmart/django-jaiminho/compare/1.2.0...1.3.0 37 | [1.2.0]: https://github.com/loadsmart/django-jaiminho/compare/1.1.1...1.2.0 38 | [1.1.1]: https://github.com/loadsmart/django-jaiminho/compare/1.1.0...1.1.1 39 | [1.1.0]: https://github.com/loadsmart/django-jaiminho/compare/1.0.0...1.1.0 40 | [1.0.0]: https://github.com/loadsmart/django-jaiminho/compare/0.0.0...1.0.0 41 | -------------------------------------------------------------------------------- /jaiminho/send.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import wraps 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | def save_to_outbox(func): 8 | @wraps(func) 9 | def inner(*args, **kwargs): 10 | # Load Django dependencies lazily to ensure the Django environment is ready 11 | from jaiminho.publish_strategies import create_publish_strategy 12 | from jaiminho import settings 13 | 14 | publish_strategy = create_publish_strategy(settings.publish_strategy) 15 | publish_strategy.publish(args, kwargs, func) 16 | 17 | inner.original_func = func 18 | return inner 19 | 20 | 21 | def save_to_outbox_stream(stream, overwrite_strategy_with=None): 22 | def decorator(func): 23 | @wraps(func) 24 | def inner(*args, **kwargs): 25 | # Load Django dependencies lazily to ensure the Django environment is ready 26 | from jaiminho.publish_strategies import create_publish_strategy 27 | from jaiminho import settings 28 | 29 | _publish_strategy = ( 30 | overwrite_strategy_with 31 | if overwrite_strategy_with 32 | else settings.publish_strategy 33 | ) 34 | publish_strategy = create_publish_strategy(_publish_strategy) 35 | publish_strategy.publish(args, kwargs, func, stream) 36 | 37 | inner.original_func = func 38 | return inner 39 | 40 | return decorator 41 | -------------------------------------------------------------------------------- /jaiminho/tests/test_settings.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from importlib import reload 3 | 4 | import pytest 5 | 6 | from jaiminho.constants import PublishStrategyType 7 | 8 | 9 | class TestSettings: 10 | @pytest.mark.parametrize( 11 | "publish_strategy", 12 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 13 | ) 14 | @pytest.mark.parametrize( 15 | ("persist_all_events", "delete_after_send"), ((True, False), (True, False)) 16 | ) 17 | def test_load_settings( 18 | self, mocker, persist_all_events, delete_after_send, publish_strategy 19 | ): 20 | settings_mock = mocker.patch("django.conf.settings") 21 | time_to_delete = mocker.MagicMock() 22 | persist_all_events = True 23 | delete_after_send = True 24 | 25 | settings_mock.JAIMINHO_CONFIG = { 26 | "PERSIST_ALL_EVENTS": persist_all_events, 27 | "TIME_TO_DELETE": time_to_delete, 28 | "DELETE_AFTER_SEND": delete_after_send, 29 | "PUBLISH_STRATEGY": publish_strategy, 30 | } 31 | settings_module = import_module("jaiminho.settings") 32 | settings_module = reload(settings_module) 33 | 34 | assert getattr(settings_module, "persist_all_events") == persist_all_events 35 | assert getattr(settings_module, "time_to_delete") == time_to_delete 36 | assert getattr(settings_module, "delete_after_send") == delete_after_send 37 | assert getattr(settings_module, "publish_strategy") == publish_strategy 38 | -------------------------------------------------------------------------------- /jaiminho/migrations/0003_remove_event_action_remove_event_encoder_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.4 on 2022-07-26 18:55 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("jaiminho", "0002_event_encoder_event_function_signature_event_options"), 9 | ] 10 | 11 | operations = [ 12 | migrations.RemoveField( 13 | model_name="event", 14 | name="action", 15 | ), 16 | migrations.RemoveField( 17 | model_name="event", 18 | name="encoder", 19 | ), 20 | migrations.RemoveField( 21 | model_name="event", 22 | name="function_signature", 23 | ), 24 | migrations.RemoveField( 25 | model_name="event", 26 | name="options", 27 | ), 28 | migrations.RemoveField( 29 | model_name="event", 30 | name="payload", 31 | ), 32 | migrations.RemoveField( 33 | model_name="event", 34 | name="type", 35 | ), 36 | migrations.AddField( 37 | model_name="event", 38 | name="function", 39 | field=models.BinaryField(max_length=65535, null=True), 40 | ), 41 | migrations.AddField( 42 | model_name="event", 43 | name="kwargs", 44 | field=models.BinaryField(max_length=65535, null=True), 45 | ), 46 | migrations.AddField( 47 | model_name="event", 48 | name="message", 49 | field=models.BinaryField(max_length=65535, null=True), 50 | ), 51 | ] 52 | -------------------------------------------------------------------------------- /jaiminho/management/commands/events_relay.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from time import sleep 3 | 4 | from django.core.management import BaseCommand 5 | from jaiminho.relayer import EventRelayer 6 | 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | class Command(BaseCommand): 12 | event_relayer = EventRelayer() 13 | 14 | def add_arguments(self, parser): 15 | parser.add_argument( 16 | "--run-in-loop", 17 | action="store_true", 18 | help="Define if command should run in loop or just once", 19 | default=False, 20 | ) 21 | 22 | parser.add_argument( 23 | "--loop-interval", 24 | nargs="?", 25 | type=float, 26 | default=1, 27 | help="Define the sleep interval (in seconds) between each loop", 28 | ) 29 | parser.add_argument( 30 | "--stream", 31 | nargs="?", 32 | type=str, 33 | default=None, 34 | help="Define which stream events should be relayed. If not provided, all events will be relayed.", 35 | ) 36 | 37 | def handle(self, *args, **options): 38 | loop_interval = options["loop_interval"] 39 | run_in_loop = options["run_in_loop"] 40 | stream = options["stream"] 41 | 42 | print(f"run_in_loop: {run_in_loop}") 43 | print(f"loop_interval: {loop_interval}") 44 | print(f"stream: {stream}") 45 | if options["run_in_loop"]: 46 | log.info("EVENTS-RELAY-COMMAND: Started to relay events in loop mode") 47 | 48 | while True: 49 | self.event_relayer.relay(stream=options["stream"]) 50 | sleep(options["loop_interval"]) 51 | log.info("EVENTS-RELAY-COMMAND: Relay iteration finished") 52 | 53 | else: 54 | log.info("EVENTS-RELAY-COMMAND: Started to relay events only once") 55 | self.event_relayer.relay(stream=options["stream"]) 56 | log.info("EVENTS-RELAY-COMMAND: Relay finished") 57 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # io.open is needed for projects that support Python 2.7 5 | # It ensures open() defaults to text mode with universal newlines, 6 | # and accepts an argument to specify the text encoding 7 | # Python 3 only projects can skip this import 8 | from io import open 9 | 10 | from setuptools import setup, find_packages 11 | 12 | here = os.path.abspath(os.path.dirname(__file__)) 13 | 14 | # Get the long description from the README file 15 | with open(os.path.join(here, "README.md"), encoding="utf-8") as f: 16 | long_description = f.read() 17 | 18 | 19 | # The placeholder is to be able to install it as editable 20 | version = os.getenv("CIRCLE_TAG", os.getenv("CIRCLE_SHA1")) or "0.0.0" 21 | 22 | 23 | setup( 24 | name="django-jaiminho", 25 | version=version, 26 | description="A broker agnostic implementation of outbox and other message resilience patterns for Django apps", 27 | long_description=long_description, 28 | long_description_content_type="text/markdown", 29 | url="https://github.com/loadsmart/django-jaiminho", 30 | author="Loadsmart", 31 | author_email="jaiminho@loadsmart.com", 32 | classifiers=[ 33 | "Development Status :: 4 - Beta", 34 | "Intended Audience :: Developers", 35 | "License :: Other/Proprietary License", 36 | "Programming Language :: Python :: 3.8", 37 | "Programming Language :: Python :: 3.9", 38 | "Programming Language :: Python :: 3.10", 39 | "Programming Language :: Python :: 3.11", 40 | "Programming Language :: Python :: 3.12", 41 | ], 42 | packages=find_packages(exclude=["docs", "tests", "jaiminho_django_test_project"]), 43 | python_requires=">=3.8, <4", 44 | install_requires=["Django", "sentry_sdk", "dill==0.4.0"], 45 | project_urls={ 46 | "Documentation": "https://github.com/loadsmart/django-jaiminho/blob/master/README.md", 47 | "Source": "https://github.com/loadsmart/django-jaiminho", 48 | "Changelog": "https://github.com/loadsmart/django-jaiminho/blob/master/CHANGELOG.md", 49 | }, 50 | ) 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docs/source/reference/*.rst 2 | 3 | # Created by https://www.gitignore.io/api/python 4 | # Edit at https://www.gitignore.io/?templates=python 5 | 6 | ### Python ### 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don’t work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # celery beat schedule file 99 | celerybeat-schedule 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # End of https://www.gitignore.io/api/python 132 | 133 | .idea/ 134 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests_functional/test_send_to_outbox.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pytest 4 | import shutil 5 | from django.test import TestCase 6 | from django.core.management import call_command 7 | 8 | from jaiminho_django_test_project.management.commands import validate_events_relay 9 | from jaiminho.constants import PublishStrategyType 10 | from jaiminho.models import Event 11 | import jaiminho_django_test_project.send 12 | 13 | 14 | EVENTS_FOLDER_PATH = "./outbox_events_confirmation" 15 | 16 | 17 | @pytest.fixture 18 | def events_confirmation_folder(): 19 | if os.path.exists(EVENTS_FOLDER_PATH): 20 | shutil.rmtree(EVENTS_FOLDER_PATH) 21 | os.mkdir(EVENTS_FOLDER_PATH) 22 | yield 23 | if os.path.exists(EVENTS_FOLDER_PATH): 24 | shutil.rmtree(EVENTS_FOLDER_PATH) 25 | 26 | 27 | @pytest.mark.django_db 28 | class TestSendToOutbox: 29 | def test_should_relay_when_keep_order_strategy_from_decorator( 30 | self, mocker, events_confirmation_folder 31 | ): 32 | mocker.patch( 33 | "jaiminho.settings.publish_strategy", PublishStrategyType.PUBLISH_ON_COMMIT 34 | ) 35 | assert Event.objects.count() == 0 36 | 37 | first_args = [{"some": "data"}] 38 | second_args = [{"other": "data"}] 39 | first_file_path = f"{EVENTS_FOLDER_PATH}/event_1.json" 40 | second_file_path = f"{EVENTS_FOLDER_PATH}/event_2.json" 41 | 42 | with TestCase.captureOnCommitCallbacks(execute=True): 43 | jaiminho_django_test_project.send.notify_functional_to_stream_overwriting_strategy( 44 | *first_args, filepath=first_file_path 45 | ) 46 | jaiminho_django_test_project.send.notify_functional_to_stream_overwriting_strategy( 47 | *second_args, filepath=second_file_path 48 | ) 49 | 50 | assert Event.objects.count() == 2 51 | outbox_events = Event.objects.all() 52 | 53 | self.assertEvent(outbox_events[0]) 54 | self.assertEvent(outbox_events[1]) 55 | 56 | call_command( 57 | validate_events_relay.Command(), 58 | run_in_loop=False, 59 | stream=jaiminho_django_test_project.send.EXAMPLE_STREAM, 60 | ) 61 | 62 | relayed_events = Event.objects.all().order_by("id") 63 | assert relayed_events.count() == 2 64 | for event in relayed_events: 65 | assert event.sent_at is not None 66 | assert relayed_events[0].sent_at < relayed_events[1].sent_at 67 | 68 | assert os.path.exists(first_file_path) 69 | assert os.path.exists(second_file_path) 70 | 71 | first_file = open(first_file_path) 72 | second_file = open(second_file_path) 73 | assert json.load(first_file) == first_args 74 | assert json.load(second_file) == second_args 75 | 76 | def assertEvent(self, event): 77 | assert event.strategy == PublishStrategyType.KEEP_ORDER 78 | assert event.stream == jaiminho_django_test_project.send.EXAMPLE_STREAM 79 | assert event.sent_at is None 80 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests/management/commands/test_validate_event_cleaner.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | import pytest 4 | from django.core.management import call_command 5 | from django.utils import timezone 6 | 7 | from jaiminho.models import Event 8 | from jaiminho.tests.factories import EventFactory 9 | from jaiminho_django_test_project.management.commands import validate_event_cleaner 10 | 11 | pytestmark = pytest.mark.django_db 12 | 13 | 14 | class TestEventCleanerCommand: 15 | TIME_TO_DELETE = timedelta(days=5) 16 | 17 | @pytest.fixture 18 | def older_events(self): 19 | return EventFactory.create_batch( 20 | 2, sent_at=timezone.now() - self.TIME_TO_DELETE - timedelta(days=1) 21 | ) 22 | 23 | @pytest.fixture 24 | def newer_events(self): 25 | return EventFactory.create_batch( 26 | 2, sent_at=timezone.now() - self.TIME_TO_DELETE + timedelta(days=1) 27 | ) 28 | 29 | @pytest.fixture 30 | def not_sent_events(self): 31 | return EventFactory.create_batch(2, sent_at=None) 32 | 33 | @pytest.fixture 34 | def events_older_than_default_config(self): 35 | return EventFactory.create_batch( 36 | 2, sent_at=timezone.now() - self.TIME_TO_DELETE - timedelta(days=30) 37 | ) 38 | 39 | def test_command_with_misconfigured_time_to_delete_raises_error_and_doesnt_delete( 40 | self, mocker, older_events, newer_events, not_sent_events 41 | ): 42 | mocker.patch("jaiminho.settings.time_to_delete", "not-a-timedelta") 43 | 44 | assert len(Event.objects.all()) == 6 45 | 46 | with pytest.raises(AssertionError): 47 | call_command(validate_event_cleaner.Command()) 48 | 49 | assert len(Event.objects.all()) == 6 50 | 51 | def test_command_deletes_older_events( 52 | self, mocker, older_events, newer_events, not_sent_events, caplog 53 | ): 54 | mocker.patch("jaiminho.settings.time_to_delete", self.TIME_TO_DELETE) 55 | 56 | assert len(Event.objects.all()) == 6 57 | call_command(validate_event_cleaner.Command()) 58 | 59 | remaining_events = Event.objects.all() 60 | assert len(remaining_events) == 4 61 | assert set(remaining_events) == set([*newer_events, *not_sent_events]) 62 | assert "JAIMINHO-EVENT-CLEANER: Successfully deleted" in caplog.text 63 | 64 | def test_command_doesnt_delete_when_there_are_no_older_events( 65 | self, mocker, newer_events, not_sent_events, caplog 66 | ): 67 | mocker.patch("jaiminho.settings.time_to_delete", self.TIME_TO_DELETE) 68 | 69 | assert len(Event.objects.all()) == 4 70 | call_command(validate_event_cleaner.Command()) 71 | assert len(Event.objects.all()) == 4 72 | 73 | def test_command_without_time_to_delete_configuration_uses_default_7_days( 74 | self, 75 | older_events, 76 | newer_events, 77 | not_sent_events, 78 | events_older_than_default_config, 79 | ): 80 | assert len(Event.objects.all()) == 8 81 | call_command(validate_event_cleaner.Command()) 82 | 83 | remaining_events = Event.objects.all() 84 | assert set(remaining_events) == {*older_events, *newer_events, *not_sent_events} 85 | assert len(Event.objects.all()) == 6 86 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | tag-pattern: &tag-pattern 2 | only: /^\d+\.\d+\.\d+$/ 3 | 4 | version: 2.1 5 | 6 | commands: 7 | install-dependencies: 8 | steps: 9 | - run: 10 | command: | 11 | pip install tox 12 | 13 | jobs: 14 | tests38: 15 | working_directory: /home/circleci/project 16 | docker: 17 | - image: cimg/python:3.8 18 | steps: 19 | - checkout 20 | - install-dependencies 21 | - run: 22 | command: tox -e py38 23 | - store_test_results: 24 | path: build 25 | tests39: 26 | working_directory: /home/circleci/project 27 | docker: 28 | - image: cimg/python:3.9 29 | steps: 30 | - checkout 31 | - install-dependencies 32 | - run: 33 | command: tox -e py39 34 | - store_test_results: 35 | path: build 36 | tests310: 37 | working_directory: /home/circleci/project 38 | docker: 39 | - image: cimg/python:3.10 40 | steps: 41 | - checkout 42 | - install-dependencies 43 | - run: 44 | command: tox -e py310 45 | - store_test_results: 46 | path: build 47 | tests311: 48 | working_directory: /home/circleci/project 49 | docker: 50 | - image: cimg/python:3.11 51 | steps: 52 | - checkout 53 | - install-dependencies 54 | - run: 55 | command: tox -e py311 56 | - store_test_results: 57 | path: build 58 | tests312: 59 | working_directory: /home/circleci/project 60 | docker: 61 | - image: cimg/python:3.12 62 | steps: 63 | - checkout 64 | - install-dependencies 65 | - run: 66 | command: tox -e py312 67 | - store_test_results: 68 | path: build 69 | release: 70 | working_directory: /home/circleci/project 71 | docker: 72 | - image: cimg/python:3.8 73 | steps: 74 | - checkout 75 | - run: 76 | command: make release 77 | - persist_to_workspace: 78 | root: . 79 | paths: 80 | - "dist" 81 | 82 | workflows: 83 | version: 2 84 | main: 85 | jobs: 86 | - tests38: 87 | context: org-global 88 | filters: 89 | tags: 90 | only: /^\d+\.\d+\.\d+$/ 91 | - tests39: 92 | context: org-global 93 | filters: 94 | tags: 95 | only: /^\d+\.\d+\.\d+$/ 96 | - tests310: 97 | context: org-global 98 | filters: 99 | tags: 100 | only: /^\d+\.\d+\.\d+$/ 101 | - tests311: 102 | context: org-global 103 | filters: 104 | tags: 105 | only: /^\d+\.\d+\.\d+$/ 106 | - tests312: 107 | context: org-global 108 | filters: 109 | tags: 110 | only: /^\d+\.\d+\.\d+$/ 111 | - release: 112 | name: release 113 | context: org-global 114 | requires: 115 | - tests38 116 | - tests39 117 | - tests310 118 | - tests311 119 | - tests312 120 | filters: 121 | branches: 122 | ignore: /.*/ 123 | tags: 124 | only: /^\d+\.\d+\.\d+$/ 125 | -------------------------------------------------------------------------------- /jaiminho/relayer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import dill 3 | 4 | from jaiminho.constants import PublishStrategyType 5 | from jaiminho.models import Event 6 | from jaiminho.signals import ( 7 | event_published_by_events_relay, 8 | event_failed_to_publish_by_events_relay, 9 | get_event_payload, 10 | ) 11 | from jaiminho import settings 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def _capture_exception(exception): 18 | capture_exception = settings.default_capture_exception 19 | if capture_exception: 20 | capture_exception(exception) 21 | 22 | 23 | def _extract_original_func(event): 24 | fn = dill.loads(event.function) 25 | original_fn = getattr(fn, "original_func", fn) 26 | return original_fn 27 | 28 | 29 | class EventRelayer: 30 | def relay(self, stream=None): 31 | events_qs = Event.objects.filter(sent_at__isnull=True) 32 | events_qs = events_qs.filter(stream=stream) 33 | 34 | events_qs = events_qs.order_by("created_at") 35 | 36 | if not events_qs: 37 | logger.info("No failed events found.") 38 | return 39 | 40 | for event in events_qs: 41 | args = dill.loads(event.message) 42 | kwargs = dill.loads(event.kwargs) if event.kwargs else {} 43 | event_payload = get_event_payload(args) 44 | 45 | try: 46 | original_fn = _extract_original_func(event) 47 | if isinstance(args, tuple): 48 | original_fn(*args, **kwargs) 49 | else: 50 | original_fn(args, **kwargs) 51 | 52 | logger.info(f"JAIMINHO-EVENTS-RELAY: Event sent. Event {event}") 53 | 54 | if settings.delete_after_send: 55 | event.delete() 56 | logger.info( 57 | f"JAIMINHO-EVENTS-RELAY: Event deleted after success send. Event: {event}, Payload: {args}" 58 | ) 59 | else: 60 | event.mark_as_sent() 61 | logger.info( 62 | f"JAIMINHO-EVENTS-RELAY: Event marked as sent. Event: {event}, Payload: {args}" 63 | ) 64 | 65 | event_published_by_events_relay.send( 66 | sender=original_fn, event_payload=event_payload, args=args, **kwargs 67 | ) 68 | 69 | except (ModuleNotFoundError, AttributeError) as e: 70 | logger.warning( 71 | f"JAIMINHO-EVENTS-RELAY: Function does not exist anymore, Event: {event} | Error: {str(e)}" 72 | ) 73 | _capture_exception(e) 74 | 75 | if self.__stuck_on_error(event): 76 | logger.warning( 77 | f"JAIMINHO-EVENTS-RELAY: Events relaying are stuck due to failing Event: {event}" 78 | ) 79 | return 80 | 81 | except BaseException as e: 82 | logger.warning( 83 | f"JAIMINHO-EVENTS-RELAY: An error occurred when relaying event: {event} | Error: {str(e)}" 84 | ) 85 | original_fn = _extract_original_func(event) 86 | event_failed_to_publish_by_events_relay.send( 87 | sender=original_fn, event_payload=event_payload, args=args, **kwargs 88 | ) 89 | _capture_exception(e) 90 | 91 | if self.__stuck_on_error(event): 92 | logger.warning( 93 | f"JAIMINHO-EVENTS-RELAY: Events relaying are stuck due to failing Event: {event}" 94 | ) 95 | return 96 | 97 | def __stuck_on_error(self, event): 98 | if not event.strategy: 99 | return settings.publish_strategy == PublishStrategyType.KEEP_ORDER 100 | return event.strategy == PublishStrategyType.KEEP_ORDER 101 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for jaiminho_django_test_project project. 3 | 4 | Generated by 'django-admin startproject' using Django 4.0.4. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/4.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | from pathlib import Path 15 | 16 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 17 | BASE_DIR = Path(__file__).resolve().parent.parent 18 | 19 | 20 | # Quick-start development settings - unsuitable for production 21 | # See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/ 22 | 23 | SECRET_KEY = os.environ.get( 24 | "TEST_SECRET_KEY", "do-not-use-in-production-only-for-tests" 25 | ) 26 | 27 | # SECURITY WARNING: don't run with debug turned on in production! 28 | DEBUG = True 29 | 30 | ALLOWED_HOSTS = [] 31 | 32 | 33 | # Application definition 34 | 35 | INSTALLED_APPS = [ 36 | "django.contrib.admin", 37 | "django.contrib.auth", 38 | "django.contrib.contenttypes", 39 | "django.contrib.messages", 40 | "django.contrib.staticfiles", 41 | "jaiminho", 42 | "jaiminho_django_test_project.app", 43 | ] 44 | 45 | MIDDLEWARE = [ 46 | "django.middleware.security.SecurityMiddleware", 47 | "django.contrib.sessions.middleware.SessionMiddleware", 48 | "django.middleware.common.CommonMiddleware", 49 | "django.middleware.csrf.CsrfViewMiddleware", 50 | "django.contrib.auth.middleware.AuthenticationMiddleware", 51 | "django.contrib.messages.middleware.MessageMiddleware", 52 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 53 | ] 54 | 55 | ROOT_URLCONF = "jaiminho_django_test_project.urls" 56 | 57 | TEMPLATES = [ 58 | { 59 | "BACKEND": "django.template.backends.django.DjangoTemplates", 60 | "DIRS": [], 61 | "APP_DIRS": True, 62 | "OPTIONS": { 63 | "context_processors": [ 64 | "django.template.context_processors.debug", 65 | "django.template.context_processors.request", 66 | "django.contrib.auth.context_processors.auth", 67 | "django.contrib.messages.context_processors.messages", 68 | ], 69 | }, 70 | }, 71 | ] 72 | 73 | WSGI_APPLICATION = "jaiminho_django_test_project.wsgi.application" 74 | 75 | 76 | # Database 77 | # https://docs.djangoproject.com/en/4.0/ref/settings/#databases 78 | 79 | DATABASES = { 80 | "default": { 81 | "ENGINE": "django.db.backends.sqlite3", 82 | "NAME": BASE_DIR / "db.sqlite3", 83 | } 84 | } 85 | 86 | 87 | # Password validation 88 | # https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators 89 | 90 | AUTH_PASSWORD_VALIDATORS = [ 91 | { 92 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", 93 | }, 94 | { 95 | "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", 96 | }, 97 | { 98 | "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", 99 | }, 100 | { 101 | "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", 102 | }, 103 | ] 104 | 105 | 106 | # Internationalization 107 | # https://docs.djangoproject.com/en/4.0/topics/i18n/ 108 | 109 | LANGUAGE_CODE = "en-us" 110 | 111 | TIME_ZONE = "UTC" 112 | 113 | USE_I18N = True 114 | 115 | USE_TZ = True 116 | 117 | 118 | # Static files (CSS, JavaScript, Images) 119 | # https://docs.djangoproject.com/en/4.0/howto/static-files/ 120 | 121 | STATIC_URL = "static/" 122 | 123 | # Default primary key field type 124 | # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field 125 | 126 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 127 | 128 | 129 | # JAIMINHO 130 | 131 | JAIMINHO_CONFIG = { 132 | "PERSIST_ALL_EVENTS": False, 133 | } 134 | 135 | LOGLEVEL = os.environ.get("LOGLEVEL", "INFO") 136 | LOGGING = { 137 | "version": 1, 138 | "disable_existing_loggers": False, 139 | "handlers": { 140 | "console": { 141 | "class": "logging.StreamHandler", 142 | }, 143 | }, 144 | "root": { 145 | "handlers": ["console"], 146 | "level": "INFO", 147 | }, 148 | } 149 | -------------------------------------------------------------------------------- /jaiminho/publish_strategies.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from abc import ABC, abstractmethod 3 | import dill 4 | 5 | from django.db import transaction 6 | 7 | from jaiminho.constants import PublishStrategyType 8 | from jaiminho.models import Event 9 | from jaiminho.relayer import EventRelayer 10 | from jaiminho.signals import event_published, event_failed_to_publish, get_event_payload 11 | from jaiminho import settings 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def create_event_data(func_signature, args, kwargs, strategy, stream=None): 18 | return { 19 | "message": dill.dumps(args), 20 | "function": func_signature, 21 | "kwargs": dill.dumps(kwargs) if bool(kwargs) else None, 22 | "strategy": strategy, 23 | "stream": stream, 24 | } 25 | 26 | 27 | class BaseStrategy(ABC): 28 | @abstractmethod 29 | def publish(self, args, kwargs, func, stream=None): 30 | raise NotImplementedError 31 | 32 | 33 | class PublishOnCommitStrategy(BaseStrategy): 34 | def publish(self, args, kwargs, func, stream=None): 35 | func_signature = dill.dumps(func) 36 | event_data = create_event_data( 37 | func_signature, 38 | args, 39 | kwargs, 40 | PublishStrategyType.PUBLISH_ON_COMMIT, 41 | stream=stream, 42 | ) 43 | 44 | event = None 45 | if settings.persist_all_events: 46 | event = Event.objects.create(**event_data) 47 | logger.info( 48 | f"JAIMINHO-SAVE-TO-OUTBOX: Event created: Event {event}, Payload: {args}" 49 | ) 50 | 51 | on_commit_hook_kwargs = { 52 | "func": func, 53 | "event_data": event_data, 54 | "event": event, 55 | "args": args, 56 | "kwargs": kwargs, 57 | } 58 | transaction.on_commit(lambda: on_commit_hook(**on_commit_hook_kwargs)) 59 | logger.info( 60 | f"JAIMINHO-SAVE-TO-OUTBOX: On commit hook configured. Event: {event}" 61 | ) 62 | 63 | 64 | class KeepOrderStrategy(BaseStrategy): 65 | def publish(self, args, kwargs, func, stream=None): 66 | func_signature = dill.dumps(func) 67 | event_data = create_event_data( 68 | func_signature, 69 | args, 70 | kwargs, 71 | PublishStrategyType.KEEP_ORDER, 72 | stream=stream, 73 | ) 74 | event = Event.objects.create(**event_data) 75 | logger.info( 76 | f"JAIMINHO-SAVE-TO-OUTBOX: Event created: Event {event}, Payload: {args}" 77 | ) 78 | 79 | 80 | def create_publish_strategy(strategy_type): 81 | strategy_map = { 82 | PublishStrategyType.PUBLISH_ON_COMMIT: PublishOnCommitStrategy, 83 | PublishStrategyType.KEEP_ORDER: KeepOrderStrategy, 84 | } 85 | 86 | try: 87 | return strategy_map[strategy_type]() 88 | except KeyError as exc: 89 | raise ValueError(f"Unknow strategy type: {strategy_type}") 90 | 91 | 92 | def on_commit_hook(func, event, event_data, args, kwargs): 93 | event_payload = get_event_payload(args) 94 | 95 | try: 96 | func(*args, **kwargs) 97 | logger.info( 98 | f"JAIMINHO-ON-COMMIT-HOOK: Event sent successfully. Payload: {args}" 99 | ) 100 | 101 | event_published.send( 102 | sender=func, event_payload=event_payload, args=args, **kwargs 103 | ) 104 | except BaseException as exc: 105 | if not event: 106 | event = Event.objects.create(**event_data) 107 | 108 | logger.warning( 109 | f"JAIMINHO-ON-COMMIT-HOOK: Event failed to be published. Event: {event}, Payload: {args}, " 110 | f"Exception: {exc}" 111 | ) 112 | event_failed_to_publish.send( 113 | sender=func, event_payload=event_payload, args=args, **kwargs 114 | ) 115 | return 116 | 117 | if event: 118 | if settings.delete_after_send: 119 | logger.info( 120 | f"JAIMINHO-ON-COMMIT-HOOK: Event deleted after success send. Event: {event}, Payload: {args}" 121 | ) 122 | event.delete() 123 | else: 124 | logger.info( 125 | f"JAIMINHO-ON-COMMIT-HOOK: Event marked as sent. Event: {event}, Payload: {args}" 126 | ) 127 | event.mark_as_sent() 128 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jaiminho 2 | 3 | [![CircleCI](https://dl.circleci.com/status-badge/img/gh/loadsmart/django-jaiminho/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/loadsmart/django-jaiminho/tree/master) 4 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) 5 | 6 | A broker agnostic implementation of the outbox and other message resilience patterns for Django apps. 7 | 8 | ![Jaiminho](https://github.com/loadsmart/django-jaiminho/blob/master/assets/jaiminho.jpg?raw=true) 9 | 10 | ## Getting Started 11 | 12 | To use jaiminho with your project, you just need to do 6 steps: 13 | 14 | ### 1 - Install it 15 | 16 | ```sh 17 | python -m pip install jaiminho 18 | ``` 19 | 20 | ### 2 - Add jaiminho to the INSTALLED_APPS 21 | 22 | ```python 23 | INSTALLED_APPS = [ 24 | ... 25 | "jaiminho" 26 | ] 27 | ``` 28 | 29 | ### 3 - Run migrations 30 | 31 | ```sh 32 | python manage.py migrate 33 | ``` 34 | 35 | ### 4 - Configure jaiminho options in Django settings.py: 36 | ```python 37 | JAIMINHO_CONFIG = { 38 | "PERSIST_ALL_EVENTS": False, 39 | "DELETE_AFTER_SEND": True, 40 | "DEFAULT_ENCODER": DjangoJSONEncoder, 41 | "PUBLISH_STRATEGY": "publish-on-commit", 42 | } 43 | 44 | ``` 45 | 46 | ### 5 - Decorate your functions with @save_to_outbox 47 | ```python 48 | from jaiminho.send import save_to_outbox 49 | 50 | @save_to_outbox 51 | def any_external_call(**kwargs): 52 | # do something 53 | return 54 | ``` 55 | 56 | ### 6 - Run the relay events command 57 | 58 | ``` 59 | python manage.py events_relay --run-in-loop --loop-interval 1 60 | 61 | ``` 62 | 63 | If you don't use `--run-in-loop` option, the relay command will run only 1 time. This is useful in case you want to configure it as a cronjob. 64 | 65 | 66 | ## Details 67 | 68 | Jaiminho `@save_to_outbox` decorator will **intercept** decorated function and **persist** it in a **database table** in the same **transaction** that is active in the decorated function context. The event relay **command**, is a **separated process** that fetches the rows from this table and execute the functions. When an outage happens, the event relay command will **keep retrying until it succeeds**. This way, **eventual consistency is ensured** by design. 69 | 70 | ### Configuration options 71 | 72 | - `PUBLISH_STRATEGY` - Strategy used to publish events (publish-on-commit, keep-order) 73 | - `PERSIST_ALL_EVENTS` - Saves all events and not only the ones that fail, default is `False`. Only applicable for `{ "PUBLISH_STRATEGY": "publish-on-commit" }` since all events needs to be stored on keep-order strategy. 74 | - `DELETE_AFTER_SEND` - Delete the event from the outbox table immediately, after a successful send 75 | - `DEFAULT_ENCODER` - Default Encoder for the payload (overwritable in the function call) 76 | 77 | ### Strategies 78 | 79 | #### Keep Order 80 | This strategy is similar to transactional outbox [described by Chris Richardson](https://microservices.io/patterns/data/transactional-outbox.html). The decorated function intercepts the function call and saves it on the local DB to be executed later. A separate command relayer will keep polling local DB and executing those functions in the same order it was stored. 81 | Be carefully with this approach, **if any execution fails, the relayer will get stuck** as it would not be possible to guarantee delivery order. 82 | 83 | #### Publish on commit 84 | 85 | This strategy will always execute the decorated function after current transaction commit. With this approach, we don't depend on a relayer (separate process / cronjob) to execute the decorated function and deliver the message. Failed items will only be retried 86 | through relayer. Although this solution has a better performance as only failed items is delivered by the relay command, **we cannot guarantee delivery order**. 87 | 88 | 89 | ### Relay Command 90 | We already provide a command to relay items from DB, [EventRelayCommand](https://github.com/loadsmart/django-jaiminho/blob/master/jaiminho/management/commands/events_relay.py). The way you should configure depends on the strategy you choose. 91 | For example, on **Publish on Commit Strategy** you can configure a cronjob to run every a couple of minutes since only failed items are published by the command relay. If you are using **Keep Order Strategy**, you should run relay command in loop mode as all items will be published by the command, e.g `call_command(events_relay.Command(), run_in_loop=True, loop_interval=0.1)`. 92 | 93 | 94 | ### How to clean older events 95 | 96 | You can use Jaiminho's [EventCleanerCommand](https://github.com/loadsmart/django-jaiminho/blob/master/jaiminho/management/commands/event_cleaner.py) in order to do that. It will query for all events that were sent before a given time interval (e.g. last 7 days) and will delete them from the outbox table. 97 | 98 | The default time interval is `7 days`. You can use the `TIME_TO_DELETE` setting to change it. It should be added to `JAIMINHO_CONFIG` and must be a valid [timedelta](https://docs.python.org/3/library/datetime.html#timedelta-objects). 99 | 100 | ### Running as cron jobs 101 | 102 | You can run those commands in a cron job. Here are some config examples: 103 | 104 | ```yaml 105 | - name: relay-failed-outbox-events 106 | schedule: "*/15 * * * *" 107 | suspend: false 108 | args: 109 | - ddtrace-run 110 | - python 111 | - manage.py 112 | - events_relay 113 | resources: 114 | requests: 115 | cpu: 1 116 | limits: 117 | memory: 384Mi 118 | 119 | - name: delete-old-outbox-events 120 | schedule: "0 5 * * *" 121 | suspend: false 122 | args: 123 | - ddtrace-run 124 | - python 125 | - manage.py 126 | - event_cleaner 127 | resources: 128 | requests: 129 | cpu: 1 130 | limits: 131 | memory: 384Mi 132 | ``` 133 | 134 | ### Relay per stream and Overwrite publish strategy 135 | 136 | Different streams can have different requirements. You can save separate events per streams by using the `@save_to_outbox_stream` decorator: 137 | 138 | ````python 139 | @save_to_outbox_stream("my-stream") 140 | def any_external_call(payload, **kwargs): 141 | # do something 142 | pass 143 | ```` 144 | 145 | you can also overwrite publish strategy configure on settings: 146 | 147 | ````python 148 | @save_to_outbox_stream("my-stream", PublishStrategyType.KEEP_ORDER) 149 | def any_external_call(payload, **kwargs): 150 | # do something 151 | pass 152 | ```` 153 | 154 | And then, run relay command with stream filter option 155 | ````shell 156 | python manage.py relay_event True 0.1 my-stream 157 | ```` 158 | 159 | In the example above, `True` is the option for run_in_loop; `0.1` for loop_interval; and `my_stream` is the name of the stream. 160 | 161 | ### Signals 162 | 163 | Jaiminho triggers the following Django signals: 164 | 165 | | Signal | Description | 166 | |-------------------------|---------------------------------------------------------------------------------| 167 | | event_published | Triggered when an event is sent successfully | 168 | | event_failed_to_publish | Triggered when an event is not sent, being added to the Outbox table queue | 169 | 170 | 171 | ### How to collect metrics from Jaiminho? 172 | 173 | You could use the Django signals triggered by Jaiminho to collect metrics. 174 | Consider the following code as example: 175 | 176 | ````python 177 | from django.dispatch import receiver 178 | 179 | @receiver(event_published) 180 | def on_event_sent(sender, event_payload, **kwargs): 181 | metrics.count(f"event_sent_successfully {event_payload.get('type')}") 182 | 183 | @receiver(event_failed_to_publish) 184 | def on_event_send_error(sender, event_payload, **kwargs): 185 | metrics.count(f"event_failed {event_payload.get('type')}") 186 | 187 | ```` 188 | 189 | ### Jaiminho with Celery 190 | 191 | Jaiminho can be very useful for adding reliability to Celery workflows. Writing to the database and enqueuing Celery tasks in the same workflow is very common in many applications, and this pattern can benefit greatly from the outbox pattern to ensure message delivery reliability. 192 | 193 | Instead of configuring the `@save_to_outbox` decorator for every individual Celery task, you can integrate it at the Celery class level by overriding the `send_task` method, which is used by Celery to enqueue new tasks. This way, all tasks automatically benefit from the outbox pattern without requiring individual configuration. 194 | 195 | Here's how to implement this: 196 | 197 | ```python 198 | from celery import Celery 199 | from jaiminho import save_to_outbox 200 | 201 | 202 | class CeleryWithJaiminho(Celery): 203 | """ 204 | Custom Celery class that inherits from Celery base class 205 | and adds Jaiminho functionality 206 | """ 207 | 208 | @save_to_outbox 209 | def send_task(self, *args, **kwargs): 210 | """Send task with outbox pattern for reliability""" 211 | return super().send_task(*args, **kwargs) 212 | 213 | 214 | app = CeleryWithJaiminho("tasks") 215 | ``` 216 | 217 | With this approach, all tasks sent through your Celery app will automatically use the outbox pattern, ensuring that task enqueuing is resilient to transient failures and network issues. 218 | 219 | ## Development 220 | 221 | Create a virtualenv 222 | 223 | ```bash 224 | virtualenv venv 225 | pip install -r requirements-dev.txt 226 | tox -e py39 227 | ``` 228 | ## Collaboration 229 | 230 | If you want to improve or suggest improvements, check our [CONTRIBUTING.md](https://github.com/loadsmart/django-jaiminho/blob/master/CONTRIBUTING.md) file. 231 | 232 | 233 | ## License 234 | 235 | This project is licensed under MIT License. 236 | 237 | ## Security 238 | 239 | If you have any security concern or report feel free to reach out to security@loadsmart.com; 240 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests/management/commands/test_validate_relay_events.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from unittest import mock 3 | from unittest.mock import call 4 | 5 | import dill 6 | import pytest 7 | from dateutil.tz import UTC 8 | from django.core.management import call_command 9 | from django.core.serializers.json import DjangoJSONEncoder 10 | from freezegun import freeze_time 11 | 12 | from jaiminho.constants import PublishStrategyType 13 | from jaiminho.models import Event 14 | from jaiminho.relayer import EventRelayer 15 | from jaiminho.tests.factories import EventFactory 16 | from jaiminho_django_test_project.management.commands import validate_events_relay 17 | from jaiminho_django_test_project.send import ( 18 | notify, 19 | notify_without_decorator, 20 | notify_to_stream, 21 | ) 22 | 23 | pytestmark = pytest.mark.django_db 24 | 25 | 26 | class TestValidateEventsRelay: 27 | @pytest.fixture 28 | def mock_log_metric(self, mocker): 29 | return mocker.patch("jaiminho_django_test_project.app.signals.log_metric") 30 | 31 | @pytest.fixture 32 | def mock_capture_exception(self, mocker): 33 | return mocker.patch( 34 | "jaiminho_django_test_project.management.commands.validate_events_relay.Command.capture_message_fn" 35 | ) 36 | 37 | @pytest.fixture 38 | def mock_event_published_signal(self, mocker): 39 | return mocker.patch("jaiminho.publish_strategies.event_published.send") 40 | 41 | @pytest.fixture 42 | def mock_event_failed_to_publish_signal(self, mocker): 43 | return mocker.patch("jaiminho.publish_strategies.event_failed_to_publish.send") 44 | 45 | @pytest.fixture 46 | def mock_event_failed_to_publish_by_events_relay_signal(self, mocker): 47 | return mocker.patch( 48 | "jaiminho.management.commands.events_relay.event_failed_to_publish_by_events_relay.send" 49 | ) 50 | 51 | @pytest.fixture 52 | def mock_internal_notify(self, mocker): 53 | return mocker.patch("jaiminho_django_test_project.send.internal_notify") 54 | 55 | @pytest.fixture 56 | def mock_internal_notify_fail(self, mocker): 57 | mock = mocker.patch("jaiminho_django_test_project.send.internal_notify") 58 | mock.side_effect = Exception("Some error") 59 | return mock 60 | 61 | @pytest.fixture 62 | def failed_event(self): 63 | return EventFactory( 64 | function=dill.dumps(notify), message=dill.dumps(({"b": 1},)) 65 | ) 66 | 67 | @pytest.fixture 68 | def failed_event_with_kwargs(self): 69 | return EventFactory( 70 | function=dill.dumps(notify), kwargs=dill.dumps({"ab": 2, "ac": 3}) 71 | ) 72 | 73 | @pytest.fixture 74 | def successful_event(self): 75 | return EventFactory( 76 | function=dill.dumps(notify), 77 | sent_at=datetime(2022, 2, 19, tzinfo=UTC), 78 | ) 79 | 80 | @pytest.fixture 81 | def mock_capture_exception_fn(self): 82 | return mock.Mock() 83 | 84 | @pytest.fixture 85 | def mock_should_delete_after_send(self, mocker): 86 | return mocker.patch("jaiminho.settings.delete_after_send", True) 87 | 88 | @pytest.fixture 89 | def mock_should_not_delete_after_send(self, mocker): 90 | return mocker.patch("jaiminho.settings.delete_after_send", False) 91 | 92 | @pytest.mark.parametrize( 93 | "publish_strategy", 94 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 95 | ) 96 | def test_relay_failed_event( 97 | self, 98 | mock_log_metric, 99 | failed_event, 100 | mock_internal_notify, 101 | mock_should_not_delete_after_send, 102 | publish_strategy, 103 | mocker, 104 | ): 105 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 106 | 107 | assert Event.objects.all().count() == 1 108 | 109 | with freeze_time("2022-10-31"): 110 | call_command(validate_events_relay.Command()) 111 | 112 | mock_internal_notify.assert_called_once() 113 | mock_internal_notify.assert_called_with(*dill.loads(failed_event.message)) 114 | assert Event.objects.all().count() == 1 115 | event = Event.objects.all()[0] 116 | assert event.sent_at == datetime(2022, 10, 31, tzinfo=UTC) 117 | 118 | @pytest.mark.parametrize( 119 | "publish_strategy", 120 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 121 | ) 122 | def test_relay_failed_event_when_message_is_not_a_tuple( 123 | self, 124 | mock_log_metric, 125 | mock_internal_notify, 126 | mock_should_not_delete_after_send, 127 | publish_strategy, 128 | mocker, 129 | ): 130 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 131 | 132 | payload = {"b": 1} 133 | failed_event = EventFactory( 134 | function=dill.dumps(notify), 135 | message=dill.dumps(payload), 136 | ) 137 | 138 | assert Event.objects.all().count() == 1 139 | 140 | with freeze_time("2022-10-31"): 141 | call_command(validate_events_relay.Command()) 142 | 143 | assert Event.objects.all().count() == 1 144 | event = Event.objects.all()[0] 145 | assert event == failed_event 146 | assert event.sent_at == datetime(2022, 10, 31, tzinfo=UTC) 147 | mock_internal_notify.assert_called_once() 148 | mock_internal_notify.assert_called_with(payload) 149 | 150 | @pytest.mark.parametrize( 151 | "publish_strategy", 152 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 153 | ) 154 | def test_relay_failed_event_from_empty_stream( 155 | self, 156 | mock_log_metric, 157 | mock_internal_notify, 158 | mock_should_not_delete_after_send, 159 | publish_strategy, 160 | mocker, 161 | ): 162 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 163 | first_event = EventFactory( 164 | function=dill.dumps(notify), 165 | message=dill.dumps(({"b": 1},)), 166 | ) 167 | second_event = EventFactory( 168 | function=dill.dumps(notify_to_stream), 169 | stream="my-stream", 170 | message=dill.dumps(({"b": 2},)), 171 | ) 172 | third_event = EventFactory( 173 | function=dill.dumps(notify_to_stream), 174 | stream="my-other-stream", 175 | message=dill.dumps(({"b": 3},)), 176 | ) 177 | assert Event.objects.all().count() == 3 178 | 179 | with freeze_time("2022-10-31"): 180 | call_command(validate_events_relay.Command()) 181 | 182 | mock_internal_notify.assert_called_once_with(*dill.loads(first_event.message)) 183 | 184 | assert Event.objects.all().count() == 3 185 | assert Event.objects.get(id=first_event.id).sent_at == datetime( 186 | 2022, 10, 31, tzinfo=UTC 187 | ) 188 | assert Event.objects.get(id=second_event.id).sent_at is None 189 | assert Event.objects.get(id=third_event.id).sent_at is None 190 | 191 | @pytest.mark.parametrize( 192 | "publish_strategy", 193 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 194 | ) 195 | def test_relay_failed_event_from_specific_stream( 196 | self, 197 | mock_log_metric, 198 | mock_internal_notify, 199 | mock_should_not_delete_after_send, 200 | publish_strategy, 201 | mocker, 202 | ): 203 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 204 | first_event = EventFactory( 205 | function=dill.dumps(notify), 206 | message=dill.dumps(({"b": 1},)), 207 | ) 208 | second_event = EventFactory( 209 | function=dill.dumps(notify_to_stream), 210 | stream="my-stream", 211 | message=dill.dumps(({"b": 2},)), 212 | ) 213 | third_event = EventFactory( 214 | function=dill.dumps(notify_to_stream), 215 | stream="my-other-stream", 216 | message=dill.dumps(({"b": 3},)), 217 | ) 218 | assert Event.objects.all().count() == 3 219 | 220 | with freeze_time("2022-10-31"): 221 | call_command( 222 | validate_events_relay.Command(), 223 | stream="my-stream", 224 | ) 225 | 226 | mock_internal_notify.assert_called_once_with(*dill.loads(second_event.message)) 227 | 228 | assert Event.objects.all().count() == 3 229 | assert Event.objects.get(id=first_event.id).sent_at is None 230 | assert Event.objects.get(id=second_event.id).sent_at == datetime( 231 | 2022, 10, 31, tzinfo=UTC 232 | ) 233 | assert Event.objects.get(id=third_event.id).sent_at is None 234 | 235 | def test_relay_must_loop_when_run_in_loop_using_kwargs( 236 | self, 237 | mock_log_metric, 238 | mocker, 239 | ): 240 | event_relayer_mock = mocker.MagicMock(spec=EventRelayer) 241 | event_relayer_mock.relay.side_effect = [None, None, Exception()] 242 | 243 | with pytest.raises(Exception): 244 | command = validate_events_relay.Command() 245 | command.event_relayer = event_relayer_mock 246 | call_command(command, run_in_loop=True, loop_interval=0.1) 247 | 248 | assert event_relayer_mock.relay.call_count == 3 249 | 250 | def test_relay_must_loop_when_run_using_param( 251 | self, 252 | mock_log_metric, 253 | mocker, 254 | ): 255 | event_relayer_mock = mocker.MagicMock(spec=EventRelayer) 256 | event_relayer_mock.relay.side_effect = [None, None, Exception()] 257 | 258 | with pytest.raises(Exception): 259 | command = validate_events_relay.Command() 260 | command.event_relayer = event_relayer_mock 261 | call_command(command, "--run-in-loop", "--loop-interval", 0.1) 262 | 263 | assert event_relayer_mock.relay.call_count == 3 264 | 265 | def test_does_not_run_in_loop_by_default( 266 | self, 267 | mock_log_metric, 268 | mocker, 269 | ): 270 | event_relayer_mock = mocker.MagicMock(spec=EventRelayer) 271 | event_relayer_mock.relay.side_effect = [None, None, Exception()] 272 | 273 | command = validate_events_relay.Command() 274 | command.event_relayer = event_relayer_mock 275 | call_command(command) 276 | 277 | assert event_relayer_mock.relay.call_count == 1 278 | 279 | def test_does_not_run_in_loop_when_receiving_false_as_kwargs( 280 | self, 281 | mock_log_metric, 282 | mocker, 283 | ): 284 | event_relayer_mock = mocker.MagicMock(spec=EventRelayer) 285 | event_relayer_mock.relay.side_effect = [None, None, Exception()] 286 | 287 | command = validate_events_relay.Command() 288 | command.event_relayer = event_relayer_mock 289 | call_command(command, run_in_loop=False) 290 | 291 | assert event_relayer_mock.relay.call_count == 1 292 | 293 | def test_arguments_are_optionals_when_passing_the_stream_arg( 294 | self, 295 | mock_log_metric, 296 | mock_internal_notify, 297 | mock_should_not_delete_after_send, 298 | mocker, 299 | ): 300 | mocker.patch( 301 | "jaiminho.settings.publish_strategy", PublishStrategyType.PUBLISH_ON_COMMIT 302 | ) 303 | args = ({"b": 1},) 304 | event = EventFactory( 305 | function=dill.dumps(notify_to_stream), 306 | stream="my-stream", 307 | message=dill.dumps(args), 308 | ) 309 | 310 | with freeze_time("2022-10-31"): 311 | call_command( 312 | validate_events_relay.Command(), 313 | "--stream", 314 | "my-stream", 315 | ) 316 | 317 | mock_internal_notify.assert_called_once_with(*dill.loads(event.message)) 318 | 319 | assert Event.objects.all().count() == 1 320 | assert Event.objects.get(id=event.id).sent_at == datetime( 321 | 2022, 10, 31, tzinfo=UTC 322 | ) 323 | 324 | @pytest.mark.parametrize( 325 | "publish_strategy", 326 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 327 | ) 328 | def test_relay_failed_event_should_delete_after_send( 329 | self, 330 | mock_log_metric, 331 | failed_event, 332 | mock_internal_notify, 333 | mock_should_delete_after_send, 334 | publish_strategy, 335 | mocker, 336 | ): 337 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 338 | assert Event.objects.all().count() == 1 339 | 340 | with freeze_time("2022-10-31"): 341 | call_command(validate_events_relay.Command()) 342 | 343 | mock_internal_notify.assert_called_once() 344 | mock_internal_notify.assert_called_with( 345 | *dill.loads(failed_event.message), 346 | ) 347 | assert Event.objects.all().count() == 0 348 | 349 | @pytest.mark.parametrize( 350 | "publish_strategy", 351 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 352 | ) 353 | def test_trigger_the_correct_signal_when_resent_successfully( 354 | self, 355 | failed_event, 356 | mock_log_metric, 357 | mock_internal_notify, 358 | mock_event_published_signal, 359 | publish_strategy, 360 | mocker, 361 | ): 362 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 363 | 364 | call_command(validate_events_relay.Command()) 365 | 366 | mock_internal_notify.assert_called_once() 367 | mock_event_published_signal.assert_not_called() 368 | expected_args = dill.loads(failed_event.message) 369 | mock_log_metric.assert_called_once_with( 370 | "event-published-through-outbox", 371 | expected_args[0], 372 | args=expected_args, 373 | ) 374 | 375 | @pytest.mark.parametrize( 376 | "publish_strategy", 377 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 378 | ) 379 | def test_trigger_the_correct_signal_when_resent_successfully_with_kwargs( 380 | self, 381 | failed_event_with_kwargs, 382 | mock_log_metric, 383 | mock_internal_notify, 384 | mock_event_published_signal, 385 | publish_strategy, 386 | mocker, 387 | ): 388 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 389 | 390 | call_command(validate_events_relay.Command()) 391 | 392 | mock_internal_notify.assert_called_once() 393 | mock_event_published_signal.assert_not_called() 394 | expected_args = dill.loads(failed_event_with_kwargs.message) 395 | mock_log_metric.assert_called_once_with( 396 | "event-published-through-outbox", 397 | expected_args[0], 398 | args=expected_args, 399 | **dill.loads(failed_event_with_kwargs.kwargs), 400 | ) 401 | 402 | @pytest.mark.parametrize( 403 | "publish_strategy", 404 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 405 | ) 406 | def test_trigger_the_correct_signal_when_resent_failed( 407 | self, 408 | failed_event, 409 | mock_log_metric, 410 | mock_internal_notify_fail, 411 | mock_event_failed_to_publish_signal, 412 | publish_strategy, 413 | mocker, 414 | ): 415 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 416 | 417 | call_command(validate_events_relay.Command()) 418 | 419 | mock_internal_notify_fail.assert_called_once() 420 | mock_event_failed_to_publish_signal.assert_not_called() 421 | expected_args = dill.loads(failed_event.message) 422 | mock_log_metric.assert_called_once_with( 423 | "event-failed-to-publish-through-outbox", 424 | expected_args[0], 425 | args=expected_args, 426 | ) 427 | 428 | @pytest.mark.parametrize( 429 | "publish_strategy", 430 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 431 | ) 432 | def test_trigger_the_correct_signal_when_resent_failed_with_kwargs( 433 | self, 434 | failed_event_with_kwargs, 435 | mock_log_metric, 436 | mock_internal_notify_fail, 437 | mock_event_failed_to_publish_signal, 438 | publish_strategy, 439 | mocker, 440 | ): 441 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 442 | 443 | call_command(validate_events_relay.Command()) 444 | 445 | mock_internal_notify_fail.assert_called_once() 446 | mock_event_failed_to_publish_signal.assert_not_called() 447 | expected_args = dill.loads(failed_event_with_kwargs.message) 448 | mock_log_metric.assert_called_once_with( 449 | "event-failed-to-publish-through-outbox", 450 | expected_args[0], 451 | args=expected_args, 452 | **dill.loads(failed_event_with_kwargs.kwargs), 453 | ) 454 | 455 | @pytest.mark.parametrize( 456 | "publish_strategy", 457 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 458 | ) 459 | def test_doest_not_relay_when_does_not_exist_failed_events( 460 | self, 461 | successful_event, 462 | caplog, 463 | publish_strategy, 464 | mocker, 465 | ): 466 | assert Event.objects.filter(sent_at__isnull=True).count() == 0 467 | assert Event.objects.filter(sent_at__isnull=False).count() == 1 468 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 469 | 470 | call_command(validate_events_relay.Command()) 471 | 472 | assert "No failed events found." in caplog.text 473 | assert Event.objects.all().count() == 1 474 | 475 | @pytest.mark.parametrize( 476 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 477 | ) 478 | def test_relay_every_event_even_at_lest_one_fail( 479 | self, 480 | mock_internal_notify_fail, 481 | publish_strategy, 482 | mocker, 483 | ): 484 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 485 | args1 = ({"b": 1},) 486 | args2 = ({"b": 2},) 487 | event_1 = EventFactory( 488 | function=dill.dumps(notify), 489 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "1"}), 490 | message=dill.dumps(args1), 491 | ) 492 | event_2 = EventFactory( 493 | function=dill.dumps(notify), 494 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "2"}), 495 | message=dill.dumps(args2), 496 | ) 497 | 498 | call_command(validate_events_relay.Command()) 499 | 500 | call_1 = call(args1[0], encoder=DjangoJSONEncoder, a="1") 501 | call_2 = call(args2[0], encoder=DjangoJSONEncoder, a="2") 502 | mock_internal_notify_fail.assert_has_calls([call_1, call_2], any_order=True) 503 | 504 | @pytest.mark.parametrize("publish_strategy", (PublishStrategyType.KEEP_ORDER,)) 505 | def test_relay_stuck_when_one_fail( 506 | self, 507 | mock_internal_notify_fail, 508 | publish_strategy, 509 | mocker, 510 | caplog, 511 | ): 512 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 513 | 514 | args1 = ({"b": 1},) 515 | args2 = ({"b": 2},) 516 | event_1 = EventFactory( 517 | function=dill.dumps(notify), 518 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "1"}), 519 | strategy=publish_strategy, 520 | message=dill.dumps(args1), 521 | ) 522 | event_2 = EventFactory( 523 | function=dill.dumps(notify), 524 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "2"}), 525 | strategy=publish_strategy, 526 | message=dill.dumps(args2), 527 | ) 528 | 529 | call_command(validate_events_relay.Command()) 530 | mock_internal_notify_fail.assert_called_once_with( 531 | args1[0], 532 | encoder=DjangoJSONEncoder, 533 | a="1", 534 | ) 535 | assert "Events relaying are stuck due to failing Event" in caplog.text 536 | 537 | @pytest.mark.parametrize("publish_strategy", (PublishStrategyType.KEEP_ORDER,)) 538 | def test_relay_stuck_when_one_fail_and_no_strategy_on_event( 539 | self, 540 | mock_internal_notify_fail, 541 | publish_strategy, 542 | mocker, 543 | caplog, 544 | ): 545 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 546 | args1 = ({"b": 1},) 547 | args2 = ({"b": 2},) 548 | event_1 = EventFactory( 549 | function=dill.dumps(notify), 550 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "1"}), 551 | message=dill.dumps(args1), 552 | ) 553 | event_2 = EventFactory( 554 | function=dill.dumps(notify), 555 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "2"}), 556 | message=dill.dumps(args2), 557 | ) 558 | 559 | call_command(validate_events_relay.Command()) 560 | mock_internal_notify_fail.assert_called_once_with( 561 | args1[0], 562 | encoder=DjangoJSONEncoder, 563 | a="1", 564 | ) 565 | assert "Events relaying are stuck due to failing Event" in caplog.text 566 | 567 | @pytest.mark.parametrize( 568 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 569 | ) 570 | def test_relay_not_stuck_when_one_fail_and_no_strategy_on_event( 571 | self, 572 | mock_internal_notify_fail, 573 | publish_strategy, 574 | mocker, 575 | caplog, 576 | ): 577 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 578 | 579 | args1 = ({"b": 1},) 580 | args2 = ({"b": 2},) 581 | event_1 = EventFactory( 582 | function=dill.dumps(notify), 583 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "1"}), 584 | message=dill.dumps(args1), 585 | ) 586 | event_2 = EventFactory( 587 | function=dill.dumps(notify), 588 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "2"}), 589 | message=dill.dumps(args2), 590 | ) 591 | 592 | call_command(validate_events_relay.Command()) 593 | call_1 = call(args1[0], encoder=DjangoJSONEncoder, a="1") 594 | call_2 = call(args2[0], encoder=DjangoJSONEncoder, a="2") 595 | mock_internal_notify_fail.assert_has_calls([call_1, call_2], any_order=True) 596 | assert "Events relaying are stuck due to failing Event" not in caplog.text 597 | 598 | @pytest.mark.parametrize("publish_strategy", (PublishStrategyType.KEEP_ORDER,)) 599 | def test_relay_stuck_when_one_fail_and_specific_stream( 600 | self, 601 | mock_internal_notify_fail, 602 | publish_strategy, 603 | mocker, 604 | caplog, 605 | ): 606 | args1 = ({"b": 1},) 607 | args2 = ({"b": 2},) 608 | event_1 = EventFactory( 609 | function=dill.dumps(notify), 610 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "1"}), 611 | strategy=publish_strategy, 612 | stream="my-stream", 613 | message=dill.dumps(args1), 614 | ) 615 | event_2 = EventFactory( 616 | function=dill.dumps(notify), 617 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "2"}), 618 | strategy=publish_strategy, 619 | stream="my-stream", 620 | message=dill.dumps(args2), 621 | ) 622 | 623 | call_command( 624 | validate_events_relay.Command(), 625 | stream="my-stream", 626 | ) 627 | mock_internal_notify_fail.assert_called_once_with( 628 | args1[0], 629 | encoder=DjangoJSONEncoder, 630 | a="1", 631 | ) 632 | assert "Events relaying are stuck due to failing Event" in caplog.text 633 | 634 | @pytest.mark.parametrize( 635 | "publish_strategy", 636 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 637 | ) 638 | def test_events_ordered_by_created_by_relay( 639 | self, mock_internal_notify, publish_strategy, mocker 640 | ): 641 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 642 | 643 | message1 = ({"b": 1},) 644 | message2 = ({"b": 2},) 645 | message3 = ({"b": 3},) 646 | 647 | with freeze_time("2022-01-03"): 648 | event_1 = EventFactory( 649 | function=dill.dumps(notify), 650 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "1"}), 651 | message=dill.dumps(message1), 652 | ) 653 | with freeze_time("2022-01-01"): 654 | event_2 = EventFactory( 655 | function=dill.dumps(notify), 656 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "2"}), 657 | message=dill.dumps(message2), 658 | ) 659 | with freeze_time("2022-01-02"): 660 | event_3 = EventFactory( 661 | function=dill.dumps(notify), 662 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder, "a": "3"}), 663 | message=dill.dumps(message3), 664 | ) 665 | 666 | call_command(validate_events_relay.Command()) 667 | 668 | call_1 = call(message1[0], encoder=DjangoJSONEncoder, a="1") 669 | call_2 = call(message2[0], encoder=DjangoJSONEncoder, a="2") 670 | call_3 = call(message3[0], encoder=DjangoJSONEncoder, a="3") 671 | mock_internal_notify.assert_has_calls([call_2, call_3, call_1], any_order=False) 672 | 673 | @pytest.mark.parametrize( 674 | "publish_strategy", 675 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 676 | ) 677 | def test_relay_message_when_notify_function_is_not_decorated( 678 | self, 679 | mock_internal_notify, 680 | mock_should_not_delete_after_send, 681 | publish_strategy, 682 | mocker, 683 | ): 684 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 685 | args = ({"a": 1},) 686 | event = EventFactory( 687 | function=dill.dumps(notify_without_decorator), 688 | message=dill.dumps(args), 689 | kwargs=dill.dumps({"encoder": DjangoJSONEncoder}), 690 | ) 691 | 692 | with freeze_time("2022-10-31"): 693 | call_command(validate_events_relay.Command()) 694 | 695 | mock_internal_notify.assert_called_once() 696 | mock_internal_notify.assert_called_with(args[0], encoder=DjangoJSONEncoder) 697 | assert Event.objects.all().count() == 1 698 | event = Event.objects.all()[0] 699 | assert event.sent_at == datetime(2022, 10, 31, tzinfo=UTC) 700 | 701 | @pytest.mark.parametrize( 702 | "publish_strategy", 703 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 704 | ) 705 | def test_dont_create_another_event_when_relay_fails( 706 | self, 707 | failed_event, 708 | mock_internal_notify_fail, 709 | mock_capture_exception_fn, 710 | publish_strategy, 711 | mocker, 712 | ): 713 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 714 | mocker.patch( 715 | "jaiminho.settings.default_capture_exception", 716 | mock_capture_exception_fn, 717 | ) 718 | assert Event.objects.all().count() == 1 719 | 720 | call_command(validate_events_relay.Command()) 721 | 722 | mock_capture_exception_fn.assert_called_once() 723 | assert Event.objects.all().count() == 1 724 | 725 | @pytest.mark.parametrize( 726 | "publish_strategy", 727 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 728 | ) 729 | def test_raise_exception_when_module_does_not_exist_anymore( 730 | self, mocker, caplog, mock_capture_exception_fn, publish_strategy 731 | ): 732 | missing_module = b'\x80\x04\x95.\x00\x00\x00\x00\x00\x00\x00\x8c"jaiminho_django_test_project.send2\x94\x8c\x03foo\x94\x93\x94.' 733 | 734 | mocker.patch( 735 | "jaiminho.settings.default_capture_exception", 736 | mock_capture_exception_fn, 737 | ) 738 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 739 | 740 | EventFactory(function=missing_module) 741 | 742 | call_command(validate_events_relay.Command()) 743 | 744 | assert "Function does not exist anymore" in caplog.text 745 | assert "No module named 'jaiminho_django_test_project.send2'" in caplog.text 746 | capture_exception_call = mock_capture_exception_fn.call_args[0][0] 747 | assert "No module named 'jaiminho_django_test_project.send2'" == str( 748 | capture_exception_call 749 | ) 750 | 751 | @pytest.mark.parametrize( 752 | "publish_strategy", 753 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 754 | ) 755 | def test_raise_exception_when_function_does_not_exist_anymore( 756 | self, caplog, publish_strategy, mocker 757 | ): 758 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 759 | 760 | missing_function = b"\x80\x04\x957\x00\x00\x00\x00\x00\x00\x00\x8c!jaiminho_django_test_project.send\x94\x8c\rnever_existed\x94\x93\x94." 761 | EventFactory( 762 | function=missing_function, 763 | ) 764 | 765 | call_command(validate_events_relay.Command()) 766 | 767 | assert "Function does not exist anymore" in caplog.text 768 | assert "Can't get attribute 'never_existed' on" in caplog.text 769 | 770 | @pytest.mark.parametrize( 771 | "publish_strategy", 772 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 773 | ) 774 | def test_works_fine_without_capture_message_fn( 775 | self, 776 | mocker, 777 | failed_event, 778 | mock_internal_notify_fail, 779 | publish_strategy, 780 | ): 781 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 782 | mocker.patch("jaiminho.settings.default_capture_exception", None) 783 | 784 | call_command(validate_events_relay.Command()) 785 | 786 | mock_internal_notify_fail.assert_called_once() 787 | 788 | @pytest.mark.parametrize( 789 | "publish_strategy", 790 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 791 | ) 792 | def test_works_with_custom_capture_message_fn( 793 | self, mocker, failed_event, mock_internal_notify_fail, publish_strategy 794 | ): 795 | mock_custom_capture_fn = mock.Mock() 796 | mocker.patch( 797 | "jaiminho.settings.default_capture_exception", mock_custom_capture_fn 798 | ) 799 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 800 | 801 | call_command(validate_events_relay.Command()) 802 | 803 | mock_internal_notify_fail.assert_called_once() 804 | exception_raised = mock_custom_capture_fn.call_args[0][0] 805 | assert exception_raised == mock_internal_notify_fail.side_effect 806 | assert "Some error" == str(exception_raised) 807 | -------------------------------------------------------------------------------- /jaiminho_django_test_project/tests/test_send.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import dill 4 | import pytest 5 | from dateutil.tz import UTC 6 | from django.core.serializers.json import DjangoJSONEncoder 7 | from freezegun import freeze_time 8 | from django.test import TestCase 9 | 10 | from jaiminho.constants import PublishStrategyType 11 | from jaiminho.models import Event 12 | import jaiminho_django_test_project.send 13 | from jaiminho.publish_strategies import KeepOrderStrategy 14 | 15 | pytestmark = pytest.mark.django_db 16 | 17 | 18 | @pytest.fixture 19 | def mock_log_metric(mocker): 20 | return mocker.patch("jaiminho_django_test_project.app.signals.log_metric") 21 | 22 | 23 | @pytest.fixture 24 | def mock_internal_notify(mocker): 25 | return mocker.patch("jaiminho_django_test_project.send.internal_notify") 26 | 27 | 28 | @pytest.fixture 29 | def mock_should_delete_after_send(mocker): 30 | return mocker.patch("jaiminho.settings.delete_after_send", True) 31 | 32 | 33 | @pytest.fixture 34 | def mock_should_not_delete_after_send(mocker): 35 | return mocker.patch("jaiminho.settings.delete_after_send", False) 36 | 37 | 38 | @pytest.fixture 39 | def mock_should_persist_all_events(mocker): 40 | return mocker.patch("jaiminho.settings.persist_all_events", True) 41 | 42 | 43 | @pytest.fixture 44 | def mock_internal_notify_fail(mocker): 45 | mock = mocker.patch("jaiminho_django_test_project.send.internal_notify") 46 | mock.side_effect = Exception("ups") 47 | return mock 48 | 49 | 50 | @pytest.fixture 51 | def mock_event_published_signal(mocker): 52 | return mocker.patch("jaiminho.publish_strategies.event_published.send") 53 | 54 | 55 | @pytest.fixture 56 | def mock_event_failed_to_publish_signal(mocker): 57 | return mocker.patch("jaiminho.publish_strategies.event_failed_to_publish.send") 58 | 59 | 60 | # we need this in the globals so we don't need to mock A LOT of things 61 | class Encoder: 62 | pass 63 | 64 | 65 | @pytest.fixture 66 | def encoder(): 67 | return Encoder 68 | 69 | 70 | class TestNotify: 71 | @pytest.mark.parametrize( 72 | "publish_strategy", 73 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 74 | ) 75 | def test_send_success_should_persist_all_events( 76 | self, 77 | mock_internal_notify, 78 | mock_log_metric, 79 | mock_should_not_delete_after_send, 80 | mock_should_persist_all_events, 81 | publish_strategy, 82 | caplog, 83 | mocker, 84 | ): 85 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 86 | 87 | payload = {"action": "a", "type": "t", "c": "d"} 88 | with TestCase.captureOnCommitCallbacks(execute=True): 89 | jaiminho_django_test_project.send.notify(payload) 90 | 91 | assert Event.objects.all().count() == 1 92 | assert Event.objects.first().stream is None 93 | assert "JAIMINHO-SAVE-TO-OUTBOX: Event created" in caplog.text 94 | 95 | @pytest.mark.parametrize( 96 | "publish_strategy", 97 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 98 | ) 99 | def test_send_success_should_persist_strategy( 100 | self, 101 | mock_internal_notify, 102 | mock_log_metric, 103 | mock_should_not_delete_after_send, 104 | mock_should_persist_all_events, 105 | publish_strategy, 106 | caplog, 107 | mocker, 108 | ): 109 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 110 | 111 | payload = {"action": "a", "type": "t", "c": "d"} 112 | with TestCase.captureOnCommitCallbacks(execute=True): 113 | jaiminho_django_test_project.send.notify(payload) 114 | 115 | assert Event.objects.all().count() == 1 116 | assert Event.objects.get().strategy == publish_strategy 117 | 118 | @pytest.mark.parametrize("publish_strategy", (PublishStrategyType.KEEP_ORDER,)) 119 | @pytest.mark.parametrize( 120 | ("persist_all_events", "delete_after_send"), ((True, False), (True, False)) 121 | ) 122 | def test_send_success_should_not_send_event_when_keep_order_strategy( 123 | self, 124 | mock_internal_notify, 125 | mock_log_metric, 126 | mock_should_not_delete_after_send, 127 | mock_should_persist_all_events, 128 | publish_strategy, 129 | persist_all_events, 130 | delete_after_send, 131 | caplog, 132 | mocker, 133 | ): 134 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 135 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 136 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 137 | 138 | payload = {"action": "a", "type": "t", "c": "d"} 139 | with TestCase.captureOnCommitCallbacks(execute=True): 140 | jaiminho_django_test_project.send.notify(payload) 141 | 142 | assert not mock_internal_notify.called 143 | 144 | @pytest.mark.parametrize( 145 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 146 | ) 147 | def test_send_success_should_publish_event( 148 | self, 149 | mock_internal_notify, 150 | mock_log_metric, 151 | mock_should_not_delete_after_send, 152 | mock_should_persist_all_events, 153 | publish_strategy, 154 | caplog, 155 | mocker, 156 | ): 157 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 158 | 159 | args = ({"action": "a", "type": "t", "c": "d"},) 160 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 161 | jaiminho_django_test_project.send.notify(*args) 162 | 163 | assert len(callbacks) == 1 164 | mock_internal_notify.assert_called_once_with(*args) 165 | assert Event.objects.all().count() == 1 166 | mock_log_metric.assert_called_once_with("event-published", args[0], args=args) 167 | 168 | @pytest.mark.parametrize( 169 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 170 | ) 171 | def test_send_success_should_not_persist_all_events( 172 | self, 173 | mock_internal_notify, 174 | mock_log_metric, 175 | mock_should_not_delete_after_send, 176 | publish_strategy, 177 | mocker, 178 | ): 179 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 180 | args = ({"action": "a", "type": "t", "c": "d"},) 181 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 182 | jaiminho_django_test_project.send.notify(*args) 183 | mock_internal_notify.assert_called_once_with(*args) 184 | assert Event.objects.all().count() == 0 185 | mock_log_metric.assert_called_once_with("event-published", args[0], args=args) 186 | assert len(callbacks) == 1 187 | 188 | @pytest.mark.parametrize( 189 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 190 | ) 191 | def test_send_success_should_delete_after_send( 192 | self, 193 | mock_internal_notify, 194 | mock_log_metric, 195 | mock_should_persist_all_events, 196 | mock_should_delete_after_send, 197 | caplog, 198 | publish_strategy, 199 | mocker, 200 | ): 201 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 202 | 203 | args = ({"action": "a", "type": "t", "c": "d"},) 204 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 205 | jaiminho_django_test_project.send.notify(*args) 206 | assert Event.objects.all().count() == 1 207 | event = Event.objects.get() 208 | assert event.sent_at is None 209 | 210 | mock_internal_notify.assert_called_once_with(*args) 211 | assert Event.objects.all().count() == 0 212 | mock_log_metric.assert_called_once_with("event-published", args[0], args=args) 213 | assert len(callbacks) == 1 214 | assert ( 215 | "JAIMINHO-ON-COMMIT-HOOK: Event deleted after success send" in caplog.text 216 | ) 217 | 218 | @pytest.mark.parametrize( 219 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 220 | ) 221 | def test_send_success_when_should_not_delete_after_send( 222 | self, 223 | mock_log_metric, 224 | mock_internal_notify, 225 | mock_should_not_delete_after_send, 226 | mock_should_persist_all_events, 227 | caplog, 228 | publish_strategy, 229 | mocker, 230 | ): 231 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 232 | 233 | payload = {"action": "a", "type": "t", "c": "d"} 234 | 235 | with freeze_time("2022-01-01"): 236 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 237 | jaiminho_django_test_project.send.notify(payload) 238 | assert len(callbacks) == 1 239 | 240 | assert Event.objects.all().count() == 1 241 | event = Event.objects.first() 242 | assert Event.objects.first().stream is None 243 | assert event.sent_at == datetime(2022, 1, 1, tzinfo=UTC) 244 | assert "JAIMINHO-ON-COMMIT-HOOK: Event marked as sent" in caplog.text 245 | 246 | @pytest.mark.parametrize( 247 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 248 | ) 249 | @pytest.mark.parametrize( 250 | ("persist_all_events", "delete_after_send"), ((True, False), (True, False)) 251 | ) 252 | def test_send_fail( 253 | self, 254 | mock_log_metric, 255 | mock_internal_notify_fail, 256 | persist_all_events, 257 | delete_after_send, 258 | publish_strategy, 259 | mocker, 260 | caplog, 261 | ): 262 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 263 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 264 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 265 | 266 | args = ({"action": "a", "type": "t", "c": "d"},) 267 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 268 | jaiminho_django_test_project.send.notify(*args) 269 | 270 | mock_internal_notify_fail.assert_called_once_with(*args) 271 | assert Event.objects.all().count() == 1 272 | assert Event.objects.first().sent_at is None 273 | assert Event.objects.first().stream is None 274 | mock_log_metric.assert_called_once_with( 275 | "event-failed-to-publish", args[0], args=args 276 | ) 277 | assert len(callbacks) == 1 278 | assert "JAIMINHO-ON-COMMIT-HOOK: Event failed to be published" in caplog.text 279 | 280 | @pytest.mark.parametrize( 281 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 282 | ) 283 | @pytest.mark.parametrize( 284 | "exception", 285 | (AssertionError, AttributeError, Exception, SystemError, SystemExit), 286 | ) 287 | def test_send_fail_handles_multiple_exceptions_type( 288 | self, 289 | mock_log_metric, 290 | mock_internal_notify_fail, 291 | exception, 292 | publish_strategy, 293 | mocker, 294 | ): 295 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 296 | 297 | mock_internal_notify_fail.side_effect = exception 298 | 299 | args = ({"action": "a", "type": "t", "c": "d"},) 300 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 301 | jaiminho_django_test_project.send.notify(*args) 302 | 303 | mock_internal_notify_fail.assert_called_once_with(*args) 304 | 305 | mock_log_metric.assert_called_once_with( 306 | "event-failed-to-publish", args[0], args=args 307 | ) 308 | assert len(callbacks) == 1 309 | 310 | @pytest.mark.parametrize( 311 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 312 | ) 313 | @pytest.mark.parametrize( 314 | ("delete_after_send", "persist_all_events"), ((True, False), (True, False)) 315 | ) 316 | def test_send_trigger_event_published_signal( 317 | self, 318 | mock_internal_notify, 319 | mock_event_published_signal, 320 | mock_should_persist_all_events, 321 | mocker, 322 | delete_after_send, 323 | persist_all_events, 324 | publish_strategy, 325 | ): 326 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 327 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 328 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 329 | 330 | args = ({"action": "a", "type": "t", "c": "d"},) 331 | 332 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 333 | jaiminho_django_test_project.send.notify( 334 | *args, first_param="1", second_param="2" 335 | ) 336 | 337 | original_func = jaiminho_django_test_project.send.notify.original_func 338 | mock_event_published_signal.assert_called_once_with( 339 | sender=original_func, 340 | event_payload=args[0], 341 | args=args, 342 | first_param="1", 343 | second_param="2", 344 | ) 345 | assert len(callbacks) == 1 346 | 347 | @pytest.mark.parametrize( 348 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 349 | ) 350 | @pytest.mark.parametrize( 351 | ("delete_after_send", "persist_all_events"), ((True, False), (True, False)) 352 | ) 353 | def test_send_trigger_event_failed_to_publish_signal( 354 | self, 355 | mock_internal_notify_fail, 356 | mock_event_failed_to_publish_signal, 357 | mock_should_persist_all_events, 358 | mocker, 359 | delete_after_send, 360 | persist_all_events, 361 | publish_strategy, 362 | ): 363 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 364 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 365 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 366 | 367 | args = ({"action": "a", "type": "t", "c": "d"},) 368 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 369 | jaiminho_django_test_project.send.notify( 370 | *args, first_param="1", second_param="2" 371 | ) 372 | 373 | original_func = jaiminho_django_test_project.send.notify.original_func 374 | mock_event_failed_to_publish_signal.assert_called_once_with( 375 | sender=original_func, 376 | event_payload=args[0], 377 | args=args, 378 | first_param="1", 379 | second_param="2", 380 | ) 381 | assert len(callbacks) == 1 382 | 383 | @pytest.mark.parametrize( 384 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 385 | ) 386 | @pytest.mark.parametrize( 387 | ("delete_after_send", "persist_all_events"), ((True, False), (True, False)) 388 | ) 389 | def test_send_fail_with_parameters( 390 | self, 391 | mock_internal_notify_fail, 392 | mock_should_persist_all_events, 393 | encoder, 394 | mocker, 395 | delete_after_send, 396 | persist_all_events, 397 | publish_strategy, 398 | ): 399 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 400 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 401 | mocker.patch("jaiminho.settings.delete_after_send", persist_all_events) 402 | args = ({"action": "a", "type": "t", "c": "d"},) 403 | param = {"param": 1} 404 | jaiminho_django_test_project.send.notify(*args, encoder=encoder, param=param) 405 | assert Event.objects.all().count() == 1 406 | event = Event.objects.first() 407 | assert event.sent_at is None 408 | assert event.stream is None 409 | assert dill.loads(event.kwargs)["encoder"] == Encoder 410 | assert dill.loads(event.kwargs)["param"] == param 411 | assert dill.loads(event.message) == args 412 | assert ( 413 | dill.loads(event.function).__code__.co_code 414 | == jaiminho_django_test_project.send.notify.original_func.__code__.co_code 415 | ) 416 | 417 | 418 | class TestNotifyWithStream: 419 | @pytest.mark.parametrize( 420 | "publish_strategy", 421 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 422 | ) 423 | def test_send_to_stream_success_should_persist_all_events( 424 | self, 425 | mock_internal_notify, 426 | mock_log_metric, 427 | mock_should_not_delete_after_send, 428 | mock_should_persist_all_events, 429 | publish_strategy, 430 | caplog, 431 | mocker, 432 | ): 433 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 434 | 435 | payload = {"action": "a", "type": "t", "c": "d"} 436 | with TestCase.captureOnCommitCallbacks(execute=True): 437 | jaiminho_django_test_project.send.notify_to_stream(payload) 438 | 439 | assert Event.objects.all().count() == 1 440 | assert ( 441 | Event.objects.get().stream 442 | == jaiminho_django_test_project.send.EXAMPLE_STREAM 443 | ) 444 | assert "JAIMINHO-SAVE-TO-OUTBOX: Event created" in caplog.text 445 | 446 | @pytest.mark.parametrize( 447 | "publish_strategy", 448 | (PublishStrategyType.PUBLISH_ON_COMMIT, PublishStrategyType.KEEP_ORDER), 449 | ) 450 | def test_send_to_stream_success_should_persist_strategy( 451 | self, 452 | mock_internal_notify, 453 | mock_log_metric, 454 | mock_should_not_delete_after_send, 455 | mock_should_persist_all_events, 456 | publish_strategy, 457 | caplog, 458 | mocker, 459 | ): 460 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 461 | 462 | payload = {"action": "a", "type": "t", "c": "d"} 463 | with TestCase.captureOnCommitCallbacks(execute=True): 464 | jaiminho_django_test_project.send.notify_to_stream(payload) 465 | 466 | assert Event.objects.all().count() == 1 467 | assert Event.objects.get().strategy == publish_strategy 468 | 469 | @pytest.mark.parametrize("publish_strategy", (PublishStrategyType.KEEP_ORDER,)) 470 | @pytest.mark.parametrize( 471 | ("persist_all_events", "delete_after_send"), ((True, False), (True, False)) 472 | ) 473 | def test_send_to_stream_success_should_not_send_event_when_keep_order_strategy( 474 | self, 475 | mock_internal_notify, 476 | mock_log_metric, 477 | mock_should_not_delete_after_send, 478 | mock_should_persist_all_events, 479 | publish_strategy, 480 | persist_all_events, 481 | delete_after_send, 482 | caplog, 483 | mocker, 484 | ): 485 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 486 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 487 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 488 | 489 | payload = {"action": "a", "type": "t", "c": "d"} 490 | with TestCase.captureOnCommitCallbacks(execute=True): 491 | jaiminho_django_test_project.send.notify_to_stream(payload) 492 | 493 | assert not mock_internal_notify.called 494 | 495 | @pytest.mark.parametrize( 496 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 497 | ) 498 | def test_send_to_stream_success_should_publish_event( 499 | self, 500 | mock_internal_notify, 501 | mock_log_metric, 502 | mock_should_not_delete_after_send, 503 | mock_should_persist_all_events, 504 | publish_strategy, 505 | caplog, 506 | mocker, 507 | ): 508 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 509 | 510 | args = ({"action": "a", "type": "t", "c": "d"},) 511 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 512 | jaiminho_django_test_project.send.notify_to_stream(*args) 513 | 514 | assert len(callbacks) == 1 515 | mock_internal_notify.assert_called_once_with(*args) 516 | assert Event.objects.all().count() == 1 517 | mock_log_metric.assert_called_once_with("event-published", args[0], args=args) 518 | 519 | @pytest.mark.parametrize( 520 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 521 | ) 522 | def test_send_to_stream_success_should_not_persist_all_events( 523 | self, 524 | mock_internal_notify, 525 | mock_log_metric, 526 | mock_should_not_delete_after_send, 527 | publish_strategy, 528 | mocker, 529 | ): 530 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 531 | args = ({"action": "a", "type": "t", "c": "d"},) 532 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 533 | jaiminho_django_test_project.send.notify_to_stream(*args) 534 | mock_internal_notify.assert_called_once_with(*args) 535 | assert Event.objects.all().count() == 0 536 | mock_log_metric.assert_called_once_with("event-published", args[0], args=args) 537 | assert len(callbacks) == 1 538 | 539 | @pytest.mark.parametrize( 540 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 541 | ) 542 | def test_send_to_stream_success_should_delete_after_send( 543 | self, 544 | mock_internal_notify, 545 | mock_log_metric, 546 | mock_should_persist_all_events, 547 | mock_should_delete_after_send, 548 | caplog, 549 | publish_strategy, 550 | mocker, 551 | ): 552 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 553 | 554 | args = ({"action": "a", "type": "t", "c": "d"},) 555 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 556 | jaiminho_django_test_project.send.notify_to_stream(*args) 557 | assert Event.objects.all().count() == 1 558 | event = Event.objects.get() 559 | assert event.sent_at is None 560 | 561 | mock_internal_notify.assert_called_once_with(*args) 562 | assert Event.objects.all().count() == 0 563 | mock_log_metric.assert_called_once_with("event-published", args[0], args=args) 564 | assert len(callbacks) == 1 565 | assert ( 566 | "JAIMINHO-ON-COMMIT-HOOK: Event deleted after success send" in caplog.text 567 | ) 568 | 569 | @pytest.mark.parametrize( 570 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 571 | ) 572 | def test_send_to_stream_success_when_should_not_delete_after_send( 573 | self, 574 | mock_log_metric, 575 | mock_internal_notify, 576 | mock_should_not_delete_after_send, 577 | mock_should_persist_all_events, 578 | caplog, 579 | publish_strategy, 580 | mocker, 581 | ): 582 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 583 | 584 | payload = {"action": "a", "type": "t", "c": "d"} 585 | 586 | with freeze_time("2022-01-01"): 587 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 588 | jaiminho_django_test_project.send.notify_to_stream(payload) 589 | assert len(callbacks) == 1 590 | 591 | assert Event.objects.all().count() == 1 592 | event = Event.objects.first() 593 | assert ( 594 | Event.objects.first().stream 595 | == jaiminho_django_test_project.send.EXAMPLE_STREAM 596 | ) 597 | assert event.sent_at == datetime(2022, 1, 1, tzinfo=UTC) 598 | assert "JAIMINHO-ON-COMMIT-HOOK: Event marked as sent" in caplog.text 599 | 600 | @pytest.mark.parametrize( 601 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 602 | ) 603 | @pytest.mark.parametrize( 604 | ("persist_all_events", "delete_after_send"), ((True, False), (True, False)) 605 | ) 606 | def test_send_to_stream_fail( 607 | self, 608 | mock_log_metric, 609 | mock_internal_notify_fail, 610 | persist_all_events, 611 | delete_after_send, 612 | publish_strategy, 613 | mocker, 614 | caplog, 615 | ): 616 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 617 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 618 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 619 | 620 | args = ({"action": "a", "type": "t", "c": "d"},) 621 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 622 | jaiminho_django_test_project.send.notify_to_stream(*args) 623 | 624 | mock_internal_notify_fail.assert_called_once_with(*args) 625 | assert Event.objects.all().count() == 1 626 | assert Event.objects.first().sent_at is None 627 | assert ( 628 | Event.objects.first().stream 629 | == jaiminho_django_test_project.send.EXAMPLE_STREAM 630 | ) 631 | mock_log_metric.assert_called_once_with( 632 | "event-failed-to-publish", args[0], args=args 633 | ) 634 | assert len(callbacks) == 1 635 | assert "JAIMINHO-ON-COMMIT-HOOK: Event failed to be published" in caplog.text 636 | 637 | @pytest.mark.parametrize( 638 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 639 | ) 640 | @pytest.mark.parametrize( 641 | "exception", 642 | (AssertionError, AttributeError, Exception, SystemError, SystemExit), 643 | ) 644 | def test_send_to_stream_fail_handles_multiple_exceptions_type( 645 | self, 646 | mock_log_metric, 647 | mock_internal_notify_fail, 648 | exception, 649 | publish_strategy, 650 | mocker, 651 | ): 652 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 653 | 654 | mock_internal_notify_fail.side_effect = exception 655 | 656 | args = ({"action": "a", "type": "t", "c": "d"},) 657 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 658 | jaiminho_django_test_project.send.notify_to_stream(*args) 659 | 660 | mock_internal_notify_fail.assert_called_once_with(*args) 661 | 662 | mock_log_metric.assert_called_once_with( 663 | "event-failed-to-publish", args[0], args=args 664 | ) 665 | assert len(callbacks) == 1 666 | 667 | @pytest.mark.parametrize( 668 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 669 | ) 670 | @pytest.mark.parametrize( 671 | ("delete_after_send", "persist_all_events"), ((True, False), (True, False)) 672 | ) 673 | def test_send_to_stream_trigger_event_published_signal( 674 | self, 675 | mock_internal_notify, 676 | mock_event_published_signal, 677 | mock_should_persist_all_events, 678 | mocker, 679 | delete_after_send, 680 | persist_all_events, 681 | publish_strategy, 682 | ): 683 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 684 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 685 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 686 | 687 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 688 | jaiminho_django_test_project.send.notify_to_stream( 689 | {"action": "a", "type": "t", "c": "d"} 690 | ) 691 | mock_event_published_signal.assert_called_once() 692 | assert len(callbacks) == 1 693 | 694 | @pytest.mark.parametrize( 695 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 696 | ) 697 | @pytest.mark.parametrize( 698 | ("delete_after_send", "persist_all_events"), ((True, False), (True, False)) 699 | ) 700 | def test_send_to_stream_trigger_event_failed_to_publish_signal( 701 | self, 702 | mock_internal_notify_fail, 703 | mock_event_failed_to_publish_signal, 704 | mock_should_persist_all_events, 705 | mocker, 706 | delete_after_send, 707 | persist_all_events, 708 | publish_strategy, 709 | ): 710 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 711 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 712 | mocker.patch("jaiminho.settings.persist_all_events", persist_all_events) 713 | 714 | with TestCase.captureOnCommitCallbacks(execute=True) as callbacks: 715 | jaiminho_django_test_project.send.notify_to_stream( 716 | {"action": "a", "type": "t", "c": "d"} 717 | ) 718 | 719 | mock_event_failed_to_publish_signal.assert_called_once() 720 | assert len(callbacks) == 1 721 | 722 | @pytest.mark.parametrize( 723 | "publish_strategy", (PublishStrategyType.PUBLISH_ON_COMMIT,) 724 | ) 725 | @pytest.mark.parametrize( 726 | ("delete_after_send", "persist_all_events"), ((True, False), (True, False)) 727 | ) 728 | def test_send_to_stream_fail_with_parameters( 729 | self, 730 | mock_internal_notify_fail, 731 | mock_should_persist_all_events, 732 | encoder, 733 | mocker, 734 | delete_after_send, 735 | persist_all_events, 736 | publish_strategy, 737 | ): 738 | mocker.patch("jaiminho.settings.publish_strategy", publish_strategy) 739 | mocker.patch("jaiminho.settings.delete_after_send", delete_after_send) 740 | mocker.patch("jaiminho.settings.delete_after_send", persist_all_events) 741 | args = ({"action": "a", "type": "t", "c": "d"},) 742 | param = {"param": 1} 743 | jaiminho_django_test_project.send.notify_to_stream( 744 | *args, encoder=encoder, param=param 745 | ) 746 | assert Event.objects.all().count() == 1 747 | event = Event.objects.first() 748 | assert event.stream == jaiminho_django_test_project.send.EXAMPLE_STREAM 749 | assert event.sent_at is None 750 | assert dill.loads(event.kwargs)["encoder"] == Encoder 751 | assert dill.loads(event.kwargs)["param"] == param 752 | assert dill.loads(event.message) == args 753 | assert ( 754 | dill.loads(event.function).__code__.co_code 755 | == jaiminho_django_test_project.send.notify.original_func.__code__.co_code 756 | ) 757 | 758 | 759 | class TestNofityWithStreamOverwritingStrategy: 760 | def test_send_to_stream_success_should_persist_all_events( 761 | self, 762 | mock_internal_notify, 763 | mock_log_metric, 764 | mock_should_not_delete_after_send, 765 | mock_should_persist_all_events, 766 | caplog, 767 | mocker, 768 | ): 769 | strategy = KeepOrderStrategy() 770 | mocker.patch( 771 | "jaiminho.settings.publish_strategy", PublishStrategyType.PUBLISH_ON_COMMIT 772 | ) 773 | create_publish_strategy_mock = mocker.patch( 774 | "jaiminho.publish_strategies.create_publish_strategy", 775 | autospec=True, 776 | return_value=strategy, 777 | ) 778 | 779 | payload = {"action": "a", "type": "t", "c": "d"} 780 | with TestCase.captureOnCommitCallbacks(execute=True): 781 | jaiminho_django_test_project.send.notify_to_stream_overwriting_strategy( 782 | payload 783 | ) 784 | 785 | create_publish_strategy_mock.assert_called_once_with( 786 | PublishStrategyType.KEEP_ORDER 787 | ) 788 | assert Event.objects.all().count() == 1 789 | assert ( 790 | Event.objects.get().stream 791 | == jaiminho_django_test_project.send.EXAMPLE_STREAM 792 | ) 793 | assert "JAIMINHO-SAVE-TO-OUTBOX: Event created" in caplog.text 794 | 795 | def test_send_to_stream_should_persist_strategy( 796 | self, 797 | mock_internal_notify, 798 | mock_log_metric, 799 | mock_should_not_delete_after_send, 800 | mock_should_persist_all_events, 801 | caplog, 802 | mocker, 803 | ): 804 | strategy = KeepOrderStrategy() 805 | mocker.patch( 806 | "jaiminho.settings.publish_strategy", PublishStrategyType.PUBLISH_ON_COMMIT 807 | ) 808 | create_publish_strategy_mock = mocker.patch( 809 | "jaiminho.publish_strategies.create_publish_strategy", 810 | autospec=True, 811 | return_value=strategy, 812 | ) 813 | 814 | payload = {"action": "a", "type": "t", "c": "d"} 815 | with TestCase.captureOnCommitCallbacks(execute=True): 816 | jaiminho_django_test_project.send.notify_to_stream_overwriting_strategy( 817 | payload 818 | ) 819 | 820 | create_publish_strategy_mock.assert_called_once_with( 821 | PublishStrategyType.KEEP_ORDER 822 | ) 823 | assert Event.objects.all().count() == 1 824 | assert Event.objects.get().strategy == PublishStrategyType.KEEP_ORDER 825 | --------------------------------------------------------------------------------